1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/pci_ids.h> 17 #include <linux/kthread.h> 18 #include <linux/sched.h> 19 #include <linux/interrupt.h> 20 #include <linux/spinlock.h> 21 #include <linux/mutex.h> 22 #include <linux/delay.h> 23 #include <linux/ccp.h> 24 #include <linux/scatterlist.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/sha.h> 27 28 #include "ccp-dev.h" 29 30 enum ccp_memtype { 31 CCP_MEMTYPE_SYSTEM = 0, 32 CCP_MEMTYPE_KSB, 33 CCP_MEMTYPE_LOCAL, 34 CCP_MEMTYPE__LAST, 35 }; 36 37 struct ccp_dma_info { 38 dma_addr_t address; 39 unsigned int offset; 40 unsigned int length; 41 enum dma_data_direction dir; 42 }; 43 44 struct ccp_dm_workarea { 45 struct device *dev; 46 struct dma_pool *dma_pool; 47 unsigned int length; 48 49 u8 *address; 50 struct ccp_dma_info dma; 51 }; 52 53 struct ccp_sg_workarea { 54 struct scatterlist *sg; 55 unsigned int nents; 56 unsigned int length; 57 58 struct scatterlist *dma_sg; 59 struct device *dma_dev; 60 unsigned int dma_count; 61 enum dma_data_direction dma_dir; 62 63 unsigned int sg_used; 64 65 u64 bytes_left; 66 }; 67 68 struct ccp_data { 69 struct ccp_sg_workarea sg_wa; 70 struct ccp_dm_workarea dm_wa; 71 }; 72 73 struct ccp_mem { 74 enum ccp_memtype type; 75 union { 76 struct ccp_dma_info dma; 77 u32 ksb; 78 } u; 79 }; 80 81 struct ccp_aes_op { 82 enum ccp_aes_type type; 83 enum ccp_aes_mode mode; 84 enum ccp_aes_action action; 85 }; 86 87 struct ccp_xts_aes_op { 88 enum ccp_aes_action action; 89 enum ccp_xts_aes_unit_size unit_size; 90 }; 91 92 struct ccp_sha_op { 93 enum ccp_sha_type type; 94 u64 msg_bits; 95 }; 96 97 struct ccp_rsa_op { 98 u32 mod_size; 99 u32 input_len; 100 }; 101 102 struct ccp_passthru_op { 103 enum ccp_passthru_bitwise bit_mod; 104 enum ccp_passthru_byteswap byte_swap; 105 }; 106 107 struct ccp_ecc_op { 108 enum ccp_ecc_function function; 109 }; 110 111 struct ccp_op { 112 struct ccp_cmd_queue *cmd_q; 113 114 u32 jobid; 115 u32 ioc; 116 u32 soc; 117 u32 ksb_key; 118 u32 ksb_ctx; 119 u32 init; 120 u32 eom; 121 122 struct ccp_mem src; 123 struct ccp_mem dst; 124 125 union { 126 struct ccp_aes_op aes; 127 struct ccp_xts_aes_op xts; 128 struct ccp_sha_op sha; 129 struct ccp_rsa_op rsa; 130 struct ccp_passthru_op passthru; 131 struct ccp_ecc_op ecc; 132 } u; 133 }; 134 135 /* SHA initial context values */ 136 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 137 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 138 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 139 cpu_to_be32(SHA1_H4), 0, 0, 0, 140 }; 141 142 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 143 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 144 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 145 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 146 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 147 }; 148 149 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 150 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 151 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 152 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 153 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 154 }; 155 156 /* The CCP cannot perform zero-length sha operations so the caller 157 * is required to buffer data for the final operation. However, a 158 * sha operation for a message with a total length of zero is valid 159 * so known values are required to supply the result. 160 */ 161 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { 162 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 163 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 164 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, 165 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 166 }; 167 168 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { 169 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 170 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 171 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 172 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, 173 }; 174 175 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { 176 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 177 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 178 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 179 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, 180 }; 181 182 static u32 ccp_addr_lo(struct ccp_dma_info *info) 183 { 184 return lower_32_bits(info->address + info->offset); 185 } 186 187 static u32 ccp_addr_hi(struct ccp_dma_info *info) 188 { 189 return upper_32_bits(info->address + info->offset) & 0x0000ffff; 190 } 191 192 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 193 { 194 struct ccp_cmd_queue *cmd_q = op->cmd_q; 195 struct ccp_device *ccp = cmd_q->ccp; 196 void __iomem *cr_addr; 197 u32 cr0, cmd; 198 unsigned int i; 199 int ret = 0; 200 201 /* We could read a status register to see how many free slots 202 * are actually available, but reading that register resets it 203 * and you could lose some error information. 204 */ 205 cmd_q->free_slots--; 206 207 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 208 | (op->jobid << REQ0_JOBID_SHIFT) 209 | REQ0_WAIT_FOR_WRITE; 210 211 if (op->soc) 212 cr0 |= REQ0_STOP_ON_COMPLETE 213 | REQ0_INT_ON_COMPLETE; 214 215 if (op->ioc || !cmd_q->free_slots) 216 cr0 |= REQ0_INT_ON_COMPLETE; 217 218 /* Start at CMD_REQ1 */ 219 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 220 221 mutex_lock(&ccp->req_mutex); 222 223 /* Write CMD_REQ1 through CMD_REQx first */ 224 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 225 iowrite32(*(cr + i), cr_addr); 226 227 /* Tell the CCP to start */ 228 wmb(); 229 iowrite32(cr0, ccp->io_regs + CMD_REQ0); 230 231 mutex_unlock(&ccp->req_mutex); 232 233 if (cr0 & REQ0_INT_ON_COMPLETE) { 234 /* Wait for the job to complete */ 235 ret = wait_event_interruptible(cmd_q->int_queue, 236 cmd_q->int_rcvd); 237 if (ret || cmd_q->cmd_error) { 238 /* On error delete all related jobs from the queue */ 239 cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 240 | op->jobid; 241 242 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 243 244 if (!ret) 245 ret = -EIO; 246 } else if (op->soc) { 247 /* Delete just head job from the queue on SoC */ 248 cmd = DEL_Q_ACTIVE 249 | (cmd_q->id << DEL_Q_ID_SHIFT) 250 | op->jobid; 251 252 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 253 } 254 255 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 256 257 cmd_q->int_rcvd = 0; 258 } 259 260 return ret; 261 } 262 263 static int ccp_perform_aes(struct ccp_op *op) 264 { 265 u32 cr[6]; 266 267 /* Fill out the register contents for REQ1 through REQ6 */ 268 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 269 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 270 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 271 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 272 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 273 cr[1] = op->src.u.dma.length - 1; 274 cr[2] = ccp_addr_lo(&op->src.u.dma); 275 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 276 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 277 | ccp_addr_hi(&op->src.u.dma); 278 cr[4] = ccp_addr_lo(&op->dst.u.dma); 279 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 280 | ccp_addr_hi(&op->dst.u.dma); 281 282 if (op->u.aes.mode == CCP_AES_MODE_CFB) 283 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 284 285 if (op->eom) 286 cr[0] |= REQ1_EOM; 287 288 if (op->init) 289 cr[0] |= REQ1_INIT; 290 291 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 292 } 293 294 static int ccp_perform_xts_aes(struct ccp_op *op) 295 { 296 u32 cr[6]; 297 298 /* Fill out the register contents for REQ1 through REQ6 */ 299 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 300 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 301 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 302 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 303 cr[1] = op->src.u.dma.length - 1; 304 cr[2] = ccp_addr_lo(&op->src.u.dma); 305 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 306 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 307 | ccp_addr_hi(&op->src.u.dma); 308 cr[4] = ccp_addr_lo(&op->dst.u.dma); 309 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 310 | ccp_addr_hi(&op->dst.u.dma); 311 312 if (op->eom) 313 cr[0] |= REQ1_EOM; 314 315 if (op->init) 316 cr[0] |= REQ1_INIT; 317 318 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 319 } 320 321 static int ccp_perform_sha(struct ccp_op *op) 322 { 323 u32 cr[6]; 324 325 /* Fill out the register contents for REQ1 through REQ6 */ 326 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 327 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 328 | REQ1_INIT; 329 cr[1] = op->src.u.dma.length - 1; 330 cr[2] = ccp_addr_lo(&op->src.u.dma); 331 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 332 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 333 | ccp_addr_hi(&op->src.u.dma); 334 335 if (op->eom) { 336 cr[0] |= REQ1_EOM; 337 cr[4] = lower_32_bits(op->u.sha.msg_bits); 338 cr[5] = upper_32_bits(op->u.sha.msg_bits); 339 } else { 340 cr[4] = 0; 341 cr[5] = 0; 342 } 343 344 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 345 } 346 347 static int ccp_perform_rsa(struct ccp_op *op) 348 { 349 u32 cr[6]; 350 351 /* Fill out the register contents for REQ1 through REQ6 */ 352 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 353 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 354 | (op->ksb_key << REQ1_KEY_KSB_SHIFT) 355 | REQ1_EOM; 356 cr[1] = op->u.rsa.input_len - 1; 357 cr[2] = ccp_addr_lo(&op->src.u.dma); 358 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 359 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 360 | ccp_addr_hi(&op->src.u.dma); 361 cr[4] = ccp_addr_lo(&op->dst.u.dma); 362 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 363 | ccp_addr_hi(&op->dst.u.dma); 364 365 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 366 } 367 368 static int ccp_perform_passthru(struct ccp_op *op) 369 { 370 u32 cr[6]; 371 372 /* Fill out the register contents for REQ1 through REQ6 */ 373 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 374 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 375 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 376 377 if (op->src.type == CCP_MEMTYPE_SYSTEM) 378 cr[1] = op->src.u.dma.length - 1; 379 else 380 cr[1] = op->dst.u.dma.length - 1; 381 382 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 383 cr[2] = ccp_addr_lo(&op->src.u.dma); 384 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 385 | ccp_addr_hi(&op->src.u.dma); 386 387 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 388 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); 389 } else { 390 cr[2] = op->src.u.ksb * CCP_KSB_BYTES; 391 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); 392 } 393 394 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 395 cr[4] = ccp_addr_lo(&op->dst.u.dma); 396 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 397 | ccp_addr_hi(&op->dst.u.dma); 398 } else { 399 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; 400 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); 401 } 402 403 if (op->eom) 404 cr[0] |= REQ1_EOM; 405 406 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 407 } 408 409 static int ccp_perform_ecc(struct ccp_op *op) 410 { 411 u32 cr[6]; 412 413 /* Fill out the register contents for REQ1 through REQ6 */ 414 cr[0] = REQ1_ECC_AFFINE_CONVERT 415 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 416 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 417 | REQ1_EOM; 418 cr[1] = op->src.u.dma.length - 1; 419 cr[2] = ccp_addr_lo(&op->src.u.dma); 420 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 421 | ccp_addr_hi(&op->src.u.dma); 422 cr[4] = ccp_addr_lo(&op->dst.u.dma); 423 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 424 | ccp_addr_hi(&op->dst.u.dma); 425 426 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 427 } 428 429 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) 430 { 431 int start; 432 433 for (;;) { 434 mutex_lock(&ccp->ksb_mutex); 435 436 start = (u32)bitmap_find_next_zero_area(ccp->ksb, 437 ccp->ksb_count, 438 ccp->ksb_start, 439 count, 0); 440 if (start <= ccp->ksb_count) { 441 bitmap_set(ccp->ksb, start, count); 442 443 mutex_unlock(&ccp->ksb_mutex); 444 break; 445 } 446 447 ccp->ksb_avail = 0; 448 449 mutex_unlock(&ccp->ksb_mutex); 450 451 /* Wait for KSB entries to become available */ 452 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail)) 453 return 0; 454 } 455 456 return KSB_START + start; 457 } 458 459 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start, 460 unsigned int count) 461 { 462 if (!start) 463 return; 464 465 mutex_lock(&ccp->ksb_mutex); 466 467 bitmap_clear(ccp->ksb, start - KSB_START, count); 468 469 ccp->ksb_avail = 1; 470 471 mutex_unlock(&ccp->ksb_mutex); 472 473 wake_up_interruptible_all(&ccp->ksb_queue); 474 } 475 476 static u32 ccp_gen_jobid(struct ccp_device *ccp) 477 { 478 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 479 } 480 481 static void ccp_sg_free(struct ccp_sg_workarea *wa) 482 { 483 if (wa->dma_count) 484 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 485 486 wa->dma_count = 0; 487 } 488 489 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 490 struct scatterlist *sg, u64 len, 491 enum dma_data_direction dma_dir) 492 { 493 memset(wa, 0, sizeof(*wa)); 494 495 wa->sg = sg; 496 if (!sg) 497 return 0; 498 499 wa->nents = sg_nents(sg); 500 wa->length = sg->length; 501 wa->bytes_left = len; 502 wa->sg_used = 0; 503 504 if (len == 0) 505 return 0; 506 507 if (dma_dir == DMA_NONE) 508 return 0; 509 510 wa->dma_sg = sg; 511 wa->dma_dev = dev; 512 wa->dma_dir = dma_dir; 513 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 514 if (!wa->dma_count) 515 return -ENOMEM; 516 517 return 0; 518 } 519 520 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 521 { 522 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 523 524 if (!wa->sg) 525 return; 526 527 wa->sg_used += nbytes; 528 wa->bytes_left -= nbytes; 529 if (wa->sg_used == wa->sg->length) { 530 wa->sg = sg_next(wa->sg); 531 wa->sg_used = 0; 532 } 533 } 534 535 static void ccp_dm_free(struct ccp_dm_workarea *wa) 536 { 537 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 538 if (wa->address) 539 dma_pool_free(wa->dma_pool, wa->address, 540 wa->dma.address); 541 } else { 542 if (wa->dma.address) 543 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 544 wa->dma.dir); 545 kfree(wa->address); 546 } 547 548 wa->address = NULL; 549 wa->dma.address = 0; 550 } 551 552 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 553 struct ccp_cmd_queue *cmd_q, 554 unsigned int len, 555 enum dma_data_direction dir) 556 { 557 memset(wa, 0, sizeof(*wa)); 558 559 if (!len) 560 return 0; 561 562 wa->dev = cmd_q->ccp->dev; 563 wa->length = len; 564 565 if (len <= CCP_DMAPOOL_MAX_SIZE) { 566 wa->dma_pool = cmd_q->dma_pool; 567 568 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 569 &wa->dma.address); 570 if (!wa->address) 571 return -ENOMEM; 572 573 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 574 575 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 576 } else { 577 wa->address = kzalloc(len, GFP_KERNEL); 578 if (!wa->address) 579 return -ENOMEM; 580 581 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 582 dir); 583 if (!wa->dma.address) 584 return -ENOMEM; 585 586 wa->dma.length = len; 587 } 588 wa->dma.dir = dir; 589 590 return 0; 591 } 592 593 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 594 struct scatterlist *sg, unsigned int sg_offset, 595 unsigned int len) 596 { 597 WARN_ON(!wa->address); 598 599 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 600 0); 601 } 602 603 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 604 struct scatterlist *sg, unsigned int sg_offset, 605 unsigned int len) 606 { 607 WARN_ON(!wa->address); 608 609 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 610 1); 611 } 612 613 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 614 struct scatterlist *sg, 615 unsigned int len, unsigned int se_len, 616 bool sign_extend) 617 { 618 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 619 u8 buffer[CCP_REVERSE_BUF_SIZE]; 620 621 BUG_ON(se_len > sizeof(buffer)); 622 623 sg_offset = len; 624 dm_offset = 0; 625 nbytes = len; 626 while (nbytes) { 627 ksb_len = min_t(unsigned int, nbytes, se_len); 628 sg_offset -= ksb_len; 629 630 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0); 631 for (i = 0; i < ksb_len; i++) 632 wa->address[dm_offset + i] = buffer[ksb_len - i - 1]; 633 634 dm_offset += ksb_len; 635 nbytes -= ksb_len; 636 637 if ((ksb_len != se_len) && sign_extend) { 638 /* Must sign-extend to nearest sign-extend length */ 639 if (wa->address[dm_offset - 1] & 0x80) 640 memset(wa->address + dm_offset, 0xff, 641 se_len - ksb_len); 642 } 643 } 644 } 645 646 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 647 struct scatterlist *sg, 648 unsigned int len) 649 { 650 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 651 u8 buffer[CCP_REVERSE_BUF_SIZE]; 652 653 sg_offset = 0; 654 dm_offset = len; 655 nbytes = len; 656 while (nbytes) { 657 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer)); 658 dm_offset -= ksb_len; 659 660 for (i = 0; i < ksb_len; i++) 661 buffer[ksb_len - i - 1] = wa->address[dm_offset + i]; 662 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1); 663 664 sg_offset += ksb_len; 665 nbytes -= ksb_len; 666 } 667 } 668 669 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 670 { 671 ccp_dm_free(&data->dm_wa); 672 ccp_sg_free(&data->sg_wa); 673 } 674 675 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 676 struct scatterlist *sg, u64 sg_len, 677 unsigned int dm_len, 678 enum dma_data_direction dir) 679 { 680 int ret; 681 682 memset(data, 0, sizeof(*data)); 683 684 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 685 dir); 686 if (ret) 687 goto e_err; 688 689 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 690 if (ret) 691 goto e_err; 692 693 return 0; 694 695 e_err: 696 ccp_free_data(data, cmd_q); 697 698 return ret; 699 } 700 701 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 702 { 703 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 704 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 705 unsigned int buf_count, nbytes; 706 707 /* Clear the buffer if setting it */ 708 if (!from) 709 memset(dm_wa->address, 0, dm_wa->length); 710 711 if (!sg_wa->sg) 712 return 0; 713 714 /* Perform the copy operation 715 * nbytes will always be <= UINT_MAX because dm_wa->length is 716 * an unsigned int 717 */ 718 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 719 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 720 nbytes, from); 721 722 /* Update the structures and generate the count */ 723 buf_count = 0; 724 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 725 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 726 dm_wa->length - buf_count); 727 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 728 729 buf_count += nbytes; 730 ccp_update_sg_workarea(sg_wa, nbytes); 731 } 732 733 return buf_count; 734 } 735 736 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 737 { 738 return ccp_queue_buf(data, 0); 739 } 740 741 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 742 { 743 return ccp_queue_buf(data, 1); 744 } 745 746 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 747 struct ccp_op *op, unsigned int block_size, 748 bool blocksize_op) 749 { 750 unsigned int sg_src_len, sg_dst_len, op_len; 751 752 /* The CCP can only DMA from/to one address each per operation. This 753 * requires that we find the smallest DMA area between the source 754 * and destination. The resulting len values will always be <= UINT_MAX 755 * because the dma length is an unsigned int. 756 */ 757 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 758 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 759 760 if (dst) { 761 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 762 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 763 op_len = min(sg_src_len, sg_dst_len); 764 } else { 765 op_len = sg_src_len; 766 } 767 768 /* The data operation length will be at least block_size in length 769 * or the smaller of available sg room remaining for the source or 770 * the destination 771 */ 772 op_len = max(op_len, block_size); 773 774 /* Unless we have to buffer data, there's no reason to wait */ 775 op->soc = 0; 776 777 if (sg_src_len < block_size) { 778 /* Not enough data in the sg element, so it 779 * needs to be buffered into a blocksize chunk 780 */ 781 int cp_len = ccp_fill_queue_buf(src); 782 783 op->soc = 1; 784 op->src.u.dma.address = src->dm_wa.dma.address; 785 op->src.u.dma.offset = 0; 786 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 787 } else { 788 /* Enough data in the sg element, but we need to 789 * adjust for any previously copied data 790 */ 791 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 792 op->src.u.dma.offset = src->sg_wa.sg_used; 793 op->src.u.dma.length = op_len & ~(block_size - 1); 794 795 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 796 } 797 798 if (dst) { 799 if (sg_dst_len < block_size) { 800 /* Not enough room in the sg element or we're on the 801 * last piece of data (when using padding), so the 802 * output needs to be buffered into a blocksize chunk 803 */ 804 op->soc = 1; 805 op->dst.u.dma.address = dst->dm_wa.dma.address; 806 op->dst.u.dma.offset = 0; 807 op->dst.u.dma.length = op->src.u.dma.length; 808 } else { 809 /* Enough room in the sg element, but we need to 810 * adjust for any previously used area 811 */ 812 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 813 op->dst.u.dma.offset = dst->sg_wa.sg_used; 814 op->dst.u.dma.length = op->src.u.dma.length; 815 } 816 } 817 } 818 819 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 820 struct ccp_op *op) 821 { 822 op->init = 0; 823 824 if (dst) { 825 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 826 ccp_empty_queue_buf(dst); 827 else 828 ccp_update_sg_workarea(&dst->sg_wa, 829 op->dst.u.dma.length); 830 } 831 } 832 833 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, 834 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 835 u32 byte_swap, bool from) 836 { 837 struct ccp_op op; 838 839 memset(&op, 0, sizeof(op)); 840 841 op.cmd_q = cmd_q; 842 op.jobid = jobid; 843 op.eom = 1; 844 845 if (from) { 846 op.soc = 1; 847 op.src.type = CCP_MEMTYPE_KSB; 848 op.src.u.ksb = ksb; 849 op.dst.type = CCP_MEMTYPE_SYSTEM; 850 op.dst.u.dma.address = wa->dma.address; 851 op.dst.u.dma.length = wa->length; 852 } else { 853 op.src.type = CCP_MEMTYPE_SYSTEM; 854 op.src.u.dma.address = wa->dma.address; 855 op.src.u.dma.length = wa->length; 856 op.dst.type = CCP_MEMTYPE_KSB; 857 op.dst.u.ksb = ksb; 858 } 859 860 op.u.passthru.byte_swap = byte_swap; 861 862 return ccp_perform_passthru(&op); 863 } 864 865 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, 866 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 867 u32 byte_swap) 868 { 869 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false); 870 } 871 872 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q, 873 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 874 u32 byte_swap) 875 { 876 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true); 877 } 878 879 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 880 struct ccp_cmd *cmd) 881 { 882 struct ccp_aes_engine *aes = &cmd->u.aes; 883 struct ccp_dm_workarea key, ctx; 884 struct ccp_data src; 885 struct ccp_op op; 886 unsigned int dm_offset; 887 int ret; 888 889 if (!((aes->key_len == AES_KEYSIZE_128) || 890 (aes->key_len == AES_KEYSIZE_192) || 891 (aes->key_len == AES_KEYSIZE_256))) 892 return -EINVAL; 893 894 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 895 return -EINVAL; 896 897 if (aes->iv_len != AES_BLOCK_SIZE) 898 return -EINVAL; 899 900 if (!aes->key || !aes->iv || !aes->src) 901 return -EINVAL; 902 903 if (aes->cmac_final) { 904 if (aes->cmac_key_len != AES_BLOCK_SIZE) 905 return -EINVAL; 906 907 if (!aes->cmac_key) 908 return -EINVAL; 909 } 910 911 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 912 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 913 914 ret = -EIO; 915 memset(&op, 0, sizeof(op)); 916 op.cmd_q = cmd_q; 917 op.jobid = ccp_gen_jobid(cmd_q->ccp); 918 op.ksb_key = cmd_q->ksb_key; 919 op.ksb_ctx = cmd_q->ksb_ctx; 920 op.init = 1; 921 op.u.aes.type = aes->type; 922 op.u.aes.mode = aes->mode; 923 op.u.aes.action = aes->action; 924 925 /* All supported key sizes fit in a single (32-byte) KSB entry 926 * and must be in little endian format. Use the 256-bit byte 927 * swap passthru option to convert from big endian to little 928 * endian. 929 */ 930 ret = ccp_init_dm_workarea(&key, cmd_q, 931 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 932 DMA_TO_DEVICE); 933 if (ret) 934 return ret; 935 936 dm_offset = CCP_KSB_BYTES - aes->key_len; 937 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 938 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 939 CCP_PASSTHRU_BYTESWAP_256BIT); 940 if (ret) { 941 cmd->engine_error = cmd_q->cmd_error; 942 goto e_key; 943 } 944 945 /* The AES context fits in a single (32-byte) KSB entry and 946 * must be in little endian format. Use the 256-bit byte swap 947 * passthru option to convert from big endian to little endian. 948 */ 949 ret = ccp_init_dm_workarea(&ctx, cmd_q, 950 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 951 DMA_BIDIRECTIONAL); 952 if (ret) 953 goto e_key; 954 955 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 956 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 957 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 958 CCP_PASSTHRU_BYTESWAP_256BIT); 959 if (ret) { 960 cmd->engine_error = cmd_q->cmd_error; 961 goto e_ctx; 962 } 963 964 /* Send data to the CCP AES engine */ 965 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 966 AES_BLOCK_SIZE, DMA_TO_DEVICE); 967 if (ret) 968 goto e_ctx; 969 970 while (src.sg_wa.bytes_left) { 971 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 972 if (aes->cmac_final && !src.sg_wa.bytes_left) { 973 op.eom = 1; 974 975 /* Push the K1/K2 key to the CCP now */ 976 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, 977 op.ksb_ctx, 978 CCP_PASSTHRU_BYTESWAP_256BIT); 979 if (ret) { 980 cmd->engine_error = cmd_q->cmd_error; 981 goto e_src; 982 } 983 984 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 985 aes->cmac_key_len); 986 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 987 CCP_PASSTHRU_BYTESWAP_256BIT); 988 if (ret) { 989 cmd->engine_error = cmd_q->cmd_error; 990 goto e_src; 991 } 992 } 993 994 ret = ccp_perform_aes(&op); 995 if (ret) { 996 cmd->engine_error = cmd_q->cmd_error; 997 goto e_src; 998 } 999 1000 ccp_process_data(&src, NULL, &op); 1001 } 1002 1003 /* Retrieve the AES context - convert from LE to BE using 1004 * 32-byte (256-bit) byteswapping 1005 */ 1006 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1007 CCP_PASSTHRU_BYTESWAP_256BIT); 1008 if (ret) { 1009 cmd->engine_error = cmd_q->cmd_error; 1010 goto e_src; 1011 } 1012 1013 /* ...but we only need AES_BLOCK_SIZE bytes */ 1014 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1015 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1016 1017 e_src: 1018 ccp_free_data(&src, cmd_q); 1019 1020 e_ctx: 1021 ccp_dm_free(&ctx); 1022 1023 e_key: 1024 ccp_dm_free(&key); 1025 1026 return ret; 1027 } 1028 1029 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1030 { 1031 struct ccp_aes_engine *aes = &cmd->u.aes; 1032 struct ccp_dm_workarea key, ctx; 1033 struct ccp_data src, dst; 1034 struct ccp_op op; 1035 unsigned int dm_offset; 1036 bool in_place = false; 1037 int ret; 1038 1039 if (aes->mode == CCP_AES_MODE_CMAC) 1040 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 1041 1042 if (!((aes->key_len == AES_KEYSIZE_128) || 1043 (aes->key_len == AES_KEYSIZE_192) || 1044 (aes->key_len == AES_KEYSIZE_256))) 1045 return -EINVAL; 1046 1047 if (((aes->mode == CCP_AES_MODE_ECB) || 1048 (aes->mode == CCP_AES_MODE_CBC) || 1049 (aes->mode == CCP_AES_MODE_CFB)) && 1050 (aes->src_len & (AES_BLOCK_SIZE - 1))) 1051 return -EINVAL; 1052 1053 if (!aes->key || !aes->src || !aes->dst) 1054 return -EINVAL; 1055 1056 if (aes->mode != CCP_AES_MODE_ECB) { 1057 if (aes->iv_len != AES_BLOCK_SIZE) 1058 return -EINVAL; 1059 1060 if (!aes->iv) 1061 return -EINVAL; 1062 } 1063 1064 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 1065 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 1066 1067 ret = -EIO; 1068 memset(&op, 0, sizeof(op)); 1069 op.cmd_q = cmd_q; 1070 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1071 op.ksb_key = cmd_q->ksb_key; 1072 op.ksb_ctx = cmd_q->ksb_ctx; 1073 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 1074 op.u.aes.type = aes->type; 1075 op.u.aes.mode = aes->mode; 1076 op.u.aes.action = aes->action; 1077 1078 /* All supported key sizes fit in a single (32-byte) KSB entry 1079 * and must be in little endian format. Use the 256-bit byte 1080 * swap passthru option to convert from big endian to little 1081 * endian. 1082 */ 1083 ret = ccp_init_dm_workarea(&key, cmd_q, 1084 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1085 DMA_TO_DEVICE); 1086 if (ret) 1087 return ret; 1088 1089 dm_offset = CCP_KSB_BYTES - aes->key_len; 1090 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 1091 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1092 CCP_PASSTHRU_BYTESWAP_256BIT); 1093 if (ret) { 1094 cmd->engine_error = cmd_q->cmd_error; 1095 goto e_key; 1096 } 1097 1098 /* The AES context fits in a single (32-byte) KSB entry and 1099 * must be in little endian format. Use the 256-bit byte swap 1100 * passthru option to convert from big endian to little endian. 1101 */ 1102 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1103 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1104 DMA_BIDIRECTIONAL); 1105 if (ret) 1106 goto e_key; 1107 1108 if (aes->mode != CCP_AES_MODE_ECB) { 1109 /* Load the AES context - conver to LE */ 1110 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1111 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1112 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1113 CCP_PASSTHRU_BYTESWAP_256BIT); 1114 if (ret) { 1115 cmd->engine_error = cmd_q->cmd_error; 1116 goto e_ctx; 1117 } 1118 } 1119 1120 /* Prepare the input and output data workareas. For in-place 1121 * operations we need to set the dma direction to BIDIRECTIONAL 1122 * and copy the src workarea to the dst workarea. 1123 */ 1124 if (sg_virt(aes->src) == sg_virt(aes->dst)) 1125 in_place = true; 1126 1127 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 1128 AES_BLOCK_SIZE, 1129 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1130 if (ret) 1131 goto e_ctx; 1132 1133 if (in_place) { 1134 dst = src; 1135 } else { 1136 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1137 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1138 if (ret) 1139 goto e_src; 1140 } 1141 1142 /* Send data to the CCP AES engine */ 1143 while (src.sg_wa.bytes_left) { 1144 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1145 if (!src.sg_wa.bytes_left) { 1146 op.eom = 1; 1147 1148 /* Since we don't retrieve the AES context in ECB 1149 * mode we have to wait for the operation to complete 1150 * on the last piece of data 1151 */ 1152 if (aes->mode == CCP_AES_MODE_ECB) 1153 op.soc = 1; 1154 } 1155 1156 ret = ccp_perform_aes(&op); 1157 if (ret) { 1158 cmd->engine_error = cmd_q->cmd_error; 1159 goto e_dst; 1160 } 1161 1162 ccp_process_data(&src, &dst, &op); 1163 } 1164 1165 if (aes->mode != CCP_AES_MODE_ECB) { 1166 /* Retrieve the AES context - convert from LE to BE using 1167 * 32-byte (256-bit) byteswapping 1168 */ 1169 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1170 CCP_PASSTHRU_BYTESWAP_256BIT); 1171 if (ret) { 1172 cmd->engine_error = cmd_q->cmd_error; 1173 goto e_dst; 1174 } 1175 1176 /* ...but we only need AES_BLOCK_SIZE bytes */ 1177 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1178 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1179 } 1180 1181 e_dst: 1182 if (!in_place) 1183 ccp_free_data(&dst, cmd_q); 1184 1185 e_src: 1186 ccp_free_data(&src, cmd_q); 1187 1188 e_ctx: 1189 ccp_dm_free(&ctx); 1190 1191 e_key: 1192 ccp_dm_free(&key); 1193 1194 return ret; 1195 } 1196 1197 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1198 struct ccp_cmd *cmd) 1199 { 1200 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1201 struct ccp_dm_workarea key, ctx; 1202 struct ccp_data src, dst; 1203 struct ccp_op op; 1204 unsigned int unit_size, dm_offset; 1205 bool in_place = false; 1206 int ret; 1207 1208 switch (xts->unit_size) { 1209 case CCP_XTS_AES_UNIT_SIZE_16: 1210 unit_size = 16; 1211 break; 1212 case CCP_XTS_AES_UNIT_SIZE_512: 1213 unit_size = 512; 1214 break; 1215 case CCP_XTS_AES_UNIT_SIZE_1024: 1216 unit_size = 1024; 1217 break; 1218 case CCP_XTS_AES_UNIT_SIZE_2048: 1219 unit_size = 2048; 1220 break; 1221 case CCP_XTS_AES_UNIT_SIZE_4096: 1222 unit_size = 4096; 1223 break; 1224 1225 default: 1226 return -EINVAL; 1227 } 1228 1229 if (xts->key_len != AES_KEYSIZE_128) 1230 return -EINVAL; 1231 1232 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1233 return -EINVAL; 1234 1235 if (xts->iv_len != AES_BLOCK_SIZE) 1236 return -EINVAL; 1237 1238 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1239 return -EINVAL; 1240 1241 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1); 1242 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1); 1243 1244 ret = -EIO; 1245 memset(&op, 0, sizeof(op)); 1246 op.cmd_q = cmd_q; 1247 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1248 op.ksb_key = cmd_q->ksb_key; 1249 op.ksb_ctx = cmd_q->ksb_ctx; 1250 op.init = 1; 1251 op.u.xts.action = xts->action; 1252 op.u.xts.unit_size = xts->unit_size; 1253 1254 /* All supported key sizes fit in a single (32-byte) KSB entry 1255 * and must be in little endian format. Use the 256-bit byte 1256 * swap passthru option to convert from big endian to little 1257 * endian. 1258 */ 1259 ret = ccp_init_dm_workarea(&key, cmd_q, 1260 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1261 DMA_TO_DEVICE); 1262 if (ret) 1263 return ret; 1264 1265 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128; 1266 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1267 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1268 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1269 CCP_PASSTHRU_BYTESWAP_256BIT); 1270 if (ret) { 1271 cmd->engine_error = cmd_q->cmd_error; 1272 goto e_key; 1273 } 1274 1275 /* The AES context fits in a single (32-byte) KSB entry and 1276 * for XTS is already in little endian format so no byte swapping 1277 * is needed. 1278 */ 1279 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1280 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1281 DMA_BIDIRECTIONAL); 1282 if (ret) 1283 goto e_key; 1284 1285 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1286 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1287 CCP_PASSTHRU_BYTESWAP_NOOP); 1288 if (ret) { 1289 cmd->engine_error = cmd_q->cmd_error; 1290 goto e_ctx; 1291 } 1292 1293 /* Prepare the input and output data workareas. For in-place 1294 * operations we need to set the dma direction to BIDIRECTIONAL 1295 * and copy the src workarea to the dst workarea. 1296 */ 1297 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1298 in_place = true; 1299 1300 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1301 unit_size, 1302 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1303 if (ret) 1304 goto e_ctx; 1305 1306 if (in_place) { 1307 dst = src; 1308 } else { 1309 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1310 unit_size, DMA_FROM_DEVICE); 1311 if (ret) 1312 goto e_src; 1313 } 1314 1315 /* Send data to the CCP AES engine */ 1316 while (src.sg_wa.bytes_left) { 1317 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1318 if (!src.sg_wa.bytes_left) 1319 op.eom = 1; 1320 1321 ret = ccp_perform_xts_aes(&op); 1322 if (ret) { 1323 cmd->engine_error = cmd_q->cmd_error; 1324 goto e_dst; 1325 } 1326 1327 ccp_process_data(&src, &dst, &op); 1328 } 1329 1330 /* Retrieve the AES context - convert from LE to BE using 1331 * 32-byte (256-bit) byteswapping 1332 */ 1333 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1334 CCP_PASSTHRU_BYTESWAP_256BIT); 1335 if (ret) { 1336 cmd->engine_error = cmd_q->cmd_error; 1337 goto e_dst; 1338 } 1339 1340 /* ...but we only need AES_BLOCK_SIZE bytes */ 1341 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1342 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1343 1344 e_dst: 1345 if (!in_place) 1346 ccp_free_data(&dst, cmd_q); 1347 1348 e_src: 1349 ccp_free_data(&src, cmd_q); 1350 1351 e_ctx: 1352 ccp_dm_free(&ctx); 1353 1354 e_key: 1355 ccp_dm_free(&key); 1356 1357 return ret; 1358 } 1359 1360 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1361 { 1362 struct ccp_sha_engine *sha = &cmd->u.sha; 1363 struct ccp_dm_workarea ctx; 1364 struct ccp_data src; 1365 struct ccp_op op; 1366 int ret; 1367 1368 if (sha->ctx_len != CCP_SHA_CTXSIZE) 1369 return -EINVAL; 1370 1371 if (!sha->ctx) 1372 return -EINVAL; 1373 1374 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) 1375 return -EINVAL; 1376 1377 if (!sha->src_len) { 1378 const u8 *sha_zero; 1379 1380 /* Not final, just return */ 1381 if (!sha->final) 1382 return 0; 1383 1384 /* CCP can't do a zero length sha operation so the caller 1385 * must buffer the data. 1386 */ 1387 if (sha->msg_bits) 1388 return -EINVAL; 1389 1390 /* A sha operation for a message with a total length of zero, 1391 * return known result. 1392 */ 1393 switch (sha->type) { 1394 case CCP_SHA_TYPE_1: 1395 sha_zero = ccp_sha1_zero; 1396 break; 1397 case CCP_SHA_TYPE_224: 1398 sha_zero = ccp_sha224_zero; 1399 break; 1400 case CCP_SHA_TYPE_256: 1401 sha_zero = ccp_sha256_zero; 1402 break; 1403 default: 1404 return -EINVAL; 1405 } 1406 1407 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1408 sha->ctx_len, 1); 1409 1410 return 0; 1411 } 1412 1413 if (!sha->src) 1414 return -EINVAL; 1415 1416 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1); 1417 1418 memset(&op, 0, sizeof(op)); 1419 op.cmd_q = cmd_q; 1420 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1421 op.ksb_ctx = cmd_q->ksb_ctx; 1422 op.u.sha.type = sha->type; 1423 op.u.sha.msg_bits = sha->msg_bits; 1424 1425 /* The SHA context fits in a single (32-byte) KSB entry and 1426 * must be in little endian format. Use the 256-bit byte swap 1427 * passthru option to convert from big endian to little endian. 1428 */ 1429 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1430 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES, 1431 DMA_BIDIRECTIONAL); 1432 if (ret) 1433 return ret; 1434 1435 if (sha->first) { 1436 const __be32 *init; 1437 1438 switch (sha->type) { 1439 case CCP_SHA_TYPE_1: 1440 init = ccp_sha1_init; 1441 break; 1442 case CCP_SHA_TYPE_224: 1443 init = ccp_sha224_init; 1444 break; 1445 case CCP_SHA_TYPE_256: 1446 init = ccp_sha256_init; 1447 break; 1448 default: 1449 ret = -EINVAL; 1450 goto e_ctx; 1451 } 1452 memcpy(ctx.address, init, CCP_SHA_CTXSIZE); 1453 } else { 1454 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1455 } 1456 1457 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1458 CCP_PASSTHRU_BYTESWAP_256BIT); 1459 if (ret) { 1460 cmd->engine_error = cmd_q->cmd_error; 1461 goto e_ctx; 1462 } 1463 1464 /* Send data to the CCP SHA engine */ 1465 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1466 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); 1467 if (ret) 1468 goto e_ctx; 1469 1470 while (src.sg_wa.bytes_left) { 1471 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); 1472 if (sha->final && !src.sg_wa.bytes_left) 1473 op.eom = 1; 1474 1475 ret = ccp_perform_sha(&op); 1476 if (ret) { 1477 cmd->engine_error = cmd_q->cmd_error; 1478 goto e_data; 1479 } 1480 1481 ccp_process_data(&src, NULL, &op); 1482 } 1483 1484 /* Retrieve the SHA context - convert from LE to BE using 1485 * 32-byte (256-bit) byteswapping to BE 1486 */ 1487 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1488 CCP_PASSTHRU_BYTESWAP_256BIT); 1489 if (ret) { 1490 cmd->engine_error = cmd_q->cmd_error; 1491 goto e_data; 1492 } 1493 1494 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1495 1496 if (sha->final && sha->opad) { 1497 /* HMAC operation, recursively perform final SHA */ 1498 struct ccp_cmd hmac_cmd; 1499 struct scatterlist sg; 1500 u64 block_size, digest_size; 1501 u8 *hmac_buf; 1502 1503 switch (sha->type) { 1504 case CCP_SHA_TYPE_1: 1505 block_size = SHA1_BLOCK_SIZE; 1506 digest_size = SHA1_DIGEST_SIZE; 1507 break; 1508 case CCP_SHA_TYPE_224: 1509 block_size = SHA224_BLOCK_SIZE; 1510 digest_size = SHA224_DIGEST_SIZE; 1511 break; 1512 case CCP_SHA_TYPE_256: 1513 block_size = SHA256_BLOCK_SIZE; 1514 digest_size = SHA256_DIGEST_SIZE; 1515 break; 1516 default: 1517 ret = -EINVAL; 1518 goto e_data; 1519 } 1520 1521 if (sha->opad_len != block_size) { 1522 ret = -EINVAL; 1523 goto e_data; 1524 } 1525 1526 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1527 if (!hmac_buf) { 1528 ret = -ENOMEM; 1529 goto e_data; 1530 } 1531 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1532 1533 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1534 memcpy(hmac_buf + block_size, ctx.address, digest_size); 1535 1536 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1537 hmac_cmd.engine = CCP_ENGINE_SHA; 1538 hmac_cmd.u.sha.type = sha->type; 1539 hmac_cmd.u.sha.ctx = sha->ctx; 1540 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1541 hmac_cmd.u.sha.src = &sg; 1542 hmac_cmd.u.sha.src_len = block_size + digest_size; 1543 hmac_cmd.u.sha.opad = NULL; 1544 hmac_cmd.u.sha.opad_len = 0; 1545 hmac_cmd.u.sha.first = 1; 1546 hmac_cmd.u.sha.final = 1; 1547 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1548 1549 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1550 if (ret) 1551 cmd->engine_error = hmac_cmd.engine_error; 1552 1553 kfree(hmac_buf); 1554 } 1555 1556 e_data: 1557 ccp_free_data(&src, cmd_q); 1558 1559 e_ctx: 1560 ccp_dm_free(&ctx); 1561 1562 return ret; 1563 } 1564 1565 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1566 { 1567 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1568 struct ccp_dm_workarea exp, src; 1569 struct ccp_data dst; 1570 struct ccp_op op; 1571 unsigned int ksb_count, i_len, o_len; 1572 int ret; 1573 1574 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1575 return -EINVAL; 1576 1577 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1578 return -EINVAL; 1579 1580 /* The RSA modulus must precede the message being acted upon, so 1581 * it must be copied to a DMA area where the message and the 1582 * modulus can be concatenated. Therefore the input buffer 1583 * length required is twice the output buffer length (which 1584 * must be a multiple of 256-bits). 1585 */ 1586 o_len = ((rsa->key_size + 255) / 256) * 32; 1587 i_len = o_len * 2; 1588 1589 ksb_count = o_len / CCP_KSB_BYTES; 1590 1591 memset(&op, 0, sizeof(op)); 1592 op.cmd_q = cmd_q; 1593 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1594 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count); 1595 if (!op.ksb_key) 1596 return -EIO; 1597 1598 /* The RSA exponent may span multiple (32-byte) KSB entries and must 1599 * be in little endian format. Reverse copy each 32-byte chunk 1600 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1601 * and each byte within that chunk and do not perform any byte swap 1602 * operations on the passthru operation. 1603 */ 1604 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1605 if (ret) 1606 goto e_ksb; 1607 1608 ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, 1609 false); 1610 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, 1611 CCP_PASSTHRU_BYTESWAP_NOOP); 1612 if (ret) { 1613 cmd->engine_error = cmd_q->cmd_error; 1614 goto e_exp; 1615 } 1616 1617 /* Concatenate the modulus and the message. Both the modulus and 1618 * the operands must be in little endian format. Since the input 1619 * is in big endian format it must be converted. 1620 */ 1621 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1622 if (ret) 1623 goto e_exp; 1624 1625 ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, 1626 false); 1627 src.address += o_len; /* Adjust the address for the copy operation */ 1628 ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, 1629 false); 1630 src.address -= o_len; /* Reset the address to original value */ 1631 1632 /* Prepare the output area for the operation */ 1633 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1634 o_len, DMA_FROM_DEVICE); 1635 if (ret) 1636 goto e_src; 1637 1638 op.soc = 1; 1639 op.src.u.dma.address = src.dma.address; 1640 op.src.u.dma.offset = 0; 1641 op.src.u.dma.length = i_len; 1642 op.dst.u.dma.address = dst.dm_wa.dma.address; 1643 op.dst.u.dma.offset = 0; 1644 op.dst.u.dma.length = o_len; 1645 1646 op.u.rsa.mod_size = rsa->key_size; 1647 op.u.rsa.input_len = i_len; 1648 1649 ret = ccp_perform_rsa(&op); 1650 if (ret) { 1651 cmd->engine_error = cmd_q->cmd_error; 1652 goto e_dst; 1653 } 1654 1655 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); 1656 1657 e_dst: 1658 ccp_free_data(&dst, cmd_q); 1659 1660 e_src: 1661 ccp_dm_free(&src); 1662 1663 e_exp: 1664 ccp_dm_free(&exp); 1665 1666 e_ksb: 1667 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count); 1668 1669 return ret; 1670 } 1671 1672 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1673 struct ccp_cmd *cmd) 1674 { 1675 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1676 struct ccp_dm_workarea mask; 1677 struct ccp_data src, dst; 1678 struct ccp_op op; 1679 bool in_place = false; 1680 unsigned int i; 1681 int ret; 1682 1683 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1684 return -EINVAL; 1685 1686 if (!pt->src || !pt->dst) 1687 return -EINVAL; 1688 1689 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1690 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1691 return -EINVAL; 1692 if (!pt->mask) 1693 return -EINVAL; 1694 } 1695 1696 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1); 1697 1698 memset(&op, 0, sizeof(op)); 1699 op.cmd_q = cmd_q; 1700 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1701 1702 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1703 /* Load the mask */ 1704 op.ksb_key = cmd_q->ksb_key; 1705 1706 ret = ccp_init_dm_workarea(&mask, cmd_q, 1707 CCP_PASSTHRU_KSB_COUNT * 1708 CCP_KSB_BYTES, 1709 DMA_TO_DEVICE); 1710 if (ret) 1711 return ret; 1712 1713 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1714 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key, 1715 CCP_PASSTHRU_BYTESWAP_NOOP); 1716 if (ret) { 1717 cmd->engine_error = cmd_q->cmd_error; 1718 goto e_mask; 1719 } 1720 } 1721 1722 /* Prepare the input and output data workareas. For in-place 1723 * operations we need to set the dma direction to BIDIRECTIONAL 1724 * and copy the src workarea to the dst workarea. 1725 */ 1726 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1727 in_place = true; 1728 1729 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1730 CCP_PASSTHRU_MASKSIZE, 1731 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1732 if (ret) 1733 goto e_mask; 1734 1735 if (in_place) { 1736 dst = src; 1737 } else { 1738 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1739 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1740 if (ret) 1741 goto e_src; 1742 } 1743 1744 /* Send data to the CCP Passthru engine 1745 * Because the CCP engine works on a single source and destination 1746 * dma address at a time, each entry in the source scatterlist 1747 * (after the dma_map_sg call) must be less than or equal to the 1748 * (remaining) length in the destination scatterlist entry and the 1749 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1750 */ 1751 dst.sg_wa.sg_used = 0; 1752 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1753 if (!dst.sg_wa.sg || 1754 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1755 ret = -EINVAL; 1756 goto e_dst; 1757 } 1758 1759 if (i == src.sg_wa.dma_count) { 1760 op.eom = 1; 1761 op.soc = 1; 1762 } 1763 1764 op.src.type = CCP_MEMTYPE_SYSTEM; 1765 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1766 op.src.u.dma.offset = 0; 1767 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1768 1769 op.dst.type = CCP_MEMTYPE_SYSTEM; 1770 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1771 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1772 op.dst.u.dma.length = op.src.u.dma.length; 1773 1774 ret = ccp_perform_passthru(&op); 1775 if (ret) { 1776 cmd->engine_error = cmd_q->cmd_error; 1777 goto e_dst; 1778 } 1779 1780 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1781 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1782 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1783 dst.sg_wa.sg_used = 0; 1784 } 1785 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1786 } 1787 1788 e_dst: 1789 if (!in_place) 1790 ccp_free_data(&dst, cmd_q); 1791 1792 e_src: 1793 ccp_free_data(&src, cmd_q); 1794 1795 e_mask: 1796 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1797 ccp_dm_free(&mask); 1798 1799 return ret; 1800 } 1801 1802 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1803 { 1804 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1805 struct ccp_dm_workarea src, dst; 1806 struct ccp_op op; 1807 int ret; 1808 u8 *save; 1809 1810 if (!ecc->u.mm.operand_1 || 1811 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1812 return -EINVAL; 1813 1814 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1815 if (!ecc->u.mm.operand_2 || 1816 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1817 return -EINVAL; 1818 1819 if (!ecc->u.mm.result || 1820 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1821 return -EINVAL; 1822 1823 memset(&op, 0, sizeof(op)); 1824 op.cmd_q = cmd_q; 1825 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1826 1827 /* Concatenate the modulus and the operands. Both the modulus and 1828 * the operands must be in little endian format. Since the input 1829 * is in big endian format it must be converted and placed in a 1830 * fixed length buffer. 1831 */ 1832 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1833 DMA_TO_DEVICE); 1834 if (ret) 1835 return ret; 1836 1837 /* Save the workarea address since it is updated in order to perform 1838 * the concatenation 1839 */ 1840 save = src.address; 1841 1842 /* Copy the ECC modulus */ 1843 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1844 CCP_ECC_OPERAND_SIZE, false); 1845 src.address += CCP_ECC_OPERAND_SIZE; 1846 1847 /* Copy the first operand */ 1848 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, 1849 ecc->u.mm.operand_1_len, 1850 CCP_ECC_OPERAND_SIZE, false); 1851 src.address += CCP_ECC_OPERAND_SIZE; 1852 1853 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1854 /* Copy the second operand */ 1855 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, 1856 ecc->u.mm.operand_2_len, 1857 CCP_ECC_OPERAND_SIZE, false); 1858 src.address += CCP_ECC_OPERAND_SIZE; 1859 } 1860 1861 /* Restore the workarea address */ 1862 src.address = save; 1863 1864 /* Prepare the output area for the operation */ 1865 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1866 DMA_FROM_DEVICE); 1867 if (ret) 1868 goto e_src; 1869 1870 op.soc = 1; 1871 op.src.u.dma.address = src.dma.address; 1872 op.src.u.dma.offset = 0; 1873 op.src.u.dma.length = src.length; 1874 op.dst.u.dma.address = dst.dma.address; 1875 op.dst.u.dma.offset = 0; 1876 op.dst.u.dma.length = dst.length; 1877 1878 op.u.ecc.function = cmd->u.ecc.function; 1879 1880 ret = ccp_perform_ecc(&op); 1881 if (ret) { 1882 cmd->engine_error = cmd_q->cmd_error; 1883 goto e_dst; 1884 } 1885 1886 ecc->ecc_result = le16_to_cpup( 1887 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1888 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1889 ret = -EIO; 1890 goto e_dst; 1891 } 1892 1893 /* Save the ECC result */ 1894 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); 1895 1896 e_dst: 1897 ccp_dm_free(&dst); 1898 1899 e_src: 1900 ccp_dm_free(&src); 1901 1902 return ret; 1903 } 1904 1905 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1906 { 1907 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1908 struct ccp_dm_workarea src, dst; 1909 struct ccp_op op; 1910 int ret; 1911 u8 *save; 1912 1913 if (!ecc->u.pm.point_1.x || 1914 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1915 !ecc->u.pm.point_1.y || 1916 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1917 return -EINVAL; 1918 1919 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1920 if (!ecc->u.pm.point_2.x || 1921 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1922 !ecc->u.pm.point_2.y || 1923 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1924 return -EINVAL; 1925 } else { 1926 if (!ecc->u.pm.domain_a || 1927 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1928 return -EINVAL; 1929 1930 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1931 if (!ecc->u.pm.scalar || 1932 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1933 return -EINVAL; 1934 } 1935 1936 if (!ecc->u.pm.result.x || 1937 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1938 !ecc->u.pm.result.y || 1939 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1940 return -EINVAL; 1941 1942 memset(&op, 0, sizeof(op)); 1943 op.cmd_q = cmd_q; 1944 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1945 1946 /* Concatenate the modulus and the operands. Both the modulus and 1947 * the operands must be in little endian format. Since the input 1948 * is in big endian format it must be converted and placed in a 1949 * fixed length buffer. 1950 */ 1951 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1952 DMA_TO_DEVICE); 1953 if (ret) 1954 return ret; 1955 1956 /* Save the workarea address since it is updated in order to perform 1957 * the concatenation 1958 */ 1959 save = src.address; 1960 1961 /* Copy the ECC modulus */ 1962 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1963 CCP_ECC_OPERAND_SIZE, false); 1964 src.address += CCP_ECC_OPERAND_SIZE; 1965 1966 /* Copy the first point X and Y coordinate */ 1967 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, 1968 ecc->u.pm.point_1.x_len, 1969 CCP_ECC_OPERAND_SIZE, false); 1970 src.address += CCP_ECC_OPERAND_SIZE; 1971 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, 1972 ecc->u.pm.point_1.y_len, 1973 CCP_ECC_OPERAND_SIZE, false); 1974 src.address += CCP_ECC_OPERAND_SIZE; 1975 1976 /* Set the first point Z coordianate to 1 */ 1977 *src.address = 0x01; 1978 src.address += CCP_ECC_OPERAND_SIZE; 1979 1980 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1981 /* Copy the second point X and Y coordinate */ 1982 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, 1983 ecc->u.pm.point_2.x_len, 1984 CCP_ECC_OPERAND_SIZE, false); 1985 src.address += CCP_ECC_OPERAND_SIZE; 1986 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, 1987 ecc->u.pm.point_2.y_len, 1988 CCP_ECC_OPERAND_SIZE, false); 1989 src.address += CCP_ECC_OPERAND_SIZE; 1990 1991 /* Set the second point Z coordianate to 1 */ 1992 *src.address = 0x01; 1993 src.address += CCP_ECC_OPERAND_SIZE; 1994 } else { 1995 /* Copy the Domain "a" parameter */ 1996 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, 1997 ecc->u.pm.domain_a_len, 1998 CCP_ECC_OPERAND_SIZE, false); 1999 src.address += CCP_ECC_OPERAND_SIZE; 2000 2001 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2002 /* Copy the scalar value */ 2003 ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, 2004 ecc->u.pm.scalar_len, 2005 CCP_ECC_OPERAND_SIZE, false); 2006 src.address += CCP_ECC_OPERAND_SIZE; 2007 } 2008 } 2009 2010 /* Restore the workarea address */ 2011 src.address = save; 2012 2013 /* Prepare the output area for the operation */ 2014 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2015 DMA_FROM_DEVICE); 2016 if (ret) 2017 goto e_src; 2018 2019 op.soc = 1; 2020 op.src.u.dma.address = src.dma.address; 2021 op.src.u.dma.offset = 0; 2022 op.src.u.dma.length = src.length; 2023 op.dst.u.dma.address = dst.dma.address; 2024 op.dst.u.dma.offset = 0; 2025 op.dst.u.dma.length = dst.length; 2026 2027 op.u.ecc.function = cmd->u.ecc.function; 2028 2029 ret = ccp_perform_ecc(&op); 2030 if (ret) { 2031 cmd->engine_error = cmd_q->cmd_error; 2032 goto e_dst; 2033 } 2034 2035 ecc->ecc_result = le16_to_cpup( 2036 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2037 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2038 ret = -EIO; 2039 goto e_dst; 2040 } 2041 2042 /* Save the workarea address since it is updated as we walk through 2043 * to copy the point math result 2044 */ 2045 save = dst.address; 2046 2047 /* Save the ECC result X and Y coordinates */ 2048 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, 2049 CCP_ECC_MODULUS_BYTES); 2050 dst.address += CCP_ECC_OUTPUT_SIZE; 2051 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, 2052 CCP_ECC_MODULUS_BYTES); 2053 dst.address += CCP_ECC_OUTPUT_SIZE; 2054 2055 /* Restore the workarea address */ 2056 dst.address = save; 2057 2058 e_dst: 2059 ccp_dm_free(&dst); 2060 2061 e_src: 2062 ccp_dm_free(&src); 2063 2064 return ret; 2065 } 2066 2067 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2068 { 2069 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2070 2071 ecc->ecc_result = 0; 2072 2073 if (!ecc->mod || 2074 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2075 return -EINVAL; 2076 2077 switch (ecc->function) { 2078 case CCP_ECC_FUNCTION_MMUL_384BIT: 2079 case CCP_ECC_FUNCTION_MADD_384BIT: 2080 case CCP_ECC_FUNCTION_MINV_384BIT: 2081 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2082 2083 case CCP_ECC_FUNCTION_PADD_384BIT: 2084 case CCP_ECC_FUNCTION_PMUL_384BIT: 2085 case CCP_ECC_FUNCTION_PDBL_384BIT: 2086 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2087 2088 default: 2089 return -EINVAL; 2090 } 2091 } 2092 2093 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2094 { 2095 int ret; 2096 2097 cmd->engine_error = 0; 2098 cmd_q->cmd_error = 0; 2099 cmd_q->int_rcvd = 0; 2100 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 2101 2102 switch (cmd->engine) { 2103 case CCP_ENGINE_AES: 2104 ret = ccp_run_aes_cmd(cmd_q, cmd); 2105 break; 2106 case CCP_ENGINE_XTS_AES_128: 2107 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2108 break; 2109 case CCP_ENGINE_SHA: 2110 ret = ccp_run_sha_cmd(cmd_q, cmd); 2111 break; 2112 case CCP_ENGINE_RSA: 2113 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2114 break; 2115 case CCP_ENGINE_PASSTHRU: 2116 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2117 break; 2118 case CCP_ENGINE_ECC: 2119 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2120 break; 2121 default: 2122 ret = -EINVAL; 2123 } 2124 2125 return ret; 2126 } 2127