1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * talitos - Freescale Integrated Security Engine (SEC) device driver 4 * 5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 6 * 7 * Scatterlist Crypto API glue code copied from files with the following: 8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 9 * 10 * Crypto algorithm registration code copied from hifn driver: 11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 12 * All rights reserved. 13 */ 14 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/device.h> 18 #include <linux/interrupt.h> 19 #include <linux/crypto.h> 20 #include <linux/hw_random.h> 21 #include <linux/of.h> 22 #include <linux/of_irq.h> 23 #include <linux/platform_device.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/io.h> 26 #include <linux/spinlock.h> 27 #include <linux/rtnetlink.h> 28 #include <linux/slab.h> 29 30 #include <crypto/algapi.h> 31 #include <crypto/aes.h> 32 #include <crypto/internal/des.h> 33 #include <crypto/sha1.h> 34 #include <crypto/sha2.h> 35 #include <crypto/md5.h> 36 #include <crypto/internal/aead.h> 37 #include <crypto/authenc.h> 38 #include <crypto/internal/skcipher.h> 39 #include <crypto/hash.h> 40 #include <crypto/internal/hash.h> 41 #include <crypto/scatterwalk.h> 42 43 #include "talitos.h" 44 45 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 46 unsigned int len, bool is_sec1) 47 { 48 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 49 if (is_sec1) { 50 ptr->len1 = cpu_to_be16(len); 51 } else { 52 ptr->len = cpu_to_be16(len); 53 ptr->eptr = upper_32_bits(dma_addr); 54 } 55 } 56 57 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 58 struct talitos_ptr *src_ptr, bool is_sec1) 59 { 60 dst_ptr->ptr = src_ptr->ptr; 61 if (is_sec1) { 62 dst_ptr->len1 = src_ptr->len1; 63 } else { 64 dst_ptr->len = src_ptr->len; 65 dst_ptr->eptr = src_ptr->eptr; 66 } 67 } 68 69 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 70 bool is_sec1) 71 { 72 if (is_sec1) 73 return be16_to_cpu(ptr->len1); 74 else 75 return be16_to_cpu(ptr->len); 76 } 77 78 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, 79 bool is_sec1) 80 { 81 if (!is_sec1) 82 ptr->j_extent = val; 83 } 84 85 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) 86 { 87 if (!is_sec1) 88 ptr->j_extent |= val; 89 } 90 91 /* 92 * map virtual single (contiguous) pointer to h/w descriptor pointer 93 */ 94 static void __map_single_talitos_ptr(struct device *dev, 95 struct talitos_ptr *ptr, 96 unsigned int len, void *data, 97 enum dma_data_direction dir, 98 unsigned long attrs) 99 { 100 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); 101 struct talitos_private *priv = dev_get_drvdata(dev); 102 bool is_sec1 = has_ftr_sec1(priv); 103 104 to_talitos_ptr(ptr, dma_addr, len, is_sec1); 105 } 106 107 static void map_single_talitos_ptr(struct device *dev, 108 struct talitos_ptr *ptr, 109 unsigned int len, void *data, 110 enum dma_data_direction dir) 111 { 112 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); 113 } 114 115 static void map_single_talitos_ptr_nosync(struct device *dev, 116 struct talitos_ptr *ptr, 117 unsigned int len, void *data, 118 enum dma_data_direction dir) 119 { 120 __map_single_talitos_ptr(dev, ptr, len, data, dir, 121 DMA_ATTR_SKIP_CPU_SYNC); 122 } 123 124 /* 125 * unmap bus single (contiguous) h/w descriptor pointer 126 */ 127 static void unmap_single_talitos_ptr(struct device *dev, 128 struct talitos_ptr *ptr, 129 enum dma_data_direction dir) 130 { 131 struct talitos_private *priv = dev_get_drvdata(dev); 132 bool is_sec1 = has_ftr_sec1(priv); 133 134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 135 from_talitos_ptr_len(ptr, is_sec1), dir); 136 } 137 138 static int reset_channel(struct device *dev, int ch) 139 { 140 struct talitos_private *priv = dev_get_drvdata(dev); 141 unsigned int timeout = TALITOS_TIMEOUT; 142 bool is_sec1 = has_ftr_sec1(priv); 143 144 if (is_sec1) { 145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 146 TALITOS1_CCCR_LO_RESET); 147 148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 149 TALITOS1_CCCR_LO_RESET) && --timeout) 150 cpu_relax(); 151 } else { 152 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 153 TALITOS2_CCCR_RESET); 154 155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 156 TALITOS2_CCCR_RESET) && --timeout) 157 cpu_relax(); 158 } 159 160 if (timeout == 0) { 161 dev_err(dev, "failed to reset channel %d\n", ch); 162 return -EIO; 163 } 164 165 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 168 /* enable chaining descriptors */ 169 if (is_sec1) 170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 171 TALITOS_CCCR_LO_NE); 172 173 /* and ICCR writeback, if available */ 174 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 175 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 176 TALITOS_CCCR_LO_IWSE); 177 178 return 0; 179 } 180 181 static int reset_device(struct device *dev) 182 { 183 struct talitos_private *priv = dev_get_drvdata(dev); 184 unsigned int timeout = TALITOS_TIMEOUT; 185 bool is_sec1 = has_ftr_sec1(priv); 186 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 187 188 setbits32(priv->reg + TALITOS_MCR, mcr); 189 190 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 191 && --timeout) 192 cpu_relax(); 193 194 if (priv->irq[1]) { 195 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 196 setbits32(priv->reg + TALITOS_MCR, mcr); 197 } 198 199 if (timeout == 0) { 200 dev_err(dev, "failed to reset device\n"); 201 return -EIO; 202 } 203 204 return 0; 205 } 206 207 /* 208 * Reset and initialize the device 209 */ 210 static int init_device(struct device *dev) 211 { 212 struct talitos_private *priv = dev_get_drvdata(dev); 213 int ch, err; 214 bool is_sec1 = has_ftr_sec1(priv); 215 216 /* 217 * Master reset 218 * errata documentation: warning: certain SEC interrupts 219 * are not fully cleared by writing the MCR:SWR bit, 220 * set bit twice to completely reset 221 */ 222 err = reset_device(dev); 223 if (err) 224 return err; 225 226 err = reset_device(dev); 227 if (err) 228 return err; 229 230 /* reset channels */ 231 for (ch = 0; ch < priv->num_channels; ch++) { 232 err = reset_channel(dev, ch); 233 if (err) 234 return err; 235 } 236 237 /* enable channel done and error interrupts */ 238 if (is_sec1) { 239 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 240 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 241 /* disable parity error check in DEU (erroneous? test vect.) */ 242 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 243 } else { 244 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 245 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 246 } 247 248 /* disable integrity check error interrupts (use writeback instead) */ 249 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 250 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 251 TALITOS_MDEUICR_LO_ICE); 252 253 return 0; 254 } 255 256 static void dma_map_request(struct device *dev, struct talitos_request *request, 257 struct talitos_desc *desc, bool is_sec1) 258 { 259 struct talitos_edesc *edesc = 260 container_of(desc, struct talitos_edesc, desc); 261 dma_addr_t dma_desc, prev_dma_desc; 262 struct talitos_edesc *prev_edesc = NULL; 263 264 if (is_sec1) { 265 while (edesc) { 266 edesc->desc.hdr1 = edesc->desc.hdr; 267 268 dma_desc = dma_map_single(dev, &edesc->desc.hdr1, 269 TALITOS_DESC_SIZE, 270 DMA_BIDIRECTIONAL); 271 272 if (!prev_edesc) { 273 request->dma_desc = dma_desc; 274 goto next; 275 } 276 277 /* Chain in any previous descriptors. */ 278 279 prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); 280 281 dma_sync_single_for_device(dev, prev_dma_desc, 282 TALITOS_DESC_SIZE, 283 DMA_TO_DEVICE); 284 285 next: 286 prev_edesc = edesc; 287 prev_dma_desc = dma_desc; 288 edesc = edesc->next_desc; 289 } 290 } else { 291 request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, 292 DMA_BIDIRECTIONAL); 293 } 294 } 295 296 /** 297 * talitos_submit - submits a descriptor to the device for processing 298 * @dev: the SEC device to be used 299 * @ch: the SEC device channel to be used 300 * @desc: the descriptor to be processed by the device 301 * @callback: whom to call when processing is complete 302 * @context: a handle for use by caller (optional) 303 * 304 * desc must contain valid dma-mapped (bus physical) address pointers. 305 * callback must check err and feedback in descriptor header 306 * for device processing status. 307 */ 308 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 309 void (*callback)(struct device *dev, 310 struct talitos_desc *desc, 311 void *context, int error), 312 void *context) 313 { 314 struct talitos_private *priv = dev_get_drvdata(dev); 315 struct talitos_request *request; 316 unsigned long flags; 317 int head; 318 bool is_sec1 = has_ftr_sec1(priv); 319 320 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 321 322 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 323 /* h/w fifo is full */ 324 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 325 return -EAGAIN; 326 } 327 328 head = priv->chan[ch].head; 329 request = &priv->chan[ch].fifo[head]; 330 331 /* map descriptor and save caller data */ 332 dma_map_request(dev, request, desc, is_sec1); 333 request->callback = callback; 334 request->context = context; 335 336 /* increment fifo head */ 337 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 338 339 smp_wmb(); 340 request->desc = desc; 341 342 /* GO! */ 343 wmb(); 344 out_be32(priv->chan[ch].reg + TALITOS_FF, 345 upper_32_bits(request->dma_desc)); 346 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 347 lower_32_bits(request->dma_desc)); 348 349 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 350 351 return -EINPROGRESS; 352 } 353 354 static __be32 get_request_hdr(struct device *dev, 355 struct talitos_request *request, bool is_sec1) 356 { 357 struct talitos_edesc *edesc; 358 dma_addr_t dma_desc; 359 360 if (!is_sec1) { 361 dma_sync_single_for_cpu(dev, request->dma_desc, 362 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 363 364 return request->desc->hdr; 365 } 366 367 edesc = container_of(request->desc, struct talitos_edesc, desc); 368 dma_desc = request->dma_desc; 369 while (edesc->next_desc) { 370 dma_desc = be32_to_cpu(edesc->desc.next_desc); 371 edesc = edesc->next_desc; 372 } 373 374 dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, 375 DMA_BIDIRECTIONAL); 376 377 return edesc->desc.hdr1; 378 } 379 380 static void dma_unmap_request(struct device *dev, 381 struct talitos_request *request, bool is_sec1) 382 { 383 struct talitos_edesc *edesc; 384 385 if (is_sec1) { 386 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 387 DMA_BIDIRECTIONAL); 388 edesc = container_of(request->desc, struct talitos_edesc, desc); 389 while (edesc->next_desc) { 390 dma_unmap_single(dev, 391 be32_to_cpu(edesc->desc.next_desc), 392 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 393 edesc = edesc->next_desc; 394 } 395 } else { 396 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 397 DMA_BIDIRECTIONAL); 398 } 399 } 400 401 /* 402 * process what was done, notify callback of error if not 403 */ 404 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 405 { 406 struct talitos_private *priv = dev_get_drvdata(dev); 407 struct talitos_request *request, saved_req; 408 unsigned long flags; 409 int tail, status; 410 bool is_sec1 = has_ftr_sec1(priv); 411 412 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 413 414 tail = priv->chan[ch].tail; 415 while (priv->chan[ch].fifo[tail].desc) { 416 __be32 hdr; 417 418 request = &priv->chan[ch].fifo[tail]; 419 420 /* descriptors with their done bits set don't get the error */ 421 rmb(); 422 hdr = get_request_hdr(dev, request, is_sec1); 423 424 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 425 status = 0; 426 else 427 if (!error) 428 break; 429 else 430 status = error; 431 432 dma_unmap_request(dev, request, is_sec1); 433 434 /* copy entries so we can call callback outside lock */ 435 saved_req.desc = request->desc; 436 saved_req.callback = request->callback; 437 saved_req.context = request->context; 438 439 /* release request entry in fifo */ 440 smp_wmb(); 441 request->desc = NULL; 442 443 /* increment fifo tail */ 444 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 445 446 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 447 448 atomic_dec(&priv->chan[ch].submit_count); 449 450 saved_req.callback(dev, saved_req.desc, saved_req.context, 451 status); 452 /* channel may resume processing in single desc error case */ 453 if (error && !reset_ch && status == error) 454 return; 455 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 456 tail = priv->chan[ch].tail; 457 } 458 459 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 460 } 461 462 /* 463 * process completed requests for channels that have done status 464 */ 465 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 466 static void talitos1_done_##name(unsigned long data) \ 467 { \ 468 struct device *dev = (struct device *)data; \ 469 struct talitos_private *priv = dev_get_drvdata(dev); \ 470 unsigned long flags; \ 471 \ 472 if (ch_done_mask & 0x10000000) \ 473 flush_channel(dev, 0, 0, 0); \ 474 if (ch_done_mask & 0x40000000) \ 475 flush_channel(dev, 1, 0, 0); \ 476 if (ch_done_mask & 0x00010000) \ 477 flush_channel(dev, 2, 0, 0); \ 478 if (ch_done_mask & 0x00040000) \ 479 flush_channel(dev, 3, 0, 0); \ 480 \ 481 /* At this point, all completed channels have been processed */ \ 482 /* Unmask done interrupts for channels completed later on. */ \ 483 spin_lock_irqsave(&priv->reg_lock, flags); \ 484 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 485 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 486 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 487 } 488 489 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 490 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) 491 492 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 493 static void talitos2_done_##name(unsigned long data) \ 494 { \ 495 struct device *dev = (struct device *)data; \ 496 struct talitos_private *priv = dev_get_drvdata(dev); \ 497 unsigned long flags; \ 498 \ 499 if (ch_done_mask & 1) \ 500 flush_channel(dev, 0, 0, 0); \ 501 if (ch_done_mask & (1 << 2)) \ 502 flush_channel(dev, 1, 0, 0); \ 503 if (ch_done_mask & (1 << 4)) \ 504 flush_channel(dev, 2, 0, 0); \ 505 if (ch_done_mask & (1 << 6)) \ 506 flush_channel(dev, 3, 0, 0); \ 507 \ 508 /* At this point, all completed channels have been processed */ \ 509 /* Unmask done interrupts for channels completed later on. */ \ 510 spin_lock_irqsave(&priv->reg_lock, flags); \ 511 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 512 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 513 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 514 } 515 516 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 517 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) 518 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 519 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 520 521 static __be32 search_desc_hdr_in_request(struct talitos_request *request, 522 dma_addr_t cur_desc, bool is_sec1) 523 { 524 struct talitos_edesc *edesc; 525 526 if (request->dma_desc == cur_desc) { 527 return request->desc->hdr; 528 } else if (is_sec1) { 529 edesc = container_of(request->desc, struct talitos_edesc, desc); 530 while (edesc->next_desc) { 531 if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) 532 return edesc->next_desc->desc.hdr1; 533 edesc = edesc->next_desc; 534 } 535 } 536 return 0; 537 } 538 539 /* 540 * locate current (offending) descriptor 541 */ 542 static __be32 current_desc_hdr(struct device *dev, int ch) 543 { 544 struct talitos_private *priv = dev_get_drvdata(dev); 545 bool is_sec1 = has_ftr_sec1(priv); 546 struct talitos_request *request; 547 int tail, iter; 548 dma_addr_t cur_desc; 549 __be32 hdr = 0; 550 551 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 552 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 553 554 if (!cur_desc) { 555 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 556 return 0; 557 } 558 559 tail = priv->chan[ch].tail; 560 iter = tail; 561 do { 562 request = &priv->chan[ch].fifo[iter]; 563 564 hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); 565 if (hdr) 566 break; 567 568 iter = (iter + 1) & (priv->fifo_len - 1); 569 } while (iter != tail); 570 571 if (!hdr) 572 dev_err(dev, "couldn't locate current descriptor\n"); 573 574 return hdr; 575 } 576 577 /* 578 * user diagnostics; report root cause of error based on execution unit status 579 */ 580 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) 581 { 582 struct talitos_private *priv = dev_get_drvdata(dev); 583 int i; 584 585 if (!desc_hdr) 586 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); 587 588 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 589 case DESC_HDR_SEL0_AFEU: 590 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 591 in_be32(priv->reg_afeu + TALITOS_EUISR), 592 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 593 break; 594 case DESC_HDR_SEL0_DEU: 595 dev_err(dev, "DEUISR 0x%08x_%08x\n", 596 in_be32(priv->reg_deu + TALITOS_EUISR), 597 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 598 break; 599 case DESC_HDR_SEL0_MDEUA: 600 case DESC_HDR_SEL0_MDEUB: 601 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 602 in_be32(priv->reg_mdeu + TALITOS_EUISR), 603 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 604 break; 605 case DESC_HDR_SEL0_RNG: 606 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 607 in_be32(priv->reg_rngu + TALITOS_ISR), 608 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 609 break; 610 case DESC_HDR_SEL0_PKEU: 611 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 612 in_be32(priv->reg_pkeu + TALITOS_EUISR), 613 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 614 break; 615 case DESC_HDR_SEL0_AESU: 616 dev_err(dev, "AESUISR 0x%08x_%08x\n", 617 in_be32(priv->reg_aesu + TALITOS_EUISR), 618 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 619 break; 620 case DESC_HDR_SEL0_CRCU: 621 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 622 in_be32(priv->reg_crcu + TALITOS_EUISR), 623 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 624 break; 625 case DESC_HDR_SEL0_KEU: 626 dev_err(dev, "KEUISR 0x%08x_%08x\n", 627 in_be32(priv->reg_pkeu + TALITOS_EUISR), 628 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 629 break; 630 } 631 632 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 633 case DESC_HDR_SEL1_MDEUA: 634 case DESC_HDR_SEL1_MDEUB: 635 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 636 in_be32(priv->reg_mdeu + TALITOS_EUISR), 637 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 638 break; 639 case DESC_HDR_SEL1_CRCU: 640 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 641 in_be32(priv->reg_crcu + TALITOS_EUISR), 642 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 643 break; 644 } 645 646 for (i = 0; i < 8; i++) 647 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 648 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 649 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 650 } 651 652 /* 653 * recover from error interrupts 654 */ 655 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 656 { 657 struct talitos_private *priv = dev_get_drvdata(dev); 658 unsigned int timeout = TALITOS_TIMEOUT; 659 int ch, error, reset_dev = 0; 660 u32 v_lo; 661 bool is_sec1 = has_ftr_sec1(priv); 662 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 663 664 for (ch = 0; ch < priv->num_channels; ch++) { 665 /* skip channels without errors */ 666 if (is_sec1) { 667 /* bits 29, 31, 17, 19 */ 668 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 669 continue; 670 } else { 671 if (!(isr & (1 << (ch * 2 + 1)))) 672 continue; 673 } 674 675 error = -EINVAL; 676 677 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 678 679 if (v_lo & TALITOS_CCPSR_LO_DOF) { 680 dev_err(dev, "double fetch fifo overflow error\n"); 681 error = -EAGAIN; 682 reset_ch = 1; 683 } 684 if (v_lo & TALITOS_CCPSR_LO_SOF) { 685 /* h/w dropped descriptor */ 686 dev_err(dev, "single fetch fifo overflow error\n"); 687 error = -EAGAIN; 688 } 689 if (v_lo & TALITOS_CCPSR_LO_MDTE) 690 dev_err(dev, "master data transfer error\n"); 691 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 692 dev_err(dev, is_sec1 ? "pointer not complete error\n" 693 : "s/g data length zero error\n"); 694 if (v_lo & TALITOS_CCPSR_LO_FPZ) 695 dev_err(dev, is_sec1 ? "parity error\n" 696 : "fetch pointer zero error\n"); 697 if (v_lo & TALITOS_CCPSR_LO_IDH) 698 dev_err(dev, "illegal descriptor header error\n"); 699 if (v_lo & TALITOS_CCPSR_LO_IEU) 700 dev_err(dev, is_sec1 ? "static assignment error\n" 701 : "invalid exec unit error\n"); 702 if (v_lo & TALITOS_CCPSR_LO_EU) 703 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 704 if (!is_sec1) { 705 if (v_lo & TALITOS_CCPSR_LO_GB) 706 dev_err(dev, "gather boundary error\n"); 707 if (v_lo & TALITOS_CCPSR_LO_GRL) 708 dev_err(dev, "gather return/length error\n"); 709 if (v_lo & TALITOS_CCPSR_LO_SB) 710 dev_err(dev, "scatter boundary error\n"); 711 if (v_lo & TALITOS_CCPSR_LO_SRL) 712 dev_err(dev, "scatter return/length error\n"); 713 } 714 715 flush_channel(dev, ch, error, reset_ch); 716 717 if (reset_ch) { 718 reset_channel(dev, ch); 719 } else { 720 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 721 TALITOS2_CCCR_CONT); 722 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 723 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 724 TALITOS2_CCCR_CONT) && --timeout) 725 cpu_relax(); 726 if (timeout == 0) { 727 dev_err(dev, "failed to restart channel %d\n", 728 ch); 729 reset_dev = 1; 730 } 731 } 732 } 733 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 734 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 735 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 736 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 737 isr, isr_lo); 738 else 739 dev_err(dev, "done overflow, internal time out, or " 740 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 741 742 /* purge request queues */ 743 for (ch = 0; ch < priv->num_channels; ch++) 744 flush_channel(dev, ch, -EIO, 1); 745 746 /* reset and reinitialize the device */ 747 init_device(dev); 748 } 749 } 750 751 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 752 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 753 { \ 754 struct device *dev = data; \ 755 struct talitos_private *priv = dev_get_drvdata(dev); \ 756 u32 isr, isr_lo; \ 757 unsigned long flags; \ 758 \ 759 spin_lock_irqsave(&priv->reg_lock, flags); \ 760 isr = in_be32(priv->reg + TALITOS_ISR); \ 761 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 762 /* Acknowledge interrupt */ \ 763 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 764 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 765 \ 766 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 767 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 768 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 769 } \ 770 else { \ 771 if (likely(isr & ch_done_mask)) { \ 772 /* mask further done interrupts. */ \ 773 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 774 /* done_task will unmask done interrupts at exit */ \ 775 tasklet_schedule(&priv->done_task[tlet]); \ 776 } \ 777 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 778 } \ 779 \ 780 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 781 IRQ_NONE; \ 782 } 783 784 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 785 786 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 787 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 788 { \ 789 struct device *dev = data; \ 790 struct talitos_private *priv = dev_get_drvdata(dev); \ 791 u32 isr, isr_lo; \ 792 unsigned long flags; \ 793 \ 794 spin_lock_irqsave(&priv->reg_lock, flags); \ 795 isr = in_be32(priv->reg + TALITOS_ISR); \ 796 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 797 /* Acknowledge interrupt */ \ 798 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 799 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 800 \ 801 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 802 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 803 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 804 } \ 805 else { \ 806 if (likely(isr & ch_done_mask)) { \ 807 /* mask further done interrupts. */ \ 808 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 809 /* done_task will unmask done interrupts at exit */ \ 810 tasklet_schedule(&priv->done_task[tlet]); \ 811 } \ 812 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 813 } \ 814 \ 815 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 816 IRQ_NONE; \ 817 } 818 819 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 820 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 821 0) 822 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 823 1) 824 825 /* 826 * hwrng 827 */ 828 static int talitos_rng_data_present(struct hwrng *rng, int wait) 829 { 830 struct device *dev = (struct device *)rng->priv; 831 struct talitos_private *priv = dev_get_drvdata(dev); 832 u32 ofl; 833 int i; 834 835 for (i = 0; i < 20; i++) { 836 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 837 TALITOS_RNGUSR_LO_OFL; 838 if (ofl || !wait) 839 break; 840 udelay(10); 841 } 842 843 return !!ofl; 844 } 845 846 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 847 { 848 struct device *dev = (struct device *)rng->priv; 849 struct talitos_private *priv = dev_get_drvdata(dev); 850 851 /* rng fifo requires 64-bit accesses */ 852 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 853 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 854 855 return sizeof(u32); 856 } 857 858 static int talitos_rng_init(struct hwrng *rng) 859 { 860 struct device *dev = (struct device *)rng->priv; 861 struct talitos_private *priv = dev_get_drvdata(dev); 862 unsigned int timeout = TALITOS_TIMEOUT; 863 864 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 865 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 866 & TALITOS_RNGUSR_LO_RD) 867 && --timeout) 868 cpu_relax(); 869 if (timeout == 0) { 870 dev_err(dev, "failed to reset rng hw\n"); 871 return -ENODEV; 872 } 873 874 /* start generating */ 875 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 876 877 return 0; 878 } 879 880 static int talitos_register_rng(struct device *dev) 881 { 882 struct talitos_private *priv = dev_get_drvdata(dev); 883 int err; 884 885 priv->rng.name = dev_driver_string(dev); 886 priv->rng.init = talitos_rng_init; 887 priv->rng.data_present = talitos_rng_data_present; 888 priv->rng.data_read = talitos_rng_data_read; 889 priv->rng.priv = (unsigned long)dev; 890 891 err = hwrng_register(&priv->rng); 892 if (!err) 893 priv->rng_registered = true; 894 895 return err; 896 } 897 898 static void talitos_unregister_rng(struct device *dev) 899 { 900 struct talitos_private *priv = dev_get_drvdata(dev); 901 902 if (!priv->rng_registered) 903 return; 904 905 hwrng_unregister(&priv->rng); 906 priv->rng_registered = false; 907 } 908 909 /* 910 * crypto alg 911 */ 912 #define TALITOS_CRA_PRIORITY 3000 913 /* 914 * Defines a priority for doing AEAD with descriptors type 915 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP 916 */ 917 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) 918 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 919 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) 920 #else 921 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) 922 #endif 923 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 924 925 struct talitos_ctx { 926 struct device *dev; 927 int ch; 928 __be32 desc_hdr_template; 929 u8 key[TALITOS_MAX_KEY_SIZE]; 930 u8 iv[TALITOS_MAX_IV_LENGTH]; 931 dma_addr_t dma_key; 932 unsigned int keylen; 933 unsigned int enckeylen; 934 unsigned int authkeylen; 935 }; 936 937 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 938 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 939 940 struct talitos_ahash_req_ctx { 941 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 942 unsigned int hw_context_size; 943 u8 buf[2][HASH_MAX_BLOCK_SIZE]; 944 int buf_idx; 945 unsigned int swinit; 946 unsigned int first_request; 947 unsigned int last_request; 948 unsigned int to_hash_later; 949 unsigned int nbuf; 950 struct scatterlist bufsl[2]; 951 struct scatterlist *psrc; 952 }; 953 954 struct talitos_export_state { 955 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 956 u8 buf[HASH_MAX_BLOCK_SIZE]; 957 unsigned int swinit; 958 unsigned int first_request; 959 unsigned int last_request; 960 unsigned int to_hash_later; 961 unsigned int nbuf; 962 }; 963 964 static int aead_setkey(struct crypto_aead *authenc, 965 const u8 *key, unsigned int keylen) 966 { 967 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 968 struct device *dev = ctx->dev; 969 struct crypto_authenc_keys keys; 970 971 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 972 goto badkey; 973 974 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 975 goto badkey; 976 977 if (ctx->keylen) 978 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 979 980 memcpy(ctx->key, keys.authkey, keys.authkeylen); 981 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 982 983 ctx->keylen = keys.authkeylen + keys.enckeylen; 984 ctx->enckeylen = keys.enckeylen; 985 ctx->authkeylen = keys.authkeylen; 986 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 987 DMA_TO_DEVICE); 988 989 memzero_explicit(&keys, sizeof(keys)); 990 return 0; 991 992 badkey: 993 memzero_explicit(&keys, sizeof(keys)); 994 return -EINVAL; 995 } 996 997 static int aead_des3_setkey(struct crypto_aead *authenc, 998 const u8 *key, unsigned int keylen) 999 { 1000 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1001 struct device *dev = ctx->dev; 1002 struct crypto_authenc_keys keys; 1003 int err; 1004 1005 err = crypto_authenc_extractkeys(&keys, key, keylen); 1006 if (unlikely(err)) 1007 goto out; 1008 1009 err = -EINVAL; 1010 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 1011 goto out; 1012 1013 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); 1014 if (err) 1015 goto out; 1016 1017 if (ctx->keylen) 1018 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1019 1020 memcpy(ctx->key, keys.authkey, keys.authkeylen); 1021 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 1022 1023 ctx->keylen = keys.authkeylen + keys.enckeylen; 1024 ctx->enckeylen = keys.enckeylen; 1025 ctx->authkeylen = keys.authkeylen; 1026 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 1027 DMA_TO_DEVICE); 1028 1029 out: 1030 memzero_explicit(&keys, sizeof(keys)); 1031 return err; 1032 } 1033 1034 static void talitos_sg_unmap(struct device *dev, 1035 struct talitos_edesc *edesc, 1036 struct scatterlist *src, 1037 struct scatterlist *dst, 1038 unsigned int len, unsigned int offset) 1039 { 1040 struct talitos_private *priv = dev_get_drvdata(dev); 1041 bool is_sec1 = has_ftr_sec1(priv); 1042 unsigned int src_nents = edesc->src_nents ? : 1; 1043 unsigned int dst_nents = edesc->dst_nents ? : 1; 1044 1045 if (is_sec1 && dst && dst_nents > 1) { 1046 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, 1047 len, DMA_FROM_DEVICE); 1048 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, 1049 offset); 1050 } 1051 if (src != dst) { 1052 if (src_nents == 1 || !is_sec1) 1053 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 1054 1055 if (dst && (dst_nents == 1 || !is_sec1)) 1056 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 1057 } else if (src_nents == 1 || !is_sec1) { 1058 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 1059 } 1060 } 1061 1062 static void ipsec_esp_unmap(struct device *dev, 1063 struct talitos_edesc *edesc, 1064 struct aead_request *areq, bool encrypt) 1065 { 1066 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1067 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1068 unsigned int ivsize = crypto_aead_ivsize(aead); 1069 unsigned int authsize = crypto_aead_authsize(aead); 1070 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1071 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; 1072 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; 1073 1074 if (is_ipsec_esp) 1075 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], 1076 DMA_FROM_DEVICE); 1077 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); 1078 1079 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, 1080 cryptlen + authsize, areq->assoclen); 1081 1082 if (edesc->dma_len) 1083 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1084 DMA_BIDIRECTIONAL); 1085 1086 if (!is_ipsec_esp) { 1087 unsigned int dst_nents = edesc->dst_nents ? : 1; 1088 1089 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, 1090 areq->assoclen + cryptlen - ivsize); 1091 } 1092 } 1093 1094 /* 1095 * ipsec_esp descriptor callbacks 1096 */ 1097 static void ipsec_esp_encrypt_done(struct device *dev, 1098 struct talitos_desc *desc, void *context, 1099 int err) 1100 { 1101 struct aead_request *areq = context; 1102 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1103 unsigned int ivsize = crypto_aead_ivsize(authenc); 1104 struct talitos_edesc *edesc; 1105 1106 edesc = container_of(desc, struct talitos_edesc, desc); 1107 1108 ipsec_esp_unmap(dev, edesc, areq, true); 1109 1110 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); 1111 1112 kfree(edesc); 1113 1114 aead_request_complete(areq, err); 1115 } 1116 1117 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 1118 struct talitos_desc *desc, 1119 void *context, int err) 1120 { 1121 struct aead_request *req = context; 1122 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1123 unsigned int authsize = crypto_aead_authsize(authenc); 1124 struct talitos_edesc *edesc; 1125 char *oicv, *icv; 1126 1127 edesc = container_of(desc, struct talitos_edesc, desc); 1128 1129 ipsec_esp_unmap(dev, edesc, req, false); 1130 1131 if (!err) { 1132 /* auth check */ 1133 oicv = edesc->buf + edesc->dma_len; 1134 icv = oicv - authsize; 1135 1136 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 1137 } 1138 1139 kfree(edesc); 1140 1141 aead_request_complete(req, err); 1142 } 1143 1144 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 1145 struct talitos_desc *desc, 1146 void *context, int err) 1147 { 1148 struct aead_request *req = context; 1149 struct talitos_edesc *edesc; 1150 1151 edesc = container_of(desc, struct talitos_edesc, desc); 1152 1153 ipsec_esp_unmap(dev, edesc, req, false); 1154 1155 /* check ICV auth status */ 1156 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1157 DESC_HDR_LO_ICCR1_PASS)) 1158 err = -EBADMSG; 1159 1160 kfree(edesc); 1161 1162 aead_request_complete(req, err); 1163 } 1164 1165 /* 1166 * convert scatterlist to SEC h/w link table format 1167 * stop at cryptlen bytes 1168 */ 1169 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1170 unsigned int offset, int datalen, int elen, 1171 struct talitos_ptr *link_tbl_ptr, int align) 1172 { 1173 int n_sg = elen ? sg_count + 1 : sg_count; 1174 int count = 0; 1175 int cryptlen = datalen + elen; 1176 int padding = ALIGN(cryptlen, align) - cryptlen; 1177 1178 while (cryptlen && sg && n_sg--) { 1179 unsigned int len = sg_dma_len(sg); 1180 1181 if (offset >= len) { 1182 offset -= len; 1183 goto next; 1184 } 1185 1186 len -= offset; 1187 1188 if (len > cryptlen) 1189 len = cryptlen; 1190 1191 if (datalen > 0 && len > datalen) { 1192 to_talitos_ptr(link_tbl_ptr + count, 1193 sg_dma_address(sg) + offset, datalen, 0); 1194 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1195 count++; 1196 len -= datalen; 1197 offset += datalen; 1198 } 1199 to_talitos_ptr(link_tbl_ptr + count, 1200 sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); 1201 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1202 count++; 1203 cryptlen -= len; 1204 datalen -= len; 1205 offset = 0; 1206 1207 next: 1208 sg = sg_next(sg); 1209 } 1210 1211 /* tag end of link table */ 1212 if (count > 0) 1213 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, 1214 DESC_PTR_LNKTBL_RET, 0); 1215 1216 return count; 1217 } 1218 1219 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, 1220 unsigned int len, struct talitos_edesc *edesc, 1221 struct talitos_ptr *ptr, int sg_count, 1222 unsigned int offset, int tbl_off, int elen, 1223 bool force, int align) 1224 { 1225 struct talitos_private *priv = dev_get_drvdata(dev); 1226 bool is_sec1 = has_ftr_sec1(priv); 1227 int aligned_len = ALIGN(len, align); 1228 1229 if (!src) { 1230 to_talitos_ptr(ptr, 0, 0, is_sec1); 1231 return 1; 1232 } 1233 to_talitos_ptr_ext_set(ptr, elen, is_sec1); 1234 if (sg_count == 1 && !force) { 1235 to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); 1236 return sg_count; 1237 } 1238 if (is_sec1) { 1239 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); 1240 return sg_count; 1241 } 1242 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, 1243 &edesc->link_tbl[tbl_off], align); 1244 if (sg_count == 1 && !force) { 1245 /* Only one segment now, so no link tbl needed*/ 1246 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); 1247 return sg_count; 1248 } 1249 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1250 tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); 1251 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); 1252 1253 return sg_count; 1254 } 1255 1256 static int talitos_sg_map(struct device *dev, struct scatterlist *src, 1257 unsigned int len, struct talitos_edesc *edesc, 1258 struct talitos_ptr *ptr, int sg_count, 1259 unsigned int offset, int tbl_off) 1260 { 1261 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, 1262 tbl_off, 0, false, 1); 1263 } 1264 1265 /* 1266 * fill in and submit ipsec_esp descriptor 1267 */ 1268 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1269 bool encrypt, 1270 void (*callback)(struct device *dev, 1271 struct talitos_desc *desc, 1272 void *context, int error)) 1273 { 1274 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1275 unsigned int authsize = crypto_aead_authsize(aead); 1276 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1277 struct device *dev = ctx->dev; 1278 struct talitos_desc *desc = &edesc->desc; 1279 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1280 unsigned int ivsize = crypto_aead_ivsize(aead); 1281 int tbl_off = 0; 1282 int sg_count, ret; 1283 int elen = 0; 1284 bool sync_needed = false; 1285 struct talitos_private *priv = dev_get_drvdata(dev); 1286 bool is_sec1 = has_ftr_sec1(priv); 1287 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; 1288 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; 1289 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; 1290 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; 1291 1292 /* hmac key */ 1293 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); 1294 1295 sg_count = edesc->src_nents ?: 1; 1296 if (is_sec1 && sg_count > 1) 1297 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1298 areq->assoclen + cryptlen); 1299 else 1300 sg_count = dma_map_sg(dev, areq->src, sg_count, 1301 (areq->src == areq->dst) ? 1302 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1303 1304 /* hmac data */ 1305 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, 1306 &desc->ptr[1], sg_count, 0, tbl_off); 1307 1308 if (ret > 1) { 1309 tbl_off += ret; 1310 sync_needed = true; 1311 } 1312 1313 /* cipher iv */ 1314 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); 1315 1316 /* cipher key */ 1317 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, 1318 ctx->enckeylen, is_sec1); 1319 1320 /* 1321 * cipher in 1322 * map and adjust cipher len to aead request cryptlen. 1323 * extent is bytes of HMAC postpended to ciphertext, 1324 * typically 12 for ipsec 1325 */ 1326 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) 1327 elen = authsize; 1328 1329 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], 1330 sg_count, areq->assoclen, tbl_off, elen, 1331 false, 1); 1332 1333 if (ret > 1) { 1334 tbl_off += ret; 1335 sync_needed = true; 1336 } 1337 1338 /* cipher out */ 1339 if (areq->src != areq->dst) { 1340 sg_count = edesc->dst_nents ? : 1; 1341 if (!is_sec1 || sg_count == 1) 1342 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1343 } 1344 1345 if (is_ipsec_esp && encrypt) 1346 elen = authsize; 1347 else 1348 elen = 0; 1349 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], 1350 sg_count, areq->assoclen, tbl_off, elen, 1351 is_ipsec_esp && !encrypt, 1); 1352 tbl_off += ret; 1353 1354 if (!encrypt && is_ipsec_esp) { 1355 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1356 1357 /* Add an entry to the link table for ICV data */ 1358 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); 1359 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); 1360 1361 /* icv data follows link tables */ 1362 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); 1363 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); 1364 sync_needed = true; 1365 } else if (!encrypt) { 1366 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); 1367 sync_needed = true; 1368 } else if (!is_ipsec_esp) { 1369 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], 1370 sg_count, areq->assoclen + cryptlen, tbl_off); 1371 } 1372 1373 /* iv out */ 1374 if (is_ipsec_esp) 1375 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1376 DMA_FROM_DEVICE); 1377 1378 if (sync_needed) 1379 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1380 edesc->dma_len, 1381 DMA_BIDIRECTIONAL); 1382 1383 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1384 if (ret != -EINPROGRESS) { 1385 ipsec_esp_unmap(dev, edesc, areq, encrypt); 1386 kfree(edesc); 1387 } 1388 return ret; 1389 } 1390 1391 /* 1392 * allocate and map the extended descriptor 1393 */ 1394 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1395 struct scatterlist *src, 1396 struct scatterlist *dst, 1397 u8 *iv, 1398 unsigned int assoclen, 1399 unsigned int cryptlen, 1400 unsigned int authsize, 1401 unsigned int ivsize, 1402 int icv_stashing, 1403 u32 cryptoflags, 1404 bool encrypt) 1405 { 1406 struct talitos_edesc *edesc; 1407 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; 1408 dma_addr_t iv_dma = 0; 1409 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1410 GFP_ATOMIC; 1411 struct talitos_private *priv = dev_get_drvdata(dev); 1412 bool is_sec1 = has_ftr_sec1(priv); 1413 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1414 1415 if (cryptlen + authsize > max_len) { 1416 dev_err(dev, "length exceeds h/w max limit\n"); 1417 return ERR_PTR(-EINVAL); 1418 } 1419 1420 if (!dst || dst == src) { 1421 src_len = assoclen + cryptlen + authsize; 1422 src_nents = sg_nents_for_len(src, src_len); 1423 if (src_nents < 0) { 1424 dev_err(dev, "Invalid number of src SG.\n"); 1425 return ERR_PTR(-EINVAL); 1426 } 1427 src_nents = (src_nents == 1) ? 0 : src_nents; 1428 dst_nents = dst ? src_nents : 0; 1429 dst_len = 0; 1430 } else { /* dst && dst != src*/ 1431 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); 1432 src_nents = sg_nents_for_len(src, src_len); 1433 if (src_nents < 0) { 1434 dev_err(dev, "Invalid number of src SG.\n"); 1435 return ERR_PTR(-EINVAL); 1436 } 1437 src_nents = (src_nents == 1) ? 0 : src_nents; 1438 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); 1439 dst_nents = sg_nents_for_len(dst, dst_len); 1440 if (dst_nents < 0) { 1441 dev_err(dev, "Invalid number of dst SG.\n"); 1442 return ERR_PTR(-EINVAL); 1443 } 1444 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1445 } 1446 1447 /* 1448 * allocate space for base edesc plus the link tables, 1449 * allowing for two separate entries for AD and generated ICV (+ 2), 1450 * and space for two sets of ICVs (stashed and generated) 1451 */ 1452 alloc_len = sizeof(struct talitos_edesc); 1453 if (src_nents || dst_nents || !encrypt) { 1454 if (is_sec1) 1455 dma_len = (src_nents ? src_len : 0) + 1456 (dst_nents ? dst_len : 0) + authsize; 1457 else 1458 dma_len = (src_nents + dst_nents + 2) * 1459 sizeof(struct talitos_ptr) + authsize; 1460 alloc_len += dma_len; 1461 } else { 1462 dma_len = 0; 1463 } 1464 alloc_len += icv_stashing ? authsize : 0; 1465 alloc_len += ivsize; 1466 1467 edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); 1468 if (!edesc) 1469 return ERR_PTR(-ENOMEM); 1470 if (ivsize) { 1471 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); 1472 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1473 } 1474 memset(&edesc->desc, 0, sizeof(edesc->desc)); 1475 1476 edesc->src_nents = src_nents; 1477 edesc->dst_nents = dst_nents; 1478 edesc->iv_dma = iv_dma; 1479 edesc->dma_len = dma_len; 1480 edesc->next_desc = NULL; 1481 if (dma_len) 1482 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1483 edesc->dma_len, 1484 DMA_BIDIRECTIONAL); 1485 1486 return edesc; 1487 } 1488 1489 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1490 int icv_stashing, bool encrypt) 1491 { 1492 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1493 unsigned int authsize = crypto_aead_authsize(authenc); 1494 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1495 unsigned int ivsize = crypto_aead_ivsize(authenc); 1496 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1497 1498 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1499 iv, areq->assoclen, cryptlen, 1500 authsize, ivsize, icv_stashing, 1501 areq->base.flags, encrypt); 1502 } 1503 1504 static int aead_encrypt(struct aead_request *req) 1505 { 1506 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1507 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1508 struct talitos_edesc *edesc; 1509 1510 /* allocate extended descriptor */ 1511 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1512 if (IS_ERR(edesc)) 1513 return PTR_ERR(edesc); 1514 1515 /* set encrypt */ 1516 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1517 1518 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); 1519 } 1520 1521 static int aead_decrypt(struct aead_request *req) 1522 { 1523 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1524 unsigned int authsize = crypto_aead_authsize(authenc); 1525 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1526 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1527 struct talitos_edesc *edesc; 1528 void *icvdata; 1529 1530 /* allocate extended descriptor */ 1531 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1532 if (IS_ERR(edesc)) 1533 return PTR_ERR(edesc); 1534 1535 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && 1536 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1537 ((!edesc->src_nents && !edesc->dst_nents) || 1538 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1539 1540 /* decrypt and check the ICV */ 1541 edesc->desc.hdr = ctx->desc_hdr_template | 1542 DESC_HDR_DIR_INBOUND | 1543 DESC_HDR_MODE1_MDEU_CICV; 1544 1545 /* reset integrity check result bits */ 1546 1547 return ipsec_esp(edesc, req, false, 1548 ipsec_esp_decrypt_hwauth_done); 1549 } 1550 1551 /* Have to check the ICV with software */ 1552 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1553 1554 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1555 icvdata = edesc->buf + edesc->dma_len; 1556 1557 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, 1558 req->assoclen + req->cryptlen - authsize); 1559 1560 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); 1561 } 1562 1563 static int skcipher_setkey(struct crypto_skcipher *cipher, 1564 const u8 *key, unsigned int keylen) 1565 { 1566 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1567 struct device *dev = ctx->dev; 1568 1569 if (ctx->keylen) 1570 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1571 1572 memcpy(&ctx->key, key, keylen); 1573 ctx->keylen = keylen; 1574 1575 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); 1576 1577 return 0; 1578 } 1579 1580 static int skcipher_des_setkey(struct crypto_skcipher *cipher, 1581 const u8 *key, unsigned int keylen) 1582 { 1583 return verify_skcipher_des_key(cipher, key) ?: 1584 skcipher_setkey(cipher, key, keylen); 1585 } 1586 1587 static int skcipher_des3_setkey(struct crypto_skcipher *cipher, 1588 const u8 *key, unsigned int keylen) 1589 { 1590 return verify_skcipher_des3_key(cipher, key) ?: 1591 skcipher_setkey(cipher, key, keylen); 1592 } 1593 1594 static int skcipher_aes_setkey(struct crypto_skcipher *cipher, 1595 const u8 *key, unsigned int keylen) 1596 { 1597 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || 1598 keylen == AES_KEYSIZE_256) 1599 return skcipher_setkey(cipher, key, keylen); 1600 1601 return -EINVAL; 1602 } 1603 1604 static void common_nonsnoop_unmap(struct device *dev, 1605 struct talitos_edesc *edesc, 1606 struct skcipher_request *areq) 1607 { 1608 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1609 1610 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); 1611 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1612 1613 if (edesc->dma_len) 1614 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1615 DMA_BIDIRECTIONAL); 1616 } 1617 1618 static void skcipher_done(struct device *dev, 1619 struct talitos_desc *desc, void *context, 1620 int err) 1621 { 1622 struct skcipher_request *areq = context; 1623 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1624 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1625 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1626 struct talitos_edesc *edesc; 1627 1628 edesc = container_of(desc, struct talitos_edesc, desc); 1629 1630 common_nonsnoop_unmap(dev, edesc, areq); 1631 memcpy(areq->iv, ctx->iv, ivsize); 1632 1633 kfree(edesc); 1634 1635 skcipher_request_complete(areq, err); 1636 } 1637 1638 static int common_nonsnoop(struct talitos_edesc *edesc, 1639 struct skcipher_request *areq, 1640 void (*callback) (struct device *dev, 1641 struct talitos_desc *desc, 1642 void *context, int error)) 1643 { 1644 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1645 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1646 struct device *dev = ctx->dev; 1647 struct talitos_desc *desc = &edesc->desc; 1648 unsigned int cryptlen = areq->cryptlen; 1649 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1650 int sg_count, ret; 1651 bool sync_needed = false; 1652 struct talitos_private *priv = dev_get_drvdata(dev); 1653 bool is_sec1 = has_ftr_sec1(priv); 1654 bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && 1655 (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; 1656 1657 /* first DWORD empty */ 1658 1659 /* cipher iv */ 1660 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); 1661 1662 /* cipher key */ 1663 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); 1664 1665 sg_count = edesc->src_nents ?: 1; 1666 if (is_sec1 && sg_count > 1) 1667 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1668 cryptlen); 1669 else 1670 sg_count = dma_map_sg(dev, areq->src, sg_count, 1671 (areq->src == areq->dst) ? 1672 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1673 /* 1674 * cipher in 1675 */ 1676 sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], 1677 sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); 1678 if (sg_count > 1) 1679 sync_needed = true; 1680 1681 /* cipher out */ 1682 if (areq->src != areq->dst) { 1683 sg_count = edesc->dst_nents ? : 1; 1684 if (!is_sec1 || sg_count == 1) 1685 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1686 } 1687 1688 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], 1689 sg_count, 0, (edesc->src_nents + 1)); 1690 if (ret > 1) 1691 sync_needed = true; 1692 1693 /* iv out */ 1694 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1695 DMA_FROM_DEVICE); 1696 1697 /* last DWORD empty */ 1698 1699 if (sync_needed) 1700 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1701 edesc->dma_len, DMA_BIDIRECTIONAL); 1702 1703 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1704 if (ret != -EINPROGRESS) { 1705 common_nonsnoop_unmap(dev, edesc, areq); 1706 kfree(edesc); 1707 } 1708 return ret; 1709 } 1710 1711 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * 1712 areq, bool encrypt) 1713 { 1714 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1715 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1716 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1717 1718 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1719 areq->iv, 0, areq->cryptlen, 0, ivsize, 0, 1720 areq->base.flags, encrypt); 1721 } 1722 1723 static int skcipher_encrypt(struct skcipher_request *areq) 1724 { 1725 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1726 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1727 struct talitos_edesc *edesc; 1728 unsigned int blocksize = 1729 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1730 1731 if (!areq->cryptlen) 1732 return 0; 1733 1734 if (areq->cryptlen % blocksize) 1735 return -EINVAL; 1736 1737 /* allocate extended descriptor */ 1738 edesc = skcipher_edesc_alloc(areq, true); 1739 if (IS_ERR(edesc)) 1740 return PTR_ERR(edesc); 1741 1742 /* set encrypt */ 1743 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1744 1745 return common_nonsnoop(edesc, areq, skcipher_done); 1746 } 1747 1748 static int skcipher_decrypt(struct skcipher_request *areq) 1749 { 1750 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1751 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1752 struct talitos_edesc *edesc; 1753 unsigned int blocksize = 1754 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1755 1756 if (!areq->cryptlen) 1757 return 0; 1758 1759 if (areq->cryptlen % blocksize) 1760 return -EINVAL; 1761 1762 /* allocate extended descriptor */ 1763 edesc = skcipher_edesc_alloc(areq, false); 1764 if (IS_ERR(edesc)) 1765 return PTR_ERR(edesc); 1766 1767 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1768 1769 return common_nonsnoop(edesc, areq, skcipher_done); 1770 } 1771 1772 static void common_nonsnoop_hash_unmap(struct device *dev, 1773 struct talitos_edesc *edesc, 1774 struct ahash_request *areq) 1775 { 1776 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1777 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1778 struct talitos_private *priv = dev_get_drvdata(dev); 1779 bool is_sec1 = has_ftr_sec1(priv); 1780 struct talitos_desc *desc = &edesc->desc; 1781 1782 unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); 1783 1784 if (edesc->last && req_ctx->last_request) 1785 memcpy(areq->result, req_ctx->hw_context, 1786 crypto_ahash_digestsize(tfm)); 1787 1788 if (edesc->src) 1789 talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); 1790 1791 /* When using hashctx-in, must unmap it. */ 1792 if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) 1793 unmap_single_talitos_ptr(dev, &desc->ptr[1], 1794 DMA_TO_DEVICE); 1795 1796 if (edesc->dma_len) 1797 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1798 DMA_BIDIRECTIONAL); 1799 } 1800 1801 static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) 1802 { 1803 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1804 struct talitos_edesc *next; 1805 1806 while (edesc) { 1807 next = edesc->next_desc; 1808 common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); 1809 kfree(edesc); 1810 edesc = next; 1811 } 1812 } 1813 1814 static void ahash_done(struct device *dev, 1815 struct talitos_desc *desc, void *context, 1816 int err) 1817 { 1818 struct ahash_request *areq = context; 1819 struct talitos_edesc *edesc = 1820 container_of(desc, struct talitos_edesc, desc); 1821 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1822 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1823 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); 1824 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1825 struct talitos_edesc *next; 1826 1827 if (is_sec1) { 1828 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1829 /* Position any partial block for next update/final/finup */ 1830 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1831 req_ctx->nbuf = req_ctx->to_hash_later; 1832 } 1833 1834 free_edesc_list_from(areq, edesc); 1835 ahash_request_complete(areq, err); 1836 } else { 1837 next = edesc->next_desc; 1838 1839 common_nonsnoop_hash_unmap(dev, edesc, areq); 1840 kfree(edesc); 1841 1842 if (err) 1843 goto out; 1844 1845 if (next) { 1846 err = talitos_submit(dev, ctx->ch, &next->desc, 1847 ahash_done, areq); 1848 if (err != -EINPROGRESS) 1849 goto out; 1850 return; 1851 } 1852 out: 1853 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1854 /* Position any partial block for next update/final/finup */ 1855 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1856 req_ctx->nbuf = req_ctx->to_hash_later; 1857 } 1858 if (err && next) 1859 free_edesc_list_from(areq, next); 1860 ahash_request_complete(areq, err); 1861 } 1862 } 1863 1864 /* 1865 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1866 * ourself and submit a padded block 1867 */ 1868 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1869 struct talitos_edesc *edesc, 1870 struct talitos_ptr *ptr) 1871 { 1872 static u8 padded_hash[64] = { 1873 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1874 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877 }; 1878 1879 pr_err_once("Bug in SEC1, padding ourself\n"); 1880 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1881 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1882 (char *)padded_hash, DMA_TO_DEVICE); 1883 } 1884 1885 static void common_nonsnoop_hash(struct talitos_edesc *edesc, 1886 struct ahash_request *areq, 1887 unsigned int length) 1888 { 1889 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1890 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1891 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1892 struct device *dev = ctx->dev; 1893 struct talitos_desc *desc = &edesc->desc; 1894 bool sync_needed = false; 1895 struct talitos_private *priv = dev_get_drvdata(dev); 1896 bool is_sec1 = has_ftr_sec1(priv); 1897 int sg_count; 1898 1899 /* first DWORD empty */ 1900 1901 /* hash context in */ 1902 if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { 1903 map_single_talitos_ptr_nosync(dev, &desc->ptr[1], 1904 req_ctx->hw_context_size, 1905 req_ctx->hw_context, 1906 DMA_TO_DEVICE); 1907 req_ctx->swinit = 0; 1908 } 1909 /* Indicate next op is not the first. */ 1910 req_ctx->first_request = 0; 1911 1912 /* HMAC key */ 1913 if (ctx->keylen) 1914 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, 1915 is_sec1); 1916 1917 sg_count = edesc->src_nents ?: 1; 1918 if (is_sec1 && sg_count > 1) 1919 sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); 1920 else if (length) 1921 sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); 1922 1923 /* 1924 * data in 1925 */ 1926 sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], 1927 sg_count, 0, 0); 1928 if (sg_count > 1) 1929 sync_needed = true; 1930 1931 /* fifth DWORD empty */ 1932 1933 /* hash/HMAC out -or- hash context out */ 1934 if (edesc->last && req_ctx->last_request) 1935 map_single_talitos_ptr(dev, &desc->ptr[5], 1936 crypto_ahash_digestsize(tfm), 1937 req_ctx->hw_context, DMA_FROM_DEVICE); 1938 else 1939 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1940 req_ctx->hw_context_size, 1941 req_ctx->hw_context, 1942 DMA_FROM_DEVICE); 1943 1944 /* last DWORD empty */ 1945 1946 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1947 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1948 1949 if (sync_needed) 1950 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1951 edesc->dma_len, DMA_BIDIRECTIONAL); 1952 } 1953 1954 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1955 struct scatterlist *src, 1956 unsigned int nbytes) 1957 { 1958 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1959 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1960 1961 return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, 1962 nbytes, 0, 0, 0, areq->base.flags, false); 1963 } 1964 1965 static struct talitos_edesc * 1966 ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, 1967 unsigned int blocksize, bool is_sec1) 1968 { 1969 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1970 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1971 struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; 1972 size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : 1973 TALITOS2_MAX_DATA_LEN; 1974 struct scatterlist tmp[2]; 1975 size_t to_hash_this_desc; 1976 struct scatterlist *src; 1977 size_t offset = 0; 1978 1979 do { 1980 src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); 1981 1982 to_hash_this_desc = 1983 min(nbytes, ALIGN_DOWN(desc_max, blocksize)); 1984 1985 /* Allocate extended descriptor */ 1986 edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); 1987 if (IS_ERR(edesc)) { 1988 if (first) 1989 free_edesc_list_from(areq, first); 1990 return edesc; 1991 } 1992 1993 edesc->src = 1994 scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); 1995 edesc->desc.hdr = ctx->desc_hdr_template; 1996 edesc->first = offset == 0; 1997 edesc->last = nbytes - to_hash_this_desc == 0; 1998 1999 /* On last one, request SEC to pad; otherwise continue */ 2000 if (req_ctx->last_request && edesc->last) 2001 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 2002 else 2003 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 2004 2005 /* request SEC to INIT hash. */ 2006 if (req_ctx->first_request && edesc->first && !req_ctx->swinit) 2007 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 2008 2009 /* 2010 * When the tfm context has a keylen, it's an HMAC. 2011 * A first or last (ie. not middle) descriptor must request HMAC. 2012 */ 2013 if (ctx->keylen && ((req_ctx->first_request && edesc->first) || 2014 (req_ctx->last_request && edesc->last))) 2015 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 2016 2017 /* clear the DN bit */ 2018 if (is_sec1 && !edesc->last) 2019 edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; 2020 2021 common_nonsnoop_hash(edesc, areq, to_hash_this_desc); 2022 2023 offset += to_hash_this_desc; 2024 nbytes -= to_hash_this_desc; 2025 2026 if (!prev_edesc) 2027 first = edesc; 2028 else 2029 prev_edesc->next_desc = edesc; 2030 prev_edesc = edesc; 2031 } while (nbytes); 2032 2033 return first; 2034 } 2035 2036 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 2037 { 2038 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2039 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2040 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2041 struct talitos_edesc *edesc; 2042 unsigned int blocksize = 2043 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2044 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); 2045 unsigned int nbytes_to_hash; 2046 unsigned int to_hash_later; 2047 unsigned int nsg; 2048 int nents; 2049 struct device *dev = ctx->dev; 2050 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; 2051 int ret; 2052 2053 if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { 2054 /* Buffer up to one whole block */ 2055 nents = sg_nents_for_len(areq->src, nbytes); 2056 if (nents < 0) { 2057 dev_err(dev, "Invalid number of src SG.\n"); 2058 return nents; 2059 } 2060 sg_copy_to_buffer(areq->src, nents, 2061 ctx_buf + req_ctx->nbuf, nbytes); 2062 req_ctx->nbuf += nbytes; 2063 return 0; 2064 } 2065 2066 /* At least (blocksize + 1) bytes are available to hash */ 2067 nbytes_to_hash = nbytes + req_ctx->nbuf; 2068 to_hash_later = nbytes_to_hash & (blocksize - 1); 2069 2070 if (req_ctx->last_request) 2071 to_hash_later = 0; 2072 else if (to_hash_later) 2073 /* There is a partial block. Hash the full block(s) now */ 2074 nbytes_to_hash -= to_hash_later; 2075 else { 2076 /* Keep one block buffered */ 2077 nbytes_to_hash -= blocksize; 2078 to_hash_later = blocksize; 2079 } 2080 2081 /* Chain in any previously buffered data */ 2082 if (req_ctx->nbuf) { 2083 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 2084 sg_init_table(req_ctx->bufsl, nsg); 2085 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); 2086 if (nsg > 1) 2087 sg_chain(req_ctx->bufsl, 2, areq->src); 2088 req_ctx->psrc = req_ctx->bufsl; 2089 } else 2090 req_ctx->psrc = areq->src; 2091 2092 if (to_hash_later) { 2093 nents = sg_nents_for_len(areq->src, nbytes); 2094 if (nents < 0) { 2095 dev_err(dev, "Invalid number of src SG.\n"); 2096 return nents; 2097 } 2098 sg_pcopy_to_buffer(areq->src, nents, 2099 req_ctx->buf[(req_ctx->buf_idx + 1) & 1], 2100 to_hash_later, 2101 nbytes - to_hash_later); 2102 } 2103 req_ctx->to_hash_later = to_hash_later; 2104 2105 edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, 2106 is_sec1); 2107 if (IS_ERR(edesc)) 2108 return PTR_ERR(edesc); 2109 2110 ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); 2111 if (ret != -EINPROGRESS) 2112 free_edesc_list_from(areq, edesc); 2113 2114 return ret; 2115 } 2116 2117 static int ahash_init(struct ahash_request *areq) 2118 { 2119 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2120 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2121 struct device *dev = ctx->dev; 2122 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2123 unsigned int size; 2124 dma_addr_t dma; 2125 2126 /* Initialize the context */ 2127 req_ctx->buf_idx = 0; 2128 req_ctx->nbuf = 0; 2129 req_ctx->first_request = 1; 2130 req_ctx->swinit = 0; /* assume h/w init of context */ 2131 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2132 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2133 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2134 req_ctx->hw_context_size = size; 2135 req_ctx->last_request = 0; 2136 2137 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2138 DMA_TO_DEVICE); 2139 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2140 2141 return 0; 2142 } 2143 2144 /* 2145 * on h/w without explicit sha224 support, we initialize h/w context 2146 * manually with sha224 constants, and tell it to run sha256. 2147 */ 2148 static int ahash_init_sha224_swinit(struct ahash_request *areq) 2149 { 2150 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2151 2152 req_ctx->hw_context[0] = SHA224_H0; 2153 req_ctx->hw_context[1] = SHA224_H1; 2154 req_ctx->hw_context[2] = SHA224_H2; 2155 req_ctx->hw_context[3] = SHA224_H3; 2156 req_ctx->hw_context[4] = SHA224_H4; 2157 req_ctx->hw_context[5] = SHA224_H5; 2158 req_ctx->hw_context[6] = SHA224_H6; 2159 req_ctx->hw_context[7] = SHA224_H7; 2160 2161 /* init 64-bit count */ 2162 req_ctx->hw_context[8] = 0; 2163 req_ctx->hw_context[9] = 0; 2164 2165 ahash_init(areq); 2166 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 2167 2168 return 0; 2169 } 2170 2171 static int ahash_update(struct ahash_request *areq) 2172 { 2173 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2174 2175 req_ctx->last_request = 0; 2176 2177 return ahash_process_req(areq, areq->nbytes); 2178 } 2179 2180 static int ahash_final(struct ahash_request *areq) 2181 { 2182 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2183 2184 req_ctx->last_request = 1; 2185 2186 return ahash_process_req(areq, 0); 2187 } 2188 2189 static int ahash_finup(struct ahash_request *areq) 2190 { 2191 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2192 2193 req_ctx->last_request = 1; 2194 2195 return ahash_process_req(areq, areq->nbytes); 2196 } 2197 2198 static int ahash_digest(struct ahash_request *areq) 2199 { 2200 ahash_init(areq); 2201 return ahash_finup(areq); 2202 } 2203 2204 static int ahash_digest_sha224_swinit(struct ahash_request *areq) 2205 { 2206 ahash_init_sha224_swinit(areq); 2207 return ahash_finup(areq); 2208 } 2209 2210 static int ahash_export(struct ahash_request *areq, void *out) 2211 { 2212 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2213 struct talitos_export_state *export = out; 2214 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2215 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2216 struct device *dev = ctx->dev; 2217 dma_addr_t dma; 2218 2219 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2220 DMA_FROM_DEVICE); 2221 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); 2222 2223 memcpy(export->hw_context, req_ctx->hw_context, 2224 req_ctx->hw_context_size); 2225 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); 2226 export->swinit = req_ctx->swinit; 2227 export->first_request = req_ctx->first_request; 2228 export->last_request = req_ctx->last_request; 2229 export->to_hash_later = req_ctx->to_hash_later; 2230 export->nbuf = req_ctx->nbuf; 2231 2232 return 0; 2233 } 2234 2235 static int ahash_import(struct ahash_request *areq, const void *in) 2236 { 2237 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2238 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2239 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2240 struct device *dev = ctx->dev; 2241 const struct talitos_export_state *export = in; 2242 unsigned int size; 2243 dma_addr_t dma; 2244 2245 memset(req_ctx, 0, sizeof(*req_ctx)); 2246 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2247 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2248 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2249 req_ctx->hw_context_size = size; 2250 memcpy(req_ctx->hw_context, export->hw_context, size); 2251 memcpy(req_ctx->buf[0], export->buf, export->nbuf); 2252 req_ctx->swinit = export->swinit; 2253 req_ctx->first_request = export->first_request; 2254 req_ctx->last_request = export->last_request; 2255 req_ctx->to_hash_later = export->to_hash_later; 2256 req_ctx->nbuf = export->nbuf; 2257 2258 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2259 DMA_TO_DEVICE); 2260 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2261 2262 return 0; 2263 } 2264 2265 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2266 u8 *hash) 2267 { 2268 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2269 2270 struct scatterlist sg[1]; 2271 struct ahash_request *req; 2272 struct crypto_wait wait; 2273 int ret; 2274 2275 crypto_init_wait(&wait); 2276 2277 req = ahash_request_alloc(tfm, GFP_KERNEL); 2278 if (!req) 2279 return -ENOMEM; 2280 2281 /* Keep tfm keylen == 0 during hash of the long key */ 2282 ctx->keylen = 0; 2283 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2284 crypto_req_done, &wait); 2285 2286 sg_init_one(&sg[0], key, keylen); 2287 2288 ahash_request_set_crypt(req, sg, hash, keylen); 2289 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 2290 2291 ahash_request_free(req); 2292 2293 return ret; 2294 } 2295 2296 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2297 unsigned int keylen) 2298 { 2299 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2300 struct device *dev = ctx->dev; 2301 unsigned int blocksize = 2302 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2303 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2304 unsigned int keysize = keylen; 2305 u8 hash[SHA512_DIGEST_SIZE]; 2306 int ret; 2307 2308 if (keylen <= blocksize) 2309 memcpy(ctx->key, key, keysize); 2310 else { 2311 /* Must get the hash of the long key */ 2312 ret = keyhash(tfm, key, keylen, hash); 2313 2314 if (ret) 2315 return -EINVAL; 2316 2317 keysize = digestsize; 2318 memcpy(ctx->key, hash, digestsize); 2319 } 2320 2321 if (ctx->keylen) 2322 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 2323 2324 ctx->keylen = keysize; 2325 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); 2326 2327 return 0; 2328 } 2329 2330 2331 struct talitos_alg_template { 2332 u32 type; 2333 u32 priority; 2334 union { 2335 struct skcipher_alg skcipher; 2336 struct ahash_alg hash; 2337 struct aead_alg aead; 2338 } alg; 2339 __be32 desc_hdr_template; 2340 }; 2341 2342 static struct talitos_alg_template driver_algs[] = { 2343 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2344 { .type = CRYPTO_ALG_TYPE_AEAD, 2345 .alg.aead = { 2346 .base = { 2347 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2348 .cra_driver_name = "authenc-hmac-sha1-" 2349 "cbc-aes-talitos", 2350 .cra_blocksize = AES_BLOCK_SIZE, 2351 .cra_flags = CRYPTO_ALG_ASYNC | 2352 CRYPTO_ALG_ALLOCATES_MEMORY, 2353 }, 2354 .ivsize = AES_BLOCK_SIZE, 2355 .maxauthsize = SHA1_DIGEST_SIZE, 2356 }, 2357 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2358 DESC_HDR_SEL0_AESU | 2359 DESC_HDR_MODE0_AESU_CBC | 2360 DESC_HDR_SEL1_MDEUA | 2361 DESC_HDR_MODE1_MDEU_INIT | 2362 DESC_HDR_MODE1_MDEU_PAD | 2363 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2364 }, 2365 { .type = CRYPTO_ALG_TYPE_AEAD, 2366 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2367 .alg.aead = { 2368 .base = { 2369 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2370 .cra_driver_name = "authenc-hmac-sha1-" 2371 "cbc-aes-talitos-hsna", 2372 .cra_blocksize = AES_BLOCK_SIZE, 2373 .cra_flags = CRYPTO_ALG_ASYNC | 2374 CRYPTO_ALG_ALLOCATES_MEMORY, 2375 }, 2376 .ivsize = AES_BLOCK_SIZE, 2377 .maxauthsize = SHA1_DIGEST_SIZE, 2378 }, 2379 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2380 DESC_HDR_SEL0_AESU | 2381 DESC_HDR_MODE0_AESU_CBC | 2382 DESC_HDR_SEL1_MDEUA | 2383 DESC_HDR_MODE1_MDEU_INIT | 2384 DESC_HDR_MODE1_MDEU_PAD | 2385 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2386 }, 2387 { .type = CRYPTO_ALG_TYPE_AEAD, 2388 .alg.aead = { 2389 .base = { 2390 .cra_name = "authenc(hmac(sha1)," 2391 "cbc(des3_ede))", 2392 .cra_driver_name = "authenc-hmac-sha1-" 2393 "cbc-3des-talitos", 2394 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2395 .cra_flags = CRYPTO_ALG_ASYNC | 2396 CRYPTO_ALG_ALLOCATES_MEMORY, 2397 }, 2398 .ivsize = DES3_EDE_BLOCK_SIZE, 2399 .maxauthsize = SHA1_DIGEST_SIZE, 2400 .setkey = aead_des3_setkey, 2401 }, 2402 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2403 DESC_HDR_SEL0_DEU | 2404 DESC_HDR_MODE0_DEU_CBC | 2405 DESC_HDR_MODE0_DEU_3DES | 2406 DESC_HDR_SEL1_MDEUA | 2407 DESC_HDR_MODE1_MDEU_INIT | 2408 DESC_HDR_MODE1_MDEU_PAD | 2409 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2410 }, 2411 { .type = CRYPTO_ALG_TYPE_AEAD, 2412 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2413 .alg.aead = { 2414 .base = { 2415 .cra_name = "authenc(hmac(sha1)," 2416 "cbc(des3_ede))", 2417 .cra_driver_name = "authenc-hmac-sha1-" 2418 "cbc-3des-talitos-hsna", 2419 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2420 .cra_flags = CRYPTO_ALG_ASYNC | 2421 CRYPTO_ALG_ALLOCATES_MEMORY, 2422 }, 2423 .ivsize = DES3_EDE_BLOCK_SIZE, 2424 .maxauthsize = SHA1_DIGEST_SIZE, 2425 .setkey = aead_des3_setkey, 2426 }, 2427 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2428 DESC_HDR_SEL0_DEU | 2429 DESC_HDR_MODE0_DEU_CBC | 2430 DESC_HDR_MODE0_DEU_3DES | 2431 DESC_HDR_SEL1_MDEUA | 2432 DESC_HDR_MODE1_MDEU_INIT | 2433 DESC_HDR_MODE1_MDEU_PAD | 2434 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2435 }, 2436 { .type = CRYPTO_ALG_TYPE_AEAD, 2437 .alg.aead = { 2438 .base = { 2439 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2440 .cra_driver_name = "authenc-hmac-sha224-" 2441 "cbc-aes-talitos", 2442 .cra_blocksize = AES_BLOCK_SIZE, 2443 .cra_flags = CRYPTO_ALG_ASYNC | 2444 CRYPTO_ALG_ALLOCATES_MEMORY, 2445 }, 2446 .ivsize = AES_BLOCK_SIZE, 2447 .maxauthsize = SHA224_DIGEST_SIZE, 2448 }, 2449 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2450 DESC_HDR_SEL0_AESU | 2451 DESC_HDR_MODE0_AESU_CBC | 2452 DESC_HDR_SEL1_MDEUA | 2453 DESC_HDR_MODE1_MDEU_INIT | 2454 DESC_HDR_MODE1_MDEU_PAD | 2455 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2456 }, 2457 { .type = CRYPTO_ALG_TYPE_AEAD, 2458 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2459 .alg.aead = { 2460 .base = { 2461 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2462 .cra_driver_name = "authenc-hmac-sha224-" 2463 "cbc-aes-talitos-hsna", 2464 .cra_blocksize = AES_BLOCK_SIZE, 2465 .cra_flags = CRYPTO_ALG_ASYNC | 2466 CRYPTO_ALG_ALLOCATES_MEMORY, 2467 }, 2468 .ivsize = AES_BLOCK_SIZE, 2469 .maxauthsize = SHA224_DIGEST_SIZE, 2470 }, 2471 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2472 DESC_HDR_SEL0_AESU | 2473 DESC_HDR_MODE0_AESU_CBC | 2474 DESC_HDR_SEL1_MDEUA | 2475 DESC_HDR_MODE1_MDEU_INIT | 2476 DESC_HDR_MODE1_MDEU_PAD | 2477 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2478 }, 2479 { .type = CRYPTO_ALG_TYPE_AEAD, 2480 .alg.aead = { 2481 .base = { 2482 .cra_name = "authenc(hmac(sha224)," 2483 "cbc(des3_ede))", 2484 .cra_driver_name = "authenc-hmac-sha224-" 2485 "cbc-3des-talitos", 2486 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2487 .cra_flags = CRYPTO_ALG_ASYNC | 2488 CRYPTO_ALG_ALLOCATES_MEMORY, 2489 }, 2490 .ivsize = DES3_EDE_BLOCK_SIZE, 2491 .maxauthsize = SHA224_DIGEST_SIZE, 2492 .setkey = aead_des3_setkey, 2493 }, 2494 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2495 DESC_HDR_SEL0_DEU | 2496 DESC_HDR_MODE0_DEU_CBC | 2497 DESC_HDR_MODE0_DEU_3DES | 2498 DESC_HDR_SEL1_MDEUA | 2499 DESC_HDR_MODE1_MDEU_INIT | 2500 DESC_HDR_MODE1_MDEU_PAD | 2501 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2502 }, 2503 { .type = CRYPTO_ALG_TYPE_AEAD, 2504 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2505 .alg.aead = { 2506 .base = { 2507 .cra_name = "authenc(hmac(sha224)," 2508 "cbc(des3_ede))", 2509 .cra_driver_name = "authenc-hmac-sha224-" 2510 "cbc-3des-talitos-hsna", 2511 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2512 .cra_flags = CRYPTO_ALG_ASYNC | 2513 CRYPTO_ALG_ALLOCATES_MEMORY, 2514 }, 2515 .ivsize = DES3_EDE_BLOCK_SIZE, 2516 .maxauthsize = SHA224_DIGEST_SIZE, 2517 .setkey = aead_des3_setkey, 2518 }, 2519 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2520 DESC_HDR_SEL0_DEU | 2521 DESC_HDR_MODE0_DEU_CBC | 2522 DESC_HDR_MODE0_DEU_3DES | 2523 DESC_HDR_SEL1_MDEUA | 2524 DESC_HDR_MODE1_MDEU_INIT | 2525 DESC_HDR_MODE1_MDEU_PAD | 2526 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2527 }, 2528 { .type = CRYPTO_ALG_TYPE_AEAD, 2529 .alg.aead = { 2530 .base = { 2531 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2532 .cra_driver_name = "authenc-hmac-sha256-" 2533 "cbc-aes-talitos", 2534 .cra_blocksize = AES_BLOCK_SIZE, 2535 .cra_flags = CRYPTO_ALG_ASYNC | 2536 CRYPTO_ALG_ALLOCATES_MEMORY, 2537 }, 2538 .ivsize = AES_BLOCK_SIZE, 2539 .maxauthsize = SHA256_DIGEST_SIZE, 2540 }, 2541 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2542 DESC_HDR_SEL0_AESU | 2543 DESC_HDR_MODE0_AESU_CBC | 2544 DESC_HDR_SEL1_MDEUA | 2545 DESC_HDR_MODE1_MDEU_INIT | 2546 DESC_HDR_MODE1_MDEU_PAD | 2547 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2548 }, 2549 { .type = CRYPTO_ALG_TYPE_AEAD, 2550 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2551 .alg.aead = { 2552 .base = { 2553 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2554 .cra_driver_name = "authenc-hmac-sha256-" 2555 "cbc-aes-talitos-hsna", 2556 .cra_blocksize = AES_BLOCK_SIZE, 2557 .cra_flags = CRYPTO_ALG_ASYNC | 2558 CRYPTO_ALG_ALLOCATES_MEMORY, 2559 }, 2560 .ivsize = AES_BLOCK_SIZE, 2561 .maxauthsize = SHA256_DIGEST_SIZE, 2562 }, 2563 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2564 DESC_HDR_SEL0_AESU | 2565 DESC_HDR_MODE0_AESU_CBC | 2566 DESC_HDR_SEL1_MDEUA | 2567 DESC_HDR_MODE1_MDEU_INIT | 2568 DESC_HDR_MODE1_MDEU_PAD | 2569 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2570 }, 2571 { .type = CRYPTO_ALG_TYPE_AEAD, 2572 .alg.aead = { 2573 .base = { 2574 .cra_name = "authenc(hmac(sha256)," 2575 "cbc(des3_ede))", 2576 .cra_driver_name = "authenc-hmac-sha256-" 2577 "cbc-3des-talitos", 2578 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2579 .cra_flags = CRYPTO_ALG_ASYNC | 2580 CRYPTO_ALG_ALLOCATES_MEMORY, 2581 }, 2582 .ivsize = DES3_EDE_BLOCK_SIZE, 2583 .maxauthsize = SHA256_DIGEST_SIZE, 2584 .setkey = aead_des3_setkey, 2585 }, 2586 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2587 DESC_HDR_SEL0_DEU | 2588 DESC_HDR_MODE0_DEU_CBC | 2589 DESC_HDR_MODE0_DEU_3DES | 2590 DESC_HDR_SEL1_MDEUA | 2591 DESC_HDR_MODE1_MDEU_INIT | 2592 DESC_HDR_MODE1_MDEU_PAD | 2593 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2594 }, 2595 { .type = CRYPTO_ALG_TYPE_AEAD, 2596 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2597 .alg.aead = { 2598 .base = { 2599 .cra_name = "authenc(hmac(sha256)," 2600 "cbc(des3_ede))", 2601 .cra_driver_name = "authenc-hmac-sha256-" 2602 "cbc-3des-talitos-hsna", 2603 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2604 .cra_flags = CRYPTO_ALG_ASYNC | 2605 CRYPTO_ALG_ALLOCATES_MEMORY, 2606 }, 2607 .ivsize = DES3_EDE_BLOCK_SIZE, 2608 .maxauthsize = SHA256_DIGEST_SIZE, 2609 .setkey = aead_des3_setkey, 2610 }, 2611 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2612 DESC_HDR_SEL0_DEU | 2613 DESC_HDR_MODE0_DEU_CBC | 2614 DESC_HDR_MODE0_DEU_3DES | 2615 DESC_HDR_SEL1_MDEUA | 2616 DESC_HDR_MODE1_MDEU_INIT | 2617 DESC_HDR_MODE1_MDEU_PAD | 2618 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2619 }, 2620 { .type = CRYPTO_ALG_TYPE_AEAD, 2621 .alg.aead = { 2622 .base = { 2623 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2624 .cra_driver_name = "authenc-hmac-sha384-" 2625 "cbc-aes-talitos", 2626 .cra_blocksize = AES_BLOCK_SIZE, 2627 .cra_flags = CRYPTO_ALG_ASYNC | 2628 CRYPTO_ALG_ALLOCATES_MEMORY, 2629 }, 2630 .ivsize = AES_BLOCK_SIZE, 2631 .maxauthsize = SHA384_DIGEST_SIZE, 2632 }, 2633 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2634 DESC_HDR_SEL0_AESU | 2635 DESC_HDR_MODE0_AESU_CBC | 2636 DESC_HDR_SEL1_MDEUB | 2637 DESC_HDR_MODE1_MDEU_INIT | 2638 DESC_HDR_MODE1_MDEU_PAD | 2639 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2640 }, 2641 { .type = CRYPTO_ALG_TYPE_AEAD, 2642 .alg.aead = { 2643 .base = { 2644 .cra_name = "authenc(hmac(sha384)," 2645 "cbc(des3_ede))", 2646 .cra_driver_name = "authenc-hmac-sha384-" 2647 "cbc-3des-talitos", 2648 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2649 .cra_flags = CRYPTO_ALG_ASYNC | 2650 CRYPTO_ALG_ALLOCATES_MEMORY, 2651 }, 2652 .ivsize = DES3_EDE_BLOCK_SIZE, 2653 .maxauthsize = SHA384_DIGEST_SIZE, 2654 .setkey = aead_des3_setkey, 2655 }, 2656 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2657 DESC_HDR_SEL0_DEU | 2658 DESC_HDR_MODE0_DEU_CBC | 2659 DESC_HDR_MODE0_DEU_3DES | 2660 DESC_HDR_SEL1_MDEUB | 2661 DESC_HDR_MODE1_MDEU_INIT | 2662 DESC_HDR_MODE1_MDEU_PAD | 2663 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2664 }, 2665 { .type = CRYPTO_ALG_TYPE_AEAD, 2666 .alg.aead = { 2667 .base = { 2668 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2669 .cra_driver_name = "authenc-hmac-sha512-" 2670 "cbc-aes-talitos", 2671 .cra_blocksize = AES_BLOCK_SIZE, 2672 .cra_flags = CRYPTO_ALG_ASYNC | 2673 CRYPTO_ALG_ALLOCATES_MEMORY, 2674 }, 2675 .ivsize = AES_BLOCK_SIZE, 2676 .maxauthsize = SHA512_DIGEST_SIZE, 2677 }, 2678 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2679 DESC_HDR_SEL0_AESU | 2680 DESC_HDR_MODE0_AESU_CBC | 2681 DESC_HDR_SEL1_MDEUB | 2682 DESC_HDR_MODE1_MDEU_INIT | 2683 DESC_HDR_MODE1_MDEU_PAD | 2684 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2685 }, 2686 { .type = CRYPTO_ALG_TYPE_AEAD, 2687 .alg.aead = { 2688 .base = { 2689 .cra_name = "authenc(hmac(sha512)," 2690 "cbc(des3_ede))", 2691 .cra_driver_name = "authenc-hmac-sha512-" 2692 "cbc-3des-talitos", 2693 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2694 .cra_flags = CRYPTO_ALG_ASYNC | 2695 CRYPTO_ALG_ALLOCATES_MEMORY, 2696 }, 2697 .ivsize = DES3_EDE_BLOCK_SIZE, 2698 .maxauthsize = SHA512_DIGEST_SIZE, 2699 .setkey = aead_des3_setkey, 2700 }, 2701 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2702 DESC_HDR_SEL0_DEU | 2703 DESC_HDR_MODE0_DEU_CBC | 2704 DESC_HDR_MODE0_DEU_3DES | 2705 DESC_HDR_SEL1_MDEUB | 2706 DESC_HDR_MODE1_MDEU_INIT | 2707 DESC_HDR_MODE1_MDEU_PAD | 2708 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2709 }, 2710 { .type = CRYPTO_ALG_TYPE_AEAD, 2711 .alg.aead = { 2712 .base = { 2713 .cra_name = "authenc(hmac(md5),cbc(aes))", 2714 .cra_driver_name = "authenc-hmac-md5-" 2715 "cbc-aes-talitos", 2716 .cra_blocksize = AES_BLOCK_SIZE, 2717 .cra_flags = CRYPTO_ALG_ASYNC | 2718 CRYPTO_ALG_ALLOCATES_MEMORY, 2719 }, 2720 .ivsize = AES_BLOCK_SIZE, 2721 .maxauthsize = MD5_DIGEST_SIZE, 2722 }, 2723 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2724 DESC_HDR_SEL0_AESU | 2725 DESC_HDR_MODE0_AESU_CBC | 2726 DESC_HDR_SEL1_MDEUA | 2727 DESC_HDR_MODE1_MDEU_INIT | 2728 DESC_HDR_MODE1_MDEU_PAD | 2729 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2730 }, 2731 { .type = CRYPTO_ALG_TYPE_AEAD, 2732 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2733 .alg.aead = { 2734 .base = { 2735 .cra_name = "authenc(hmac(md5),cbc(aes))", 2736 .cra_driver_name = "authenc-hmac-md5-" 2737 "cbc-aes-talitos-hsna", 2738 .cra_blocksize = AES_BLOCK_SIZE, 2739 .cra_flags = CRYPTO_ALG_ASYNC | 2740 CRYPTO_ALG_ALLOCATES_MEMORY, 2741 }, 2742 .ivsize = AES_BLOCK_SIZE, 2743 .maxauthsize = MD5_DIGEST_SIZE, 2744 }, 2745 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2746 DESC_HDR_SEL0_AESU | 2747 DESC_HDR_MODE0_AESU_CBC | 2748 DESC_HDR_SEL1_MDEUA | 2749 DESC_HDR_MODE1_MDEU_INIT | 2750 DESC_HDR_MODE1_MDEU_PAD | 2751 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2752 }, 2753 { .type = CRYPTO_ALG_TYPE_AEAD, 2754 .alg.aead = { 2755 .base = { 2756 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2757 .cra_driver_name = "authenc-hmac-md5-" 2758 "cbc-3des-talitos", 2759 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2760 .cra_flags = CRYPTO_ALG_ASYNC | 2761 CRYPTO_ALG_ALLOCATES_MEMORY, 2762 }, 2763 .ivsize = DES3_EDE_BLOCK_SIZE, 2764 .maxauthsize = MD5_DIGEST_SIZE, 2765 .setkey = aead_des3_setkey, 2766 }, 2767 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2768 DESC_HDR_SEL0_DEU | 2769 DESC_HDR_MODE0_DEU_CBC | 2770 DESC_HDR_MODE0_DEU_3DES | 2771 DESC_HDR_SEL1_MDEUA | 2772 DESC_HDR_MODE1_MDEU_INIT | 2773 DESC_HDR_MODE1_MDEU_PAD | 2774 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2775 }, 2776 { .type = CRYPTO_ALG_TYPE_AEAD, 2777 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2778 .alg.aead = { 2779 .base = { 2780 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2781 .cra_driver_name = "authenc-hmac-md5-" 2782 "cbc-3des-talitos-hsna", 2783 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2784 .cra_flags = CRYPTO_ALG_ASYNC | 2785 CRYPTO_ALG_ALLOCATES_MEMORY, 2786 }, 2787 .ivsize = DES3_EDE_BLOCK_SIZE, 2788 .maxauthsize = MD5_DIGEST_SIZE, 2789 .setkey = aead_des3_setkey, 2790 }, 2791 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2792 DESC_HDR_SEL0_DEU | 2793 DESC_HDR_MODE0_DEU_CBC | 2794 DESC_HDR_MODE0_DEU_3DES | 2795 DESC_HDR_SEL1_MDEUA | 2796 DESC_HDR_MODE1_MDEU_INIT | 2797 DESC_HDR_MODE1_MDEU_PAD | 2798 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2799 }, 2800 /* SKCIPHER algorithms. */ 2801 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2802 .alg.skcipher = { 2803 .base.cra_name = "ecb(aes)", 2804 .base.cra_driver_name = "ecb-aes-talitos", 2805 .base.cra_blocksize = AES_BLOCK_SIZE, 2806 .base.cra_flags = CRYPTO_ALG_ASYNC | 2807 CRYPTO_ALG_ALLOCATES_MEMORY, 2808 .min_keysize = AES_MIN_KEY_SIZE, 2809 .max_keysize = AES_MAX_KEY_SIZE, 2810 .setkey = skcipher_aes_setkey, 2811 }, 2812 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2813 DESC_HDR_SEL0_AESU, 2814 }, 2815 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2816 .alg.skcipher = { 2817 .base.cra_name = "cbc(aes)", 2818 .base.cra_driver_name = "cbc-aes-talitos", 2819 .base.cra_blocksize = AES_BLOCK_SIZE, 2820 .base.cra_flags = CRYPTO_ALG_ASYNC | 2821 CRYPTO_ALG_ALLOCATES_MEMORY, 2822 .min_keysize = AES_MIN_KEY_SIZE, 2823 .max_keysize = AES_MAX_KEY_SIZE, 2824 .ivsize = AES_BLOCK_SIZE, 2825 .setkey = skcipher_aes_setkey, 2826 }, 2827 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2828 DESC_HDR_SEL0_AESU | 2829 DESC_HDR_MODE0_AESU_CBC, 2830 }, 2831 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2832 .alg.skcipher = { 2833 .base.cra_name = "ctr(aes)", 2834 .base.cra_driver_name = "ctr-aes-talitos", 2835 .base.cra_blocksize = 1, 2836 .base.cra_flags = CRYPTO_ALG_ASYNC | 2837 CRYPTO_ALG_ALLOCATES_MEMORY, 2838 .min_keysize = AES_MIN_KEY_SIZE, 2839 .max_keysize = AES_MAX_KEY_SIZE, 2840 .ivsize = AES_BLOCK_SIZE, 2841 .setkey = skcipher_aes_setkey, 2842 }, 2843 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | 2844 DESC_HDR_SEL0_AESU | 2845 DESC_HDR_MODE0_AESU_CTR, 2846 }, 2847 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2848 .alg.skcipher = { 2849 .base.cra_name = "ctr(aes)", 2850 .base.cra_driver_name = "ctr-aes-talitos", 2851 .base.cra_blocksize = 1, 2852 .base.cra_flags = CRYPTO_ALG_ASYNC | 2853 CRYPTO_ALG_ALLOCATES_MEMORY, 2854 .min_keysize = AES_MIN_KEY_SIZE, 2855 .max_keysize = AES_MAX_KEY_SIZE, 2856 .ivsize = AES_BLOCK_SIZE, 2857 .setkey = skcipher_aes_setkey, 2858 }, 2859 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2860 DESC_HDR_SEL0_AESU | 2861 DESC_HDR_MODE0_AESU_CTR, 2862 }, 2863 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2864 .alg.skcipher = { 2865 .base.cra_name = "ecb(des)", 2866 .base.cra_driver_name = "ecb-des-talitos", 2867 .base.cra_blocksize = DES_BLOCK_SIZE, 2868 .base.cra_flags = CRYPTO_ALG_ASYNC | 2869 CRYPTO_ALG_ALLOCATES_MEMORY, 2870 .min_keysize = DES_KEY_SIZE, 2871 .max_keysize = DES_KEY_SIZE, 2872 .setkey = skcipher_des_setkey, 2873 }, 2874 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2875 DESC_HDR_SEL0_DEU, 2876 }, 2877 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2878 .alg.skcipher = { 2879 .base.cra_name = "cbc(des)", 2880 .base.cra_driver_name = "cbc-des-talitos", 2881 .base.cra_blocksize = DES_BLOCK_SIZE, 2882 .base.cra_flags = CRYPTO_ALG_ASYNC | 2883 CRYPTO_ALG_ALLOCATES_MEMORY, 2884 .min_keysize = DES_KEY_SIZE, 2885 .max_keysize = DES_KEY_SIZE, 2886 .ivsize = DES_BLOCK_SIZE, 2887 .setkey = skcipher_des_setkey, 2888 }, 2889 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2890 DESC_HDR_SEL0_DEU | 2891 DESC_HDR_MODE0_DEU_CBC, 2892 }, 2893 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2894 .alg.skcipher = { 2895 .base.cra_name = "ecb(des3_ede)", 2896 .base.cra_driver_name = "ecb-3des-talitos", 2897 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2898 .base.cra_flags = CRYPTO_ALG_ASYNC | 2899 CRYPTO_ALG_ALLOCATES_MEMORY, 2900 .min_keysize = DES3_EDE_KEY_SIZE, 2901 .max_keysize = DES3_EDE_KEY_SIZE, 2902 .setkey = skcipher_des3_setkey, 2903 }, 2904 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2905 DESC_HDR_SEL0_DEU | 2906 DESC_HDR_MODE0_DEU_3DES, 2907 }, 2908 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2909 .alg.skcipher = { 2910 .base.cra_name = "cbc(des3_ede)", 2911 .base.cra_driver_name = "cbc-3des-talitos", 2912 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2913 .base.cra_flags = CRYPTO_ALG_ASYNC | 2914 CRYPTO_ALG_ALLOCATES_MEMORY, 2915 .min_keysize = DES3_EDE_KEY_SIZE, 2916 .max_keysize = DES3_EDE_KEY_SIZE, 2917 .ivsize = DES3_EDE_BLOCK_SIZE, 2918 .setkey = skcipher_des3_setkey, 2919 }, 2920 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2921 DESC_HDR_SEL0_DEU | 2922 DESC_HDR_MODE0_DEU_CBC | 2923 DESC_HDR_MODE0_DEU_3DES, 2924 }, 2925 /* AHASH algorithms. */ 2926 { .type = CRYPTO_ALG_TYPE_AHASH, 2927 .alg.hash = { 2928 .halg.digestsize = MD5_DIGEST_SIZE, 2929 .halg.statesize = sizeof(struct talitos_export_state), 2930 .halg.base = { 2931 .cra_name = "md5", 2932 .cra_driver_name = "md5-talitos", 2933 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2934 .cra_flags = CRYPTO_ALG_ASYNC | 2935 CRYPTO_ALG_ALLOCATES_MEMORY, 2936 } 2937 }, 2938 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2939 DESC_HDR_SEL0_MDEUA | 2940 DESC_HDR_MODE0_MDEU_MD5, 2941 }, 2942 { .type = CRYPTO_ALG_TYPE_AHASH, 2943 .alg.hash = { 2944 .halg.digestsize = SHA1_DIGEST_SIZE, 2945 .halg.statesize = sizeof(struct talitos_export_state), 2946 .halg.base = { 2947 .cra_name = "sha1", 2948 .cra_driver_name = "sha1-talitos", 2949 .cra_blocksize = SHA1_BLOCK_SIZE, 2950 .cra_flags = CRYPTO_ALG_ASYNC | 2951 CRYPTO_ALG_ALLOCATES_MEMORY, 2952 } 2953 }, 2954 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2955 DESC_HDR_SEL0_MDEUA | 2956 DESC_HDR_MODE0_MDEU_SHA1, 2957 }, 2958 { .type = CRYPTO_ALG_TYPE_AHASH, 2959 .alg.hash = { 2960 .halg.digestsize = SHA224_DIGEST_SIZE, 2961 .halg.statesize = sizeof(struct talitos_export_state), 2962 .halg.base = { 2963 .cra_name = "sha224", 2964 .cra_driver_name = "sha224-talitos", 2965 .cra_blocksize = SHA224_BLOCK_SIZE, 2966 .cra_flags = CRYPTO_ALG_ASYNC | 2967 CRYPTO_ALG_ALLOCATES_MEMORY, 2968 } 2969 }, 2970 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2971 DESC_HDR_SEL0_MDEUA | 2972 DESC_HDR_MODE0_MDEU_SHA224, 2973 }, 2974 { .type = CRYPTO_ALG_TYPE_AHASH, 2975 .alg.hash = { 2976 .halg.digestsize = SHA256_DIGEST_SIZE, 2977 .halg.statesize = sizeof(struct talitos_export_state), 2978 .halg.base = { 2979 .cra_name = "sha256", 2980 .cra_driver_name = "sha256-talitos", 2981 .cra_blocksize = SHA256_BLOCK_SIZE, 2982 .cra_flags = CRYPTO_ALG_ASYNC | 2983 CRYPTO_ALG_ALLOCATES_MEMORY, 2984 } 2985 }, 2986 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2987 DESC_HDR_SEL0_MDEUA | 2988 DESC_HDR_MODE0_MDEU_SHA256, 2989 }, 2990 { .type = CRYPTO_ALG_TYPE_AHASH, 2991 .alg.hash = { 2992 .halg.digestsize = SHA384_DIGEST_SIZE, 2993 .halg.statesize = sizeof(struct talitos_export_state), 2994 .halg.base = { 2995 .cra_name = "sha384", 2996 .cra_driver_name = "sha384-talitos", 2997 .cra_blocksize = SHA384_BLOCK_SIZE, 2998 .cra_flags = CRYPTO_ALG_ASYNC | 2999 CRYPTO_ALG_ALLOCATES_MEMORY, 3000 } 3001 }, 3002 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3003 DESC_HDR_SEL0_MDEUB | 3004 DESC_HDR_MODE0_MDEUB_SHA384, 3005 }, 3006 { .type = CRYPTO_ALG_TYPE_AHASH, 3007 .alg.hash = { 3008 .halg.digestsize = SHA512_DIGEST_SIZE, 3009 .halg.statesize = sizeof(struct talitos_export_state), 3010 .halg.base = { 3011 .cra_name = "sha512", 3012 .cra_driver_name = "sha512-talitos", 3013 .cra_blocksize = SHA512_BLOCK_SIZE, 3014 .cra_flags = CRYPTO_ALG_ASYNC | 3015 CRYPTO_ALG_ALLOCATES_MEMORY, 3016 } 3017 }, 3018 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3019 DESC_HDR_SEL0_MDEUB | 3020 DESC_HDR_MODE0_MDEUB_SHA512, 3021 }, 3022 { .type = CRYPTO_ALG_TYPE_AHASH, 3023 .alg.hash = { 3024 .halg.digestsize = MD5_DIGEST_SIZE, 3025 .halg.statesize = sizeof(struct talitos_export_state), 3026 .halg.base = { 3027 .cra_name = "hmac(md5)", 3028 .cra_driver_name = "hmac-md5-talitos", 3029 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 3030 .cra_flags = CRYPTO_ALG_ASYNC | 3031 CRYPTO_ALG_ALLOCATES_MEMORY, 3032 } 3033 }, 3034 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3035 DESC_HDR_SEL0_MDEUA | 3036 DESC_HDR_MODE0_MDEU_MD5, 3037 }, 3038 { .type = CRYPTO_ALG_TYPE_AHASH, 3039 .alg.hash = { 3040 .halg.digestsize = SHA1_DIGEST_SIZE, 3041 .halg.statesize = sizeof(struct talitos_export_state), 3042 .halg.base = { 3043 .cra_name = "hmac(sha1)", 3044 .cra_driver_name = "hmac-sha1-talitos", 3045 .cra_blocksize = SHA1_BLOCK_SIZE, 3046 .cra_flags = CRYPTO_ALG_ASYNC | 3047 CRYPTO_ALG_ALLOCATES_MEMORY, 3048 } 3049 }, 3050 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3051 DESC_HDR_SEL0_MDEUA | 3052 DESC_HDR_MODE0_MDEU_SHA1, 3053 }, 3054 { .type = CRYPTO_ALG_TYPE_AHASH, 3055 .alg.hash = { 3056 .halg.digestsize = SHA224_DIGEST_SIZE, 3057 .halg.statesize = sizeof(struct talitos_export_state), 3058 .halg.base = { 3059 .cra_name = "hmac(sha224)", 3060 .cra_driver_name = "hmac-sha224-talitos", 3061 .cra_blocksize = SHA224_BLOCK_SIZE, 3062 .cra_flags = CRYPTO_ALG_ASYNC | 3063 CRYPTO_ALG_ALLOCATES_MEMORY, 3064 } 3065 }, 3066 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3067 DESC_HDR_SEL0_MDEUA | 3068 DESC_HDR_MODE0_MDEU_SHA224, 3069 }, 3070 { .type = CRYPTO_ALG_TYPE_AHASH, 3071 .alg.hash = { 3072 .halg.digestsize = SHA256_DIGEST_SIZE, 3073 .halg.statesize = sizeof(struct talitos_export_state), 3074 .halg.base = { 3075 .cra_name = "hmac(sha256)", 3076 .cra_driver_name = "hmac-sha256-talitos", 3077 .cra_blocksize = SHA256_BLOCK_SIZE, 3078 .cra_flags = CRYPTO_ALG_ASYNC | 3079 CRYPTO_ALG_ALLOCATES_MEMORY, 3080 } 3081 }, 3082 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3083 DESC_HDR_SEL0_MDEUA | 3084 DESC_HDR_MODE0_MDEU_SHA256, 3085 }, 3086 { .type = CRYPTO_ALG_TYPE_AHASH, 3087 .alg.hash = { 3088 .halg.digestsize = SHA384_DIGEST_SIZE, 3089 .halg.statesize = sizeof(struct talitos_export_state), 3090 .halg.base = { 3091 .cra_name = "hmac(sha384)", 3092 .cra_driver_name = "hmac-sha384-talitos", 3093 .cra_blocksize = SHA384_BLOCK_SIZE, 3094 .cra_flags = CRYPTO_ALG_ASYNC | 3095 CRYPTO_ALG_ALLOCATES_MEMORY, 3096 } 3097 }, 3098 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3099 DESC_HDR_SEL0_MDEUB | 3100 DESC_HDR_MODE0_MDEUB_SHA384, 3101 }, 3102 { .type = CRYPTO_ALG_TYPE_AHASH, 3103 .alg.hash = { 3104 .halg.digestsize = SHA512_DIGEST_SIZE, 3105 .halg.statesize = sizeof(struct talitos_export_state), 3106 .halg.base = { 3107 .cra_name = "hmac(sha512)", 3108 .cra_driver_name = "hmac-sha512-talitos", 3109 .cra_blocksize = SHA512_BLOCK_SIZE, 3110 .cra_flags = CRYPTO_ALG_ASYNC | 3111 CRYPTO_ALG_ALLOCATES_MEMORY, 3112 } 3113 }, 3114 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3115 DESC_HDR_SEL0_MDEUB | 3116 DESC_HDR_MODE0_MDEUB_SHA512, 3117 } 3118 }; 3119 3120 struct talitos_crypto_alg { 3121 struct list_head entry; 3122 struct device *dev; 3123 struct talitos_alg_template algt; 3124 }; 3125 3126 static int talitos_init_common(struct talitos_ctx *ctx, 3127 struct talitos_crypto_alg *talitos_alg) 3128 { 3129 struct talitos_private *priv; 3130 3131 /* update context with ptr to dev */ 3132 ctx->dev = talitos_alg->dev; 3133 3134 /* assign SEC channel to tfm in round-robin fashion */ 3135 priv = dev_get_drvdata(ctx->dev); 3136 ctx->ch = atomic_inc_return(&priv->last_chan) & 3137 (priv->num_channels - 1); 3138 3139 /* copy descriptor header template value */ 3140 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 3141 3142 /* select done notification */ 3143 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 3144 3145 return 0; 3146 } 3147 3148 static int talitos_cra_init_aead(struct crypto_aead *tfm) 3149 { 3150 struct aead_alg *alg = crypto_aead_alg(tfm); 3151 struct talitos_crypto_alg *talitos_alg; 3152 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 3153 3154 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3155 algt.alg.aead); 3156 3157 return talitos_init_common(ctx, talitos_alg); 3158 } 3159 3160 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) 3161 { 3162 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3163 struct talitos_crypto_alg *talitos_alg; 3164 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); 3165 3166 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3167 algt.alg.skcipher); 3168 3169 return talitos_init_common(ctx, talitos_alg); 3170 } 3171 3172 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 3173 { 3174 struct crypto_alg *alg = tfm->__crt_alg; 3175 struct talitos_crypto_alg *talitos_alg; 3176 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3177 3178 talitos_alg = container_of(__crypto_ahash_alg(alg), 3179 struct talitos_crypto_alg, 3180 algt.alg.hash); 3181 3182 ctx->keylen = 0; 3183 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 3184 sizeof(struct talitos_ahash_req_ctx)); 3185 3186 return talitos_init_common(ctx, talitos_alg); 3187 } 3188 3189 static void talitos_cra_exit(struct crypto_tfm *tfm) 3190 { 3191 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3192 struct device *dev = ctx->dev; 3193 3194 if (ctx->keylen) 3195 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 3196 } 3197 3198 /* 3199 * given the alg's descriptor header template, determine whether descriptor 3200 * type and primary/secondary execution units required match the hw 3201 * capabilities description provided in the device tree node. 3202 */ 3203 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 3204 { 3205 struct talitos_private *priv = dev_get_drvdata(dev); 3206 int ret; 3207 3208 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 3209 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 3210 3211 if (SECONDARY_EU(desc_hdr_template)) 3212 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 3213 & priv->exec_units); 3214 3215 return ret; 3216 } 3217 3218 static void talitos_remove(struct platform_device *ofdev) 3219 { 3220 struct device *dev = &ofdev->dev; 3221 struct talitos_private *priv = dev_get_drvdata(dev); 3222 struct talitos_crypto_alg *t_alg, *n; 3223 int i; 3224 3225 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 3226 switch (t_alg->algt.type) { 3227 case CRYPTO_ALG_TYPE_SKCIPHER: 3228 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); 3229 break; 3230 case CRYPTO_ALG_TYPE_AEAD: 3231 crypto_unregister_aead(&t_alg->algt.alg.aead); 3232 break; 3233 case CRYPTO_ALG_TYPE_AHASH: 3234 crypto_unregister_ahash(&t_alg->algt.alg.hash); 3235 break; 3236 } 3237 list_del(&t_alg->entry); 3238 } 3239 3240 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 3241 talitos_unregister_rng(dev); 3242 3243 for (i = 0; i < 2; i++) 3244 if (priv->irq[i]) { 3245 free_irq(priv->irq[i], dev); 3246 irq_dispose_mapping(priv->irq[i]); 3247 } 3248 3249 tasklet_kill(&priv->done_task[0]); 3250 if (priv->irq[1]) 3251 tasklet_kill(&priv->done_task[1]); 3252 } 3253 3254 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 3255 struct talitos_alg_template 3256 *template) 3257 { 3258 struct talitos_private *priv = dev_get_drvdata(dev); 3259 struct talitos_crypto_alg *t_alg; 3260 struct crypto_alg *alg; 3261 3262 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), 3263 GFP_KERNEL); 3264 if (!t_alg) 3265 return ERR_PTR(-ENOMEM); 3266 3267 t_alg->algt = *template; 3268 3269 switch (t_alg->algt.type) { 3270 case CRYPTO_ALG_TYPE_SKCIPHER: 3271 alg = &t_alg->algt.alg.skcipher.base; 3272 alg->cra_exit = talitos_cra_exit; 3273 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; 3274 t_alg->algt.alg.skcipher.setkey = 3275 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; 3276 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; 3277 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; 3278 if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && 3279 DESC_TYPE(t_alg->algt.desc_hdr_template) != 3280 DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { 3281 devm_kfree(dev, t_alg); 3282 return ERR_PTR(-ENOTSUPP); 3283 } 3284 break; 3285 case CRYPTO_ALG_TYPE_AEAD: 3286 alg = &t_alg->algt.alg.aead.base; 3287 alg->cra_exit = talitos_cra_exit; 3288 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 3289 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: 3290 aead_setkey; 3291 t_alg->algt.alg.aead.encrypt = aead_encrypt; 3292 t_alg->algt.alg.aead.decrypt = aead_decrypt; 3293 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3294 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { 3295 devm_kfree(dev, t_alg); 3296 return ERR_PTR(-ENOTSUPP); 3297 } 3298 break; 3299 case CRYPTO_ALG_TYPE_AHASH: 3300 alg = &t_alg->algt.alg.hash.halg.base; 3301 alg->cra_init = talitos_cra_init_ahash; 3302 alg->cra_exit = talitos_cra_exit; 3303 t_alg->algt.alg.hash.init = ahash_init; 3304 t_alg->algt.alg.hash.update = ahash_update; 3305 t_alg->algt.alg.hash.final = ahash_final; 3306 t_alg->algt.alg.hash.finup = ahash_finup; 3307 t_alg->algt.alg.hash.digest = ahash_digest; 3308 if (!strncmp(alg->cra_name, "hmac", 4)) 3309 t_alg->algt.alg.hash.setkey = ahash_setkey; 3310 t_alg->algt.alg.hash.import = ahash_import; 3311 t_alg->algt.alg.hash.export = ahash_export; 3312 3313 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 3314 !strncmp(alg->cra_name, "hmac", 4)) { 3315 devm_kfree(dev, t_alg); 3316 return ERR_PTR(-ENOTSUPP); 3317 } 3318 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3319 (!strcmp(alg->cra_name, "sha224") || 3320 !strcmp(alg->cra_name, "hmac(sha224)"))) { 3321 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 3322 t_alg->algt.alg.hash.digest = 3323 ahash_digest_sha224_swinit; 3324 t_alg->algt.desc_hdr_template = 3325 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3326 DESC_HDR_SEL0_MDEUA | 3327 DESC_HDR_MODE0_MDEU_SHA256; 3328 } 3329 break; 3330 default: 3331 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 3332 devm_kfree(dev, t_alg); 3333 return ERR_PTR(-EINVAL); 3334 } 3335 3336 alg->cra_module = THIS_MODULE; 3337 if (t_alg->algt.priority) 3338 alg->cra_priority = t_alg->algt.priority; 3339 else 3340 alg->cra_priority = TALITOS_CRA_PRIORITY; 3341 if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH) 3342 alg->cra_alignmask = 3; 3343 else 3344 alg->cra_alignmask = 0; 3345 alg->cra_ctxsize = sizeof(struct talitos_ctx); 3346 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 3347 3348 t_alg->dev = dev; 3349 3350 return t_alg; 3351 } 3352 3353 static int talitos_probe_irq(struct platform_device *ofdev) 3354 { 3355 struct device *dev = &ofdev->dev; 3356 struct device_node *np = ofdev->dev.of_node; 3357 struct talitos_private *priv = dev_get_drvdata(dev); 3358 int err; 3359 bool is_sec1 = has_ftr_sec1(priv); 3360 3361 priv->irq[0] = irq_of_parse_and_map(np, 0); 3362 if (!priv->irq[0]) { 3363 dev_err(dev, "failed to map irq\n"); 3364 return -EINVAL; 3365 } 3366 if (is_sec1) { 3367 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 3368 dev_driver_string(dev), dev); 3369 goto primary_out; 3370 } 3371 3372 priv->irq[1] = irq_of_parse_and_map(np, 1); 3373 3374 /* get the primary irq line */ 3375 if (!priv->irq[1]) { 3376 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 3377 dev_driver_string(dev), dev); 3378 goto primary_out; 3379 } 3380 3381 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 3382 dev_driver_string(dev), dev); 3383 if (err) 3384 goto primary_out; 3385 3386 /* get the secondary irq line */ 3387 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 3388 dev_driver_string(dev), dev); 3389 if (err) { 3390 dev_err(dev, "failed to request secondary irq\n"); 3391 irq_dispose_mapping(priv->irq[1]); 3392 priv->irq[1] = 0; 3393 } 3394 3395 return err; 3396 3397 primary_out: 3398 if (err) { 3399 dev_err(dev, "failed to request primary irq\n"); 3400 irq_dispose_mapping(priv->irq[0]); 3401 priv->irq[0] = 0; 3402 } 3403 3404 return err; 3405 } 3406 3407 static int talitos_probe(struct platform_device *ofdev) 3408 { 3409 struct device *dev = &ofdev->dev; 3410 struct device_node *np = ofdev->dev.of_node; 3411 struct talitos_private *priv; 3412 unsigned int num_channels; 3413 int i, err; 3414 int stride; 3415 3416 if (of_property_read_u32(np, "fsl,num-channels", &num_channels)) 3417 return -EINVAL; 3418 3419 priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL); 3420 if (!priv) 3421 return -ENOMEM; 3422 3423 priv->num_channels = num_channels; 3424 3425 INIT_LIST_HEAD(&priv->alg_list); 3426 3427 dev_set_drvdata(dev, priv); 3428 3429 priv->ofdev = ofdev; 3430 3431 spin_lock_init(&priv->reg_lock); 3432 3433 priv->reg = devm_platform_ioremap_resource(ofdev, 0); 3434 if (IS_ERR(priv->reg)) { 3435 dev_err(dev, "failed to of_iomap\n"); 3436 err = PTR_ERR(priv->reg); 3437 goto err_out; 3438 } 3439 3440 /* get SEC version capabilities from device tree */ 3441 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); 3442 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); 3443 of_property_read_u32(np, "fsl,descriptor-types-mask", 3444 &priv->desc_types); 3445 3446 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 3447 !priv->exec_units || !priv->desc_types) { 3448 dev_err(dev, "invalid property data in device tree node\n"); 3449 err = -EINVAL; 3450 goto err_out; 3451 } 3452 3453 if (of_device_is_compatible(np, "fsl,sec3.0")) 3454 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 3455 3456 if (of_device_is_compatible(np, "fsl,sec2.1")) 3457 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 3458 TALITOS_FTR_SHA224_HWINIT | 3459 TALITOS_FTR_HMAC_OK; 3460 3461 if (of_device_is_compatible(np, "fsl,sec1.0")) 3462 priv->features |= TALITOS_FTR_SEC1; 3463 3464 if (of_device_is_compatible(np, "fsl,sec1.2")) { 3465 priv->reg_deu = priv->reg + TALITOS12_DEU; 3466 priv->reg_aesu = priv->reg + TALITOS12_AESU; 3467 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 3468 stride = TALITOS1_CH_STRIDE; 3469 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 3470 priv->reg_deu = priv->reg + TALITOS10_DEU; 3471 priv->reg_aesu = priv->reg + TALITOS10_AESU; 3472 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 3473 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 3474 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 3475 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 3476 stride = TALITOS1_CH_STRIDE; 3477 } else { 3478 priv->reg_deu = priv->reg + TALITOS2_DEU; 3479 priv->reg_aesu = priv->reg + TALITOS2_AESU; 3480 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 3481 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 3482 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 3483 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 3484 priv->reg_keu = priv->reg + TALITOS2_KEU; 3485 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 3486 stride = TALITOS2_CH_STRIDE; 3487 } 3488 3489 err = talitos_probe_irq(ofdev); 3490 if (err) 3491 goto err_out; 3492 3493 if (has_ftr_sec1(priv)) { 3494 if (priv->num_channels == 1) 3495 tasklet_init(&priv->done_task[0], talitos1_done_ch0, 3496 (unsigned long)dev); 3497 else 3498 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 3499 (unsigned long)dev); 3500 } else { 3501 if (priv->irq[1]) { 3502 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3503 (unsigned long)dev); 3504 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3505 (unsigned long)dev); 3506 } else if (priv->num_channels == 1) { 3507 tasklet_init(&priv->done_task[0], talitos2_done_ch0, 3508 (unsigned long)dev); 3509 } else { 3510 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3511 (unsigned long)dev); 3512 } 3513 } 3514 3515 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3516 3517 for (i = 0; i < priv->num_channels; i++) { 3518 priv->chan[i].reg = priv->reg + stride * (i + 1); 3519 if (!priv->irq[1] || !(i & 1)) 3520 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3521 3522 spin_lock_init(&priv->chan[i].head_lock); 3523 spin_lock_init(&priv->chan[i].tail_lock); 3524 3525 priv->chan[i].fifo = devm_kcalloc(dev, 3526 priv->fifo_len, 3527 sizeof(struct talitos_request), 3528 GFP_KERNEL); 3529 if (!priv->chan[i].fifo) { 3530 dev_err(dev, "failed to allocate request fifo %d\n", i); 3531 err = -ENOMEM; 3532 goto err_out; 3533 } 3534 3535 atomic_set(&priv->chan[i].submit_count, 3536 -(priv->chfifo_len - 1)); 3537 } 3538 3539 dma_set_mask(dev, DMA_BIT_MASK(36)); 3540 3541 /* reset and initialize the h/w */ 3542 err = init_device(dev); 3543 if (err) { 3544 dev_err(dev, "failed to initialize device\n"); 3545 goto err_out; 3546 } 3547 3548 /* register the RNG, if available */ 3549 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3550 err = talitos_register_rng(dev); 3551 if (err) { 3552 dev_err(dev, "failed to register hwrng: %d\n", err); 3553 goto err_out; 3554 } else 3555 dev_info(dev, "hwrng\n"); 3556 } 3557 3558 /* register crypto algorithms the device supports */ 3559 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3560 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3561 struct talitos_crypto_alg *t_alg; 3562 struct crypto_alg *alg = NULL; 3563 3564 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3565 if (IS_ERR(t_alg)) { 3566 err = PTR_ERR(t_alg); 3567 if (err == -ENOTSUPP) 3568 continue; 3569 goto err_out; 3570 } 3571 3572 switch (t_alg->algt.type) { 3573 case CRYPTO_ALG_TYPE_SKCIPHER: 3574 err = crypto_register_skcipher( 3575 &t_alg->algt.alg.skcipher); 3576 alg = &t_alg->algt.alg.skcipher.base; 3577 break; 3578 3579 case CRYPTO_ALG_TYPE_AEAD: 3580 err = crypto_register_aead( 3581 &t_alg->algt.alg.aead); 3582 alg = &t_alg->algt.alg.aead.base; 3583 break; 3584 3585 case CRYPTO_ALG_TYPE_AHASH: 3586 err = crypto_register_ahash( 3587 &t_alg->algt.alg.hash); 3588 alg = &t_alg->algt.alg.hash.halg.base; 3589 break; 3590 } 3591 if (err) { 3592 dev_err(dev, "%s alg registration failed\n", 3593 alg->cra_driver_name); 3594 devm_kfree(dev, t_alg); 3595 } else 3596 list_add_tail(&t_alg->entry, &priv->alg_list); 3597 } 3598 } 3599 if (!list_empty(&priv->alg_list)) 3600 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3601 (char *)of_get_property(np, "compatible", NULL)); 3602 3603 return 0; 3604 3605 err_out: 3606 talitos_remove(ofdev); 3607 3608 return err; 3609 } 3610 3611 static const struct of_device_id talitos_match[] = { 3612 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3613 { 3614 .compatible = "fsl,sec1.0", 3615 }, 3616 #endif 3617 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3618 { 3619 .compatible = "fsl,sec2.0", 3620 }, 3621 #endif 3622 {}, 3623 }; 3624 MODULE_DEVICE_TABLE(of, talitos_match); 3625 3626 static struct platform_driver talitos_driver = { 3627 .driver = { 3628 .name = "talitos", 3629 .of_match_table = talitos_match, 3630 }, 3631 .probe = talitos_probe, 3632 .remove = talitos_remove, 3633 }; 3634 3635 module_platform_driver(talitos_driver); 3636 3637 MODULE_LICENSE("GPL"); 3638 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3639 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3640