1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * talitos - Freescale Integrated Security Engine (SEC) device driver 4 * 5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 6 * 7 * Scatterlist Crypto API glue code copied from files with the following: 8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 9 * 10 * Crypto algorithm registration code copied from hifn driver: 11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 12 * All rights reserved. 13 */ 14 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/mod_devicetable.h> 18 #include <linux/device.h> 19 #include <linux/interrupt.h> 20 #include <linux/crypto.h> 21 #include <linux/hw_random.h> 22 #include <linux/of.h> 23 #include <linux/of_irq.h> 24 #include <linux/platform_device.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/io.h> 27 #include <linux/spinlock.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/slab.h> 30 31 #include <crypto/algapi.h> 32 #include <crypto/aes.h> 33 #include <crypto/internal/des.h> 34 #include <crypto/sha1.h> 35 #include <crypto/sha2.h> 36 #include <crypto/md5.h> 37 #include <crypto/internal/aead.h> 38 #include <crypto/authenc.h> 39 #include <crypto/internal/skcipher.h> 40 #include <crypto/hash.h> 41 #include <crypto/internal/hash.h> 42 #include <crypto/scatterwalk.h> 43 44 #include "talitos.h" 45 46 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 47 unsigned int len, bool is_sec1) 48 { 49 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 50 if (is_sec1) { 51 ptr->len1 = cpu_to_be16(len); 52 } else { 53 ptr->len = cpu_to_be16(len); 54 ptr->eptr = upper_32_bits(dma_addr); 55 } 56 } 57 58 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 59 struct talitos_ptr *src_ptr, bool is_sec1) 60 { 61 dst_ptr->ptr = src_ptr->ptr; 62 if (is_sec1) { 63 dst_ptr->len1 = src_ptr->len1; 64 } else { 65 dst_ptr->len = src_ptr->len; 66 dst_ptr->eptr = src_ptr->eptr; 67 } 68 } 69 70 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 71 bool is_sec1) 72 { 73 if (is_sec1) 74 return be16_to_cpu(ptr->len1); 75 else 76 return be16_to_cpu(ptr->len); 77 } 78 79 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, 80 bool is_sec1) 81 { 82 if (!is_sec1) 83 ptr->j_extent = val; 84 } 85 86 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) 87 { 88 if (!is_sec1) 89 ptr->j_extent |= val; 90 } 91 92 /* 93 * map virtual single (contiguous) pointer to h/w descriptor pointer 94 */ 95 static void __map_single_talitos_ptr(struct device *dev, 96 struct talitos_ptr *ptr, 97 unsigned int len, void *data, 98 enum dma_data_direction dir, 99 unsigned long attrs) 100 { 101 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); 102 struct talitos_private *priv = dev_get_drvdata(dev); 103 bool is_sec1 = has_ftr_sec1(priv); 104 105 to_talitos_ptr(ptr, dma_addr, len, is_sec1); 106 } 107 108 static void map_single_talitos_ptr(struct device *dev, 109 struct talitos_ptr *ptr, 110 unsigned int len, void *data, 111 enum dma_data_direction dir) 112 { 113 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); 114 } 115 116 static void map_single_talitos_ptr_nosync(struct device *dev, 117 struct talitos_ptr *ptr, 118 unsigned int len, void *data, 119 enum dma_data_direction dir) 120 { 121 __map_single_talitos_ptr(dev, ptr, len, data, dir, 122 DMA_ATTR_SKIP_CPU_SYNC); 123 } 124 125 /* 126 * unmap bus single (contiguous) h/w descriptor pointer 127 */ 128 static void unmap_single_talitos_ptr(struct device *dev, 129 struct talitos_ptr *ptr, 130 enum dma_data_direction dir) 131 { 132 struct talitos_private *priv = dev_get_drvdata(dev); 133 bool is_sec1 = has_ftr_sec1(priv); 134 135 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 136 from_talitos_ptr_len(ptr, is_sec1), dir); 137 } 138 139 static int reset_channel(struct device *dev, int ch) 140 { 141 struct talitos_private *priv = dev_get_drvdata(dev); 142 unsigned int timeout = TALITOS_TIMEOUT; 143 bool is_sec1 = has_ftr_sec1(priv); 144 145 if (is_sec1) { 146 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 147 TALITOS1_CCCR_LO_RESET); 148 149 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 150 TALITOS1_CCCR_LO_RESET) && --timeout) 151 cpu_relax(); 152 } else { 153 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 154 TALITOS2_CCCR_RESET); 155 156 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 157 TALITOS2_CCCR_RESET) && --timeout) 158 cpu_relax(); 159 } 160 161 if (timeout == 0) { 162 dev_err(dev, "failed to reset channel %d\n", ch); 163 return -EIO; 164 } 165 166 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 167 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 168 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 169 /* enable chaining descriptors */ 170 if (is_sec1) 171 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 172 TALITOS_CCCR_LO_NE); 173 174 /* and ICCR writeback, if available */ 175 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 176 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 177 TALITOS_CCCR_LO_IWSE); 178 179 return 0; 180 } 181 182 static int reset_device(struct device *dev) 183 { 184 struct talitos_private *priv = dev_get_drvdata(dev); 185 unsigned int timeout = TALITOS_TIMEOUT; 186 bool is_sec1 = has_ftr_sec1(priv); 187 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 188 189 setbits32(priv->reg + TALITOS_MCR, mcr); 190 191 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 192 && --timeout) 193 cpu_relax(); 194 195 if (priv->irq[1]) { 196 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 197 setbits32(priv->reg + TALITOS_MCR, mcr); 198 } 199 200 if (timeout == 0) { 201 dev_err(dev, "failed to reset device\n"); 202 return -EIO; 203 } 204 205 return 0; 206 } 207 208 /* 209 * Reset and initialize the device 210 */ 211 static int init_device(struct device *dev) 212 { 213 struct talitos_private *priv = dev_get_drvdata(dev); 214 int ch, err; 215 bool is_sec1 = has_ftr_sec1(priv); 216 217 /* 218 * Master reset 219 * errata documentation: warning: certain SEC interrupts 220 * are not fully cleared by writing the MCR:SWR bit, 221 * set bit twice to completely reset 222 */ 223 err = reset_device(dev); 224 if (err) 225 return err; 226 227 err = reset_device(dev); 228 if (err) 229 return err; 230 231 /* reset channels */ 232 for (ch = 0; ch < priv->num_channels; ch++) { 233 err = reset_channel(dev, ch); 234 if (err) 235 return err; 236 } 237 238 /* enable channel done and error interrupts */ 239 if (is_sec1) { 240 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 241 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 242 /* disable parity error check in DEU (erroneous? test vect.) */ 243 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 244 } else { 245 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 246 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 247 } 248 249 /* disable integrity check error interrupts (use writeback instead) */ 250 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 251 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 252 TALITOS_MDEUICR_LO_ICE); 253 254 return 0; 255 } 256 257 static void dma_map_request(struct device *dev, struct talitos_request *request, 258 struct talitos_desc *desc, bool is_sec1) 259 { 260 struct talitos_edesc *edesc = 261 container_of(desc, struct talitos_edesc, desc); 262 dma_addr_t dma_desc, prev_dma_desc; 263 struct talitos_edesc *prev_edesc = NULL; 264 265 if (is_sec1) { 266 while (edesc) { 267 edesc->desc.hdr1 = edesc->desc.hdr; 268 269 dma_desc = dma_map_single(dev, &edesc->desc.hdr1, 270 TALITOS_DESC_SIZE, 271 DMA_BIDIRECTIONAL); 272 273 if (!prev_edesc) { 274 request->dma_desc = dma_desc; 275 goto next; 276 } 277 278 /* Chain in any previous descriptors. */ 279 280 prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); 281 282 dma_sync_single_for_device(dev, prev_dma_desc, 283 TALITOS_DESC_SIZE, 284 DMA_TO_DEVICE); 285 286 next: 287 prev_edesc = edesc; 288 prev_dma_desc = dma_desc; 289 edesc = edesc->next_desc; 290 } 291 } else { 292 request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, 293 DMA_BIDIRECTIONAL); 294 } 295 } 296 297 /** 298 * talitos_submit - submits a descriptor to the device for processing 299 * @dev: the SEC device to be used 300 * @ch: the SEC device channel to be used 301 * @desc: the descriptor to be processed by the device 302 * @callback: whom to call when processing is complete 303 * @context: a handle for use by caller (optional) 304 * 305 * desc must contain valid dma-mapped (bus physical) address pointers. 306 * callback must check err and feedback in descriptor header 307 * for device processing status. 308 */ 309 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 310 void (*callback)(struct device *dev, 311 struct talitos_desc *desc, 312 void *context, int error), 313 void *context) 314 { 315 struct talitos_private *priv = dev_get_drvdata(dev); 316 struct talitos_request *request; 317 unsigned long flags; 318 int head; 319 bool is_sec1 = has_ftr_sec1(priv); 320 321 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 322 323 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 324 /* h/w fifo is full */ 325 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 326 return -EAGAIN; 327 } 328 329 head = priv->chan[ch].head; 330 request = &priv->chan[ch].fifo[head]; 331 332 /* map descriptor and save caller data */ 333 dma_map_request(dev, request, desc, is_sec1); 334 request->callback = callback; 335 request->context = context; 336 337 /* increment fifo head */ 338 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 339 340 smp_wmb(); 341 request->desc = desc; 342 343 /* GO! */ 344 wmb(); 345 out_be32(priv->chan[ch].reg + TALITOS_FF, 346 upper_32_bits(request->dma_desc)); 347 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 348 lower_32_bits(request->dma_desc)); 349 350 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 351 352 return -EINPROGRESS; 353 } 354 355 static __be32 get_request_hdr(struct device *dev, 356 struct talitos_request *request, bool is_sec1) 357 { 358 struct talitos_edesc *edesc; 359 dma_addr_t dma_desc; 360 361 if (!is_sec1) { 362 dma_sync_single_for_cpu(dev, request->dma_desc, 363 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 364 365 return request->desc->hdr; 366 } 367 368 edesc = container_of(request->desc, struct talitos_edesc, desc); 369 dma_desc = request->dma_desc; 370 while (edesc->next_desc) { 371 dma_desc = be32_to_cpu(edesc->desc.next_desc); 372 edesc = edesc->next_desc; 373 } 374 375 dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, 376 DMA_BIDIRECTIONAL); 377 378 return edesc->desc.hdr1; 379 } 380 381 static void dma_unmap_request(struct device *dev, 382 struct talitos_request *request, bool is_sec1) 383 { 384 struct talitos_edesc *edesc; 385 386 if (is_sec1) { 387 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 388 DMA_BIDIRECTIONAL); 389 edesc = container_of(request->desc, struct talitos_edesc, desc); 390 while (edesc->next_desc) { 391 dma_unmap_single(dev, 392 be32_to_cpu(edesc->desc.next_desc), 393 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 394 edesc = edesc->next_desc; 395 } 396 } else { 397 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 398 DMA_BIDIRECTIONAL); 399 } 400 } 401 402 /* 403 * process what was done, notify callback of error if not 404 */ 405 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 406 { 407 struct talitos_private *priv = dev_get_drvdata(dev); 408 struct talitos_request *request, saved_req; 409 unsigned long flags; 410 int tail, status; 411 bool is_sec1 = has_ftr_sec1(priv); 412 413 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 414 415 tail = priv->chan[ch].tail; 416 while (priv->chan[ch].fifo[tail].desc) { 417 __be32 hdr; 418 419 request = &priv->chan[ch].fifo[tail]; 420 421 /* descriptors with their done bits set don't get the error */ 422 rmb(); 423 hdr = get_request_hdr(dev, request, is_sec1); 424 425 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 426 status = 0; 427 else 428 if (!error) 429 break; 430 else 431 status = error; 432 433 dma_unmap_request(dev, request, is_sec1); 434 435 /* copy entries so we can call callback outside lock */ 436 saved_req.desc = request->desc; 437 saved_req.callback = request->callback; 438 saved_req.context = request->context; 439 440 /* release request entry in fifo */ 441 smp_wmb(); 442 request->desc = NULL; 443 444 /* increment fifo tail */ 445 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 446 447 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 448 449 atomic_dec(&priv->chan[ch].submit_count); 450 451 saved_req.callback(dev, saved_req.desc, saved_req.context, 452 status); 453 /* channel may resume processing in single desc error case */ 454 if (error && !reset_ch && status == error) 455 return; 456 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 457 tail = priv->chan[ch].tail; 458 } 459 460 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 461 } 462 463 /* 464 * process completed requests for channels that have done status 465 */ 466 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 467 static void talitos1_done_##name(unsigned long data) \ 468 { \ 469 struct device *dev = (struct device *)data; \ 470 struct talitos_private *priv = dev_get_drvdata(dev); \ 471 unsigned long flags; \ 472 \ 473 if (ch_done_mask & 0x10000000) \ 474 flush_channel(dev, 0, 0, 0); \ 475 if (ch_done_mask & 0x40000000) \ 476 flush_channel(dev, 1, 0, 0); \ 477 if (ch_done_mask & 0x00010000) \ 478 flush_channel(dev, 2, 0, 0); \ 479 if (ch_done_mask & 0x00040000) \ 480 flush_channel(dev, 3, 0, 0); \ 481 \ 482 /* At this point, all completed channels have been processed */ \ 483 /* Unmask done interrupts for channels completed later on. */ \ 484 spin_lock_irqsave(&priv->reg_lock, flags); \ 485 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 486 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 487 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 488 } 489 490 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 491 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) 492 493 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 494 static void talitos2_done_##name(unsigned long data) \ 495 { \ 496 struct device *dev = (struct device *)data; \ 497 struct talitos_private *priv = dev_get_drvdata(dev); \ 498 unsigned long flags; \ 499 \ 500 if (ch_done_mask & 1) \ 501 flush_channel(dev, 0, 0, 0); \ 502 if (ch_done_mask & (1 << 2)) \ 503 flush_channel(dev, 1, 0, 0); \ 504 if (ch_done_mask & (1 << 4)) \ 505 flush_channel(dev, 2, 0, 0); \ 506 if (ch_done_mask & (1 << 6)) \ 507 flush_channel(dev, 3, 0, 0); \ 508 \ 509 /* At this point, all completed channels have been processed */ \ 510 /* Unmask done interrupts for channels completed later on. */ \ 511 spin_lock_irqsave(&priv->reg_lock, flags); \ 512 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 513 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 514 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 515 } 516 517 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 518 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) 519 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 520 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 521 522 static __be32 search_desc_hdr_in_request(struct talitos_request *request, 523 dma_addr_t cur_desc, bool is_sec1) 524 { 525 struct talitos_edesc *edesc; 526 527 if (request->dma_desc == cur_desc) { 528 return request->desc->hdr; 529 } else if (is_sec1) { 530 edesc = container_of(request->desc, struct talitos_edesc, desc); 531 while (edesc->next_desc) { 532 if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) 533 return edesc->next_desc->desc.hdr1; 534 edesc = edesc->next_desc; 535 } 536 } 537 return 0; 538 } 539 540 /* 541 * locate current (offending) descriptor 542 */ 543 static __be32 current_desc_hdr(struct device *dev, int ch) 544 { 545 struct talitos_private *priv = dev_get_drvdata(dev); 546 bool is_sec1 = has_ftr_sec1(priv); 547 struct talitos_request *request; 548 int tail, iter; 549 dma_addr_t cur_desc; 550 __be32 hdr = 0; 551 552 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 553 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 554 555 if (!cur_desc) { 556 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 557 return 0; 558 } 559 560 tail = priv->chan[ch].tail; 561 iter = tail; 562 do { 563 request = &priv->chan[ch].fifo[iter]; 564 565 hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); 566 if (hdr) 567 break; 568 569 iter = (iter + 1) & (priv->fifo_len - 1); 570 } while (iter != tail); 571 572 if (!hdr) 573 dev_err(dev, "couldn't locate current descriptor\n"); 574 575 return hdr; 576 } 577 578 /* 579 * user diagnostics; report root cause of error based on execution unit status 580 */ 581 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) 582 { 583 struct talitos_private *priv = dev_get_drvdata(dev); 584 int i; 585 586 if (!desc_hdr) 587 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); 588 589 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 590 case DESC_HDR_SEL0_AFEU: 591 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 592 in_be32(priv->reg_afeu + TALITOS_EUISR), 593 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 594 break; 595 case DESC_HDR_SEL0_DEU: 596 dev_err(dev, "DEUISR 0x%08x_%08x\n", 597 in_be32(priv->reg_deu + TALITOS_EUISR), 598 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 599 break; 600 case DESC_HDR_SEL0_MDEUA: 601 case DESC_HDR_SEL0_MDEUB: 602 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 603 in_be32(priv->reg_mdeu + TALITOS_EUISR), 604 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 605 break; 606 case DESC_HDR_SEL0_RNG: 607 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 608 in_be32(priv->reg_rngu + TALITOS_ISR), 609 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 610 break; 611 case DESC_HDR_SEL0_PKEU: 612 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 613 in_be32(priv->reg_pkeu + TALITOS_EUISR), 614 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 615 break; 616 case DESC_HDR_SEL0_AESU: 617 dev_err(dev, "AESUISR 0x%08x_%08x\n", 618 in_be32(priv->reg_aesu + TALITOS_EUISR), 619 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 620 break; 621 case DESC_HDR_SEL0_CRCU: 622 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 623 in_be32(priv->reg_crcu + TALITOS_EUISR), 624 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 625 break; 626 case DESC_HDR_SEL0_KEU: 627 dev_err(dev, "KEUISR 0x%08x_%08x\n", 628 in_be32(priv->reg_pkeu + TALITOS_EUISR), 629 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 630 break; 631 } 632 633 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 634 case DESC_HDR_SEL1_MDEUA: 635 case DESC_HDR_SEL1_MDEUB: 636 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 637 in_be32(priv->reg_mdeu + TALITOS_EUISR), 638 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 639 break; 640 case DESC_HDR_SEL1_CRCU: 641 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 642 in_be32(priv->reg_crcu + TALITOS_EUISR), 643 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 644 break; 645 } 646 647 for (i = 0; i < 8; i++) 648 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 649 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 650 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 651 } 652 653 /* 654 * recover from error interrupts 655 */ 656 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 657 { 658 struct talitos_private *priv = dev_get_drvdata(dev); 659 unsigned int timeout = TALITOS_TIMEOUT; 660 int ch, error, reset_dev = 0; 661 u32 v_lo; 662 bool is_sec1 = has_ftr_sec1(priv); 663 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 664 665 for (ch = 0; ch < priv->num_channels; ch++) { 666 /* skip channels without errors */ 667 if (is_sec1) { 668 /* bits 29, 31, 17, 19 */ 669 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 670 continue; 671 } else { 672 if (!(isr & (1 << (ch * 2 + 1)))) 673 continue; 674 } 675 676 error = -EINVAL; 677 678 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 679 680 if (v_lo & TALITOS_CCPSR_LO_DOF) { 681 dev_err(dev, "double fetch fifo overflow error\n"); 682 error = -EAGAIN; 683 reset_ch = 1; 684 } 685 if (v_lo & TALITOS_CCPSR_LO_SOF) { 686 /* h/w dropped descriptor */ 687 dev_err(dev, "single fetch fifo overflow error\n"); 688 error = -EAGAIN; 689 } 690 if (v_lo & TALITOS_CCPSR_LO_MDTE) 691 dev_err(dev, "master data transfer error\n"); 692 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 693 dev_err(dev, is_sec1 ? "pointer not complete error\n" 694 : "s/g data length zero error\n"); 695 if (v_lo & TALITOS_CCPSR_LO_FPZ) 696 dev_err(dev, is_sec1 ? "parity error\n" 697 : "fetch pointer zero error\n"); 698 if (v_lo & TALITOS_CCPSR_LO_IDH) 699 dev_err(dev, "illegal descriptor header error\n"); 700 if (v_lo & TALITOS_CCPSR_LO_IEU) 701 dev_err(dev, is_sec1 ? "static assignment error\n" 702 : "invalid exec unit error\n"); 703 if (v_lo & TALITOS_CCPSR_LO_EU) 704 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 705 if (!is_sec1) { 706 if (v_lo & TALITOS_CCPSR_LO_GB) 707 dev_err(dev, "gather boundary error\n"); 708 if (v_lo & TALITOS_CCPSR_LO_GRL) 709 dev_err(dev, "gather return/length error\n"); 710 if (v_lo & TALITOS_CCPSR_LO_SB) 711 dev_err(dev, "scatter boundary error\n"); 712 if (v_lo & TALITOS_CCPSR_LO_SRL) 713 dev_err(dev, "scatter return/length error\n"); 714 } 715 716 flush_channel(dev, ch, error, reset_ch); 717 718 if (reset_ch) { 719 reset_channel(dev, ch); 720 } else { 721 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 722 TALITOS2_CCCR_CONT); 723 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 724 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 725 TALITOS2_CCCR_CONT) && --timeout) 726 cpu_relax(); 727 if (timeout == 0) { 728 dev_err(dev, "failed to restart channel %d\n", 729 ch); 730 reset_dev = 1; 731 } 732 } 733 } 734 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 735 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 736 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 737 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 738 isr, isr_lo); 739 else 740 dev_err(dev, "done overflow, internal time out, or " 741 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 742 743 /* purge request queues */ 744 for (ch = 0; ch < priv->num_channels; ch++) 745 flush_channel(dev, ch, -EIO, 1); 746 747 /* reset and reinitialize the device */ 748 init_device(dev); 749 } 750 } 751 752 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 753 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 754 { \ 755 struct device *dev = data; \ 756 struct talitos_private *priv = dev_get_drvdata(dev); \ 757 u32 isr, isr_lo; \ 758 unsigned long flags; \ 759 \ 760 spin_lock_irqsave(&priv->reg_lock, flags); \ 761 isr = in_be32(priv->reg + TALITOS_ISR); \ 762 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 763 /* Acknowledge interrupt */ \ 764 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 765 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 766 \ 767 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 768 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 769 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 770 } \ 771 else { \ 772 if (likely(isr & ch_done_mask)) { \ 773 /* mask further done interrupts. */ \ 774 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 775 /* done_task will unmask done interrupts at exit */ \ 776 tasklet_schedule(&priv->done_task[tlet]); \ 777 } \ 778 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 779 } \ 780 \ 781 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 782 IRQ_NONE; \ 783 } 784 785 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 786 787 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 788 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 789 { \ 790 struct device *dev = data; \ 791 struct talitos_private *priv = dev_get_drvdata(dev); \ 792 u32 isr, isr_lo; \ 793 unsigned long flags; \ 794 \ 795 spin_lock_irqsave(&priv->reg_lock, flags); \ 796 isr = in_be32(priv->reg + TALITOS_ISR); \ 797 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 798 /* Acknowledge interrupt */ \ 799 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 800 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 801 \ 802 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 803 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 804 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 805 } \ 806 else { \ 807 if (likely(isr & ch_done_mask)) { \ 808 /* mask further done interrupts. */ \ 809 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 810 /* done_task will unmask done interrupts at exit */ \ 811 tasklet_schedule(&priv->done_task[tlet]); \ 812 } \ 813 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 814 } \ 815 \ 816 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 817 IRQ_NONE; \ 818 } 819 820 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 821 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 822 0) 823 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 824 1) 825 826 /* 827 * hwrng 828 */ 829 static int talitos_rng_data_present(struct hwrng *rng, int wait) 830 { 831 struct device *dev = (struct device *)rng->priv; 832 struct talitos_private *priv = dev_get_drvdata(dev); 833 u32 ofl; 834 int i; 835 836 for (i = 0; i < 20; i++) { 837 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 838 TALITOS_RNGUSR_LO_OFL; 839 if (ofl || !wait) 840 break; 841 udelay(10); 842 } 843 844 return !!ofl; 845 } 846 847 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 848 { 849 struct device *dev = (struct device *)rng->priv; 850 struct talitos_private *priv = dev_get_drvdata(dev); 851 852 /* rng fifo requires 64-bit accesses */ 853 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 854 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 855 856 return sizeof(u32); 857 } 858 859 static int talitos_rng_init(struct hwrng *rng) 860 { 861 struct device *dev = (struct device *)rng->priv; 862 struct talitos_private *priv = dev_get_drvdata(dev); 863 unsigned int timeout = TALITOS_TIMEOUT; 864 865 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 866 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 867 & TALITOS_RNGUSR_LO_RD) 868 && --timeout) 869 cpu_relax(); 870 if (timeout == 0) { 871 dev_err(dev, "failed to reset rng hw\n"); 872 return -ENODEV; 873 } 874 875 /* start generating */ 876 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 877 878 return 0; 879 } 880 881 static int talitos_register_rng(struct device *dev) 882 { 883 struct talitos_private *priv = dev_get_drvdata(dev); 884 int err; 885 886 priv->rng.name = dev_driver_string(dev); 887 priv->rng.init = talitos_rng_init; 888 priv->rng.data_present = talitos_rng_data_present; 889 priv->rng.data_read = talitos_rng_data_read; 890 priv->rng.priv = (unsigned long)dev; 891 892 err = hwrng_register(&priv->rng); 893 if (!err) 894 priv->rng_registered = true; 895 896 return err; 897 } 898 899 static void talitos_unregister_rng(struct device *dev) 900 { 901 struct talitos_private *priv = dev_get_drvdata(dev); 902 903 if (!priv->rng_registered) 904 return; 905 906 hwrng_unregister(&priv->rng); 907 priv->rng_registered = false; 908 } 909 910 /* 911 * crypto alg 912 */ 913 #define TALITOS_CRA_PRIORITY 3000 914 /* 915 * Defines a priority for doing AEAD with descriptors type 916 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP 917 */ 918 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) 919 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 920 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) 921 #else 922 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) 923 #endif 924 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 925 926 struct talitos_ctx { 927 struct device *dev; 928 int ch; 929 __be32 desc_hdr_template; 930 u8 key[TALITOS_MAX_KEY_SIZE]; 931 u8 iv[TALITOS_MAX_IV_LENGTH]; 932 dma_addr_t dma_key; 933 unsigned int keylen; 934 unsigned int enckeylen; 935 unsigned int authkeylen; 936 }; 937 938 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 939 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 940 941 struct talitos_ahash_req_ctx { 942 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 943 unsigned int hw_context_size; 944 u8 buf[2][HASH_MAX_BLOCK_SIZE]; 945 int buf_idx; 946 unsigned int swinit; 947 unsigned int first_request; 948 unsigned int last_request; 949 unsigned int to_hash_later; 950 unsigned int nbuf; 951 struct scatterlist bufsl[2]; 952 struct scatterlist *psrc; 953 }; 954 955 struct talitos_export_state { 956 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 957 u8 buf[HASH_MAX_BLOCK_SIZE]; 958 unsigned int swinit; 959 unsigned int first_request; 960 unsigned int last_request; 961 unsigned int to_hash_later; 962 unsigned int nbuf; 963 }; 964 965 static int aead_setkey(struct crypto_aead *authenc, 966 const u8 *key, unsigned int keylen) 967 { 968 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 969 struct device *dev = ctx->dev; 970 struct crypto_authenc_keys keys; 971 972 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 973 goto badkey; 974 975 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 976 goto badkey; 977 978 if (ctx->keylen) 979 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 980 981 memcpy(ctx->key, keys.authkey, keys.authkeylen); 982 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 983 984 ctx->keylen = keys.authkeylen + keys.enckeylen; 985 ctx->enckeylen = keys.enckeylen; 986 ctx->authkeylen = keys.authkeylen; 987 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 988 DMA_TO_DEVICE); 989 990 memzero_explicit(&keys, sizeof(keys)); 991 return 0; 992 993 badkey: 994 memzero_explicit(&keys, sizeof(keys)); 995 return -EINVAL; 996 } 997 998 static int aead_des3_setkey(struct crypto_aead *authenc, 999 const u8 *key, unsigned int keylen) 1000 { 1001 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1002 struct device *dev = ctx->dev; 1003 struct crypto_authenc_keys keys; 1004 int err; 1005 1006 err = crypto_authenc_extractkeys(&keys, key, keylen); 1007 if (unlikely(err)) 1008 goto out; 1009 1010 err = -EINVAL; 1011 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 1012 goto out; 1013 1014 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); 1015 if (err) 1016 goto out; 1017 1018 if (ctx->keylen) 1019 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1020 1021 memcpy(ctx->key, keys.authkey, keys.authkeylen); 1022 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 1023 1024 ctx->keylen = keys.authkeylen + keys.enckeylen; 1025 ctx->enckeylen = keys.enckeylen; 1026 ctx->authkeylen = keys.authkeylen; 1027 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 1028 DMA_TO_DEVICE); 1029 1030 out: 1031 memzero_explicit(&keys, sizeof(keys)); 1032 return err; 1033 } 1034 1035 static void talitos_sg_unmap(struct device *dev, 1036 struct talitos_edesc *edesc, 1037 struct scatterlist *src, 1038 struct scatterlist *dst, 1039 unsigned int len, unsigned int offset) 1040 { 1041 struct talitos_private *priv = dev_get_drvdata(dev); 1042 bool is_sec1 = has_ftr_sec1(priv); 1043 unsigned int src_nents = edesc->src_nents ? : 1; 1044 unsigned int dst_nents = edesc->dst_nents ? : 1; 1045 1046 if (is_sec1 && dst && dst_nents > 1) { 1047 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, 1048 len, DMA_FROM_DEVICE); 1049 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, 1050 offset); 1051 } 1052 if (src != dst) { 1053 if (src_nents == 1 || !is_sec1) 1054 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 1055 1056 if (dst && (dst_nents == 1 || !is_sec1)) 1057 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 1058 } else if (src_nents == 1 || !is_sec1) { 1059 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 1060 } 1061 } 1062 1063 static void ipsec_esp_unmap(struct device *dev, 1064 struct talitos_edesc *edesc, 1065 struct aead_request *areq, bool encrypt) 1066 { 1067 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1068 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1069 unsigned int ivsize = crypto_aead_ivsize(aead); 1070 unsigned int authsize = crypto_aead_authsize(aead); 1071 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1072 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; 1073 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; 1074 1075 if (is_ipsec_esp) 1076 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], 1077 DMA_FROM_DEVICE); 1078 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); 1079 1080 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, 1081 cryptlen + authsize, areq->assoclen); 1082 1083 if (edesc->dma_len) 1084 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1085 DMA_BIDIRECTIONAL); 1086 1087 if (!is_ipsec_esp) { 1088 unsigned int dst_nents = edesc->dst_nents ? : 1; 1089 1090 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, 1091 areq->assoclen + cryptlen - ivsize); 1092 } 1093 } 1094 1095 /* 1096 * ipsec_esp descriptor callbacks 1097 */ 1098 static void ipsec_esp_encrypt_done(struct device *dev, 1099 struct talitos_desc *desc, void *context, 1100 int err) 1101 { 1102 struct aead_request *areq = context; 1103 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1104 unsigned int ivsize = crypto_aead_ivsize(authenc); 1105 struct talitos_edesc *edesc; 1106 1107 edesc = container_of(desc, struct talitos_edesc, desc); 1108 1109 ipsec_esp_unmap(dev, edesc, areq, true); 1110 1111 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); 1112 1113 kfree(edesc); 1114 1115 aead_request_complete(areq, err); 1116 } 1117 1118 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 1119 struct talitos_desc *desc, 1120 void *context, int err) 1121 { 1122 struct aead_request *req = context; 1123 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1124 unsigned int authsize = crypto_aead_authsize(authenc); 1125 struct talitos_edesc *edesc; 1126 char *oicv, *icv; 1127 1128 edesc = container_of(desc, struct talitos_edesc, desc); 1129 1130 ipsec_esp_unmap(dev, edesc, req, false); 1131 1132 if (!err) { 1133 /* auth check */ 1134 oicv = edesc->buf + edesc->dma_len; 1135 icv = oicv - authsize; 1136 1137 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 1138 } 1139 1140 kfree(edesc); 1141 1142 aead_request_complete(req, err); 1143 } 1144 1145 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 1146 struct talitos_desc *desc, 1147 void *context, int err) 1148 { 1149 struct aead_request *req = context; 1150 struct talitos_edesc *edesc; 1151 1152 edesc = container_of(desc, struct talitos_edesc, desc); 1153 1154 ipsec_esp_unmap(dev, edesc, req, false); 1155 1156 /* check ICV auth status */ 1157 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1158 DESC_HDR_LO_ICCR1_PASS)) 1159 err = -EBADMSG; 1160 1161 kfree(edesc); 1162 1163 aead_request_complete(req, err); 1164 } 1165 1166 /* 1167 * convert scatterlist to SEC h/w link table format 1168 * stop at cryptlen bytes 1169 */ 1170 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1171 unsigned int offset, int datalen, int elen, 1172 struct talitos_ptr *link_tbl_ptr, int align) 1173 { 1174 int n_sg = elen ? sg_count + 1 : sg_count; 1175 int count = 0; 1176 int cryptlen = datalen + elen; 1177 int padding = ALIGN(cryptlen, align) - cryptlen; 1178 1179 while (cryptlen && sg && n_sg--) { 1180 unsigned int len = sg_dma_len(sg); 1181 1182 if (offset >= len) { 1183 offset -= len; 1184 goto next; 1185 } 1186 1187 len -= offset; 1188 1189 if (len > cryptlen) 1190 len = cryptlen; 1191 1192 if (datalen > 0 && len > datalen) { 1193 to_talitos_ptr(link_tbl_ptr + count, 1194 sg_dma_address(sg) + offset, datalen, 0); 1195 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1196 count++; 1197 len -= datalen; 1198 offset += datalen; 1199 } 1200 to_talitos_ptr(link_tbl_ptr + count, 1201 sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); 1202 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1203 count++; 1204 cryptlen -= len; 1205 datalen -= len; 1206 offset = 0; 1207 1208 next: 1209 sg = sg_next(sg); 1210 } 1211 1212 /* tag end of link table */ 1213 if (count > 0) 1214 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, 1215 DESC_PTR_LNKTBL_RET, 0); 1216 1217 return count; 1218 } 1219 1220 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, 1221 unsigned int len, struct talitos_edesc *edesc, 1222 struct talitos_ptr *ptr, int sg_count, 1223 unsigned int offset, int tbl_off, int elen, 1224 bool force, int align) 1225 { 1226 struct talitos_private *priv = dev_get_drvdata(dev); 1227 bool is_sec1 = has_ftr_sec1(priv); 1228 int aligned_len = ALIGN(len, align); 1229 1230 if (!src) { 1231 to_talitos_ptr(ptr, 0, 0, is_sec1); 1232 return 1; 1233 } 1234 to_talitos_ptr_ext_set(ptr, elen, is_sec1); 1235 if (sg_count == 1 && !force) { 1236 to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); 1237 return sg_count; 1238 } 1239 if (is_sec1) { 1240 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); 1241 return sg_count; 1242 } 1243 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, 1244 &edesc->link_tbl[tbl_off], align); 1245 if (sg_count == 1 && !force) { 1246 /* Only one segment now, so no link tbl needed*/ 1247 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); 1248 return sg_count; 1249 } 1250 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1251 tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); 1252 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); 1253 1254 return sg_count; 1255 } 1256 1257 static int talitos_sg_map(struct device *dev, struct scatterlist *src, 1258 unsigned int len, struct talitos_edesc *edesc, 1259 struct talitos_ptr *ptr, int sg_count, 1260 unsigned int offset, int tbl_off) 1261 { 1262 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, 1263 tbl_off, 0, false, 1); 1264 } 1265 1266 /* 1267 * fill in and submit ipsec_esp descriptor 1268 */ 1269 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1270 bool encrypt, 1271 void (*callback)(struct device *dev, 1272 struct talitos_desc *desc, 1273 void *context, int error)) 1274 { 1275 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1276 unsigned int authsize = crypto_aead_authsize(aead); 1277 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1278 struct device *dev = ctx->dev; 1279 struct talitos_desc *desc = &edesc->desc; 1280 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1281 unsigned int ivsize = crypto_aead_ivsize(aead); 1282 int tbl_off = 0; 1283 int sg_count, ret; 1284 int elen = 0; 1285 bool sync_needed = false; 1286 struct talitos_private *priv = dev_get_drvdata(dev); 1287 bool is_sec1 = has_ftr_sec1(priv); 1288 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; 1289 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; 1290 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; 1291 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; 1292 1293 /* hmac key */ 1294 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); 1295 1296 sg_count = edesc->src_nents ?: 1; 1297 if (is_sec1 && sg_count > 1) 1298 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1299 areq->assoclen + cryptlen); 1300 else 1301 sg_count = dma_map_sg(dev, areq->src, sg_count, 1302 (areq->src == areq->dst) ? 1303 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1304 1305 /* hmac data */ 1306 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, 1307 &desc->ptr[1], sg_count, 0, tbl_off); 1308 1309 if (ret > 1) { 1310 tbl_off += ret; 1311 sync_needed = true; 1312 } 1313 1314 /* cipher iv */ 1315 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); 1316 1317 /* cipher key */ 1318 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, 1319 ctx->enckeylen, is_sec1); 1320 1321 /* 1322 * cipher in 1323 * map and adjust cipher len to aead request cryptlen. 1324 * extent is bytes of HMAC postpended to ciphertext, 1325 * typically 12 for ipsec 1326 */ 1327 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) 1328 elen = authsize; 1329 1330 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], 1331 sg_count, areq->assoclen, tbl_off, elen, 1332 false, 1); 1333 1334 if (ret > 1) { 1335 tbl_off += ret; 1336 sync_needed = true; 1337 } 1338 1339 /* cipher out */ 1340 if (areq->src != areq->dst) { 1341 sg_count = edesc->dst_nents ? : 1; 1342 if (!is_sec1 || sg_count == 1) 1343 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1344 } 1345 1346 if (is_ipsec_esp && encrypt) 1347 elen = authsize; 1348 else 1349 elen = 0; 1350 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], 1351 sg_count, areq->assoclen, tbl_off, elen, 1352 is_ipsec_esp && !encrypt, 1); 1353 tbl_off += ret; 1354 1355 if (!encrypt && is_ipsec_esp) { 1356 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1357 1358 /* Add an entry to the link table for ICV data */ 1359 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); 1360 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); 1361 1362 /* icv data follows link tables */ 1363 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); 1364 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); 1365 sync_needed = true; 1366 } else if (!encrypt) { 1367 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); 1368 sync_needed = true; 1369 } else if (!is_ipsec_esp) { 1370 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], 1371 sg_count, areq->assoclen + cryptlen, tbl_off); 1372 } 1373 1374 /* iv out */ 1375 if (is_ipsec_esp) 1376 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1377 DMA_FROM_DEVICE); 1378 1379 if (sync_needed) 1380 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1381 edesc->dma_len, 1382 DMA_BIDIRECTIONAL); 1383 1384 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1385 if (ret != -EINPROGRESS) { 1386 ipsec_esp_unmap(dev, edesc, areq, encrypt); 1387 kfree(edesc); 1388 } 1389 return ret; 1390 } 1391 1392 /* 1393 * allocate and map the extended descriptor 1394 */ 1395 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1396 struct scatterlist *src, 1397 struct scatterlist *dst, 1398 u8 *iv, 1399 unsigned int assoclen, 1400 unsigned int cryptlen, 1401 unsigned int authsize, 1402 unsigned int ivsize, 1403 int icv_stashing, 1404 u32 cryptoflags, 1405 bool encrypt) 1406 { 1407 struct talitos_edesc *edesc; 1408 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; 1409 dma_addr_t iv_dma = 0; 1410 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1411 GFP_ATOMIC; 1412 struct talitos_private *priv = dev_get_drvdata(dev); 1413 bool is_sec1 = has_ftr_sec1(priv); 1414 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1415 1416 if (cryptlen + authsize > max_len) { 1417 dev_err(dev, "length exceeds h/w max limit\n"); 1418 return ERR_PTR(-EINVAL); 1419 } 1420 1421 if (!dst || dst == src) { 1422 src_len = assoclen + cryptlen + authsize; 1423 src_nents = sg_nents_for_len(src, src_len); 1424 if (src_nents < 0) { 1425 dev_err(dev, "Invalid number of src SG.\n"); 1426 return ERR_PTR(-EINVAL); 1427 } 1428 src_nents = (src_nents == 1) ? 0 : src_nents; 1429 dst_nents = dst ? src_nents : 0; 1430 dst_len = 0; 1431 } else { /* dst && dst != src*/ 1432 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); 1433 src_nents = sg_nents_for_len(src, src_len); 1434 if (src_nents < 0) { 1435 dev_err(dev, "Invalid number of src SG.\n"); 1436 return ERR_PTR(-EINVAL); 1437 } 1438 src_nents = (src_nents == 1) ? 0 : src_nents; 1439 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); 1440 dst_nents = sg_nents_for_len(dst, dst_len); 1441 if (dst_nents < 0) { 1442 dev_err(dev, "Invalid number of dst SG.\n"); 1443 return ERR_PTR(-EINVAL); 1444 } 1445 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1446 } 1447 1448 /* 1449 * allocate space for base edesc plus the link tables, 1450 * allowing for two separate entries for AD and generated ICV (+ 2), 1451 * and space for two sets of ICVs (stashed and generated) 1452 */ 1453 alloc_len = sizeof(struct talitos_edesc); 1454 if (src_nents || dst_nents || !encrypt) { 1455 if (is_sec1) 1456 dma_len = (src_nents ? src_len : 0) + 1457 (dst_nents ? dst_len : 0) + authsize; 1458 else 1459 dma_len = (src_nents + dst_nents + 2) * 1460 sizeof(struct talitos_ptr) + authsize; 1461 alloc_len += dma_len; 1462 } else { 1463 dma_len = 0; 1464 } 1465 alloc_len += icv_stashing ? authsize : 0; 1466 alloc_len += ivsize; 1467 1468 edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); 1469 if (!edesc) 1470 return ERR_PTR(-ENOMEM); 1471 if (ivsize) { 1472 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); 1473 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1474 } 1475 memset(&edesc->desc, 0, sizeof(edesc->desc)); 1476 1477 edesc->src_nents = src_nents; 1478 edesc->dst_nents = dst_nents; 1479 edesc->iv_dma = iv_dma; 1480 edesc->dma_len = dma_len; 1481 edesc->next_desc = NULL; 1482 if (dma_len) 1483 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1484 edesc->dma_len, 1485 DMA_BIDIRECTIONAL); 1486 1487 return edesc; 1488 } 1489 1490 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1491 int icv_stashing, bool encrypt) 1492 { 1493 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1494 unsigned int authsize = crypto_aead_authsize(authenc); 1495 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1496 unsigned int ivsize = crypto_aead_ivsize(authenc); 1497 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1498 1499 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1500 iv, areq->assoclen, cryptlen, 1501 authsize, ivsize, icv_stashing, 1502 areq->base.flags, encrypt); 1503 } 1504 1505 static int aead_encrypt(struct aead_request *req) 1506 { 1507 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1508 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1509 struct talitos_edesc *edesc; 1510 1511 /* allocate extended descriptor */ 1512 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1513 if (IS_ERR(edesc)) 1514 return PTR_ERR(edesc); 1515 1516 /* set encrypt */ 1517 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1518 1519 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); 1520 } 1521 1522 static int aead_decrypt(struct aead_request *req) 1523 { 1524 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1525 unsigned int authsize = crypto_aead_authsize(authenc); 1526 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1527 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1528 struct talitos_edesc *edesc; 1529 void *icvdata; 1530 1531 /* allocate extended descriptor */ 1532 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1533 if (IS_ERR(edesc)) 1534 return PTR_ERR(edesc); 1535 1536 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && 1537 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1538 ((!edesc->src_nents && !edesc->dst_nents) || 1539 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1540 1541 /* decrypt and check the ICV */ 1542 edesc->desc.hdr = ctx->desc_hdr_template | 1543 DESC_HDR_DIR_INBOUND | 1544 DESC_HDR_MODE1_MDEU_CICV; 1545 1546 /* reset integrity check result bits */ 1547 1548 return ipsec_esp(edesc, req, false, 1549 ipsec_esp_decrypt_hwauth_done); 1550 } 1551 1552 /* Have to check the ICV with software */ 1553 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1554 1555 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1556 icvdata = edesc->buf + edesc->dma_len; 1557 1558 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, 1559 req->assoclen + req->cryptlen - authsize); 1560 1561 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); 1562 } 1563 1564 static int skcipher_setkey(struct crypto_skcipher *cipher, 1565 const u8 *key, unsigned int keylen) 1566 { 1567 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1568 struct device *dev = ctx->dev; 1569 1570 if (ctx->keylen) 1571 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1572 1573 memcpy(&ctx->key, key, keylen); 1574 ctx->keylen = keylen; 1575 1576 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); 1577 1578 return 0; 1579 } 1580 1581 static int skcipher_des_setkey(struct crypto_skcipher *cipher, 1582 const u8 *key, unsigned int keylen) 1583 { 1584 return verify_skcipher_des_key(cipher, key) ?: 1585 skcipher_setkey(cipher, key, keylen); 1586 } 1587 1588 static int skcipher_des3_setkey(struct crypto_skcipher *cipher, 1589 const u8 *key, unsigned int keylen) 1590 { 1591 return verify_skcipher_des3_key(cipher, key) ?: 1592 skcipher_setkey(cipher, key, keylen); 1593 } 1594 1595 static int skcipher_aes_setkey(struct crypto_skcipher *cipher, 1596 const u8 *key, unsigned int keylen) 1597 { 1598 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || 1599 keylen == AES_KEYSIZE_256) 1600 return skcipher_setkey(cipher, key, keylen); 1601 1602 return -EINVAL; 1603 } 1604 1605 static void common_nonsnoop_unmap(struct device *dev, 1606 struct talitos_edesc *edesc, 1607 struct skcipher_request *areq) 1608 { 1609 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1610 1611 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); 1612 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1613 1614 if (edesc->dma_len) 1615 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1616 DMA_BIDIRECTIONAL); 1617 } 1618 1619 static void skcipher_done(struct device *dev, 1620 struct talitos_desc *desc, void *context, 1621 int err) 1622 { 1623 struct skcipher_request *areq = context; 1624 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1625 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1626 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1627 struct talitos_edesc *edesc; 1628 1629 edesc = container_of(desc, struct talitos_edesc, desc); 1630 1631 common_nonsnoop_unmap(dev, edesc, areq); 1632 memcpy(areq->iv, ctx->iv, ivsize); 1633 1634 kfree(edesc); 1635 1636 skcipher_request_complete(areq, err); 1637 } 1638 1639 static int common_nonsnoop(struct talitos_edesc *edesc, 1640 struct skcipher_request *areq, 1641 void (*callback) (struct device *dev, 1642 struct talitos_desc *desc, 1643 void *context, int error)) 1644 { 1645 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1646 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1647 struct device *dev = ctx->dev; 1648 struct talitos_desc *desc = &edesc->desc; 1649 unsigned int cryptlen = areq->cryptlen; 1650 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1651 int sg_count, ret; 1652 bool sync_needed = false; 1653 struct talitos_private *priv = dev_get_drvdata(dev); 1654 bool is_sec1 = has_ftr_sec1(priv); 1655 bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && 1656 (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; 1657 1658 /* first DWORD empty */ 1659 1660 /* cipher iv */ 1661 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); 1662 1663 /* cipher key */ 1664 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); 1665 1666 sg_count = edesc->src_nents ?: 1; 1667 if (is_sec1 && sg_count > 1) 1668 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1669 cryptlen); 1670 else 1671 sg_count = dma_map_sg(dev, areq->src, sg_count, 1672 (areq->src == areq->dst) ? 1673 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1674 /* 1675 * cipher in 1676 */ 1677 sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], 1678 sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); 1679 if (sg_count > 1) 1680 sync_needed = true; 1681 1682 /* cipher out */ 1683 if (areq->src != areq->dst) { 1684 sg_count = edesc->dst_nents ? : 1; 1685 if (!is_sec1 || sg_count == 1) 1686 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1687 } 1688 1689 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], 1690 sg_count, 0, (edesc->src_nents + 1)); 1691 if (ret > 1) 1692 sync_needed = true; 1693 1694 /* iv out */ 1695 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1696 DMA_FROM_DEVICE); 1697 1698 /* last DWORD empty */ 1699 1700 if (sync_needed) 1701 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1702 edesc->dma_len, DMA_BIDIRECTIONAL); 1703 1704 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1705 if (ret != -EINPROGRESS) { 1706 common_nonsnoop_unmap(dev, edesc, areq); 1707 kfree(edesc); 1708 } 1709 return ret; 1710 } 1711 1712 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * 1713 areq, bool encrypt) 1714 { 1715 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1716 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1717 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1718 1719 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1720 areq->iv, 0, areq->cryptlen, 0, ivsize, 0, 1721 areq->base.flags, encrypt); 1722 } 1723 1724 static int skcipher_encrypt(struct skcipher_request *areq) 1725 { 1726 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1727 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1728 struct talitos_edesc *edesc; 1729 unsigned int blocksize = 1730 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1731 1732 if (!areq->cryptlen) 1733 return 0; 1734 1735 if (areq->cryptlen % blocksize) 1736 return -EINVAL; 1737 1738 /* allocate extended descriptor */ 1739 edesc = skcipher_edesc_alloc(areq, true); 1740 if (IS_ERR(edesc)) 1741 return PTR_ERR(edesc); 1742 1743 /* set encrypt */ 1744 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1745 1746 return common_nonsnoop(edesc, areq, skcipher_done); 1747 } 1748 1749 static int skcipher_decrypt(struct skcipher_request *areq) 1750 { 1751 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1752 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1753 struct talitos_edesc *edesc; 1754 unsigned int blocksize = 1755 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1756 1757 if (!areq->cryptlen) 1758 return 0; 1759 1760 if (areq->cryptlen % blocksize) 1761 return -EINVAL; 1762 1763 /* allocate extended descriptor */ 1764 edesc = skcipher_edesc_alloc(areq, false); 1765 if (IS_ERR(edesc)) 1766 return PTR_ERR(edesc); 1767 1768 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1769 1770 return common_nonsnoop(edesc, areq, skcipher_done); 1771 } 1772 1773 static void common_nonsnoop_hash_unmap(struct device *dev, 1774 struct talitos_edesc *edesc, 1775 struct ahash_request *areq) 1776 { 1777 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1778 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1779 struct talitos_private *priv = dev_get_drvdata(dev); 1780 bool is_sec1 = has_ftr_sec1(priv); 1781 struct talitos_desc *desc = &edesc->desc; 1782 1783 unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); 1784 1785 if (edesc->last && req_ctx->last_request) 1786 memcpy(areq->result, req_ctx->hw_context, 1787 crypto_ahash_digestsize(tfm)); 1788 1789 if (edesc->src) 1790 talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); 1791 1792 /* When using hashctx-in, must unmap it. */ 1793 if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) 1794 unmap_single_talitos_ptr(dev, &desc->ptr[1], 1795 DMA_TO_DEVICE); 1796 1797 if (edesc->dma_len) 1798 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1799 DMA_BIDIRECTIONAL); 1800 } 1801 1802 static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) 1803 { 1804 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1805 struct talitos_edesc *next; 1806 1807 while (edesc) { 1808 next = edesc->next_desc; 1809 common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); 1810 kfree(edesc); 1811 edesc = next; 1812 } 1813 } 1814 1815 static void ahash_done(struct device *dev, 1816 struct talitos_desc *desc, void *context, 1817 int err) 1818 { 1819 struct ahash_request *areq = context; 1820 struct talitos_edesc *edesc = 1821 container_of(desc, struct talitos_edesc, desc); 1822 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1823 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1824 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); 1825 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1826 struct talitos_edesc *next; 1827 1828 if (is_sec1) { 1829 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1830 /* Position any partial block for next update/final/finup */ 1831 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1832 req_ctx->nbuf = req_ctx->to_hash_later; 1833 } 1834 1835 free_edesc_list_from(areq, edesc); 1836 ahash_request_complete(areq, err); 1837 } else { 1838 next = edesc->next_desc; 1839 1840 common_nonsnoop_hash_unmap(dev, edesc, areq); 1841 kfree(edesc); 1842 1843 if (err) 1844 goto out; 1845 1846 if (next) { 1847 err = talitos_submit(dev, ctx->ch, &next->desc, 1848 ahash_done, areq); 1849 if (err != -EINPROGRESS) 1850 goto out; 1851 return; 1852 } 1853 out: 1854 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1855 /* Position any partial block for next update/final/finup */ 1856 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1857 req_ctx->nbuf = req_ctx->to_hash_later; 1858 } 1859 if (err && next) 1860 free_edesc_list_from(areq, next); 1861 ahash_request_complete(areq, err); 1862 } 1863 } 1864 1865 /* 1866 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1867 * ourself and submit a padded block 1868 */ 1869 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1870 struct talitos_edesc *edesc, 1871 struct talitos_ptr *ptr) 1872 { 1873 static u8 padded_hash[64] = { 1874 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878 }; 1879 1880 pr_err_once("Bug in SEC1, padding ourself\n"); 1881 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1882 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1883 (char *)padded_hash, DMA_TO_DEVICE); 1884 } 1885 1886 static void common_nonsnoop_hash(struct talitos_edesc *edesc, 1887 struct ahash_request *areq, 1888 unsigned int length) 1889 { 1890 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1891 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1892 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1893 struct device *dev = ctx->dev; 1894 struct talitos_desc *desc = &edesc->desc; 1895 bool sync_needed = false; 1896 struct talitos_private *priv = dev_get_drvdata(dev); 1897 bool is_sec1 = has_ftr_sec1(priv); 1898 int sg_count; 1899 1900 /* first DWORD empty */ 1901 1902 /* hash context in */ 1903 if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { 1904 map_single_talitos_ptr_nosync(dev, &desc->ptr[1], 1905 req_ctx->hw_context_size, 1906 req_ctx->hw_context, 1907 DMA_TO_DEVICE); 1908 req_ctx->swinit = 0; 1909 } 1910 /* Indicate next op is not the first. */ 1911 req_ctx->first_request = 0; 1912 1913 /* HMAC key */ 1914 if (ctx->keylen) 1915 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, 1916 is_sec1); 1917 1918 sg_count = edesc->src_nents ?: 1; 1919 if (is_sec1 && sg_count > 1) 1920 sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); 1921 else if (length) 1922 sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); 1923 1924 /* 1925 * data in 1926 */ 1927 sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], 1928 sg_count, 0, 0); 1929 if (sg_count > 1) 1930 sync_needed = true; 1931 1932 /* fifth DWORD empty */ 1933 1934 /* hash/HMAC out -or- hash context out */ 1935 if (edesc->last && req_ctx->last_request) 1936 map_single_talitos_ptr(dev, &desc->ptr[5], 1937 crypto_ahash_digestsize(tfm), 1938 req_ctx->hw_context, DMA_FROM_DEVICE); 1939 else 1940 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1941 req_ctx->hw_context_size, 1942 req_ctx->hw_context, 1943 DMA_FROM_DEVICE); 1944 1945 /* last DWORD empty */ 1946 1947 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1948 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1949 1950 if (sync_needed) 1951 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1952 edesc->dma_len, DMA_BIDIRECTIONAL); 1953 } 1954 1955 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1956 struct scatterlist *src, 1957 unsigned int nbytes) 1958 { 1959 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1960 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1961 1962 return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, 1963 nbytes, 0, 0, 0, areq->base.flags, false); 1964 } 1965 1966 static struct talitos_edesc * 1967 ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, 1968 unsigned int blocksize, bool is_sec1) 1969 { 1970 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1971 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1972 struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; 1973 size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : 1974 TALITOS2_MAX_DATA_LEN; 1975 struct scatterlist tmp[2]; 1976 size_t to_hash_this_desc; 1977 struct scatterlist *src; 1978 size_t offset = 0; 1979 1980 do { 1981 src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); 1982 1983 to_hash_this_desc = 1984 min(nbytes, ALIGN_DOWN(desc_max, blocksize)); 1985 1986 /* Allocate extended descriptor */ 1987 edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); 1988 if (IS_ERR(edesc)) { 1989 if (first) 1990 free_edesc_list_from(areq, first); 1991 return edesc; 1992 } 1993 1994 edesc->src = 1995 scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); 1996 edesc->desc.hdr = ctx->desc_hdr_template; 1997 edesc->first = offset == 0; 1998 edesc->last = nbytes - to_hash_this_desc == 0; 1999 2000 /* On last one, request SEC to pad; otherwise continue */ 2001 if (req_ctx->last_request && edesc->last) 2002 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 2003 else 2004 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 2005 2006 /* request SEC to INIT hash. */ 2007 if (req_ctx->first_request && edesc->first && !req_ctx->swinit) 2008 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 2009 2010 /* 2011 * When the tfm context has a keylen, it's an HMAC. 2012 * A first or last (ie. not middle) descriptor must request HMAC. 2013 */ 2014 if (ctx->keylen && ((req_ctx->first_request && edesc->first) || 2015 (req_ctx->last_request && edesc->last))) 2016 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 2017 2018 /* clear the DN bit */ 2019 if (is_sec1 && !edesc->last) 2020 edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; 2021 2022 common_nonsnoop_hash(edesc, areq, to_hash_this_desc); 2023 2024 offset += to_hash_this_desc; 2025 nbytes -= to_hash_this_desc; 2026 2027 if (!prev_edesc) 2028 first = edesc; 2029 else 2030 prev_edesc->next_desc = edesc; 2031 prev_edesc = edesc; 2032 } while (nbytes); 2033 2034 return first; 2035 } 2036 2037 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 2038 { 2039 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2040 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2041 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2042 struct talitos_edesc *edesc; 2043 unsigned int blocksize = 2044 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2045 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); 2046 unsigned int nbytes_to_hash; 2047 unsigned int to_hash_later; 2048 unsigned int nsg; 2049 int nents; 2050 struct device *dev = ctx->dev; 2051 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; 2052 int ret; 2053 2054 if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { 2055 /* Buffer up to one whole block */ 2056 nents = sg_nents_for_len(areq->src, nbytes); 2057 if (nents < 0) { 2058 dev_err(dev, "Invalid number of src SG.\n"); 2059 return nents; 2060 } 2061 sg_copy_to_buffer(areq->src, nents, 2062 ctx_buf + req_ctx->nbuf, nbytes); 2063 req_ctx->nbuf += nbytes; 2064 return 0; 2065 } 2066 2067 /* At least (blocksize + 1) bytes are available to hash */ 2068 nbytes_to_hash = nbytes + req_ctx->nbuf; 2069 to_hash_later = nbytes_to_hash & (blocksize - 1); 2070 2071 if (req_ctx->last_request) 2072 to_hash_later = 0; 2073 else if (to_hash_later) 2074 /* There is a partial block. Hash the full block(s) now */ 2075 nbytes_to_hash -= to_hash_later; 2076 else { 2077 /* Keep one block buffered */ 2078 nbytes_to_hash -= blocksize; 2079 to_hash_later = blocksize; 2080 } 2081 2082 /* Chain in any previously buffered data */ 2083 if (req_ctx->nbuf) { 2084 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 2085 sg_init_table(req_ctx->bufsl, nsg); 2086 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); 2087 if (nsg > 1) 2088 sg_chain(req_ctx->bufsl, 2, areq->src); 2089 req_ctx->psrc = req_ctx->bufsl; 2090 } else 2091 req_ctx->psrc = areq->src; 2092 2093 if (to_hash_later) { 2094 nents = sg_nents_for_len(areq->src, nbytes); 2095 if (nents < 0) { 2096 dev_err(dev, "Invalid number of src SG.\n"); 2097 return nents; 2098 } 2099 sg_pcopy_to_buffer(areq->src, nents, 2100 req_ctx->buf[(req_ctx->buf_idx + 1) & 1], 2101 to_hash_later, 2102 nbytes - to_hash_later); 2103 } 2104 req_ctx->to_hash_later = to_hash_later; 2105 2106 edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, 2107 is_sec1); 2108 if (IS_ERR(edesc)) 2109 return PTR_ERR(edesc); 2110 2111 ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); 2112 if (ret != -EINPROGRESS) 2113 free_edesc_list_from(areq, edesc); 2114 2115 return ret; 2116 } 2117 2118 static int ahash_init(struct ahash_request *areq) 2119 { 2120 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2121 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2122 struct device *dev = ctx->dev; 2123 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2124 unsigned int size; 2125 dma_addr_t dma; 2126 2127 /* Initialize the context */ 2128 req_ctx->buf_idx = 0; 2129 req_ctx->nbuf = 0; 2130 req_ctx->first_request = 1; 2131 req_ctx->swinit = 0; /* assume h/w init of context */ 2132 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2133 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2134 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2135 req_ctx->hw_context_size = size; 2136 req_ctx->last_request = 0; 2137 2138 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2139 DMA_TO_DEVICE); 2140 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2141 2142 return 0; 2143 } 2144 2145 /* 2146 * on h/w without explicit sha224 support, we initialize h/w context 2147 * manually with sha224 constants, and tell it to run sha256. 2148 */ 2149 static int ahash_init_sha224_swinit(struct ahash_request *areq) 2150 { 2151 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2152 2153 req_ctx->hw_context[0] = SHA224_H0; 2154 req_ctx->hw_context[1] = SHA224_H1; 2155 req_ctx->hw_context[2] = SHA224_H2; 2156 req_ctx->hw_context[3] = SHA224_H3; 2157 req_ctx->hw_context[4] = SHA224_H4; 2158 req_ctx->hw_context[5] = SHA224_H5; 2159 req_ctx->hw_context[6] = SHA224_H6; 2160 req_ctx->hw_context[7] = SHA224_H7; 2161 2162 /* init 64-bit count */ 2163 req_ctx->hw_context[8] = 0; 2164 req_ctx->hw_context[9] = 0; 2165 2166 ahash_init(areq); 2167 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 2168 2169 return 0; 2170 } 2171 2172 static int ahash_update(struct ahash_request *areq) 2173 { 2174 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2175 2176 req_ctx->last_request = 0; 2177 2178 return ahash_process_req(areq, areq->nbytes); 2179 } 2180 2181 static int ahash_final(struct ahash_request *areq) 2182 { 2183 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2184 2185 req_ctx->last_request = 1; 2186 2187 return ahash_process_req(areq, 0); 2188 } 2189 2190 static int ahash_finup(struct ahash_request *areq) 2191 { 2192 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2193 2194 req_ctx->last_request = 1; 2195 2196 return ahash_process_req(areq, areq->nbytes); 2197 } 2198 2199 static int ahash_digest(struct ahash_request *areq) 2200 { 2201 ahash_init(areq); 2202 return ahash_finup(areq); 2203 } 2204 2205 static int ahash_digest_sha224_swinit(struct ahash_request *areq) 2206 { 2207 ahash_init_sha224_swinit(areq); 2208 return ahash_finup(areq); 2209 } 2210 2211 static int ahash_export(struct ahash_request *areq, void *out) 2212 { 2213 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2214 struct talitos_export_state *export = out; 2215 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2216 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2217 struct device *dev = ctx->dev; 2218 dma_addr_t dma; 2219 2220 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2221 DMA_FROM_DEVICE); 2222 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); 2223 2224 memcpy(export->hw_context, req_ctx->hw_context, 2225 req_ctx->hw_context_size); 2226 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); 2227 export->swinit = req_ctx->swinit; 2228 export->first_request = req_ctx->first_request; 2229 export->last_request = req_ctx->last_request; 2230 export->to_hash_later = req_ctx->to_hash_later; 2231 export->nbuf = req_ctx->nbuf; 2232 2233 return 0; 2234 } 2235 2236 static int ahash_import(struct ahash_request *areq, const void *in) 2237 { 2238 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2239 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2240 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2241 struct device *dev = ctx->dev; 2242 const struct talitos_export_state *export = in; 2243 unsigned int size; 2244 dma_addr_t dma; 2245 2246 memset(req_ctx, 0, sizeof(*req_ctx)); 2247 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2248 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2249 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2250 req_ctx->hw_context_size = size; 2251 memcpy(req_ctx->hw_context, export->hw_context, size); 2252 memcpy(req_ctx->buf[0], export->buf, export->nbuf); 2253 req_ctx->swinit = export->swinit; 2254 req_ctx->first_request = export->first_request; 2255 req_ctx->last_request = export->last_request; 2256 req_ctx->to_hash_later = export->to_hash_later; 2257 req_ctx->nbuf = export->nbuf; 2258 2259 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2260 DMA_TO_DEVICE); 2261 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2262 2263 return 0; 2264 } 2265 2266 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2267 u8 *hash) 2268 { 2269 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2270 2271 struct scatterlist sg[1]; 2272 struct ahash_request *req; 2273 struct crypto_wait wait; 2274 int ret; 2275 2276 crypto_init_wait(&wait); 2277 2278 req = ahash_request_alloc(tfm, GFP_KERNEL); 2279 if (!req) 2280 return -ENOMEM; 2281 2282 /* Keep tfm keylen == 0 during hash of the long key */ 2283 ctx->keylen = 0; 2284 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2285 crypto_req_done, &wait); 2286 2287 sg_init_one(&sg[0], key, keylen); 2288 2289 ahash_request_set_crypt(req, sg, hash, keylen); 2290 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 2291 2292 ahash_request_free(req); 2293 2294 return ret; 2295 } 2296 2297 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2298 unsigned int keylen) 2299 { 2300 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2301 struct device *dev = ctx->dev; 2302 unsigned int blocksize = 2303 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2304 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2305 unsigned int keysize = keylen; 2306 u8 hash[SHA512_DIGEST_SIZE]; 2307 int ret; 2308 2309 if (keylen <= blocksize) 2310 memcpy(ctx->key, key, keysize); 2311 else { 2312 /* Must get the hash of the long key */ 2313 ret = keyhash(tfm, key, keylen, hash); 2314 2315 if (ret) 2316 return -EINVAL; 2317 2318 keysize = digestsize; 2319 memcpy(ctx->key, hash, digestsize); 2320 } 2321 2322 if (ctx->keylen) 2323 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 2324 2325 ctx->keylen = keysize; 2326 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); 2327 2328 return 0; 2329 } 2330 2331 2332 struct talitos_alg_template { 2333 u32 type; 2334 u32 priority; 2335 union { 2336 struct skcipher_alg skcipher; 2337 struct ahash_alg hash; 2338 struct aead_alg aead; 2339 } alg; 2340 __be32 desc_hdr_template; 2341 }; 2342 2343 static struct talitos_alg_template driver_algs[] = { 2344 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2345 { .type = CRYPTO_ALG_TYPE_AEAD, 2346 .alg.aead = { 2347 .base = { 2348 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2349 .cra_driver_name = "authenc-hmac-sha1-" 2350 "cbc-aes-talitos", 2351 .cra_blocksize = AES_BLOCK_SIZE, 2352 .cra_flags = CRYPTO_ALG_ASYNC | 2353 CRYPTO_ALG_ALLOCATES_MEMORY, 2354 }, 2355 .ivsize = AES_BLOCK_SIZE, 2356 .maxauthsize = SHA1_DIGEST_SIZE, 2357 }, 2358 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2359 DESC_HDR_SEL0_AESU | 2360 DESC_HDR_MODE0_AESU_CBC | 2361 DESC_HDR_SEL1_MDEUA | 2362 DESC_HDR_MODE1_MDEU_INIT | 2363 DESC_HDR_MODE1_MDEU_PAD | 2364 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2365 }, 2366 { .type = CRYPTO_ALG_TYPE_AEAD, 2367 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2368 .alg.aead = { 2369 .base = { 2370 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2371 .cra_driver_name = "authenc-hmac-sha1-" 2372 "cbc-aes-talitos-hsna", 2373 .cra_blocksize = AES_BLOCK_SIZE, 2374 .cra_flags = CRYPTO_ALG_ASYNC | 2375 CRYPTO_ALG_ALLOCATES_MEMORY, 2376 }, 2377 .ivsize = AES_BLOCK_SIZE, 2378 .maxauthsize = SHA1_DIGEST_SIZE, 2379 }, 2380 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2381 DESC_HDR_SEL0_AESU | 2382 DESC_HDR_MODE0_AESU_CBC | 2383 DESC_HDR_SEL1_MDEUA | 2384 DESC_HDR_MODE1_MDEU_INIT | 2385 DESC_HDR_MODE1_MDEU_PAD | 2386 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2387 }, 2388 { .type = CRYPTO_ALG_TYPE_AEAD, 2389 .alg.aead = { 2390 .base = { 2391 .cra_name = "authenc(hmac(sha1)," 2392 "cbc(des3_ede))", 2393 .cra_driver_name = "authenc-hmac-sha1-" 2394 "cbc-3des-talitos", 2395 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2396 .cra_flags = CRYPTO_ALG_ASYNC | 2397 CRYPTO_ALG_ALLOCATES_MEMORY, 2398 }, 2399 .ivsize = DES3_EDE_BLOCK_SIZE, 2400 .maxauthsize = SHA1_DIGEST_SIZE, 2401 .setkey = aead_des3_setkey, 2402 }, 2403 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2404 DESC_HDR_SEL0_DEU | 2405 DESC_HDR_MODE0_DEU_CBC | 2406 DESC_HDR_MODE0_DEU_3DES | 2407 DESC_HDR_SEL1_MDEUA | 2408 DESC_HDR_MODE1_MDEU_INIT | 2409 DESC_HDR_MODE1_MDEU_PAD | 2410 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2411 }, 2412 { .type = CRYPTO_ALG_TYPE_AEAD, 2413 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2414 .alg.aead = { 2415 .base = { 2416 .cra_name = "authenc(hmac(sha1)," 2417 "cbc(des3_ede))", 2418 .cra_driver_name = "authenc-hmac-sha1-" 2419 "cbc-3des-talitos-hsna", 2420 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2421 .cra_flags = CRYPTO_ALG_ASYNC | 2422 CRYPTO_ALG_ALLOCATES_MEMORY, 2423 }, 2424 .ivsize = DES3_EDE_BLOCK_SIZE, 2425 .maxauthsize = SHA1_DIGEST_SIZE, 2426 .setkey = aead_des3_setkey, 2427 }, 2428 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2429 DESC_HDR_SEL0_DEU | 2430 DESC_HDR_MODE0_DEU_CBC | 2431 DESC_HDR_MODE0_DEU_3DES | 2432 DESC_HDR_SEL1_MDEUA | 2433 DESC_HDR_MODE1_MDEU_INIT | 2434 DESC_HDR_MODE1_MDEU_PAD | 2435 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2436 }, 2437 { .type = CRYPTO_ALG_TYPE_AEAD, 2438 .alg.aead = { 2439 .base = { 2440 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2441 .cra_driver_name = "authenc-hmac-sha224-" 2442 "cbc-aes-talitos", 2443 .cra_blocksize = AES_BLOCK_SIZE, 2444 .cra_flags = CRYPTO_ALG_ASYNC | 2445 CRYPTO_ALG_ALLOCATES_MEMORY, 2446 }, 2447 .ivsize = AES_BLOCK_SIZE, 2448 .maxauthsize = SHA224_DIGEST_SIZE, 2449 }, 2450 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2451 DESC_HDR_SEL0_AESU | 2452 DESC_HDR_MODE0_AESU_CBC | 2453 DESC_HDR_SEL1_MDEUA | 2454 DESC_HDR_MODE1_MDEU_INIT | 2455 DESC_HDR_MODE1_MDEU_PAD | 2456 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2457 }, 2458 { .type = CRYPTO_ALG_TYPE_AEAD, 2459 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2460 .alg.aead = { 2461 .base = { 2462 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2463 .cra_driver_name = "authenc-hmac-sha224-" 2464 "cbc-aes-talitos-hsna", 2465 .cra_blocksize = AES_BLOCK_SIZE, 2466 .cra_flags = CRYPTO_ALG_ASYNC | 2467 CRYPTO_ALG_ALLOCATES_MEMORY, 2468 }, 2469 .ivsize = AES_BLOCK_SIZE, 2470 .maxauthsize = SHA224_DIGEST_SIZE, 2471 }, 2472 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2473 DESC_HDR_SEL0_AESU | 2474 DESC_HDR_MODE0_AESU_CBC | 2475 DESC_HDR_SEL1_MDEUA | 2476 DESC_HDR_MODE1_MDEU_INIT | 2477 DESC_HDR_MODE1_MDEU_PAD | 2478 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2479 }, 2480 { .type = CRYPTO_ALG_TYPE_AEAD, 2481 .alg.aead = { 2482 .base = { 2483 .cra_name = "authenc(hmac(sha224)," 2484 "cbc(des3_ede))", 2485 .cra_driver_name = "authenc-hmac-sha224-" 2486 "cbc-3des-talitos", 2487 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2488 .cra_flags = CRYPTO_ALG_ASYNC | 2489 CRYPTO_ALG_ALLOCATES_MEMORY, 2490 }, 2491 .ivsize = DES3_EDE_BLOCK_SIZE, 2492 .maxauthsize = SHA224_DIGEST_SIZE, 2493 .setkey = aead_des3_setkey, 2494 }, 2495 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2496 DESC_HDR_SEL0_DEU | 2497 DESC_HDR_MODE0_DEU_CBC | 2498 DESC_HDR_MODE0_DEU_3DES | 2499 DESC_HDR_SEL1_MDEUA | 2500 DESC_HDR_MODE1_MDEU_INIT | 2501 DESC_HDR_MODE1_MDEU_PAD | 2502 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2503 }, 2504 { .type = CRYPTO_ALG_TYPE_AEAD, 2505 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2506 .alg.aead = { 2507 .base = { 2508 .cra_name = "authenc(hmac(sha224)," 2509 "cbc(des3_ede))", 2510 .cra_driver_name = "authenc-hmac-sha224-" 2511 "cbc-3des-talitos-hsna", 2512 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2513 .cra_flags = CRYPTO_ALG_ASYNC | 2514 CRYPTO_ALG_ALLOCATES_MEMORY, 2515 }, 2516 .ivsize = DES3_EDE_BLOCK_SIZE, 2517 .maxauthsize = SHA224_DIGEST_SIZE, 2518 .setkey = aead_des3_setkey, 2519 }, 2520 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2521 DESC_HDR_SEL0_DEU | 2522 DESC_HDR_MODE0_DEU_CBC | 2523 DESC_HDR_MODE0_DEU_3DES | 2524 DESC_HDR_SEL1_MDEUA | 2525 DESC_HDR_MODE1_MDEU_INIT | 2526 DESC_HDR_MODE1_MDEU_PAD | 2527 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2528 }, 2529 { .type = CRYPTO_ALG_TYPE_AEAD, 2530 .alg.aead = { 2531 .base = { 2532 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2533 .cra_driver_name = "authenc-hmac-sha256-" 2534 "cbc-aes-talitos", 2535 .cra_blocksize = AES_BLOCK_SIZE, 2536 .cra_flags = CRYPTO_ALG_ASYNC | 2537 CRYPTO_ALG_ALLOCATES_MEMORY, 2538 }, 2539 .ivsize = AES_BLOCK_SIZE, 2540 .maxauthsize = SHA256_DIGEST_SIZE, 2541 }, 2542 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2543 DESC_HDR_SEL0_AESU | 2544 DESC_HDR_MODE0_AESU_CBC | 2545 DESC_HDR_SEL1_MDEUA | 2546 DESC_HDR_MODE1_MDEU_INIT | 2547 DESC_HDR_MODE1_MDEU_PAD | 2548 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2549 }, 2550 { .type = CRYPTO_ALG_TYPE_AEAD, 2551 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2552 .alg.aead = { 2553 .base = { 2554 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2555 .cra_driver_name = "authenc-hmac-sha256-" 2556 "cbc-aes-talitos-hsna", 2557 .cra_blocksize = AES_BLOCK_SIZE, 2558 .cra_flags = CRYPTO_ALG_ASYNC | 2559 CRYPTO_ALG_ALLOCATES_MEMORY, 2560 }, 2561 .ivsize = AES_BLOCK_SIZE, 2562 .maxauthsize = SHA256_DIGEST_SIZE, 2563 }, 2564 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2565 DESC_HDR_SEL0_AESU | 2566 DESC_HDR_MODE0_AESU_CBC | 2567 DESC_HDR_SEL1_MDEUA | 2568 DESC_HDR_MODE1_MDEU_INIT | 2569 DESC_HDR_MODE1_MDEU_PAD | 2570 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2571 }, 2572 { .type = CRYPTO_ALG_TYPE_AEAD, 2573 .alg.aead = { 2574 .base = { 2575 .cra_name = "authenc(hmac(sha256)," 2576 "cbc(des3_ede))", 2577 .cra_driver_name = "authenc-hmac-sha256-" 2578 "cbc-3des-talitos", 2579 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2580 .cra_flags = CRYPTO_ALG_ASYNC | 2581 CRYPTO_ALG_ALLOCATES_MEMORY, 2582 }, 2583 .ivsize = DES3_EDE_BLOCK_SIZE, 2584 .maxauthsize = SHA256_DIGEST_SIZE, 2585 .setkey = aead_des3_setkey, 2586 }, 2587 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2588 DESC_HDR_SEL0_DEU | 2589 DESC_HDR_MODE0_DEU_CBC | 2590 DESC_HDR_MODE0_DEU_3DES | 2591 DESC_HDR_SEL1_MDEUA | 2592 DESC_HDR_MODE1_MDEU_INIT | 2593 DESC_HDR_MODE1_MDEU_PAD | 2594 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2595 }, 2596 { .type = CRYPTO_ALG_TYPE_AEAD, 2597 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2598 .alg.aead = { 2599 .base = { 2600 .cra_name = "authenc(hmac(sha256)," 2601 "cbc(des3_ede))", 2602 .cra_driver_name = "authenc-hmac-sha256-" 2603 "cbc-3des-talitos-hsna", 2604 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2605 .cra_flags = CRYPTO_ALG_ASYNC | 2606 CRYPTO_ALG_ALLOCATES_MEMORY, 2607 }, 2608 .ivsize = DES3_EDE_BLOCK_SIZE, 2609 .maxauthsize = SHA256_DIGEST_SIZE, 2610 .setkey = aead_des3_setkey, 2611 }, 2612 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2613 DESC_HDR_SEL0_DEU | 2614 DESC_HDR_MODE0_DEU_CBC | 2615 DESC_HDR_MODE0_DEU_3DES | 2616 DESC_HDR_SEL1_MDEUA | 2617 DESC_HDR_MODE1_MDEU_INIT | 2618 DESC_HDR_MODE1_MDEU_PAD | 2619 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2620 }, 2621 { .type = CRYPTO_ALG_TYPE_AEAD, 2622 .alg.aead = { 2623 .base = { 2624 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2625 .cra_driver_name = "authenc-hmac-sha384-" 2626 "cbc-aes-talitos", 2627 .cra_blocksize = AES_BLOCK_SIZE, 2628 .cra_flags = CRYPTO_ALG_ASYNC | 2629 CRYPTO_ALG_ALLOCATES_MEMORY, 2630 }, 2631 .ivsize = AES_BLOCK_SIZE, 2632 .maxauthsize = SHA384_DIGEST_SIZE, 2633 }, 2634 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2635 DESC_HDR_SEL0_AESU | 2636 DESC_HDR_MODE0_AESU_CBC | 2637 DESC_HDR_SEL1_MDEUB | 2638 DESC_HDR_MODE1_MDEU_INIT | 2639 DESC_HDR_MODE1_MDEU_PAD | 2640 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2641 }, 2642 { .type = CRYPTO_ALG_TYPE_AEAD, 2643 .alg.aead = { 2644 .base = { 2645 .cra_name = "authenc(hmac(sha384)," 2646 "cbc(des3_ede))", 2647 .cra_driver_name = "authenc-hmac-sha384-" 2648 "cbc-3des-talitos", 2649 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2650 .cra_flags = CRYPTO_ALG_ASYNC | 2651 CRYPTO_ALG_ALLOCATES_MEMORY, 2652 }, 2653 .ivsize = DES3_EDE_BLOCK_SIZE, 2654 .maxauthsize = SHA384_DIGEST_SIZE, 2655 .setkey = aead_des3_setkey, 2656 }, 2657 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2658 DESC_HDR_SEL0_DEU | 2659 DESC_HDR_MODE0_DEU_CBC | 2660 DESC_HDR_MODE0_DEU_3DES | 2661 DESC_HDR_SEL1_MDEUB | 2662 DESC_HDR_MODE1_MDEU_INIT | 2663 DESC_HDR_MODE1_MDEU_PAD | 2664 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2665 }, 2666 { .type = CRYPTO_ALG_TYPE_AEAD, 2667 .alg.aead = { 2668 .base = { 2669 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2670 .cra_driver_name = "authenc-hmac-sha512-" 2671 "cbc-aes-talitos", 2672 .cra_blocksize = AES_BLOCK_SIZE, 2673 .cra_flags = CRYPTO_ALG_ASYNC | 2674 CRYPTO_ALG_ALLOCATES_MEMORY, 2675 }, 2676 .ivsize = AES_BLOCK_SIZE, 2677 .maxauthsize = SHA512_DIGEST_SIZE, 2678 }, 2679 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2680 DESC_HDR_SEL0_AESU | 2681 DESC_HDR_MODE0_AESU_CBC | 2682 DESC_HDR_SEL1_MDEUB | 2683 DESC_HDR_MODE1_MDEU_INIT | 2684 DESC_HDR_MODE1_MDEU_PAD | 2685 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2686 }, 2687 { .type = CRYPTO_ALG_TYPE_AEAD, 2688 .alg.aead = { 2689 .base = { 2690 .cra_name = "authenc(hmac(sha512)," 2691 "cbc(des3_ede))", 2692 .cra_driver_name = "authenc-hmac-sha512-" 2693 "cbc-3des-talitos", 2694 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2695 .cra_flags = CRYPTO_ALG_ASYNC | 2696 CRYPTO_ALG_ALLOCATES_MEMORY, 2697 }, 2698 .ivsize = DES3_EDE_BLOCK_SIZE, 2699 .maxauthsize = SHA512_DIGEST_SIZE, 2700 .setkey = aead_des3_setkey, 2701 }, 2702 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2703 DESC_HDR_SEL0_DEU | 2704 DESC_HDR_MODE0_DEU_CBC | 2705 DESC_HDR_MODE0_DEU_3DES | 2706 DESC_HDR_SEL1_MDEUB | 2707 DESC_HDR_MODE1_MDEU_INIT | 2708 DESC_HDR_MODE1_MDEU_PAD | 2709 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2710 }, 2711 { .type = CRYPTO_ALG_TYPE_AEAD, 2712 .alg.aead = { 2713 .base = { 2714 .cra_name = "authenc(hmac(md5),cbc(aes))", 2715 .cra_driver_name = "authenc-hmac-md5-" 2716 "cbc-aes-talitos", 2717 .cra_blocksize = AES_BLOCK_SIZE, 2718 .cra_flags = CRYPTO_ALG_ASYNC | 2719 CRYPTO_ALG_ALLOCATES_MEMORY, 2720 }, 2721 .ivsize = AES_BLOCK_SIZE, 2722 .maxauthsize = MD5_DIGEST_SIZE, 2723 }, 2724 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2725 DESC_HDR_SEL0_AESU | 2726 DESC_HDR_MODE0_AESU_CBC | 2727 DESC_HDR_SEL1_MDEUA | 2728 DESC_HDR_MODE1_MDEU_INIT | 2729 DESC_HDR_MODE1_MDEU_PAD | 2730 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2731 }, 2732 { .type = CRYPTO_ALG_TYPE_AEAD, 2733 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2734 .alg.aead = { 2735 .base = { 2736 .cra_name = "authenc(hmac(md5),cbc(aes))", 2737 .cra_driver_name = "authenc-hmac-md5-" 2738 "cbc-aes-talitos-hsna", 2739 .cra_blocksize = AES_BLOCK_SIZE, 2740 .cra_flags = CRYPTO_ALG_ASYNC | 2741 CRYPTO_ALG_ALLOCATES_MEMORY, 2742 }, 2743 .ivsize = AES_BLOCK_SIZE, 2744 .maxauthsize = MD5_DIGEST_SIZE, 2745 }, 2746 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2747 DESC_HDR_SEL0_AESU | 2748 DESC_HDR_MODE0_AESU_CBC | 2749 DESC_HDR_SEL1_MDEUA | 2750 DESC_HDR_MODE1_MDEU_INIT | 2751 DESC_HDR_MODE1_MDEU_PAD | 2752 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2753 }, 2754 { .type = CRYPTO_ALG_TYPE_AEAD, 2755 .alg.aead = { 2756 .base = { 2757 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2758 .cra_driver_name = "authenc-hmac-md5-" 2759 "cbc-3des-talitos", 2760 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2761 .cra_flags = CRYPTO_ALG_ASYNC | 2762 CRYPTO_ALG_ALLOCATES_MEMORY, 2763 }, 2764 .ivsize = DES3_EDE_BLOCK_SIZE, 2765 .maxauthsize = MD5_DIGEST_SIZE, 2766 .setkey = aead_des3_setkey, 2767 }, 2768 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2769 DESC_HDR_SEL0_DEU | 2770 DESC_HDR_MODE0_DEU_CBC | 2771 DESC_HDR_MODE0_DEU_3DES | 2772 DESC_HDR_SEL1_MDEUA | 2773 DESC_HDR_MODE1_MDEU_INIT | 2774 DESC_HDR_MODE1_MDEU_PAD | 2775 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2776 }, 2777 { .type = CRYPTO_ALG_TYPE_AEAD, 2778 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2779 .alg.aead = { 2780 .base = { 2781 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2782 .cra_driver_name = "authenc-hmac-md5-" 2783 "cbc-3des-talitos-hsna", 2784 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2785 .cra_flags = CRYPTO_ALG_ASYNC | 2786 CRYPTO_ALG_ALLOCATES_MEMORY, 2787 }, 2788 .ivsize = DES3_EDE_BLOCK_SIZE, 2789 .maxauthsize = MD5_DIGEST_SIZE, 2790 .setkey = aead_des3_setkey, 2791 }, 2792 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2793 DESC_HDR_SEL0_DEU | 2794 DESC_HDR_MODE0_DEU_CBC | 2795 DESC_HDR_MODE0_DEU_3DES | 2796 DESC_HDR_SEL1_MDEUA | 2797 DESC_HDR_MODE1_MDEU_INIT | 2798 DESC_HDR_MODE1_MDEU_PAD | 2799 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2800 }, 2801 /* SKCIPHER algorithms. */ 2802 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2803 .alg.skcipher = { 2804 .base.cra_name = "ecb(aes)", 2805 .base.cra_driver_name = "ecb-aes-talitos", 2806 .base.cra_blocksize = AES_BLOCK_SIZE, 2807 .base.cra_flags = CRYPTO_ALG_ASYNC | 2808 CRYPTO_ALG_ALLOCATES_MEMORY, 2809 .min_keysize = AES_MIN_KEY_SIZE, 2810 .max_keysize = AES_MAX_KEY_SIZE, 2811 .setkey = skcipher_aes_setkey, 2812 }, 2813 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2814 DESC_HDR_SEL0_AESU, 2815 }, 2816 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2817 .alg.skcipher = { 2818 .base.cra_name = "cbc(aes)", 2819 .base.cra_driver_name = "cbc-aes-talitos", 2820 .base.cra_blocksize = AES_BLOCK_SIZE, 2821 .base.cra_flags = CRYPTO_ALG_ASYNC | 2822 CRYPTO_ALG_ALLOCATES_MEMORY, 2823 .min_keysize = AES_MIN_KEY_SIZE, 2824 .max_keysize = AES_MAX_KEY_SIZE, 2825 .ivsize = AES_BLOCK_SIZE, 2826 .setkey = skcipher_aes_setkey, 2827 }, 2828 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2829 DESC_HDR_SEL0_AESU | 2830 DESC_HDR_MODE0_AESU_CBC, 2831 }, 2832 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2833 .alg.skcipher = { 2834 .base.cra_name = "ctr(aes)", 2835 .base.cra_driver_name = "ctr-aes-talitos", 2836 .base.cra_blocksize = 1, 2837 .base.cra_flags = CRYPTO_ALG_ASYNC | 2838 CRYPTO_ALG_ALLOCATES_MEMORY, 2839 .min_keysize = AES_MIN_KEY_SIZE, 2840 .max_keysize = AES_MAX_KEY_SIZE, 2841 .ivsize = AES_BLOCK_SIZE, 2842 .setkey = skcipher_aes_setkey, 2843 }, 2844 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | 2845 DESC_HDR_SEL0_AESU | 2846 DESC_HDR_MODE0_AESU_CTR, 2847 }, 2848 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2849 .alg.skcipher = { 2850 .base.cra_name = "ctr(aes)", 2851 .base.cra_driver_name = "ctr-aes-talitos", 2852 .base.cra_blocksize = 1, 2853 .base.cra_flags = CRYPTO_ALG_ASYNC | 2854 CRYPTO_ALG_ALLOCATES_MEMORY, 2855 .min_keysize = AES_MIN_KEY_SIZE, 2856 .max_keysize = AES_MAX_KEY_SIZE, 2857 .ivsize = AES_BLOCK_SIZE, 2858 .setkey = skcipher_aes_setkey, 2859 }, 2860 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2861 DESC_HDR_SEL0_AESU | 2862 DESC_HDR_MODE0_AESU_CTR, 2863 }, 2864 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2865 .alg.skcipher = { 2866 .base.cra_name = "ecb(des)", 2867 .base.cra_driver_name = "ecb-des-talitos", 2868 .base.cra_blocksize = DES_BLOCK_SIZE, 2869 .base.cra_flags = CRYPTO_ALG_ASYNC | 2870 CRYPTO_ALG_ALLOCATES_MEMORY, 2871 .min_keysize = DES_KEY_SIZE, 2872 .max_keysize = DES_KEY_SIZE, 2873 .setkey = skcipher_des_setkey, 2874 }, 2875 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2876 DESC_HDR_SEL0_DEU, 2877 }, 2878 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2879 .alg.skcipher = { 2880 .base.cra_name = "cbc(des)", 2881 .base.cra_driver_name = "cbc-des-talitos", 2882 .base.cra_blocksize = DES_BLOCK_SIZE, 2883 .base.cra_flags = CRYPTO_ALG_ASYNC | 2884 CRYPTO_ALG_ALLOCATES_MEMORY, 2885 .min_keysize = DES_KEY_SIZE, 2886 .max_keysize = DES_KEY_SIZE, 2887 .ivsize = DES_BLOCK_SIZE, 2888 .setkey = skcipher_des_setkey, 2889 }, 2890 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2891 DESC_HDR_SEL0_DEU | 2892 DESC_HDR_MODE0_DEU_CBC, 2893 }, 2894 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2895 .alg.skcipher = { 2896 .base.cra_name = "ecb(des3_ede)", 2897 .base.cra_driver_name = "ecb-3des-talitos", 2898 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2899 .base.cra_flags = CRYPTO_ALG_ASYNC | 2900 CRYPTO_ALG_ALLOCATES_MEMORY, 2901 .min_keysize = DES3_EDE_KEY_SIZE, 2902 .max_keysize = DES3_EDE_KEY_SIZE, 2903 .setkey = skcipher_des3_setkey, 2904 }, 2905 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2906 DESC_HDR_SEL0_DEU | 2907 DESC_HDR_MODE0_DEU_3DES, 2908 }, 2909 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2910 .alg.skcipher = { 2911 .base.cra_name = "cbc(des3_ede)", 2912 .base.cra_driver_name = "cbc-3des-talitos", 2913 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2914 .base.cra_flags = CRYPTO_ALG_ASYNC | 2915 CRYPTO_ALG_ALLOCATES_MEMORY, 2916 .min_keysize = DES3_EDE_KEY_SIZE, 2917 .max_keysize = DES3_EDE_KEY_SIZE, 2918 .ivsize = DES3_EDE_BLOCK_SIZE, 2919 .setkey = skcipher_des3_setkey, 2920 }, 2921 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2922 DESC_HDR_SEL0_DEU | 2923 DESC_HDR_MODE0_DEU_CBC | 2924 DESC_HDR_MODE0_DEU_3DES, 2925 }, 2926 /* AHASH algorithms. */ 2927 { .type = CRYPTO_ALG_TYPE_AHASH, 2928 .alg.hash = { 2929 .halg.digestsize = MD5_DIGEST_SIZE, 2930 .halg.statesize = sizeof(struct talitos_export_state), 2931 .halg.base = { 2932 .cra_name = "md5", 2933 .cra_driver_name = "md5-talitos", 2934 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2935 .cra_flags = CRYPTO_ALG_ASYNC | 2936 CRYPTO_ALG_ALLOCATES_MEMORY, 2937 } 2938 }, 2939 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2940 DESC_HDR_SEL0_MDEUA | 2941 DESC_HDR_MODE0_MDEU_MD5, 2942 }, 2943 { .type = CRYPTO_ALG_TYPE_AHASH, 2944 .alg.hash = { 2945 .halg.digestsize = SHA1_DIGEST_SIZE, 2946 .halg.statesize = sizeof(struct talitos_export_state), 2947 .halg.base = { 2948 .cra_name = "sha1", 2949 .cra_driver_name = "sha1-talitos", 2950 .cra_blocksize = SHA1_BLOCK_SIZE, 2951 .cra_flags = CRYPTO_ALG_ASYNC | 2952 CRYPTO_ALG_ALLOCATES_MEMORY, 2953 } 2954 }, 2955 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2956 DESC_HDR_SEL0_MDEUA | 2957 DESC_HDR_MODE0_MDEU_SHA1, 2958 }, 2959 { .type = CRYPTO_ALG_TYPE_AHASH, 2960 .alg.hash = { 2961 .halg.digestsize = SHA224_DIGEST_SIZE, 2962 .halg.statesize = sizeof(struct talitos_export_state), 2963 .halg.base = { 2964 .cra_name = "sha224", 2965 .cra_driver_name = "sha224-talitos", 2966 .cra_blocksize = SHA224_BLOCK_SIZE, 2967 .cra_flags = CRYPTO_ALG_ASYNC | 2968 CRYPTO_ALG_ALLOCATES_MEMORY, 2969 } 2970 }, 2971 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2972 DESC_HDR_SEL0_MDEUA | 2973 DESC_HDR_MODE0_MDEU_SHA224, 2974 }, 2975 { .type = CRYPTO_ALG_TYPE_AHASH, 2976 .alg.hash = { 2977 .halg.digestsize = SHA256_DIGEST_SIZE, 2978 .halg.statesize = sizeof(struct talitos_export_state), 2979 .halg.base = { 2980 .cra_name = "sha256", 2981 .cra_driver_name = "sha256-talitos", 2982 .cra_blocksize = SHA256_BLOCK_SIZE, 2983 .cra_flags = CRYPTO_ALG_ASYNC | 2984 CRYPTO_ALG_ALLOCATES_MEMORY, 2985 } 2986 }, 2987 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2988 DESC_HDR_SEL0_MDEUA | 2989 DESC_HDR_MODE0_MDEU_SHA256, 2990 }, 2991 { .type = CRYPTO_ALG_TYPE_AHASH, 2992 .alg.hash = { 2993 .halg.digestsize = SHA384_DIGEST_SIZE, 2994 .halg.statesize = sizeof(struct talitos_export_state), 2995 .halg.base = { 2996 .cra_name = "sha384", 2997 .cra_driver_name = "sha384-talitos", 2998 .cra_blocksize = SHA384_BLOCK_SIZE, 2999 .cra_flags = CRYPTO_ALG_ASYNC | 3000 CRYPTO_ALG_ALLOCATES_MEMORY, 3001 } 3002 }, 3003 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3004 DESC_HDR_SEL0_MDEUB | 3005 DESC_HDR_MODE0_MDEUB_SHA384, 3006 }, 3007 { .type = CRYPTO_ALG_TYPE_AHASH, 3008 .alg.hash = { 3009 .halg.digestsize = SHA512_DIGEST_SIZE, 3010 .halg.statesize = sizeof(struct talitos_export_state), 3011 .halg.base = { 3012 .cra_name = "sha512", 3013 .cra_driver_name = "sha512-talitos", 3014 .cra_blocksize = SHA512_BLOCK_SIZE, 3015 .cra_flags = CRYPTO_ALG_ASYNC | 3016 CRYPTO_ALG_ALLOCATES_MEMORY, 3017 } 3018 }, 3019 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3020 DESC_HDR_SEL0_MDEUB | 3021 DESC_HDR_MODE0_MDEUB_SHA512, 3022 }, 3023 { .type = CRYPTO_ALG_TYPE_AHASH, 3024 .alg.hash = { 3025 .halg.digestsize = MD5_DIGEST_SIZE, 3026 .halg.statesize = sizeof(struct talitos_export_state), 3027 .halg.base = { 3028 .cra_name = "hmac(md5)", 3029 .cra_driver_name = "hmac-md5-talitos", 3030 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 3031 .cra_flags = CRYPTO_ALG_ASYNC | 3032 CRYPTO_ALG_ALLOCATES_MEMORY, 3033 } 3034 }, 3035 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3036 DESC_HDR_SEL0_MDEUA | 3037 DESC_HDR_MODE0_MDEU_MD5, 3038 }, 3039 { .type = CRYPTO_ALG_TYPE_AHASH, 3040 .alg.hash = { 3041 .halg.digestsize = SHA1_DIGEST_SIZE, 3042 .halg.statesize = sizeof(struct talitos_export_state), 3043 .halg.base = { 3044 .cra_name = "hmac(sha1)", 3045 .cra_driver_name = "hmac-sha1-talitos", 3046 .cra_blocksize = SHA1_BLOCK_SIZE, 3047 .cra_flags = CRYPTO_ALG_ASYNC | 3048 CRYPTO_ALG_ALLOCATES_MEMORY, 3049 } 3050 }, 3051 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3052 DESC_HDR_SEL0_MDEUA | 3053 DESC_HDR_MODE0_MDEU_SHA1, 3054 }, 3055 { .type = CRYPTO_ALG_TYPE_AHASH, 3056 .alg.hash = { 3057 .halg.digestsize = SHA224_DIGEST_SIZE, 3058 .halg.statesize = sizeof(struct talitos_export_state), 3059 .halg.base = { 3060 .cra_name = "hmac(sha224)", 3061 .cra_driver_name = "hmac-sha224-talitos", 3062 .cra_blocksize = SHA224_BLOCK_SIZE, 3063 .cra_flags = CRYPTO_ALG_ASYNC | 3064 CRYPTO_ALG_ALLOCATES_MEMORY, 3065 } 3066 }, 3067 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3068 DESC_HDR_SEL0_MDEUA | 3069 DESC_HDR_MODE0_MDEU_SHA224, 3070 }, 3071 { .type = CRYPTO_ALG_TYPE_AHASH, 3072 .alg.hash = { 3073 .halg.digestsize = SHA256_DIGEST_SIZE, 3074 .halg.statesize = sizeof(struct talitos_export_state), 3075 .halg.base = { 3076 .cra_name = "hmac(sha256)", 3077 .cra_driver_name = "hmac-sha256-talitos", 3078 .cra_blocksize = SHA256_BLOCK_SIZE, 3079 .cra_flags = CRYPTO_ALG_ASYNC | 3080 CRYPTO_ALG_ALLOCATES_MEMORY, 3081 } 3082 }, 3083 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3084 DESC_HDR_SEL0_MDEUA | 3085 DESC_HDR_MODE0_MDEU_SHA256, 3086 }, 3087 { .type = CRYPTO_ALG_TYPE_AHASH, 3088 .alg.hash = { 3089 .halg.digestsize = SHA384_DIGEST_SIZE, 3090 .halg.statesize = sizeof(struct talitos_export_state), 3091 .halg.base = { 3092 .cra_name = "hmac(sha384)", 3093 .cra_driver_name = "hmac-sha384-talitos", 3094 .cra_blocksize = SHA384_BLOCK_SIZE, 3095 .cra_flags = CRYPTO_ALG_ASYNC | 3096 CRYPTO_ALG_ALLOCATES_MEMORY, 3097 } 3098 }, 3099 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3100 DESC_HDR_SEL0_MDEUB | 3101 DESC_HDR_MODE0_MDEUB_SHA384, 3102 }, 3103 { .type = CRYPTO_ALG_TYPE_AHASH, 3104 .alg.hash = { 3105 .halg.digestsize = SHA512_DIGEST_SIZE, 3106 .halg.statesize = sizeof(struct talitos_export_state), 3107 .halg.base = { 3108 .cra_name = "hmac(sha512)", 3109 .cra_driver_name = "hmac-sha512-talitos", 3110 .cra_blocksize = SHA512_BLOCK_SIZE, 3111 .cra_flags = CRYPTO_ALG_ASYNC | 3112 CRYPTO_ALG_ALLOCATES_MEMORY, 3113 } 3114 }, 3115 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3116 DESC_HDR_SEL0_MDEUB | 3117 DESC_HDR_MODE0_MDEUB_SHA512, 3118 } 3119 }; 3120 3121 struct talitos_crypto_alg { 3122 struct list_head entry; 3123 struct device *dev; 3124 struct talitos_alg_template algt; 3125 }; 3126 3127 static int talitos_init_common(struct talitos_ctx *ctx, 3128 struct talitos_crypto_alg *talitos_alg) 3129 { 3130 struct talitos_private *priv; 3131 3132 /* update context with ptr to dev */ 3133 ctx->dev = talitos_alg->dev; 3134 3135 /* assign SEC channel to tfm in round-robin fashion */ 3136 priv = dev_get_drvdata(ctx->dev); 3137 ctx->ch = atomic_inc_return(&priv->last_chan) & 3138 (priv->num_channels - 1); 3139 3140 /* copy descriptor header template value */ 3141 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 3142 3143 /* select done notification */ 3144 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 3145 3146 return 0; 3147 } 3148 3149 static int talitos_cra_init_aead(struct crypto_aead *tfm) 3150 { 3151 struct aead_alg *alg = crypto_aead_alg(tfm); 3152 struct talitos_crypto_alg *talitos_alg; 3153 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 3154 3155 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3156 algt.alg.aead); 3157 3158 return talitos_init_common(ctx, talitos_alg); 3159 } 3160 3161 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) 3162 { 3163 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3164 struct talitos_crypto_alg *talitos_alg; 3165 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); 3166 3167 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3168 algt.alg.skcipher); 3169 3170 return talitos_init_common(ctx, talitos_alg); 3171 } 3172 3173 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 3174 { 3175 struct crypto_alg *alg = tfm->__crt_alg; 3176 struct talitos_crypto_alg *talitos_alg; 3177 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3178 3179 talitos_alg = container_of(__crypto_ahash_alg(alg), 3180 struct talitos_crypto_alg, 3181 algt.alg.hash); 3182 3183 ctx->keylen = 0; 3184 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 3185 sizeof(struct talitos_ahash_req_ctx)); 3186 3187 return talitos_init_common(ctx, talitos_alg); 3188 } 3189 3190 static void talitos_cra_exit(struct crypto_tfm *tfm) 3191 { 3192 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3193 struct device *dev = ctx->dev; 3194 3195 if (ctx->keylen) 3196 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 3197 } 3198 3199 /* 3200 * given the alg's descriptor header template, determine whether descriptor 3201 * type and primary/secondary execution units required match the hw 3202 * capabilities description provided in the device tree node. 3203 */ 3204 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 3205 { 3206 struct talitos_private *priv = dev_get_drvdata(dev); 3207 int ret; 3208 3209 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 3210 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 3211 3212 if (SECONDARY_EU(desc_hdr_template)) 3213 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 3214 & priv->exec_units); 3215 3216 return ret; 3217 } 3218 3219 static void talitos_remove(struct platform_device *ofdev) 3220 { 3221 struct device *dev = &ofdev->dev; 3222 struct talitos_private *priv = dev_get_drvdata(dev); 3223 struct talitos_crypto_alg *t_alg, *n; 3224 int i; 3225 3226 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 3227 switch (t_alg->algt.type) { 3228 case CRYPTO_ALG_TYPE_SKCIPHER: 3229 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); 3230 break; 3231 case CRYPTO_ALG_TYPE_AEAD: 3232 crypto_unregister_aead(&t_alg->algt.alg.aead); 3233 break; 3234 case CRYPTO_ALG_TYPE_AHASH: 3235 crypto_unregister_ahash(&t_alg->algt.alg.hash); 3236 break; 3237 } 3238 list_del(&t_alg->entry); 3239 } 3240 3241 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 3242 talitos_unregister_rng(dev); 3243 3244 for (i = 0; i < 2; i++) 3245 if (priv->irq[i]) { 3246 free_irq(priv->irq[i], dev); 3247 irq_dispose_mapping(priv->irq[i]); 3248 } 3249 3250 tasklet_kill(&priv->done_task[0]); 3251 if (priv->irq[1]) 3252 tasklet_kill(&priv->done_task[1]); 3253 } 3254 3255 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 3256 struct talitos_alg_template 3257 *template) 3258 { 3259 struct talitos_private *priv = dev_get_drvdata(dev); 3260 struct talitos_crypto_alg *t_alg; 3261 struct crypto_alg *alg; 3262 3263 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), 3264 GFP_KERNEL); 3265 if (!t_alg) 3266 return ERR_PTR(-ENOMEM); 3267 3268 t_alg->algt = *template; 3269 3270 switch (t_alg->algt.type) { 3271 case CRYPTO_ALG_TYPE_SKCIPHER: 3272 alg = &t_alg->algt.alg.skcipher.base; 3273 alg->cra_exit = talitos_cra_exit; 3274 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; 3275 t_alg->algt.alg.skcipher.setkey = 3276 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; 3277 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; 3278 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; 3279 if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && 3280 DESC_TYPE(t_alg->algt.desc_hdr_template) != 3281 DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { 3282 devm_kfree(dev, t_alg); 3283 return ERR_PTR(-ENOTSUPP); 3284 } 3285 break; 3286 case CRYPTO_ALG_TYPE_AEAD: 3287 alg = &t_alg->algt.alg.aead.base; 3288 alg->cra_exit = talitos_cra_exit; 3289 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 3290 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: 3291 aead_setkey; 3292 t_alg->algt.alg.aead.encrypt = aead_encrypt; 3293 t_alg->algt.alg.aead.decrypt = aead_decrypt; 3294 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3295 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { 3296 devm_kfree(dev, t_alg); 3297 return ERR_PTR(-ENOTSUPP); 3298 } 3299 break; 3300 case CRYPTO_ALG_TYPE_AHASH: 3301 alg = &t_alg->algt.alg.hash.halg.base; 3302 alg->cra_init = talitos_cra_init_ahash; 3303 alg->cra_exit = talitos_cra_exit; 3304 t_alg->algt.alg.hash.init = ahash_init; 3305 t_alg->algt.alg.hash.update = ahash_update; 3306 t_alg->algt.alg.hash.final = ahash_final; 3307 t_alg->algt.alg.hash.finup = ahash_finup; 3308 t_alg->algt.alg.hash.digest = ahash_digest; 3309 if (!strncmp(alg->cra_name, "hmac", 4)) 3310 t_alg->algt.alg.hash.setkey = ahash_setkey; 3311 t_alg->algt.alg.hash.import = ahash_import; 3312 t_alg->algt.alg.hash.export = ahash_export; 3313 3314 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 3315 !strncmp(alg->cra_name, "hmac", 4)) { 3316 devm_kfree(dev, t_alg); 3317 return ERR_PTR(-ENOTSUPP); 3318 } 3319 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3320 (!strcmp(alg->cra_name, "sha224") || 3321 !strcmp(alg->cra_name, "hmac(sha224)"))) { 3322 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 3323 t_alg->algt.alg.hash.digest = 3324 ahash_digest_sha224_swinit; 3325 t_alg->algt.desc_hdr_template = 3326 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3327 DESC_HDR_SEL0_MDEUA | 3328 DESC_HDR_MODE0_MDEU_SHA256; 3329 } 3330 break; 3331 default: 3332 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 3333 devm_kfree(dev, t_alg); 3334 return ERR_PTR(-EINVAL); 3335 } 3336 3337 alg->cra_module = THIS_MODULE; 3338 if (t_alg->algt.priority) 3339 alg->cra_priority = t_alg->algt.priority; 3340 else 3341 alg->cra_priority = TALITOS_CRA_PRIORITY; 3342 if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH) 3343 alg->cra_alignmask = 3; 3344 else 3345 alg->cra_alignmask = 0; 3346 alg->cra_ctxsize = sizeof(struct talitos_ctx); 3347 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 3348 3349 t_alg->dev = dev; 3350 3351 return t_alg; 3352 } 3353 3354 static int talitos_probe_irq(struct platform_device *ofdev) 3355 { 3356 struct device *dev = &ofdev->dev; 3357 struct device_node *np = ofdev->dev.of_node; 3358 struct talitos_private *priv = dev_get_drvdata(dev); 3359 int err; 3360 bool is_sec1 = has_ftr_sec1(priv); 3361 3362 priv->irq[0] = irq_of_parse_and_map(np, 0); 3363 if (!priv->irq[0]) { 3364 dev_err(dev, "failed to map irq\n"); 3365 return -EINVAL; 3366 } 3367 if (is_sec1) { 3368 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 3369 dev_driver_string(dev), dev); 3370 goto primary_out; 3371 } 3372 3373 priv->irq[1] = irq_of_parse_and_map(np, 1); 3374 3375 /* get the primary irq line */ 3376 if (!priv->irq[1]) { 3377 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 3378 dev_driver_string(dev), dev); 3379 goto primary_out; 3380 } 3381 3382 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 3383 dev_driver_string(dev), dev); 3384 if (err) 3385 goto primary_out; 3386 3387 /* get the secondary irq line */ 3388 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 3389 dev_driver_string(dev), dev); 3390 if (err) { 3391 dev_err(dev, "failed to request secondary irq\n"); 3392 irq_dispose_mapping(priv->irq[1]); 3393 priv->irq[1] = 0; 3394 } 3395 3396 return err; 3397 3398 primary_out: 3399 if (err) { 3400 dev_err(dev, "failed to request primary irq\n"); 3401 irq_dispose_mapping(priv->irq[0]); 3402 priv->irq[0] = 0; 3403 } 3404 3405 return err; 3406 } 3407 3408 static int talitos_probe(struct platform_device *ofdev) 3409 { 3410 struct device *dev = &ofdev->dev; 3411 struct device_node *np = ofdev->dev.of_node; 3412 struct talitos_private *priv; 3413 unsigned int num_channels; 3414 int i, err; 3415 int stride; 3416 3417 if (of_property_read_u32(np, "fsl,num-channels", &num_channels)) 3418 return -EINVAL; 3419 3420 priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL); 3421 if (!priv) 3422 return -ENOMEM; 3423 3424 priv->num_channels = num_channels; 3425 3426 INIT_LIST_HEAD(&priv->alg_list); 3427 3428 dev_set_drvdata(dev, priv); 3429 3430 priv->ofdev = ofdev; 3431 3432 spin_lock_init(&priv->reg_lock); 3433 3434 priv->reg = devm_platform_ioremap_resource(ofdev, 0); 3435 if (IS_ERR(priv->reg)) { 3436 dev_err(dev, "failed to of_iomap\n"); 3437 err = PTR_ERR(priv->reg); 3438 goto err_out; 3439 } 3440 3441 /* get SEC version capabilities from device tree */ 3442 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); 3443 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); 3444 of_property_read_u32(np, "fsl,descriptor-types-mask", 3445 &priv->desc_types); 3446 3447 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 3448 !priv->exec_units || !priv->desc_types) { 3449 dev_err(dev, "invalid property data in device tree node\n"); 3450 err = -EINVAL; 3451 goto err_out; 3452 } 3453 3454 if (of_device_is_compatible(np, "fsl,sec3.0")) 3455 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 3456 3457 if (of_device_is_compatible(np, "fsl,sec2.1")) 3458 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 3459 TALITOS_FTR_SHA224_HWINIT | 3460 TALITOS_FTR_HMAC_OK; 3461 3462 if (of_device_is_compatible(np, "fsl,sec1.0")) 3463 priv->features |= TALITOS_FTR_SEC1; 3464 3465 if (of_device_is_compatible(np, "fsl,sec1.2")) { 3466 priv->reg_deu = priv->reg + TALITOS12_DEU; 3467 priv->reg_aesu = priv->reg + TALITOS12_AESU; 3468 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 3469 stride = TALITOS1_CH_STRIDE; 3470 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 3471 priv->reg_deu = priv->reg + TALITOS10_DEU; 3472 priv->reg_aesu = priv->reg + TALITOS10_AESU; 3473 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 3474 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 3475 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 3476 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 3477 stride = TALITOS1_CH_STRIDE; 3478 } else { 3479 priv->reg_deu = priv->reg + TALITOS2_DEU; 3480 priv->reg_aesu = priv->reg + TALITOS2_AESU; 3481 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 3482 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 3483 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 3484 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 3485 priv->reg_keu = priv->reg + TALITOS2_KEU; 3486 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 3487 stride = TALITOS2_CH_STRIDE; 3488 } 3489 3490 err = talitos_probe_irq(ofdev); 3491 if (err) 3492 goto err_out; 3493 3494 if (has_ftr_sec1(priv)) { 3495 if (priv->num_channels == 1) 3496 tasklet_init(&priv->done_task[0], talitos1_done_ch0, 3497 (unsigned long)dev); 3498 else 3499 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 3500 (unsigned long)dev); 3501 } else { 3502 if (priv->irq[1]) { 3503 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3504 (unsigned long)dev); 3505 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3506 (unsigned long)dev); 3507 } else if (priv->num_channels == 1) { 3508 tasklet_init(&priv->done_task[0], talitos2_done_ch0, 3509 (unsigned long)dev); 3510 } else { 3511 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3512 (unsigned long)dev); 3513 } 3514 } 3515 3516 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3517 3518 for (i = 0; i < priv->num_channels; i++) { 3519 priv->chan[i].reg = priv->reg + stride * (i + 1); 3520 if (!priv->irq[1] || !(i & 1)) 3521 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3522 3523 spin_lock_init(&priv->chan[i].head_lock); 3524 spin_lock_init(&priv->chan[i].tail_lock); 3525 3526 priv->chan[i].fifo = devm_kcalloc(dev, 3527 priv->fifo_len, 3528 sizeof(struct talitos_request), 3529 GFP_KERNEL); 3530 if (!priv->chan[i].fifo) { 3531 dev_err(dev, "failed to allocate request fifo %d\n", i); 3532 err = -ENOMEM; 3533 goto err_out; 3534 } 3535 3536 atomic_set(&priv->chan[i].submit_count, 3537 -(priv->chfifo_len - 1)); 3538 } 3539 3540 dma_set_mask(dev, DMA_BIT_MASK(36)); 3541 3542 /* reset and initialize the h/w */ 3543 err = init_device(dev); 3544 if (err) { 3545 dev_err(dev, "failed to initialize device\n"); 3546 goto err_out; 3547 } 3548 3549 /* register the RNG, if available */ 3550 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3551 err = talitos_register_rng(dev); 3552 if (err) { 3553 dev_err(dev, "failed to register hwrng: %d\n", err); 3554 goto err_out; 3555 } else 3556 dev_info(dev, "hwrng\n"); 3557 } 3558 3559 /* register crypto algorithms the device supports */ 3560 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3561 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3562 struct talitos_crypto_alg *t_alg; 3563 struct crypto_alg *alg = NULL; 3564 3565 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3566 if (IS_ERR(t_alg)) { 3567 err = PTR_ERR(t_alg); 3568 if (err == -ENOTSUPP) 3569 continue; 3570 goto err_out; 3571 } 3572 3573 switch (t_alg->algt.type) { 3574 case CRYPTO_ALG_TYPE_SKCIPHER: 3575 err = crypto_register_skcipher( 3576 &t_alg->algt.alg.skcipher); 3577 alg = &t_alg->algt.alg.skcipher.base; 3578 break; 3579 3580 case CRYPTO_ALG_TYPE_AEAD: 3581 err = crypto_register_aead( 3582 &t_alg->algt.alg.aead); 3583 alg = &t_alg->algt.alg.aead.base; 3584 break; 3585 3586 case CRYPTO_ALG_TYPE_AHASH: 3587 err = crypto_register_ahash( 3588 &t_alg->algt.alg.hash); 3589 alg = &t_alg->algt.alg.hash.halg.base; 3590 break; 3591 } 3592 if (err) { 3593 dev_err(dev, "%s alg registration failed\n", 3594 alg->cra_driver_name); 3595 devm_kfree(dev, t_alg); 3596 } else 3597 list_add_tail(&t_alg->entry, &priv->alg_list); 3598 } 3599 } 3600 if (!list_empty(&priv->alg_list)) 3601 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3602 (char *)of_get_property(np, "compatible", NULL)); 3603 3604 return 0; 3605 3606 err_out: 3607 talitos_remove(ofdev); 3608 3609 return err; 3610 } 3611 3612 static const struct of_device_id talitos_match[] = { 3613 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3614 { 3615 .compatible = "fsl,sec1.0", 3616 }, 3617 #endif 3618 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3619 { 3620 .compatible = "fsl,sec2.0", 3621 }, 3622 #endif 3623 {}, 3624 }; 3625 MODULE_DEVICE_TABLE(of, talitos_match); 3626 3627 static struct platform_driver talitos_driver = { 3628 .driver = { 3629 .name = "talitos", 3630 .of_match_table = talitos_match, 3631 }, 3632 .probe = talitos_probe, 3633 .remove = talitos_remove, 3634 }; 3635 3636 module_platform_driver(talitos_driver); 3637 3638 MODULE_LICENSE("GPL"); 3639 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3640 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3641