1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * talitos - Freescale Integrated Security Engine (SEC) device driver 4 * 5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 6 * 7 * Scatterlist Crypto API glue code copied from files with the following: 8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 9 * 10 * Crypto algorithm registration code copied from hifn driver: 11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 12 * All rights reserved. 13 */ 14 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/device.h> 18 #include <linux/interrupt.h> 19 #include <linux/crypto.h> 20 #include <linux/hw_random.h> 21 #include <linux/of.h> 22 #include <linux/of_irq.h> 23 #include <linux/platform_device.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/io.h> 26 #include <linux/spinlock.h> 27 #include <linux/rtnetlink.h> 28 #include <linux/slab.h> 29 30 #include <crypto/algapi.h> 31 #include <crypto/aes.h> 32 #include <crypto/internal/des.h> 33 #include <crypto/sha1.h> 34 #include <crypto/sha2.h> 35 #include <crypto/md5.h> 36 #include <crypto/internal/aead.h> 37 #include <crypto/authenc.h> 38 #include <crypto/internal/skcipher.h> 39 #include <crypto/hash.h> 40 #include <crypto/internal/hash.h> 41 #include <crypto/scatterwalk.h> 42 43 #include "talitos.h" 44 45 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 46 unsigned int len, bool is_sec1) 47 { 48 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 49 if (is_sec1) { 50 ptr->len1 = cpu_to_be16(len); 51 } else { 52 ptr->len = cpu_to_be16(len); 53 ptr->eptr = upper_32_bits(dma_addr); 54 } 55 } 56 57 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 58 struct talitos_ptr *src_ptr, bool is_sec1) 59 { 60 dst_ptr->ptr = src_ptr->ptr; 61 if (is_sec1) { 62 dst_ptr->len1 = src_ptr->len1; 63 } else { 64 dst_ptr->len = src_ptr->len; 65 dst_ptr->eptr = src_ptr->eptr; 66 } 67 } 68 69 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 70 bool is_sec1) 71 { 72 if (is_sec1) 73 return be16_to_cpu(ptr->len1); 74 else 75 return be16_to_cpu(ptr->len); 76 } 77 78 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, 79 bool is_sec1) 80 { 81 if (!is_sec1) 82 ptr->j_extent = val; 83 } 84 85 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) 86 { 87 if (!is_sec1) 88 ptr->j_extent |= val; 89 } 90 91 /* 92 * map virtual single (contiguous) pointer to h/w descriptor pointer 93 */ 94 static void __map_single_talitos_ptr(struct device *dev, 95 struct talitos_ptr *ptr, 96 unsigned int len, void *data, 97 enum dma_data_direction dir, 98 unsigned long attrs) 99 { 100 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); 101 struct talitos_private *priv = dev_get_drvdata(dev); 102 bool is_sec1 = has_ftr_sec1(priv); 103 104 to_talitos_ptr(ptr, dma_addr, len, is_sec1); 105 } 106 107 static void map_single_talitos_ptr(struct device *dev, 108 struct talitos_ptr *ptr, 109 unsigned int len, void *data, 110 enum dma_data_direction dir) 111 { 112 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); 113 } 114 115 static void map_single_talitos_ptr_nosync(struct device *dev, 116 struct talitos_ptr *ptr, 117 unsigned int len, void *data, 118 enum dma_data_direction dir) 119 { 120 __map_single_talitos_ptr(dev, ptr, len, data, dir, 121 DMA_ATTR_SKIP_CPU_SYNC); 122 } 123 124 /* 125 * unmap bus single (contiguous) h/w descriptor pointer 126 */ 127 static void unmap_single_talitos_ptr(struct device *dev, 128 struct talitos_ptr *ptr, 129 enum dma_data_direction dir) 130 { 131 struct talitos_private *priv = dev_get_drvdata(dev); 132 bool is_sec1 = has_ftr_sec1(priv); 133 134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 135 from_talitos_ptr_len(ptr, is_sec1), dir); 136 } 137 138 static int reset_channel(struct device *dev, int ch) 139 { 140 struct talitos_private *priv = dev_get_drvdata(dev); 141 unsigned int timeout = TALITOS_TIMEOUT; 142 bool is_sec1 = has_ftr_sec1(priv); 143 144 if (is_sec1) { 145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 146 TALITOS1_CCCR_LO_RESET); 147 148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 149 TALITOS1_CCCR_LO_RESET) && --timeout) 150 cpu_relax(); 151 } else { 152 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 153 TALITOS2_CCCR_RESET); 154 155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 156 TALITOS2_CCCR_RESET) && --timeout) 157 cpu_relax(); 158 } 159 160 if (timeout == 0) { 161 dev_err(dev, "failed to reset channel %d\n", ch); 162 return -EIO; 163 } 164 165 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 168 /* enable chaining descriptors */ 169 if (is_sec1) 170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 171 TALITOS_CCCR_LO_NE); 172 173 /* and ICCR writeback, if available */ 174 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 175 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 176 TALITOS_CCCR_LO_IWSE); 177 178 return 0; 179 } 180 181 static int reset_device(struct device *dev) 182 { 183 struct talitos_private *priv = dev_get_drvdata(dev); 184 unsigned int timeout = TALITOS_TIMEOUT; 185 bool is_sec1 = has_ftr_sec1(priv); 186 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 187 188 setbits32(priv->reg + TALITOS_MCR, mcr); 189 190 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 191 && --timeout) 192 cpu_relax(); 193 194 if (priv->irq[1] > 0) { 195 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 196 setbits32(priv->reg + TALITOS_MCR, mcr); 197 } 198 199 if (timeout == 0) { 200 dev_err(dev, "failed to reset device\n"); 201 return -EIO; 202 } 203 204 return 0; 205 } 206 207 /* 208 * Reset and initialize the device 209 */ 210 static int init_device(struct device *dev) 211 { 212 struct talitos_private *priv = dev_get_drvdata(dev); 213 int ch, err; 214 bool is_sec1 = has_ftr_sec1(priv); 215 216 /* 217 * Master reset 218 * errata documentation: warning: certain SEC interrupts 219 * are not fully cleared by writing the MCR:SWR bit, 220 * set bit twice to completely reset 221 */ 222 err = reset_device(dev); 223 if (err) 224 return err; 225 226 err = reset_device(dev); 227 if (err) 228 return err; 229 230 /* reset channels */ 231 for (ch = 0; ch < priv->num_channels; ch++) { 232 err = reset_channel(dev, ch); 233 if (err) 234 return err; 235 } 236 237 /* enable channel done and error interrupts */ 238 if (is_sec1) { 239 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 240 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 241 /* disable parity error check in DEU (erroneous? test vect.) */ 242 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 243 } else { 244 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 245 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 246 } 247 248 /* disable integrity check error interrupts (use writeback instead) */ 249 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 250 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 251 TALITOS_MDEUICR_LO_ICE); 252 253 return 0; 254 } 255 256 static void dma_map_request(struct device *dev, struct talitos_request *request, 257 struct talitos_desc *desc, bool is_sec1) 258 { 259 struct talitos_edesc *edesc = 260 container_of(desc, struct talitos_edesc, desc); 261 dma_addr_t dma_desc, prev_dma_desc; 262 struct talitos_edesc *prev_edesc = NULL; 263 264 if (is_sec1) { 265 while (edesc) { 266 edesc->desc.hdr1 = edesc->desc.hdr; 267 268 dma_desc = dma_map_single(dev, &edesc->desc.hdr1, 269 TALITOS_DESC_SIZE, 270 DMA_BIDIRECTIONAL); 271 272 if (!prev_edesc) { 273 request->dma_desc = dma_desc; 274 goto next; 275 } 276 277 /* Chain in any previous descriptors. */ 278 279 prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); 280 281 dma_sync_single_for_device(dev, prev_dma_desc, 282 TALITOS_DESC_SIZE, 283 DMA_TO_DEVICE); 284 285 next: 286 prev_edesc = edesc; 287 prev_dma_desc = dma_desc; 288 edesc = edesc->next_desc; 289 } 290 } else { 291 request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, 292 DMA_BIDIRECTIONAL); 293 } 294 } 295 296 /** 297 * talitos_submit - submits a descriptor to the device for processing 298 * @dev: the SEC device to be used 299 * @ch: the SEC device channel to be used 300 * @desc: the descriptor to be processed by the device 301 * @callback: whom to call when processing is complete 302 * @context: a handle for use by caller (optional) 303 * 304 * desc must contain valid dma-mapped (bus physical) address pointers. 305 * callback must check err and feedback in descriptor header 306 * for device processing status. 307 */ 308 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 309 void (*callback)(struct device *dev, 310 struct talitos_desc *desc, 311 void *context, int error), 312 void *context) 313 { 314 struct talitos_private *priv = dev_get_drvdata(dev); 315 struct talitos_request *request; 316 unsigned long flags; 317 int head; 318 bool is_sec1 = has_ftr_sec1(priv); 319 320 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 321 322 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 323 /* h/w fifo is full */ 324 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 325 return -EAGAIN; 326 } 327 328 head = priv->chan[ch].head; 329 request = &priv->chan[ch].fifo[head]; 330 331 /* map descriptor and save caller data */ 332 dma_map_request(dev, request, desc, is_sec1); 333 request->callback = callback; 334 request->context = context; 335 336 /* increment fifo head */ 337 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 338 339 smp_wmb(); 340 request->desc = desc; 341 342 /* GO! */ 343 wmb(); 344 out_be32(priv->chan[ch].reg + TALITOS_FF, 345 upper_32_bits(request->dma_desc)); 346 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 347 lower_32_bits(request->dma_desc)); 348 349 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 350 351 return -EINPROGRESS; 352 } 353 354 static __be32 get_request_hdr(struct device *dev, 355 struct talitos_request *request, bool is_sec1) 356 { 357 struct talitos_edesc *edesc; 358 dma_addr_t dma_desc; 359 360 if (!is_sec1) { 361 dma_sync_single_for_cpu(dev, request->dma_desc, 362 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 363 364 return request->desc->hdr; 365 } 366 367 edesc = container_of(request->desc, struct talitos_edesc, desc); 368 dma_desc = request->dma_desc; 369 while (edesc->next_desc) { 370 dma_desc = be32_to_cpu(edesc->desc.next_desc); 371 edesc = edesc->next_desc; 372 } 373 374 dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, 375 DMA_BIDIRECTIONAL); 376 377 return edesc->desc.hdr1; 378 } 379 380 static void dma_unmap_request(struct device *dev, 381 struct talitos_request *request, bool is_sec1) 382 { 383 struct talitos_edesc *edesc; 384 385 if (is_sec1) { 386 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 387 DMA_BIDIRECTIONAL); 388 edesc = container_of(request->desc, struct talitos_edesc, desc); 389 while (edesc->next_desc) { 390 dma_unmap_single(dev, 391 be32_to_cpu(edesc->desc.next_desc), 392 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 393 edesc = edesc->next_desc; 394 } 395 } else { 396 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, 397 DMA_BIDIRECTIONAL); 398 } 399 } 400 401 /* 402 * process what was done, notify callback of error if not 403 */ 404 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 405 { 406 struct talitos_private *priv = dev_get_drvdata(dev); 407 struct talitos_request *request, saved_req; 408 unsigned long flags; 409 int tail, status; 410 bool is_sec1 = has_ftr_sec1(priv); 411 412 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 413 414 tail = priv->chan[ch].tail; 415 while (priv->chan[ch].fifo[tail].desc) { 416 __be32 hdr; 417 418 request = &priv->chan[ch].fifo[tail]; 419 420 /* descriptors with their done bits set don't get the error */ 421 rmb(); 422 hdr = get_request_hdr(dev, request, is_sec1); 423 424 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 425 status = 0; 426 else 427 if (!error) 428 break; 429 else 430 status = error; 431 432 dma_unmap_request(dev, request, is_sec1); 433 434 /* copy entries so we can call callback outside lock */ 435 saved_req.desc = request->desc; 436 saved_req.callback = request->callback; 437 saved_req.context = request->context; 438 439 /* release request entry in fifo */ 440 smp_wmb(); 441 request->desc = NULL; 442 443 /* increment fifo tail */ 444 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 445 446 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 447 448 atomic_dec(&priv->chan[ch].submit_count); 449 450 saved_req.callback(dev, saved_req.desc, saved_req.context, 451 status); 452 /* channel may resume processing in single desc error case */ 453 if (error && !reset_ch && status == error) 454 return; 455 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 456 tail = priv->chan[ch].tail; 457 } 458 459 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 460 } 461 462 /* 463 * process completed requests for channels that have done status 464 */ 465 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 466 static void talitos1_done_##name(unsigned long data) \ 467 { \ 468 struct device *dev = (struct device *)data; \ 469 struct talitos_private *priv = dev_get_drvdata(dev); \ 470 unsigned long flags; \ 471 \ 472 if (ch_done_mask & 0x10000000) \ 473 flush_channel(dev, 0, 0, 0); \ 474 if (ch_done_mask & 0x40000000) \ 475 flush_channel(dev, 1, 0, 0); \ 476 if (ch_done_mask & 0x00010000) \ 477 flush_channel(dev, 2, 0, 0); \ 478 if (ch_done_mask & 0x00040000) \ 479 flush_channel(dev, 3, 0, 0); \ 480 \ 481 /* At this point, all completed channels have been processed */ \ 482 /* Unmask done interrupts for channels completed later on. */ \ 483 spin_lock_irqsave(&priv->reg_lock, flags); \ 484 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 485 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 486 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 487 } 488 489 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 490 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) 491 492 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 493 static void talitos2_done_##name(unsigned long data) \ 494 { \ 495 struct device *dev = (struct device *)data; \ 496 struct talitos_private *priv = dev_get_drvdata(dev); \ 497 unsigned long flags; \ 498 \ 499 if (ch_done_mask & 1) \ 500 flush_channel(dev, 0, 0, 0); \ 501 if (ch_done_mask & (1 << 2)) \ 502 flush_channel(dev, 1, 0, 0); \ 503 if (ch_done_mask & (1 << 4)) \ 504 flush_channel(dev, 2, 0, 0); \ 505 if (ch_done_mask & (1 << 6)) \ 506 flush_channel(dev, 3, 0, 0); \ 507 \ 508 /* At this point, all completed channels have been processed */ \ 509 /* Unmask done interrupts for channels completed later on. */ \ 510 spin_lock_irqsave(&priv->reg_lock, flags); \ 511 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 512 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 513 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 514 } 515 516 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 517 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) 518 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 519 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 520 521 static __be32 search_desc_hdr_in_request(struct talitos_request *request, 522 dma_addr_t cur_desc, bool is_sec1) 523 { 524 struct talitos_edesc *edesc; 525 526 if (request->dma_desc == cur_desc) { 527 return request->desc->hdr; 528 } else if (is_sec1) { 529 edesc = container_of(request->desc, struct talitos_edesc, desc); 530 while (edesc->next_desc) { 531 if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) 532 return edesc->next_desc->desc.hdr1; 533 edesc = edesc->next_desc; 534 } 535 } 536 return 0; 537 } 538 539 /* 540 * locate current (offending) descriptor 541 */ 542 static __be32 current_desc_hdr(struct device *dev, int ch) 543 { 544 struct talitos_private *priv = dev_get_drvdata(dev); 545 bool is_sec1 = has_ftr_sec1(priv); 546 struct talitos_request *request; 547 int tail, iter; 548 dma_addr_t cur_desc; 549 __be32 hdr = 0; 550 551 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 552 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 553 554 if (!cur_desc) { 555 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 556 return 0; 557 } 558 559 tail = priv->chan[ch].tail; 560 iter = tail; 561 do { 562 request = &priv->chan[ch].fifo[iter]; 563 564 hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); 565 if (hdr) 566 break; 567 568 iter = (iter + 1) & (priv->fifo_len - 1); 569 } while (iter != tail); 570 571 if (!hdr) 572 dev_err(dev, "couldn't locate current descriptor\n"); 573 574 return hdr; 575 } 576 577 /* 578 * user diagnostics; report root cause of error based on execution unit status 579 */ 580 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) 581 { 582 struct talitos_private *priv = dev_get_drvdata(dev); 583 int i; 584 585 if (!desc_hdr) 586 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); 587 588 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 589 case DESC_HDR_SEL0_AFEU: 590 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 591 in_be32(priv->reg_afeu + TALITOS_EUISR), 592 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 593 break; 594 case DESC_HDR_SEL0_DEU: 595 dev_err(dev, "DEUISR 0x%08x_%08x\n", 596 in_be32(priv->reg_deu + TALITOS_EUISR), 597 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 598 break; 599 case DESC_HDR_SEL0_MDEUA: 600 case DESC_HDR_SEL0_MDEUB: 601 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 602 in_be32(priv->reg_mdeu + TALITOS_EUISR), 603 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 604 break; 605 case DESC_HDR_SEL0_RNG: 606 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 607 in_be32(priv->reg_rngu + TALITOS_ISR), 608 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 609 break; 610 case DESC_HDR_SEL0_PKEU: 611 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 612 in_be32(priv->reg_pkeu + TALITOS_EUISR), 613 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 614 break; 615 case DESC_HDR_SEL0_AESU: 616 dev_err(dev, "AESUISR 0x%08x_%08x\n", 617 in_be32(priv->reg_aesu + TALITOS_EUISR), 618 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 619 break; 620 case DESC_HDR_SEL0_CRCU: 621 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 622 in_be32(priv->reg_crcu + TALITOS_EUISR), 623 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 624 break; 625 case DESC_HDR_SEL0_KEU: 626 dev_err(dev, "KEUISR 0x%08x_%08x\n", 627 in_be32(priv->reg_pkeu + TALITOS_EUISR), 628 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 629 break; 630 } 631 632 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 633 case DESC_HDR_SEL1_MDEUA: 634 case DESC_HDR_SEL1_MDEUB: 635 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 636 in_be32(priv->reg_mdeu + TALITOS_EUISR), 637 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 638 break; 639 case DESC_HDR_SEL1_CRCU: 640 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 641 in_be32(priv->reg_crcu + TALITOS_EUISR), 642 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 643 break; 644 } 645 646 for (i = 0; i < 8; i++) 647 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 648 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 649 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 650 } 651 652 /* 653 * recover from error interrupts 654 */ 655 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 656 { 657 struct talitos_private *priv = dev_get_drvdata(dev); 658 unsigned int timeout = TALITOS_TIMEOUT; 659 int ch, error, reset_dev = 0; 660 u32 v_lo; 661 bool is_sec1 = has_ftr_sec1(priv); 662 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 663 664 for (ch = 0; ch < priv->num_channels; ch++) { 665 /* skip channels without errors */ 666 if (is_sec1) { 667 /* bits 29, 31, 17, 19 */ 668 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 669 continue; 670 } else { 671 if (!(isr & (1 << (ch * 2 + 1)))) 672 continue; 673 } 674 675 error = -EINVAL; 676 677 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 678 679 if (v_lo & TALITOS_CCPSR_LO_DOF) { 680 dev_err(dev, "double fetch fifo overflow error\n"); 681 error = -EAGAIN; 682 reset_ch = 1; 683 } 684 if (v_lo & TALITOS_CCPSR_LO_SOF) { 685 /* h/w dropped descriptor */ 686 dev_err(dev, "single fetch fifo overflow error\n"); 687 error = -EAGAIN; 688 } 689 if (v_lo & TALITOS_CCPSR_LO_MDTE) 690 dev_err(dev, "master data transfer error\n"); 691 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 692 dev_err(dev, is_sec1 ? "pointer not complete error\n" 693 : "s/g data length zero error\n"); 694 if (v_lo & TALITOS_CCPSR_LO_FPZ) 695 dev_err(dev, is_sec1 ? "parity error\n" 696 : "fetch pointer zero error\n"); 697 if (v_lo & TALITOS_CCPSR_LO_IDH) 698 dev_err(dev, "illegal descriptor header error\n"); 699 if (v_lo & TALITOS_CCPSR_LO_IEU) 700 dev_err(dev, is_sec1 ? "static assignment error\n" 701 : "invalid exec unit error\n"); 702 if (v_lo & TALITOS_CCPSR_LO_EU) 703 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 704 if (!is_sec1) { 705 if (v_lo & TALITOS_CCPSR_LO_GB) 706 dev_err(dev, "gather boundary error\n"); 707 if (v_lo & TALITOS_CCPSR_LO_GRL) 708 dev_err(dev, "gather return/length error\n"); 709 if (v_lo & TALITOS_CCPSR_LO_SB) 710 dev_err(dev, "scatter boundary error\n"); 711 if (v_lo & TALITOS_CCPSR_LO_SRL) 712 dev_err(dev, "scatter return/length error\n"); 713 } 714 715 flush_channel(dev, ch, error, reset_ch); 716 717 if (reset_ch) { 718 reset_channel(dev, ch); 719 } else { 720 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 721 TALITOS2_CCCR_CONT); 722 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 723 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 724 TALITOS2_CCCR_CONT) && --timeout) 725 cpu_relax(); 726 if (timeout == 0) { 727 dev_err(dev, "failed to restart channel %d\n", 728 ch); 729 reset_dev = 1; 730 } 731 } 732 } 733 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 734 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 735 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 736 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 737 isr, isr_lo); 738 else 739 dev_err(dev, "done overflow, internal time out, or " 740 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 741 742 /* purge request queues */ 743 for (ch = 0; ch < priv->num_channels; ch++) 744 flush_channel(dev, ch, -EIO, 1); 745 746 /* reset and reinitialize the device */ 747 init_device(dev); 748 } 749 } 750 751 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 752 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 753 { \ 754 struct talitos_private *priv = data; \ 755 u32 isr, isr_lo; \ 756 unsigned long flags; \ 757 \ 758 spin_lock_irqsave(&priv->reg_lock, flags); \ 759 isr = in_be32(priv->reg + TALITOS_ISR); \ 760 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 761 /* Acknowledge interrupt */ \ 762 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 763 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 764 \ 765 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 766 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 767 talitos_error(priv->dev, isr & ch_err_mask, isr_lo); \ 768 } \ 769 else { \ 770 if (likely(isr & ch_done_mask)) { \ 771 /* mask further done interrupts. */ \ 772 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 773 /* done_task will unmask done interrupts at exit */ \ 774 tasklet_schedule(&priv->done_task[tlet]); \ 775 } \ 776 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 777 } \ 778 \ 779 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 780 IRQ_NONE; \ 781 } 782 783 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 784 785 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 786 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 787 { \ 788 struct talitos_private *priv = data; \ 789 u32 isr, isr_lo; \ 790 unsigned long flags; \ 791 \ 792 spin_lock_irqsave(&priv->reg_lock, flags); \ 793 isr = in_be32(priv->reg + TALITOS_ISR); \ 794 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 795 /* Acknowledge interrupt */ \ 796 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 797 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 798 \ 799 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 800 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 801 talitos_error(priv->dev, isr & ch_err_mask, isr_lo); \ 802 } \ 803 else { \ 804 if (likely(isr & ch_done_mask)) { \ 805 /* mask further done interrupts. */ \ 806 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 807 /* done_task will unmask done interrupts at exit */ \ 808 tasklet_schedule(&priv->done_task[tlet]); \ 809 } \ 810 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 811 } \ 812 \ 813 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 814 IRQ_NONE; \ 815 } 816 817 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 818 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 819 0) 820 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 821 1) 822 823 /* 824 * hwrng 825 */ 826 static int talitos_rng_data_present(struct hwrng *rng, int wait) 827 { 828 struct device *dev = (struct device *)rng->priv; 829 struct talitos_private *priv = dev_get_drvdata(dev); 830 u32 ofl; 831 int i; 832 833 for (i = 0; i < 20; i++) { 834 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 835 TALITOS_RNGUSR_LO_OFL; 836 if (ofl || !wait) 837 break; 838 udelay(10); 839 } 840 841 return !!ofl; 842 } 843 844 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 845 { 846 struct device *dev = (struct device *)rng->priv; 847 struct talitos_private *priv = dev_get_drvdata(dev); 848 849 /* rng fifo requires 64-bit accesses */ 850 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 851 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 852 853 return sizeof(u32); 854 } 855 856 static int talitos_rng_init(struct hwrng *rng) 857 { 858 struct device *dev = (struct device *)rng->priv; 859 struct talitos_private *priv = dev_get_drvdata(dev); 860 unsigned int timeout = TALITOS_TIMEOUT; 861 862 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 863 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 864 & TALITOS_RNGUSR_LO_RD) 865 && --timeout) 866 cpu_relax(); 867 if (timeout == 0) { 868 dev_err(dev, "failed to reset rng hw\n"); 869 return -ENODEV; 870 } 871 872 /* start generating */ 873 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 874 875 return 0; 876 } 877 878 static int talitos_register_rng(struct device *dev) 879 { 880 struct talitos_private *priv = dev_get_drvdata(dev); 881 int err; 882 883 priv->rng.name = dev_driver_string(dev); 884 priv->rng.init = talitos_rng_init; 885 priv->rng.data_present = talitos_rng_data_present; 886 priv->rng.data_read = talitos_rng_data_read; 887 priv->rng.priv = (unsigned long)dev; 888 889 err = hwrng_register(&priv->rng); 890 if (!err) 891 priv->rng_registered = true; 892 893 return err; 894 } 895 896 static void talitos_unregister_rng(struct device *dev) 897 { 898 struct talitos_private *priv = dev_get_drvdata(dev); 899 900 if (!priv->rng_registered) 901 return; 902 903 hwrng_unregister(&priv->rng); 904 priv->rng_registered = false; 905 } 906 907 /* 908 * crypto alg 909 */ 910 #define TALITOS_CRA_PRIORITY 3000 911 /* 912 * Defines a priority for doing AEAD with descriptors type 913 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP 914 */ 915 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) 916 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 917 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) 918 #else 919 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) 920 #endif 921 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 922 923 struct talitos_ctx { 924 struct device *dev; 925 int ch; 926 __be32 desc_hdr_template; 927 u8 key[TALITOS_MAX_KEY_SIZE]; 928 u8 iv[TALITOS_MAX_IV_LENGTH]; 929 dma_addr_t dma_key; 930 unsigned int keylen; 931 unsigned int enckeylen; 932 unsigned int authkeylen; 933 }; 934 935 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 936 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 937 938 struct talitos_ahash_req_ctx { 939 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 940 unsigned int hw_context_size; 941 u8 buf[2][HASH_MAX_BLOCK_SIZE]; 942 int buf_idx; 943 unsigned int swinit; 944 unsigned int first_request; 945 unsigned int last_request; 946 unsigned int to_hash_later; 947 unsigned int nbuf; 948 struct scatterlist bufsl[2]; 949 struct scatterlist *psrc; 950 }; 951 952 struct talitos_export_state { 953 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 954 u8 buf[HASH_MAX_BLOCK_SIZE]; 955 unsigned int swinit; 956 unsigned int first_request; 957 unsigned int last_request; 958 unsigned int to_hash_later; 959 unsigned int nbuf; 960 }; 961 962 static int aead_setkey(struct crypto_aead *authenc, 963 const u8 *key, unsigned int keylen) 964 { 965 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 966 struct device *dev = ctx->dev; 967 struct crypto_authenc_keys keys; 968 969 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 970 goto badkey; 971 972 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 973 goto badkey; 974 975 if (ctx->keylen) 976 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 977 978 memcpy(ctx->key, keys.authkey, keys.authkeylen); 979 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 980 981 ctx->keylen = keys.authkeylen + keys.enckeylen; 982 ctx->enckeylen = keys.enckeylen; 983 ctx->authkeylen = keys.authkeylen; 984 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 985 DMA_TO_DEVICE); 986 987 memzero_explicit(&keys, sizeof(keys)); 988 return 0; 989 990 badkey: 991 memzero_explicit(&keys, sizeof(keys)); 992 return -EINVAL; 993 } 994 995 static int aead_des3_setkey(struct crypto_aead *authenc, 996 const u8 *key, unsigned int keylen) 997 { 998 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 999 struct device *dev = ctx->dev; 1000 struct crypto_authenc_keys keys; 1001 int err; 1002 1003 err = crypto_authenc_extractkeys(&keys, key, keylen); 1004 if (unlikely(err)) 1005 goto out; 1006 1007 err = -EINVAL; 1008 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 1009 goto out; 1010 1011 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); 1012 if (err) 1013 goto out; 1014 1015 if (ctx->keylen) 1016 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1017 1018 memcpy(ctx->key, keys.authkey, keys.authkeylen); 1019 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 1020 1021 ctx->keylen = keys.authkeylen + keys.enckeylen; 1022 ctx->enckeylen = keys.enckeylen; 1023 ctx->authkeylen = keys.authkeylen; 1024 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 1025 DMA_TO_DEVICE); 1026 1027 out: 1028 memzero_explicit(&keys, sizeof(keys)); 1029 return err; 1030 } 1031 1032 static void talitos_sg_unmap(struct device *dev, 1033 struct talitos_edesc *edesc, 1034 struct scatterlist *src, 1035 struct scatterlist *dst, 1036 unsigned int len, unsigned int offset) 1037 { 1038 struct talitos_private *priv = dev_get_drvdata(dev); 1039 bool is_sec1 = has_ftr_sec1(priv); 1040 unsigned int src_nents = edesc->src_nents ? : 1; 1041 unsigned int dst_nents = edesc->dst_nents ? : 1; 1042 1043 if (is_sec1 && dst && dst_nents > 1) { 1044 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, 1045 len, DMA_FROM_DEVICE); 1046 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, 1047 offset); 1048 } 1049 if (src != dst) { 1050 if (src_nents == 1 || !is_sec1) 1051 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 1052 1053 if (dst && (dst_nents == 1 || !is_sec1)) 1054 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 1055 } else if (src_nents == 1 || !is_sec1) { 1056 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 1057 } 1058 } 1059 1060 static void ipsec_esp_unmap(struct device *dev, 1061 struct talitos_edesc *edesc, 1062 struct aead_request *areq, bool encrypt) 1063 { 1064 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1065 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1066 unsigned int ivsize = crypto_aead_ivsize(aead); 1067 unsigned int authsize = crypto_aead_authsize(aead); 1068 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1069 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; 1070 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; 1071 1072 if (is_ipsec_esp) 1073 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], 1074 DMA_FROM_DEVICE); 1075 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); 1076 1077 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, 1078 cryptlen + authsize, areq->assoclen); 1079 1080 if (edesc->dma_len) 1081 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1082 DMA_BIDIRECTIONAL); 1083 1084 if (!is_ipsec_esp) { 1085 unsigned int dst_nents = edesc->dst_nents ? : 1; 1086 1087 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, 1088 areq->assoclen + cryptlen - ivsize); 1089 } 1090 } 1091 1092 /* 1093 * ipsec_esp descriptor callbacks 1094 */ 1095 static void ipsec_esp_encrypt_done(struct device *dev, 1096 struct talitos_desc *desc, void *context, 1097 int err) 1098 { 1099 struct aead_request *areq = context; 1100 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1101 unsigned int ivsize = crypto_aead_ivsize(authenc); 1102 struct talitos_edesc *edesc; 1103 1104 edesc = container_of(desc, struct talitos_edesc, desc); 1105 1106 ipsec_esp_unmap(dev, edesc, areq, true); 1107 1108 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); 1109 1110 kfree(edesc); 1111 1112 aead_request_complete(areq, err); 1113 } 1114 1115 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 1116 struct talitos_desc *desc, 1117 void *context, int err) 1118 { 1119 struct aead_request *req = context; 1120 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1121 unsigned int authsize = crypto_aead_authsize(authenc); 1122 struct talitos_edesc *edesc; 1123 char *oicv, *icv; 1124 1125 edesc = container_of(desc, struct talitos_edesc, desc); 1126 1127 ipsec_esp_unmap(dev, edesc, req, false); 1128 1129 if (!err) { 1130 /* auth check */ 1131 oicv = edesc->buf + edesc->dma_len; 1132 icv = oicv - authsize; 1133 1134 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 1135 } 1136 1137 kfree(edesc); 1138 1139 aead_request_complete(req, err); 1140 } 1141 1142 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 1143 struct talitos_desc *desc, 1144 void *context, int err) 1145 { 1146 struct aead_request *req = context; 1147 struct talitos_edesc *edesc; 1148 1149 edesc = container_of(desc, struct talitos_edesc, desc); 1150 1151 ipsec_esp_unmap(dev, edesc, req, false); 1152 1153 /* check ICV auth status */ 1154 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1155 DESC_HDR_LO_ICCR1_PASS)) 1156 err = -EBADMSG; 1157 1158 kfree(edesc); 1159 1160 aead_request_complete(req, err); 1161 } 1162 1163 /* 1164 * convert scatterlist to SEC h/w link table format 1165 * stop at cryptlen bytes 1166 */ 1167 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1168 unsigned int offset, int datalen, int elen, 1169 struct talitos_ptr *link_tbl_ptr, int align) 1170 { 1171 int n_sg = elen ? sg_count + 1 : sg_count; 1172 int count = 0; 1173 int cryptlen = datalen + elen; 1174 int padding = ALIGN(cryptlen, align) - cryptlen; 1175 1176 while (cryptlen && sg && n_sg--) { 1177 unsigned int len = sg_dma_len(sg); 1178 1179 if (offset >= len) { 1180 offset -= len; 1181 goto next; 1182 } 1183 1184 len -= offset; 1185 1186 if (len > cryptlen) 1187 len = cryptlen; 1188 1189 if (datalen > 0 && len > datalen) { 1190 to_talitos_ptr(link_tbl_ptr + count, 1191 sg_dma_address(sg) + offset, datalen, 0); 1192 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1193 count++; 1194 len -= datalen; 1195 offset += datalen; 1196 } 1197 to_talitos_ptr(link_tbl_ptr + count, 1198 sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); 1199 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1200 count++; 1201 cryptlen -= len; 1202 datalen -= len; 1203 offset = 0; 1204 1205 next: 1206 sg = sg_next(sg); 1207 } 1208 1209 /* tag end of link table */ 1210 if (count > 0) 1211 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, 1212 DESC_PTR_LNKTBL_RET, 0); 1213 1214 return count; 1215 } 1216 1217 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, 1218 unsigned int len, struct talitos_edesc *edesc, 1219 struct talitos_ptr *ptr, int sg_count, 1220 unsigned int offset, int tbl_off, int elen, 1221 bool force, int align) 1222 { 1223 struct talitos_private *priv = dev_get_drvdata(dev); 1224 bool is_sec1 = has_ftr_sec1(priv); 1225 int aligned_len = ALIGN(len, align); 1226 1227 if (!src) { 1228 to_talitos_ptr(ptr, 0, 0, is_sec1); 1229 return 1; 1230 } 1231 to_talitos_ptr_ext_set(ptr, elen, is_sec1); 1232 if (sg_count == 1 && !force) { 1233 to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); 1234 return sg_count; 1235 } 1236 if (is_sec1) { 1237 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); 1238 return sg_count; 1239 } 1240 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, 1241 &edesc->link_tbl[tbl_off], align); 1242 if (sg_count == 1 && !force) { 1243 /* Only one segment now, so no link tbl needed*/ 1244 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); 1245 return sg_count; 1246 } 1247 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1248 tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); 1249 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); 1250 1251 return sg_count; 1252 } 1253 1254 static int talitos_sg_map(struct device *dev, struct scatterlist *src, 1255 unsigned int len, struct talitos_edesc *edesc, 1256 struct talitos_ptr *ptr, int sg_count, 1257 unsigned int offset, int tbl_off) 1258 { 1259 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, 1260 tbl_off, 0, false, 1); 1261 } 1262 1263 /* 1264 * fill in and submit ipsec_esp descriptor 1265 */ 1266 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1267 bool encrypt, 1268 void (*callback)(struct device *dev, 1269 struct talitos_desc *desc, 1270 void *context, int error)) 1271 { 1272 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1273 unsigned int authsize = crypto_aead_authsize(aead); 1274 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1275 struct device *dev = ctx->dev; 1276 struct talitos_desc *desc = &edesc->desc; 1277 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1278 unsigned int ivsize = crypto_aead_ivsize(aead); 1279 int tbl_off = 0; 1280 int sg_count, ret; 1281 int elen = 0; 1282 bool sync_needed = false; 1283 struct talitos_private *priv = dev_get_drvdata(dev); 1284 bool is_sec1 = has_ftr_sec1(priv); 1285 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; 1286 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; 1287 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; 1288 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; 1289 1290 /* hmac key */ 1291 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); 1292 1293 sg_count = edesc->src_nents ?: 1; 1294 if (is_sec1 && sg_count > 1) 1295 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1296 areq->assoclen + cryptlen); 1297 else 1298 sg_count = dma_map_sg(dev, areq->src, sg_count, 1299 (areq->src == areq->dst) ? 1300 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1301 1302 /* hmac data */ 1303 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, 1304 &desc->ptr[1], sg_count, 0, tbl_off); 1305 1306 if (ret > 1) { 1307 tbl_off += ret; 1308 sync_needed = true; 1309 } 1310 1311 /* cipher iv */ 1312 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); 1313 1314 /* cipher key */ 1315 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, 1316 ctx->enckeylen, is_sec1); 1317 1318 /* 1319 * cipher in 1320 * map and adjust cipher len to aead request cryptlen. 1321 * extent is bytes of HMAC postpended to ciphertext, 1322 * typically 12 for ipsec 1323 */ 1324 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) 1325 elen = authsize; 1326 1327 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], 1328 sg_count, areq->assoclen, tbl_off, elen, 1329 false, 1); 1330 1331 if (ret > 1) { 1332 tbl_off += ret; 1333 sync_needed = true; 1334 } 1335 1336 /* cipher out */ 1337 if (areq->src != areq->dst) { 1338 sg_count = edesc->dst_nents ? : 1; 1339 if (!is_sec1 || sg_count == 1) 1340 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1341 } 1342 1343 if (is_ipsec_esp && encrypt) 1344 elen = authsize; 1345 else 1346 elen = 0; 1347 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], 1348 sg_count, areq->assoclen, tbl_off, elen, 1349 is_ipsec_esp && !encrypt, 1); 1350 tbl_off += ret; 1351 1352 if (!encrypt && is_ipsec_esp) { 1353 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1354 1355 /* Add an entry to the link table for ICV data */ 1356 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); 1357 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); 1358 1359 /* icv data follows link tables */ 1360 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); 1361 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); 1362 sync_needed = true; 1363 } else if (!encrypt) { 1364 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); 1365 sync_needed = true; 1366 } else if (!is_ipsec_esp) { 1367 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], 1368 sg_count, areq->assoclen + cryptlen, tbl_off); 1369 } 1370 1371 /* iv out */ 1372 if (is_ipsec_esp) 1373 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1374 DMA_FROM_DEVICE); 1375 1376 if (sync_needed) 1377 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1378 edesc->dma_len, 1379 DMA_BIDIRECTIONAL); 1380 1381 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1382 if (ret != -EINPROGRESS) { 1383 ipsec_esp_unmap(dev, edesc, areq, encrypt); 1384 kfree(edesc); 1385 } 1386 return ret; 1387 } 1388 1389 /* 1390 * allocate and map the extended descriptor 1391 */ 1392 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1393 struct scatterlist *src, 1394 struct scatterlist *dst, 1395 u8 *iv, 1396 unsigned int assoclen, 1397 unsigned int cryptlen, 1398 unsigned int authsize, 1399 unsigned int ivsize, 1400 int icv_stashing, 1401 u32 cryptoflags, 1402 bool encrypt) 1403 { 1404 struct talitos_edesc *edesc; 1405 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; 1406 dma_addr_t iv_dma = 0; 1407 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1408 GFP_ATOMIC; 1409 struct talitos_private *priv = dev_get_drvdata(dev); 1410 bool is_sec1 = has_ftr_sec1(priv); 1411 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1412 1413 if (cryptlen + authsize > max_len) { 1414 dev_err(dev, "length exceeds h/w max limit\n"); 1415 return ERR_PTR(-EINVAL); 1416 } 1417 1418 if (!dst || dst == src) { 1419 src_len = assoclen + cryptlen + authsize; 1420 src_nents = sg_nents_for_len(src, src_len); 1421 if (src_nents < 0) { 1422 dev_err(dev, "Invalid number of src SG.\n"); 1423 return ERR_PTR(-EINVAL); 1424 } 1425 src_nents = (src_nents == 1) ? 0 : src_nents; 1426 dst_nents = dst ? src_nents : 0; 1427 dst_len = 0; 1428 } else { /* dst && dst != src*/ 1429 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); 1430 src_nents = sg_nents_for_len(src, src_len); 1431 if (src_nents < 0) { 1432 dev_err(dev, "Invalid number of src SG.\n"); 1433 return ERR_PTR(-EINVAL); 1434 } 1435 src_nents = (src_nents == 1) ? 0 : src_nents; 1436 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); 1437 dst_nents = sg_nents_for_len(dst, dst_len); 1438 if (dst_nents < 0) { 1439 dev_err(dev, "Invalid number of dst SG.\n"); 1440 return ERR_PTR(-EINVAL); 1441 } 1442 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1443 } 1444 1445 /* 1446 * allocate space for base edesc plus the link tables, 1447 * allowing for two separate entries for AD and generated ICV (+ 2), 1448 * and space for two sets of ICVs (stashed and generated) 1449 */ 1450 alloc_len = sizeof(struct talitos_edesc); 1451 if (src_nents || dst_nents || !encrypt) { 1452 if (is_sec1) 1453 dma_len = (src_nents ? src_len : 0) + 1454 (dst_nents ? dst_len : 0) + authsize; 1455 else 1456 dma_len = (src_nents + dst_nents + 2) * 1457 sizeof(struct talitos_ptr) + authsize; 1458 alloc_len += dma_len; 1459 } else { 1460 dma_len = 0; 1461 } 1462 alloc_len += icv_stashing ? authsize : 0; 1463 alloc_len += ivsize; 1464 1465 edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); 1466 if (!edesc) 1467 return ERR_PTR(-ENOMEM); 1468 if (ivsize) { 1469 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); 1470 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1471 } 1472 memset(&edesc->desc, 0, sizeof(edesc->desc)); 1473 1474 edesc->src_nents = src_nents; 1475 edesc->dst_nents = dst_nents; 1476 edesc->iv_dma = iv_dma; 1477 edesc->dma_len = dma_len; 1478 edesc->next_desc = NULL; 1479 if (dma_len) 1480 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1481 edesc->dma_len, 1482 DMA_BIDIRECTIONAL); 1483 1484 return edesc; 1485 } 1486 1487 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1488 int icv_stashing, bool encrypt) 1489 { 1490 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1491 unsigned int authsize = crypto_aead_authsize(authenc); 1492 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1493 unsigned int ivsize = crypto_aead_ivsize(authenc); 1494 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1495 1496 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1497 iv, areq->assoclen, cryptlen, 1498 authsize, ivsize, icv_stashing, 1499 areq->base.flags, encrypt); 1500 } 1501 1502 static int aead_encrypt(struct aead_request *req) 1503 { 1504 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1505 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1506 struct talitos_edesc *edesc; 1507 1508 /* allocate extended descriptor */ 1509 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1510 if (IS_ERR(edesc)) 1511 return PTR_ERR(edesc); 1512 1513 /* set encrypt */ 1514 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1515 1516 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); 1517 } 1518 1519 static int aead_decrypt(struct aead_request *req) 1520 { 1521 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1522 unsigned int authsize = crypto_aead_authsize(authenc); 1523 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1524 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1525 struct talitos_edesc *edesc; 1526 void *icvdata; 1527 1528 /* allocate extended descriptor */ 1529 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1530 if (IS_ERR(edesc)) 1531 return PTR_ERR(edesc); 1532 1533 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && 1534 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1535 ((!edesc->src_nents && !edesc->dst_nents) || 1536 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1537 1538 /* decrypt and check the ICV */ 1539 edesc->desc.hdr = ctx->desc_hdr_template | 1540 DESC_HDR_DIR_INBOUND | 1541 DESC_HDR_MODE1_MDEU_CICV; 1542 1543 /* reset integrity check result bits */ 1544 1545 return ipsec_esp(edesc, req, false, 1546 ipsec_esp_decrypt_hwauth_done); 1547 } 1548 1549 /* Have to check the ICV with software */ 1550 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1551 1552 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1553 icvdata = edesc->buf + edesc->dma_len; 1554 1555 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, 1556 req->assoclen + req->cryptlen - authsize); 1557 1558 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); 1559 } 1560 1561 static int skcipher_setkey(struct crypto_skcipher *cipher, 1562 const u8 *key, unsigned int keylen) 1563 { 1564 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1565 struct device *dev = ctx->dev; 1566 1567 if (ctx->keylen) 1568 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1569 1570 memcpy(&ctx->key, key, keylen); 1571 ctx->keylen = keylen; 1572 1573 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); 1574 1575 return 0; 1576 } 1577 1578 static int skcipher_des_setkey(struct crypto_skcipher *cipher, 1579 const u8 *key, unsigned int keylen) 1580 { 1581 return verify_skcipher_des_key(cipher, key) ?: 1582 skcipher_setkey(cipher, key, keylen); 1583 } 1584 1585 static int skcipher_des3_setkey(struct crypto_skcipher *cipher, 1586 const u8 *key, unsigned int keylen) 1587 { 1588 return verify_skcipher_des3_key(cipher, key) ?: 1589 skcipher_setkey(cipher, key, keylen); 1590 } 1591 1592 static int skcipher_aes_setkey(struct crypto_skcipher *cipher, 1593 const u8 *key, unsigned int keylen) 1594 { 1595 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || 1596 keylen == AES_KEYSIZE_256) 1597 return skcipher_setkey(cipher, key, keylen); 1598 1599 return -EINVAL; 1600 } 1601 1602 static void common_nonsnoop_unmap(struct device *dev, 1603 struct talitos_edesc *edesc, 1604 struct skcipher_request *areq) 1605 { 1606 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1607 1608 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); 1609 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1610 1611 if (edesc->dma_len) 1612 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1613 DMA_BIDIRECTIONAL); 1614 } 1615 1616 static void skcipher_done(struct device *dev, 1617 struct talitos_desc *desc, void *context, 1618 int err) 1619 { 1620 struct skcipher_request *areq = context; 1621 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1622 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1623 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1624 struct talitos_edesc *edesc; 1625 1626 edesc = container_of(desc, struct talitos_edesc, desc); 1627 1628 common_nonsnoop_unmap(dev, edesc, areq); 1629 memcpy(areq->iv, ctx->iv, ivsize); 1630 1631 kfree(edesc); 1632 1633 skcipher_request_complete(areq, err); 1634 } 1635 1636 static int common_nonsnoop(struct talitos_edesc *edesc, 1637 struct skcipher_request *areq, 1638 void (*callback) (struct device *dev, 1639 struct talitos_desc *desc, 1640 void *context, int error)) 1641 { 1642 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1643 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1644 struct device *dev = ctx->dev; 1645 struct talitos_desc *desc = &edesc->desc; 1646 unsigned int cryptlen = areq->cryptlen; 1647 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1648 int sg_count, ret; 1649 bool sync_needed = false; 1650 struct talitos_private *priv = dev_get_drvdata(dev); 1651 bool is_sec1 = has_ftr_sec1(priv); 1652 bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && 1653 (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; 1654 1655 /* first DWORD empty */ 1656 1657 /* cipher iv */ 1658 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); 1659 1660 /* cipher key */ 1661 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); 1662 1663 sg_count = edesc->src_nents ?: 1; 1664 if (is_sec1 && sg_count > 1) 1665 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1666 cryptlen); 1667 else 1668 sg_count = dma_map_sg(dev, areq->src, sg_count, 1669 (areq->src == areq->dst) ? 1670 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1671 /* 1672 * cipher in 1673 */ 1674 sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], 1675 sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); 1676 if (sg_count > 1) 1677 sync_needed = true; 1678 1679 /* cipher out */ 1680 if (areq->src != areq->dst) { 1681 sg_count = edesc->dst_nents ? : 1; 1682 if (!is_sec1 || sg_count == 1) 1683 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1684 } 1685 1686 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], 1687 sg_count, 0, (edesc->src_nents + 1)); 1688 if (ret > 1) 1689 sync_needed = true; 1690 1691 /* iv out */ 1692 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1693 DMA_FROM_DEVICE); 1694 1695 /* last DWORD empty */ 1696 1697 if (sync_needed) 1698 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1699 edesc->dma_len, DMA_BIDIRECTIONAL); 1700 1701 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1702 if (ret != -EINPROGRESS) { 1703 common_nonsnoop_unmap(dev, edesc, areq); 1704 kfree(edesc); 1705 } 1706 return ret; 1707 } 1708 1709 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * 1710 areq, bool encrypt) 1711 { 1712 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1713 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1714 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1715 1716 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1717 areq->iv, 0, areq->cryptlen, 0, ivsize, 0, 1718 areq->base.flags, encrypt); 1719 } 1720 1721 static int skcipher_encrypt(struct skcipher_request *areq) 1722 { 1723 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1724 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1725 struct talitos_edesc *edesc; 1726 unsigned int blocksize = 1727 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1728 1729 if (!areq->cryptlen) 1730 return 0; 1731 1732 if (areq->cryptlen % blocksize) 1733 return -EINVAL; 1734 1735 /* allocate extended descriptor */ 1736 edesc = skcipher_edesc_alloc(areq, true); 1737 if (IS_ERR(edesc)) 1738 return PTR_ERR(edesc); 1739 1740 /* set encrypt */ 1741 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1742 1743 return common_nonsnoop(edesc, areq, skcipher_done); 1744 } 1745 1746 static int skcipher_decrypt(struct skcipher_request *areq) 1747 { 1748 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1749 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1750 struct talitos_edesc *edesc; 1751 unsigned int blocksize = 1752 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1753 1754 if (!areq->cryptlen) 1755 return 0; 1756 1757 if (areq->cryptlen % blocksize) 1758 return -EINVAL; 1759 1760 /* allocate extended descriptor */ 1761 edesc = skcipher_edesc_alloc(areq, false); 1762 if (IS_ERR(edesc)) 1763 return PTR_ERR(edesc); 1764 1765 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1766 1767 return common_nonsnoop(edesc, areq, skcipher_done); 1768 } 1769 1770 static void common_nonsnoop_hash_unmap(struct device *dev, 1771 struct talitos_edesc *edesc, 1772 struct ahash_request *areq) 1773 { 1774 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1775 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1776 struct talitos_private *priv = dev_get_drvdata(dev); 1777 bool is_sec1 = has_ftr_sec1(priv); 1778 struct talitos_desc *desc = &edesc->desc; 1779 1780 unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); 1781 1782 if (edesc->last && req_ctx->last_request) 1783 memcpy(areq->result, req_ctx->hw_context, 1784 crypto_ahash_digestsize(tfm)); 1785 1786 if (edesc->src) 1787 talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); 1788 1789 /* When using hashctx-in, must unmap it. */ 1790 if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) 1791 unmap_single_talitos_ptr(dev, &desc->ptr[1], 1792 DMA_TO_DEVICE); 1793 1794 if (edesc->dma_len) 1795 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1796 DMA_BIDIRECTIONAL); 1797 } 1798 1799 static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) 1800 { 1801 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1802 struct talitos_edesc *next; 1803 1804 while (edesc) { 1805 next = edesc->next_desc; 1806 common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); 1807 kfree(edesc); 1808 edesc = next; 1809 } 1810 } 1811 1812 static void ahash_done(struct device *dev, 1813 struct talitos_desc *desc, void *context, 1814 int err) 1815 { 1816 struct ahash_request *areq = context; 1817 struct talitos_edesc *edesc = 1818 container_of(desc, struct talitos_edesc, desc); 1819 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1820 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1821 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); 1822 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1823 struct talitos_edesc *next; 1824 1825 if (is_sec1) { 1826 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1827 /* Position any partial block for next update/final/finup */ 1828 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1829 req_ctx->nbuf = req_ctx->to_hash_later; 1830 } 1831 1832 free_edesc_list_from(areq, edesc); 1833 ahash_request_complete(areq, err); 1834 } else { 1835 next = edesc->next_desc; 1836 1837 common_nonsnoop_hash_unmap(dev, edesc, areq); 1838 kfree(edesc); 1839 1840 if (err) 1841 goto out; 1842 1843 if (next) { 1844 err = talitos_submit(dev, ctx->ch, &next->desc, 1845 ahash_done, areq); 1846 if (err != -EINPROGRESS) 1847 goto out; 1848 return; 1849 } 1850 out: 1851 if (!req_ctx->last_request && req_ctx->to_hash_later) { 1852 /* Position any partial block for next update/final/finup */ 1853 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1854 req_ctx->nbuf = req_ctx->to_hash_later; 1855 } 1856 if (err && next) 1857 free_edesc_list_from(areq, next); 1858 ahash_request_complete(areq, err); 1859 } 1860 } 1861 1862 /* 1863 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1864 * ourself and submit a padded block 1865 */ 1866 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1867 struct talitos_edesc *edesc, 1868 struct talitos_ptr *ptr) 1869 { 1870 static u8 padded_hash[64] = { 1871 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1872 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1873 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1874 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875 }; 1876 1877 pr_err_once("Bug in SEC1, padding ourself\n"); 1878 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1879 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1880 (char *)padded_hash, DMA_TO_DEVICE); 1881 } 1882 1883 static void common_nonsnoop_hash(struct talitos_edesc *edesc, 1884 struct ahash_request *areq, 1885 unsigned int length) 1886 { 1887 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1888 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1889 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1890 struct device *dev = ctx->dev; 1891 struct talitos_desc *desc = &edesc->desc; 1892 bool sync_needed = false; 1893 struct talitos_private *priv = dev_get_drvdata(dev); 1894 bool is_sec1 = has_ftr_sec1(priv); 1895 int sg_count; 1896 1897 /* first DWORD empty */ 1898 1899 /* hash context in */ 1900 if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { 1901 map_single_talitos_ptr_nosync(dev, &desc->ptr[1], 1902 req_ctx->hw_context_size, 1903 req_ctx->hw_context, 1904 DMA_TO_DEVICE); 1905 req_ctx->swinit = 0; 1906 } 1907 /* Indicate next op is not the first. */ 1908 req_ctx->first_request = 0; 1909 1910 /* HMAC key */ 1911 if (ctx->keylen) 1912 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, 1913 is_sec1); 1914 1915 sg_count = edesc->src_nents ?: 1; 1916 if (is_sec1 && sg_count > 1) 1917 sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); 1918 else if (length) 1919 sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); 1920 1921 /* 1922 * data in 1923 */ 1924 sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], 1925 sg_count, 0, 0); 1926 if (sg_count > 1) 1927 sync_needed = true; 1928 1929 /* fifth DWORD empty */ 1930 1931 /* hash/HMAC out -or- hash context out */ 1932 if (edesc->last && req_ctx->last_request) 1933 map_single_talitos_ptr(dev, &desc->ptr[5], 1934 crypto_ahash_digestsize(tfm), 1935 req_ctx->hw_context, DMA_FROM_DEVICE); 1936 else 1937 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1938 req_ctx->hw_context_size, 1939 req_ctx->hw_context, 1940 DMA_FROM_DEVICE); 1941 1942 /* last DWORD empty */ 1943 1944 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1945 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1946 1947 if (sync_needed) 1948 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1949 edesc->dma_len, DMA_BIDIRECTIONAL); 1950 } 1951 1952 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1953 struct scatterlist *src, 1954 unsigned int nbytes) 1955 { 1956 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1957 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1958 1959 return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, 1960 nbytes, 0, 0, 0, areq->base.flags, false); 1961 } 1962 1963 static struct talitos_edesc * 1964 ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, 1965 unsigned int blocksize, bool is_sec1) 1966 { 1967 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1968 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1969 struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; 1970 size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : 1971 TALITOS2_MAX_DATA_LEN; 1972 struct scatterlist tmp[2]; 1973 size_t to_hash_this_desc; 1974 struct scatterlist *src; 1975 size_t offset = 0; 1976 1977 do { 1978 src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); 1979 1980 to_hash_this_desc = 1981 min(nbytes, ALIGN_DOWN(desc_max, blocksize)); 1982 1983 /* Allocate extended descriptor */ 1984 edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); 1985 if (IS_ERR(edesc)) { 1986 if (first) 1987 free_edesc_list_from(areq, first); 1988 return edesc; 1989 } 1990 1991 edesc->src = 1992 scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); 1993 edesc->desc.hdr = ctx->desc_hdr_template; 1994 edesc->first = offset == 0; 1995 edesc->last = nbytes - to_hash_this_desc == 0; 1996 1997 /* On last one, request SEC to pad; otherwise continue */ 1998 if (req_ctx->last_request && edesc->last) 1999 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 2000 else 2001 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 2002 2003 /* request SEC to INIT hash. */ 2004 if (req_ctx->first_request && edesc->first && !req_ctx->swinit) 2005 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 2006 2007 /* 2008 * When the tfm context has a keylen, it's an HMAC. 2009 * A first or last (ie. not middle) descriptor must request HMAC. 2010 */ 2011 if (ctx->keylen && ((req_ctx->first_request && edesc->first) || 2012 (req_ctx->last_request && edesc->last))) 2013 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 2014 2015 /* clear the DN bit */ 2016 if (is_sec1 && !edesc->last) 2017 edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; 2018 2019 common_nonsnoop_hash(edesc, areq, to_hash_this_desc); 2020 2021 offset += to_hash_this_desc; 2022 nbytes -= to_hash_this_desc; 2023 2024 if (!prev_edesc) 2025 first = edesc; 2026 else 2027 prev_edesc->next_desc = edesc; 2028 prev_edesc = edesc; 2029 } while (nbytes); 2030 2031 return first; 2032 } 2033 2034 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 2035 { 2036 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2037 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2038 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2039 struct talitos_edesc *edesc; 2040 unsigned int blocksize = 2041 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2042 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); 2043 unsigned int nbytes_to_hash; 2044 unsigned int to_hash_later; 2045 unsigned int nsg; 2046 int nents; 2047 struct device *dev = ctx->dev; 2048 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; 2049 int ret; 2050 2051 if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { 2052 /* Buffer up to one whole block */ 2053 nents = sg_nents_for_len(areq->src, nbytes); 2054 if (nents < 0) { 2055 dev_err(dev, "Invalid number of src SG.\n"); 2056 return nents; 2057 } 2058 sg_copy_to_buffer(areq->src, nents, 2059 ctx_buf + req_ctx->nbuf, nbytes); 2060 req_ctx->nbuf += nbytes; 2061 return 0; 2062 } 2063 2064 /* At least (blocksize + 1) bytes are available to hash */ 2065 nbytes_to_hash = nbytes + req_ctx->nbuf; 2066 to_hash_later = nbytes_to_hash & (blocksize - 1); 2067 2068 if (req_ctx->last_request) 2069 to_hash_later = 0; 2070 else if (to_hash_later) 2071 /* There is a partial block. Hash the full block(s) now */ 2072 nbytes_to_hash -= to_hash_later; 2073 else { 2074 /* Keep one block buffered */ 2075 nbytes_to_hash -= blocksize; 2076 to_hash_later = blocksize; 2077 } 2078 2079 /* Chain in any previously buffered data */ 2080 if (req_ctx->nbuf) { 2081 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 2082 sg_init_table(req_ctx->bufsl, nsg); 2083 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); 2084 if (nsg > 1) 2085 sg_chain(req_ctx->bufsl, 2, areq->src); 2086 req_ctx->psrc = req_ctx->bufsl; 2087 } else 2088 req_ctx->psrc = areq->src; 2089 2090 if (to_hash_later) { 2091 nents = sg_nents_for_len(areq->src, nbytes); 2092 if (nents < 0) { 2093 dev_err(dev, "Invalid number of src SG.\n"); 2094 return nents; 2095 } 2096 sg_pcopy_to_buffer(areq->src, nents, 2097 req_ctx->buf[(req_ctx->buf_idx + 1) & 1], 2098 to_hash_later, 2099 nbytes - to_hash_later); 2100 } 2101 req_ctx->to_hash_later = to_hash_later; 2102 2103 edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, 2104 is_sec1); 2105 if (IS_ERR(edesc)) 2106 return PTR_ERR(edesc); 2107 2108 ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); 2109 if (ret != -EINPROGRESS) 2110 free_edesc_list_from(areq, edesc); 2111 2112 return ret; 2113 } 2114 2115 static int ahash_init(struct ahash_request *areq) 2116 { 2117 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2118 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2119 struct device *dev = ctx->dev; 2120 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2121 unsigned int size; 2122 dma_addr_t dma; 2123 2124 /* Initialize the context */ 2125 req_ctx->buf_idx = 0; 2126 req_ctx->nbuf = 0; 2127 req_ctx->first_request = 1; 2128 req_ctx->swinit = 0; /* assume h/w init of context */ 2129 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2130 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2131 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2132 req_ctx->hw_context_size = size; 2133 req_ctx->last_request = 0; 2134 2135 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2136 DMA_TO_DEVICE); 2137 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2138 2139 return 0; 2140 } 2141 2142 /* 2143 * on h/w without explicit sha224 support, we initialize h/w context 2144 * manually with sha224 constants, and tell it to run sha256. 2145 */ 2146 static int ahash_init_sha224_swinit(struct ahash_request *areq) 2147 { 2148 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2149 2150 req_ctx->hw_context[0] = SHA224_H0; 2151 req_ctx->hw_context[1] = SHA224_H1; 2152 req_ctx->hw_context[2] = SHA224_H2; 2153 req_ctx->hw_context[3] = SHA224_H3; 2154 req_ctx->hw_context[4] = SHA224_H4; 2155 req_ctx->hw_context[5] = SHA224_H5; 2156 req_ctx->hw_context[6] = SHA224_H6; 2157 req_ctx->hw_context[7] = SHA224_H7; 2158 2159 /* init 64-bit count */ 2160 req_ctx->hw_context[8] = 0; 2161 req_ctx->hw_context[9] = 0; 2162 2163 ahash_init(areq); 2164 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 2165 2166 return 0; 2167 } 2168 2169 static int ahash_update(struct ahash_request *areq) 2170 { 2171 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2172 2173 req_ctx->last_request = 0; 2174 2175 return ahash_process_req(areq, areq->nbytes); 2176 } 2177 2178 static int ahash_final(struct ahash_request *areq) 2179 { 2180 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2181 2182 req_ctx->last_request = 1; 2183 2184 return ahash_process_req(areq, 0); 2185 } 2186 2187 static int ahash_finup(struct ahash_request *areq) 2188 { 2189 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2190 2191 req_ctx->last_request = 1; 2192 2193 return ahash_process_req(areq, areq->nbytes); 2194 } 2195 2196 static int ahash_digest(struct ahash_request *areq) 2197 { 2198 ahash_init(areq); 2199 return ahash_finup(areq); 2200 } 2201 2202 static int ahash_digest_sha224_swinit(struct ahash_request *areq) 2203 { 2204 ahash_init_sha224_swinit(areq); 2205 return ahash_finup(areq); 2206 } 2207 2208 static int ahash_export(struct ahash_request *areq, void *out) 2209 { 2210 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2211 struct talitos_export_state *export = out; 2212 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2213 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2214 struct device *dev = ctx->dev; 2215 dma_addr_t dma; 2216 2217 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2218 DMA_FROM_DEVICE); 2219 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); 2220 2221 memcpy(export->hw_context, req_ctx->hw_context, 2222 req_ctx->hw_context_size); 2223 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); 2224 export->swinit = req_ctx->swinit; 2225 export->first_request = req_ctx->first_request; 2226 export->last_request = req_ctx->last_request; 2227 export->to_hash_later = req_ctx->to_hash_later; 2228 export->nbuf = req_ctx->nbuf; 2229 2230 return 0; 2231 } 2232 2233 static int ahash_import(struct ahash_request *areq, const void *in) 2234 { 2235 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2236 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2237 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2238 struct device *dev = ctx->dev; 2239 const struct talitos_export_state *export = in; 2240 unsigned int size; 2241 dma_addr_t dma; 2242 2243 memset(req_ctx, 0, sizeof(*req_ctx)); 2244 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2245 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2246 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2247 req_ctx->hw_context_size = size; 2248 memcpy(req_ctx->hw_context, export->hw_context, size); 2249 memcpy(req_ctx->buf[0], export->buf, export->nbuf); 2250 req_ctx->swinit = export->swinit; 2251 req_ctx->first_request = export->first_request; 2252 req_ctx->last_request = export->last_request; 2253 req_ctx->to_hash_later = export->to_hash_later; 2254 req_ctx->nbuf = export->nbuf; 2255 2256 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2257 DMA_TO_DEVICE); 2258 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2259 2260 return 0; 2261 } 2262 2263 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2264 u8 *hash) 2265 { 2266 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2267 2268 struct scatterlist sg[1]; 2269 struct ahash_request *req; 2270 struct crypto_wait wait; 2271 int ret; 2272 2273 crypto_init_wait(&wait); 2274 2275 req = ahash_request_alloc(tfm, GFP_KERNEL); 2276 if (!req) 2277 return -ENOMEM; 2278 2279 /* Keep tfm keylen == 0 during hash of the long key */ 2280 ctx->keylen = 0; 2281 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2282 crypto_req_done, &wait); 2283 2284 sg_init_one(&sg[0], key, keylen); 2285 2286 ahash_request_set_crypt(req, sg, hash, keylen); 2287 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 2288 2289 ahash_request_free(req); 2290 2291 return ret; 2292 } 2293 2294 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2295 unsigned int keylen) 2296 { 2297 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2298 struct device *dev = ctx->dev; 2299 unsigned int blocksize = 2300 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2301 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2302 unsigned int keysize = keylen; 2303 u8 hash[SHA512_DIGEST_SIZE]; 2304 int ret; 2305 2306 if (keylen <= blocksize) 2307 memcpy(ctx->key, key, keysize); 2308 else { 2309 /* Must get the hash of the long key */ 2310 ret = keyhash(tfm, key, keylen, hash); 2311 2312 if (ret) 2313 return -EINVAL; 2314 2315 keysize = digestsize; 2316 memcpy(ctx->key, hash, digestsize); 2317 } 2318 2319 if (ctx->keylen) 2320 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 2321 2322 ctx->keylen = keysize; 2323 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); 2324 2325 return 0; 2326 } 2327 2328 2329 struct talitos_alg_template { 2330 u32 type; 2331 u32 priority; 2332 union { 2333 struct skcipher_alg skcipher; 2334 struct ahash_alg hash; 2335 struct aead_alg aead; 2336 } alg; 2337 __be32 desc_hdr_template; 2338 }; 2339 2340 static struct talitos_alg_template driver_algs[] = { 2341 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2342 { .type = CRYPTO_ALG_TYPE_AEAD, 2343 .alg.aead = { 2344 .base = { 2345 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2346 .cra_driver_name = "authenc-hmac-sha1-" 2347 "cbc-aes-talitos", 2348 .cra_blocksize = AES_BLOCK_SIZE, 2349 .cra_flags = CRYPTO_ALG_ASYNC | 2350 CRYPTO_ALG_ALLOCATES_MEMORY, 2351 }, 2352 .ivsize = AES_BLOCK_SIZE, 2353 .maxauthsize = SHA1_DIGEST_SIZE, 2354 }, 2355 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2356 DESC_HDR_SEL0_AESU | 2357 DESC_HDR_MODE0_AESU_CBC | 2358 DESC_HDR_SEL1_MDEUA | 2359 DESC_HDR_MODE1_MDEU_INIT | 2360 DESC_HDR_MODE1_MDEU_PAD | 2361 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2362 }, 2363 { .type = CRYPTO_ALG_TYPE_AEAD, 2364 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2365 .alg.aead = { 2366 .base = { 2367 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2368 .cra_driver_name = "authenc-hmac-sha1-" 2369 "cbc-aes-talitos-hsna", 2370 .cra_blocksize = AES_BLOCK_SIZE, 2371 .cra_flags = CRYPTO_ALG_ASYNC | 2372 CRYPTO_ALG_ALLOCATES_MEMORY, 2373 }, 2374 .ivsize = AES_BLOCK_SIZE, 2375 .maxauthsize = SHA1_DIGEST_SIZE, 2376 }, 2377 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2378 DESC_HDR_SEL0_AESU | 2379 DESC_HDR_MODE0_AESU_CBC | 2380 DESC_HDR_SEL1_MDEUA | 2381 DESC_HDR_MODE1_MDEU_INIT | 2382 DESC_HDR_MODE1_MDEU_PAD | 2383 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2384 }, 2385 { .type = CRYPTO_ALG_TYPE_AEAD, 2386 .alg.aead = { 2387 .base = { 2388 .cra_name = "authenc(hmac(sha1)," 2389 "cbc(des3_ede))", 2390 .cra_driver_name = "authenc-hmac-sha1-" 2391 "cbc-3des-talitos", 2392 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2393 .cra_flags = CRYPTO_ALG_ASYNC | 2394 CRYPTO_ALG_ALLOCATES_MEMORY, 2395 }, 2396 .ivsize = DES3_EDE_BLOCK_SIZE, 2397 .maxauthsize = SHA1_DIGEST_SIZE, 2398 .setkey = aead_des3_setkey, 2399 }, 2400 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2401 DESC_HDR_SEL0_DEU | 2402 DESC_HDR_MODE0_DEU_CBC | 2403 DESC_HDR_MODE0_DEU_3DES | 2404 DESC_HDR_SEL1_MDEUA | 2405 DESC_HDR_MODE1_MDEU_INIT | 2406 DESC_HDR_MODE1_MDEU_PAD | 2407 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2408 }, 2409 { .type = CRYPTO_ALG_TYPE_AEAD, 2410 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2411 .alg.aead = { 2412 .base = { 2413 .cra_name = "authenc(hmac(sha1)," 2414 "cbc(des3_ede))", 2415 .cra_driver_name = "authenc-hmac-sha1-" 2416 "cbc-3des-talitos-hsna", 2417 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2418 .cra_flags = CRYPTO_ALG_ASYNC | 2419 CRYPTO_ALG_ALLOCATES_MEMORY, 2420 }, 2421 .ivsize = DES3_EDE_BLOCK_SIZE, 2422 .maxauthsize = SHA1_DIGEST_SIZE, 2423 .setkey = aead_des3_setkey, 2424 }, 2425 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2426 DESC_HDR_SEL0_DEU | 2427 DESC_HDR_MODE0_DEU_CBC | 2428 DESC_HDR_MODE0_DEU_3DES | 2429 DESC_HDR_SEL1_MDEUA | 2430 DESC_HDR_MODE1_MDEU_INIT | 2431 DESC_HDR_MODE1_MDEU_PAD | 2432 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2433 }, 2434 { .type = CRYPTO_ALG_TYPE_AEAD, 2435 .alg.aead = { 2436 .base = { 2437 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2438 .cra_driver_name = "authenc-hmac-sha224-" 2439 "cbc-aes-talitos", 2440 .cra_blocksize = AES_BLOCK_SIZE, 2441 .cra_flags = CRYPTO_ALG_ASYNC | 2442 CRYPTO_ALG_ALLOCATES_MEMORY, 2443 }, 2444 .ivsize = AES_BLOCK_SIZE, 2445 .maxauthsize = SHA224_DIGEST_SIZE, 2446 }, 2447 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2448 DESC_HDR_SEL0_AESU | 2449 DESC_HDR_MODE0_AESU_CBC | 2450 DESC_HDR_SEL1_MDEUA | 2451 DESC_HDR_MODE1_MDEU_INIT | 2452 DESC_HDR_MODE1_MDEU_PAD | 2453 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2454 }, 2455 { .type = CRYPTO_ALG_TYPE_AEAD, 2456 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2457 .alg.aead = { 2458 .base = { 2459 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2460 .cra_driver_name = "authenc-hmac-sha224-" 2461 "cbc-aes-talitos-hsna", 2462 .cra_blocksize = AES_BLOCK_SIZE, 2463 .cra_flags = CRYPTO_ALG_ASYNC | 2464 CRYPTO_ALG_ALLOCATES_MEMORY, 2465 }, 2466 .ivsize = AES_BLOCK_SIZE, 2467 .maxauthsize = SHA224_DIGEST_SIZE, 2468 }, 2469 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2470 DESC_HDR_SEL0_AESU | 2471 DESC_HDR_MODE0_AESU_CBC | 2472 DESC_HDR_SEL1_MDEUA | 2473 DESC_HDR_MODE1_MDEU_INIT | 2474 DESC_HDR_MODE1_MDEU_PAD | 2475 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2476 }, 2477 { .type = CRYPTO_ALG_TYPE_AEAD, 2478 .alg.aead = { 2479 .base = { 2480 .cra_name = "authenc(hmac(sha224)," 2481 "cbc(des3_ede))", 2482 .cra_driver_name = "authenc-hmac-sha224-" 2483 "cbc-3des-talitos", 2484 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2485 .cra_flags = CRYPTO_ALG_ASYNC | 2486 CRYPTO_ALG_ALLOCATES_MEMORY, 2487 }, 2488 .ivsize = DES3_EDE_BLOCK_SIZE, 2489 .maxauthsize = SHA224_DIGEST_SIZE, 2490 .setkey = aead_des3_setkey, 2491 }, 2492 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2493 DESC_HDR_SEL0_DEU | 2494 DESC_HDR_MODE0_DEU_CBC | 2495 DESC_HDR_MODE0_DEU_3DES | 2496 DESC_HDR_SEL1_MDEUA | 2497 DESC_HDR_MODE1_MDEU_INIT | 2498 DESC_HDR_MODE1_MDEU_PAD | 2499 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2500 }, 2501 { .type = CRYPTO_ALG_TYPE_AEAD, 2502 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2503 .alg.aead = { 2504 .base = { 2505 .cra_name = "authenc(hmac(sha224)," 2506 "cbc(des3_ede))", 2507 .cra_driver_name = "authenc-hmac-sha224-" 2508 "cbc-3des-talitos-hsna", 2509 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2510 .cra_flags = CRYPTO_ALG_ASYNC | 2511 CRYPTO_ALG_ALLOCATES_MEMORY, 2512 }, 2513 .ivsize = DES3_EDE_BLOCK_SIZE, 2514 .maxauthsize = SHA224_DIGEST_SIZE, 2515 .setkey = aead_des3_setkey, 2516 }, 2517 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2518 DESC_HDR_SEL0_DEU | 2519 DESC_HDR_MODE0_DEU_CBC | 2520 DESC_HDR_MODE0_DEU_3DES | 2521 DESC_HDR_SEL1_MDEUA | 2522 DESC_HDR_MODE1_MDEU_INIT | 2523 DESC_HDR_MODE1_MDEU_PAD | 2524 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2525 }, 2526 { .type = CRYPTO_ALG_TYPE_AEAD, 2527 .alg.aead = { 2528 .base = { 2529 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2530 .cra_driver_name = "authenc-hmac-sha256-" 2531 "cbc-aes-talitos", 2532 .cra_blocksize = AES_BLOCK_SIZE, 2533 .cra_flags = CRYPTO_ALG_ASYNC | 2534 CRYPTO_ALG_ALLOCATES_MEMORY, 2535 }, 2536 .ivsize = AES_BLOCK_SIZE, 2537 .maxauthsize = SHA256_DIGEST_SIZE, 2538 }, 2539 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2540 DESC_HDR_SEL0_AESU | 2541 DESC_HDR_MODE0_AESU_CBC | 2542 DESC_HDR_SEL1_MDEUA | 2543 DESC_HDR_MODE1_MDEU_INIT | 2544 DESC_HDR_MODE1_MDEU_PAD | 2545 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2546 }, 2547 { .type = CRYPTO_ALG_TYPE_AEAD, 2548 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2549 .alg.aead = { 2550 .base = { 2551 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2552 .cra_driver_name = "authenc-hmac-sha256-" 2553 "cbc-aes-talitos-hsna", 2554 .cra_blocksize = AES_BLOCK_SIZE, 2555 .cra_flags = CRYPTO_ALG_ASYNC | 2556 CRYPTO_ALG_ALLOCATES_MEMORY, 2557 }, 2558 .ivsize = AES_BLOCK_SIZE, 2559 .maxauthsize = SHA256_DIGEST_SIZE, 2560 }, 2561 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2562 DESC_HDR_SEL0_AESU | 2563 DESC_HDR_MODE0_AESU_CBC | 2564 DESC_HDR_SEL1_MDEUA | 2565 DESC_HDR_MODE1_MDEU_INIT | 2566 DESC_HDR_MODE1_MDEU_PAD | 2567 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2568 }, 2569 { .type = CRYPTO_ALG_TYPE_AEAD, 2570 .alg.aead = { 2571 .base = { 2572 .cra_name = "authenc(hmac(sha256)," 2573 "cbc(des3_ede))", 2574 .cra_driver_name = "authenc-hmac-sha256-" 2575 "cbc-3des-talitos", 2576 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2577 .cra_flags = CRYPTO_ALG_ASYNC | 2578 CRYPTO_ALG_ALLOCATES_MEMORY, 2579 }, 2580 .ivsize = DES3_EDE_BLOCK_SIZE, 2581 .maxauthsize = SHA256_DIGEST_SIZE, 2582 .setkey = aead_des3_setkey, 2583 }, 2584 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2585 DESC_HDR_SEL0_DEU | 2586 DESC_HDR_MODE0_DEU_CBC | 2587 DESC_HDR_MODE0_DEU_3DES | 2588 DESC_HDR_SEL1_MDEUA | 2589 DESC_HDR_MODE1_MDEU_INIT | 2590 DESC_HDR_MODE1_MDEU_PAD | 2591 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2592 }, 2593 { .type = CRYPTO_ALG_TYPE_AEAD, 2594 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2595 .alg.aead = { 2596 .base = { 2597 .cra_name = "authenc(hmac(sha256)," 2598 "cbc(des3_ede))", 2599 .cra_driver_name = "authenc-hmac-sha256-" 2600 "cbc-3des-talitos-hsna", 2601 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2602 .cra_flags = CRYPTO_ALG_ASYNC | 2603 CRYPTO_ALG_ALLOCATES_MEMORY, 2604 }, 2605 .ivsize = DES3_EDE_BLOCK_SIZE, 2606 .maxauthsize = SHA256_DIGEST_SIZE, 2607 .setkey = aead_des3_setkey, 2608 }, 2609 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2610 DESC_HDR_SEL0_DEU | 2611 DESC_HDR_MODE0_DEU_CBC | 2612 DESC_HDR_MODE0_DEU_3DES | 2613 DESC_HDR_SEL1_MDEUA | 2614 DESC_HDR_MODE1_MDEU_INIT | 2615 DESC_HDR_MODE1_MDEU_PAD | 2616 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2617 }, 2618 { .type = CRYPTO_ALG_TYPE_AEAD, 2619 .alg.aead = { 2620 .base = { 2621 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2622 .cra_driver_name = "authenc-hmac-sha384-" 2623 "cbc-aes-talitos", 2624 .cra_blocksize = AES_BLOCK_SIZE, 2625 .cra_flags = CRYPTO_ALG_ASYNC | 2626 CRYPTO_ALG_ALLOCATES_MEMORY, 2627 }, 2628 .ivsize = AES_BLOCK_SIZE, 2629 .maxauthsize = SHA384_DIGEST_SIZE, 2630 }, 2631 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2632 DESC_HDR_SEL0_AESU | 2633 DESC_HDR_MODE0_AESU_CBC | 2634 DESC_HDR_SEL1_MDEUB | 2635 DESC_HDR_MODE1_MDEU_INIT | 2636 DESC_HDR_MODE1_MDEU_PAD | 2637 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2638 }, 2639 { .type = CRYPTO_ALG_TYPE_AEAD, 2640 .alg.aead = { 2641 .base = { 2642 .cra_name = "authenc(hmac(sha384)," 2643 "cbc(des3_ede))", 2644 .cra_driver_name = "authenc-hmac-sha384-" 2645 "cbc-3des-talitos", 2646 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2647 .cra_flags = CRYPTO_ALG_ASYNC | 2648 CRYPTO_ALG_ALLOCATES_MEMORY, 2649 }, 2650 .ivsize = DES3_EDE_BLOCK_SIZE, 2651 .maxauthsize = SHA384_DIGEST_SIZE, 2652 .setkey = aead_des3_setkey, 2653 }, 2654 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2655 DESC_HDR_SEL0_DEU | 2656 DESC_HDR_MODE0_DEU_CBC | 2657 DESC_HDR_MODE0_DEU_3DES | 2658 DESC_HDR_SEL1_MDEUB | 2659 DESC_HDR_MODE1_MDEU_INIT | 2660 DESC_HDR_MODE1_MDEU_PAD | 2661 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2662 }, 2663 { .type = CRYPTO_ALG_TYPE_AEAD, 2664 .alg.aead = { 2665 .base = { 2666 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2667 .cra_driver_name = "authenc-hmac-sha512-" 2668 "cbc-aes-talitos", 2669 .cra_blocksize = AES_BLOCK_SIZE, 2670 .cra_flags = CRYPTO_ALG_ASYNC | 2671 CRYPTO_ALG_ALLOCATES_MEMORY, 2672 }, 2673 .ivsize = AES_BLOCK_SIZE, 2674 .maxauthsize = SHA512_DIGEST_SIZE, 2675 }, 2676 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2677 DESC_HDR_SEL0_AESU | 2678 DESC_HDR_MODE0_AESU_CBC | 2679 DESC_HDR_SEL1_MDEUB | 2680 DESC_HDR_MODE1_MDEU_INIT | 2681 DESC_HDR_MODE1_MDEU_PAD | 2682 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2683 }, 2684 { .type = CRYPTO_ALG_TYPE_AEAD, 2685 .alg.aead = { 2686 .base = { 2687 .cra_name = "authenc(hmac(sha512)," 2688 "cbc(des3_ede))", 2689 .cra_driver_name = "authenc-hmac-sha512-" 2690 "cbc-3des-talitos", 2691 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2692 .cra_flags = CRYPTO_ALG_ASYNC | 2693 CRYPTO_ALG_ALLOCATES_MEMORY, 2694 }, 2695 .ivsize = DES3_EDE_BLOCK_SIZE, 2696 .maxauthsize = SHA512_DIGEST_SIZE, 2697 .setkey = aead_des3_setkey, 2698 }, 2699 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2700 DESC_HDR_SEL0_DEU | 2701 DESC_HDR_MODE0_DEU_CBC | 2702 DESC_HDR_MODE0_DEU_3DES | 2703 DESC_HDR_SEL1_MDEUB | 2704 DESC_HDR_MODE1_MDEU_INIT | 2705 DESC_HDR_MODE1_MDEU_PAD | 2706 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2707 }, 2708 { .type = CRYPTO_ALG_TYPE_AEAD, 2709 .alg.aead = { 2710 .base = { 2711 .cra_name = "authenc(hmac(md5),cbc(aes))", 2712 .cra_driver_name = "authenc-hmac-md5-" 2713 "cbc-aes-talitos", 2714 .cra_blocksize = AES_BLOCK_SIZE, 2715 .cra_flags = CRYPTO_ALG_ASYNC | 2716 CRYPTO_ALG_ALLOCATES_MEMORY, 2717 }, 2718 .ivsize = AES_BLOCK_SIZE, 2719 .maxauthsize = MD5_DIGEST_SIZE, 2720 }, 2721 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2722 DESC_HDR_SEL0_AESU | 2723 DESC_HDR_MODE0_AESU_CBC | 2724 DESC_HDR_SEL1_MDEUA | 2725 DESC_HDR_MODE1_MDEU_INIT | 2726 DESC_HDR_MODE1_MDEU_PAD | 2727 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2728 }, 2729 { .type = CRYPTO_ALG_TYPE_AEAD, 2730 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2731 .alg.aead = { 2732 .base = { 2733 .cra_name = "authenc(hmac(md5),cbc(aes))", 2734 .cra_driver_name = "authenc-hmac-md5-" 2735 "cbc-aes-talitos-hsna", 2736 .cra_blocksize = AES_BLOCK_SIZE, 2737 .cra_flags = CRYPTO_ALG_ASYNC | 2738 CRYPTO_ALG_ALLOCATES_MEMORY, 2739 }, 2740 .ivsize = AES_BLOCK_SIZE, 2741 .maxauthsize = MD5_DIGEST_SIZE, 2742 }, 2743 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2744 DESC_HDR_SEL0_AESU | 2745 DESC_HDR_MODE0_AESU_CBC | 2746 DESC_HDR_SEL1_MDEUA | 2747 DESC_HDR_MODE1_MDEU_INIT | 2748 DESC_HDR_MODE1_MDEU_PAD | 2749 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2750 }, 2751 { .type = CRYPTO_ALG_TYPE_AEAD, 2752 .alg.aead = { 2753 .base = { 2754 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2755 .cra_driver_name = "authenc-hmac-md5-" 2756 "cbc-3des-talitos", 2757 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2758 .cra_flags = CRYPTO_ALG_ASYNC | 2759 CRYPTO_ALG_ALLOCATES_MEMORY, 2760 }, 2761 .ivsize = DES3_EDE_BLOCK_SIZE, 2762 .maxauthsize = MD5_DIGEST_SIZE, 2763 .setkey = aead_des3_setkey, 2764 }, 2765 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2766 DESC_HDR_SEL0_DEU | 2767 DESC_HDR_MODE0_DEU_CBC | 2768 DESC_HDR_MODE0_DEU_3DES | 2769 DESC_HDR_SEL1_MDEUA | 2770 DESC_HDR_MODE1_MDEU_INIT | 2771 DESC_HDR_MODE1_MDEU_PAD | 2772 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2773 }, 2774 { .type = CRYPTO_ALG_TYPE_AEAD, 2775 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2776 .alg.aead = { 2777 .base = { 2778 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2779 .cra_driver_name = "authenc-hmac-md5-" 2780 "cbc-3des-talitos-hsna", 2781 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2782 .cra_flags = CRYPTO_ALG_ASYNC | 2783 CRYPTO_ALG_ALLOCATES_MEMORY, 2784 }, 2785 .ivsize = DES3_EDE_BLOCK_SIZE, 2786 .maxauthsize = MD5_DIGEST_SIZE, 2787 .setkey = aead_des3_setkey, 2788 }, 2789 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2790 DESC_HDR_SEL0_DEU | 2791 DESC_HDR_MODE0_DEU_CBC | 2792 DESC_HDR_MODE0_DEU_3DES | 2793 DESC_HDR_SEL1_MDEUA | 2794 DESC_HDR_MODE1_MDEU_INIT | 2795 DESC_HDR_MODE1_MDEU_PAD | 2796 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2797 }, 2798 /* SKCIPHER algorithms. */ 2799 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2800 .alg.skcipher = { 2801 .base.cra_name = "ecb(aes)", 2802 .base.cra_driver_name = "ecb-aes-talitos", 2803 .base.cra_blocksize = AES_BLOCK_SIZE, 2804 .base.cra_flags = CRYPTO_ALG_ASYNC | 2805 CRYPTO_ALG_ALLOCATES_MEMORY, 2806 .min_keysize = AES_MIN_KEY_SIZE, 2807 .max_keysize = AES_MAX_KEY_SIZE, 2808 .setkey = skcipher_aes_setkey, 2809 }, 2810 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2811 DESC_HDR_SEL0_AESU, 2812 }, 2813 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2814 .alg.skcipher = { 2815 .base.cra_name = "cbc(aes)", 2816 .base.cra_driver_name = "cbc-aes-talitos", 2817 .base.cra_blocksize = AES_BLOCK_SIZE, 2818 .base.cra_flags = CRYPTO_ALG_ASYNC | 2819 CRYPTO_ALG_ALLOCATES_MEMORY, 2820 .min_keysize = AES_MIN_KEY_SIZE, 2821 .max_keysize = AES_MAX_KEY_SIZE, 2822 .ivsize = AES_BLOCK_SIZE, 2823 .setkey = skcipher_aes_setkey, 2824 }, 2825 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2826 DESC_HDR_SEL0_AESU | 2827 DESC_HDR_MODE0_AESU_CBC, 2828 }, 2829 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2830 .alg.skcipher = { 2831 .base.cra_name = "ctr(aes)", 2832 .base.cra_driver_name = "ctr-aes-talitos", 2833 .base.cra_blocksize = 1, 2834 .base.cra_flags = CRYPTO_ALG_ASYNC | 2835 CRYPTO_ALG_ALLOCATES_MEMORY, 2836 .min_keysize = AES_MIN_KEY_SIZE, 2837 .max_keysize = AES_MAX_KEY_SIZE, 2838 .ivsize = AES_BLOCK_SIZE, 2839 .setkey = skcipher_aes_setkey, 2840 }, 2841 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | 2842 DESC_HDR_SEL0_AESU | 2843 DESC_HDR_MODE0_AESU_CTR, 2844 }, 2845 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2846 .alg.skcipher = { 2847 .base.cra_name = "ctr(aes)", 2848 .base.cra_driver_name = "ctr-aes-talitos", 2849 .base.cra_blocksize = 1, 2850 .base.cra_flags = CRYPTO_ALG_ASYNC | 2851 CRYPTO_ALG_ALLOCATES_MEMORY, 2852 .min_keysize = AES_MIN_KEY_SIZE, 2853 .max_keysize = AES_MAX_KEY_SIZE, 2854 .ivsize = AES_BLOCK_SIZE, 2855 .setkey = skcipher_aes_setkey, 2856 }, 2857 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2858 DESC_HDR_SEL0_AESU | 2859 DESC_HDR_MODE0_AESU_CTR, 2860 }, 2861 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2862 .alg.skcipher = { 2863 .base.cra_name = "ecb(des)", 2864 .base.cra_driver_name = "ecb-des-talitos", 2865 .base.cra_blocksize = DES_BLOCK_SIZE, 2866 .base.cra_flags = CRYPTO_ALG_ASYNC | 2867 CRYPTO_ALG_ALLOCATES_MEMORY, 2868 .min_keysize = DES_KEY_SIZE, 2869 .max_keysize = DES_KEY_SIZE, 2870 .setkey = skcipher_des_setkey, 2871 }, 2872 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2873 DESC_HDR_SEL0_DEU, 2874 }, 2875 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2876 .alg.skcipher = { 2877 .base.cra_name = "cbc(des)", 2878 .base.cra_driver_name = "cbc-des-talitos", 2879 .base.cra_blocksize = DES_BLOCK_SIZE, 2880 .base.cra_flags = CRYPTO_ALG_ASYNC | 2881 CRYPTO_ALG_ALLOCATES_MEMORY, 2882 .min_keysize = DES_KEY_SIZE, 2883 .max_keysize = DES_KEY_SIZE, 2884 .ivsize = DES_BLOCK_SIZE, 2885 .setkey = skcipher_des_setkey, 2886 }, 2887 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2888 DESC_HDR_SEL0_DEU | 2889 DESC_HDR_MODE0_DEU_CBC, 2890 }, 2891 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2892 .alg.skcipher = { 2893 .base.cra_name = "ecb(des3_ede)", 2894 .base.cra_driver_name = "ecb-3des-talitos", 2895 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2896 .base.cra_flags = CRYPTO_ALG_ASYNC | 2897 CRYPTO_ALG_ALLOCATES_MEMORY, 2898 .min_keysize = DES3_EDE_KEY_SIZE, 2899 .max_keysize = DES3_EDE_KEY_SIZE, 2900 .setkey = skcipher_des3_setkey, 2901 }, 2902 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2903 DESC_HDR_SEL0_DEU | 2904 DESC_HDR_MODE0_DEU_3DES, 2905 }, 2906 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2907 .alg.skcipher = { 2908 .base.cra_name = "cbc(des3_ede)", 2909 .base.cra_driver_name = "cbc-3des-talitos", 2910 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2911 .base.cra_flags = CRYPTO_ALG_ASYNC | 2912 CRYPTO_ALG_ALLOCATES_MEMORY, 2913 .min_keysize = DES3_EDE_KEY_SIZE, 2914 .max_keysize = DES3_EDE_KEY_SIZE, 2915 .ivsize = DES3_EDE_BLOCK_SIZE, 2916 .setkey = skcipher_des3_setkey, 2917 }, 2918 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2919 DESC_HDR_SEL0_DEU | 2920 DESC_HDR_MODE0_DEU_CBC | 2921 DESC_HDR_MODE0_DEU_3DES, 2922 }, 2923 /* AHASH algorithms. */ 2924 { .type = CRYPTO_ALG_TYPE_AHASH, 2925 .alg.hash = { 2926 .halg.digestsize = MD5_DIGEST_SIZE, 2927 .halg.statesize = sizeof(struct talitos_export_state), 2928 .halg.base = { 2929 .cra_name = "md5", 2930 .cra_driver_name = "md5-talitos", 2931 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2932 .cra_flags = CRYPTO_ALG_ASYNC | 2933 CRYPTO_ALG_ALLOCATES_MEMORY, 2934 } 2935 }, 2936 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2937 DESC_HDR_SEL0_MDEUA | 2938 DESC_HDR_MODE0_MDEU_MD5, 2939 }, 2940 { .type = CRYPTO_ALG_TYPE_AHASH, 2941 .alg.hash = { 2942 .halg.digestsize = SHA1_DIGEST_SIZE, 2943 .halg.statesize = sizeof(struct talitos_export_state), 2944 .halg.base = { 2945 .cra_name = "sha1", 2946 .cra_driver_name = "sha1-talitos", 2947 .cra_blocksize = SHA1_BLOCK_SIZE, 2948 .cra_flags = CRYPTO_ALG_ASYNC | 2949 CRYPTO_ALG_ALLOCATES_MEMORY, 2950 } 2951 }, 2952 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2953 DESC_HDR_SEL0_MDEUA | 2954 DESC_HDR_MODE0_MDEU_SHA1, 2955 }, 2956 { .type = CRYPTO_ALG_TYPE_AHASH, 2957 .alg.hash = { 2958 .halg.digestsize = SHA224_DIGEST_SIZE, 2959 .halg.statesize = sizeof(struct talitos_export_state), 2960 .halg.base = { 2961 .cra_name = "sha224", 2962 .cra_driver_name = "sha224-talitos", 2963 .cra_blocksize = SHA224_BLOCK_SIZE, 2964 .cra_flags = CRYPTO_ALG_ASYNC | 2965 CRYPTO_ALG_ALLOCATES_MEMORY, 2966 } 2967 }, 2968 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2969 DESC_HDR_SEL0_MDEUA | 2970 DESC_HDR_MODE0_MDEU_SHA224, 2971 }, 2972 { .type = CRYPTO_ALG_TYPE_AHASH, 2973 .alg.hash = { 2974 .halg.digestsize = SHA256_DIGEST_SIZE, 2975 .halg.statesize = sizeof(struct talitos_export_state), 2976 .halg.base = { 2977 .cra_name = "sha256", 2978 .cra_driver_name = "sha256-talitos", 2979 .cra_blocksize = SHA256_BLOCK_SIZE, 2980 .cra_flags = CRYPTO_ALG_ASYNC | 2981 CRYPTO_ALG_ALLOCATES_MEMORY, 2982 } 2983 }, 2984 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2985 DESC_HDR_SEL0_MDEUA | 2986 DESC_HDR_MODE0_MDEU_SHA256, 2987 }, 2988 { .type = CRYPTO_ALG_TYPE_AHASH, 2989 .alg.hash = { 2990 .halg.digestsize = SHA384_DIGEST_SIZE, 2991 .halg.statesize = sizeof(struct talitos_export_state), 2992 .halg.base = { 2993 .cra_name = "sha384", 2994 .cra_driver_name = "sha384-talitos", 2995 .cra_blocksize = SHA384_BLOCK_SIZE, 2996 .cra_flags = CRYPTO_ALG_ASYNC | 2997 CRYPTO_ALG_ALLOCATES_MEMORY, 2998 } 2999 }, 3000 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3001 DESC_HDR_SEL0_MDEUB | 3002 DESC_HDR_MODE0_MDEUB_SHA384, 3003 }, 3004 { .type = CRYPTO_ALG_TYPE_AHASH, 3005 .alg.hash = { 3006 .halg.digestsize = SHA512_DIGEST_SIZE, 3007 .halg.statesize = sizeof(struct talitos_export_state), 3008 .halg.base = { 3009 .cra_name = "sha512", 3010 .cra_driver_name = "sha512-talitos", 3011 .cra_blocksize = SHA512_BLOCK_SIZE, 3012 .cra_flags = CRYPTO_ALG_ASYNC | 3013 CRYPTO_ALG_ALLOCATES_MEMORY, 3014 } 3015 }, 3016 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3017 DESC_HDR_SEL0_MDEUB | 3018 DESC_HDR_MODE0_MDEUB_SHA512, 3019 }, 3020 { .type = CRYPTO_ALG_TYPE_AHASH, 3021 .alg.hash = { 3022 .halg.digestsize = MD5_DIGEST_SIZE, 3023 .halg.statesize = sizeof(struct talitos_export_state), 3024 .halg.base = { 3025 .cra_name = "hmac(md5)", 3026 .cra_driver_name = "hmac-md5-talitos", 3027 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 3028 .cra_flags = CRYPTO_ALG_ASYNC | 3029 CRYPTO_ALG_ALLOCATES_MEMORY, 3030 } 3031 }, 3032 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3033 DESC_HDR_SEL0_MDEUA | 3034 DESC_HDR_MODE0_MDEU_MD5, 3035 }, 3036 { .type = CRYPTO_ALG_TYPE_AHASH, 3037 .alg.hash = { 3038 .halg.digestsize = SHA1_DIGEST_SIZE, 3039 .halg.statesize = sizeof(struct talitos_export_state), 3040 .halg.base = { 3041 .cra_name = "hmac(sha1)", 3042 .cra_driver_name = "hmac-sha1-talitos", 3043 .cra_blocksize = SHA1_BLOCK_SIZE, 3044 .cra_flags = CRYPTO_ALG_ASYNC | 3045 CRYPTO_ALG_ALLOCATES_MEMORY, 3046 } 3047 }, 3048 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3049 DESC_HDR_SEL0_MDEUA | 3050 DESC_HDR_MODE0_MDEU_SHA1, 3051 }, 3052 { .type = CRYPTO_ALG_TYPE_AHASH, 3053 .alg.hash = { 3054 .halg.digestsize = SHA224_DIGEST_SIZE, 3055 .halg.statesize = sizeof(struct talitos_export_state), 3056 .halg.base = { 3057 .cra_name = "hmac(sha224)", 3058 .cra_driver_name = "hmac-sha224-talitos", 3059 .cra_blocksize = SHA224_BLOCK_SIZE, 3060 .cra_flags = CRYPTO_ALG_ASYNC | 3061 CRYPTO_ALG_ALLOCATES_MEMORY, 3062 } 3063 }, 3064 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3065 DESC_HDR_SEL0_MDEUA | 3066 DESC_HDR_MODE0_MDEU_SHA224, 3067 }, 3068 { .type = CRYPTO_ALG_TYPE_AHASH, 3069 .alg.hash = { 3070 .halg.digestsize = SHA256_DIGEST_SIZE, 3071 .halg.statesize = sizeof(struct talitos_export_state), 3072 .halg.base = { 3073 .cra_name = "hmac(sha256)", 3074 .cra_driver_name = "hmac-sha256-talitos", 3075 .cra_blocksize = SHA256_BLOCK_SIZE, 3076 .cra_flags = CRYPTO_ALG_ASYNC | 3077 CRYPTO_ALG_ALLOCATES_MEMORY, 3078 } 3079 }, 3080 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3081 DESC_HDR_SEL0_MDEUA | 3082 DESC_HDR_MODE0_MDEU_SHA256, 3083 }, 3084 { .type = CRYPTO_ALG_TYPE_AHASH, 3085 .alg.hash = { 3086 .halg.digestsize = SHA384_DIGEST_SIZE, 3087 .halg.statesize = sizeof(struct talitos_export_state), 3088 .halg.base = { 3089 .cra_name = "hmac(sha384)", 3090 .cra_driver_name = "hmac-sha384-talitos", 3091 .cra_blocksize = SHA384_BLOCK_SIZE, 3092 .cra_flags = CRYPTO_ALG_ASYNC | 3093 CRYPTO_ALG_ALLOCATES_MEMORY, 3094 } 3095 }, 3096 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3097 DESC_HDR_SEL0_MDEUB | 3098 DESC_HDR_MODE0_MDEUB_SHA384, 3099 }, 3100 { .type = CRYPTO_ALG_TYPE_AHASH, 3101 .alg.hash = { 3102 .halg.digestsize = SHA512_DIGEST_SIZE, 3103 .halg.statesize = sizeof(struct talitos_export_state), 3104 .halg.base = { 3105 .cra_name = "hmac(sha512)", 3106 .cra_driver_name = "hmac-sha512-talitos", 3107 .cra_blocksize = SHA512_BLOCK_SIZE, 3108 .cra_flags = CRYPTO_ALG_ASYNC | 3109 CRYPTO_ALG_ALLOCATES_MEMORY, 3110 } 3111 }, 3112 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3113 DESC_HDR_SEL0_MDEUB | 3114 DESC_HDR_MODE0_MDEUB_SHA512, 3115 } 3116 }; 3117 3118 struct talitos_crypto_alg { 3119 struct list_head entry; 3120 struct device *dev; 3121 struct talitos_alg_template algt; 3122 }; 3123 3124 static int talitos_init_common(struct talitos_ctx *ctx, 3125 struct talitos_crypto_alg *talitos_alg) 3126 { 3127 struct talitos_private *priv; 3128 3129 /* update context with ptr to dev */ 3130 ctx->dev = talitos_alg->dev; 3131 3132 /* assign SEC channel to tfm in round-robin fashion */ 3133 priv = dev_get_drvdata(ctx->dev); 3134 ctx->ch = atomic_inc_return(&priv->last_chan) & 3135 (priv->num_channels - 1); 3136 3137 /* copy descriptor header template value */ 3138 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 3139 3140 /* select done notification */ 3141 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 3142 3143 return 0; 3144 } 3145 3146 static int talitos_cra_init_aead(struct crypto_aead *tfm) 3147 { 3148 struct aead_alg *alg = crypto_aead_alg(tfm); 3149 struct talitos_crypto_alg *talitos_alg; 3150 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 3151 3152 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3153 algt.alg.aead); 3154 3155 return talitos_init_common(ctx, talitos_alg); 3156 } 3157 3158 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) 3159 { 3160 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3161 struct talitos_crypto_alg *talitos_alg; 3162 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); 3163 3164 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3165 algt.alg.skcipher); 3166 3167 return talitos_init_common(ctx, talitos_alg); 3168 } 3169 3170 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 3171 { 3172 struct crypto_alg *alg = tfm->__crt_alg; 3173 struct talitos_crypto_alg *talitos_alg; 3174 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3175 3176 talitos_alg = container_of(__crypto_ahash_alg(alg), 3177 struct talitos_crypto_alg, 3178 algt.alg.hash); 3179 3180 ctx->keylen = 0; 3181 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 3182 sizeof(struct talitos_ahash_req_ctx)); 3183 3184 return talitos_init_common(ctx, talitos_alg); 3185 } 3186 3187 static void talitos_cra_exit(struct crypto_tfm *tfm) 3188 { 3189 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3190 struct device *dev = ctx->dev; 3191 3192 if (ctx->keylen) 3193 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 3194 } 3195 3196 /* 3197 * given the alg's descriptor header template, determine whether descriptor 3198 * type and primary/secondary execution units required match the hw 3199 * capabilities description provided in the device tree node. 3200 */ 3201 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 3202 { 3203 struct talitos_private *priv = dev_get_drvdata(dev); 3204 int ret; 3205 3206 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 3207 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 3208 3209 if (SECONDARY_EU(desc_hdr_template)) 3210 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 3211 & priv->exec_units); 3212 3213 return ret; 3214 } 3215 3216 static void talitos_remove(struct platform_device *ofdev) 3217 { 3218 struct device *dev = &ofdev->dev; 3219 struct talitos_private *priv = dev_get_drvdata(dev); 3220 struct talitos_crypto_alg *t_alg, *n; 3221 int i; 3222 3223 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 3224 switch (t_alg->algt.type) { 3225 case CRYPTO_ALG_TYPE_SKCIPHER: 3226 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); 3227 break; 3228 case CRYPTO_ALG_TYPE_AEAD: 3229 crypto_unregister_aead(&t_alg->algt.alg.aead); 3230 break; 3231 case CRYPTO_ALG_TYPE_AHASH: 3232 crypto_unregister_ahash(&t_alg->algt.alg.hash); 3233 break; 3234 } 3235 list_del(&t_alg->entry); 3236 } 3237 3238 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 3239 talitos_unregister_rng(dev); 3240 3241 for (i = 0; i < 2; i++) 3242 if (priv->irq[i] > 0) 3243 free_irq(priv->irq[i], priv); 3244 3245 tasklet_kill(&priv->done_task[0]); 3246 if (priv->irq[1] > 0) 3247 tasklet_kill(&priv->done_task[1]); 3248 } 3249 3250 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 3251 struct talitos_alg_template 3252 *template) 3253 { 3254 struct talitos_private *priv = dev_get_drvdata(dev); 3255 struct talitos_crypto_alg *t_alg; 3256 struct crypto_alg *alg; 3257 3258 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), 3259 GFP_KERNEL); 3260 if (!t_alg) 3261 return ERR_PTR(-ENOMEM); 3262 3263 t_alg->algt = *template; 3264 3265 switch (t_alg->algt.type) { 3266 case CRYPTO_ALG_TYPE_SKCIPHER: 3267 alg = &t_alg->algt.alg.skcipher.base; 3268 alg->cra_exit = talitos_cra_exit; 3269 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; 3270 t_alg->algt.alg.skcipher.setkey = 3271 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; 3272 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; 3273 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; 3274 if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && 3275 DESC_TYPE(t_alg->algt.desc_hdr_template) != 3276 DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { 3277 devm_kfree(dev, t_alg); 3278 return ERR_PTR(-ENOTSUPP); 3279 } 3280 break; 3281 case CRYPTO_ALG_TYPE_AEAD: 3282 alg = &t_alg->algt.alg.aead.base; 3283 alg->cra_exit = talitos_cra_exit; 3284 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 3285 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: 3286 aead_setkey; 3287 t_alg->algt.alg.aead.encrypt = aead_encrypt; 3288 t_alg->algt.alg.aead.decrypt = aead_decrypt; 3289 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3290 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { 3291 devm_kfree(dev, t_alg); 3292 return ERR_PTR(-ENOTSUPP); 3293 } 3294 break; 3295 case CRYPTO_ALG_TYPE_AHASH: 3296 alg = &t_alg->algt.alg.hash.halg.base; 3297 alg->cra_init = talitos_cra_init_ahash; 3298 alg->cra_exit = talitos_cra_exit; 3299 t_alg->algt.alg.hash.init = ahash_init; 3300 t_alg->algt.alg.hash.update = ahash_update; 3301 t_alg->algt.alg.hash.final = ahash_final; 3302 t_alg->algt.alg.hash.finup = ahash_finup; 3303 t_alg->algt.alg.hash.digest = ahash_digest; 3304 if (!strncmp(alg->cra_name, "hmac", 4)) 3305 t_alg->algt.alg.hash.setkey = ahash_setkey; 3306 t_alg->algt.alg.hash.import = ahash_import; 3307 t_alg->algt.alg.hash.export = ahash_export; 3308 3309 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 3310 !strncmp(alg->cra_name, "hmac", 4)) { 3311 devm_kfree(dev, t_alg); 3312 return ERR_PTR(-ENOTSUPP); 3313 } 3314 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3315 (!strcmp(alg->cra_name, "sha224") || 3316 !strcmp(alg->cra_name, "hmac(sha224)"))) { 3317 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 3318 t_alg->algt.alg.hash.digest = 3319 ahash_digest_sha224_swinit; 3320 t_alg->algt.desc_hdr_template = 3321 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3322 DESC_HDR_SEL0_MDEUA | 3323 DESC_HDR_MODE0_MDEU_SHA256; 3324 } 3325 break; 3326 default: 3327 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 3328 devm_kfree(dev, t_alg); 3329 return ERR_PTR(-EINVAL); 3330 } 3331 3332 alg->cra_module = THIS_MODULE; 3333 if (t_alg->algt.priority) 3334 alg->cra_priority = t_alg->algt.priority; 3335 else 3336 alg->cra_priority = TALITOS_CRA_PRIORITY; 3337 if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH) 3338 alg->cra_alignmask = 3; 3339 else 3340 alg->cra_alignmask = 0; 3341 alg->cra_ctxsize = sizeof(struct talitos_ctx); 3342 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 3343 3344 t_alg->dev = dev; 3345 3346 return t_alg; 3347 } 3348 3349 static int talitos_probe_irq(struct platform_device *ofdev) 3350 { 3351 struct device *dev = &ofdev->dev; 3352 struct talitos_private *priv = dev_get_drvdata(dev); 3353 int err; 3354 bool is_sec1 = has_ftr_sec1(priv); 3355 3356 if (is_sec1) { 3357 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 3358 dev_driver_string(dev), priv); 3359 goto primary_out; 3360 } 3361 3362 /* single (or primary) irq line */ 3363 if (priv->irq[1] < 0) { 3364 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 3365 dev_driver_string(dev), priv); 3366 goto primary_out; 3367 } 3368 3369 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 3370 dev_driver_string(dev), priv); 3371 if (err) 3372 goto primary_out; 3373 3374 /* secondary irq line */ 3375 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 3376 dev_driver_string(dev), priv); 3377 if (err) { 3378 dev_err(dev, "failed to request secondary irq\n"); 3379 priv->irq[1] = 0; 3380 } 3381 3382 return err; 3383 3384 primary_out: 3385 if (err) { 3386 dev_err(dev, "failed to request primary irq\n"); 3387 /* neither line is requested, keep remove() from freeing them */ 3388 priv->irq[0] = 0; 3389 priv->irq[1] = 0; 3390 } 3391 3392 return err; 3393 } 3394 3395 static int talitos_probe(struct platform_device *ofdev) 3396 { 3397 struct device *dev = &ofdev->dev; 3398 struct device_node *np = ofdev->dev.of_node; 3399 struct talitos_private *priv; 3400 unsigned int num_channels; 3401 void __iomem *reg; 3402 int i, err; 3403 int stride; 3404 int irq0; 3405 int irq1; 3406 3407 if (of_property_read_u32(np, "fsl,num-channels", &num_channels)) 3408 return -EINVAL; 3409 3410 irq0 = platform_get_irq(ofdev, 0); 3411 if (irq0 < 0) 3412 return irq0; 3413 3414 irq1 = platform_get_irq_optional(ofdev, 1); 3415 if (irq1 == -EPROBE_DEFER) 3416 return irq1; 3417 3418 reg = devm_platform_ioremap_resource(ofdev, 0); 3419 if (IS_ERR(reg)) 3420 return PTR_ERR(reg); 3421 3422 priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL); 3423 if (!priv) 3424 return -ENOMEM; 3425 3426 priv->num_channels = num_channels; 3427 3428 INIT_LIST_HEAD(&priv->alg_list); 3429 3430 dev_set_drvdata(dev, priv); 3431 3432 priv->dev = dev; 3433 priv->ofdev = ofdev; 3434 3435 spin_lock_init(&priv->reg_lock); 3436 3437 priv->reg = reg; 3438 3439 /* get SEC version capabilities from device tree */ 3440 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); 3441 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); 3442 of_property_read_u32(np, "fsl,descriptor-types-mask", 3443 &priv->desc_types); 3444 3445 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 3446 !priv->exec_units || !priv->desc_types) { 3447 dev_err(dev, "invalid property data in device tree node\n"); 3448 err = -EINVAL; 3449 goto err_out; 3450 } 3451 3452 if (of_device_is_compatible(np, "fsl,sec3.0")) 3453 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 3454 3455 if (of_device_is_compatible(np, "fsl,sec2.1")) 3456 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 3457 TALITOS_FTR_SHA224_HWINIT | 3458 TALITOS_FTR_HMAC_OK; 3459 3460 if (of_device_is_compatible(np, "fsl,sec1.0")) 3461 priv->features |= TALITOS_FTR_SEC1; 3462 3463 if (of_device_is_compatible(np, "fsl,sec1.2")) { 3464 priv->reg_deu = priv->reg + TALITOS12_DEU; 3465 priv->reg_aesu = priv->reg + TALITOS12_AESU; 3466 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 3467 stride = TALITOS1_CH_STRIDE; 3468 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 3469 priv->reg_deu = priv->reg + TALITOS10_DEU; 3470 priv->reg_aesu = priv->reg + TALITOS10_AESU; 3471 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 3472 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 3473 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 3474 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 3475 stride = TALITOS1_CH_STRIDE; 3476 } else { 3477 priv->reg_deu = priv->reg + TALITOS2_DEU; 3478 priv->reg_aesu = priv->reg + TALITOS2_AESU; 3479 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 3480 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 3481 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 3482 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 3483 priv->reg_keu = priv->reg + TALITOS2_KEU; 3484 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 3485 stride = TALITOS2_CH_STRIDE; 3486 } 3487 3488 priv->irq[0] = irq0; 3489 3490 if (has_ftr_sec1(priv)) { 3491 if (priv->num_channels == 1) 3492 tasklet_init(&priv->done_task[0], talitos1_done_ch0, 3493 (unsigned long)dev); 3494 else 3495 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 3496 (unsigned long)dev); 3497 } else { 3498 priv->irq[1] = irq1; 3499 if (priv->irq[1] > 0) { 3500 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3501 (unsigned long)dev); 3502 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3503 (unsigned long)dev); 3504 } else if (priv->num_channels == 1) { 3505 tasklet_init(&priv->done_task[0], talitos2_done_ch0, 3506 (unsigned long)dev); 3507 } else { 3508 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3509 (unsigned long)dev); 3510 } 3511 } 3512 3513 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3514 3515 for (i = 0; i < priv->num_channels; i++) { 3516 priv->chan[i].reg = priv->reg + stride * (i + 1); 3517 if (priv->irq[1] < 0 || !(i & 1)) 3518 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3519 3520 spin_lock_init(&priv->chan[i].head_lock); 3521 spin_lock_init(&priv->chan[i].tail_lock); 3522 3523 priv->chan[i].fifo = devm_kcalloc(dev, 3524 priv->fifo_len, 3525 sizeof(struct talitos_request), 3526 GFP_KERNEL); 3527 if (!priv->chan[i].fifo) { 3528 dev_err(dev, "failed to allocate request fifo %d\n", i); 3529 err = -ENOMEM; 3530 goto err_noirq; 3531 } 3532 3533 atomic_set(&priv->chan[i].submit_count, 3534 -(priv->chfifo_len - 1)); 3535 } 3536 3537 dma_set_mask(dev, DMA_BIT_MASK(36)); 3538 3539 /* reset and initialize the h/w */ 3540 err = init_device(dev); 3541 if (err) { 3542 dev_err(dev, "failed to initialize device\n"); 3543 goto err_noirq; 3544 } 3545 3546 /* enable interrupts once the channel fifos and tasklets are set up */ 3547 err = talitos_probe_irq(ofdev); 3548 if (err) 3549 goto err_out; 3550 3551 /* register the RNG, if available */ 3552 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3553 err = talitos_register_rng(dev); 3554 if (err) { 3555 dev_err(dev, "failed to register hwrng: %d\n", err); 3556 goto err_out; 3557 } else 3558 dev_info(dev, "hwrng\n"); 3559 } 3560 3561 /* register crypto algorithms the device supports */ 3562 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3563 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3564 struct talitos_crypto_alg *t_alg; 3565 struct crypto_alg *alg = NULL; 3566 3567 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3568 if (IS_ERR(t_alg)) { 3569 err = PTR_ERR(t_alg); 3570 if (err == -ENOTSUPP) 3571 continue; 3572 goto err_out; 3573 } 3574 3575 switch (t_alg->algt.type) { 3576 case CRYPTO_ALG_TYPE_SKCIPHER: 3577 err = crypto_register_skcipher( 3578 &t_alg->algt.alg.skcipher); 3579 alg = &t_alg->algt.alg.skcipher.base; 3580 break; 3581 3582 case CRYPTO_ALG_TYPE_AEAD: 3583 err = crypto_register_aead( 3584 &t_alg->algt.alg.aead); 3585 alg = &t_alg->algt.alg.aead.base; 3586 break; 3587 3588 case CRYPTO_ALG_TYPE_AHASH: 3589 err = crypto_register_ahash( 3590 &t_alg->algt.alg.hash); 3591 alg = &t_alg->algt.alg.hash.halg.base; 3592 break; 3593 } 3594 if (err) { 3595 dev_err(dev, "%s alg registration failed\n", 3596 alg->cra_driver_name); 3597 devm_kfree(dev, t_alg); 3598 } else 3599 list_add_tail(&t_alg->entry, &priv->alg_list); 3600 } 3601 } 3602 if (!list_empty(&priv->alg_list)) 3603 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3604 (char *)of_get_property(np, "compatible", NULL)); 3605 3606 return 0; 3607 3608 err_noirq: 3609 /* the irq numbers are known, but the lines are not requested yet */ 3610 priv->irq[0] = 0; 3611 priv->irq[1] = 0; 3612 err_out: 3613 talitos_remove(ofdev); 3614 3615 return err; 3616 } 3617 3618 static const struct of_device_id talitos_match[] = { 3619 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3620 { 3621 .compatible = "fsl,sec1.0", 3622 }, 3623 #endif 3624 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3625 { 3626 .compatible = "fsl,sec2.0", 3627 }, 3628 #endif 3629 {}, 3630 }; 3631 MODULE_DEVICE_TABLE(of, talitos_match); 3632 3633 static struct platform_driver talitos_driver = { 3634 .driver = { 3635 .name = "talitos", 3636 .of_match_table = talitos_match, 3637 }, 3638 .probe = talitos_probe, 3639 .remove = talitos_remove, 3640 }; 3641 3642 module_platform_driver(talitos_driver); 3643 3644 MODULE_LICENSE("GPL"); 3645 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3646 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3647