1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2009-2011 Semihalf. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * CESA SRAM Memory Map: 31 * 32 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE 33 * | | 34 * | DATA | 35 * | | 36 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0) 37 * | struct cesa_sa_data | 38 * +------------------------+ 39 * | struct cesa_sa_hdesc | 40 * +------------------------+ <= sc->sc_sram_base_va 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/bus.h> 49 #include <sys/endian.h> 50 #include <sys/kernel.h> 51 #include <sys/lock.h> 52 #include <sys/mbuf.h> 53 #include <sys/module.h> 54 #include <sys/mutex.h> 55 #include <sys/rman.h> 56 57 #include <machine/bus.h> 58 #include <machine/intr.h> 59 #include <machine/resource.h> 60 #include <machine/fdt.h> 61 62 #include <dev/fdt/simplebus.h> 63 #include <dev/fdt/fdt_common.h> 64 #include <dev/ofw/ofw_bus.h> 65 #include <dev/ofw/ofw_bus_subr.h> 66 67 #include <crypto/sha1.h> 68 #include <crypto/sha2/sha256.h> 69 #include <crypto/rijndael/rijndael.h> 70 #include <opencrypto/cryptodev.h> 71 #include <opencrypto/xform.h> 72 #include "cryptodev_if.h" 73 74 #include <arm/mv/mvreg.h> 75 #include <arm/mv/mvvar.h> 76 #include "cesa.h" 77 78 static int cesa_probe(device_t); 79 static int cesa_attach(device_t); 80 static int cesa_attach_late(device_t); 81 static int cesa_detach(device_t); 82 static void cesa_intr(void *); 83 static int cesa_probesession(device_t, 84 const struct crypto_session_params *); 85 static int cesa_newsession(device_t, crypto_session_t, 86 const struct crypto_session_params *); 87 static int cesa_process(device_t, struct cryptop *, int); 88 89 static struct resource_spec cesa_res_spec[] = { 90 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 91 { SYS_RES_MEMORY, 1, RF_ACTIVE }, 92 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 93 { -1, 0 } 94 }; 95 96 static device_method_t cesa_methods[] = { 97 /* Device interface */ 98 DEVMETHOD(device_probe, cesa_probe), 99 DEVMETHOD(device_attach, cesa_attach), 100 DEVMETHOD(device_detach, cesa_detach), 101 102 /* Crypto device methods */ 103 DEVMETHOD(cryptodev_probesession, cesa_probesession), 104 DEVMETHOD(cryptodev_newsession, cesa_newsession), 105 DEVMETHOD(cryptodev_process, cesa_process), 106 107 DEVMETHOD_END 108 }; 109 110 static driver_t cesa_driver = { 111 "cesa", 112 cesa_methods, 113 sizeof (struct cesa_softc) 114 }; 115 116 DRIVER_MODULE(cesa, simplebus, cesa_driver, 0, 0); 117 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 118 119 static void 120 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 121 { 122 #ifdef DEBUG 123 device_t dev; 124 125 dev = sc->sc_dev; 126 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 127 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 128 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 129 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 130 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 131 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 132 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 133 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 134 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 135 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 136 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 137 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 138 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 139 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 140 #endif 141 } 142 143 static void 144 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 145 { 146 struct cesa_dma_mem *cdm; 147 148 if (error) 149 return; 150 151 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 152 cdm = arg; 153 cdm->cdm_paddr = segs->ds_addr; 154 } 155 156 static int 157 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 158 bus_size_t size) 159 { 160 int error; 161 162 KASSERT(cdm->cdm_vaddr == NULL, 163 ("%s(): DMA memory descriptor in use.", __func__)); 164 165 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 166 PAGE_SIZE, 0, /* alignment, boundary */ 167 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 168 BUS_SPACE_MAXADDR, /* highaddr */ 169 NULL, NULL, /* filtfunc, filtfuncarg */ 170 size, 1, /* maxsize, nsegments */ 171 size, 0, /* maxsegsz, flags */ 172 NULL, NULL, /* lockfunc, lockfuncarg */ 173 &cdm->cdm_tag); /* dmat */ 174 if (error) { 175 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 176 " %i!\n", error); 177 178 goto err1; 179 } 180 181 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 182 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 183 if (error) { 184 device_printf(sc->sc_dev, "failed to allocate DMA safe" 185 " memory, error %i!\n", error); 186 187 goto err2; 188 } 189 190 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 191 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 192 if (error) { 193 device_printf(sc->sc_dev, "cannot get address of the DMA" 194 " memory, error %i\n", error); 195 196 goto err3; 197 } 198 199 return (0); 200 err3: 201 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 202 err2: 203 bus_dma_tag_destroy(cdm->cdm_tag); 204 err1: 205 cdm->cdm_vaddr = NULL; 206 return (error); 207 } 208 209 static void 210 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 211 { 212 213 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 214 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 215 bus_dma_tag_destroy(cdm->cdm_tag); 216 cdm->cdm_vaddr = NULL; 217 } 218 219 static void 220 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 221 { 222 223 /* Sync only if dma memory is valid */ 224 if (cdm->cdm_vaddr != NULL) 225 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 226 } 227 228 static void 229 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 230 { 231 232 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 233 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 234 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 235 } 236 237 static struct cesa_request * 238 cesa_alloc_request(struct cesa_softc *sc) 239 { 240 struct cesa_request *cr; 241 242 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 243 if (!cr) 244 return (NULL); 245 246 STAILQ_INIT(&cr->cr_tdesc); 247 STAILQ_INIT(&cr->cr_sdesc); 248 249 return (cr); 250 } 251 252 static void 253 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 254 { 255 256 /* Free TDMA descriptors assigned to this request */ 257 CESA_LOCK(sc, tdesc); 258 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 259 CESA_UNLOCK(sc, tdesc); 260 261 /* Free SA descriptors assigned to this request */ 262 CESA_LOCK(sc, sdesc); 263 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 264 CESA_UNLOCK(sc, sdesc); 265 266 /* Unload DMA memory associated with request */ 267 if (cr->cr_dmap_loaded) { 268 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 269 cr->cr_dmap_loaded = 0; 270 } 271 272 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 273 } 274 275 static void 276 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 277 { 278 279 CESA_LOCK(sc, requests); 280 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 281 CESA_UNLOCK(sc, requests); 282 } 283 284 static struct cesa_tdma_desc * 285 cesa_alloc_tdesc(struct cesa_softc *sc) 286 { 287 struct cesa_tdma_desc *ctd; 288 289 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 290 291 if (!ctd) 292 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 293 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 294 295 return (ctd); 296 } 297 298 static struct cesa_sa_desc * 299 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 300 { 301 struct cesa_sa_desc *csd; 302 303 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 304 if (!csd) { 305 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 306 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 307 return (NULL); 308 } 309 310 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 311 312 /* Fill-in SA descriptor with default values */ 313 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 314 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 315 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 316 csd->csd_cshd->cshd_enc_src = 0; 317 csd->csd_cshd->cshd_enc_dst = 0; 318 csd->csd_cshd->cshd_enc_dlen = 0; 319 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 320 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 321 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 322 csd->csd_cshd->cshd_mac_src = 0; 323 csd->csd_cshd->cshd_mac_dlen = 0; 324 325 return (csd); 326 } 327 328 static struct cesa_tdma_desc * 329 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 330 bus_size_t size) 331 { 332 struct cesa_tdma_desc *ctd; 333 334 ctd = cesa_alloc_tdesc(sc); 335 if (!ctd) 336 return (NULL); 337 338 ctd->ctd_cthd->cthd_dst = dst; 339 ctd->ctd_cthd->cthd_src = src; 340 ctd->ctd_cthd->cthd_byte_count = size; 341 342 /* Handle special control packet */ 343 if (size != 0) 344 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 345 else 346 ctd->ctd_cthd->cthd_flags = 0; 347 348 return (ctd); 349 } 350 351 static struct cesa_tdma_desc * 352 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 353 { 354 355 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa + 356 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 357 sizeof(struct cesa_sa_data))); 358 } 359 360 static struct cesa_tdma_desc * 361 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 362 { 363 364 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa + 365 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 366 } 367 368 static struct cesa_tdma_desc * 369 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 370 { 371 372 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr, 373 sizeof(struct cesa_sa_hdesc))); 374 } 375 376 static void 377 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 378 { 379 struct cesa_tdma_desc *ctd_prev; 380 381 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 382 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 383 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 384 } 385 386 ctd->ctd_cthd->cthd_next = 0; 387 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 388 } 389 390 static int 391 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 392 struct cesa_packet *cp, struct cesa_sa_desc *csd) 393 { 394 struct cesa_tdma_desc *ctd, *tmp; 395 396 /* Copy SA descriptor for this packet */ 397 ctd = cesa_tdma_copy_sdesc(sc, csd); 398 if (!ctd) 399 return (ENOMEM); 400 401 cesa_append_tdesc(cr, ctd); 402 403 /* Copy data to be processed */ 404 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 405 cesa_append_tdesc(cr, ctd); 406 STAILQ_INIT(&cp->cp_copyin); 407 408 /* Insert control descriptor */ 409 ctd = cesa_tdma_copy(sc, 0, 0, 0); 410 if (!ctd) 411 return (ENOMEM); 412 413 cesa_append_tdesc(cr, ctd); 414 415 /* Copy back results */ 416 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 417 cesa_append_tdesc(cr, ctd); 418 STAILQ_INIT(&cp->cp_copyout); 419 420 return (0); 421 } 422 423 static void 424 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 425 { 426 union authctx auth_ctx; 427 uint32_t *hout; 428 uint32_t *hin; 429 int i; 430 431 hin = (uint32_t *)cs->cs_hiv_in; 432 hout = (uint32_t *)cs->cs_hiv_out; 433 434 switch (alg) { 435 case CRYPTO_SHA1_HMAC: 436 hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx); 437 memcpy(hin, auth_ctx.sha1ctx.h.b32, 438 sizeof(auth_ctx.sha1ctx.h.b32)); 439 hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx); 440 memcpy(hout, auth_ctx.sha1ctx.h.b32, 441 sizeof(auth_ctx.sha1ctx.h.b32)); 442 break; 443 case CRYPTO_SHA2_256_HMAC: 444 hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen, 445 &auth_ctx); 446 memcpy(hin, auth_ctx.sha256ctx.state, 447 sizeof(auth_ctx.sha256ctx.state)); 448 hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen, 449 &auth_ctx); 450 memcpy(hout, auth_ctx.sha256ctx.state, 451 sizeof(auth_ctx.sha256ctx.state)); 452 break; 453 default: 454 panic("shouldn't get here"); 455 } 456 457 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 458 hin[i] = htobe32(hin[i]); 459 hout[i] = htobe32(hout[i]); 460 } 461 explicit_bzero(&auth_ctx, sizeof(auth_ctx)); 462 } 463 464 static int 465 cesa_prep_aes_key(struct cesa_session *cs, 466 const struct crypto_session_params *csp) 467 { 468 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 469 uint32_t *dkey; 470 int i; 471 472 rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8); 473 474 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 475 dkey = (uint32_t *)cs->cs_aes_dkey; 476 477 switch (csp->csp_cipher_klen) { 478 case 16: 479 cs->cs_config |= CESA_CSH_AES_KLEN_128; 480 for (i = 0; i < 4; i++) 481 *dkey++ = htobe32(ek[4 * 10 + i]); 482 break; 483 case 24: 484 cs->cs_config |= CESA_CSH_AES_KLEN_192; 485 for (i = 0; i < 4; i++) 486 *dkey++ = htobe32(ek[4 * 12 + i]); 487 for (i = 0; i < 2; i++) 488 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 489 break; 490 case 32: 491 cs->cs_config |= CESA_CSH_AES_KLEN_256; 492 for (i = 0; i < 4; i++) 493 *dkey++ = htobe32(ek[4 * 14 + i]); 494 for (i = 0; i < 4; i++) 495 *dkey++ = htobe32(ek[4 * 13 + i]); 496 break; 497 default: 498 return (EINVAL); 499 } 500 501 return (0); 502 } 503 504 static void 505 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 506 { 507 508 cp->cp_size = size; 509 cp->cp_offset = 0; 510 STAILQ_INIT(&cp->cp_copyin); 511 STAILQ_INIT(&cp->cp_copyout); 512 } 513 514 static int 515 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 516 bus_dma_segment_t *seg) 517 { 518 struct cesa_tdma_desc *ctd; 519 unsigned int bsize; 520 521 /* Calculate size of block copy */ 522 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 523 524 if (bsize > 0) { 525 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa + 526 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 527 if (!ctd) 528 return (-ENOMEM); 529 530 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 531 532 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa + 533 CESA_DATA(cp->cp_offset), bsize); 534 if (!ctd) 535 return (-ENOMEM); 536 537 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 538 539 seg->ds_len -= bsize; 540 seg->ds_addr += bsize; 541 cp->cp_offset += bsize; 542 } 543 544 return (bsize); 545 } 546 547 static void 548 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 549 { 550 unsigned int mpsize, fragmented; 551 unsigned int mlen, mskip, tmlen; 552 struct cesa_chain_info *cci; 553 unsigned int elen, eskip; 554 unsigned int skip, len; 555 struct cesa_sa_desc *csd; 556 struct cesa_request *cr; 557 struct cryptop *crp; 558 struct cesa_softc *sc; 559 struct cesa_packet cp; 560 bus_dma_segment_t seg; 561 uint32_t config; 562 int size; 563 564 cci = arg; 565 sc = cci->cci_sc; 566 cr = cci->cci_cr; 567 crp = cr->cr_crp; 568 569 if (error) { 570 cci->cci_error = error; 571 return; 572 } 573 574 /* 575 * Only do a combined op if the AAD is adjacent to the payload 576 * and the AAD length is a multiple of the IV length. The 577 * checks against 'config' are to avoid recursing when the 578 * logic below invokes separate operations. 579 */ 580 config = cci->cci_config; 581 if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC || 582 (config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) && 583 crp->crp_aad_length != 0 && 584 (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) { 585 /* 586 * Data alignment in the request does not meet CESA requiremnts 587 * for combined encryption/decryption and hashing. We have to 588 * split the request to separate operations and process them 589 * one by one. 590 */ 591 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 592 config &= ~CESA_CSHD_OP_MASK; 593 594 cci->cci_config = config | CESA_CSHD_MAC; 595 cesa_create_chain_cb(cci, segs, nseg, 0); 596 597 cci->cci_config = config | CESA_CSHD_ENC; 598 cesa_create_chain_cb(cci, segs, nseg, 0); 599 } else { 600 config &= ~CESA_CSHD_OP_MASK; 601 602 cci->cci_config = config | CESA_CSHD_ENC; 603 cesa_create_chain_cb(cci, segs, nseg, 0); 604 605 cci->cci_config = config | CESA_CSHD_MAC; 606 cesa_create_chain_cb(cci, segs, nseg, 0); 607 } 608 609 return; 610 } 611 612 mskip = mlen = eskip = elen = 0; 613 614 if (crp->crp_aad_length == 0) { 615 skip = crp->crp_payload_start; 616 len = crp->crp_payload_length; 617 switch (config & CESA_CSHD_OP_MASK) { 618 case CESA_CSHD_ENC: 619 eskip = skip; 620 elen = len; 621 break; 622 case CESA_CSHD_MAC: 623 mskip = skip; 624 mlen = len; 625 break; 626 default: 627 eskip = skip; 628 elen = len; 629 mskip = skip; 630 mlen = len; 631 break; 632 } 633 } else { 634 /* 635 * For an encryption-only separate request, only 636 * process the payload. For combined requests and 637 * hash-only requests, process the entire region. 638 */ 639 switch (config & CESA_CSHD_OP_MASK) { 640 case CESA_CSHD_ENC: 641 skip = crp->crp_payload_start; 642 len = crp->crp_payload_length; 643 eskip = skip; 644 elen = len; 645 break; 646 case CESA_CSHD_MAC: 647 skip = crp->crp_aad_start; 648 len = crp->crp_aad_length + crp->crp_payload_length; 649 mskip = skip; 650 mlen = len; 651 break; 652 default: 653 skip = crp->crp_aad_start; 654 len = crp->crp_aad_length + crp->crp_payload_length; 655 mskip = skip; 656 mlen = len; 657 eskip = crp->crp_payload_start; 658 elen = crp->crp_payload_length; 659 break; 660 } 661 } 662 663 tmlen = mlen; 664 fragmented = 0; 665 mpsize = CESA_MAX_PACKET_SIZE; 666 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 667 668 /* Start first packet in chain */ 669 cesa_start_packet(&cp, MIN(mpsize, len)); 670 671 while (nseg-- && len > 0) { 672 seg = *(segs++); 673 674 /* 675 * Skip data in buffer on which neither ENC nor MAC operation 676 * is requested. 677 */ 678 if (skip > 0) { 679 size = MIN(skip, seg.ds_len); 680 skip -= size; 681 682 seg.ds_addr += size; 683 seg.ds_len -= size; 684 685 if (eskip > 0) 686 eskip -= size; 687 688 if (mskip > 0) 689 mskip -= size; 690 691 if (seg.ds_len == 0) 692 continue; 693 } 694 695 while (1) { 696 /* 697 * Fill in current packet with data. Break if there is 698 * no more data in current DMA segment or an error 699 * occurred. 700 */ 701 size = cesa_fill_packet(sc, &cp, &seg); 702 if (size <= 0) { 703 error = -size; 704 break; 705 } 706 707 len -= size; 708 709 /* If packet is full, append it to the chain */ 710 if (cp.cp_size == cp.cp_offset) { 711 csd = cesa_alloc_sdesc(sc, cr); 712 if (!csd) { 713 error = ENOMEM; 714 break; 715 } 716 717 /* Create SA descriptor for this packet */ 718 csd->csd_cshd->cshd_config = cci->cci_config; 719 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 720 721 /* 722 * Enable fragmentation if request will not fit 723 * into one packet. 724 */ 725 if (len > 0) { 726 if (!fragmented) { 727 fragmented = 1; 728 csd->csd_cshd->cshd_config |= 729 CESA_CSHD_FRAG_FIRST; 730 } else 731 csd->csd_cshd->cshd_config |= 732 CESA_CSHD_FRAG_MIDDLE; 733 } else if (fragmented) 734 csd->csd_cshd->cshd_config |= 735 CESA_CSHD_FRAG_LAST; 736 737 if (eskip < cp.cp_size && elen > 0) { 738 csd->csd_cshd->cshd_enc_src = 739 CESA_DATA(eskip); 740 csd->csd_cshd->cshd_enc_dst = 741 CESA_DATA(eskip); 742 csd->csd_cshd->cshd_enc_dlen = 743 MIN(elen, cp.cp_size - eskip); 744 } 745 746 if (mskip < cp.cp_size && mlen > 0) { 747 csd->csd_cshd->cshd_mac_src = 748 CESA_DATA(mskip); 749 csd->csd_cshd->cshd_mac_dlen = 750 MIN(mlen, cp.cp_size - mskip); 751 } 752 753 elen -= csd->csd_cshd->cshd_enc_dlen; 754 eskip -= MIN(eskip, cp.cp_size); 755 mlen -= csd->csd_cshd->cshd_mac_dlen; 756 mskip -= MIN(mskip, cp.cp_size); 757 758 cesa_dump_cshd(sc, csd->csd_cshd); 759 760 /* Append packet to the request */ 761 error = cesa_append_packet(sc, cr, &cp, csd); 762 if (error) 763 break; 764 765 /* Start a new packet, as current is full */ 766 cesa_start_packet(&cp, MIN(mpsize, len)); 767 } 768 } 769 770 if (error) 771 break; 772 } 773 774 if (error) { 775 /* 776 * Move all allocated resources to the request. They will be 777 * freed later. 778 */ 779 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 780 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 781 cci->cci_error = error; 782 } 783 } 784 785 static int 786 cesa_create_chain(struct cesa_softc *sc, 787 const struct crypto_session_params *csp, struct cesa_request *cr) 788 { 789 struct cesa_chain_info cci; 790 struct cesa_tdma_desc *ctd; 791 uint32_t config; 792 int error; 793 794 error = 0; 795 CESA_LOCK_ASSERT(sc, sessions); 796 797 /* Create request metadata */ 798 if (csp->csp_cipher_klen != 0) { 799 if (csp->csp_cipher_alg == CRYPTO_AES_CBC && 800 !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) 801 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 802 csp->csp_cipher_klen); 803 else 804 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 805 csp->csp_cipher_klen); 806 } 807 808 if (csp->csp_auth_klen != 0) { 809 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 810 CESA_MAX_HASH_LEN); 811 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 812 CESA_MAX_HASH_LEN); 813 } 814 815 ctd = cesa_tdma_copyin_sa_data(sc, cr); 816 if (!ctd) 817 return (ENOMEM); 818 819 cesa_append_tdesc(cr, ctd); 820 821 /* Prepare SA configuration */ 822 config = cr->cr_cs->cs_config; 823 824 if (csp->csp_cipher_alg != 0 && 825 !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) 826 config |= CESA_CSHD_DECRYPT; 827 switch (csp->csp_mode) { 828 case CSP_MODE_CIPHER: 829 config |= CESA_CSHD_ENC; 830 break; 831 case CSP_MODE_DIGEST: 832 config |= CESA_CSHD_MAC; 833 break; 834 case CSP_MODE_ETA: 835 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 836 CESA_CSHD_ENC_AND_MAC; 837 break; 838 } 839 840 /* Create data packets */ 841 cci.cci_sc = sc; 842 cci.cci_cr = cr; 843 cci.cci_config = config; 844 cci.cci_error = 0; 845 846 error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp, 847 cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT); 848 849 if (!error) 850 cr->cr_dmap_loaded = 1; 851 852 if (cci.cci_error) 853 error = cci.cci_error; 854 855 if (error) 856 return (error); 857 858 /* Read back request metadata */ 859 ctd = cesa_tdma_copyout_sa_data(sc, cr); 860 if (!ctd) 861 return (ENOMEM); 862 863 cesa_append_tdesc(cr, ctd); 864 865 return (0); 866 } 867 868 static void 869 cesa_execute(struct cesa_softc *sc) 870 { 871 struct cesa_tdma_desc *prev_ctd, *ctd; 872 struct cesa_request *prev_cr, *cr; 873 874 CESA_LOCK(sc, requests); 875 876 /* 877 * If ready list is empty, there is nothing to execute. If queued list 878 * is not empty, the hardware is busy and we cannot start another 879 * execution. 880 */ 881 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 882 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 883 CESA_UNLOCK(sc, requests); 884 return; 885 } 886 887 /* Move all ready requests to queued list */ 888 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 889 STAILQ_INIT(&sc->sc_ready_requests); 890 891 /* Create one execution chain from all requests on the list */ 892 if (STAILQ_FIRST(&sc->sc_queued_requests) != 893 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 894 prev_cr = NULL; 895 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 896 BUS_DMASYNC_POSTWRITE); 897 898 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 899 if (prev_cr) { 900 ctd = STAILQ_FIRST(&cr->cr_tdesc); 901 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 902 cesa_tdma_desc, ctd_stq); 903 904 prev_ctd->ctd_cthd->cthd_next = 905 ctd->ctd_cthd_paddr; 906 } 907 908 prev_cr = cr; 909 } 910 911 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 912 BUS_DMASYNC_PREWRITE); 913 } 914 915 /* Start chain execution in hardware */ 916 cr = STAILQ_FIRST(&sc->sc_queued_requests); 917 ctd = STAILQ_FIRST(&cr->cr_tdesc); 918 919 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 920 921 if (sc->sc_soc_id == MV_DEV_88F6828 || 922 sc->sc_soc_id == MV_DEV_88F6820 || 923 sc->sc_soc_id == MV_DEV_88F6810) 924 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2); 925 else 926 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 927 928 CESA_UNLOCK(sc, requests); 929 } 930 931 static int 932 cesa_setup_sram(struct cesa_softc *sc) 933 { 934 phandle_t sram_node; 935 ihandle_t sram_ihandle; 936 pcell_t sram_handle, sram_reg[2]; 937 void *sram_va; 938 int rv; 939 940 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 941 (void *)&sram_handle, sizeof(sram_handle)); 942 if (rv <= 0) 943 return (rv); 944 945 sram_ihandle = (ihandle_t)sram_handle; 946 sram_node = OF_instance_to_package(sram_ihandle); 947 948 rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg)); 949 if (rv <= 0) 950 return (rv); 951 952 sc->sc_sram_base_pa = sram_reg[0]; 953 /* Store SRAM size to be able to unmap in detach() */ 954 sc->sc_sram_size = sram_reg[1]; 955 956 if (sc->sc_soc_id != MV_DEV_88F6828 && 957 sc->sc_soc_id != MV_DEV_88F6820 && 958 sc->sc_soc_id != MV_DEV_88F6810) 959 return (0); 960 961 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 962 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 963 if (sram_va == NULL) 964 return (ENOMEM); 965 sc->sc_sram_base_va = sram_va; 966 967 return (0); 968 } 969 970 /* 971 * Function: device_from_node 972 * This function returns appropriate device_t to phandle_t 973 * Parameters: 974 * root - device where you want to start search 975 * if you provide NULL here, function will take 976 * "root0" device as root. 977 * node - we are checking every device_t to be 978 * appropriate with this. 979 */ 980 static device_t 981 device_from_node(device_t root, phandle_t node) 982 { 983 device_t *children, retval; 984 int nkid, i; 985 986 /* Nothing matches no node */ 987 if (node == -1) 988 return (NULL); 989 990 if (root == NULL) 991 /* Get root of device tree */ 992 if ((root = device_lookup_by_name("root0")) == NULL) 993 return (NULL); 994 995 if (device_get_children(root, &children, &nkid) != 0) 996 return (NULL); 997 998 retval = NULL; 999 for (i = 0; i < nkid; i++) { 1000 /* Check if device and node matches */ 1001 if (OFW_BUS_GET_NODE(root, children[i]) == node) { 1002 retval = children[i]; 1003 break; 1004 } 1005 /* or go deeper */ 1006 if ((retval = device_from_node(children[i], node)) != NULL) 1007 break; 1008 } 1009 free(children, M_TEMP); 1010 1011 return (retval); 1012 } 1013 1014 static int 1015 cesa_setup_sram_armada(struct cesa_softc *sc) 1016 { 1017 phandle_t sram_node; 1018 ihandle_t sram_ihandle; 1019 pcell_t sram_handle[2]; 1020 void *sram_va; 1021 int rv, j; 1022 struct resource_list rl; 1023 struct resource_list_entry *rle; 1024 struct simplebus_softc *ssc; 1025 device_t sdev; 1026 1027 /* Get refs to SRAMS from CESA node */ 1028 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams", 1029 (void *)sram_handle, sizeof(sram_handle)); 1030 if (rv <= 0) 1031 return (rv); 1032 1033 if (sc->sc_cesa_engine_id >= 2) 1034 return (ENXIO); 1035 1036 /* Get SRAM node on the basis of sc_cesa_engine_id */ 1037 sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id]; 1038 sram_node = OF_instance_to_package(sram_ihandle); 1039 1040 /* Get device_t of simplebus (sram_node parent) */ 1041 sdev = device_from_node(NULL, OF_parent(sram_node)); 1042 if (!sdev) 1043 return (ENXIO); 1044 1045 ssc = device_get_softc(sdev); 1046 1047 resource_list_init(&rl); 1048 /* Parse reg property to resource list */ 1049 ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells, 1050 ssc->scells, &rl); 1051 1052 /* We expect only one resource */ 1053 rle = resource_list_find(&rl, SYS_RES_MEMORY, 0); 1054 if (rle == NULL) 1055 return (ENXIO); 1056 1057 /* Remap through ranges property */ 1058 for (j = 0; j < ssc->nranges; j++) { 1059 if (rle->start >= ssc->ranges[j].bus && 1060 rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) { 1061 rle->start -= ssc->ranges[j].bus; 1062 rle->start += ssc->ranges[j].host; 1063 rle->end -= ssc->ranges[j].bus; 1064 rle->end += ssc->ranges[j].host; 1065 } 1066 } 1067 1068 sc->sc_sram_base_pa = rle->start; 1069 sc->sc_sram_size = rle->count; 1070 1071 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 1072 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 1073 if (sram_va == NULL) 1074 return (ENOMEM); 1075 sc->sc_sram_base_va = sram_va; 1076 1077 return (0); 1078 } 1079 1080 struct ofw_compat_data cesa_devices[] = { 1081 { "mrvl,cesa", (uintptr_t)true }, 1082 { "marvell,armada-38x-crypto", (uintptr_t)true }, 1083 { NULL, 0 } 1084 }; 1085 1086 static int 1087 cesa_probe(device_t dev) 1088 { 1089 1090 if (!ofw_bus_status_okay(dev)) 1091 return (ENXIO); 1092 1093 if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data) 1094 return (ENXIO); 1095 1096 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 1097 "Accelerator"); 1098 1099 return (BUS_PROBE_DEFAULT); 1100 } 1101 1102 static int 1103 cesa_attach(device_t dev) 1104 { 1105 static int engine_idx = 0; 1106 struct simplebus_devinfo *ndi; 1107 struct resource_list *rl; 1108 struct cesa_softc *sc; 1109 1110 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) 1111 return (cesa_attach_late(dev)); 1112 1113 /* 1114 * Get simplebus_devinfo which contains 1115 * resource list filled with adresses and 1116 * interrupts read form FDT. 1117 * Let's correct it by splitting resources 1118 * for each engine. 1119 */ 1120 if ((ndi = device_get_ivars(dev)) == NULL) 1121 return (ENXIO); 1122 1123 rl = &ndi->rl; 1124 1125 switch (engine_idx) { 1126 case 0: 1127 /* Update regs values */ 1128 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA0_TDMA_ADDR, 1129 CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); 1130 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA0_CESA_ADDR, 1131 CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); 1132 1133 /* Remove unused interrupt */ 1134 resource_list_delete(rl, SYS_RES_IRQ, 1); 1135 break; 1136 1137 case 1: 1138 /* Update regs values */ 1139 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA1_TDMA_ADDR, 1140 CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); 1141 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA1_CESA_ADDR, 1142 CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); 1143 1144 /* Remove unused interrupt */ 1145 resource_list_delete(rl, SYS_RES_IRQ, 0); 1146 resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0; 1147 break; 1148 1149 default: 1150 device_printf(dev, "Bad cesa engine_idx\n"); 1151 return (ENXIO); 1152 } 1153 1154 sc = device_get_softc(dev); 1155 sc->sc_cesa_engine_id = engine_idx; 1156 1157 /* 1158 * Call simplebus_add_device only once. 1159 * It will create second cesa driver instance 1160 * with the same FDT node as first instance. 1161 * When second driver reach this function, 1162 * it will be configured to use second cesa engine 1163 */ 1164 if (engine_idx == 0) 1165 simplebus_add_device(device_get_parent(dev), ofw_bus_get_node(dev), 1166 0, "cesa", 1, NULL); 1167 1168 engine_idx++; 1169 1170 return (cesa_attach_late(dev)); 1171 } 1172 1173 static int 1174 cesa_attach_late(device_t dev) 1175 { 1176 struct cesa_softc *sc; 1177 uint32_t d, r, val; 1178 int error; 1179 int i; 1180 1181 sc = device_get_softc(dev); 1182 sc->sc_blocked = 0; 1183 sc->sc_error = 0; 1184 sc->sc_dev = dev; 1185 1186 soc_id(&d, &r); 1187 1188 switch (d) { 1189 case MV_DEV_88F6281: 1190 case MV_DEV_88F6282: 1191 /* Check if CESA peripheral device has power turned on */ 1192 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) == 1193 CPU_PM_CTRL_CRYPTO) { 1194 device_printf(dev, "not powered on\n"); 1195 return (ENXIO); 1196 } 1197 sc->sc_tperr = 0; 1198 break; 1199 case MV_DEV_88F6828: 1200 case MV_DEV_88F6820: 1201 case MV_DEV_88F6810: 1202 sc->sc_tperr = 0; 1203 break; 1204 case MV_DEV_MV78100: 1205 case MV_DEV_MV78100_Z0: 1206 /* Check if CESA peripheral device has power turned on */ 1207 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != 1208 CPU_PM_CTRL_CRYPTO) { 1209 device_printf(dev, "not powered on\n"); 1210 return (ENXIO); 1211 } 1212 sc->sc_tperr = CESA_ICR_TPERR; 1213 break; 1214 default: 1215 return (ENXIO); 1216 } 1217 1218 sc->sc_soc_id = d; 1219 1220 /* Initialize mutexes */ 1221 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1222 "CESA Shared Data", MTX_DEF); 1223 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1224 "CESA TDMA Descriptors Pool", MTX_DEF); 1225 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1226 "CESA SA Descriptors Pool", MTX_DEF); 1227 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1228 "CESA Requests Pool", MTX_DEF); 1229 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1230 "CESA Sessions Pool", MTX_DEF); 1231 1232 /* Allocate I/O and IRQ resources */ 1233 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1234 if (error) { 1235 device_printf(dev, "could not allocate resources\n"); 1236 goto err0; 1237 } 1238 1239 /* Acquire SRAM base address */ 1240 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) 1241 error = cesa_setup_sram(sc); 1242 else 1243 error = cesa_setup_sram_armada(sc); 1244 1245 if (error) { 1246 device_printf(dev, "could not setup SRAM\n"); 1247 goto err1; 1248 } 1249 1250 /* Setup interrupt handler */ 1251 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET | 1252 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie)); 1253 if (error) { 1254 device_printf(dev, "could not setup engine completion irq\n"); 1255 goto err2; 1256 } 1257 1258 /* Create DMA tag for processed data */ 1259 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1260 1, 0, /* alignment, boundary */ 1261 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1262 BUS_SPACE_MAXADDR, /* highaddr */ 1263 NULL, NULL, /* filtfunc, filtfuncarg */ 1264 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1265 CESA_MAX_FRAGMENTS, /* nsegments */ 1266 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1267 NULL, NULL, /* lockfunc, lockfuncarg */ 1268 &sc->sc_data_dtag); /* dmat */ 1269 if (error) 1270 goto err3; 1271 1272 /* Initialize data structures: TDMA Descriptors Pool */ 1273 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1274 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1275 if (error) 1276 goto err4; 1277 1278 STAILQ_INIT(&sc->sc_free_tdesc); 1279 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1280 sc->sc_tdesc[i].ctd_cthd = 1281 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1282 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1283 (i * sizeof(struct cesa_tdma_hdesc)); 1284 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1285 ctd_stq); 1286 } 1287 1288 /* Initialize data structures: SA Descriptors Pool */ 1289 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1290 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1291 if (error) 1292 goto err5; 1293 1294 STAILQ_INIT(&sc->sc_free_sdesc); 1295 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1296 sc->sc_sdesc[i].csd_cshd = 1297 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1298 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1299 (i * sizeof(struct cesa_sa_hdesc)); 1300 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1301 csd_stq); 1302 } 1303 1304 /* Initialize data structures: Requests Pool */ 1305 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1306 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1307 if (error) 1308 goto err6; 1309 1310 STAILQ_INIT(&sc->sc_free_requests); 1311 STAILQ_INIT(&sc->sc_ready_requests); 1312 STAILQ_INIT(&sc->sc_queued_requests); 1313 for (i = 0; i < CESA_REQUESTS; i++) { 1314 sc->sc_requests[i].cr_csd = 1315 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1316 sc->sc_requests[i].cr_csd_paddr = 1317 sc->sc_requests_cdm.cdm_paddr + 1318 (i * sizeof(struct cesa_sa_data)); 1319 1320 /* Preallocate DMA maps */ 1321 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1322 &sc->sc_requests[i].cr_dmap); 1323 if (error && i > 0) { 1324 i--; 1325 do { 1326 bus_dmamap_destroy(sc->sc_data_dtag, 1327 sc->sc_requests[i].cr_dmap); 1328 } while (i--); 1329 1330 goto err7; 1331 } 1332 1333 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1334 cr_stq); 1335 } 1336 1337 /* 1338 * Initialize TDMA: 1339 * - Burst limit: 128 bytes, 1340 * - Outstanding reads enabled, 1341 * - No byte-swap. 1342 */ 1343 val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1344 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE; 1345 1346 if (sc->sc_soc_id == MV_DEV_88F6828 || 1347 sc->sc_soc_id == MV_DEV_88F6820 || 1348 sc->sc_soc_id == MV_DEV_88F6810) 1349 val |= CESA_TDMA_NUM_OUTSTAND; 1350 1351 CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val); 1352 1353 /* 1354 * Initialize SA: 1355 * - SA descriptor is present at beginning of CESA SRAM, 1356 * - Multi-packet chain mode, 1357 * - Cooperation with TDMA enabled. 1358 */ 1359 CESA_REG_WRITE(sc, CESA_SA_DPR, 0); 1360 CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1361 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1362 1363 /* Unmask interrupts */ 1364 CESA_REG_WRITE(sc, CESA_ICR, 0); 1365 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1366 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1367 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1368 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1369 CESA_TDMA_EMR_DATA_ERROR); 1370 1371 /* Register in OCF */ 1372 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session), 1373 CRYPTOCAP_F_HARDWARE); 1374 if (sc->sc_cid < 0) { 1375 device_printf(dev, "could not get crypto driver id\n"); 1376 goto err8; 1377 } 1378 1379 return (0); 1380 err8: 1381 for (i = 0; i < CESA_REQUESTS; i++) 1382 bus_dmamap_destroy(sc->sc_data_dtag, 1383 sc->sc_requests[i].cr_dmap); 1384 err7: 1385 cesa_free_dma_mem(&sc->sc_requests_cdm); 1386 err6: 1387 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1388 err5: 1389 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1390 err4: 1391 bus_dma_tag_destroy(sc->sc_data_dtag); 1392 err3: 1393 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1394 err2: 1395 if (sc->sc_soc_id == MV_DEV_88F6828 || 1396 sc->sc_soc_id == MV_DEV_88F6820 || 1397 sc->sc_soc_id == MV_DEV_88F6810) 1398 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1399 err1: 1400 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1401 err0: 1402 mtx_destroy(&sc->sc_sessions_lock); 1403 mtx_destroy(&sc->sc_requests_lock); 1404 mtx_destroy(&sc->sc_sdesc_lock); 1405 mtx_destroy(&sc->sc_tdesc_lock); 1406 mtx_destroy(&sc->sc_sc_lock); 1407 return (ENXIO); 1408 } 1409 1410 static int 1411 cesa_detach(device_t dev) 1412 { 1413 struct cesa_softc *sc; 1414 int i; 1415 1416 sc = device_get_softc(dev); 1417 1418 /* TODO: Wait for queued requests completion before shutdown. */ 1419 1420 /* Mask interrupts */ 1421 CESA_REG_WRITE(sc, CESA_ICM, 0); 1422 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0); 1423 1424 /* Unregister from OCF */ 1425 crypto_unregister_all(sc->sc_cid); 1426 1427 /* Free DMA Maps */ 1428 for (i = 0; i < CESA_REQUESTS; i++) 1429 bus_dmamap_destroy(sc->sc_data_dtag, 1430 sc->sc_requests[i].cr_dmap); 1431 1432 /* Free DMA Memory */ 1433 cesa_free_dma_mem(&sc->sc_requests_cdm); 1434 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1435 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1436 1437 /* Free DMA Tag */ 1438 bus_dma_tag_destroy(sc->sc_data_dtag); 1439 1440 /* Stop interrupt */ 1441 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1442 1443 /* Relase I/O and IRQ resources */ 1444 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1445 1446 /* Unmap SRAM memory */ 1447 if (sc->sc_soc_id == MV_DEV_88F6828 || 1448 sc->sc_soc_id == MV_DEV_88F6820 || 1449 sc->sc_soc_id == MV_DEV_88F6810) 1450 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1451 1452 /* Destroy mutexes */ 1453 mtx_destroy(&sc->sc_sessions_lock); 1454 mtx_destroy(&sc->sc_requests_lock); 1455 mtx_destroy(&sc->sc_sdesc_lock); 1456 mtx_destroy(&sc->sc_tdesc_lock); 1457 mtx_destroy(&sc->sc_sc_lock); 1458 1459 return (0); 1460 } 1461 1462 static void 1463 cesa_intr(void *arg) 1464 { 1465 STAILQ_HEAD(, cesa_request) requests; 1466 struct cesa_request *cr, *tmp; 1467 struct cesa_softc *sc; 1468 uint32_t ecr, icr; 1469 uint8_t hash[HASH_MAX_LEN]; 1470 int blocked; 1471 1472 sc = arg; 1473 1474 /* Ack interrupt */ 1475 ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR); 1476 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1477 icr = CESA_REG_READ(sc, CESA_ICR); 1478 CESA_REG_WRITE(sc, CESA_ICR, 0); 1479 1480 /* Check for TDMA errors */ 1481 if (ecr & CESA_TDMA_ECR_MISS) { 1482 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1483 sc->sc_error = EIO; 1484 } 1485 1486 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1487 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1488 sc->sc_error = EIO; 1489 } 1490 1491 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1492 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1493 sc->sc_error = EIO; 1494 } 1495 1496 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1497 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1498 sc->sc_error = EIO; 1499 } 1500 1501 /* Check for CESA errors */ 1502 if (icr & sc->sc_tperr) { 1503 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1504 sc->sc_error = EIO; 1505 } 1506 1507 /* If there is nothing more to do, return */ 1508 if ((icr & CESA_ICR_ACCTDMA) == 0) 1509 return; 1510 1511 /* Get all finished requests */ 1512 CESA_LOCK(sc, requests); 1513 STAILQ_INIT(&requests); 1514 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1515 STAILQ_INIT(&sc->sc_queued_requests); 1516 CESA_UNLOCK(sc, requests); 1517 1518 /* Execute all ready requests */ 1519 cesa_execute(sc); 1520 1521 /* Process completed requests */ 1522 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1523 BUS_DMASYNC_POSTWRITE); 1524 1525 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1526 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1527 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1528 1529 cr->cr_crp->crp_etype = sc->sc_error; 1530 if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) { 1531 if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1532 crypto_copydata(cr->cr_crp, 1533 cr->cr_crp->crp_digest_start, 1534 cr->cr_cs->cs_hlen, hash); 1535 if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash, 1536 cr->cr_cs->cs_hlen) != 0) 1537 cr->cr_crp->crp_etype = EBADMSG; 1538 } else 1539 crypto_copyback(cr->cr_crp, 1540 cr->cr_crp->crp_digest_start, 1541 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1542 } 1543 crypto_done(cr->cr_crp); 1544 cesa_free_request(sc, cr); 1545 } 1546 1547 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1548 BUS_DMASYNC_PREWRITE); 1549 1550 sc->sc_error = 0; 1551 1552 /* Unblock driver if it ran out of resources */ 1553 CESA_LOCK(sc, sc); 1554 blocked = sc->sc_blocked; 1555 sc->sc_blocked = 0; 1556 CESA_UNLOCK(sc, sc); 1557 1558 if (blocked) 1559 crypto_unblock(sc->sc_cid, blocked); 1560 } 1561 1562 static bool 1563 cesa_cipher_supported(const struct crypto_session_params *csp) 1564 { 1565 1566 switch (csp->csp_cipher_alg) { 1567 case CRYPTO_AES_CBC: 1568 if (csp->csp_ivlen != AES_BLOCK_LEN) 1569 return (false); 1570 break; 1571 default: 1572 return (false); 1573 } 1574 1575 if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN) 1576 return (false); 1577 1578 return (true); 1579 } 1580 1581 static bool 1582 cesa_auth_supported(struct cesa_softc *sc, 1583 const struct crypto_session_params *csp) 1584 { 1585 1586 switch (csp->csp_auth_alg) { 1587 case CRYPTO_SHA2_256_HMAC: 1588 if (!(sc->sc_soc_id == MV_DEV_88F6828 || 1589 sc->sc_soc_id == MV_DEV_88F6820 || 1590 sc->sc_soc_id == MV_DEV_88F6810)) 1591 return (false); 1592 /* FALLTHROUGH */ 1593 case CRYPTO_SHA1: 1594 case CRYPTO_SHA1_HMAC: 1595 break; 1596 default: 1597 return (false); 1598 } 1599 1600 if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN) 1601 return (false); 1602 1603 return (true); 1604 } 1605 1606 static int 1607 cesa_probesession(device_t dev, const struct crypto_session_params *csp) 1608 { 1609 struct cesa_softc *sc; 1610 1611 sc = device_get_softc(dev); 1612 if (csp->csp_flags != 0) 1613 return (EINVAL); 1614 switch (csp->csp_mode) { 1615 case CSP_MODE_DIGEST: 1616 if (!cesa_auth_supported(sc, csp)) 1617 return (EINVAL); 1618 break; 1619 case CSP_MODE_CIPHER: 1620 if (!cesa_cipher_supported(csp)) 1621 return (EINVAL); 1622 break; 1623 case CSP_MODE_ETA: 1624 if (!cesa_auth_supported(sc, csp) || 1625 !cesa_cipher_supported(csp)) 1626 return (EINVAL); 1627 break; 1628 default: 1629 return (EINVAL); 1630 } 1631 return (CRYPTODEV_PROBE_HARDWARE); 1632 } 1633 1634 static int 1635 cesa_newsession(device_t dev, crypto_session_t cses, 1636 const struct crypto_session_params *csp) 1637 { 1638 struct cesa_session *cs; 1639 int error; 1640 1641 error = 0; 1642 1643 /* Allocate session */ 1644 cs = crypto_get_driver_session(cses); 1645 1646 /* Prepare CESA configuration */ 1647 cs->cs_config = 0; 1648 cs->cs_ivlen = 1; 1649 cs->cs_mblen = 1; 1650 1651 switch (csp->csp_cipher_alg) { 1652 case CRYPTO_AES_CBC: 1653 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1654 cs->cs_ivlen = AES_BLOCK_LEN; 1655 break; 1656 } 1657 1658 switch (csp->csp_auth_alg) { 1659 case CRYPTO_SHA1: 1660 cs->cs_mblen = 1; 1661 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : 1662 csp->csp_auth_mlen; 1663 cs->cs_config |= CESA_CSHD_SHA1; 1664 break; 1665 case CRYPTO_SHA1_HMAC: 1666 cs->cs_mblen = SHA1_BLOCK_LEN; 1667 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : 1668 csp->csp_auth_mlen; 1669 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1670 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1671 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1672 break; 1673 case CRYPTO_SHA2_256_HMAC: 1674 cs->cs_mblen = SHA2_256_BLOCK_LEN; 1675 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN : 1676 csp->csp_auth_mlen; 1677 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; 1678 break; 1679 } 1680 1681 /* Save cipher key */ 1682 if (csp->csp_cipher_key != NULL) { 1683 memcpy(cs->cs_key, csp->csp_cipher_key, 1684 csp->csp_cipher_klen); 1685 if (csp->csp_cipher_alg == CRYPTO_AES_CBC) 1686 error = cesa_prep_aes_key(cs, csp); 1687 } 1688 1689 /* Save digest key */ 1690 if (csp->csp_auth_key != NULL) 1691 cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key, 1692 csp->csp_auth_klen); 1693 1694 return (error); 1695 } 1696 1697 static int 1698 cesa_process(device_t dev, struct cryptop *crp, int hint) 1699 { 1700 const struct crypto_session_params *csp; 1701 struct cesa_request *cr; 1702 struct cesa_session *cs; 1703 struct cesa_softc *sc; 1704 int error; 1705 1706 sc = device_get_softc(dev); 1707 error = 0; 1708 1709 cs = crypto_get_driver_session(crp->crp_session); 1710 csp = crypto_get_params(crp->crp_session); 1711 1712 /* Check and parse input */ 1713 if (crypto_buffer_len(&crp->crp_buf) > CESA_MAX_REQUEST_SIZE) { 1714 crp->crp_etype = E2BIG; 1715 crypto_done(crp); 1716 return (0); 1717 } 1718 1719 /* 1720 * For requests with AAD, only requests where the AAD is 1721 * immediately adjacent to the payload are supported. 1722 */ 1723 if (crp->crp_aad_length != 0 && 1724 (crp->crp_aad_start + crp->crp_aad_length) != 1725 crp->crp_payload_start) { 1726 crp->crp_etype = EINVAL; 1727 crypto_done(crp); 1728 return (0); 1729 } 1730 1731 /* 1732 * Get request descriptor. Block driver if there is no free 1733 * descriptors in pool. 1734 */ 1735 cr = cesa_alloc_request(sc); 1736 if (!cr) { 1737 CESA_LOCK(sc, sc); 1738 sc->sc_blocked = CRYPTO_SYMQ; 1739 CESA_UNLOCK(sc, sc); 1740 return (ERESTART); 1741 } 1742 1743 /* Prepare request */ 1744 cr->cr_crp = crp; 1745 cr->cr_cs = cs; 1746 1747 CESA_LOCK(sc, sessions); 1748 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1749 1750 if (csp->csp_cipher_alg != 0) 1751 crypto_read_iv(crp, cr->cr_csd->csd_iv); 1752 1753 if (crp->crp_cipher_key != NULL) { 1754 memcpy(cs->cs_key, crp->crp_cipher_key, 1755 csp->csp_cipher_klen); 1756 if (csp->csp_cipher_alg == CRYPTO_AES_CBC) 1757 error = cesa_prep_aes_key(cs, csp); 1758 } 1759 1760 if (!error && crp->crp_auth_key != NULL) 1761 cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key, 1762 csp->csp_auth_klen); 1763 1764 /* Convert request to chain of TDMA and SA descriptors */ 1765 if (!error) 1766 error = cesa_create_chain(sc, csp, cr); 1767 1768 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1769 CESA_UNLOCK(sc, sessions); 1770 1771 if (error) { 1772 cesa_free_request(sc, cr); 1773 crp->crp_etype = error; 1774 crypto_done(crp); 1775 return (0); 1776 } 1777 1778 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1779 BUS_DMASYNC_PREWRITE); 1780 1781 /* Enqueue request to execution */ 1782 cesa_enqueue_request(sc, cr); 1783 1784 /* Start execution, if we have no more requests in queue */ 1785 if ((hint & CRYPTO_HINT_MORE) == 0) 1786 cesa_execute(sc); 1787 1788 return (0); 1789 } 1790