Lines Matching +full:cs +full:- +full:x
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (C) 2009-2011 Semihalf.
32 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE
36 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0)
38 * +------------------------+
40 * +------------------------+ <= sc->sc_sram_base_va
90 { -1, 0 }
122 dev = sc->sc_dev; in cesa_dump_cshd()
124 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); in cesa_dump_cshd()
125 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); in cesa_dump_cshd()
126 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); in cesa_dump_cshd()
127 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); in cesa_dump_cshd()
128 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); in cesa_dump_cshd()
129 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); in cesa_dump_cshd()
130 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); in cesa_dump_cshd()
131 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); in cesa_dump_cshd()
132 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); in cesa_dump_cshd()
133 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); in cesa_dump_cshd()
134 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); in cesa_dump_cshd()
135 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); in cesa_dump_cshd()
136 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); in cesa_dump_cshd()
150 cdm->cdm_paddr = segs->ds_addr; in cesa_alloc_dma_mem_cb()
159 KASSERT(cdm->cdm_vaddr == NULL, in cesa_alloc_dma_mem()
162 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ in cesa_alloc_dma_mem()
170 &cdm->cdm_tag); /* dmat */ in cesa_alloc_dma_mem()
172 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" in cesa_alloc_dma_mem()
178 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, in cesa_alloc_dma_mem()
179 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); in cesa_alloc_dma_mem()
181 device_printf(sc->sc_dev, "failed to allocate DMA safe" in cesa_alloc_dma_mem()
187 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, in cesa_alloc_dma_mem()
190 device_printf(sc->sc_dev, "cannot get address of the DMA" in cesa_alloc_dma_mem()
198 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); in cesa_alloc_dma_mem()
200 bus_dma_tag_destroy(cdm->cdm_tag); in cesa_alloc_dma_mem()
202 cdm->cdm_vaddr = NULL; in cesa_alloc_dma_mem()
210 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); in cesa_free_dma_mem()
211 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); in cesa_free_dma_mem()
212 bus_dma_tag_destroy(cdm->cdm_tag); in cesa_free_dma_mem()
213 cdm->cdm_vaddr = NULL; in cesa_free_dma_mem()
221 if (cdm->cdm_vaddr != NULL) in cesa_sync_dma_mem()
222 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); in cesa_sync_dma_mem()
229 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); in cesa_sync_desc()
230 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); in cesa_sync_desc()
231 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); in cesa_sync_desc()
243 STAILQ_INIT(&cr->cr_tdesc); in cesa_alloc_request()
244 STAILQ_INIT(&cr->cr_sdesc); in cesa_alloc_request()
255 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); in cesa_free_request()
260 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); in cesa_free_request()
264 if (cr->cr_dmap_loaded) { in cesa_free_request()
265 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); in cesa_free_request()
266 cr->cr_dmap_loaded = 0; in cesa_free_request()
277 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); in cesa_enqueue_request()
289 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " in cesa_alloc_tdesc()
302 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " in cesa_alloc_sdesc()
307 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); in cesa_alloc_sdesc()
309 /* Fill-in SA descriptor with default values */ in cesa_alloc_sdesc()
310 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); in cesa_alloc_sdesc()
311 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); in cesa_alloc_sdesc()
312 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); in cesa_alloc_sdesc()
313 csd->csd_cshd->cshd_enc_src = 0; in cesa_alloc_sdesc()
314 csd->csd_cshd->cshd_enc_dst = 0; in cesa_alloc_sdesc()
315 csd->csd_cshd->cshd_enc_dlen = 0; in cesa_alloc_sdesc()
316 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); in cesa_alloc_sdesc()
317 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); in cesa_alloc_sdesc()
318 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); in cesa_alloc_sdesc()
319 csd->csd_cshd->cshd_mac_src = 0; in cesa_alloc_sdesc()
320 csd->csd_cshd->cshd_mac_dlen = 0; in cesa_alloc_sdesc()
335 ctd->ctd_cthd->cthd_dst = dst; in cesa_tdma_copy()
336 ctd->ctd_cthd->cthd_src = src; in cesa_tdma_copy()
337 ctd->ctd_cthd->cthd_byte_count = size; in cesa_tdma_copy()
341 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; in cesa_tdma_copy()
343 ctd->ctd_cthd->cthd_flags = 0; in cesa_tdma_copy()
352 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa + in cesa_tdma_copyin_sa_data()
353 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, in cesa_tdma_copyin_sa_data()
361 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa + in cesa_tdma_copyout_sa_data()
369 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr, in cesa_tdma_copy_sdesc()
378 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { in cesa_append_tdesc()
379 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); in cesa_append_tdesc()
380 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; in cesa_append_tdesc()
383 ctd->ctd_cthd->cthd_next = 0; in cesa_append_tdesc()
384 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); in cesa_append_tdesc()
401 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) in cesa_append_packet()
403 STAILQ_INIT(&cp->cp_copyin); in cesa_append_packet()
413 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) in cesa_append_packet()
415 STAILQ_INIT(&cp->cp_copyout); in cesa_append_packet()
421 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) in cesa_set_mkey() argument
428 hin = (uint32_t *)cs->cs_hiv_in; in cesa_set_mkey()
429 hout = (uint32_t *)cs->cs_hiv_out; in cesa_set_mkey()
462 cesa_prep_aes_key(struct cesa_session *cs, in cesa_prep_aes_key() argument
469 rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8); in cesa_prep_aes_key()
471 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; in cesa_prep_aes_key()
472 dkey = (uint32_t *)cs->cs_aes_dkey; in cesa_prep_aes_key()
474 switch (csp->csp_cipher_klen) { in cesa_prep_aes_key()
476 cs->cs_config |= CESA_CSH_AES_KLEN_128; in cesa_prep_aes_key()
481 cs->cs_config |= CESA_CSH_AES_KLEN_192; in cesa_prep_aes_key()
488 cs->cs_config |= CESA_CSH_AES_KLEN_256; in cesa_prep_aes_key()
505 cp->cp_size = size; in cesa_start_packet()
506 cp->cp_offset = 0; in cesa_start_packet()
507 STAILQ_INIT(&cp->cp_copyin); in cesa_start_packet()
508 STAILQ_INIT(&cp->cp_copyout); in cesa_start_packet()
519 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); in cesa_fill_packet()
522 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa + in cesa_fill_packet()
523 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); in cesa_fill_packet()
525 return (-ENOMEM); in cesa_fill_packet()
527 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); in cesa_fill_packet()
529 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa + in cesa_fill_packet()
530 CESA_DATA(cp->cp_offset), bsize); in cesa_fill_packet()
532 return (-ENOMEM); in cesa_fill_packet()
534 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); in cesa_fill_packet()
536 seg->ds_len -= bsize; in cesa_fill_packet()
537 seg->ds_addr += bsize; in cesa_fill_packet()
538 cp->cp_offset += bsize; in cesa_fill_packet()
562 sc = cci->cci_sc; in cesa_create_chain_cb()
563 cr = cci->cci_cr; in cesa_create_chain_cb()
564 crp = cr->cr_crp; in cesa_create_chain_cb()
567 cci->cci_error = error; in cesa_create_chain_cb()
577 config = cci->cci_config; in cesa_create_chain_cb()
580 crp->crp_aad_length != 0 && in cesa_create_chain_cb()
581 (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) { in cesa_create_chain_cb()
591 cci->cci_config = config | CESA_CSHD_MAC; in cesa_create_chain_cb()
594 cci->cci_config = config | CESA_CSHD_ENC; in cesa_create_chain_cb()
599 cci->cci_config = config | CESA_CSHD_ENC; in cesa_create_chain_cb()
602 cci->cci_config = config | CESA_CSHD_MAC; in cesa_create_chain_cb()
611 if (crp->crp_aad_length == 0) { in cesa_create_chain_cb()
612 skip = crp->crp_payload_start; in cesa_create_chain_cb()
613 len = crp->crp_payload_length; in cesa_create_chain_cb()
632 * For an encryption-only separate request, only in cesa_create_chain_cb()
634 * hash-only requests, process the entire region. in cesa_create_chain_cb()
638 skip = crp->crp_payload_start; in cesa_create_chain_cb()
639 len = crp->crp_payload_length; in cesa_create_chain_cb()
644 skip = crp->crp_aad_start; in cesa_create_chain_cb()
645 len = crp->crp_aad_length + crp->crp_payload_length; in cesa_create_chain_cb()
650 skip = crp->crp_aad_start; in cesa_create_chain_cb()
651 len = crp->crp_aad_length + crp->crp_payload_length; in cesa_create_chain_cb()
654 eskip = crp->crp_payload_start; in cesa_create_chain_cb()
655 elen = crp->crp_payload_length; in cesa_create_chain_cb()
663 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); in cesa_create_chain_cb()
668 while (nseg-- && len > 0) { in cesa_create_chain_cb()
677 skip -= size; in cesa_create_chain_cb()
680 seg.ds_len -= size; in cesa_create_chain_cb()
683 eskip -= size; in cesa_create_chain_cb()
686 mskip -= size; in cesa_create_chain_cb()
700 error = -size; in cesa_create_chain_cb()
704 len -= size; in cesa_create_chain_cb()
715 csd->csd_cshd->cshd_config = cci->cci_config; in cesa_create_chain_cb()
716 csd->csd_cshd->cshd_mac_total_dlen = tmlen; in cesa_create_chain_cb()
725 csd->csd_cshd->cshd_config |= in cesa_create_chain_cb()
728 csd->csd_cshd->cshd_config |= in cesa_create_chain_cb()
731 csd->csd_cshd->cshd_config |= in cesa_create_chain_cb()
735 csd->csd_cshd->cshd_enc_src = in cesa_create_chain_cb()
737 csd->csd_cshd->cshd_enc_dst = in cesa_create_chain_cb()
739 csd->csd_cshd->cshd_enc_dlen = in cesa_create_chain_cb()
740 MIN(elen, cp.cp_size - eskip); in cesa_create_chain_cb()
744 csd->csd_cshd->cshd_mac_src = in cesa_create_chain_cb()
746 csd->csd_cshd->cshd_mac_dlen = in cesa_create_chain_cb()
747 MIN(mlen, cp.cp_size - mskip); in cesa_create_chain_cb()
750 elen -= csd->csd_cshd->cshd_enc_dlen; in cesa_create_chain_cb()
751 eskip -= MIN(eskip, cp.cp_size); in cesa_create_chain_cb()
752 mlen -= csd->csd_cshd->cshd_mac_dlen; in cesa_create_chain_cb()
753 mskip -= MIN(mskip, cp.cp_size); in cesa_create_chain_cb()
755 cesa_dump_cshd(sc, csd->csd_cshd); in cesa_create_chain_cb()
776 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); in cesa_create_chain_cb()
777 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); in cesa_create_chain_cb()
778 cci->cci_error = error; in cesa_create_chain_cb()
795 if (csp->csp_cipher_klen != 0) { in cesa_create_chain()
796 if (csp->csp_cipher_alg == CRYPTO_AES_CBC && in cesa_create_chain()
797 !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) in cesa_create_chain()
798 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, in cesa_create_chain()
799 csp->csp_cipher_klen); in cesa_create_chain()
801 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, in cesa_create_chain()
802 csp->csp_cipher_klen); in cesa_create_chain()
805 if (csp->csp_auth_klen != 0) { in cesa_create_chain()
806 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, in cesa_create_chain()
808 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, in cesa_create_chain()
819 config = cr->cr_cs->cs_config; in cesa_create_chain()
821 if (csp->csp_cipher_alg != 0 && in cesa_create_chain()
822 !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op)) in cesa_create_chain()
824 switch (csp->csp_mode) { in cesa_create_chain()
843 error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp, in cesa_create_chain()
847 cr->cr_dmap_loaded = 1; in cesa_create_chain()
878 if (STAILQ_EMPTY(&sc->sc_ready_requests) || in cesa_execute()
879 !STAILQ_EMPTY(&sc->sc_queued_requests)) { in cesa_execute()
885 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); in cesa_execute()
886 STAILQ_INIT(&sc->sc_ready_requests); in cesa_execute()
889 if (STAILQ_FIRST(&sc->sc_queued_requests) != in cesa_execute()
890 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { in cesa_execute()
892 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | in cesa_execute()
895 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { in cesa_execute()
897 ctd = STAILQ_FIRST(&cr->cr_tdesc); in cesa_execute()
898 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, in cesa_execute()
901 prev_ctd->ctd_cthd->cthd_next = in cesa_execute()
902 ctd->ctd_cthd_paddr; in cesa_execute()
908 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | in cesa_execute()
913 cr = STAILQ_FIRST(&sc->sc_queued_requests); in cesa_execute()
914 ctd = STAILQ_FIRST(&cr->cr_tdesc); in cesa_execute()
916 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); in cesa_execute()
918 if (sc->sc_soc_id == MV_DEV_88F6828 || in cesa_execute()
919 sc->sc_soc_id == MV_DEV_88F6820 || in cesa_execute()
920 sc->sc_soc_id == MV_DEV_88F6810) in cesa_execute()
937 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", in cesa_setup_sram()
949 sc->sc_sram_base_pa = sram_reg[0]; in cesa_setup_sram()
951 sc->sc_sram_size = sram_reg[1]; in cesa_setup_sram()
953 if (sc->sc_soc_id != MV_DEV_88F6828 && in cesa_setup_sram()
954 sc->sc_soc_id != MV_DEV_88F6820 && in cesa_setup_sram()
955 sc->sc_soc_id != MV_DEV_88F6810) in cesa_setup_sram()
959 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); in cesa_setup_sram()
962 sc->sc_sram_base_va = sram_va; in cesa_setup_sram()
971 * root - device where you want to start search
974 * node - we are checking every device_t to be
984 if (node == -1) in device_from_node()
1025 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams", in cesa_setup_sram_armada()
1030 if (sc->sc_cesa_engine_id >= 2) in cesa_setup_sram_armada()
1034 sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id]; in cesa_setup_sram_armada()
1046 ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells, in cesa_setup_sram_armada()
1047 ssc->scells, &rl); in cesa_setup_sram_armada()
1055 for (j = 0; j < ssc->nranges; j++) { in cesa_setup_sram_armada()
1056 if (rle->start >= ssc->ranges[j].bus && in cesa_setup_sram_armada()
1057 rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) { in cesa_setup_sram_armada()
1058 rle->start -= ssc->ranges[j].bus; in cesa_setup_sram_armada()
1059 rle->start += ssc->ranges[j].host; in cesa_setup_sram_armada()
1060 rle->end -= ssc->ranges[j].bus; in cesa_setup_sram_armada()
1061 rle->end += ssc->ranges[j].host; in cesa_setup_sram_armada()
1065 sc->sc_sram_base_pa = rle->start; in cesa_setup_sram_armada()
1066 sc->sc_sram_size = rle->count; in cesa_setup_sram_armada()
1069 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); in cesa_setup_sram_armada()
1072 sc->sc_sram_base_va = sram_va; in cesa_setup_sram_armada()
1079 { "marvell,armada-38x-crypto", (uintptr_t)true },
1090 if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data) in cesa_probe()
1107 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) in cesa_attach()
1120 rl = &ndi->rl; in cesa_attach()
1126 CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); in cesa_attach()
1128 CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); in cesa_attach()
1137 CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); in cesa_attach()
1139 CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); in cesa_attach()
1143 resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0; in cesa_attach()
1152 sc->sc_cesa_engine_id = engine_idx; in cesa_attach()
1179 sc->sc_blocked = 0; in cesa_attach_late()
1180 sc->sc_error = 0; in cesa_attach_late()
1181 sc->sc_dev = dev; in cesa_attach_late()
1189 sc->sc_tperr = 0; in cesa_attach_late()
1195 sc->sc_soc_id = d; in cesa_attach_late()
1198 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), in cesa_attach_late()
1200 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), in cesa_attach_late()
1202 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), in cesa_attach_late()
1204 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), in cesa_attach_late()
1206 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), in cesa_attach_late()
1210 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); in cesa_attach_late()
1217 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) in cesa_attach_late()
1228 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET | in cesa_attach_late()
1229 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie)); in cesa_attach_late()
1245 &sc->sc_data_dtag); /* dmat */ in cesa_attach_late()
1250 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, in cesa_attach_late()
1255 STAILQ_INIT(&sc->sc_free_tdesc); in cesa_attach_late()
1257 sc->sc_tdesc[i].ctd_cthd = in cesa_attach_late()
1258 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; in cesa_attach_late()
1259 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + in cesa_attach_late()
1261 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], in cesa_attach_late()
1266 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, in cesa_attach_late()
1271 STAILQ_INIT(&sc->sc_free_sdesc); in cesa_attach_late()
1273 sc->sc_sdesc[i].csd_cshd = in cesa_attach_late()
1274 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; in cesa_attach_late()
1275 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + in cesa_attach_late()
1277 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], in cesa_attach_late()
1282 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, in cesa_attach_late()
1287 STAILQ_INIT(&sc->sc_free_requests); in cesa_attach_late()
1288 STAILQ_INIT(&sc->sc_ready_requests); in cesa_attach_late()
1289 STAILQ_INIT(&sc->sc_queued_requests); in cesa_attach_late()
1291 sc->sc_requests[i].cr_csd = in cesa_attach_late()
1292 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; in cesa_attach_late()
1293 sc->sc_requests[i].cr_csd_paddr = in cesa_attach_late()
1294 sc->sc_requests_cdm.cdm_paddr + in cesa_attach_late()
1298 error = bus_dmamap_create(sc->sc_data_dtag, 0, in cesa_attach_late()
1299 &sc->sc_requests[i].cr_dmap); in cesa_attach_late()
1301 i--; in cesa_attach_late()
1303 bus_dmamap_destroy(sc->sc_data_dtag, in cesa_attach_late()
1304 sc->sc_requests[i].cr_dmap); in cesa_attach_late()
1305 } while (i--); in cesa_attach_late()
1310 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], in cesa_attach_late()
1316 * - Burst limit: 128 bytes, in cesa_attach_late()
1317 * - Outstanding reads enabled, in cesa_attach_late()
1318 * - No byte-swap. in cesa_attach_late()
1323 if (sc->sc_soc_id == MV_DEV_88F6828 || in cesa_attach_late()
1324 sc->sc_soc_id == MV_DEV_88F6820 || in cesa_attach_late()
1325 sc->sc_soc_id == MV_DEV_88F6810) in cesa_attach_late()
1332 * - SA descriptor is present at beginning of CESA SRAM, in cesa_attach_late()
1333 * - Multi-packet chain mode, in cesa_attach_late()
1334 * - Cooperation with TDMA enabled. in cesa_attach_late()
1342 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); in cesa_attach_late()
1349 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session), in cesa_attach_late()
1351 if (sc->sc_cid < 0) { in cesa_attach_late()
1359 bus_dmamap_destroy(sc->sc_data_dtag, in cesa_attach_late()
1360 sc->sc_requests[i].cr_dmap); in cesa_attach_late()
1362 cesa_free_dma_mem(&sc->sc_requests_cdm); in cesa_attach_late()
1364 cesa_free_dma_mem(&sc->sc_sdesc_cdm); in cesa_attach_late()
1366 cesa_free_dma_mem(&sc->sc_tdesc_cdm); in cesa_attach_late()
1368 bus_dma_tag_destroy(sc->sc_data_dtag); in cesa_attach_late()
1370 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); in cesa_attach_late()
1372 if (sc->sc_soc_id == MV_DEV_88F6828 || in cesa_attach_late()
1373 sc->sc_soc_id == MV_DEV_88F6820 || in cesa_attach_late()
1374 sc->sc_soc_id == MV_DEV_88F6810) in cesa_attach_late()
1375 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); in cesa_attach_late()
1377 bus_release_resources(dev, cesa_res_spec, sc->sc_res); in cesa_attach_late()
1379 mtx_destroy(&sc->sc_sessions_lock); in cesa_attach_late()
1380 mtx_destroy(&sc->sc_requests_lock); in cesa_attach_late()
1381 mtx_destroy(&sc->sc_sdesc_lock); in cesa_attach_late()
1382 mtx_destroy(&sc->sc_tdesc_lock); in cesa_attach_late()
1383 mtx_destroy(&sc->sc_sc_lock); in cesa_attach_late()
1402 crypto_unregister_all(sc->sc_cid); in cesa_detach()
1406 bus_dmamap_destroy(sc->sc_data_dtag, in cesa_detach()
1407 sc->sc_requests[i].cr_dmap); in cesa_detach()
1410 cesa_free_dma_mem(&sc->sc_requests_cdm); in cesa_detach()
1411 cesa_free_dma_mem(&sc->sc_sdesc_cdm); in cesa_detach()
1412 cesa_free_dma_mem(&sc->sc_tdesc_cdm); in cesa_detach()
1415 bus_dma_tag_destroy(sc->sc_data_dtag); in cesa_detach()
1418 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); in cesa_detach()
1421 bus_release_resources(dev, cesa_res_spec, sc->sc_res); in cesa_detach()
1424 if (sc->sc_soc_id == MV_DEV_88F6828 || in cesa_detach()
1425 sc->sc_soc_id == MV_DEV_88F6820 || in cesa_detach()
1426 sc->sc_soc_id == MV_DEV_88F6810) in cesa_detach()
1427 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); in cesa_detach()
1430 mtx_destroy(&sc->sc_sessions_lock); in cesa_detach()
1431 mtx_destroy(&sc->sc_requests_lock); in cesa_detach()
1432 mtx_destroy(&sc->sc_sdesc_lock); in cesa_detach()
1433 mtx_destroy(&sc->sc_tdesc_lock); in cesa_detach()
1434 mtx_destroy(&sc->sc_sc_lock); in cesa_detach()
1459 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); in cesa_intr()
1460 sc->sc_error = EIO; in cesa_intr()
1464 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); in cesa_intr()
1465 sc->sc_error = EIO; in cesa_intr()
1469 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); in cesa_intr()
1470 sc->sc_error = EIO; in cesa_intr()
1474 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); in cesa_intr()
1475 sc->sc_error = EIO; in cesa_intr()
1479 if (icr & sc->sc_tperr) { in cesa_intr()
1480 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); in cesa_intr()
1481 sc->sc_error = EIO; in cesa_intr()
1491 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); in cesa_intr()
1492 STAILQ_INIT(&sc->sc_queued_requests); in cesa_intr()
1499 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | in cesa_intr()
1503 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, in cesa_intr()
1506 cr->cr_crp->crp_etype = sc->sc_error; in cesa_intr()
1507 if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) { in cesa_intr()
1508 if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { in cesa_intr()
1509 crypto_copydata(cr->cr_crp, in cesa_intr()
1510 cr->cr_crp->crp_digest_start, in cesa_intr()
1511 cr->cr_cs->cs_hlen, hash); in cesa_intr()
1512 if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash, in cesa_intr()
1513 cr->cr_cs->cs_hlen) != 0) in cesa_intr()
1514 cr->cr_crp->crp_etype = EBADMSG; in cesa_intr()
1516 crypto_copyback(cr->cr_crp, in cesa_intr()
1517 cr->cr_crp->crp_digest_start, in cesa_intr()
1518 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); in cesa_intr()
1520 crypto_done(cr->cr_crp); in cesa_intr()
1524 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | in cesa_intr()
1527 sc->sc_error = 0; in cesa_intr()
1531 blocked = sc->sc_blocked; in cesa_intr()
1532 sc->sc_blocked = 0; in cesa_intr()
1536 crypto_unblock(sc->sc_cid, blocked); in cesa_intr()
1543 switch (csp->csp_cipher_alg) { in cesa_cipher_supported()
1545 if (csp->csp_ivlen != AES_BLOCK_LEN) in cesa_cipher_supported()
1552 if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN) in cesa_cipher_supported()
1563 switch (csp->csp_auth_alg) { in cesa_auth_supported()
1565 if (!(sc->sc_soc_id == MV_DEV_88F6828 || in cesa_auth_supported()
1566 sc->sc_soc_id == MV_DEV_88F6820 || in cesa_auth_supported()
1567 sc->sc_soc_id == MV_DEV_88F6810)) in cesa_auth_supported()
1577 if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN) in cesa_auth_supported()
1589 if (csp->csp_flags != 0) in cesa_probesession()
1591 switch (csp->csp_mode) { in cesa_probesession()
1615 struct cesa_session *cs; in cesa_newsession() local
1621 cs = crypto_get_driver_session(cses); in cesa_newsession()
1624 cs->cs_config = 0; in cesa_newsession()
1625 cs->cs_ivlen = 1; in cesa_newsession()
1626 cs->cs_mblen = 1; in cesa_newsession()
1628 switch (csp->csp_cipher_alg) { in cesa_newsession()
1630 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; in cesa_newsession()
1631 cs->cs_ivlen = AES_BLOCK_LEN; in cesa_newsession()
1635 switch (csp->csp_auth_alg) { in cesa_newsession()
1637 cs->cs_mblen = 1; in cesa_newsession()
1638 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : in cesa_newsession()
1639 csp->csp_auth_mlen; in cesa_newsession()
1640 cs->cs_config |= CESA_CSHD_SHA1; in cesa_newsession()
1643 cs->cs_mblen = SHA1_BLOCK_LEN; in cesa_newsession()
1644 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN : in cesa_newsession()
1645 csp->csp_auth_mlen; in cesa_newsession()
1646 cs->cs_config |= CESA_CSHD_SHA1_HMAC; in cesa_newsession()
1647 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) in cesa_newsession()
1648 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; in cesa_newsession()
1651 cs->cs_mblen = SHA2_256_BLOCK_LEN; in cesa_newsession()
1652 cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN : in cesa_newsession()
1653 csp->csp_auth_mlen; in cesa_newsession()
1654 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; in cesa_newsession()
1659 if (csp->csp_cipher_key != NULL) { in cesa_newsession()
1660 memcpy(cs->cs_key, csp->csp_cipher_key, in cesa_newsession()
1661 csp->csp_cipher_klen); in cesa_newsession()
1662 if (csp->csp_cipher_alg == CRYPTO_AES_CBC) in cesa_newsession()
1663 error = cesa_prep_aes_key(cs, csp); in cesa_newsession()
1667 if (csp->csp_auth_key != NULL) in cesa_newsession()
1668 cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key, in cesa_newsession()
1669 csp->csp_auth_klen); in cesa_newsession()
1679 struct cesa_session *cs; in cesa_process() local
1686 cs = crypto_get_driver_session(crp->crp_session); in cesa_process()
1687 csp = crypto_get_params(crp->crp_session); in cesa_process()
1690 if (crypto_buffer_len(&crp->crp_buf) > CESA_MAX_REQUEST_SIZE) { in cesa_process()
1691 crp->crp_etype = E2BIG; in cesa_process()
1700 if (crp->crp_aad_length != 0 && in cesa_process()
1701 (crp->crp_aad_start + crp->crp_aad_length) != in cesa_process()
1702 crp->crp_payload_start) { in cesa_process()
1703 crp->crp_etype = EINVAL; in cesa_process()
1715 sc->sc_blocked = CRYPTO_SYMQ; in cesa_process()
1721 cr->cr_crp = crp; in cesa_process()
1722 cr->cr_cs = cs; in cesa_process()
1727 if (csp->csp_cipher_alg != 0) in cesa_process()
1728 crypto_read_iv(crp, cr->cr_csd->csd_iv); in cesa_process()
1730 if (crp->crp_cipher_key != NULL) { in cesa_process()
1731 memcpy(cs->cs_key, crp->crp_cipher_key, in cesa_process()
1732 csp->csp_cipher_klen); in cesa_process()
1733 if (csp->csp_cipher_alg == CRYPTO_AES_CBC) in cesa_process()
1734 error = cesa_prep_aes_key(cs, csp); in cesa_process()
1737 if (!error && crp->crp_auth_key != NULL) in cesa_process()
1738 cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key, in cesa_process()
1739 csp->csp_auth_klen); in cesa_process()
1750 crp->crp_etype = error; in cesa_process()
1755 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | in cesa_process()