/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $"); #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include #include #include "qatreg.h" #include "qatvar.h" #include "qat_aevar.h" extern struct qat_hw qat_hw_c2xxx; #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18 static const struct qat_product { uint16_t qatp_vendor; uint16_t qatp_product; const char *qatp_name; enum qat_chip_type qatp_chip; const struct qat_hw *qatp_hw; } qat_products[] = { { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS, "Intel C2000 QuickAssist PF", QAT_CHIP_C2XXX, &qat_hw_c2xxx }, { 0, 0, NULL, 0, NULL }, }; /* Hash Algorithm specific structure */ /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = { 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0 }; /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 }; /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = { 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 }; /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 }; static const struct qat_sym_hash_alg_info sha1_info = { .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE, .qshai_init_state = sha1_initial_state, .qshai_sah = &auth_hash_hmac_sha1, .qshai_state_offset = 0, .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha256_info = { .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE, .qshai_init_state = sha256_initial_state, .qshai_sah = &auth_hash_hmac_sha2_256, .qshai_state_offset = offsetof(SHA256_CTX, state), .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha384_info = { .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE, .qshai_init_state = sha384_initial_state, .qshai_sah = &auth_hash_hmac_sha2_384, .qshai_state_offset = offsetof(SHA384_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info sha512_info = { .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE, .qshai_init_state = sha512_initial_state, .qshai_sah = &auth_hash_hmac_sha2_512, .qshai_state_offset = offsetof(SHA512_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info aes_gcm_info = { .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE, .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE, .qshai_sah = &auth_hash_nist_gmac_aes_128, }; /* Hash QAT specific structures */ static const struct qat_sym_hash_qat_info sha1_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA1, .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE, .qshqi_state1_len = HW_SHA1_STATE1_SZ, .qshqi_state2_len = HW_SHA1_STATE2_SZ, }; static const struct qat_sym_hash_qat_info sha256_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA256, .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE, .qshqi_state1_len = HW_SHA256_STATE1_SZ, .qshqi_state2_len = HW_SHA256_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha384_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA384, .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE, .qshqi_state1_len = HW_SHA384_STATE1_SZ, .qshqi_state2_len = HW_SHA384_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha512_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA512, .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE, .qshqi_state1_len = HW_SHA512_STATE1_SZ, .qshqi_state2_len = HW_SHA512_STATE2_SZ }; static const struct qat_sym_hash_qat_info aes_gcm_config = { .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128, .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ, .qshqi_state2_len = HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ, }; static const struct qat_sym_hash_def qat_sym_hash_defs[] = { [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config }, [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config }, [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config }, [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config }, [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config }, }; static const struct qat_product *qat_lookup(device_t); static int qat_probe(device_t); static int qat_attach(device_t); static int qat_init(device_t); static int qat_start(device_t); static int qat_detach(device_t); static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp); static void qat_freesession(device_t dev, crypto_session_t cses); static int qat_setup_msix_intr(struct qat_softc *); static void qat_etr_init(struct qat_softc *); static void qat_etr_deinit(struct qat_softc *); static void qat_etr_bank_init(struct qat_softc *, int); static void qat_etr_bank_deinit(struct qat_softc *sc, int); static void qat_etr_ap_bank_init(struct qat_softc *); static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int); static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *, uint32_t, int); static void qat_etr_ap_bank_setup_ring(struct qat_softc *, struct qat_ring *); static int qat_etr_verify_ring_size(uint32_t, uint32_t); static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *, struct qat_ring *); static void qat_etr_bank_intr(void *); static void qat_arb_update(struct qat_softc *, struct qat_bank *); static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie( struct qat_crypto_bank *); static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *, struct qat_sym_cookie *); static int qat_crypto_setup_ring(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_bank_init(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_init(struct qat_softc *); static void qat_crypto_deinit(struct qat_softc *); static int qat_crypto_start(struct qat_softc *); static void qat_crypto_stop(struct qat_softc *); static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *); static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver"); static const struct qat_product * qat_lookup(device_t dev) { const struct qat_product *qatp; for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) { if (pci_get_vendor(dev) == qatp->qatp_vendor && pci_get_device(dev) == qatp->qatp_product) return qatp; } return NULL; } static int qat_probe(device_t dev) { const struct qat_product *prod; prod = qat_lookup(dev); if (prod != NULL) { device_set_desc(dev, prod->qatp_name); return BUS_PROBE_DEFAULT; } return ENXIO; } static int qat_attach(device_t dev) { struct qat_softc *sc = device_get_softc(dev); const struct qat_product *qatp; int bar, count, error, i; sc->sc_dev = dev; sc->sc_rev = pci_get_revid(dev); sc->sc_crypto.qcy_cid = -1; qatp = qat_lookup(dev); memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw)); /* Determine active accelerators and engines */ sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc); sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc); sc->sc_accel_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) { if (sc->sc_accel_mask & (1 << i)) sc->sc_accel_num++; } sc->sc_ae_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) { if (sc->sc_ae_mask & (1 << i)) sc->sc_ae_num++; } if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) { device_printf(sc->sc_dev, "couldn't find acceleration"); goto fail; } MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL); MPASS(sc->sc_ae_num <= MAX_NUM_AE); /* Determine SKU and capabilities */ sc->sc_sku = sc->sc_hw.qhw_get_sku(sc); sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc); sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc); i = 0; if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) { MPASS(sc->sc_hw.qhw_sram_bar_id == 0); uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4); /* Skip SRAM BAR */ i = (fusectl & FUSECTL_MASK) ? 1 : 0; } for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) { uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4); if (val == 0 || !PCI_BAR_MEM(val)) continue; sc->sc_rid[i] = PCIR_BAR(bar); sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid[i], RF_ACTIVE); if (sc->sc_res[i] == NULL) { device_printf(dev, "couldn't map BAR %d\n", bar); goto fail; } sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]); sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]); i++; if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) bar++; } pci_enable_busmaster(dev); count = sc->sc_hw.qhw_num_banks + 1; if (pci_msix_count(dev) < count) { device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n", pci_msix_count(dev), count); goto fail; } error = pci_alloc_msix(dev, &count); if (error != 0) { device_printf(dev, "failed to allocate MSI-X vectors\n"); goto fail; } error = qat_init(dev); if (error == 0) return 0; fail: qat_detach(dev); return ENXIO; } static int qat_init(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; qat_etr_init(sc); if (sc->sc_hw.qhw_init_admin_comms != NULL && (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize admin comms: %d\n", error); return error; } if (sc->sc_hw.qhw_init_arb != NULL && (error = sc->sc_hw.qhw_init_arb(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize hw arbiter: %d\n", error); return error; } error = qat_ae_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize Acceleration Engine: %d\n", error); return error; } error = qat_aefw_load(sc); if (error) { device_printf(sc->sc_dev, "Could not load firmware: %d\n", error); return error; } error = qat_setup_msix_intr(sc); if (error) { device_printf(sc->sc_dev, "Could not setup interrupts: %d\n", error); return error; } sc->sc_hw.qhw_enable_intr(sc); error = qat_crypto_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize service: %d\n", error); return error; } if (sc->sc_hw.qhw_enable_error_correction != NULL) sc->sc_hw.qhw_enable_error_correction(sc); if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL && (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize watchdog timer: %d\n", error); return error; } error = qat_start(dev); if (error) { device_printf(sc->sc_dev, "Could not start: %d\n", error); return error; } return 0; } static int qat_start(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; error = qat_ae_start(sc); if (error) return error; if (sc->sc_hw.qhw_send_admin_init != NULL && (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) { return error; } error = qat_crypto_start(sc); if (error) return error; return 0; } static int qat_detach(device_t dev) { struct qat_softc *sc; int bar, i; sc = device_get_softc(dev); qat_crypto_stop(sc); qat_crypto_deinit(sc); qat_aefw_unload(sc); if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i]; if (qb->qb_ih_cookie != NULL) (void)bus_teardown_intr(dev, qb->qb_ih, qb->qb_ih_cookie); if (qb->qb_ih != NULL) (void)bus_release_resource(dev, SYS_RES_IRQ, i + 1, qb->qb_ih); } } if (sc->sc_ih_cookie != NULL) { (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie); sc->sc_ih_cookie = NULL; } if (sc->sc_ih != NULL) { (void)bus_release_resource(dev, SYS_RES_IRQ, sc->sc_hw.qhw_num_banks + 1, sc->sc_ih); sc->sc_ih = NULL; } pci_release_msi(dev); qat_etr_deinit(sc); for (bar = 0; bar < MAX_BARS; bar++) { if (sc->sc_res[bar] != NULL) { (void)bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid[bar], sc->sc_res[bar]); sc->sc_res[bar] = NULL; } } return 0; } void * qat_alloc_mem(size_t size) { return (malloc(size, M_QAT, M_WAITOK | M_ZERO)); } void qat_free_mem(void *ptr) { free(ptr, M_QAT); } static void qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct qat_dmamem *qdm; if (error != 0) return; KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); qdm = arg; qdm->qdm_dma_seg = segs[0]; } int qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm, int nseg, bus_size_t size, bus_size_t alignment) { int error; KASSERT(qdm->qdm_dma_vaddr == NULL, ("%s: DMA memory descriptor in use", __func__)); error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), alignment, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ nseg, /* nsegments */ size, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &qdm->qdm_dma_tag); if (error != 0) return error; error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &qdm->qdm_dma_map); if (error != 0) { device_printf(sc->sc_dev, "couldn't allocate dmamem, error = %d\n", error); goto fail_0; } error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map, qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm, BUS_DMA_NOWAIT); if (error) { device_printf(sc->sc_dev, "couldn't load dmamem map, error = %d\n", error); goto fail_1; } return 0; fail_1: bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); fail_0: bus_dma_tag_destroy(qdm->qdm_dma_tag); return error; } void qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm) { if (qdm->qdm_dma_tag != NULL) { bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map); bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); bus_dma_tag_destroy(qdm->qdm_dma_tag); explicit_bzero(qdm, sizeof(*qdm)); } } static int qat_setup_msix_intr(struct qat_softc *sc) { device_t dev; int error, i, rid; dev = sc->sc_dev; for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i - 1]; rid = i; qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (qb->qb_ih == NULL) { device_printf(dev, "failed to allocate bank intr resource\n"); return ENXIO; } error = bus_setup_intr(dev, qb->qb_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb, &qb->qb_ih_cookie); if (error != 0) { device_printf(dev, "failed to set up bank intr\n"); return error; } error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus); if (error != 0) device_printf(dev, "failed to bind intr %d\n", i); } rid = i; sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_ih == NULL) return ENXIO; error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie); return error; } static void qat_etr_init(struct qat_softc *sc) { int i; sc->sc_etr_banks = qat_alloc_mem( sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks); for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_init(sc, i); if (sc->sc_hw.qhw_num_ap_banks) { sc->sc_etr_ap_banks = qat_alloc_mem( sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks); qat_etr_ap_bank_init(sc); } } static void qat_etr_deinit(struct qat_softc *sc) { int i; if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_deinit(sc, i); qat_free_mem(sc->sc_etr_banks); sc->sc_etr_banks = NULL; } if (sc->sc_etr_ap_banks != NULL) { qat_free_mem(sc->sc_etr_ap_banks); sc->sc_etr_ap_banks = NULL; } } static void qat_etr_bank_init(struct qat_softc *sc, int bank) { struct qat_bank *qb = &sc->sc_etr_banks[bank]; int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap; MPASS(bank < sc->sc_hw.qhw_num_banks); mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF); qb->qb_sc = sc; qb->qb_bank = bank; qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT; /* Clean CSRs for all rings within the bank */ for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { struct qat_ring *qr = &qb->qb_et_rings[i]; qat_etr_bank_ring_write_4(sc, bank, i, ETR_RING_CONFIG, 0); qat_etr_bank_ring_base_write_8(sc, bank, i, 0); if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t)); } else if (sc->sc_hw.qhw_tx_rings_mask & (1 << (i - tx_rx_gap))) { /* Share inflight counter with rx and tx */ qr->qr_inflight = qb->qb_et_rings[i - tx_rx_gap].qr_inflight; } } if (sc->sc_hw.qhw_init_etr_intr != NULL) { sc->sc_hw.qhw_init_etr_intr(sc, bank); } else { /* common code in qat 1.7 */ qat_etr_bank_write_4(sc, bank, ETR_INT_REG, ETR_INT_REG_CLEAR_MASK); for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank / ETR_RINGS_PER_INT_SRCSEL; i++) { qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL + (i * ETR_INT_SRCSEL_NEXT_OFFSET), ETR_INT_SRCSEL_MASK); } } } static void qat_etr_bank_deinit(struct qat_softc *sc, int bank) { struct qat_bank *qb; struct qat_ring *qr; int i; qb = &sc->sc_etr_banks[bank]; for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr = &qb->qb_et_rings[i]; qat_free_mem(qr->qr_inflight); } } } static void qat_etr_ap_bank_init(struct qat_softc *sc) { int ap_bank; for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) { struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank]; qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK, ETR_AP_NF_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK, ETR_AP_NE_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0); memset(qab, 0, sizeof(*qab)); } } static void qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask) { if (set_mask) *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); else *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); } static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest, uint32_t ring, int set_dest) { uint32_t ae_mask; uint8_t mailbox, ae, nae; uint8_t *dest = (uint8_t *)ap_dest; mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring); nae = 0; ae_mask = sc->sc_ae_mask; for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) { if ((ae_mask & (1 << ae)) == 0) continue; if (set_dest) { dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) | __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) | ETR_AP_DEST_ENABLE; } else { dest[nae] = 0; } nae++; if (nae == ETR_MAX_AE_PER_MAILBOX) break; } } static void qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr) { struct qat_ap_bank *qab; int ap_bank; if (sc->sc_hw.qhw_num_ap_banks == 0) return; ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring); MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks); qab = &sc->sc_etr_ap_banks[ap_bank]; if (qr->qr_cb == NULL) { qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1); if (!qab->qab_ne_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, qab->qab_ne_dest); } } else { qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1); if (!qab->qab_nf_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, qab->qab_nf_dest); } } } static int qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs) { int i = QAT_MIN_RING_SIZE; for (; i <= QAT_MAX_RING_SIZE; i++) if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i)) return i; return QAT_DEFAULT_RING_SIZE; } int qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring, uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg, const char *name, struct qat_ring **rqr) { struct qat_bank *qb; struct qat_ring *qr = NULL; int error; uint32_t ring_size_bytes, ring_config; uint64_t ring_base; uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512; uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0; MPASS(bank < sc->sc_hw.qhw_num_banks); /* Allocate a ring from specified bank */ qb = &sc->sc_etr_banks[bank]; if (ring >= sc->sc_hw.qhw_num_rings_per_bank) return EINVAL; if (qb->qb_allocated_rings & (1 << ring)) return ENOENT; qr = &qb->qb_et_rings[ring]; qb->qb_allocated_rings |= 1 << ring; /* Initialize allocated ring */ qr->qr_ring = ring; qr->qr_bank = bank; qr->qr_name = name; qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring; qr->qr_ring_mask = (1 << ring); qr->qr_cb = cb; qr->qr_cb_arg = cb_arg; /* Setup the shadow variables */ qr->qr_head = 0; qr->qr_tail = 0; qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size); qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs); /* * To make sure that ring is alligned to ring size allocate * at least 4k and then tell the user it is smaller. */ ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size); ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes); error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes, ring_size_bytes); if (error) return error; qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr; qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr; memset(qr->qr_ring_vaddr, QAT_RING_PATTERN, qr->qr_dma.qdm_dma_seg.ds_len); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (cb == NULL) { ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size); } else { ring_config = ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne); } qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config); ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size); qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base); if (sc->sc_hw.qhw_init_arb != NULL) qat_arb_update(sc, qb); mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF); qat_etr_ap_bank_setup_ring(sc, qr); if (cb != NULL) { uint32_t intr_mask; qb->qb_intr_mask |= qr->qr_ring_mask; intr_mask = qb->qb_intr_mask; qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask); qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); } *rqr = qr; return 0; } static inline u_int qat_modulo(u_int data, u_int shift) { u_int div = data >> shift; u_int mult = div << shift; return data - mult; } int qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg) { uint32_t inflight; uint32_t *addr; mtx_lock(&qr->qr_ring_mtx); inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1; if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) { atomic_subtract_32(qr->qr_inflight, 1); qr->qr_need_wakeup = true; mtx_unlock(&qr->qr_ring_mtx); counter_u64_add(sc->sc_ring_full_restarts, 1); return ERESTART; } addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail); memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size)); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE); qr->qr_tail = qat_modulo(qr->qr_tail + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_TAIL_OFFSET, qr->qr_tail); mtx_unlock(&qr->qr_ring_mtx); return 0; } static int qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb, struct qat_ring *qr) { uint32_t *msg, nmsg = 0; int handled = 0; bool blocked = false; mtx_lock(&qr->qr_ring_mtx); msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) { atomic_subtract_32(qr->qr_inflight, 1); if (qr->qr_cb != NULL) { mtx_unlock(&qr->qr_ring_mtx); handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg); mtx_lock(&qr->qr_ring_mtx); } atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG); qr->qr_head = qat_modulo(qr->qr_head + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); nmsg++; msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); } bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (nmsg > 0) { qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_HEAD_OFFSET, qr->qr_head); if (qr->qr_need_wakeup) { blocked = true; qr->qr_need_wakeup = false; } } mtx_unlock(&qr->qr_ring_mtx); if (blocked) crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ); return handled; } static void qat_etr_bank_intr(void *arg) { struct qat_bank *qb = arg; struct qat_softc *sc = qb->qb_sc; uint32_t estat; int i; mtx_lock(&qb->qb_bank_mtx); qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0); /* Now handle all the responses */ estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT); estat &= qb->qb_intr_mask; qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); mtx_unlock(&qb->qb_bank_mtx); while ((i = ffs(estat)) != 0) { struct qat_ring *qr = &qb->qb_et_rings[--i]; estat &= ~(1 << i); (void)qat_etr_ring_intr(sc, qb, qr); } } void qat_arb_update(struct qat_softc *sc, struct qat_bank *qb) { qat_arb_ringsrvarben_write_4(sc, qb->qb_bank, qb->qb_allocated_rings & 0xff); } static struct qat_sym_cookie * qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb) { struct qat_sym_cookie *qsc; mtx_lock(&qcb->qcb_bank_mtx); if (qcb->qcb_symck_free_count == 0) { mtx_unlock(&qcb->qcb_bank_mtx); return NULL; } qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count]; mtx_unlock(&qcb->qcb_bank_mtx); return qsc; } static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb, struct qat_sym_cookie *qsc) { explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN); explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN); mtx_lock(&qcb->qcb_bank_mtx); qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc; mtx_unlock(&qcb->qcb_bank_mtx); } void qat_memcpy_htobe64(void *dst, const void *src, size_t len) { uint64_t *dst0 = dst; const uint64_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe64(*(src0 + i)); } void qat_memcpy_htobe32(void *dst, const void *src, size_t len) { uint32_t *dst0 = dst; const uint32_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe32(*(src0 + i)); } void qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte) { switch (wordbyte) { case 4: qat_memcpy_htobe32(dst, src, len); break; case 8: qat_memcpy_htobe64(dst, src, len); break; default: panic("invalid word size %u", wordbyte); } } void qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state) { uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; char zeros[AES_BLOCK_LEN]; int rounds; memset(zeros, 0, sizeof(zeros)); rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); rijndaelEncrypt(ks, rounds, zeros, state); explicit_bzero(ks, sizeof(ks)); } void qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state1, uint8_t *state2) { union authctx ctx; const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah; uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset; uint32_t state_size = hash_def->qshd_alg->qshai_state_size; uint32_t state_word = hash_def->qshd_alg->qshai_state_word; hmac_init_ipad(sah, key, klen, &ctx); qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size, state_word); hmac_init_opad(sah, key, klen, &ctx); qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size, state_word); explicit_bzero(&ctx, sizeof(ctx)); } static enum hw_cipher_algo qat_aes_cipher_algo(int klen) { switch (klen) { case HW_AES_128_KEY_SZ: return HW_CIPHER_ALGO_AES128; case HW_AES_192_KEY_SZ: return HW_CIPHER_ALGO_AES192; case HW_AES_256_KEY_SZ: return HW_CIPHER_ALGO_AES256; default: panic("invalid key length %d", klen); } } uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc, const struct qat_session *qs) { enum hw_cipher_algo algo; enum hw_cipher_dir dir; enum hw_cipher_convert key_convert; enum hw_cipher_mode mode; dir = desc->qcd_cipher_dir; key_convert = HW_CIPHER_NO_CONVERT; mode = qs->qs_cipher_mode; switch (mode) { case HW_CIPHER_CBC_MODE: case HW_CIPHER_XTS_MODE: algo = qs->qs_cipher_algo; /* * AES decrypt key needs to be reversed. * Instead of reversing the key at session registration, * it is instead reversed on-the-fly by setting the KEY_CONVERT * bit here. */ if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT) key_convert = HW_CIPHER_KEY_CONVERT; break; case HW_CIPHER_CTR_MODE: algo = qs->qs_cipher_algo; dir = HW_CIPHER_ENCRYPT; break; default: panic("unhandled cipher mode %d", mode); break; } return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir); } uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct qat_sym_hash_def **hash_def) { enum qat_sym_hash_algorithm algo; switch (qs->qs_auth_algo) { case HW_AUTH_ALGO_SHA1: algo = QAT_SYM_HASH_SHA1; break; case HW_AUTH_ALGO_SHA256: algo = QAT_SYM_HASH_SHA256; break; case HW_AUTH_ALGO_SHA384: algo = QAT_SYM_HASH_SHA384; break; case HW_AUTH_ALGO_SHA512: algo = QAT_SYM_HASH_SHA512; break; case HW_AUTH_ALGO_GALOIS_128: algo = QAT_SYM_HASH_AES_GCM; break; default: panic("unhandled auth algorithm %d", qs->qs_auth_algo); break; } *hash_def = &qat_sym_hash_defs[algo]; return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode, (*hash_def)->qshd_qat->qshqi_algo_enc, (*hash_def)->qshd_alg->qshai_digest_len); } struct qat_crypto_load_cb_arg { struct qat_session *qs; struct qat_sym_cookie *qsc; struct cryptop *crp; int error; }; static int qat_crypto_populate_buf_list(struct buffer_list_desc *buffers, bus_dma_segment_t *segs, int niseg, int noseg, int skip) { struct flat_buffer_desc *flatbuf; bus_addr_t addr; bus_size_t len; int iseg, oseg; for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG; iseg++) { addr = segs[iseg].ds_addr; len = segs[iseg].ds_len; if (skip > 0) { if (skip < len) { addr += skip; len -= skip; skip = 0; } else { skip -= len; continue; } } flatbuf = &buffers->flat_bufs[oseg++]; flatbuf->data_len_in_bytes = (uint32_t)len; flatbuf->phy_buffer = (uint64_t)addr; } buffers->num_buffers = oseg; return iseg < niseg ? E2BIG : 0; } static void qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct qat_crypto_load_cb_arg *arg; struct qat_sym_cookie *qsc; arg = _arg; if (error != 0) { arg->error = error; return; } qsc = arg->qsc; arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, nseg, 0, 0); } static void qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct cryptop *crp; struct qat_crypto_load_cb_arg *arg; struct qat_session *qs; struct qat_sym_cookie *qsc; int noseg, skip; arg = _arg; if (error != 0) { arg->error = error; return; } crp = arg->crp; qs = arg->qs; qsc = arg->qsc; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { /* AAD was handled in qat_crypto_load(). */ skip = crp->crp_payload_start; noseg = 0; } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) { skip = crp->crp_aad_start; noseg = 0; } else { skip = crp->crp_payload_start; noseg = crp->crp_aad == NULL ? 0 : qsc->qsc_buf_list.num_buffers; } arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, nseg, noseg, skip); } static void qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct buffer_list_desc *ibufs, *obufs; struct flat_buffer_desc *ibuf, *obuf; struct cryptop *crp; struct qat_crypto_load_cb_arg *arg; struct qat_session *qs; struct qat_sym_cookie *qsc; int buflen, osegs, tocopy; arg = _arg; if (error != 0) { arg->error = error; return; } crp = arg->crp; qs = arg->qs; qsc = arg->qsc; /* * The payload must start at the same offset in the output SG list as in * the input SG list. Copy over SG entries from the input corresponding * to the AAD buffer. */ osegs = 0; if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 && crp->crp_aad_length > 0) { tocopy = crp->crp_aad == NULL ? crp->crp_payload_start - crp->crp_aad_start : crp->crp_aad_length; ibufs = &qsc->qsc_buf_list; obufs = &qsc->qsc_obuf_list; for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) { ibuf = &ibufs->flat_bufs[osegs]; obuf = &obufs->flat_bufs[osegs]; obuf->phy_buffer = ibuf->phy_buffer; buflen = imin(ibuf->data_len_in_bytes, tocopy); obuf->data_len_in_bytes = buflen; tocopy -= buflen; } } arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs, nseg, osegs, crp->crp_payload_output_start); } static int qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc, struct qat_crypto_desc const *desc, struct cryptop *crp) { struct qat_crypto_load_cb_arg arg; int error; crypto_read_iv(crp, qsc->qsc_iv_buf); arg.crp = crp; arg.qs = qs; arg.qsc = qsc; arg.error = 0; error = 0; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 && crp->crp_aad_length > 0) { /* * The firmware expects AAD to be in a contiguous buffer and * padded to a multiple of 16 bytes. To satisfy these * constraints we bounce the AAD into a per-request buffer. * There is a small limit on the AAD size so this is not too * onerous. */ memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX); if (crp->crp_aad == NULL) { crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, qsc->qsc_gcm_aad); } else { memcpy(qsc->qsc_gcm_aad, crp->crp_aad, crp->crp_aad_length); } } else if (crp->crp_aad != NULL) { error = bus_dmamap_load( qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, crp->crp_aad, crp->crp_aad_length, qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } if (error == 0) { error = bus_dmamap_load_crp_buffer( qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, &crp->crp_buf, qat_crypto_load_buf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) { error = bus_dmamap_load_crp_buffer( qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } return error; } static inline struct qat_crypto_bank * qat_crypto_select_bank(struct qat_crypto *qcy) { u_int cpuid = PCPU_GET(cpuid); return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks]; } static int qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb) { char *name; int bank, curname, error, i, j; bank = qcb->qcb_bank; curname = 0; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size, NULL, NULL, name, &qcb->qcb_sym_tx); if (error) return error; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size, qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx); if (error) return error; for (i = 0; i < QAT_NSYMCOOKIE; i++) { struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i]; struct qat_sym_cookie *qsc; error = qat_alloc_dmamem(sc, qdm, 1, sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN); if (error) return error; qsc = qdm->qdm_dma_vaddr; qsc->qsc_self_dmamap = qdm->qdm_dma_map; qsc->qsc_self_dma_tag = qdm->qdm_dma_tag; qsc->qsc_bulk_req_params_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_bulk_cookie.qsbc_req_params_buf); qsc->qsc_buffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_buf_list); qsc->qsc_obuffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_obuf_list); qsc->qsc_obuffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_obuf_list); qsc->qsc_iv_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_iv_buf); qsc->qsc_auth_res_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_auth_res); qsc->qsc_gcm_aad_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_gcm_aad); qsc->qsc_content_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_content_desc); qcb->qcb_symck_free[i] = qsc; qcb->qcb_symck_free_count++; for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ QAT_MAXLEN, /* maxsize */ QAT_MAXSEG, /* nsegments */ QAT_MAXLEN, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &qsc->qsc_dma[j].qsd_dma_tag); if (error != 0) return error; error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag, BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap); if (error != 0) return error; } } return 0; } static int qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb) { mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF); return qat_crypto_setup_ring(sc, qcb); } static void qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb) { struct qat_dmamem *qdm; struct qat_sym_cookie *qsc; int i, j; for (i = 0; i < QAT_NSYMCOOKIE; i++) { qdm = &qcb->qcb_symck_dmamems[i]; qsc = qcb->qcb_symck_free[i]; for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag, qsc->qsc_dma[j].qsd_dmamap); bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag); } qat_free_dmamem(sc, qdm); } qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma); qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma); mtx_destroy(&qcb->qcb_bank_mtx); } static int qat_crypto_init(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; int bank, error, num_banks; qcy->qcy_sc = sc; if (sc->sc_hw.qhw_init_arb != NULL) num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks); else num_banks = sc->sc_ae_num; qcy->qcy_num_banks = num_banks; qcy->qcy_banks = qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks); for (bank = 0; bank < num_banks; bank++) { struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank]; qcb->qcb_bank = bank; error = qat_crypto_bank_init(sc, qcb); if (error) return error; } mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF); ctx = device_get_sysctl_ctx(sc->sc_dev); oid = device_get_sysctl_tree(sc->sc_dev); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts", CTLFLAG_RD, &sc->sc_gcm_aad_restarts, "GCM requests deferred due to AAD size change"); sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates", CTLFLAG_RD, &sc->sc_gcm_aad_updates, "GCM requests that required session state update"); sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full", CTLFLAG_RD, &sc->sc_ring_full_restarts, "Requests deferred due to in-flight max reached"); sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures", CTLFLAG_RD, &sc->sc_sym_alloc_failures, "Request allocation failures"); return 0; } static void qat_crypto_deinit(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct qat_crypto_bank *qcb; int bank; counter_u64_free(sc->sc_sym_alloc_failures); counter_u64_free(sc->sc_ring_full_restarts); counter_u64_free(sc->sc_gcm_aad_updates); counter_u64_free(sc->sc_gcm_aad_restarts); if (qcy->qcy_banks != NULL) { for (bank = 0; bank < qcy->qcy_num_banks; bank++) { qcb = &qcy->qcy_banks[bank]; qat_crypto_bank_deinit(sc, qcb); } qat_free_mem(qcy->qcy_banks); mtx_destroy(&qcy->qcy_crypto_mtx); } } static int qat_crypto_start(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; qcy->qcy_cid = crypto_get_driverid(sc->sc_dev, sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE); if (qcy->qcy_cid < 0) { device_printf(sc->sc_dev, "could not get opencrypto driver id\n"); return ENOENT; } return 0; } static void qat_crypto_stop(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; if (qcy->qcy_cid >= 0) (void)crypto_unregister_all(qcy->qcy_cid); } static void qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i) { bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap); } static int qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg) { char icv[QAT_SYM_HASH_BUFFER_LEN]; struct qat_crypto_bank *qcb = arg; struct qat_crypto *qcy; struct qat_session *qs; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; struct cryptop *crp; int error; uint16_t auth_sz; bool blocked; qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset); qsbc = &qsc->qsc_bulk_cookie; qcy = qsbc->qsbc_crypto; qs = qsbc->qsbc_session; crp = qsbc->qsbc_cb_tag; bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); if (crp->crp_aad != NULL) qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF); qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF); error = 0; if ((auth_sz = qs->qs_auth_mlen) != 0) { if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { crypto_copydata(crp, crp->crp_digest_start, auth_sz, icv); if (timingsafe_bcmp(icv, qsc->qsc_auth_res, auth_sz) != 0) { error = EBADMSG; } } else { crypto_copyback(crp, crp->crp_digest_start, auth_sz, qsc->qsc_auth_res); } } qat_crypto_free_sym_cookie(qcb, qsc); blocked = false; mtx_lock(&qs->qs_session_mtx); MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE); qs->qs_inflight--; if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) { blocked = true; qs->qs_need_wakeup = false; } mtx_unlock(&qs->qs_session_mtx); crp->crp_etype = error; crypto_done(crp); if (blocked) crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ); return 1; } static int qat_probesession(device_t dev, const struct crypto_session_params *csp) { if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) return EINVAL; if (csp->csp_cipher_alg == CRYPTO_AES_XTS && qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) { /* * AES-XTS is not supported by the NanoQAT. */ return EINVAL; } switch (csp->csp_mode) { case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_DIGEST: switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: break; case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: break; default: return EINVAL; } break; case CSP_MODE_ETA: switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; default: return EINVAL; } break; default: return EINVAL; } return CRYPTODEV_PROBE_HARDWARE; } static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct qat_crypto *qcy; struct qat_dmamem *qdm; struct qat_session *qs; struct qat_softc *sc; struct qat_crypto_desc *ddesc, *edesc; int error, slices; sc = device_get_softc(dev); qs = crypto_get_driver_session(cses); qcy = &sc->sc_crypto; qdm = &qs->qs_desc_mem; error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG, sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN); if (error != 0) return error; mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF); qs->qs_aad_length = -1; qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr; qs->qs_enc_desc = edesc = ddesc + 1; ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr; ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr + sizeof(struct qat_crypto_desc); edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); qs->qs_status = QAT_SESSION_STATUS_ACTIVE; qs->qs_inflight = 0; qs->qs_cipher_key = csp->csp_cipher_key; qs->qs_cipher_klen = csp->csp_cipher_klen; qs->qs_auth_key = csp->csp_auth_key; qs->qs_auth_klen = csp->csp_auth_klen; switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CBC_MODE; break; case CRYPTO_AES_ICM: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; break; case CRYPTO_AES_XTS: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen / 2); qs->qs_cipher_mode = HW_CIPHER_XTS_MODE; break; case CRYPTO_AES_NIST_GCM_16: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; break; case 0: break; default: panic("%s: unhandled cipher algorithm %d", __func__, csp->csp_cipher_alg); } switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA1: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_256_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_256: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_384_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_384: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_512_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_512: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_AES_NIST_GMAC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; qs->qs_cipher_key = qs->qs_auth_key; qs->qs_cipher_klen = qs->qs_auth_klen; break; case 0: break; default: panic("%s: unhandled auth algorithm %d", __func__, csp->csp_auth_alg); } slices = 0; switch (csp->csp_mode) { case CSP_MODE_AEAD: case CSP_MODE_ETA: /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; break; case CSP_MODE_CIPHER: /* decrypt */ ddesc->qcd_slices[0] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER; /* encrypt */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER; slices = 1; break; case CSP_MODE_DIGEST: if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; } else { ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_cmd_id = FW_LA_CMD_AUTH; edesc->qcd_slices[0] = FW_SLICE_AUTH; edesc->qcd_cmd_id = FW_LA_CMD_AUTH; slices = 1; } break; default: panic("%s: unhandled crypto algorithm %d, %d", __func__, csp->csp_cipher_alg, csp->csp_auth_alg); } ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc); qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc); if (csp->csp_auth_mlen != 0) qs->qs_auth_mlen = csp->csp_auth_mlen; else qs->qs_auth_mlen = edesc->qcd_auth_sz; /* Compute the GMAC by specifying a null cipher payload. */ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH; return 0; } static void qat_crypto_clear_desc(struct qat_crypto_desc *desc) { explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc)); explicit_bzero(desc->qcd_hash_state_prefix_buf, sizeof(desc->qcd_hash_state_prefix_buf)); explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache)); } static void qat_freesession(device_t dev, crypto_session_t cses) { struct qat_session *qs; qs = crypto_get_driver_session(cses); KASSERT(qs->qs_inflight == 0, ("%s: session %p has requests in flight", __func__, qs)); qat_crypto_clear_desc(qs->qs_enc_desc); qat_crypto_clear_desc(qs->qs_dec_desc); qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem); mtx_destroy(&qs->qs_session_mtx); } static int qat_process(device_t dev, struct cryptop *crp, int hint) { struct qat_crypto *qcy; struct qat_crypto_bank *qcb; struct qat_crypto_desc const *desc; struct qat_session *qs; struct qat_softc *sc; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; int error; sc = device_get_softc(dev); qcy = &sc->sc_crypto; qs = crypto_get_driver_session(crp->crp_session); qsc = NULL; if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) { error = E2BIG; goto fail1; } mtx_lock(&qs->qs_session_mtx); if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) { error = E2BIG; mtx_unlock(&qs->qs_session_mtx); goto fail1; } /* * The firmware interface for GCM annoyingly requires the AAD * size to be stored in the session's content descriptor, which * is not really meant to be updated after session * initialization. For IPSec the AAD size is fixed so this is * not much of a problem in practice, but we have to catch AAD * size updates here so that the device code can safely update * the session's recorded AAD size. */ if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) { if (qs->qs_inflight == 0) { if (qs->qs_aad_length != -1) { counter_u64_add(sc->sc_gcm_aad_updates, 1); } qs->qs_aad_length = crp->crp_aad_length; } else { qs->qs_need_wakeup = true; mtx_unlock(&qs->qs_session_mtx); counter_u64_add(sc->sc_gcm_aad_restarts, 1); error = ERESTART; goto fail1; } } } qs->qs_inflight++; mtx_unlock(&qs->qs_session_mtx); qcb = qat_crypto_select_bank(qcy); qsc = qat_crypto_alloc_sym_cookie(qcb); if (qsc == NULL) { counter_u64_add(sc->sc_sym_alloc_failures, 1); error = ENOBUFS; goto fail2; } if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) desc = qs->qs_enc_desc; else desc = qs->qs_dec_desc; error = qat_crypto_load(qs, qsc, desc, crp); if (error != 0) goto fail2; qsbc = &qsc->qsc_bulk_cookie; qsbc->qsbc_crypto = qcy; qsbc->qsbc_session = qs; qsbc->qsbc_cb_tag = crp; sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp); if (crp->crp_aad != NULL) { bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); } bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); } bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); error = qat_etr_put_msg(sc, qcb->qcb_sym_tx, (uint32_t *)qsbc->qsbc_msg); if (error) goto fail2; return 0; fail2: if (qsc) qat_crypto_free_sym_cookie(qcb, qsc); mtx_lock(&qs->qs_session_mtx); qs->qs_inflight--; mtx_unlock(&qs->qs_session_mtx); fail1: crp->crp_etype = error; crypto_done(crp); return 0; } static device_method_t qat_methods[] = { /* Device interface */ DEVMETHOD(device_probe, qat_probe), DEVMETHOD(device_attach, qat_attach), DEVMETHOD(device_detach, qat_detach), /* Cryptodev interface */ DEVMETHOD(cryptodev_probesession, qat_probesession), DEVMETHOD(cryptodev_newsession, qat_newsession), DEVMETHOD(cryptodev_freesession, qat_freesession), DEVMETHOD(cryptodev_process, qat_process), DEVMETHOD_END }; static driver_t qat_driver = { .name = "qat_c2xxx", .methods = qat_methods, .size = sizeof(struct qat_softc), }; DRIVER_MODULE(qat_c2xxx, pci, qat_driver, 0, 0); MODULE_VERSION(qat_c2xxx, 1); MODULE_DEPEND(qat_c2xxx, crypto, 1, 1, 1); MODULE_DEPEND(qat_c2xxx, firmware, 1, 1, 1); MODULE_DEPEND(qat_c2xxx, pci, 1, 1, 1);