1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */ 2 /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */ 3 4 /* 5 * Copyright (c) 2019 Internet Initiative Japan, Inc. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 /* 31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 37 * * Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * * Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in 41 * the documentation and/or other materials provided with the 42 * distribution. 43 * * Neither the name of Intel Corporation nor the names of its 44 * contributors may be used to endorse or promote products derived 45 * from this software without specific prior written permission. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 */ 59 60 #include <sys/cdefs.h> 61 #if 0 62 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $"); 63 #endif 64 65 #include <sys/param.h> 66 #include <sys/systm.h> 67 #include <sys/bus.h> 68 #include <sys/cpu.h> 69 #include <sys/firmware.h> 70 #include <sys/kernel.h> 71 #include <sys/mbuf.h> 72 #include <sys/md5.h> 73 #include <sys/module.h> 74 #include <sys/mutex.h> 75 #include <sys/smp.h> 76 #include <sys/sysctl.h> 77 #include <sys/rman.h> 78 79 #include <machine/bus.h> 80 81 #include <opencrypto/cryptodev.h> 82 #include <opencrypto/xform.h> 83 84 #include "cryptodev_if.h" 85 86 #include <dev/pci/pcireg.h> 87 #include <dev/pci/pcivar.h> 88 89 #include "qatreg.h" 90 #include "qatvar.h" 91 #include "qat_aevar.h" 92 93 extern struct qat_hw qat_hw_c2xxx; 94 95 #define PCI_VENDOR_INTEL 0x8086 96 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18 97 98 static const struct qat_product { 99 uint16_t qatp_vendor; 100 uint16_t qatp_product; 101 const char *qatp_name; 102 enum qat_chip_type qatp_chip; 103 const struct qat_hw *qatp_hw; 104 } qat_products[] = { 105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS, 106 "Intel C2000 QuickAssist PF", 107 QAT_CHIP_C2XXX, &qat_hw_c2xxx }, 108 { 0, 0, NULL, 0, NULL }, 109 }; 110 111 /* Hash Algorithm specific structure */ 112 113 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ 114 static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = { 115 0x67, 0x45, 0x23, 0x01, 116 0xef, 0xcd, 0xab, 0x89, 117 0x98, 0xba, 0xdc, 0xfe, 118 0x10, 0x32, 0x54, 0x76, 119 0xc3, 0xd2, 0xe1, 0xf0 120 }; 121 122 /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ 123 static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = { 124 0x6a, 0x09, 0xe6, 0x67, 125 0xbb, 0x67, 0xae, 0x85, 126 0x3c, 0x6e, 0xf3, 0x72, 127 0xa5, 0x4f, 0xf5, 0x3a, 128 0x51, 0x0e, 0x52, 0x7f, 129 0x9b, 0x05, 0x68, 0x8c, 130 0x1f, 0x83, 0xd9, 0xab, 131 0x5b, 0xe0, 0xcd, 0x19 132 }; 133 134 /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ 135 static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = { 136 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 137 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, 138 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 139 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 140 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 141 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 142 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 143 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 144 }; 145 146 /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ 147 static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = { 148 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 149 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 150 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, 151 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 152 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 153 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 154 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 155 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 156 }; 157 158 static const struct qat_sym_hash_alg_info sha1_info = { 159 .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE, 160 .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE, 161 .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE, 162 .qshai_init_state = sha1_initial_state, 163 .qshai_sah = &auth_hash_hmac_sha1, 164 .qshai_state_offset = 0, 165 .qshai_state_word = 4, 166 }; 167 168 static const struct qat_sym_hash_alg_info sha256_info = { 169 .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE, 170 .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE, 171 .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE, 172 .qshai_init_state = sha256_initial_state, 173 .qshai_sah = &auth_hash_hmac_sha2_256, 174 .qshai_state_offset = offsetof(SHA256_CTX, state), 175 .qshai_state_word = 4, 176 }; 177 178 static const struct qat_sym_hash_alg_info sha384_info = { 179 .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE, 180 .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE, 181 .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE, 182 .qshai_init_state = sha384_initial_state, 183 .qshai_sah = &auth_hash_hmac_sha2_384, 184 .qshai_state_offset = offsetof(SHA384_CTX, state), 185 .qshai_state_word = 8, 186 }; 187 188 static const struct qat_sym_hash_alg_info sha512_info = { 189 .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE, 190 .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE, 191 .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE, 192 .qshai_init_state = sha512_initial_state, 193 .qshai_sah = &auth_hash_hmac_sha2_512, 194 .qshai_state_offset = offsetof(SHA512_CTX, state), 195 .qshai_state_word = 8, 196 }; 197 198 static const struct qat_sym_hash_alg_info aes_gcm_info = { 199 .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE, 200 .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE, 201 .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE, 202 .qshai_sah = &auth_hash_nist_gmac_aes_128, 203 }; 204 205 /* Hash QAT specific structures */ 206 207 static const struct qat_sym_hash_qat_info sha1_config = { 208 .qshqi_algo_enc = HW_AUTH_ALGO_SHA1, 209 .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE, 210 .qshqi_state1_len = HW_SHA1_STATE1_SZ, 211 .qshqi_state2_len = HW_SHA1_STATE2_SZ, 212 }; 213 214 static const struct qat_sym_hash_qat_info sha256_config = { 215 .qshqi_algo_enc = HW_AUTH_ALGO_SHA256, 216 .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE, 217 .qshqi_state1_len = HW_SHA256_STATE1_SZ, 218 .qshqi_state2_len = HW_SHA256_STATE2_SZ 219 }; 220 221 static const struct qat_sym_hash_qat_info sha384_config = { 222 .qshqi_algo_enc = HW_AUTH_ALGO_SHA384, 223 .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE, 224 .qshqi_state1_len = HW_SHA384_STATE1_SZ, 225 .qshqi_state2_len = HW_SHA384_STATE2_SZ 226 }; 227 228 static const struct qat_sym_hash_qat_info sha512_config = { 229 .qshqi_algo_enc = HW_AUTH_ALGO_SHA512, 230 .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE, 231 .qshqi_state1_len = HW_SHA512_STATE1_SZ, 232 .qshqi_state2_len = HW_SHA512_STATE2_SZ 233 }; 234 235 static const struct qat_sym_hash_qat_info aes_gcm_config = { 236 .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128, 237 .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE, 238 .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ, 239 .qshqi_state2_len = 240 HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ, 241 }; 242 243 static const struct qat_sym_hash_def qat_sym_hash_defs[] = { 244 [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config }, 245 [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config }, 246 [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config }, 247 [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config }, 248 [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config }, 249 }; 250 251 static const struct qat_product *qat_lookup(device_t); 252 static int qat_probe(device_t); 253 static int qat_attach(device_t); 254 static int qat_init(device_t); 255 static int qat_start(device_t); 256 static int qat_detach(device_t); 257 258 static int qat_newsession(device_t dev, crypto_session_t cses, 259 const struct crypto_session_params *csp); 260 static void qat_freesession(device_t dev, crypto_session_t cses); 261 262 static int qat_setup_msix_intr(struct qat_softc *); 263 264 static void qat_etr_init(struct qat_softc *); 265 static void qat_etr_deinit(struct qat_softc *); 266 static void qat_etr_bank_init(struct qat_softc *, int); 267 static void qat_etr_bank_deinit(struct qat_softc *sc, int); 268 269 static void qat_etr_ap_bank_init(struct qat_softc *); 270 static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int); 271 static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *, 272 uint32_t, int); 273 static void qat_etr_ap_bank_setup_ring(struct qat_softc *, 274 struct qat_ring *); 275 static int qat_etr_verify_ring_size(uint32_t, uint32_t); 276 277 static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *, 278 struct qat_ring *); 279 static void qat_etr_bank_intr(void *); 280 281 static void qat_arb_update(struct qat_softc *, struct qat_bank *); 282 283 static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie( 284 struct qat_crypto_bank *); 285 static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *, 286 struct qat_sym_cookie *); 287 static int qat_crypto_setup_ring(struct qat_softc *, 288 struct qat_crypto_bank *); 289 static int qat_crypto_bank_init(struct qat_softc *, 290 struct qat_crypto_bank *); 291 static int qat_crypto_init(struct qat_softc *); 292 static void qat_crypto_deinit(struct qat_softc *); 293 static int qat_crypto_start(struct qat_softc *); 294 static void qat_crypto_stop(struct qat_softc *); 295 static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *); 296 297 static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver"); 298 299 static const struct qat_product * 300 qat_lookup(device_t dev) 301 { 302 const struct qat_product *qatp; 303 304 for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) { 305 if (pci_get_vendor(dev) == qatp->qatp_vendor && 306 pci_get_device(dev) == qatp->qatp_product) 307 return qatp; 308 } 309 return NULL; 310 } 311 312 static int 313 qat_probe(device_t dev) 314 { 315 const struct qat_product *prod; 316 317 prod = qat_lookup(dev); 318 if (prod != NULL) { 319 device_set_desc(dev, prod->qatp_name); 320 return BUS_PROBE_DEFAULT; 321 } 322 return ENXIO; 323 } 324 325 static int 326 qat_attach(device_t dev) 327 { 328 struct qat_softc *sc = device_get_softc(dev); 329 const struct qat_product *qatp; 330 int bar, count, error, i; 331 332 sc->sc_dev = dev; 333 sc->sc_rev = pci_get_revid(dev); 334 sc->sc_crypto.qcy_cid = -1; 335 336 qatp = qat_lookup(dev); 337 memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw)); 338 339 /* Determine active accelerators and engines */ 340 sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc); 341 sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc); 342 343 sc->sc_accel_num = 0; 344 for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) { 345 if (sc->sc_accel_mask & (1 << i)) 346 sc->sc_accel_num++; 347 } 348 sc->sc_ae_num = 0; 349 for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) { 350 if (sc->sc_ae_mask & (1 << i)) 351 sc->sc_ae_num++; 352 } 353 354 if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) { 355 device_printf(sc->sc_dev, "couldn't find acceleration"); 356 goto fail; 357 } 358 359 MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL); 360 MPASS(sc->sc_ae_num <= MAX_NUM_AE); 361 362 /* Determine SKU and capabilities */ 363 sc->sc_sku = sc->sc_hw.qhw_get_sku(sc); 364 sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc); 365 sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc); 366 367 i = 0; 368 if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) { 369 MPASS(sc->sc_hw.qhw_sram_bar_id == 0); 370 uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4); 371 /* Skip SRAM BAR */ 372 i = (fusectl & FUSECTL_MASK) ? 1 : 0; 373 } 374 for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) { 375 uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4); 376 if (val == 0 || !PCI_BAR_MEM(val)) 377 continue; 378 379 sc->sc_rid[i] = PCIR_BAR(bar); 380 sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 381 &sc->sc_rid[i], RF_ACTIVE); 382 if (sc->sc_res[i] == NULL) { 383 device_printf(dev, "couldn't map BAR %d\n", bar); 384 goto fail; 385 } 386 387 sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]); 388 sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]); 389 390 i++; 391 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) 392 bar++; 393 } 394 395 pci_enable_busmaster(dev); 396 397 count = sc->sc_hw.qhw_num_banks + 1; 398 if (pci_msix_count(dev) < count) { 399 device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n", 400 pci_msix_count(dev), count); 401 goto fail; 402 } 403 error = pci_alloc_msix(dev, &count); 404 if (error != 0) { 405 device_printf(dev, "failed to allocate MSI-X vectors\n"); 406 goto fail; 407 } 408 409 error = qat_init(dev); 410 if (error == 0) 411 return 0; 412 413 fail: 414 qat_detach(dev); 415 return ENXIO; 416 } 417 418 static int 419 qat_init(device_t dev) 420 { 421 struct qat_softc *sc = device_get_softc(dev); 422 int error; 423 424 qat_etr_init(sc); 425 426 if (sc->sc_hw.qhw_init_admin_comms != NULL && 427 (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) { 428 device_printf(sc->sc_dev, 429 "Could not initialize admin comms: %d\n", error); 430 return error; 431 } 432 433 if (sc->sc_hw.qhw_init_arb != NULL && 434 (error = sc->sc_hw.qhw_init_arb(sc)) != 0) { 435 device_printf(sc->sc_dev, 436 "Could not initialize hw arbiter: %d\n", error); 437 return error; 438 } 439 440 error = qat_ae_init(sc); 441 if (error) { 442 device_printf(sc->sc_dev, 443 "Could not initialize Acceleration Engine: %d\n", error); 444 return error; 445 } 446 447 error = qat_aefw_load(sc); 448 if (error) { 449 device_printf(sc->sc_dev, 450 "Could not load firmware: %d\n", error); 451 return error; 452 } 453 454 error = qat_setup_msix_intr(sc); 455 if (error) { 456 device_printf(sc->sc_dev, 457 "Could not setup interrupts: %d\n", error); 458 return error; 459 } 460 461 sc->sc_hw.qhw_enable_intr(sc); 462 463 error = qat_crypto_init(sc); 464 if (error) { 465 device_printf(sc->sc_dev, 466 "Could not initialize service: %d\n", error); 467 return error; 468 } 469 470 if (sc->sc_hw.qhw_enable_error_correction != NULL) 471 sc->sc_hw.qhw_enable_error_correction(sc); 472 473 if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL && 474 (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) { 475 device_printf(sc->sc_dev, 476 "Could not initialize watchdog timer: %d\n", error); 477 return error; 478 } 479 480 error = qat_start(dev); 481 if (error) { 482 device_printf(sc->sc_dev, 483 "Could not start: %d\n", error); 484 return error; 485 } 486 487 return 0; 488 } 489 490 static int 491 qat_start(device_t dev) 492 { 493 struct qat_softc *sc = device_get_softc(dev); 494 int error; 495 496 error = qat_ae_start(sc); 497 if (error) 498 return error; 499 500 if (sc->sc_hw.qhw_send_admin_init != NULL && 501 (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) { 502 return error; 503 } 504 505 error = qat_crypto_start(sc); 506 if (error) 507 return error; 508 509 return 0; 510 } 511 512 static int 513 qat_detach(device_t dev) 514 { 515 struct qat_softc *sc; 516 int bar, i; 517 518 sc = device_get_softc(dev); 519 520 qat_crypto_stop(sc); 521 qat_crypto_deinit(sc); 522 qat_aefw_unload(sc); 523 524 if (sc->sc_etr_banks != NULL) { 525 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) { 526 struct qat_bank *qb = &sc->sc_etr_banks[i]; 527 528 if (qb->qb_ih_cookie != NULL) 529 (void)bus_teardown_intr(dev, qb->qb_ih, 530 qb->qb_ih_cookie); 531 if (qb->qb_ih != NULL) 532 (void)bus_release_resource(dev, SYS_RES_IRQ, 533 i + 1, qb->qb_ih); 534 } 535 } 536 if (sc->sc_ih_cookie != NULL) { 537 (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie); 538 sc->sc_ih_cookie = NULL; 539 } 540 if (sc->sc_ih != NULL) { 541 (void)bus_release_resource(dev, SYS_RES_IRQ, 542 sc->sc_hw.qhw_num_banks + 1, sc->sc_ih); 543 sc->sc_ih = NULL; 544 } 545 pci_release_msi(dev); 546 547 qat_etr_deinit(sc); 548 549 for (bar = 0; bar < MAX_BARS; bar++) { 550 if (sc->sc_res[bar] != NULL) { 551 (void)bus_release_resource(dev, SYS_RES_MEMORY, 552 sc->sc_rid[bar], sc->sc_res[bar]); 553 sc->sc_res[bar] = NULL; 554 } 555 } 556 557 return 0; 558 } 559 560 void * 561 qat_alloc_mem(size_t size) 562 { 563 return (malloc(size, M_QAT, M_WAITOK | M_ZERO)); 564 } 565 566 void 567 qat_free_mem(void *ptr) 568 { 569 free(ptr, M_QAT); 570 } 571 572 static void 573 qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg, 574 int error) 575 { 576 struct qat_dmamem *qdm; 577 578 if (error != 0) 579 return; 580 581 KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); 582 qdm = arg; 583 qdm->qdm_dma_seg = segs[0]; 584 } 585 586 int 587 qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm, 588 int nseg, bus_size_t size, bus_size_t alignment) 589 { 590 int error; 591 592 KASSERT(qdm->qdm_dma_vaddr == NULL, 593 ("%s: DMA memory descriptor in use", __func__)); 594 595 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 596 alignment, 0, /* alignment, boundary */ 597 BUS_SPACE_MAXADDR, /* lowaddr */ 598 BUS_SPACE_MAXADDR, /* highaddr */ 599 NULL, NULL, /* filter, filterarg */ 600 size, /* maxsize */ 601 nseg, /* nsegments */ 602 size, /* maxsegsize */ 603 BUS_DMA_COHERENT, /* flags */ 604 NULL, NULL, /* lockfunc, lockarg */ 605 &qdm->qdm_dma_tag); 606 if (error != 0) 607 return error; 608 609 error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr, 610 BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, 611 &qdm->qdm_dma_map); 612 if (error != 0) { 613 device_printf(sc->sc_dev, 614 "couldn't allocate dmamem, error = %d\n", error); 615 goto fail_0; 616 } 617 618 error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map, 619 qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm, 620 BUS_DMA_NOWAIT); 621 if (error) { 622 device_printf(sc->sc_dev, 623 "couldn't load dmamem map, error = %d\n", error); 624 goto fail_1; 625 } 626 627 return 0; 628 fail_1: 629 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); 630 fail_0: 631 bus_dma_tag_destroy(qdm->qdm_dma_tag); 632 return error; 633 } 634 635 void 636 qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm) 637 { 638 if (qdm->qdm_dma_tag != NULL) { 639 bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map); 640 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, 641 qdm->qdm_dma_map); 642 bus_dma_tag_destroy(qdm->qdm_dma_tag); 643 explicit_bzero(qdm, sizeof(*qdm)); 644 } 645 } 646 647 static int 648 qat_setup_msix_intr(struct qat_softc *sc) 649 { 650 device_t dev; 651 int error, i, rid; 652 653 dev = sc->sc_dev; 654 655 for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) { 656 struct qat_bank *qb = &sc->sc_etr_banks[i - 1]; 657 658 rid = i; 659 qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 660 RF_ACTIVE); 661 if (qb->qb_ih == NULL) { 662 device_printf(dev, 663 "failed to allocate bank intr resource\n"); 664 return ENXIO; 665 } 666 error = bus_setup_intr(dev, qb->qb_ih, 667 INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb, 668 &qb->qb_ih_cookie); 669 if (error != 0) { 670 device_printf(dev, "failed to set up bank intr\n"); 671 return error; 672 } 673 error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus); 674 if (error != 0) 675 device_printf(dev, "failed to bind intr %d\n", i); 676 } 677 678 rid = i; 679 sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 680 RF_ACTIVE); 681 if (sc->sc_ih == NULL) 682 return ENXIO; 683 error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE, 684 NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie); 685 686 return error; 687 } 688 689 static void 690 qat_etr_init(struct qat_softc *sc) 691 { 692 int i; 693 694 sc->sc_etr_banks = qat_alloc_mem( 695 sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks); 696 697 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) 698 qat_etr_bank_init(sc, i); 699 700 if (sc->sc_hw.qhw_num_ap_banks) { 701 sc->sc_etr_ap_banks = qat_alloc_mem( 702 sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks); 703 qat_etr_ap_bank_init(sc); 704 } 705 } 706 707 static void 708 qat_etr_deinit(struct qat_softc *sc) 709 { 710 int i; 711 712 if (sc->sc_etr_banks != NULL) { 713 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) 714 qat_etr_bank_deinit(sc, i); 715 qat_free_mem(sc->sc_etr_banks); 716 sc->sc_etr_banks = NULL; 717 } 718 if (sc->sc_etr_ap_banks != NULL) { 719 qat_free_mem(sc->sc_etr_ap_banks); 720 sc->sc_etr_ap_banks = NULL; 721 } 722 } 723 724 static void 725 qat_etr_bank_init(struct qat_softc *sc, int bank) 726 { 727 struct qat_bank *qb = &sc->sc_etr_banks[bank]; 728 int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap; 729 730 MPASS(bank < sc->sc_hw.qhw_num_banks); 731 732 mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF); 733 734 qb->qb_sc = sc; 735 qb->qb_bank = bank; 736 qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT; 737 738 /* Clean CSRs for all rings within the bank */ 739 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { 740 struct qat_ring *qr = &qb->qb_et_rings[i]; 741 742 qat_etr_bank_ring_write_4(sc, bank, i, 743 ETR_RING_CONFIG, 0); 744 qat_etr_bank_ring_base_write_8(sc, bank, i, 0); 745 746 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { 747 qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t)); 748 } else if (sc->sc_hw.qhw_tx_rings_mask & 749 (1 << (i - tx_rx_gap))) { 750 /* Share inflight counter with rx and tx */ 751 qr->qr_inflight = 752 qb->qb_et_rings[i - tx_rx_gap].qr_inflight; 753 } 754 } 755 756 if (sc->sc_hw.qhw_init_etr_intr != NULL) { 757 sc->sc_hw.qhw_init_etr_intr(sc, bank); 758 } else { 759 /* common code in qat 1.7 */ 760 qat_etr_bank_write_4(sc, bank, ETR_INT_REG, 761 ETR_INT_REG_CLEAR_MASK); 762 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank / 763 ETR_RINGS_PER_INT_SRCSEL; i++) { 764 qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL + 765 (i * ETR_INT_SRCSEL_NEXT_OFFSET), 766 ETR_INT_SRCSEL_MASK); 767 } 768 } 769 } 770 771 static void 772 qat_etr_bank_deinit(struct qat_softc *sc, int bank) 773 { 774 struct qat_bank *qb; 775 struct qat_ring *qr; 776 int i; 777 778 qb = &sc->sc_etr_banks[bank]; 779 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { 780 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { 781 qr = &qb->qb_et_rings[i]; 782 qat_free_mem(qr->qr_inflight); 783 } 784 } 785 } 786 787 static void 788 qat_etr_ap_bank_init(struct qat_softc *sc) 789 { 790 int ap_bank; 791 792 for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) { 793 struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank]; 794 795 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK, 796 ETR_AP_NF_MASK_INIT); 797 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0); 798 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK, 799 ETR_AP_NE_MASK_INIT); 800 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0); 801 802 memset(qab, 0, sizeof(*qab)); 803 } 804 } 805 806 static void 807 qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask) 808 { 809 if (set_mask) 810 *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); 811 else 812 *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); 813 } 814 815 static void 816 qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest, 817 uint32_t ring, int set_dest) 818 { 819 uint32_t ae_mask; 820 uint8_t mailbox, ae, nae; 821 uint8_t *dest = (uint8_t *)ap_dest; 822 823 mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring); 824 825 nae = 0; 826 ae_mask = sc->sc_ae_mask; 827 for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) { 828 if ((ae_mask & (1 << ae)) == 0) 829 continue; 830 831 if (set_dest) { 832 dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) | 833 __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) | 834 ETR_AP_DEST_ENABLE; 835 } else { 836 dest[nae] = 0; 837 } 838 nae++; 839 if (nae == ETR_MAX_AE_PER_MAILBOX) 840 break; 841 } 842 } 843 844 static void 845 qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr) 846 { 847 struct qat_ap_bank *qab; 848 int ap_bank; 849 850 if (sc->sc_hw.qhw_num_ap_banks == 0) 851 return; 852 853 ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring); 854 MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks); 855 qab = &sc->sc_etr_ap_banks[ap_bank]; 856 857 if (qr->qr_cb == NULL) { 858 qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1); 859 if (!qab->qab_ne_dest) { 860 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest, 861 qr->qr_ring, 1); 862 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 863 qab->qab_ne_dest); 864 } 865 } else { 866 qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1); 867 if (!qab->qab_nf_dest) { 868 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest, 869 qr->qr_ring, 1); 870 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 871 qab->qab_nf_dest); 872 } 873 } 874 } 875 876 static int 877 qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs) 878 { 879 int i = QAT_MIN_RING_SIZE; 880 881 for (; i <= QAT_MAX_RING_SIZE; i++) 882 if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i)) 883 return i; 884 885 return QAT_DEFAULT_RING_SIZE; 886 } 887 888 int 889 qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring, 890 uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg, 891 const char *name, struct qat_ring **rqr) 892 { 893 struct qat_bank *qb; 894 struct qat_ring *qr = NULL; 895 int error; 896 uint32_t ring_size_bytes, ring_config; 897 uint64_t ring_base; 898 uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512; 899 uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0; 900 901 MPASS(bank < sc->sc_hw.qhw_num_banks); 902 903 /* Allocate a ring from specified bank */ 904 qb = &sc->sc_etr_banks[bank]; 905 906 if (ring >= sc->sc_hw.qhw_num_rings_per_bank) 907 return EINVAL; 908 if (qb->qb_allocated_rings & (1 << ring)) 909 return ENOENT; 910 qr = &qb->qb_et_rings[ring]; 911 qb->qb_allocated_rings |= 1 << ring; 912 913 /* Initialize allocated ring */ 914 qr->qr_ring = ring; 915 qr->qr_bank = bank; 916 qr->qr_name = name; 917 qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring; 918 qr->qr_ring_mask = (1 << ring); 919 qr->qr_cb = cb; 920 qr->qr_cb_arg = cb_arg; 921 922 /* Setup the shadow variables */ 923 qr->qr_head = 0; 924 qr->qr_tail = 0; 925 qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size); 926 qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs); 927 928 /* 929 * To make sure that ring is alligned to ring size allocate 930 * at least 4k and then tell the user it is smaller. 931 */ 932 ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size); 933 ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes); 934 error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes, 935 ring_size_bytes); 936 if (error) 937 return error; 938 939 qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr; 940 qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr; 941 942 memset(qr->qr_ring_vaddr, QAT_RING_PATTERN, 943 qr->qr_dma.qdm_dma_seg.ds_len); 944 945 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, 946 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 947 948 if (cb == NULL) { 949 ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size); 950 } else { 951 ring_config = 952 ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne); 953 } 954 qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config); 955 956 ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size); 957 qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base); 958 959 if (sc->sc_hw.qhw_init_arb != NULL) 960 qat_arb_update(sc, qb); 961 962 mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF); 963 964 qat_etr_ap_bank_setup_ring(sc, qr); 965 966 if (cb != NULL) { 967 uint32_t intr_mask; 968 969 qb->qb_intr_mask |= qr->qr_ring_mask; 970 intr_mask = qb->qb_intr_mask; 971 972 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask); 973 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL, 974 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); 975 } 976 977 *rqr = qr; 978 979 return 0; 980 } 981 982 static inline u_int 983 qat_modulo(u_int data, u_int shift) 984 { 985 u_int div = data >> shift; 986 u_int mult = div << shift; 987 return data - mult; 988 } 989 990 int 991 qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg) 992 { 993 uint32_t inflight; 994 uint32_t *addr; 995 996 mtx_lock(&qr->qr_ring_mtx); 997 998 inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1; 999 if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) { 1000 atomic_subtract_32(qr->qr_inflight, 1); 1001 qr->qr_need_wakeup = true; 1002 mtx_unlock(&qr->qr_ring_mtx); 1003 counter_u64_add(sc->sc_ring_full_restarts, 1); 1004 return ERESTART; 1005 } 1006 1007 addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail); 1008 1009 memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size)); 1010 1011 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, 1012 BUS_DMASYNC_PREWRITE); 1013 1014 qr->qr_tail = qat_modulo(qr->qr_tail + 1015 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), 1016 QAT_RING_SIZE_MODULO(qr->qr_ring_size)); 1017 1018 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, 1019 ETR_RING_TAIL_OFFSET, qr->qr_tail); 1020 1021 mtx_unlock(&qr->qr_ring_mtx); 1022 1023 return 0; 1024 } 1025 1026 static int 1027 qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb, 1028 struct qat_ring *qr) 1029 { 1030 uint32_t *msg, nmsg = 0; 1031 int handled = 0; 1032 bool blocked = false; 1033 1034 mtx_lock(&qr->qr_ring_mtx); 1035 1036 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); 1037 1038 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, 1039 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1040 1041 while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) { 1042 atomic_subtract_32(qr->qr_inflight, 1); 1043 1044 if (qr->qr_cb != NULL) { 1045 mtx_unlock(&qr->qr_ring_mtx); 1046 handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg); 1047 mtx_lock(&qr->qr_ring_mtx); 1048 } 1049 1050 atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG); 1051 1052 qr->qr_head = qat_modulo(qr->qr_head + 1053 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), 1054 QAT_RING_SIZE_MODULO(qr->qr_ring_size)); 1055 nmsg++; 1056 1057 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); 1058 } 1059 1060 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, 1061 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1062 1063 if (nmsg > 0) { 1064 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, 1065 ETR_RING_HEAD_OFFSET, qr->qr_head); 1066 if (qr->qr_need_wakeup) { 1067 blocked = true; 1068 qr->qr_need_wakeup = false; 1069 } 1070 } 1071 1072 mtx_unlock(&qr->qr_ring_mtx); 1073 1074 if (blocked) 1075 crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ); 1076 1077 return handled; 1078 } 1079 1080 static void 1081 qat_etr_bank_intr(void *arg) 1082 { 1083 struct qat_bank *qb = arg; 1084 struct qat_softc *sc = qb->qb_sc; 1085 uint32_t estat; 1086 int i; 1087 1088 mtx_lock(&qb->qb_bank_mtx); 1089 1090 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0); 1091 1092 /* Now handle all the responses */ 1093 estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT); 1094 estat &= qb->qb_intr_mask; 1095 1096 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 1097 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); 1098 1099 mtx_unlock(&qb->qb_bank_mtx); 1100 1101 while ((i = ffs(estat)) != 0) { 1102 struct qat_ring *qr = &qb->qb_et_rings[--i]; 1103 estat &= ~(1 << i); 1104 (void)qat_etr_ring_intr(sc, qb, qr); 1105 } 1106 } 1107 1108 void 1109 qat_arb_update(struct qat_softc *sc, struct qat_bank *qb) 1110 { 1111 1112 qat_arb_ringsrvarben_write_4(sc, qb->qb_bank, 1113 qb->qb_allocated_rings & 0xff); 1114 } 1115 1116 static struct qat_sym_cookie * 1117 qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb) 1118 { 1119 struct qat_sym_cookie *qsc; 1120 1121 mtx_lock(&qcb->qcb_bank_mtx); 1122 1123 if (qcb->qcb_symck_free_count == 0) { 1124 mtx_unlock(&qcb->qcb_bank_mtx); 1125 return NULL; 1126 } 1127 1128 qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count]; 1129 1130 mtx_unlock(&qcb->qcb_bank_mtx); 1131 1132 return qsc; 1133 } 1134 1135 static void 1136 qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb, 1137 struct qat_sym_cookie *qsc) 1138 { 1139 explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN); 1140 explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN); 1141 1142 mtx_lock(&qcb->qcb_bank_mtx); 1143 qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc; 1144 mtx_unlock(&qcb->qcb_bank_mtx); 1145 } 1146 1147 void 1148 qat_memcpy_htobe64(void *dst, const void *src, size_t len) 1149 { 1150 uint64_t *dst0 = dst; 1151 const uint64_t *src0 = src; 1152 size_t i; 1153 1154 MPASS(len % sizeof(*dst0) == 0); 1155 1156 for (i = 0; i < len / sizeof(*dst0); i++) 1157 *(dst0 + i) = htobe64(*(src0 + i)); 1158 } 1159 1160 void 1161 qat_memcpy_htobe32(void *dst, const void *src, size_t len) 1162 { 1163 uint32_t *dst0 = dst; 1164 const uint32_t *src0 = src; 1165 size_t i; 1166 1167 MPASS(len % sizeof(*dst0) == 0); 1168 1169 for (i = 0; i < len / sizeof(*dst0); i++) 1170 *(dst0 + i) = htobe32(*(src0 + i)); 1171 } 1172 1173 void 1174 qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte) 1175 { 1176 switch (wordbyte) { 1177 case 4: 1178 qat_memcpy_htobe32(dst, src, len); 1179 break; 1180 case 8: 1181 qat_memcpy_htobe64(dst, src, len); 1182 break; 1183 default: 1184 panic("invalid word size %u", wordbyte); 1185 } 1186 } 1187 1188 void 1189 qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc, 1190 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, 1191 uint8_t *state) 1192 { 1193 uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; 1194 char zeros[AES_BLOCK_LEN]; 1195 int rounds; 1196 1197 memset(zeros, 0, sizeof(zeros)); 1198 rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); 1199 rijndaelEncrypt(ks, rounds, zeros, state); 1200 explicit_bzero(ks, sizeof(ks)); 1201 } 1202 1203 void 1204 qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc, 1205 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, 1206 uint8_t *state1, uint8_t *state2) 1207 { 1208 union authctx ctx; 1209 const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah; 1210 uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset; 1211 uint32_t state_size = hash_def->qshd_alg->qshai_state_size; 1212 uint32_t state_word = hash_def->qshd_alg->qshai_state_word; 1213 1214 hmac_init_ipad(sah, key, klen, &ctx); 1215 qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size, 1216 state_word); 1217 hmac_init_opad(sah, key, klen, &ctx); 1218 qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size, 1219 state_word); 1220 explicit_bzero(&ctx, sizeof(ctx)); 1221 } 1222 1223 static enum hw_cipher_algo 1224 qat_aes_cipher_algo(int klen) 1225 { 1226 switch (klen) { 1227 case HW_AES_128_KEY_SZ: 1228 return HW_CIPHER_ALGO_AES128; 1229 case HW_AES_192_KEY_SZ: 1230 return HW_CIPHER_ALGO_AES192; 1231 case HW_AES_256_KEY_SZ: 1232 return HW_CIPHER_ALGO_AES256; 1233 default: 1234 panic("invalid key length %d", klen); 1235 } 1236 } 1237 1238 uint16_t 1239 qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc, 1240 const struct qat_session *qs) 1241 { 1242 enum hw_cipher_algo algo; 1243 enum hw_cipher_dir dir; 1244 enum hw_cipher_convert key_convert; 1245 enum hw_cipher_mode mode; 1246 1247 dir = desc->qcd_cipher_dir; 1248 key_convert = HW_CIPHER_NO_CONVERT; 1249 mode = qs->qs_cipher_mode; 1250 switch (mode) { 1251 case HW_CIPHER_CBC_MODE: 1252 case HW_CIPHER_XTS_MODE: 1253 algo = qs->qs_cipher_algo; 1254 1255 /* 1256 * AES decrypt key needs to be reversed. 1257 * Instead of reversing the key at session registration, 1258 * it is instead reversed on-the-fly by setting the KEY_CONVERT 1259 * bit here. 1260 */ 1261 if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT) 1262 key_convert = HW_CIPHER_KEY_CONVERT; 1263 break; 1264 case HW_CIPHER_CTR_MODE: 1265 algo = qs->qs_cipher_algo; 1266 dir = HW_CIPHER_ENCRYPT; 1267 break; 1268 default: 1269 panic("unhandled cipher mode %d", mode); 1270 break; 1271 } 1272 1273 return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir); 1274 } 1275 1276 uint16_t 1277 qat_crypto_load_auth_session(const struct qat_crypto_desc *desc, 1278 const struct qat_session *qs, const struct qat_sym_hash_def **hash_def) 1279 { 1280 enum qat_sym_hash_algorithm algo; 1281 1282 switch (qs->qs_auth_algo) { 1283 case HW_AUTH_ALGO_SHA1: 1284 algo = QAT_SYM_HASH_SHA1; 1285 break; 1286 case HW_AUTH_ALGO_SHA256: 1287 algo = QAT_SYM_HASH_SHA256; 1288 break; 1289 case HW_AUTH_ALGO_SHA384: 1290 algo = QAT_SYM_HASH_SHA384; 1291 break; 1292 case HW_AUTH_ALGO_SHA512: 1293 algo = QAT_SYM_HASH_SHA512; 1294 break; 1295 case HW_AUTH_ALGO_GALOIS_128: 1296 algo = QAT_SYM_HASH_AES_GCM; 1297 break; 1298 default: 1299 panic("unhandled auth algorithm %d", qs->qs_auth_algo); 1300 break; 1301 } 1302 *hash_def = &qat_sym_hash_defs[algo]; 1303 1304 return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode, 1305 (*hash_def)->qshd_qat->qshqi_algo_enc, 1306 (*hash_def)->qshd_alg->qshai_digest_len); 1307 } 1308 1309 struct qat_crypto_load_cb_arg { 1310 struct qat_session *qs; 1311 struct qat_sym_cookie *qsc; 1312 struct cryptop *crp; 1313 int error; 1314 }; 1315 1316 static int 1317 qat_crypto_populate_buf_list(struct buffer_list_desc *buffers, 1318 bus_dma_segment_t *segs, int niseg, int noseg, int skip) 1319 { 1320 struct flat_buffer_desc *flatbuf; 1321 bus_addr_t addr; 1322 bus_size_t len; 1323 int iseg, oseg; 1324 1325 for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG; 1326 iseg++) { 1327 addr = segs[iseg].ds_addr; 1328 len = segs[iseg].ds_len; 1329 1330 if (skip > 0) { 1331 if (skip < len) { 1332 addr += skip; 1333 len -= skip; 1334 skip = 0; 1335 } else { 1336 skip -= len; 1337 continue; 1338 } 1339 } 1340 1341 flatbuf = &buffers->flat_bufs[oseg++]; 1342 flatbuf->data_len_in_bytes = (uint32_t)len; 1343 flatbuf->phy_buffer = (uint64_t)addr; 1344 } 1345 buffers->num_buffers = oseg; 1346 return iseg < niseg ? E2BIG : 0; 1347 } 1348 1349 static void 1350 qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, 1351 int error) 1352 { 1353 struct qat_crypto_load_cb_arg *arg; 1354 struct qat_sym_cookie *qsc; 1355 1356 arg = _arg; 1357 if (error != 0) { 1358 arg->error = error; 1359 return; 1360 } 1361 1362 qsc = arg->qsc; 1363 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, 1364 nseg, 0, 0); 1365 } 1366 1367 static void 1368 qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, 1369 int error) 1370 { 1371 struct cryptop *crp; 1372 struct qat_crypto_load_cb_arg *arg; 1373 struct qat_session *qs; 1374 struct qat_sym_cookie *qsc; 1375 int noseg, skip; 1376 1377 arg = _arg; 1378 if (error != 0) { 1379 arg->error = error; 1380 return; 1381 } 1382 1383 crp = arg->crp; 1384 qs = arg->qs; 1385 qsc = arg->qsc; 1386 1387 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { 1388 /* AAD was handled in qat_crypto_load(). */ 1389 skip = crp->crp_payload_start; 1390 noseg = 0; 1391 } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) { 1392 skip = crp->crp_aad_start; 1393 noseg = 0; 1394 } else { 1395 skip = crp->crp_payload_start; 1396 noseg = crp->crp_aad == NULL ? 1397 0 : qsc->qsc_buf_list.num_buffers; 1398 } 1399 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, 1400 nseg, noseg, skip); 1401 } 1402 1403 static void 1404 qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, 1405 int error) 1406 { 1407 struct buffer_list_desc *ibufs, *obufs; 1408 struct flat_buffer_desc *ibuf, *obuf; 1409 struct cryptop *crp; 1410 struct qat_crypto_load_cb_arg *arg; 1411 struct qat_session *qs; 1412 struct qat_sym_cookie *qsc; 1413 int buflen, osegs, tocopy; 1414 1415 arg = _arg; 1416 if (error != 0) { 1417 arg->error = error; 1418 return; 1419 } 1420 1421 crp = arg->crp; 1422 qs = arg->qs; 1423 qsc = arg->qsc; 1424 1425 /* 1426 * The payload must start at the same offset in the output SG list as in 1427 * the input SG list. Copy over SG entries from the input corresponding 1428 * to the AAD buffer. 1429 */ 1430 osegs = 0; 1431 if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 && 1432 crp->crp_aad_length > 0) { 1433 tocopy = crp->crp_aad == NULL ? 1434 crp->crp_payload_start - crp->crp_aad_start : 1435 crp->crp_aad_length; 1436 1437 ibufs = &qsc->qsc_buf_list; 1438 obufs = &qsc->qsc_obuf_list; 1439 for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) { 1440 ibuf = &ibufs->flat_bufs[osegs]; 1441 obuf = &obufs->flat_bufs[osegs]; 1442 1443 obuf->phy_buffer = ibuf->phy_buffer; 1444 buflen = imin(ibuf->data_len_in_bytes, tocopy); 1445 obuf->data_len_in_bytes = buflen; 1446 tocopy -= buflen; 1447 } 1448 } 1449 1450 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs, 1451 nseg, osegs, crp->crp_payload_output_start); 1452 } 1453 1454 static int 1455 qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc, 1456 struct qat_crypto_desc const *desc, struct cryptop *crp) 1457 { 1458 struct qat_crypto_load_cb_arg arg; 1459 int error; 1460 1461 crypto_read_iv(crp, qsc->qsc_iv_buf); 1462 1463 arg.crp = crp; 1464 arg.qs = qs; 1465 arg.qsc = qsc; 1466 arg.error = 0; 1467 1468 error = 0; 1469 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 && 1470 crp->crp_aad_length > 0) { 1471 /* 1472 * The firmware expects AAD to be in a contiguous buffer and 1473 * padded to a multiple of 16 bytes. To satisfy these 1474 * constraints we bounce the AAD into a per-request buffer. 1475 * There is a small limit on the AAD size so this is not too 1476 * onerous. 1477 */ 1478 memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX); 1479 if (crp->crp_aad == NULL) { 1480 crypto_copydata(crp, crp->crp_aad_start, 1481 crp->crp_aad_length, qsc->qsc_gcm_aad); 1482 } else { 1483 memcpy(qsc->qsc_gcm_aad, crp->crp_aad, 1484 crp->crp_aad_length); 1485 } 1486 } else if (crp->crp_aad != NULL) { 1487 error = bus_dmamap_load( 1488 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, 1489 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, 1490 crp->crp_aad, crp->crp_aad_length, 1491 qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT); 1492 if (error == 0) 1493 error = arg.error; 1494 } 1495 if (error == 0) { 1496 error = bus_dmamap_load_crp_buffer( 1497 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, 1498 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, 1499 &crp->crp_buf, qat_crypto_load_buf_cb, &arg, 1500 BUS_DMA_NOWAIT); 1501 if (error == 0) 1502 error = arg.error; 1503 } 1504 if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1505 error = bus_dmamap_load_crp_buffer( 1506 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, 1507 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, 1508 &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg, 1509 BUS_DMA_NOWAIT); 1510 if (error == 0) 1511 error = arg.error; 1512 } 1513 return error; 1514 } 1515 1516 static inline struct qat_crypto_bank * 1517 qat_crypto_select_bank(struct qat_crypto *qcy) 1518 { 1519 u_int cpuid = PCPU_GET(cpuid); 1520 1521 return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks]; 1522 } 1523 1524 static int 1525 qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb) 1526 { 1527 char *name; 1528 int bank, curname, error, i, j; 1529 1530 bank = qcb->qcb_bank; 1531 curname = 0; 1532 1533 name = qcb->qcb_ring_names[curname++]; 1534 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank); 1535 error = qat_etr_setup_ring(sc, qcb->qcb_bank, 1536 sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size, 1537 NULL, NULL, name, &qcb->qcb_sym_tx); 1538 if (error) 1539 return error; 1540 1541 name = qcb->qcb_ring_names[curname++]; 1542 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank); 1543 error = qat_etr_setup_ring(sc, qcb->qcb_bank, 1544 sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size, 1545 qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx); 1546 if (error) 1547 return error; 1548 1549 for (i = 0; i < QAT_NSYMCOOKIE; i++) { 1550 struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i]; 1551 struct qat_sym_cookie *qsc; 1552 1553 error = qat_alloc_dmamem(sc, qdm, 1, 1554 sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN); 1555 if (error) 1556 return error; 1557 1558 qsc = qdm->qdm_dma_vaddr; 1559 qsc->qsc_self_dmamap = qdm->qdm_dma_map; 1560 qsc->qsc_self_dma_tag = qdm->qdm_dma_tag; 1561 qsc->qsc_bulk_req_params_buf_paddr = 1562 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1563 qsc_bulk_cookie.qsbc_req_params_buf); 1564 qsc->qsc_buffer_list_desc_paddr = 1565 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1566 qsc_buf_list); 1567 qsc->qsc_obuffer_list_desc_paddr = 1568 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1569 qsc_obuf_list); 1570 qsc->qsc_obuffer_list_desc_paddr = 1571 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1572 qsc_obuf_list); 1573 qsc->qsc_iv_buf_paddr = 1574 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1575 qsc_iv_buf); 1576 qsc->qsc_auth_res_paddr = 1577 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1578 qsc_auth_res); 1579 qsc->qsc_gcm_aad_paddr = 1580 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1581 qsc_gcm_aad); 1582 qsc->qsc_content_desc_paddr = 1583 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, 1584 qsc_content_desc); 1585 qcb->qcb_symck_free[i] = qsc; 1586 qcb->qcb_symck_free_count++; 1587 1588 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { 1589 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1590 1, 0, /* alignment, boundary */ 1591 BUS_SPACE_MAXADDR, /* lowaddr */ 1592 BUS_SPACE_MAXADDR, /* highaddr */ 1593 NULL, NULL, /* filter, filterarg */ 1594 QAT_MAXLEN, /* maxsize */ 1595 QAT_MAXSEG, /* nsegments */ 1596 QAT_MAXLEN, /* maxsegsize */ 1597 BUS_DMA_COHERENT, /* flags */ 1598 NULL, NULL, /* lockfunc, lockarg */ 1599 &qsc->qsc_dma[j].qsd_dma_tag); 1600 if (error != 0) 1601 return error; 1602 error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag, 1603 BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap); 1604 if (error != 0) 1605 return error; 1606 } 1607 } 1608 1609 return 0; 1610 } 1611 1612 static int 1613 qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb) 1614 { 1615 mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF); 1616 1617 return qat_crypto_setup_ring(sc, qcb); 1618 } 1619 1620 static void 1621 qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb) 1622 { 1623 struct qat_dmamem *qdm; 1624 struct qat_sym_cookie *qsc; 1625 int i, j; 1626 1627 for (i = 0; i < QAT_NSYMCOOKIE; i++) { 1628 qdm = &qcb->qcb_symck_dmamems[i]; 1629 qsc = qcb->qcb_symck_free[i]; 1630 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { 1631 bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag, 1632 qsc->qsc_dma[j].qsd_dmamap); 1633 bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag); 1634 } 1635 qat_free_dmamem(sc, qdm); 1636 } 1637 qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma); 1638 qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma); 1639 1640 mtx_destroy(&qcb->qcb_bank_mtx); 1641 } 1642 1643 static int 1644 qat_crypto_init(struct qat_softc *sc) 1645 { 1646 struct qat_crypto *qcy = &sc->sc_crypto; 1647 struct sysctl_ctx_list *ctx; 1648 struct sysctl_oid *oid; 1649 struct sysctl_oid_list *children; 1650 int bank, error, num_banks; 1651 1652 qcy->qcy_sc = sc; 1653 1654 if (sc->sc_hw.qhw_init_arb != NULL) 1655 num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks); 1656 else 1657 num_banks = sc->sc_ae_num; 1658 1659 qcy->qcy_num_banks = num_banks; 1660 1661 qcy->qcy_banks = 1662 qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks); 1663 1664 for (bank = 0; bank < num_banks; bank++) { 1665 struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank]; 1666 qcb->qcb_bank = bank; 1667 error = qat_crypto_bank_init(sc, qcb); 1668 if (error) 1669 return error; 1670 } 1671 1672 mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF); 1673 1674 ctx = device_get_sysctl_ctx(sc->sc_dev); 1675 oid = device_get_sysctl_tree(sc->sc_dev); 1676 children = SYSCTL_CHILDREN(oid); 1677 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", 1678 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); 1679 children = SYSCTL_CHILDREN(oid); 1680 1681 sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK); 1682 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts", 1683 CTLFLAG_RD, &sc->sc_gcm_aad_restarts, 1684 "GCM requests deferred due to AAD size change"); 1685 sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK); 1686 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates", 1687 CTLFLAG_RD, &sc->sc_gcm_aad_updates, 1688 "GCM requests that required session state update"); 1689 sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK); 1690 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full", 1691 CTLFLAG_RD, &sc->sc_ring_full_restarts, 1692 "Requests deferred due to in-flight max reached"); 1693 sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK); 1694 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures", 1695 CTLFLAG_RD, &sc->sc_sym_alloc_failures, 1696 "Request allocation failures"); 1697 1698 return 0; 1699 } 1700 1701 static void 1702 qat_crypto_deinit(struct qat_softc *sc) 1703 { 1704 struct qat_crypto *qcy = &sc->sc_crypto; 1705 struct qat_crypto_bank *qcb; 1706 int bank; 1707 1708 counter_u64_free(sc->sc_sym_alloc_failures); 1709 counter_u64_free(sc->sc_ring_full_restarts); 1710 counter_u64_free(sc->sc_gcm_aad_updates); 1711 counter_u64_free(sc->sc_gcm_aad_restarts); 1712 1713 if (qcy->qcy_banks != NULL) { 1714 for (bank = 0; bank < qcy->qcy_num_banks; bank++) { 1715 qcb = &qcy->qcy_banks[bank]; 1716 qat_crypto_bank_deinit(sc, qcb); 1717 } 1718 qat_free_mem(qcy->qcy_banks); 1719 mtx_destroy(&qcy->qcy_crypto_mtx); 1720 } 1721 } 1722 1723 static int 1724 qat_crypto_start(struct qat_softc *sc) 1725 { 1726 struct qat_crypto *qcy; 1727 1728 qcy = &sc->sc_crypto; 1729 qcy->qcy_cid = crypto_get_driverid(sc->sc_dev, 1730 sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE); 1731 if (qcy->qcy_cid < 0) { 1732 device_printf(sc->sc_dev, 1733 "could not get opencrypto driver id\n"); 1734 return ENOENT; 1735 } 1736 1737 return 0; 1738 } 1739 1740 static void 1741 qat_crypto_stop(struct qat_softc *sc) 1742 { 1743 struct qat_crypto *qcy; 1744 1745 qcy = &sc->sc_crypto; 1746 if (qcy->qcy_cid >= 0) 1747 (void)crypto_unregister_all(qcy->qcy_cid); 1748 } 1749 1750 static void 1751 qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i) 1752 { 1753 bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap, 1754 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1755 bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag, 1756 qsc->qsc_dma[i].qsd_dmamap); 1757 } 1758 1759 static int 1760 qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg) 1761 { 1762 char icv[QAT_SYM_HASH_BUFFER_LEN]; 1763 struct qat_crypto_bank *qcb = arg; 1764 struct qat_crypto *qcy; 1765 struct qat_session *qs; 1766 struct qat_sym_cookie *qsc; 1767 struct qat_sym_bulk_cookie *qsbc; 1768 struct cryptop *crp; 1769 int error; 1770 uint16_t auth_sz; 1771 bool blocked; 1772 1773 qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset); 1774 1775 qsbc = &qsc->qsc_bulk_cookie; 1776 qcy = qsbc->qsbc_crypto; 1777 qs = qsbc->qsbc_session; 1778 crp = qsbc->qsbc_cb_tag; 1779 1780 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, 1781 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1782 1783 if (crp->crp_aad != NULL) 1784 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF); 1785 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF); 1786 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) 1787 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF); 1788 1789 error = 0; 1790 if ((auth_sz = qs->qs_auth_mlen) != 0) { 1791 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 1792 crypto_copydata(crp, crp->crp_digest_start, 1793 auth_sz, icv); 1794 if (timingsafe_bcmp(icv, qsc->qsc_auth_res, 1795 auth_sz) != 0) { 1796 error = EBADMSG; 1797 } 1798 } else { 1799 crypto_copyback(crp, crp->crp_digest_start, 1800 auth_sz, qsc->qsc_auth_res); 1801 } 1802 } 1803 1804 qat_crypto_free_sym_cookie(qcb, qsc); 1805 1806 blocked = false; 1807 mtx_lock(&qs->qs_session_mtx); 1808 MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE); 1809 qs->qs_inflight--; 1810 if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) { 1811 blocked = true; 1812 qs->qs_need_wakeup = false; 1813 } 1814 mtx_unlock(&qs->qs_session_mtx); 1815 1816 crp->crp_etype = error; 1817 crypto_done(crp); 1818 1819 if (blocked) 1820 crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ); 1821 1822 return 1; 1823 } 1824 1825 static int 1826 qat_probesession(device_t dev, const struct crypto_session_params *csp) 1827 { 1828 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 1829 0) 1830 return EINVAL; 1831 1832 if (csp->csp_cipher_alg == CRYPTO_AES_XTS && 1833 qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) { 1834 /* 1835 * AES-XTS is not supported by the NanoQAT. 1836 */ 1837 return EINVAL; 1838 } 1839 1840 switch (csp->csp_mode) { 1841 case CSP_MODE_CIPHER: 1842 switch (csp->csp_cipher_alg) { 1843 case CRYPTO_AES_CBC: 1844 case CRYPTO_AES_ICM: 1845 if (csp->csp_ivlen != AES_BLOCK_LEN) 1846 return EINVAL; 1847 break; 1848 case CRYPTO_AES_XTS: 1849 if (csp->csp_ivlen != AES_XTS_IV_LEN) 1850 return EINVAL; 1851 break; 1852 default: 1853 return EINVAL; 1854 } 1855 break; 1856 case CSP_MODE_DIGEST: 1857 switch (csp->csp_auth_alg) { 1858 case CRYPTO_SHA1: 1859 case CRYPTO_SHA1_HMAC: 1860 case CRYPTO_SHA2_256: 1861 case CRYPTO_SHA2_256_HMAC: 1862 case CRYPTO_SHA2_384: 1863 case CRYPTO_SHA2_384_HMAC: 1864 case CRYPTO_SHA2_512: 1865 case CRYPTO_SHA2_512_HMAC: 1866 break; 1867 case CRYPTO_AES_NIST_GMAC: 1868 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1869 return EINVAL; 1870 break; 1871 default: 1872 return EINVAL; 1873 } 1874 break; 1875 case CSP_MODE_AEAD: 1876 switch (csp->csp_cipher_alg) { 1877 case CRYPTO_AES_NIST_GCM_16: 1878 break; 1879 default: 1880 return EINVAL; 1881 } 1882 break; 1883 case CSP_MODE_ETA: 1884 switch (csp->csp_auth_alg) { 1885 case CRYPTO_SHA1_HMAC: 1886 case CRYPTO_SHA2_256_HMAC: 1887 case CRYPTO_SHA2_384_HMAC: 1888 case CRYPTO_SHA2_512_HMAC: 1889 switch (csp->csp_cipher_alg) { 1890 case CRYPTO_AES_CBC: 1891 case CRYPTO_AES_ICM: 1892 if (csp->csp_ivlen != AES_BLOCK_LEN) 1893 return EINVAL; 1894 break; 1895 case CRYPTO_AES_XTS: 1896 if (csp->csp_ivlen != AES_XTS_IV_LEN) 1897 return EINVAL; 1898 break; 1899 default: 1900 return EINVAL; 1901 } 1902 break; 1903 default: 1904 return EINVAL; 1905 } 1906 break; 1907 default: 1908 return EINVAL; 1909 } 1910 1911 return CRYPTODEV_PROBE_HARDWARE; 1912 } 1913 1914 static int 1915 qat_newsession(device_t dev, crypto_session_t cses, 1916 const struct crypto_session_params *csp) 1917 { 1918 struct qat_crypto *qcy; 1919 struct qat_dmamem *qdm; 1920 struct qat_session *qs; 1921 struct qat_softc *sc; 1922 struct qat_crypto_desc *ddesc, *edesc; 1923 int error, slices; 1924 1925 sc = device_get_softc(dev); 1926 qs = crypto_get_driver_session(cses); 1927 qcy = &sc->sc_crypto; 1928 1929 qdm = &qs->qs_desc_mem; 1930 error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG, 1931 sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN); 1932 if (error != 0) 1933 return error; 1934 1935 mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF); 1936 qs->qs_aad_length = -1; 1937 1938 qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr; 1939 qs->qs_enc_desc = edesc = ddesc + 1; 1940 1941 ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr; 1942 ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr + 1943 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); 1944 edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr + 1945 sizeof(struct qat_crypto_desc); 1946 edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr + 1947 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); 1948 1949 qs->qs_status = QAT_SESSION_STATUS_ACTIVE; 1950 qs->qs_inflight = 0; 1951 1952 qs->qs_cipher_key = csp->csp_cipher_key; 1953 qs->qs_cipher_klen = csp->csp_cipher_klen; 1954 qs->qs_auth_key = csp->csp_auth_key; 1955 qs->qs_auth_klen = csp->csp_auth_klen; 1956 1957 switch (csp->csp_cipher_alg) { 1958 case CRYPTO_AES_CBC: 1959 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); 1960 qs->qs_cipher_mode = HW_CIPHER_CBC_MODE; 1961 break; 1962 case CRYPTO_AES_ICM: 1963 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); 1964 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; 1965 break; 1966 case CRYPTO_AES_XTS: 1967 qs->qs_cipher_algo = 1968 qat_aes_cipher_algo(csp->csp_cipher_klen / 2); 1969 qs->qs_cipher_mode = HW_CIPHER_XTS_MODE; 1970 break; 1971 case CRYPTO_AES_NIST_GCM_16: 1972 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); 1973 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; 1974 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; 1975 qs->qs_auth_mode = HW_AUTH_MODE1; 1976 break; 1977 case 0: 1978 break; 1979 default: 1980 panic("%s: unhandled cipher algorithm %d", __func__, 1981 csp->csp_cipher_alg); 1982 } 1983 1984 switch (csp->csp_auth_alg) { 1985 case CRYPTO_SHA1_HMAC: 1986 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; 1987 qs->qs_auth_mode = HW_AUTH_MODE1; 1988 break; 1989 case CRYPTO_SHA1: 1990 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; 1991 qs->qs_auth_mode = HW_AUTH_MODE0; 1992 break; 1993 case CRYPTO_SHA2_256_HMAC: 1994 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; 1995 qs->qs_auth_mode = HW_AUTH_MODE1; 1996 break; 1997 case CRYPTO_SHA2_256: 1998 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; 1999 qs->qs_auth_mode = HW_AUTH_MODE0; 2000 break; 2001 case CRYPTO_SHA2_384_HMAC: 2002 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; 2003 qs->qs_auth_mode = HW_AUTH_MODE1; 2004 break; 2005 case CRYPTO_SHA2_384: 2006 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; 2007 qs->qs_auth_mode = HW_AUTH_MODE0; 2008 break; 2009 case CRYPTO_SHA2_512_HMAC: 2010 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; 2011 qs->qs_auth_mode = HW_AUTH_MODE1; 2012 break; 2013 case CRYPTO_SHA2_512: 2014 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; 2015 qs->qs_auth_mode = HW_AUTH_MODE0; 2016 break; 2017 case CRYPTO_AES_NIST_GMAC: 2018 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen); 2019 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; 2020 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; 2021 qs->qs_auth_mode = HW_AUTH_MODE1; 2022 2023 qs->qs_cipher_key = qs->qs_auth_key; 2024 qs->qs_cipher_klen = qs->qs_auth_klen; 2025 break; 2026 case 0: 2027 break; 2028 default: 2029 panic("%s: unhandled auth algorithm %d", __func__, 2030 csp->csp_auth_alg); 2031 } 2032 2033 slices = 0; 2034 switch (csp->csp_mode) { 2035 case CSP_MODE_AEAD: 2036 case CSP_MODE_ETA: 2037 /* auth then decrypt */ 2038 ddesc->qcd_slices[0] = FW_SLICE_AUTH; 2039 ddesc->qcd_slices[1] = FW_SLICE_CIPHER; 2040 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; 2041 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; 2042 /* encrypt then auth */ 2043 edesc->qcd_slices[0] = FW_SLICE_CIPHER; 2044 edesc->qcd_slices[1] = FW_SLICE_AUTH; 2045 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; 2046 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; 2047 slices = 2; 2048 break; 2049 case CSP_MODE_CIPHER: 2050 /* decrypt */ 2051 ddesc->qcd_slices[0] = FW_SLICE_CIPHER; 2052 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; 2053 ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER; 2054 /* encrypt */ 2055 edesc->qcd_slices[0] = FW_SLICE_CIPHER; 2056 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; 2057 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER; 2058 slices = 1; 2059 break; 2060 case CSP_MODE_DIGEST: 2061 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { 2062 /* auth then decrypt */ 2063 ddesc->qcd_slices[0] = FW_SLICE_AUTH; 2064 ddesc->qcd_slices[1] = FW_SLICE_CIPHER; 2065 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; 2066 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; 2067 /* encrypt then auth */ 2068 edesc->qcd_slices[0] = FW_SLICE_CIPHER; 2069 edesc->qcd_slices[1] = FW_SLICE_AUTH; 2070 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; 2071 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; 2072 slices = 2; 2073 } else { 2074 ddesc->qcd_slices[0] = FW_SLICE_AUTH; 2075 ddesc->qcd_cmd_id = FW_LA_CMD_AUTH; 2076 edesc->qcd_slices[0] = FW_SLICE_AUTH; 2077 edesc->qcd_cmd_id = FW_LA_CMD_AUTH; 2078 slices = 1; 2079 } 2080 break; 2081 default: 2082 panic("%s: unhandled crypto algorithm %d, %d", __func__, 2083 csp->csp_cipher_alg, csp->csp_auth_alg); 2084 } 2085 ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; 2086 edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; 2087 2088 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc); 2089 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc); 2090 2091 if (csp->csp_auth_mlen != 0) 2092 qs->qs_auth_mlen = csp->csp_auth_mlen; 2093 else 2094 qs->qs_auth_mlen = edesc->qcd_auth_sz; 2095 2096 /* Compute the GMAC by specifying a null cipher payload. */ 2097 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) 2098 ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH; 2099 2100 return 0; 2101 } 2102 2103 static void 2104 qat_crypto_clear_desc(struct qat_crypto_desc *desc) 2105 { 2106 explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc)); 2107 explicit_bzero(desc->qcd_hash_state_prefix_buf, 2108 sizeof(desc->qcd_hash_state_prefix_buf)); 2109 explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache)); 2110 } 2111 2112 static void 2113 qat_freesession(device_t dev, crypto_session_t cses) 2114 { 2115 struct qat_session *qs; 2116 2117 qs = crypto_get_driver_session(cses); 2118 KASSERT(qs->qs_inflight == 0, 2119 ("%s: session %p has requests in flight", __func__, qs)); 2120 2121 qat_crypto_clear_desc(qs->qs_enc_desc); 2122 qat_crypto_clear_desc(qs->qs_dec_desc); 2123 qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem); 2124 mtx_destroy(&qs->qs_session_mtx); 2125 } 2126 2127 static int 2128 qat_process(device_t dev, struct cryptop *crp, int hint) 2129 { 2130 struct qat_crypto *qcy; 2131 struct qat_crypto_bank *qcb; 2132 struct qat_crypto_desc const *desc; 2133 struct qat_session *qs; 2134 struct qat_softc *sc; 2135 struct qat_sym_cookie *qsc; 2136 struct qat_sym_bulk_cookie *qsbc; 2137 int error; 2138 2139 sc = device_get_softc(dev); 2140 qcy = &sc->sc_crypto; 2141 qs = crypto_get_driver_session(crp->crp_session); 2142 qsc = NULL; 2143 2144 if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) { 2145 error = E2BIG; 2146 goto fail1; 2147 } 2148 2149 mtx_lock(&qs->qs_session_mtx); 2150 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { 2151 if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) { 2152 error = E2BIG; 2153 mtx_unlock(&qs->qs_session_mtx); 2154 goto fail1; 2155 } 2156 2157 /* 2158 * The firmware interface for GCM annoyingly requires the AAD 2159 * size to be stored in the session's content descriptor, which 2160 * is not really meant to be updated after session 2161 * initialization. For IPSec the AAD size is fixed so this is 2162 * not much of a problem in practice, but we have to catch AAD 2163 * size updates here so that the device code can safely update 2164 * the session's recorded AAD size. 2165 */ 2166 if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) { 2167 if (qs->qs_inflight == 0) { 2168 if (qs->qs_aad_length != -1) { 2169 counter_u64_add(sc->sc_gcm_aad_updates, 2170 1); 2171 } 2172 qs->qs_aad_length = crp->crp_aad_length; 2173 } else { 2174 qs->qs_need_wakeup = true; 2175 mtx_unlock(&qs->qs_session_mtx); 2176 counter_u64_add(sc->sc_gcm_aad_restarts, 1); 2177 error = ERESTART; 2178 goto fail1; 2179 } 2180 } 2181 } 2182 qs->qs_inflight++; 2183 mtx_unlock(&qs->qs_session_mtx); 2184 2185 qcb = qat_crypto_select_bank(qcy); 2186 2187 qsc = qat_crypto_alloc_sym_cookie(qcb); 2188 if (qsc == NULL) { 2189 counter_u64_add(sc->sc_sym_alloc_failures, 1); 2190 error = ENOBUFS; 2191 goto fail2; 2192 } 2193 2194 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 2195 desc = qs->qs_enc_desc; 2196 else 2197 desc = qs->qs_dec_desc; 2198 2199 error = qat_crypto_load(qs, qsc, desc, crp); 2200 if (error != 0) 2201 goto fail2; 2202 2203 qsbc = &qsc->qsc_bulk_cookie; 2204 qsbc->qsbc_crypto = qcy; 2205 qsbc->qsbc_session = qs; 2206 qsbc->qsbc_cb_tag = crp; 2207 2208 sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp); 2209 2210 if (crp->crp_aad != NULL) { 2211 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, 2212 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, 2213 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2214 } 2215 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, 2216 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, 2217 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2218 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 2219 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, 2220 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, 2221 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2222 } 2223 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, 2224 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2225 2226 error = qat_etr_put_msg(sc, qcb->qcb_sym_tx, 2227 (uint32_t *)qsbc->qsbc_msg); 2228 if (error) 2229 goto fail2; 2230 2231 return 0; 2232 2233 fail2: 2234 if (qsc) 2235 qat_crypto_free_sym_cookie(qcb, qsc); 2236 mtx_lock(&qs->qs_session_mtx); 2237 qs->qs_inflight--; 2238 mtx_unlock(&qs->qs_session_mtx); 2239 fail1: 2240 crp->crp_etype = error; 2241 crypto_done(crp); 2242 return 0; 2243 } 2244 2245 static device_method_t qat_methods[] = { 2246 /* Device interface */ 2247 DEVMETHOD(device_probe, qat_probe), 2248 DEVMETHOD(device_attach, qat_attach), 2249 DEVMETHOD(device_detach, qat_detach), 2250 2251 /* Cryptodev interface */ 2252 DEVMETHOD(cryptodev_probesession, qat_probesession), 2253 DEVMETHOD(cryptodev_newsession, qat_newsession), 2254 DEVMETHOD(cryptodev_freesession, qat_freesession), 2255 DEVMETHOD(cryptodev_process, qat_process), 2256 2257 DEVMETHOD_END 2258 }; 2259 2260 static driver_t qat_driver = { 2261 .name = "qat_c2xxx", 2262 .methods = qat_methods, 2263 .size = sizeof(struct qat_softc), 2264 }; 2265 2266 DRIVER_MODULE(qat_c2xxx, pci, qat_driver, 0, 0); 2267 MODULE_VERSION(qat_c2xxx, 1); 2268 MODULE_DEPEND(qat_c2xxx, crypto, 1, 1, 1); 2269 MODULE_DEPEND(qat_c2xxx, firmware, 1, 1, 1); 2270 MODULE_DEPEND(qat_c2xxx, pci, 1, 1, 1); 2271