1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * This file represents the QAT implementation of checksums and encryption. 25 * Internally, QAT shares the same cryptographic instances for both of these 26 * operations, so the code has been combined here. QAT data compression uses 27 * compression instances, so that code is separated into qat_compress.c 28 */ 29 30 #if defined(_KERNEL) && defined(HAVE_QAT) 31 #include <linux/slab.h> 32 #include <linux/vmalloc.h> 33 #include <linux/pagemap.h> 34 #include <linux/completion.h> 35 #include <sys/zfs_context.h> 36 #include <sys/zio_crypt.h> 37 #include "lac/cpa_cy_im.h" 38 #include "lac/cpa_cy_common.h" 39 #include <sys/qat.h> 40 41 /* 42 * Max instances in a QAT device, each instance is a channel to submit 43 * jobs to QAT hardware, this is only for pre-allocating instances 44 * and session arrays; the actual number of instances are defined in 45 * the QAT driver's configure file. 46 */ 47 #define QAT_CRYPT_MAX_INSTANCES 48 48 49 #define MAX_PAGE_NUM 1024 50 51 static Cpa32U inst_num = 0; 52 static Cpa16U num_inst = 0; 53 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; 54 static boolean_t qat_cy_init_done = B_FALSE; 55 int zfs_qat_encrypt_disable = 0; 56 int zfs_qat_checksum_disable = 0; 57 58 typedef struct cy_callback { 59 CpaBoolean verify_result; 60 struct completion complete; 61 } cy_callback_t; 62 63 static void 64 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation, 65 void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify) 66 { 67 cy_callback_t *cb = p_callback; 68 69 if (cb != NULL) { 70 /* indicate that the function has been called */ 71 cb->verify_result = verify; 72 complete(&cb->complete); 73 } 74 } 75 76 boolean_t 77 qat_crypt_use_accel(size_t s_len) 78 { 79 return (!zfs_qat_encrypt_disable && 80 qat_cy_init_done && 81 s_len >= QAT_MIN_BUF_SIZE && 82 s_len <= QAT_MAX_BUF_SIZE); 83 } 84 85 boolean_t 86 qat_checksum_use_accel(size_t s_len) 87 { 88 return (!zfs_qat_checksum_disable && 89 qat_cy_init_done && 90 s_len >= QAT_MIN_BUF_SIZE && 91 s_len <= QAT_MAX_BUF_SIZE); 92 } 93 94 void 95 qat_cy_clean(void) 96 { 97 for (Cpa16U i = 0; i < num_inst; i++) 98 cpaCyStopInstance(cy_inst_handles[i]); 99 100 num_inst = 0; 101 qat_cy_init_done = B_FALSE; 102 } 103 104 int 105 qat_cy_init(void) 106 { 107 CpaStatus status = CPA_STATUS_FAIL; 108 109 if (qat_cy_init_done) 110 return (0); 111 112 status = cpaCyGetNumInstances(&num_inst); 113 if (status != CPA_STATUS_SUCCESS) 114 return (-1); 115 116 /* if the user has configured no QAT encryption units just return */ 117 if (num_inst == 0) 118 return (0); 119 120 if (num_inst > QAT_CRYPT_MAX_INSTANCES) 121 num_inst = QAT_CRYPT_MAX_INSTANCES; 122 123 status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]); 124 if (status != CPA_STATUS_SUCCESS) 125 return (-1); 126 127 for (Cpa16U i = 0; i < num_inst; i++) { 128 status = cpaCySetAddressTranslation(cy_inst_handles[i], 129 (void *)virt_to_phys); 130 if (status != CPA_STATUS_SUCCESS) 131 goto error; 132 133 status = cpaCyStartInstance(cy_inst_handles[i]); 134 if (status != CPA_STATUS_SUCCESS) 135 goto error; 136 } 137 138 qat_cy_init_done = B_TRUE; 139 return (0); 140 141 error: 142 qat_cy_clean(); 143 return (-1); 144 } 145 146 void 147 qat_cy_fini(void) 148 { 149 if (!qat_cy_init_done) 150 return; 151 152 qat_cy_clean(); 153 } 154 155 static CpaStatus 156 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle, 157 CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key, 158 Cpa64U crypt, Cpa32U aad_len) 159 { 160 CpaStatus status = CPA_STATUS_SUCCESS; 161 Cpa32U ctx_size; 162 Cpa32U ciper_algorithm; 163 Cpa32U hash_algorithm; 164 CpaCySymSessionSetupData sd = { 0 }; 165 166 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) { 167 return (CPA_STATUS_FAIL); 168 } else { 169 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM; 170 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM; 171 } 172 173 sd.cipherSetupData.cipherAlgorithm = ciper_algorithm; 174 sd.cipherSetupData.pCipherKey = key->ck_data; 175 sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8; 176 sd.hashSetupData.hashAlgorithm = hash_algorithm; 177 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH; 178 sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN; 179 sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len; 180 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL; 181 sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; 182 sd.digestIsAppended = CPA_FALSE; 183 sd.verifyDigest = CPA_FALSE; 184 185 if (dir == QAT_ENCRYPT) { 186 sd.cipherSetupData.cipherDirection = 187 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 188 sd.algChainOrder = 189 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 190 } else { 191 ASSERT3U(dir, ==, QAT_DECRYPT); 192 sd.cipherSetupData.cipherDirection = 193 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 194 sd.algChainOrder = 195 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 196 } 197 198 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size); 199 if (status != CPA_STATUS_SUCCESS) 200 return (status); 201 202 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size); 203 if (status != CPA_STATUS_SUCCESS) 204 return (status); 205 206 status = cpaCySymInitSession(inst_handle, symcallback, &sd, 207 *cy_session_ctx); 208 if (status != CPA_STATUS_SUCCESS) { 209 QAT_PHYS_CONTIG_FREE(*cy_session_ctx); 210 return (status); 211 } 212 213 return (CPA_STATUS_SUCCESS); 214 } 215 216 static CpaStatus 217 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle, 218 CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum) 219 { 220 CpaStatus status = CPA_STATUS_SUCCESS; 221 Cpa32U ctx_size; 222 Cpa32U hash_algorithm; 223 CpaCySymSessionSetupData sd = { 0 }; 224 225 /* 226 * ZFS's SHA512 checksum is actually SHA512/256, which uses 227 * a different IV from standard SHA512. QAT does not support 228 * SHA512/256, so we can only support SHA256. 229 */ 230 if (cksum == ZIO_CHECKSUM_SHA256) 231 hash_algorithm = CPA_CY_SYM_HASH_SHA256; 232 else 233 return (CPA_STATUS_FAIL); 234 235 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL; 236 sd.symOperation = CPA_CY_SYM_OP_HASH; 237 sd.hashSetupData.hashAlgorithm = hash_algorithm; 238 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN; 239 sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t); 240 sd.digestIsAppended = CPA_FALSE; 241 sd.verifyDigest = CPA_FALSE; 242 243 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size); 244 if (status != CPA_STATUS_SUCCESS) 245 return (status); 246 247 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size); 248 if (status != CPA_STATUS_SUCCESS) 249 return (status); 250 251 status = cpaCySymInitSession(inst_handle, symcallback, &sd, 252 *cy_session_ctx); 253 if (status != CPA_STATUS_SUCCESS) { 254 QAT_PHYS_CONTIG_FREE(*cy_session_ctx); 255 return (status); 256 } 257 258 return (CPA_STATUS_SUCCESS); 259 } 260 261 static CpaStatus 262 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs, 263 CpaBufferList *src, CpaBufferList *dst) 264 { 265 CpaStatus status = CPA_STATUS_SUCCESS; 266 Cpa32U meta_size = 0; 267 268 status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size); 269 if (status != CPA_STATUS_SUCCESS) 270 return (status); 271 272 status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size); 273 if (status != CPA_STATUS_SUCCESS) 274 goto error; 275 276 if (src != dst) { 277 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData, 278 meta_size); 279 if (status != CPA_STATUS_SUCCESS) 280 goto error; 281 } 282 283 return (CPA_STATUS_SUCCESS); 284 285 error: 286 QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData); 287 if (src != dst) 288 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData); 289 290 return (status); 291 } 292 293 int 294 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, 295 uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, 296 crypto_key_t *key, uint64_t crypt, uint32_t enc_len) 297 { 298 CpaStatus status = CPA_STATUS_SUCCESS; 299 Cpa16U i; 300 CpaInstanceHandle cy_inst_handle; 301 Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2; 302 Cpa32U bytes_left = 0; 303 Cpa8S *data = NULL; 304 CpaCySymSessionCtx *cy_session_ctx = NULL; 305 cy_callback_t cb; 306 CpaCySymOpData op_data = { 0 }; 307 CpaBufferList src_buffer_list = { 0 }; 308 CpaBufferList dst_buffer_list = { 0 }; 309 CpaFlatBuffer *flat_src_buf_array = NULL; 310 CpaFlatBuffer *flat_src_buf = NULL; 311 CpaFlatBuffer *flat_dst_buf_array = NULL; 312 CpaFlatBuffer *flat_dst_buf = NULL; 313 struct page *in_pages[MAX_PAGE_NUM]; 314 struct page *out_pages[MAX_PAGE_NUM]; 315 Cpa32U in_page_num = 0; 316 Cpa32U out_page_num = 0; 317 Cpa32U in_page_off = 0; 318 Cpa32U out_page_off = 0; 319 320 if (dir == QAT_ENCRYPT) { 321 QAT_STAT_BUMP(encrypt_requests); 322 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len); 323 } else { 324 QAT_STAT_BUMP(decrypt_requests); 325 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len); 326 } 327 328 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 329 cy_inst_handle = cy_inst_handles[i]; 330 331 status = qat_init_crypt_session_ctx(dir, cy_inst_handle, 332 &cy_session_ctx, key, crypt, aad_len); 333 if (status != CPA_STATUS_SUCCESS) { 334 /* don't count CCM as a failure since it's not supported */ 335 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM) 336 QAT_STAT_BUMP(crypt_fails); 337 return (status); 338 } 339 340 /* 341 * We increment nr_bufs by 2 to allow us to handle non 342 * page-aligned buffer addresses and buffers whose sizes 343 * are not divisible by PAGE_SIZE. 344 */ 345 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, 346 &src_buffer_list, &dst_buffer_list); 347 if (status != CPA_STATUS_SUCCESS) 348 goto fail; 349 350 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array, 351 nr_bufs * sizeof (CpaFlatBuffer)); 352 if (status != CPA_STATUS_SUCCESS) 353 goto fail; 354 status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array, 355 nr_bufs * sizeof (CpaFlatBuffer)); 356 if (status != CPA_STATUS_SUCCESS) 357 goto fail; 358 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult, 359 ZIO_DATA_MAC_LEN); 360 if (status != CPA_STATUS_SUCCESS) 361 goto fail; 362 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv, 363 ZIO_DATA_IV_LEN); 364 if (status != CPA_STATUS_SUCCESS) 365 goto fail; 366 if (aad_len > 0) { 367 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData, 368 aad_len); 369 if (status != CPA_STATUS_SUCCESS) 370 goto fail; 371 memcpy(op_data.pAdditionalAuthData, aad_buf, aad_len); 372 } 373 374 bytes_left = enc_len; 375 data = src_buf; 376 flat_src_buf = flat_src_buf_array; 377 while (bytes_left > 0) { 378 in_page_off = ((long)data & ~PAGE_MASK); 379 in_pages[in_page_num] = qat_mem_to_page(data); 380 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off; 381 flat_src_buf->dataLenInBytes = 382 min((long)PAGE_SIZE - in_page_off, (long)bytes_left); 383 data += flat_src_buf->dataLenInBytes; 384 bytes_left -= flat_src_buf->dataLenInBytes; 385 flat_src_buf++; 386 in_page_num++; 387 } 388 src_buffer_list.pBuffers = flat_src_buf_array; 389 src_buffer_list.numBuffers = in_page_num; 390 391 bytes_left = enc_len; 392 data = dst_buf; 393 flat_dst_buf = flat_dst_buf_array; 394 while (bytes_left > 0) { 395 out_page_off = ((long)data & ~PAGE_MASK); 396 out_pages[out_page_num] = qat_mem_to_page(data); 397 flat_dst_buf->pData = kmap(out_pages[out_page_num]) + 398 out_page_off; 399 flat_dst_buf->dataLenInBytes = 400 min((long)PAGE_SIZE - out_page_off, (long)bytes_left); 401 data += flat_dst_buf->dataLenInBytes; 402 bytes_left -= flat_dst_buf->dataLenInBytes; 403 flat_dst_buf++; 404 out_page_num++; 405 } 406 dst_buffer_list.pBuffers = flat_dst_buf_array; 407 dst_buffer_list.numBuffers = out_page_num; 408 409 op_data.sessionCtx = cy_session_ctx; 410 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; 411 op_data.cryptoStartSrcOffsetInBytes = 0; 412 op_data.messageLenToCipherInBytes = 0; 413 op_data.hashStartSrcOffsetInBytes = 0; 414 op_data.messageLenToHashInBytes = 0; 415 op_data.messageLenToCipherInBytes = enc_len; 416 op_data.ivLenInBytes = ZIO_DATA_IV_LEN; 417 memcpy(op_data.pIv, iv_buf, ZIO_DATA_IV_LEN); 418 /* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */ 419 if (dir == QAT_DECRYPT) 420 memcpy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN); 421 422 cb.verify_result = CPA_FALSE; 423 init_completion(&cb.complete); 424 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data, 425 &src_buffer_list, &dst_buffer_list, NULL); 426 if (status != CPA_STATUS_SUCCESS) 427 goto fail; 428 429 /* we now wait until the completion of the operation. */ 430 wait_for_completion(&cb.complete); 431 432 if (cb.verify_result == CPA_FALSE) { 433 status = CPA_STATUS_FAIL; 434 goto fail; 435 } 436 437 if (dir == QAT_ENCRYPT) { 438 /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */ 439 memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN); 440 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); 441 } else { 442 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len); 443 } 444 445 fail: 446 if (status != CPA_STATUS_SUCCESS) 447 QAT_STAT_BUMP(crypt_fails); 448 449 for (i = 0; i < in_page_num; i++) 450 kunmap(in_pages[i]); 451 for (i = 0; i < out_page_num; i++) 452 kunmap(out_pages[i]); 453 454 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); 455 if (aad_len > 0) 456 QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData); 457 QAT_PHYS_CONTIG_FREE(op_data.pIv); 458 QAT_PHYS_CONTIG_FREE(op_data.pDigestResult); 459 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); 460 QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData); 461 QAT_PHYS_CONTIG_FREE(cy_session_ctx); 462 QAT_PHYS_CONTIG_FREE(flat_src_buf_array); 463 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array); 464 465 return (status); 466 } 467 468 int 469 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) 470 { 471 CpaStatus status; 472 Cpa16U i; 473 CpaInstanceHandle cy_inst_handle; 474 Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2; 475 Cpa32U bytes_left = 0; 476 Cpa8S *data = NULL; 477 CpaCySymSessionCtx *cy_session_ctx = NULL; 478 cy_callback_t cb; 479 Cpa8U *digest_buffer = NULL; 480 CpaCySymOpData op_data = { 0 }; 481 CpaBufferList src_buffer_list = { 0 }; 482 CpaFlatBuffer *flat_src_buf_array = NULL; 483 CpaFlatBuffer *flat_src_buf = NULL; 484 struct page *in_pages[MAX_PAGE_NUM]; 485 Cpa32U page_num = 0; 486 Cpa32U page_off = 0; 487 488 QAT_STAT_BUMP(cksum_requests); 489 QAT_STAT_INCR(cksum_total_in_bytes, size); 490 491 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 492 cy_inst_handle = cy_inst_handles[i]; 493 494 status = qat_init_checksum_session_ctx(cy_inst_handle, 495 &cy_session_ctx, cksum); 496 if (status != CPA_STATUS_SUCCESS) { 497 /* don't count unsupported checksums as a failure */ 498 if (cksum == ZIO_CHECKSUM_SHA256 || 499 cksum == ZIO_CHECKSUM_SHA512) 500 QAT_STAT_BUMP(cksum_fails); 501 return (status); 502 } 503 504 /* 505 * We increment nr_bufs by 2 to allow us to handle non 506 * page-aligned buffer addresses and buffers whose sizes 507 * are not divisible by PAGE_SIZE. 508 */ 509 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, 510 &src_buffer_list, &src_buffer_list); 511 if (status != CPA_STATUS_SUCCESS) 512 goto fail; 513 514 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array, 515 nr_bufs * sizeof (CpaFlatBuffer)); 516 if (status != CPA_STATUS_SUCCESS) 517 goto fail; 518 status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer, 519 sizeof (zio_cksum_t)); 520 if (status != CPA_STATUS_SUCCESS) 521 goto fail; 522 523 bytes_left = size; 524 data = buf; 525 flat_src_buf = flat_src_buf_array; 526 while (bytes_left > 0) { 527 page_off = ((long)data & ~PAGE_MASK); 528 in_pages[page_num] = qat_mem_to_page(data); 529 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off; 530 flat_src_buf->dataLenInBytes = 531 min((long)PAGE_SIZE - page_off, (long)bytes_left); 532 data += flat_src_buf->dataLenInBytes; 533 bytes_left -= flat_src_buf->dataLenInBytes; 534 flat_src_buf++; 535 page_num++; 536 } 537 src_buffer_list.pBuffers = flat_src_buf_array; 538 src_buffer_list.numBuffers = page_num; 539 540 op_data.sessionCtx = cy_session_ctx; 541 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; 542 op_data.hashStartSrcOffsetInBytes = 0; 543 op_data.messageLenToHashInBytes = size; 544 op_data.pDigestResult = digest_buffer; 545 546 cb.verify_result = CPA_FALSE; 547 init_completion(&cb.complete); 548 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data, 549 &src_buffer_list, &src_buffer_list, NULL); 550 if (status != CPA_STATUS_SUCCESS) 551 goto fail; 552 553 /* we now wait until the completion of the operation. */ 554 wait_for_completion(&cb.complete); 555 556 if (cb.verify_result == CPA_FALSE) { 557 status = CPA_STATUS_FAIL; 558 goto fail; 559 } 560 561 memcpy(zcp, digest_buffer, sizeof (zio_cksum_t)); 562 563 fail: 564 if (status != CPA_STATUS_SUCCESS) 565 QAT_STAT_BUMP(cksum_fails); 566 567 for (i = 0; i < page_num; i++) 568 kunmap(in_pages[i]); 569 570 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); 571 QAT_PHYS_CONTIG_FREE(digest_buffer); 572 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); 573 QAT_PHYS_CONTIG_FREE(cy_session_ctx); 574 QAT_PHYS_CONTIG_FREE(flat_src_buf_array); 575 576 return (status); 577 } 578 579 static int 580 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp) 581 { 582 int ret; 583 int *pvalue = kp->arg; 584 ret = param_set_int(val, kp); 585 if (ret) 586 return (ret); 587 /* 588 * zfs_qat_encrypt_disable = 0: enable qat encrypt 589 * try to initialize qat instance if it has not been done 590 */ 591 if (*pvalue == 0 && !qat_cy_init_done) { 592 ret = qat_cy_init(); 593 if (ret != 0) { 594 zfs_qat_encrypt_disable = 1; 595 return (ret); 596 } 597 } 598 return (ret); 599 } 600 601 static int 602 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp) 603 { 604 int ret; 605 int *pvalue = kp->arg; 606 ret = param_set_int(val, kp); 607 if (ret) 608 return (ret); 609 /* 610 * set_checksum_param_ops = 0: enable qat checksum 611 * try to initialize qat instance if it has not been done 612 */ 613 if (*pvalue == 0 && !qat_cy_init_done) { 614 ret = qat_cy_init(); 615 if (ret != 0) { 616 zfs_qat_checksum_disable = 1; 617 return (ret); 618 } 619 } 620 return (ret); 621 } 622 623 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt, 624 param_get_int, &zfs_qat_encrypt_disable, 0644); 625 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption"); 626 627 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum, 628 param_get_int, &zfs_qat_checksum_disable, 0644); 629 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming"); 630 631 #endif 632