1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * This file represents the QAT implementation of checksums and encryption. 24 * Internally, QAT shares the same cryptographic instances for both of these 25 * operations, so the code has been combined here. QAT data compression uses 26 * compression instances, so that code is separated into qat_compress.c 27 */ 28 29 #if defined(_KERNEL) && defined(HAVE_QAT) 30 #include <linux/slab.h> 31 #include <linux/vmalloc.h> 32 #include <linux/pagemap.h> 33 #include <linux/completion.h> 34 #include <sys/zfs_context.h> 35 #include <sys/zio_crypt.h> 36 #include "lac/cpa_cy_im.h" 37 #include "lac/cpa_cy_common.h" 38 #include <sys/qat.h> 39 40 /* 41 * Max instances in a QAT device, each instance is a channel to submit 42 * jobs to QAT hardware, this is only for pre-allocating instances 43 * and session arrays; the actual number of instances are defined in 44 * the QAT driver's configure file. 45 */ 46 #define QAT_CRYPT_MAX_INSTANCES 48 47 48 #define MAX_PAGE_NUM 1024 49 50 static Cpa32U inst_num = 0; 51 static Cpa16U num_inst = 0; 52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; 53 static boolean_t qat_cy_init_done = B_FALSE; 54 int zfs_qat_encrypt_disable = 0; 55 int zfs_qat_checksum_disable = 0; 56 57 typedef struct cy_callback { 58 CpaBoolean verify_result; 59 struct completion complete; 60 } cy_callback_t; 61 62 static void 63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation, 64 void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify) 65 { 66 cy_callback_t *cb = p_callback; 67 68 if (cb != NULL) { 69 /* indicate that the function has been called */ 70 cb->verify_result = verify; 71 complete(&cb->complete); 72 } 73 } 74 75 boolean_t 76 qat_crypt_use_accel(size_t s_len) 77 { 78 return (!zfs_qat_encrypt_disable && 79 qat_cy_init_done && 80 s_len >= QAT_MIN_BUF_SIZE && 81 s_len <= QAT_MAX_BUF_SIZE); 82 } 83 84 boolean_t 85 qat_checksum_use_accel(size_t s_len) 86 { 87 return (!zfs_qat_checksum_disable && 88 qat_cy_init_done && 89 s_len >= QAT_MIN_BUF_SIZE && 90 s_len <= QAT_MAX_BUF_SIZE); 91 } 92 93 void 94 qat_cy_clean(void) 95 { 96 for (Cpa16U i = 0; i < num_inst; i++) 97 cpaCyStopInstance(cy_inst_handles[i]); 98 99 num_inst = 0; 100 qat_cy_init_done = B_FALSE; 101 } 102 103 int 104 qat_cy_init(void) 105 { 106 CpaStatus status = CPA_STATUS_FAIL; 107 108 if (qat_cy_init_done) 109 return (0); 110 111 status = cpaCyGetNumInstances(&num_inst); 112 if (status != CPA_STATUS_SUCCESS) 113 return (-1); 114 115 /* if the user has configured no QAT encryption units just return */ 116 if (num_inst == 0) 117 return (0); 118 119 if (num_inst > QAT_CRYPT_MAX_INSTANCES) 120 num_inst = QAT_CRYPT_MAX_INSTANCES; 121 122 status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]); 123 if (status != CPA_STATUS_SUCCESS) 124 return (-1); 125 126 for (Cpa16U i = 0; i < num_inst; i++) { 127 status = cpaCySetAddressTranslation(cy_inst_handles[i], 128 (void *)virt_to_phys); 129 if (status != CPA_STATUS_SUCCESS) 130 goto error; 131 132 status = cpaCyStartInstance(cy_inst_handles[i]); 133 if (status != CPA_STATUS_SUCCESS) 134 goto error; 135 } 136 137 qat_cy_init_done = B_TRUE; 138 return (0); 139 140 error: 141 qat_cy_clean(); 142 return (-1); 143 } 144 145 void 146 qat_cy_fini(void) 147 { 148 if (!qat_cy_init_done) 149 return; 150 151 qat_cy_clean(); 152 } 153 154 static CpaStatus 155 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle, 156 CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key, 157 Cpa64U crypt, Cpa32U aad_len) 158 { 159 CpaStatus status = CPA_STATUS_SUCCESS; 160 Cpa32U ctx_size; 161 Cpa32U ciper_algorithm; 162 Cpa32U hash_algorithm; 163 CpaCySymSessionSetupData sd = { 0 }; 164 165 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) { 166 return (CPA_STATUS_FAIL); 167 } else { 168 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM; 169 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM; 170 } 171 172 sd.cipherSetupData.cipherAlgorithm = ciper_algorithm; 173 sd.cipherSetupData.pCipherKey = key->ck_data; 174 sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8; 175 sd.hashSetupData.hashAlgorithm = hash_algorithm; 176 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH; 177 sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN; 178 sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len; 179 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL; 180 sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; 181 sd.digestIsAppended = CPA_FALSE; 182 sd.verifyDigest = CPA_FALSE; 183 184 if (dir == QAT_ENCRYPT) { 185 sd.cipherSetupData.cipherDirection = 186 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 187 sd.algChainOrder = 188 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 189 } else { 190 ASSERT3U(dir, ==, QAT_DECRYPT); 191 sd.cipherSetupData.cipherDirection = 192 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 193 sd.algChainOrder = 194 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 195 } 196 197 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size); 198 if (status != CPA_STATUS_SUCCESS) 199 return (status); 200 201 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size); 202 if (status != CPA_STATUS_SUCCESS) 203 return (status); 204 205 status = cpaCySymInitSession(inst_handle, symcallback, &sd, 206 *cy_session_ctx); 207 if (status != CPA_STATUS_SUCCESS) { 208 QAT_PHYS_CONTIG_FREE(*cy_session_ctx); 209 return (status); 210 } 211 212 return (CPA_STATUS_SUCCESS); 213 } 214 215 static CpaStatus 216 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle, 217 CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum) 218 { 219 CpaStatus status = CPA_STATUS_SUCCESS; 220 Cpa32U ctx_size; 221 Cpa32U hash_algorithm; 222 CpaCySymSessionSetupData sd = { 0 }; 223 224 /* 225 * ZFS's SHA512 checksum is actually SHA512/256, which uses 226 * a different IV from standard SHA512. QAT does not support 227 * SHA512/256, so we can only support SHA256. 228 */ 229 if (cksum == ZIO_CHECKSUM_SHA256) 230 hash_algorithm = CPA_CY_SYM_HASH_SHA256; 231 else 232 return (CPA_STATUS_FAIL); 233 234 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL; 235 sd.symOperation = CPA_CY_SYM_OP_HASH; 236 sd.hashSetupData.hashAlgorithm = hash_algorithm; 237 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN; 238 sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t); 239 sd.digestIsAppended = CPA_FALSE; 240 sd.verifyDigest = CPA_FALSE; 241 242 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size); 243 if (status != CPA_STATUS_SUCCESS) 244 return (status); 245 246 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size); 247 if (status != CPA_STATUS_SUCCESS) 248 return (status); 249 250 status = cpaCySymInitSession(inst_handle, symcallback, &sd, 251 *cy_session_ctx); 252 if (status != CPA_STATUS_SUCCESS) { 253 QAT_PHYS_CONTIG_FREE(*cy_session_ctx); 254 return (status); 255 } 256 257 return (CPA_STATUS_SUCCESS); 258 } 259 260 static CpaStatus 261 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs, 262 CpaBufferList *src, CpaBufferList *dst) 263 { 264 CpaStatus status = CPA_STATUS_SUCCESS; 265 Cpa32U meta_size = 0; 266 267 status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size); 268 if (status != CPA_STATUS_SUCCESS) 269 return (status); 270 271 status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size); 272 if (status != CPA_STATUS_SUCCESS) 273 goto error; 274 275 if (src != dst) { 276 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData, 277 meta_size); 278 if (status != CPA_STATUS_SUCCESS) 279 goto error; 280 } 281 282 return (CPA_STATUS_SUCCESS); 283 284 error: 285 QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData); 286 if (src != dst) 287 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData); 288 289 return (status); 290 } 291 292 int 293 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, 294 uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, 295 crypto_key_t *key, uint64_t crypt, uint32_t enc_len) 296 { 297 CpaStatus status = CPA_STATUS_SUCCESS; 298 Cpa16U i; 299 CpaInstanceHandle cy_inst_handle; 300 Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2; 301 Cpa32U bytes_left = 0; 302 Cpa8S *data = NULL; 303 CpaCySymSessionCtx *cy_session_ctx = NULL; 304 cy_callback_t cb; 305 CpaCySymOpData op_data = { 0 }; 306 CpaBufferList src_buffer_list = { 0 }; 307 CpaBufferList dst_buffer_list = { 0 }; 308 CpaFlatBuffer *flat_src_buf_array = NULL; 309 CpaFlatBuffer *flat_src_buf = NULL; 310 CpaFlatBuffer *flat_dst_buf_array = NULL; 311 CpaFlatBuffer *flat_dst_buf = NULL; 312 struct page *in_pages[MAX_PAGE_NUM]; 313 struct page *out_pages[MAX_PAGE_NUM]; 314 Cpa32U in_page_num = 0; 315 Cpa32U out_page_num = 0; 316 Cpa32U in_page_off = 0; 317 Cpa32U out_page_off = 0; 318 319 if (dir == QAT_ENCRYPT) { 320 QAT_STAT_BUMP(encrypt_requests); 321 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len); 322 } else { 323 QAT_STAT_BUMP(decrypt_requests); 324 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len); 325 } 326 327 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 328 cy_inst_handle = cy_inst_handles[i]; 329 330 status = qat_init_crypt_session_ctx(dir, cy_inst_handle, 331 &cy_session_ctx, key, crypt, aad_len); 332 if (status != CPA_STATUS_SUCCESS) { 333 /* don't count CCM as a failure since it's not supported */ 334 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM) 335 QAT_STAT_BUMP(crypt_fails); 336 return (status); 337 } 338 339 /* 340 * We increment nr_bufs by 2 to allow us to handle non 341 * page-aligned buffer addresses and buffers whose sizes 342 * are not divisible by PAGE_SIZE. 343 */ 344 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, 345 &src_buffer_list, &dst_buffer_list); 346 if (status != CPA_STATUS_SUCCESS) 347 goto fail; 348 349 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array, 350 nr_bufs * sizeof (CpaFlatBuffer)); 351 if (status != CPA_STATUS_SUCCESS) 352 goto fail; 353 status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array, 354 nr_bufs * sizeof (CpaFlatBuffer)); 355 if (status != CPA_STATUS_SUCCESS) 356 goto fail; 357 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult, 358 ZIO_DATA_MAC_LEN); 359 if (status != CPA_STATUS_SUCCESS) 360 goto fail; 361 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv, 362 ZIO_DATA_IV_LEN); 363 if (status != CPA_STATUS_SUCCESS) 364 goto fail; 365 if (aad_len > 0) { 366 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData, 367 aad_len); 368 if (status != CPA_STATUS_SUCCESS) 369 goto fail; 370 memcpy(op_data.pAdditionalAuthData, aad_buf, aad_len); 371 } 372 373 bytes_left = enc_len; 374 data = src_buf; 375 flat_src_buf = flat_src_buf_array; 376 while (bytes_left > 0) { 377 in_page_off = ((long)data & ~PAGE_MASK); 378 in_pages[in_page_num] = qat_mem_to_page(data); 379 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off; 380 flat_src_buf->dataLenInBytes = 381 min((long)PAGE_SIZE - in_page_off, (long)bytes_left); 382 data += flat_src_buf->dataLenInBytes; 383 bytes_left -= flat_src_buf->dataLenInBytes; 384 flat_src_buf++; 385 in_page_num++; 386 } 387 src_buffer_list.pBuffers = flat_src_buf_array; 388 src_buffer_list.numBuffers = in_page_num; 389 390 bytes_left = enc_len; 391 data = dst_buf; 392 flat_dst_buf = flat_dst_buf_array; 393 while (bytes_left > 0) { 394 out_page_off = ((long)data & ~PAGE_MASK); 395 out_pages[out_page_num] = qat_mem_to_page(data); 396 flat_dst_buf->pData = kmap(out_pages[out_page_num]) + 397 out_page_off; 398 flat_dst_buf->dataLenInBytes = 399 min((long)PAGE_SIZE - out_page_off, (long)bytes_left); 400 data += flat_dst_buf->dataLenInBytes; 401 bytes_left -= flat_dst_buf->dataLenInBytes; 402 flat_dst_buf++; 403 out_page_num++; 404 } 405 dst_buffer_list.pBuffers = flat_dst_buf_array; 406 dst_buffer_list.numBuffers = out_page_num; 407 408 op_data.sessionCtx = cy_session_ctx; 409 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; 410 op_data.cryptoStartSrcOffsetInBytes = 0; 411 op_data.messageLenToCipherInBytes = 0; 412 op_data.hashStartSrcOffsetInBytes = 0; 413 op_data.messageLenToHashInBytes = 0; 414 op_data.messageLenToCipherInBytes = enc_len; 415 op_data.ivLenInBytes = ZIO_DATA_IV_LEN; 416 memcpy(op_data.pIv, iv_buf, ZIO_DATA_IV_LEN); 417 /* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */ 418 if (dir == QAT_DECRYPT) 419 memcpy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN); 420 421 cb.verify_result = CPA_FALSE; 422 init_completion(&cb.complete); 423 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data, 424 &src_buffer_list, &dst_buffer_list, NULL); 425 if (status != CPA_STATUS_SUCCESS) 426 goto fail; 427 428 /* we now wait until the completion of the operation. */ 429 wait_for_completion(&cb.complete); 430 431 if (cb.verify_result == CPA_FALSE) { 432 status = CPA_STATUS_FAIL; 433 goto fail; 434 } 435 436 if (dir == QAT_ENCRYPT) { 437 /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */ 438 memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN); 439 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); 440 } else { 441 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len); 442 } 443 444 fail: 445 if (status != CPA_STATUS_SUCCESS) 446 QAT_STAT_BUMP(crypt_fails); 447 448 for (i = 0; i < in_page_num; i++) 449 kunmap(in_pages[i]); 450 for (i = 0; i < out_page_num; i++) 451 kunmap(out_pages[i]); 452 453 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); 454 if (aad_len > 0) 455 QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData); 456 QAT_PHYS_CONTIG_FREE(op_data.pIv); 457 QAT_PHYS_CONTIG_FREE(op_data.pDigestResult); 458 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); 459 QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData); 460 QAT_PHYS_CONTIG_FREE(cy_session_ctx); 461 QAT_PHYS_CONTIG_FREE(flat_src_buf_array); 462 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array); 463 464 return (status); 465 } 466 467 int 468 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) 469 { 470 CpaStatus status; 471 Cpa16U i; 472 CpaInstanceHandle cy_inst_handle; 473 Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2; 474 Cpa32U bytes_left = 0; 475 Cpa8S *data = NULL; 476 CpaCySymSessionCtx *cy_session_ctx = NULL; 477 cy_callback_t cb; 478 Cpa8U *digest_buffer = NULL; 479 CpaCySymOpData op_data = { 0 }; 480 CpaBufferList src_buffer_list = { 0 }; 481 CpaFlatBuffer *flat_src_buf_array = NULL; 482 CpaFlatBuffer *flat_src_buf = NULL; 483 struct page *in_pages[MAX_PAGE_NUM]; 484 Cpa32U page_num = 0; 485 Cpa32U page_off = 0; 486 487 QAT_STAT_BUMP(cksum_requests); 488 QAT_STAT_INCR(cksum_total_in_bytes, size); 489 490 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 491 cy_inst_handle = cy_inst_handles[i]; 492 493 status = qat_init_checksum_session_ctx(cy_inst_handle, 494 &cy_session_ctx, cksum); 495 if (status != CPA_STATUS_SUCCESS) { 496 /* don't count unsupported checksums as a failure */ 497 if (cksum == ZIO_CHECKSUM_SHA256 || 498 cksum == ZIO_CHECKSUM_SHA512) 499 QAT_STAT_BUMP(cksum_fails); 500 return (status); 501 } 502 503 /* 504 * We increment nr_bufs by 2 to allow us to handle non 505 * page-aligned buffer addresses and buffers whose sizes 506 * are not divisible by PAGE_SIZE. 507 */ 508 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, 509 &src_buffer_list, &src_buffer_list); 510 if (status != CPA_STATUS_SUCCESS) 511 goto fail; 512 513 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array, 514 nr_bufs * sizeof (CpaFlatBuffer)); 515 if (status != CPA_STATUS_SUCCESS) 516 goto fail; 517 status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer, 518 sizeof (zio_cksum_t)); 519 if (status != CPA_STATUS_SUCCESS) 520 goto fail; 521 522 bytes_left = size; 523 data = buf; 524 flat_src_buf = flat_src_buf_array; 525 while (bytes_left > 0) { 526 page_off = ((long)data & ~PAGE_MASK); 527 in_pages[page_num] = qat_mem_to_page(data); 528 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off; 529 flat_src_buf->dataLenInBytes = 530 min((long)PAGE_SIZE - page_off, (long)bytes_left); 531 data += flat_src_buf->dataLenInBytes; 532 bytes_left -= flat_src_buf->dataLenInBytes; 533 flat_src_buf++; 534 page_num++; 535 } 536 src_buffer_list.pBuffers = flat_src_buf_array; 537 src_buffer_list.numBuffers = page_num; 538 539 op_data.sessionCtx = cy_session_ctx; 540 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; 541 op_data.hashStartSrcOffsetInBytes = 0; 542 op_data.messageLenToHashInBytes = size; 543 op_data.pDigestResult = digest_buffer; 544 545 cb.verify_result = CPA_FALSE; 546 init_completion(&cb.complete); 547 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data, 548 &src_buffer_list, &src_buffer_list, NULL); 549 if (status != CPA_STATUS_SUCCESS) 550 goto fail; 551 552 /* we now wait until the completion of the operation. */ 553 wait_for_completion(&cb.complete); 554 555 if (cb.verify_result == CPA_FALSE) { 556 status = CPA_STATUS_FAIL; 557 goto fail; 558 } 559 560 memcpy(zcp, digest_buffer, sizeof (zio_cksum_t)); 561 562 fail: 563 if (status != CPA_STATUS_SUCCESS) 564 QAT_STAT_BUMP(cksum_fails); 565 566 for (i = 0; i < page_num; i++) 567 kunmap(in_pages[i]); 568 569 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); 570 QAT_PHYS_CONTIG_FREE(digest_buffer); 571 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); 572 QAT_PHYS_CONTIG_FREE(cy_session_ctx); 573 QAT_PHYS_CONTIG_FREE(flat_src_buf_array); 574 575 return (status); 576 } 577 578 static int 579 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp) 580 { 581 int ret; 582 int *pvalue = kp->arg; 583 ret = param_set_int(val, kp); 584 if (ret) 585 return (ret); 586 /* 587 * zfs_qat_encrypt_disable = 0: enable qat encrypt 588 * try to initialize qat instance if it has not been done 589 */ 590 if (*pvalue == 0 && !qat_cy_init_done) { 591 ret = qat_cy_init(); 592 if (ret != 0) { 593 zfs_qat_encrypt_disable = 1; 594 return (ret); 595 } 596 } 597 return (ret); 598 } 599 600 static int 601 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp) 602 { 603 int ret; 604 int *pvalue = kp->arg; 605 ret = param_set_int(val, kp); 606 if (ret) 607 return (ret); 608 /* 609 * set_checksum_param_ops = 0: enable qat checksum 610 * try to initialize qat instance if it has not been done 611 */ 612 if (*pvalue == 0 && !qat_cy_init_done) { 613 ret = qat_cy_init(); 614 if (ret != 0) { 615 zfs_qat_checksum_disable = 1; 616 return (ret); 617 } 618 } 619 return (ret); 620 } 621 622 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt, 623 param_get_int, &zfs_qat_encrypt_disable, 0644); 624 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption"); 625 626 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum, 627 param_get_int, &zfs_qat_checksum_disable, 0644); 628 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming"); 629 630 #endif 631