1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * This file represents the QAT implementation of checksums and encryption.
24 * Internally, QAT shares the same cryptographic instances for both of these
25 * operations, so the code has been combined here. QAT data compression uses
26 * compression instances, so that code is separated into qat_compress.c
27 */
28
29 #if defined(_KERNEL) && defined(HAVE_QAT)
30 #include <linux/slab.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/completion.h>
34 #include <sys/zfs_context.h>
35 #include <sys/zio_crypt.h>
36 #include "lac/cpa_cy_im.h"
37 #include "lac/cpa_cy_common.h"
38 #include <sys/qat.h>
39
40 /*
41 * Max instances in a QAT device, each instance is a channel to submit
42 * jobs to QAT hardware, this is only for pre-allocating instances
43 * and session arrays; the actual number of instances are defined in
44 * the QAT driver's configure file.
45 */
46 #define QAT_CRYPT_MAX_INSTANCES 48
47
48 #define MAX_PAGE_NUM 1024
49
50 static Cpa32U inst_num = 0;
51 static Cpa16U num_inst = 0;
52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
53 static boolean_t qat_cy_init_done = B_FALSE;
54 int zfs_qat_encrypt_disable = 0;
55 int zfs_qat_checksum_disable = 0;
56
57 typedef struct cy_callback {
58 CpaBoolean verify_result;
59 struct completion complete;
60 } cy_callback_t;
61
62 static void
symcallback(void * p_callback,CpaStatus status,const CpaCySymOp operation,void * op_data,CpaBufferList * buf_list_dst,CpaBoolean verify)63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
64 void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
65 {
66 cy_callback_t *cb = p_callback;
67
68 if (cb != NULL) {
69 /* indicate that the function has been called */
70 cb->verify_result = verify;
71 complete(&cb->complete);
72 }
73 }
74
75 boolean_t
qat_crypt_use_accel(size_t s_len)76 qat_crypt_use_accel(size_t s_len)
77 {
78 return (!zfs_qat_encrypt_disable &&
79 qat_cy_init_done &&
80 s_len >= QAT_MIN_BUF_SIZE &&
81 s_len <= QAT_MAX_BUF_SIZE);
82 }
83
84 boolean_t
qat_checksum_use_accel(size_t s_len)85 qat_checksum_use_accel(size_t s_len)
86 {
87 return (!zfs_qat_checksum_disable &&
88 qat_cy_init_done &&
89 s_len >= QAT_MIN_BUF_SIZE &&
90 s_len <= QAT_MAX_BUF_SIZE);
91 }
92
93 void
qat_cy_clean(void)94 qat_cy_clean(void)
95 {
96 for (Cpa16U i = 0; i < num_inst; i++)
97 cpaCyStopInstance(cy_inst_handles[i]);
98
99 num_inst = 0;
100 qat_cy_init_done = B_FALSE;
101 }
102
103 int
qat_cy_init(void)104 qat_cy_init(void)
105 {
106 CpaStatus status = CPA_STATUS_FAIL;
107
108 if (qat_cy_init_done)
109 return (0);
110
111 status = cpaCyGetNumInstances(&num_inst);
112 if (status != CPA_STATUS_SUCCESS)
113 return (-1);
114
115 /* if the user has configured no QAT encryption units just return */
116 if (num_inst == 0)
117 return (0);
118
119 if (num_inst > QAT_CRYPT_MAX_INSTANCES)
120 num_inst = QAT_CRYPT_MAX_INSTANCES;
121
122 status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
123 if (status != CPA_STATUS_SUCCESS)
124 return (-1);
125
126 for (Cpa16U i = 0; i < num_inst; i++) {
127 status = cpaCySetAddressTranslation(cy_inst_handles[i],
128 (void *)virt_to_phys);
129 if (status != CPA_STATUS_SUCCESS)
130 goto error;
131
132 status = cpaCyStartInstance(cy_inst_handles[i]);
133 if (status != CPA_STATUS_SUCCESS)
134 goto error;
135 }
136
137 qat_cy_init_done = B_TRUE;
138 return (0);
139
140 error:
141 qat_cy_clean();
142 return (-1);
143 }
144
145 void
qat_cy_fini(void)146 qat_cy_fini(void)
147 {
148 if (!qat_cy_init_done)
149 return;
150
151 qat_cy_clean();
152 }
153
154 static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir,CpaInstanceHandle inst_handle,CpaCySymSessionCtx ** cy_session_ctx,crypto_key_t * key,Cpa64U crypt,Cpa32U aad_len)155 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
156 CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
157 Cpa64U crypt, Cpa32U aad_len)
158 {
159 CpaStatus status = CPA_STATUS_SUCCESS;
160 Cpa32U ctx_size;
161 Cpa32U ciper_algorithm;
162 Cpa32U hash_algorithm;
163 CpaCySymSessionSetupData sd = { 0 };
164
165 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
166 return (CPA_STATUS_FAIL);
167 } else {
168 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
169 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
170 }
171
172 sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
173 sd.cipherSetupData.pCipherKey = key->ck_data;
174 sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
175 sd.hashSetupData.hashAlgorithm = hash_algorithm;
176 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
177 sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
178 sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
179 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
180 sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
181 sd.digestIsAppended = CPA_FALSE;
182 sd.verifyDigest = CPA_FALSE;
183
184 if (dir == QAT_ENCRYPT) {
185 sd.cipherSetupData.cipherDirection =
186 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
187 sd.algChainOrder =
188 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
189 } else {
190 ASSERT3U(dir, ==, QAT_DECRYPT);
191 sd.cipherSetupData.cipherDirection =
192 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
193 sd.algChainOrder =
194 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
195 }
196
197 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
198 if (status != CPA_STATUS_SUCCESS)
199 return (status);
200
201 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
202 if (status != CPA_STATUS_SUCCESS)
203 return (status);
204
205 status = cpaCySymInitSession(inst_handle, symcallback, &sd,
206 *cy_session_ctx);
207 if (status != CPA_STATUS_SUCCESS) {
208 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
209 return (status);
210 }
211
212 return (CPA_STATUS_SUCCESS);
213 }
214
215 static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,CpaCySymSessionCtx ** cy_session_ctx,Cpa64U cksum)216 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
217 CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
218 {
219 CpaStatus status = CPA_STATUS_SUCCESS;
220 Cpa32U ctx_size;
221 Cpa32U hash_algorithm;
222 CpaCySymSessionSetupData sd = { 0 };
223
224 /*
225 * ZFS's SHA512 checksum is actually SHA512/256, which uses
226 * a different IV from standard SHA512. QAT does not support
227 * SHA512/256, so we can only support SHA256.
228 */
229 if (cksum == ZIO_CHECKSUM_SHA256)
230 hash_algorithm = CPA_CY_SYM_HASH_SHA256;
231 else
232 return (CPA_STATUS_FAIL);
233
234 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
235 sd.symOperation = CPA_CY_SYM_OP_HASH;
236 sd.hashSetupData.hashAlgorithm = hash_algorithm;
237 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
238 sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
239 sd.digestIsAppended = CPA_FALSE;
240 sd.verifyDigest = CPA_FALSE;
241
242 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
243 if (status != CPA_STATUS_SUCCESS)
244 return (status);
245
246 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
247 if (status != CPA_STATUS_SUCCESS)
248 return (status);
249
250 status = cpaCySymInitSession(inst_handle, symcallback, &sd,
251 *cy_session_ctx);
252 if (status != CPA_STATUS_SUCCESS) {
253 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
254 return (status);
255 }
256
257 return (CPA_STATUS_SUCCESS);
258 }
259
260 static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle,uint32_t nr_bufs,CpaBufferList * src,CpaBufferList * dst)261 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
262 CpaBufferList *src, CpaBufferList *dst)
263 {
264 CpaStatus status = CPA_STATUS_SUCCESS;
265 Cpa32U meta_size = 0;
266
267 status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
268 if (status != CPA_STATUS_SUCCESS)
269 return (status);
270
271 status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
272 if (status != CPA_STATUS_SUCCESS)
273 goto error;
274
275 if (src != dst) {
276 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
277 meta_size);
278 if (status != CPA_STATUS_SUCCESS)
279 goto error;
280 }
281
282 return (CPA_STATUS_SUCCESS);
283
284 error:
285 QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
286 if (src != dst)
287 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
288
289 return (status);
290 }
291
292 int
qat_crypt(qat_encrypt_dir_t dir,uint8_t * src_buf,uint8_t * dst_buf,uint8_t * aad_buf,uint32_t aad_len,uint8_t * iv_buf,uint8_t * digest_buf,crypto_key_t * key,uint64_t crypt,uint32_t enc_len)293 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
294 uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
295 crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
296 {
297 CpaStatus status = CPA_STATUS_SUCCESS;
298 Cpa16U i;
299 CpaInstanceHandle cy_inst_handle;
300 Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
301 Cpa32U bytes_left = 0;
302 Cpa8S *data = NULL;
303 CpaCySymSessionCtx *cy_session_ctx = NULL;
304 cy_callback_t cb;
305 CpaCySymOpData op_data = { 0 };
306 CpaBufferList src_buffer_list = { 0 };
307 CpaBufferList dst_buffer_list = { 0 };
308 CpaFlatBuffer *flat_src_buf_array = NULL;
309 CpaFlatBuffer *flat_src_buf = NULL;
310 CpaFlatBuffer *flat_dst_buf_array = NULL;
311 CpaFlatBuffer *flat_dst_buf = NULL;
312 struct page *in_pages[MAX_PAGE_NUM];
313 struct page *out_pages[MAX_PAGE_NUM];
314 Cpa32U in_page_num = 0;
315 Cpa32U out_page_num = 0;
316 Cpa32U in_page_off = 0;
317 Cpa32U out_page_off = 0;
318
319 if (dir == QAT_ENCRYPT) {
320 QAT_STAT_BUMP(encrypt_requests);
321 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
322 } else {
323 QAT_STAT_BUMP(decrypt_requests);
324 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
325 }
326
327 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
328 cy_inst_handle = cy_inst_handles[i];
329
330 status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
331 &cy_session_ctx, key, crypt, aad_len);
332 if (status != CPA_STATUS_SUCCESS) {
333 /* don't count CCM as a failure since it's not supported */
334 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
335 QAT_STAT_BUMP(crypt_fails);
336 return (status);
337 }
338
339 /*
340 * We increment nr_bufs by 2 to allow us to handle non
341 * page-aligned buffer addresses and buffers whose sizes
342 * are not divisible by PAGE_SIZE.
343 */
344 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
345 &src_buffer_list, &dst_buffer_list);
346 if (status != CPA_STATUS_SUCCESS)
347 goto fail;
348
349 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
350 nr_bufs * sizeof (CpaFlatBuffer));
351 if (status != CPA_STATUS_SUCCESS)
352 goto fail;
353 status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
354 nr_bufs * sizeof (CpaFlatBuffer));
355 if (status != CPA_STATUS_SUCCESS)
356 goto fail;
357 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
358 ZIO_DATA_MAC_LEN);
359 if (status != CPA_STATUS_SUCCESS)
360 goto fail;
361 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
362 ZIO_DATA_IV_LEN);
363 if (status != CPA_STATUS_SUCCESS)
364 goto fail;
365 if (aad_len > 0) {
366 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
367 aad_len);
368 if (status != CPA_STATUS_SUCCESS)
369 goto fail;
370 memcpy(op_data.pAdditionalAuthData, aad_buf, aad_len);
371 }
372
373 bytes_left = enc_len;
374 data = src_buf;
375 flat_src_buf = flat_src_buf_array;
376 while (bytes_left > 0) {
377 in_page_off = ((long)data & ~PAGE_MASK);
378 in_pages[in_page_num] = qat_mem_to_page(data);
379 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
380 flat_src_buf->dataLenInBytes =
381 min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
382 data += flat_src_buf->dataLenInBytes;
383 bytes_left -= flat_src_buf->dataLenInBytes;
384 flat_src_buf++;
385 in_page_num++;
386 }
387 src_buffer_list.pBuffers = flat_src_buf_array;
388 src_buffer_list.numBuffers = in_page_num;
389
390 bytes_left = enc_len;
391 data = dst_buf;
392 flat_dst_buf = flat_dst_buf_array;
393 while (bytes_left > 0) {
394 out_page_off = ((long)data & ~PAGE_MASK);
395 out_pages[out_page_num] = qat_mem_to_page(data);
396 flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
397 out_page_off;
398 flat_dst_buf->dataLenInBytes =
399 min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
400 data += flat_dst_buf->dataLenInBytes;
401 bytes_left -= flat_dst_buf->dataLenInBytes;
402 flat_dst_buf++;
403 out_page_num++;
404 }
405 dst_buffer_list.pBuffers = flat_dst_buf_array;
406 dst_buffer_list.numBuffers = out_page_num;
407
408 op_data.sessionCtx = cy_session_ctx;
409 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
410 op_data.cryptoStartSrcOffsetInBytes = 0;
411 op_data.messageLenToCipherInBytes = 0;
412 op_data.hashStartSrcOffsetInBytes = 0;
413 op_data.messageLenToHashInBytes = 0;
414 op_data.messageLenToCipherInBytes = enc_len;
415 op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
416 memcpy(op_data.pIv, iv_buf, ZIO_DATA_IV_LEN);
417 /* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */
418 if (dir == QAT_DECRYPT)
419 memcpy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
420
421 cb.verify_result = CPA_FALSE;
422 init_completion(&cb.complete);
423 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
424 &src_buffer_list, &dst_buffer_list, NULL);
425 if (status != CPA_STATUS_SUCCESS)
426 goto fail;
427
428 /* we now wait until the completion of the operation. */
429 wait_for_completion(&cb.complete);
430
431 if (cb.verify_result == CPA_FALSE) {
432 status = CPA_STATUS_FAIL;
433 goto fail;
434 }
435
436 if (dir == QAT_ENCRYPT) {
437 /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */
438 memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN);
439 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
440 } else {
441 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
442 }
443
444 fail:
445 if (status != CPA_STATUS_SUCCESS)
446 QAT_STAT_BUMP(crypt_fails);
447
448 for (i = 0; i < in_page_num; i++)
449 kunmap(in_pages[i]);
450 for (i = 0; i < out_page_num; i++)
451 kunmap(out_pages[i]);
452
453 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
454 if (aad_len > 0)
455 QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
456 QAT_PHYS_CONTIG_FREE(op_data.pIv);
457 QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
458 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
459 QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
460 QAT_PHYS_CONTIG_FREE(cy_session_ctx);
461 QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
462 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
463
464 return (status);
465 }
466
467 int
qat_checksum(uint64_t cksum,uint8_t * buf,uint64_t size,zio_cksum_t * zcp)468 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
469 {
470 CpaStatus status;
471 Cpa16U i;
472 CpaInstanceHandle cy_inst_handle;
473 Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
474 Cpa32U bytes_left = 0;
475 Cpa8S *data = NULL;
476 CpaCySymSessionCtx *cy_session_ctx = NULL;
477 cy_callback_t cb;
478 Cpa8U *digest_buffer = NULL;
479 CpaCySymOpData op_data = { 0 };
480 CpaBufferList src_buffer_list = { 0 };
481 CpaFlatBuffer *flat_src_buf_array = NULL;
482 CpaFlatBuffer *flat_src_buf = NULL;
483 struct page *in_pages[MAX_PAGE_NUM];
484 Cpa32U page_num = 0;
485 Cpa32U page_off = 0;
486
487 QAT_STAT_BUMP(cksum_requests);
488 QAT_STAT_INCR(cksum_total_in_bytes, size);
489
490 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
491 cy_inst_handle = cy_inst_handles[i];
492
493 status = qat_init_checksum_session_ctx(cy_inst_handle,
494 &cy_session_ctx, cksum);
495 if (status != CPA_STATUS_SUCCESS) {
496 /* don't count unsupported checksums as a failure */
497 if (cksum == ZIO_CHECKSUM_SHA256 ||
498 cksum == ZIO_CHECKSUM_SHA512)
499 QAT_STAT_BUMP(cksum_fails);
500 return (status);
501 }
502
503 /*
504 * We increment nr_bufs by 2 to allow us to handle non
505 * page-aligned buffer addresses and buffers whose sizes
506 * are not divisible by PAGE_SIZE.
507 */
508 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
509 &src_buffer_list, &src_buffer_list);
510 if (status != CPA_STATUS_SUCCESS)
511 goto fail;
512
513 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
514 nr_bufs * sizeof (CpaFlatBuffer));
515 if (status != CPA_STATUS_SUCCESS)
516 goto fail;
517 status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
518 sizeof (zio_cksum_t));
519 if (status != CPA_STATUS_SUCCESS)
520 goto fail;
521
522 bytes_left = size;
523 data = buf;
524 flat_src_buf = flat_src_buf_array;
525 while (bytes_left > 0) {
526 page_off = ((long)data & ~PAGE_MASK);
527 in_pages[page_num] = qat_mem_to_page(data);
528 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
529 flat_src_buf->dataLenInBytes =
530 min((long)PAGE_SIZE - page_off, (long)bytes_left);
531 data += flat_src_buf->dataLenInBytes;
532 bytes_left -= flat_src_buf->dataLenInBytes;
533 flat_src_buf++;
534 page_num++;
535 }
536 src_buffer_list.pBuffers = flat_src_buf_array;
537 src_buffer_list.numBuffers = page_num;
538
539 op_data.sessionCtx = cy_session_ctx;
540 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
541 op_data.hashStartSrcOffsetInBytes = 0;
542 op_data.messageLenToHashInBytes = size;
543 op_data.pDigestResult = digest_buffer;
544
545 cb.verify_result = CPA_FALSE;
546 init_completion(&cb.complete);
547 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
548 &src_buffer_list, &src_buffer_list, NULL);
549 if (status != CPA_STATUS_SUCCESS)
550 goto fail;
551
552 /* we now wait until the completion of the operation. */
553 wait_for_completion(&cb.complete);
554
555 if (cb.verify_result == CPA_FALSE) {
556 status = CPA_STATUS_FAIL;
557 goto fail;
558 }
559
560 memcpy(zcp, digest_buffer, sizeof (zio_cksum_t));
561
562 fail:
563 if (status != CPA_STATUS_SUCCESS)
564 QAT_STAT_BUMP(cksum_fails);
565
566 for (i = 0; i < page_num; i++)
567 kunmap(in_pages[i]);
568
569 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
570 QAT_PHYS_CONTIG_FREE(digest_buffer);
571 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
572 QAT_PHYS_CONTIG_FREE(cy_session_ctx);
573 QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
574
575 return (status);
576 }
577
578 static int
param_set_qat_encrypt(const char * val,zfs_kernel_param_t * kp)579 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp)
580 {
581 int ret;
582 int *pvalue = kp->arg;
583 ret = param_set_int(val, kp);
584 if (ret)
585 return (ret);
586 /*
587 * zfs_qat_encrypt_disable = 0: enable qat encrypt
588 * try to initialize qat instance if it has not been done
589 */
590 if (*pvalue == 0 && !qat_cy_init_done) {
591 ret = qat_cy_init();
592 if (ret != 0) {
593 zfs_qat_encrypt_disable = 1;
594 return (ret);
595 }
596 }
597 return (ret);
598 }
599
600 static int
param_set_qat_checksum(const char * val,zfs_kernel_param_t * kp)601 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp)
602 {
603 int ret;
604 int *pvalue = kp->arg;
605 ret = param_set_int(val, kp);
606 if (ret)
607 return (ret);
608 /*
609 * set_checksum_param_ops = 0: enable qat checksum
610 * try to initialize qat instance if it has not been done
611 */
612 if (*pvalue == 0 && !qat_cy_init_done) {
613 ret = qat_cy_init();
614 if (ret != 0) {
615 zfs_qat_checksum_disable = 1;
616 return (ret);
617 }
618 }
619 return (ret);
620 }
621
622 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
623 param_get_int, &zfs_qat_encrypt_disable, 0644);
624 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
625
626 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
627 param_get_int, &zfs_qat_checksum_disable, 0644);
628 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
629
630 #endif
631