xref: /freebsd/sys/contrib/openzfs/module/os/linux/zfs/qat_crypt.c (revision 61145dc2b94f12f6a47344fb9aac702321880e43)
1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3  * CDDL HEADER START
4  *
5  * The contents of this file are subject to the terms of the
6  * Common Development and Distribution License (the "License").
7  * You may not use this file except in compliance with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or https://opensource.org/licenses/CDDL-1.0.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 /*
24  * This file represents the QAT implementation of checksums and encryption.
25  * Internally, QAT shares the same cryptographic instances for both of these
26  * operations, so the code has been combined here. QAT data compression uses
27  * compression instances, so that code is separated into qat_compress.c
28  */
29 
30 #if defined(_KERNEL) && defined(HAVE_QAT)
31 #include <linux/slab.h>
32 #include <linux/vmalloc.h>
33 #include <linux/pagemap.h>
34 #include <linux/completion.h>
35 #include <sys/zfs_context.h>
36 #include <sys/zio_crypt.h>
37 #include "lac/cpa_cy_im.h"
38 #include "lac/cpa_cy_common.h"
39 #include <sys/qat.h>
40 
41 /*
42  * Max instances in a QAT device, each instance is a channel to submit
43  * jobs to QAT hardware, this is only for pre-allocating instances
44  * and session arrays; the actual number of instances are defined in
45  * the QAT driver's configure file.
46  */
47 #define	QAT_CRYPT_MAX_INSTANCES		48
48 
49 #define	MAX_PAGE_NUM			1024
50 
51 static Cpa32U inst_num = 0;
52 static Cpa16U num_inst = 0;
53 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
54 static boolean_t qat_cy_init_done = B_FALSE;
55 int zfs_qat_encrypt_disable = 0;
56 int zfs_qat_checksum_disable = 0;
57 
58 typedef struct cy_callback {
59 	CpaBoolean verify_result;
60 	struct completion complete;
61 } cy_callback_t;
62 
63 static void
symcallback(void * p_callback,CpaStatus status,const CpaCySymOp operation,void * op_data,CpaBufferList * buf_list_dst,CpaBoolean verify)64 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
65     void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
66 {
67 	cy_callback_t *cb = p_callback;
68 
69 	if (cb != NULL) {
70 		/* indicate that the function has been called */
71 		cb->verify_result = verify;
72 		complete(&cb->complete);
73 	}
74 }
75 
76 boolean_t
qat_crypt_use_accel(size_t s_len)77 qat_crypt_use_accel(size_t s_len)
78 {
79 	return (!zfs_qat_encrypt_disable &&
80 	    qat_cy_init_done &&
81 	    s_len >= QAT_MIN_BUF_SIZE &&
82 	    s_len <= QAT_MAX_BUF_SIZE);
83 }
84 
85 boolean_t
qat_checksum_use_accel(size_t s_len)86 qat_checksum_use_accel(size_t s_len)
87 {
88 	return (!zfs_qat_checksum_disable &&
89 	    qat_cy_init_done &&
90 	    s_len >= QAT_MIN_BUF_SIZE &&
91 	    s_len <= QAT_MAX_BUF_SIZE);
92 }
93 
94 void
qat_cy_clean(void)95 qat_cy_clean(void)
96 {
97 	for (Cpa16U i = 0; i < num_inst; i++)
98 		cpaCyStopInstance(cy_inst_handles[i]);
99 
100 	num_inst = 0;
101 	qat_cy_init_done = B_FALSE;
102 }
103 
104 int
qat_cy_init(void)105 qat_cy_init(void)
106 {
107 	CpaStatus status = CPA_STATUS_FAIL;
108 
109 	if (qat_cy_init_done)
110 		return (0);
111 
112 	status = cpaCyGetNumInstances(&num_inst);
113 	if (status != CPA_STATUS_SUCCESS)
114 		return (-1);
115 
116 	/* if the user has configured no QAT encryption units just return */
117 	if (num_inst == 0)
118 		return (0);
119 
120 	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
121 		num_inst = QAT_CRYPT_MAX_INSTANCES;
122 
123 	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
124 	if (status != CPA_STATUS_SUCCESS)
125 		return (-1);
126 
127 	for (Cpa16U i = 0; i < num_inst; i++) {
128 		status = cpaCySetAddressTranslation(cy_inst_handles[i],
129 		    (void *)virt_to_phys);
130 		if (status != CPA_STATUS_SUCCESS)
131 			goto error;
132 
133 		status = cpaCyStartInstance(cy_inst_handles[i]);
134 		if (status != CPA_STATUS_SUCCESS)
135 			goto error;
136 	}
137 
138 	qat_cy_init_done = B_TRUE;
139 	return (0);
140 
141 error:
142 	qat_cy_clean();
143 	return (-1);
144 }
145 
146 void
qat_cy_fini(void)147 qat_cy_fini(void)
148 {
149 	if (!qat_cy_init_done)
150 		return;
151 
152 	qat_cy_clean();
153 }
154 
155 static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir,CpaInstanceHandle inst_handle,CpaCySymSessionCtx ** cy_session_ctx,crypto_key_t * key,Cpa64U crypt,Cpa32U aad_len)156 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
157     CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
158     Cpa64U crypt, Cpa32U aad_len)
159 {
160 	CpaStatus status = CPA_STATUS_SUCCESS;
161 	Cpa32U ctx_size;
162 	Cpa32U ciper_algorithm;
163 	Cpa32U hash_algorithm;
164 	CpaCySymSessionSetupData sd = { 0 };
165 
166 	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
167 		return (CPA_STATUS_FAIL);
168 	} else {
169 		ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
170 		hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
171 	}
172 
173 	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
174 	sd.cipherSetupData.pCipherKey = key->ck_data;
175 	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
176 	sd.hashSetupData.hashAlgorithm = hash_algorithm;
177 	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
178 	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
179 	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
180 	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
181 	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
182 	sd.digestIsAppended = CPA_FALSE;
183 	sd.verifyDigest = CPA_FALSE;
184 
185 	if (dir == QAT_ENCRYPT) {
186 		sd.cipherSetupData.cipherDirection =
187 		    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
188 		sd.algChainOrder =
189 		    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
190 	} else {
191 		ASSERT3U(dir, ==, QAT_DECRYPT);
192 		sd.cipherSetupData.cipherDirection =
193 		    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
194 		sd.algChainOrder =
195 		    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
196 	}
197 
198 	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
199 	if (status != CPA_STATUS_SUCCESS)
200 		return (status);
201 
202 	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
203 	if (status != CPA_STATUS_SUCCESS)
204 		return (status);
205 
206 	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
207 	    *cy_session_ctx);
208 	if (status != CPA_STATUS_SUCCESS) {
209 		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
210 		return (status);
211 	}
212 
213 	return (CPA_STATUS_SUCCESS);
214 }
215 
216 static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,CpaCySymSessionCtx ** cy_session_ctx,Cpa64U cksum)217 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
218     CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
219 {
220 	CpaStatus status = CPA_STATUS_SUCCESS;
221 	Cpa32U ctx_size;
222 	Cpa32U hash_algorithm;
223 	CpaCySymSessionSetupData sd = { 0 };
224 
225 	/*
226 	 * ZFS's SHA512 checksum is actually SHA512/256, which uses
227 	 * a different IV from standard SHA512. QAT does not support
228 	 * SHA512/256, so we can only support SHA256.
229 	 */
230 	if (cksum == ZIO_CHECKSUM_SHA256)
231 		hash_algorithm = CPA_CY_SYM_HASH_SHA256;
232 	else
233 		return (CPA_STATUS_FAIL);
234 
235 	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
236 	sd.symOperation = CPA_CY_SYM_OP_HASH;
237 	sd.hashSetupData.hashAlgorithm = hash_algorithm;
238 	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
239 	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
240 	sd.digestIsAppended = CPA_FALSE;
241 	sd.verifyDigest = CPA_FALSE;
242 
243 	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
244 	if (status != CPA_STATUS_SUCCESS)
245 		return (status);
246 
247 	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
248 	if (status != CPA_STATUS_SUCCESS)
249 		return (status);
250 
251 	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
252 	    *cy_session_ctx);
253 	if (status != CPA_STATUS_SUCCESS) {
254 		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
255 		return (status);
256 	}
257 
258 	return (CPA_STATUS_SUCCESS);
259 }
260 
261 static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle,uint32_t nr_bufs,CpaBufferList * src,CpaBufferList * dst)262 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
263     CpaBufferList *src, CpaBufferList *dst)
264 {
265 	CpaStatus status = CPA_STATUS_SUCCESS;
266 	Cpa32U meta_size = 0;
267 
268 	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
269 	if (status != CPA_STATUS_SUCCESS)
270 		return (status);
271 
272 	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
273 	if (status != CPA_STATUS_SUCCESS)
274 		goto error;
275 
276 	if (src != dst) {
277 		status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
278 		    meta_size);
279 		if (status != CPA_STATUS_SUCCESS)
280 			goto error;
281 	}
282 
283 	return (CPA_STATUS_SUCCESS);
284 
285 error:
286 	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
287 	if (src != dst)
288 		QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
289 
290 	return (status);
291 }
292 
293 int
qat_crypt(qat_encrypt_dir_t dir,uint8_t * src_buf,uint8_t * dst_buf,uint8_t * aad_buf,uint32_t aad_len,uint8_t * iv_buf,uint8_t * digest_buf,crypto_key_t * key,uint64_t crypt,uint32_t enc_len)294 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
295     uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
296     crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
297 {
298 	CpaStatus status = CPA_STATUS_SUCCESS;
299 	Cpa16U i;
300 	CpaInstanceHandle cy_inst_handle;
301 	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
302 	Cpa32U bytes_left = 0;
303 	Cpa8S *data = NULL;
304 	CpaCySymSessionCtx *cy_session_ctx = NULL;
305 	cy_callback_t cb;
306 	CpaCySymOpData op_data = { 0 };
307 	CpaBufferList src_buffer_list = { 0 };
308 	CpaBufferList dst_buffer_list = { 0 };
309 	CpaFlatBuffer *flat_src_buf_array = NULL;
310 	CpaFlatBuffer *flat_src_buf = NULL;
311 	CpaFlatBuffer *flat_dst_buf_array = NULL;
312 	CpaFlatBuffer *flat_dst_buf = NULL;
313 	struct page *in_pages[MAX_PAGE_NUM];
314 	struct page *out_pages[MAX_PAGE_NUM];
315 	Cpa32U in_page_num = 0;
316 	Cpa32U out_page_num = 0;
317 	Cpa32U in_page_off = 0;
318 	Cpa32U out_page_off = 0;
319 
320 	if (dir == QAT_ENCRYPT) {
321 		QAT_STAT_BUMP(encrypt_requests);
322 		QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
323 	} else {
324 		QAT_STAT_BUMP(decrypt_requests);
325 		QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
326 	}
327 
328 	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
329 	cy_inst_handle = cy_inst_handles[i];
330 
331 	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
332 	    &cy_session_ctx, key, crypt, aad_len);
333 	if (status != CPA_STATUS_SUCCESS) {
334 		/* don't count CCM as a failure since it's not supported */
335 		if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
336 			QAT_STAT_BUMP(crypt_fails);
337 		return (status);
338 	}
339 
340 	/*
341 	 * We increment nr_bufs by 2 to allow us to handle non
342 	 * page-aligned buffer addresses and buffers whose sizes
343 	 * are not divisible by PAGE_SIZE.
344 	 */
345 	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
346 	    &src_buffer_list, &dst_buffer_list);
347 	if (status != CPA_STATUS_SUCCESS)
348 		goto fail;
349 
350 	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
351 	    nr_bufs * sizeof (CpaFlatBuffer));
352 	if (status != CPA_STATUS_SUCCESS)
353 		goto fail;
354 	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
355 	    nr_bufs * sizeof (CpaFlatBuffer));
356 	if (status != CPA_STATUS_SUCCESS)
357 		goto fail;
358 	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
359 	    ZIO_DATA_MAC_LEN);
360 	if (status != CPA_STATUS_SUCCESS)
361 		goto fail;
362 	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
363 	    ZIO_DATA_IV_LEN);
364 	if (status != CPA_STATUS_SUCCESS)
365 		goto fail;
366 	if (aad_len > 0) {
367 		status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
368 		    aad_len);
369 		if (status != CPA_STATUS_SUCCESS)
370 			goto fail;
371 		memcpy(op_data.pAdditionalAuthData, aad_buf, aad_len);
372 	}
373 
374 	bytes_left = enc_len;
375 	data = src_buf;
376 	flat_src_buf = flat_src_buf_array;
377 	while (bytes_left > 0) {
378 		in_page_off = ((long)data & ~PAGE_MASK);
379 		in_pages[in_page_num] = qat_mem_to_page(data);
380 		flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
381 		flat_src_buf->dataLenInBytes =
382 		    min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
383 		data += flat_src_buf->dataLenInBytes;
384 		bytes_left -= flat_src_buf->dataLenInBytes;
385 		flat_src_buf++;
386 		in_page_num++;
387 	}
388 	src_buffer_list.pBuffers = flat_src_buf_array;
389 	src_buffer_list.numBuffers = in_page_num;
390 
391 	bytes_left = enc_len;
392 	data = dst_buf;
393 	flat_dst_buf = flat_dst_buf_array;
394 	while (bytes_left > 0) {
395 		out_page_off = ((long)data & ~PAGE_MASK);
396 		out_pages[out_page_num] = qat_mem_to_page(data);
397 		flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
398 		    out_page_off;
399 		flat_dst_buf->dataLenInBytes =
400 		    min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
401 		data += flat_dst_buf->dataLenInBytes;
402 		bytes_left -= flat_dst_buf->dataLenInBytes;
403 		flat_dst_buf++;
404 		out_page_num++;
405 	}
406 	dst_buffer_list.pBuffers = flat_dst_buf_array;
407 	dst_buffer_list.numBuffers = out_page_num;
408 
409 	op_data.sessionCtx = cy_session_ctx;
410 	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
411 	op_data.cryptoStartSrcOffsetInBytes = 0;
412 	op_data.messageLenToCipherInBytes = 0;
413 	op_data.hashStartSrcOffsetInBytes = 0;
414 	op_data.messageLenToHashInBytes = 0;
415 	op_data.messageLenToCipherInBytes = enc_len;
416 	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
417 	memcpy(op_data.pIv, iv_buf, ZIO_DATA_IV_LEN);
418 	/* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */
419 	if (dir == QAT_DECRYPT)
420 		memcpy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
421 
422 	cb.verify_result = CPA_FALSE;
423 	init_completion(&cb.complete);
424 	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
425 	    &src_buffer_list, &dst_buffer_list, NULL);
426 	if (status != CPA_STATUS_SUCCESS)
427 		goto fail;
428 
429 	/* we now wait until the completion of the operation. */
430 	wait_for_completion(&cb.complete);
431 
432 	if (cb.verify_result == CPA_FALSE) {
433 		status = CPA_STATUS_FAIL;
434 		goto fail;
435 	}
436 
437 	if (dir == QAT_ENCRYPT) {
438 		/* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */
439 		memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN);
440 		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
441 	} else {
442 		QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
443 	}
444 
445 fail:
446 	if (status != CPA_STATUS_SUCCESS)
447 		QAT_STAT_BUMP(crypt_fails);
448 
449 	for (i = 0; i < in_page_num; i++)
450 		kunmap(in_pages[i]);
451 	for (i = 0; i < out_page_num; i++)
452 		kunmap(out_pages[i]);
453 
454 	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
455 	if (aad_len > 0)
456 		QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
457 	QAT_PHYS_CONTIG_FREE(op_data.pIv);
458 	QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
459 	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
460 	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
461 	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
462 	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
463 	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
464 
465 	return (status);
466 }
467 
468 int
qat_checksum(uint64_t cksum,uint8_t * buf,uint64_t size,zio_cksum_t * zcp)469 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
470 {
471 	CpaStatus status;
472 	Cpa16U i;
473 	CpaInstanceHandle cy_inst_handle;
474 	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
475 	Cpa32U bytes_left = 0;
476 	Cpa8S *data = NULL;
477 	CpaCySymSessionCtx *cy_session_ctx = NULL;
478 	cy_callback_t cb;
479 	Cpa8U *digest_buffer = NULL;
480 	CpaCySymOpData op_data = { 0 };
481 	CpaBufferList src_buffer_list = { 0 };
482 	CpaFlatBuffer *flat_src_buf_array = NULL;
483 	CpaFlatBuffer *flat_src_buf = NULL;
484 	struct page *in_pages[MAX_PAGE_NUM];
485 	Cpa32U page_num = 0;
486 	Cpa32U page_off = 0;
487 
488 	QAT_STAT_BUMP(cksum_requests);
489 	QAT_STAT_INCR(cksum_total_in_bytes, size);
490 
491 	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
492 	cy_inst_handle = cy_inst_handles[i];
493 
494 	status = qat_init_checksum_session_ctx(cy_inst_handle,
495 	    &cy_session_ctx, cksum);
496 	if (status != CPA_STATUS_SUCCESS) {
497 		/* don't count unsupported checksums as a failure */
498 		if (cksum == ZIO_CHECKSUM_SHA256 ||
499 		    cksum == ZIO_CHECKSUM_SHA512)
500 			QAT_STAT_BUMP(cksum_fails);
501 		return (status);
502 	}
503 
504 	/*
505 	 * We increment nr_bufs by 2 to allow us to handle non
506 	 * page-aligned buffer addresses and buffers whose sizes
507 	 * are not divisible by PAGE_SIZE.
508 	 */
509 	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
510 	    &src_buffer_list, &src_buffer_list);
511 	if (status != CPA_STATUS_SUCCESS)
512 		goto fail;
513 
514 	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
515 	    nr_bufs * sizeof (CpaFlatBuffer));
516 	if (status != CPA_STATUS_SUCCESS)
517 		goto fail;
518 	status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
519 	    sizeof (zio_cksum_t));
520 	if (status != CPA_STATUS_SUCCESS)
521 		goto fail;
522 
523 	bytes_left = size;
524 	data = buf;
525 	flat_src_buf = flat_src_buf_array;
526 	while (bytes_left > 0) {
527 		page_off = ((long)data & ~PAGE_MASK);
528 		in_pages[page_num] = qat_mem_to_page(data);
529 		flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
530 		flat_src_buf->dataLenInBytes =
531 		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
532 		data += flat_src_buf->dataLenInBytes;
533 		bytes_left -= flat_src_buf->dataLenInBytes;
534 		flat_src_buf++;
535 		page_num++;
536 	}
537 	src_buffer_list.pBuffers = flat_src_buf_array;
538 	src_buffer_list.numBuffers = page_num;
539 
540 	op_data.sessionCtx = cy_session_ctx;
541 	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
542 	op_data.hashStartSrcOffsetInBytes = 0;
543 	op_data.messageLenToHashInBytes = size;
544 	op_data.pDigestResult = digest_buffer;
545 
546 	cb.verify_result = CPA_FALSE;
547 	init_completion(&cb.complete);
548 	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
549 	    &src_buffer_list, &src_buffer_list, NULL);
550 	if (status != CPA_STATUS_SUCCESS)
551 		goto fail;
552 
553 	/* we now wait until the completion of the operation. */
554 	wait_for_completion(&cb.complete);
555 
556 	if (cb.verify_result == CPA_FALSE) {
557 		status = CPA_STATUS_FAIL;
558 		goto fail;
559 	}
560 
561 	memcpy(zcp, digest_buffer, sizeof (zio_cksum_t));
562 
563 fail:
564 	if (status != CPA_STATUS_SUCCESS)
565 		QAT_STAT_BUMP(cksum_fails);
566 
567 	for (i = 0; i < page_num; i++)
568 		kunmap(in_pages[i]);
569 
570 	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
571 	QAT_PHYS_CONTIG_FREE(digest_buffer);
572 	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
573 	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
574 	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
575 
576 	return (status);
577 }
578 
579 static int
param_set_qat_encrypt(const char * val,zfs_kernel_param_t * kp)580 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp)
581 {
582 	int ret;
583 	int *pvalue = kp->arg;
584 	ret = param_set_int(val, kp);
585 	if (ret)
586 		return (ret);
587 	/*
588 	 * zfs_qat_encrypt_disable = 0: enable qat encrypt
589 	 * try to initialize qat instance if it has not been done
590 	 */
591 	if (*pvalue == 0 && !qat_cy_init_done) {
592 		ret = qat_cy_init();
593 		if (ret != 0) {
594 			zfs_qat_encrypt_disable = 1;
595 			return (ret);
596 		}
597 	}
598 	return (ret);
599 }
600 
601 static int
param_set_qat_checksum(const char * val,zfs_kernel_param_t * kp)602 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp)
603 {
604 	int ret;
605 	int *pvalue = kp->arg;
606 	ret = param_set_int(val, kp);
607 	if (ret)
608 		return (ret);
609 	/*
610 	 * set_checksum_param_ops = 0: enable qat checksum
611 	 * try to initialize qat instance if it has not been done
612 	 */
613 	if (*pvalue == 0 && !qat_cy_init_done) {
614 		ret = qat_cy_init();
615 		if (ret != 0) {
616 			zfs_qat_checksum_disable = 1;
617 			return (ret);
618 		}
619 	}
620 	return (ret);
621 }
622 
623 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
624     param_get_int, &zfs_qat_encrypt_disable, 0644);
625 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
626 
627 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
628     param_get_int, &zfs_qat_checksum_disable, 0644);
629 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
630 
631 #endif
632