xref: /freebsd/sys/contrib/openzfs/module/os/linux/zfs/qat_compress.c (revision 61145dc2b94f12f6a47344fb9aac702321880e43)
1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3  * CDDL HEADER START
4  *
5  * The contents of this file are subject to the terms of the
6  * Common Development and Distribution License (the "License").
7  * You may not use this file except in compliance with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or https://opensource.org/licenses/CDDL-1.0.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 #if defined(_KERNEL) && defined(HAVE_QAT)
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <linux/pagemap.h>
27 #include <linux/completion.h>
28 #include <sys/zfs_context.h>
29 #include <sys/byteorder.h>
30 #include <sys/zio.h>
31 #include <sys/qat.h>
32 
33 /*
34  * Max instances in a QAT device, each instance is a channel to submit
35  * jobs to QAT hardware, this is only for pre-allocating instance and
36  * session arrays; the actual number of instances are defined in the
37  * QAT driver's configuration file.
38  */
39 #define	QAT_DC_MAX_INSTANCES	48
40 
41 /*
42  * ZLIB head and foot size
43  */
44 #define	ZLIB_HEAD_SZ		2
45 #define	ZLIB_FOOT_SZ		4
46 
47 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
48 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
49 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
50 static Cpa16U num_inst = 0;
51 static Cpa32U inst_num = 0;
52 static boolean_t qat_dc_init_done = B_FALSE;
53 int zfs_qat_compress_disable = 0;
54 
55 boolean_t
qat_dc_use_accel(size_t s_len)56 qat_dc_use_accel(size_t s_len)
57 {
58 	return (!zfs_qat_compress_disable &&
59 	    qat_dc_init_done &&
60 	    s_len >= QAT_MIN_BUF_SIZE &&
61 	    s_len <= QAT_MAX_BUF_SIZE);
62 }
63 
64 static void
qat_dc_callback(void * p_callback,CpaStatus status)65 qat_dc_callback(void *p_callback, CpaStatus status)
66 {
67 	if (p_callback != NULL)
68 		complete((struct completion *)p_callback);
69 }
70 
71 static void
qat_dc_clean(void)72 qat_dc_clean(void)
73 {
74 	Cpa16U buff_num = 0;
75 	Cpa16U num_inter_buff_lists = 0;
76 
77 	for (Cpa16U i = 0; i < num_inst; i++) {
78 		cpaDcStopInstance(dc_inst_handles[i]);
79 		QAT_PHYS_CONTIG_FREE(session_handles[i]);
80 		/* free intermediate buffers  */
81 		if (buffer_array[i] != NULL) {
82 			cpaDcGetNumIntermediateBuffers(
83 			    dc_inst_handles[i], &num_inter_buff_lists);
84 			for (buff_num = 0; buff_num < num_inter_buff_lists;
85 			    buff_num++) {
86 				CpaBufferList *buffer_inter =
87 				    buffer_array[i][buff_num];
88 				if (buffer_inter->pBuffers) {
89 					QAT_PHYS_CONTIG_FREE(
90 					    buffer_inter->pBuffers->pData);
91 					QAT_PHYS_CONTIG_FREE(
92 					    buffer_inter->pBuffers);
93 				}
94 				QAT_PHYS_CONTIG_FREE(
95 				    buffer_inter->pPrivateMetaData);
96 				QAT_PHYS_CONTIG_FREE(buffer_inter);
97 			}
98 		}
99 	}
100 
101 	num_inst = 0;
102 	qat_dc_init_done = B_FALSE;
103 }
104 
105 int
qat_dc_init(void)106 qat_dc_init(void)
107 {
108 	CpaStatus status = CPA_STATUS_SUCCESS;
109 	Cpa32U sess_size = 0;
110 	Cpa32U ctx_size = 0;
111 	Cpa16U num_inter_buff_lists = 0;
112 	Cpa16U buff_num = 0;
113 	Cpa32U buff_meta_size = 0;
114 	CpaDcSessionSetupData sd = {0};
115 
116 	if (qat_dc_init_done)
117 		return (0);
118 
119 	status = cpaDcGetNumInstances(&num_inst);
120 	if (status != CPA_STATUS_SUCCESS)
121 		return (-1);
122 
123 	/* if the user has configured no QAT compression units just return */
124 	if (num_inst == 0)
125 		return (0);
126 
127 	if (num_inst > QAT_DC_MAX_INSTANCES)
128 		num_inst = QAT_DC_MAX_INSTANCES;
129 
130 	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
131 	if (status != CPA_STATUS_SUCCESS)
132 		return (-1);
133 
134 	for (Cpa16U i = 0; i < num_inst; i++) {
135 		cpaDcSetAddressTranslation(dc_inst_handles[i],
136 		    (void*)virt_to_phys);
137 
138 		status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
139 		    1, &buff_meta_size);
140 
141 		if (status == CPA_STATUS_SUCCESS)
142 			status = cpaDcGetNumIntermediateBuffers(
143 			    dc_inst_handles[i], &num_inter_buff_lists);
144 
145 		if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
146 			status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
147 			    num_inter_buff_lists *
148 			    sizeof (CpaBufferList *));
149 
150 		for (buff_num = 0; buff_num < num_inter_buff_lists;
151 		    buff_num++) {
152 			if (status == CPA_STATUS_SUCCESS)
153 				status = QAT_PHYS_CONTIG_ALLOC(
154 				    &buffer_array[i][buff_num],
155 				    sizeof (CpaBufferList));
156 
157 			if (status == CPA_STATUS_SUCCESS)
158 				status = QAT_PHYS_CONTIG_ALLOC(
159 				    &buffer_array[i][buff_num]->
160 				    pPrivateMetaData,
161 				    buff_meta_size);
162 
163 			if (status == CPA_STATUS_SUCCESS)
164 				status = QAT_PHYS_CONTIG_ALLOC(
165 				    &buffer_array[i][buff_num]->pBuffers,
166 				    sizeof (CpaFlatBuffer));
167 
168 			if (status == CPA_STATUS_SUCCESS) {
169 				/*
170 				 *  implementation requires an intermediate
171 				 *  buffer approximately twice the size of
172 				 *  output buffer, which is 2x max buffer
173 				 *  size here.
174 				 */
175 				status = QAT_PHYS_CONTIG_ALLOC(
176 				    &buffer_array[i][buff_num]->pBuffers->
177 				    pData, 2 * QAT_MAX_BUF_SIZE);
178 				if (status != CPA_STATUS_SUCCESS)
179 					goto fail;
180 
181 				buffer_array[i][buff_num]->numBuffers = 1;
182 				buffer_array[i][buff_num]->pBuffers->
183 				    dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
184 			}
185 		}
186 
187 		status = cpaDcStartInstance(dc_inst_handles[i],
188 		    num_inter_buff_lists, buffer_array[i]);
189 		if (status != CPA_STATUS_SUCCESS)
190 			goto fail;
191 
192 		sd.compLevel = CPA_DC_L1;
193 		sd.compType = CPA_DC_DEFLATE;
194 		sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
195 		sd.sessDirection = CPA_DC_DIR_COMBINED;
196 		sd.sessState = CPA_DC_STATELESS;
197 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6)
198 		sd.deflateWindowSize = 7;
199 #endif
200 		sd.checksum = CPA_DC_ADLER32;
201 		status = cpaDcGetSessionSize(dc_inst_handles[i],
202 		    &sd, &sess_size, &ctx_size);
203 		if (status != CPA_STATUS_SUCCESS)
204 			goto fail;
205 
206 		QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
207 		if (session_handles[i] == NULL)
208 			goto fail;
209 
210 		status = cpaDcInitSession(dc_inst_handles[i],
211 		    session_handles[i],
212 		    &sd, NULL, qat_dc_callback);
213 		if (status != CPA_STATUS_SUCCESS)
214 			goto fail;
215 	}
216 
217 	qat_dc_init_done = B_TRUE;
218 	return (0);
219 fail:
220 	qat_dc_clean();
221 	return (-1);
222 }
223 
224 void
qat_dc_fini(void)225 qat_dc_fini(void)
226 {
227 	if (!qat_dc_init_done)
228 		return;
229 
230 	qat_dc_clean();
231 }
232 
233 /*
234  * The "add" parameter is an additional buffer which is passed
235  * to QAT as a scratch buffer alongside the destination buffer
236  * in case the "compressed" data ends up being larger than the
237  * original source data. This is necessary to prevent QAT from
238  * generating buffer overflow warnings for incompressible data.
239  */
240 static int
qat_compress_impl(qat_compress_dir_t dir,char * src,int src_len,char * dst,int dst_len,char * add,int add_len,size_t * c_len)241 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
242     char *dst, int dst_len, char *add, int add_len, size_t *c_len)
243 {
244 	CpaInstanceHandle dc_inst_handle;
245 	CpaDcSessionHandle session_handle;
246 	CpaBufferList *buf_list_src = NULL;
247 	CpaBufferList *buf_list_dst = NULL;
248 	CpaFlatBuffer *flat_buf_src = NULL;
249 	CpaFlatBuffer *flat_buf_dst = NULL;
250 	Cpa8U *buffer_meta_src = NULL;
251 	Cpa8U *buffer_meta_dst = NULL;
252 	Cpa32U buffer_meta_size = 0;
253 	CpaDcRqResults dc_results = {.checksum = 1};
254 	CpaStatus status = CPA_STATUS_FAIL;
255 	Cpa32U hdr_sz = 0;
256 	Cpa32U compressed_sz;
257 	Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
258 	Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
259 	Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
260 	Cpa32U bytes_left;
261 	Cpa32U dst_pages = 0;
262 	Cpa32U adler32 = 0;
263 	char *data;
264 	struct page *page;
265 	struct page **in_pages = NULL;
266 	struct page **out_pages = NULL;
267 	struct page **add_pages = NULL;
268 	Cpa32U page_off = 0;
269 	struct completion complete;
270 	Cpa32U page_num = 0;
271 	Cpa16U i;
272 
273 	/*
274 	 * We increment num_src_buf and num_dst_buf by 2 to allow
275 	 * us to handle non page-aligned buffer addresses and buffers
276 	 * whose sizes are not divisible by PAGE_SIZE.
277 	 */
278 	Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
279 	    (num_src_buf * sizeof (CpaFlatBuffer));
280 	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
281 	    ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
282 
283 	status = QAT_PHYS_CONTIG_ALLOC(&in_pages,
284 	    num_src_buf * sizeof (struct page *));
285 	if (status != CPA_STATUS_SUCCESS)
286 		goto fail;
287 
288 	status = QAT_PHYS_CONTIG_ALLOC(&out_pages,
289 	    num_dst_buf * sizeof (struct page *));
290 	if (status != CPA_STATUS_SUCCESS)
291 		goto fail;
292 
293 	status = QAT_PHYS_CONTIG_ALLOC(&add_pages,
294 	    num_add_buf * sizeof (struct page *));
295 	if (status != CPA_STATUS_SUCCESS)
296 		goto fail;
297 
298 	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
299 	dc_inst_handle = dc_inst_handles[i];
300 	session_handle = session_handles[i];
301 
302 	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
303 	    &buffer_meta_size);
304 	status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size);
305 	if (status != CPA_STATUS_SUCCESS)
306 		goto fail;
307 
308 	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
309 	    &buffer_meta_size);
310 	status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size);
311 	if (status != CPA_STATUS_SUCCESS)
312 		goto fail;
313 
314 	/* build source buffer list */
315 	status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size);
316 	if (status != CPA_STATUS_SUCCESS)
317 		goto fail;
318 
319 	flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
320 
321 	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
322 
323 	/* build destination buffer list */
324 	status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size);
325 	if (status != CPA_STATUS_SUCCESS)
326 		goto fail;
327 
328 	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
329 
330 	buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
331 
332 	buf_list_src->numBuffers = 0;
333 	buf_list_src->pPrivateMetaData = buffer_meta_src;
334 	bytes_left = src_len;
335 	data = src;
336 	page_num = 0;
337 	while (bytes_left > 0) {
338 		page_off = ((long)data & ~PAGE_MASK);
339 		page = qat_mem_to_page(data);
340 		in_pages[page_num] = page;
341 		flat_buf_src->pData = kmap(page) + page_off;
342 		flat_buf_src->dataLenInBytes =
343 		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
344 
345 		bytes_left -= flat_buf_src->dataLenInBytes;
346 		data += flat_buf_src->dataLenInBytes;
347 		flat_buf_src++;
348 		buf_list_src->numBuffers++;
349 		page_num++;
350 	}
351 
352 	buf_list_dst->numBuffers = 0;
353 	buf_list_dst->pPrivateMetaData = buffer_meta_dst;
354 	bytes_left = dst_len;
355 	data = dst;
356 	page_num = 0;
357 	while (bytes_left > 0) {
358 		page_off = ((long)data & ~PAGE_MASK);
359 		page = qat_mem_to_page(data);
360 		flat_buf_dst->pData = kmap(page) + page_off;
361 		out_pages[page_num] = page;
362 		flat_buf_dst->dataLenInBytes =
363 		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
364 
365 		bytes_left -= flat_buf_dst->dataLenInBytes;
366 		data += flat_buf_dst->dataLenInBytes;
367 		flat_buf_dst++;
368 		buf_list_dst->numBuffers++;
369 		page_num++;
370 		dst_pages++;
371 	}
372 
373 	/* map additional scratch pages into the destination buffer list */
374 	bytes_left = add_len;
375 	data = add;
376 	page_num = 0;
377 	while (bytes_left > 0) {
378 		page_off = ((long)data & ~PAGE_MASK);
379 		page = qat_mem_to_page(data);
380 		flat_buf_dst->pData = kmap(page) + page_off;
381 		add_pages[page_num] = page;
382 		flat_buf_dst->dataLenInBytes =
383 		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
384 
385 		bytes_left -= flat_buf_dst->dataLenInBytes;
386 		data += flat_buf_dst->dataLenInBytes;
387 		flat_buf_dst++;
388 		buf_list_dst->numBuffers++;
389 		page_num++;
390 	}
391 
392 	init_completion(&complete);
393 
394 	if (dir == QAT_COMPRESS) {
395 		QAT_STAT_BUMP(comp_requests);
396 		QAT_STAT_INCR(comp_total_in_bytes, src_len);
397 
398 		cpaDcGenerateHeader(session_handle,
399 		    buf_list_dst->pBuffers, &hdr_sz);
400 		buf_list_dst->pBuffers->pData += hdr_sz;
401 		buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
402 		status = cpaDcCompressData(
403 		    dc_inst_handle, session_handle,
404 		    buf_list_src, buf_list_dst,
405 		    &dc_results, CPA_DC_FLUSH_FINAL,
406 		    &complete);
407 		if (status != CPA_STATUS_SUCCESS) {
408 			goto fail;
409 		}
410 
411 		/* we now wait until the completion of the operation. */
412 		wait_for_completion(&complete);
413 
414 		if (dc_results.status != CPA_STATUS_SUCCESS) {
415 			status = CPA_STATUS_FAIL;
416 			goto fail;
417 		}
418 
419 		compressed_sz = dc_results.produced;
420 		if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
421 			status = CPA_STATUS_INCOMPRESSIBLE;
422 			goto fail;
423 		}
424 
425 		/* get adler32 checksum and append footer */
426 		*(Cpa32U*)(dst + hdr_sz + compressed_sz) =
427 		    BSWAP_32(dc_results.checksum);
428 
429 		*c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ;
430 		QAT_STAT_INCR(comp_total_out_bytes, *c_len);
431 	} else {
432 		ASSERT3U(dir, ==, QAT_DECOMPRESS);
433 		QAT_STAT_BUMP(decomp_requests);
434 		QAT_STAT_INCR(decomp_total_in_bytes, src_len);
435 
436 		buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
437 		buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
438 		status = cpaDcDecompressData(dc_inst_handle, session_handle,
439 		    buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
440 		    &complete);
441 
442 		if (CPA_STATUS_SUCCESS != status) {
443 			status = CPA_STATUS_FAIL;
444 			goto fail;
445 		}
446 
447 		/* we now wait until the completion of the operation. */
448 		wait_for_completion(&complete);
449 
450 		if (dc_results.status != CPA_STATUS_SUCCESS) {
451 			status = CPA_STATUS_FAIL;
452 			goto fail;
453 		}
454 
455 		/* verify adler checksum */
456 		adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
457 		if (adler32 != BSWAP_32(dc_results.checksum)) {
458 			status = CPA_STATUS_FAIL;
459 			goto fail;
460 		}
461 		*c_len = dc_results.produced;
462 		QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
463 	}
464 
465 fail:
466 	if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
467 		QAT_STAT_BUMP(dc_fails);
468 
469 	if (in_pages) {
470 		for (page_num = 0;
471 		    page_num < buf_list_src->numBuffers;
472 		    page_num++) {
473 			kunmap(in_pages[page_num]);
474 		}
475 		QAT_PHYS_CONTIG_FREE(in_pages);
476 	}
477 
478 	if (out_pages) {
479 		for (page_num = 0; page_num < dst_pages; page_num++) {
480 			kunmap(out_pages[page_num]);
481 		}
482 		QAT_PHYS_CONTIG_FREE(out_pages);
483 	}
484 
485 	if (add_pages) {
486 		for (page_num = 0;
487 		    page_num < buf_list_dst->numBuffers - dst_pages;
488 		    page_num++) {
489 			kunmap(add_pages[page_num]);
490 		}
491 		QAT_PHYS_CONTIG_FREE(add_pages);
492 	}
493 
494 	QAT_PHYS_CONTIG_FREE(buffer_meta_src);
495 	QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
496 	QAT_PHYS_CONTIG_FREE(buf_list_src);
497 	QAT_PHYS_CONTIG_FREE(buf_list_dst);
498 
499 	return (status);
500 }
501 
502 /*
503  * Entry point for QAT accelerated compression / decompression.
504  */
505 int
qat_compress(qat_compress_dir_t dir,char * src,int src_len,char * dst,int dst_len,size_t * c_len)506 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
507     char *dst, int dst_len, size_t *c_len)
508 {
509 	int ret;
510 	size_t add_len = 0;
511 	void *add = NULL;
512 
513 	if (dir == QAT_COMPRESS) {
514 		add_len = dst_len;
515 		add = zio_data_buf_alloc(add_len);
516 	}
517 
518 	ret = qat_compress_impl(dir, src, src_len, dst,
519 	    dst_len, add, add_len, c_len);
520 
521 	if (dir == QAT_COMPRESS)
522 		zio_data_buf_free(add, add_len);
523 
524 	return (ret);
525 }
526 
527 static int
param_set_qat_compress(const char * val,zfs_kernel_param_t * kp)528 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp)
529 {
530 	int ret;
531 	int *pvalue = kp->arg;
532 	ret = param_set_int(val, kp);
533 	if (ret)
534 		return (ret);
535 	/*
536 	 * zfs_qat_compress_disable = 0: enable qat compress
537 	 * try to initialize qat instance if it has not been done
538 	 */
539 	if (*pvalue == 0 && !qat_dc_init_done) {
540 		ret = qat_dc_init();
541 		if (ret != 0) {
542 			zfs_qat_compress_disable = 1;
543 			return (ret);
544 		}
545 	}
546 	return (ret);
547 }
548 
549 module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
550     param_get_int, &zfs_qat_compress_disable, 0644);
551 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
552 
553 #endif
554