1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 #if defined(_KERNEL) && defined(HAVE_QAT)
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/pagemap.h>
17 #include <linux/completion.h>
18 #include <sys/zfs_context.h>
19 #include <sys/byteorder.h>
20 #include <sys/zio.h>
21 #include <sys/qat.h>
22
23 /*
24 * Max instances in a QAT device, each instance is a channel to submit
25 * jobs to QAT hardware, this is only for pre-allocating instance and
26 * session arrays; the actual number of instances are defined in the
27 * QAT driver's configuration file.
28 */
29 #define QAT_DC_MAX_INSTANCES 48
30
31 /*
32 * ZLIB head and foot size
33 */
34 #define ZLIB_HEAD_SZ 2
35 #define ZLIB_FOOT_SZ 4
36
37 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
38 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
39 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
40 static Cpa16U num_inst = 0;
41 static Cpa32U inst_num = 0;
42 static boolean_t qat_dc_init_done = B_FALSE;
43 int zfs_qat_compress_disable = 0;
44
45 boolean_t
qat_dc_use_accel(size_t s_len)46 qat_dc_use_accel(size_t s_len)
47 {
48 return (!zfs_qat_compress_disable &&
49 qat_dc_init_done &&
50 s_len >= QAT_MIN_BUF_SIZE &&
51 s_len <= QAT_MAX_BUF_SIZE);
52 }
53
54 static void
qat_dc_callback(void * p_callback,CpaStatus status)55 qat_dc_callback(void *p_callback, CpaStatus status)
56 {
57 if (p_callback != NULL)
58 complete((struct completion *)p_callback);
59 }
60
61 static void
qat_dc_clean(void)62 qat_dc_clean(void)
63 {
64 Cpa16U buff_num = 0;
65 Cpa16U num_inter_buff_lists = 0;
66
67 for (Cpa16U i = 0; i < num_inst; i++) {
68 cpaDcStopInstance(dc_inst_handles[i]);
69 QAT_PHYS_CONTIG_FREE(session_handles[i]);
70 /* free intermediate buffers */
71 if (buffer_array[i] != NULL) {
72 cpaDcGetNumIntermediateBuffers(
73 dc_inst_handles[i], &num_inter_buff_lists);
74 for (buff_num = 0; buff_num < num_inter_buff_lists;
75 buff_num++) {
76 CpaBufferList *buffer_inter =
77 buffer_array[i][buff_num];
78 if (buffer_inter->pBuffers) {
79 QAT_PHYS_CONTIG_FREE(
80 buffer_inter->pBuffers->pData);
81 QAT_PHYS_CONTIG_FREE(
82 buffer_inter->pBuffers);
83 }
84 QAT_PHYS_CONTIG_FREE(
85 buffer_inter->pPrivateMetaData);
86 QAT_PHYS_CONTIG_FREE(buffer_inter);
87 }
88 }
89 }
90
91 num_inst = 0;
92 qat_dc_init_done = B_FALSE;
93 }
94
95 int
qat_dc_init(void)96 qat_dc_init(void)
97 {
98 CpaStatus status = CPA_STATUS_SUCCESS;
99 Cpa32U sess_size = 0;
100 Cpa32U ctx_size = 0;
101 Cpa16U num_inter_buff_lists = 0;
102 Cpa16U buff_num = 0;
103 Cpa32U buff_meta_size = 0;
104 CpaDcSessionSetupData sd = {0};
105
106 if (qat_dc_init_done)
107 return (0);
108
109 status = cpaDcGetNumInstances(&num_inst);
110 if (status != CPA_STATUS_SUCCESS)
111 return (-1);
112
113 /* if the user has configured no QAT compression units just return */
114 if (num_inst == 0)
115 return (0);
116
117 if (num_inst > QAT_DC_MAX_INSTANCES)
118 num_inst = QAT_DC_MAX_INSTANCES;
119
120 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
121 if (status != CPA_STATUS_SUCCESS)
122 return (-1);
123
124 for (Cpa16U i = 0; i < num_inst; i++) {
125 cpaDcSetAddressTranslation(dc_inst_handles[i],
126 (void*)virt_to_phys);
127
128 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
129 1, &buff_meta_size);
130
131 if (status == CPA_STATUS_SUCCESS)
132 status = cpaDcGetNumIntermediateBuffers(
133 dc_inst_handles[i], &num_inter_buff_lists);
134
135 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
136 status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
137 num_inter_buff_lists *
138 sizeof (CpaBufferList *));
139
140 for (buff_num = 0; buff_num < num_inter_buff_lists;
141 buff_num++) {
142 if (status == CPA_STATUS_SUCCESS)
143 status = QAT_PHYS_CONTIG_ALLOC(
144 &buffer_array[i][buff_num],
145 sizeof (CpaBufferList));
146
147 if (status == CPA_STATUS_SUCCESS)
148 status = QAT_PHYS_CONTIG_ALLOC(
149 &buffer_array[i][buff_num]->
150 pPrivateMetaData,
151 buff_meta_size);
152
153 if (status == CPA_STATUS_SUCCESS)
154 status = QAT_PHYS_CONTIG_ALLOC(
155 &buffer_array[i][buff_num]->pBuffers,
156 sizeof (CpaFlatBuffer));
157
158 if (status == CPA_STATUS_SUCCESS) {
159 /*
160 * implementation requires an intermediate
161 * buffer approximately twice the size of
162 * output buffer, which is 2x max buffer
163 * size here.
164 */
165 status = QAT_PHYS_CONTIG_ALLOC(
166 &buffer_array[i][buff_num]->pBuffers->
167 pData, 2 * QAT_MAX_BUF_SIZE);
168 if (status != CPA_STATUS_SUCCESS)
169 goto fail;
170
171 buffer_array[i][buff_num]->numBuffers = 1;
172 buffer_array[i][buff_num]->pBuffers->
173 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
174 }
175 }
176
177 status = cpaDcStartInstance(dc_inst_handles[i],
178 num_inter_buff_lists, buffer_array[i]);
179 if (status != CPA_STATUS_SUCCESS)
180 goto fail;
181
182 sd.compLevel = CPA_DC_L1;
183 sd.compType = CPA_DC_DEFLATE;
184 sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
185 sd.sessDirection = CPA_DC_DIR_COMBINED;
186 sd.sessState = CPA_DC_STATELESS;
187 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6)
188 sd.deflateWindowSize = 7;
189 #endif
190 sd.checksum = CPA_DC_ADLER32;
191 status = cpaDcGetSessionSize(dc_inst_handles[i],
192 &sd, &sess_size, &ctx_size);
193 if (status != CPA_STATUS_SUCCESS)
194 goto fail;
195
196 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
197 if (session_handles[i] == NULL)
198 goto fail;
199
200 status = cpaDcInitSession(dc_inst_handles[i],
201 session_handles[i],
202 &sd, NULL, qat_dc_callback);
203 if (status != CPA_STATUS_SUCCESS)
204 goto fail;
205 }
206
207 qat_dc_init_done = B_TRUE;
208 return (0);
209 fail:
210 qat_dc_clean();
211 return (-1);
212 }
213
214 void
qat_dc_fini(void)215 qat_dc_fini(void)
216 {
217 if (!qat_dc_init_done)
218 return;
219
220 qat_dc_clean();
221 }
222
223 /*
224 * The "add" parameter is an additional buffer which is passed
225 * to QAT as a scratch buffer alongside the destination buffer
226 * in case the "compressed" data ends up being larger than the
227 * original source data. This is necessary to prevent QAT from
228 * generating buffer overflow warnings for incompressible data.
229 */
230 static int
qat_compress_impl(qat_compress_dir_t dir,char * src,int src_len,char * dst,int dst_len,char * add,int add_len,size_t * c_len)231 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
232 char *dst, int dst_len, char *add, int add_len, size_t *c_len)
233 {
234 CpaInstanceHandle dc_inst_handle;
235 CpaDcSessionHandle session_handle;
236 CpaBufferList *buf_list_src = NULL;
237 CpaBufferList *buf_list_dst = NULL;
238 CpaFlatBuffer *flat_buf_src = NULL;
239 CpaFlatBuffer *flat_buf_dst = NULL;
240 Cpa8U *buffer_meta_src = NULL;
241 Cpa8U *buffer_meta_dst = NULL;
242 Cpa32U buffer_meta_size = 0;
243 CpaDcRqResults dc_results = {.checksum = 1};
244 CpaStatus status = CPA_STATUS_FAIL;
245 Cpa32U hdr_sz = 0;
246 Cpa32U compressed_sz;
247 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
248 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
249 Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
250 Cpa32U bytes_left;
251 Cpa32U dst_pages = 0;
252 Cpa32U adler32 = 0;
253 char *data;
254 struct page *page;
255 struct page **in_pages = NULL;
256 struct page **out_pages = NULL;
257 struct page **add_pages = NULL;
258 Cpa32U page_off = 0;
259 struct completion complete;
260 Cpa32U page_num = 0;
261 Cpa16U i;
262
263 /*
264 * We increment num_src_buf and num_dst_buf by 2 to allow
265 * us to handle non page-aligned buffer addresses and buffers
266 * whose sizes are not divisible by PAGE_SIZE.
267 */
268 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
269 (num_src_buf * sizeof (CpaFlatBuffer));
270 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
271 ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
272
273 status = QAT_PHYS_CONTIG_ALLOC(&in_pages,
274 num_src_buf * sizeof (struct page *));
275 if (status != CPA_STATUS_SUCCESS)
276 goto fail;
277
278 status = QAT_PHYS_CONTIG_ALLOC(&out_pages,
279 num_dst_buf * sizeof (struct page *));
280 if (status != CPA_STATUS_SUCCESS)
281 goto fail;
282
283 status = QAT_PHYS_CONTIG_ALLOC(&add_pages,
284 num_add_buf * sizeof (struct page *));
285 if (status != CPA_STATUS_SUCCESS)
286 goto fail;
287
288 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
289 dc_inst_handle = dc_inst_handles[i];
290 session_handle = session_handles[i];
291
292 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
293 &buffer_meta_size);
294 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size);
295 if (status != CPA_STATUS_SUCCESS)
296 goto fail;
297
298 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
299 &buffer_meta_size);
300 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size);
301 if (status != CPA_STATUS_SUCCESS)
302 goto fail;
303
304 /* build source buffer list */
305 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size);
306 if (status != CPA_STATUS_SUCCESS)
307 goto fail;
308
309 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
310
311 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
312
313 /* build destination buffer list */
314 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size);
315 if (status != CPA_STATUS_SUCCESS)
316 goto fail;
317
318 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
319
320 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
321
322 buf_list_src->numBuffers = 0;
323 buf_list_src->pPrivateMetaData = buffer_meta_src;
324 bytes_left = src_len;
325 data = src;
326 page_num = 0;
327 while (bytes_left > 0) {
328 page_off = ((long)data & ~PAGE_MASK);
329 page = qat_mem_to_page(data);
330 in_pages[page_num] = page;
331 flat_buf_src->pData = kmap(page) + page_off;
332 flat_buf_src->dataLenInBytes =
333 min((long)PAGE_SIZE - page_off, (long)bytes_left);
334
335 bytes_left -= flat_buf_src->dataLenInBytes;
336 data += flat_buf_src->dataLenInBytes;
337 flat_buf_src++;
338 buf_list_src->numBuffers++;
339 page_num++;
340 }
341
342 buf_list_dst->numBuffers = 0;
343 buf_list_dst->pPrivateMetaData = buffer_meta_dst;
344 bytes_left = dst_len;
345 data = dst;
346 page_num = 0;
347 while (bytes_left > 0) {
348 page_off = ((long)data & ~PAGE_MASK);
349 page = qat_mem_to_page(data);
350 flat_buf_dst->pData = kmap(page) + page_off;
351 out_pages[page_num] = page;
352 flat_buf_dst->dataLenInBytes =
353 min((long)PAGE_SIZE - page_off, (long)bytes_left);
354
355 bytes_left -= flat_buf_dst->dataLenInBytes;
356 data += flat_buf_dst->dataLenInBytes;
357 flat_buf_dst++;
358 buf_list_dst->numBuffers++;
359 page_num++;
360 dst_pages++;
361 }
362
363 /* map additional scratch pages into the destination buffer list */
364 bytes_left = add_len;
365 data = add;
366 page_num = 0;
367 while (bytes_left > 0) {
368 page_off = ((long)data & ~PAGE_MASK);
369 page = qat_mem_to_page(data);
370 flat_buf_dst->pData = kmap(page) + page_off;
371 add_pages[page_num] = page;
372 flat_buf_dst->dataLenInBytes =
373 min((long)PAGE_SIZE - page_off, (long)bytes_left);
374
375 bytes_left -= flat_buf_dst->dataLenInBytes;
376 data += flat_buf_dst->dataLenInBytes;
377 flat_buf_dst++;
378 buf_list_dst->numBuffers++;
379 page_num++;
380 }
381
382 init_completion(&complete);
383
384 if (dir == QAT_COMPRESS) {
385 QAT_STAT_BUMP(comp_requests);
386 QAT_STAT_INCR(comp_total_in_bytes, src_len);
387
388 cpaDcGenerateHeader(session_handle,
389 buf_list_dst->pBuffers, &hdr_sz);
390 buf_list_dst->pBuffers->pData += hdr_sz;
391 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
392 status = cpaDcCompressData(
393 dc_inst_handle, session_handle,
394 buf_list_src, buf_list_dst,
395 &dc_results, CPA_DC_FLUSH_FINAL,
396 &complete);
397 if (status != CPA_STATUS_SUCCESS) {
398 goto fail;
399 }
400
401 /* we now wait until the completion of the operation. */
402 wait_for_completion(&complete);
403
404 if (dc_results.status != CPA_STATUS_SUCCESS) {
405 status = CPA_STATUS_FAIL;
406 goto fail;
407 }
408
409 compressed_sz = dc_results.produced;
410 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
411 status = CPA_STATUS_INCOMPRESSIBLE;
412 goto fail;
413 }
414
415 /* get adler32 checksum and append footer */
416 *(Cpa32U*)(dst + hdr_sz + compressed_sz) =
417 BSWAP_32(dc_results.checksum);
418
419 *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ;
420 QAT_STAT_INCR(comp_total_out_bytes, *c_len);
421 } else {
422 ASSERT3U(dir, ==, QAT_DECOMPRESS);
423 QAT_STAT_BUMP(decomp_requests);
424 QAT_STAT_INCR(decomp_total_in_bytes, src_len);
425
426 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
427 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
428 status = cpaDcDecompressData(dc_inst_handle, session_handle,
429 buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
430 &complete);
431
432 if (CPA_STATUS_SUCCESS != status) {
433 status = CPA_STATUS_FAIL;
434 goto fail;
435 }
436
437 /* we now wait until the completion of the operation. */
438 wait_for_completion(&complete);
439
440 if (dc_results.status != CPA_STATUS_SUCCESS) {
441 status = CPA_STATUS_FAIL;
442 goto fail;
443 }
444
445 /* verify adler checksum */
446 adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
447 if (adler32 != BSWAP_32(dc_results.checksum)) {
448 status = CPA_STATUS_FAIL;
449 goto fail;
450 }
451 *c_len = dc_results.produced;
452 QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
453 }
454
455 fail:
456 if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
457 QAT_STAT_BUMP(dc_fails);
458
459 if (in_pages) {
460 for (page_num = 0;
461 page_num < buf_list_src->numBuffers;
462 page_num++) {
463 kunmap(in_pages[page_num]);
464 }
465 QAT_PHYS_CONTIG_FREE(in_pages);
466 }
467
468 if (out_pages) {
469 for (page_num = 0; page_num < dst_pages; page_num++) {
470 kunmap(out_pages[page_num]);
471 }
472 QAT_PHYS_CONTIG_FREE(out_pages);
473 }
474
475 if (add_pages) {
476 for (page_num = 0;
477 page_num < buf_list_dst->numBuffers - dst_pages;
478 page_num++) {
479 kunmap(add_pages[page_num]);
480 }
481 QAT_PHYS_CONTIG_FREE(add_pages);
482 }
483
484 QAT_PHYS_CONTIG_FREE(buffer_meta_src);
485 QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
486 QAT_PHYS_CONTIG_FREE(buf_list_src);
487 QAT_PHYS_CONTIG_FREE(buf_list_dst);
488
489 return (status);
490 }
491
492 /*
493 * Entry point for QAT accelerated compression / decompression.
494 */
495 int
qat_compress(qat_compress_dir_t dir,char * src,int src_len,char * dst,int dst_len,size_t * c_len)496 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
497 char *dst, int dst_len, size_t *c_len)
498 {
499 int ret;
500 size_t add_len = 0;
501 void *add = NULL;
502
503 if (dir == QAT_COMPRESS) {
504 add_len = dst_len;
505 add = zio_data_buf_alloc(add_len);
506 }
507
508 ret = qat_compress_impl(dir, src, src_len, dst,
509 dst_len, add, add_len, c_len);
510
511 if (dir == QAT_COMPRESS)
512 zio_data_buf_free(add, add_len);
513
514 return (ret);
515 }
516
517 static int
param_set_qat_compress(const char * val,zfs_kernel_param_t * kp)518 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp)
519 {
520 int ret;
521 int *pvalue = kp->arg;
522 ret = param_set_int(val, kp);
523 if (ret)
524 return (ret);
525 /*
526 * zfs_qat_compress_disable = 0: enable qat compress
527 * try to initialize qat instance if it has not been done
528 */
529 if (*pvalue == 0 && !qat_dc_init_done) {
530 ret = qat_dc_init();
531 if (ret != 0) {
532 zfs_qat_compress_disable = 1;
533 return (ret);
534 }
535 }
536 return (ret);
537 }
538
539 module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
540 param_get_int, &zfs_qat_compress_disable, 0644);
541 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
542
543 #endif
544