1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 13 #ifdef HAVE_QAT 14 15 #include <linux/slab.h> 16 #include <linux/vmalloc.h> 17 #include <linux/pagemap.h> 18 #include <linux/completion.h> 19 #include <sys/zfs_context.h> 20 #include <sys/byteorder.h> 21 #include <sys/zio.h> 22 #include <sys/qat.h> 23 24 /* 25 * Max instances in a QAT device, each instance is a channel to submit 26 * jobs to QAT hardware, this is only for pre-allocating instance and 27 * session arrays; the actual number of instances are defined in the 28 * QAT driver's configuration file. 29 */ 30 #define QAT_DC_MAX_INSTANCES 48 31 32 /* 33 * ZLIB head and foot size 34 */ 35 #define ZLIB_HEAD_SZ 2 36 #define ZLIB_FOOT_SZ 4 37 38 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; 39 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; 40 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; 41 static Cpa16U num_inst = 0; 42 static Cpa32U inst_num = 0; 43 static boolean_t qat_dc_init_done = B_FALSE; 44 int zfs_qat_compress_disable = 0; 45 46 boolean_t 47 qat_dc_use_accel(size_t s_len) 48 { 49 return (!zfs_qat_compress_disable && 50 qat_dc_init_done && 51 s_len >= QAT_MIN_BUF_SIZE && 52 s_len <= QAT_MAX_BUF_SIZE); 53 } 54 55 static void 56 qat_dc_callback(void *p_callback, CpaStatus status) 57 { 58 if (p_callback != NULL) 59 complete((struct completion *)p_callback); 60 } 61 62 static void 63 qat_dc_clean(void) 64 { 65 Cpa16U buff_num = 0; 66 Cpa16U num_inter_buff_lists = 0; 67 68 for (Cpa16U i = 0; i < num_inst; i++) { 69 cpaDcStopInstance(dc_inst_handles[i]); 70 QAT_PHYS_CONTIG_FREE(session_handles[i]); 71 /* free intermediate buffers */ 72 if (buffer_array[i] != NULL) { 73 cpaDcGetNumIntermediateBuffers( 74 dc_inst_handles[i], &num_inter_buff_lists); 75 for (buff_num = 0; buff_num < num_inter_buff_lists; 76 buff_num++) { 77 CpaBufferList *buffer_inter = 78 buffer_array[i][buff_num]; 79 if (buffer_inter->pBuffers) { 80 QAT_PHYS_CONTIG_FREE( 81 buffer_inter->pBuffers->pData); 82 QAT_PHYS_CONTIG_FREE( 83 buffer_inter->pBuffers); 84 } 85 QAT_PHYS_CONTIG_FREE( 86 buffer_inter->pPrivateMetaData); 87 QAT_PHYS_CONTIG_FREE(buffer_inter); 88 } 89 } 90 } 91 92 num_inst = 0; 93 qat_dc_init_done = B_FALSE; 94 } 95 96 int 97 qat_dc_init(void) 98 { 99 CpaStatus status = CPA_STATUS_SUCCESS; 100 Cpa32U sess_size = 0; 101 Cpa32U ctx_size = 0; 102 Cpa16U num_inter_buff_lists = 0; 103 Cpa16U buff_num = 0; 104 Cpa32U buff_meta_size = 0; 105 CpaDcSessionSetupData sd = {0}; 106 107 if (qat_dc_init_done) 108 return (0); 109 110 status = cpaDcGetNumInstances(&num_inst); 111 if (status != CPA_STATUS_SUCCESS) 112 return (-1); 113 114 /* if the user has configured no QAT compression units just return */ 115 if (num_inst == 0) 116 return (0); 117 118 if (num_inst > QAT_DC_MAX_INSTANCES) 119 num_inst = QAT_DC_MAX_INSTANCES; 120 121 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); 122 if (status != CPA_STATUS_SUCCESS) 123 return (-1); 124 125 for (Cpa16U i = 0; i < num_inst; i++) { 126 cpaDcSetAddressTranslation(dc_inst_handles[i], 127 (void*)virt_to_phys); 128 129 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], 130 1, &buff_meta_size); 131 132 if (status == CPA_STATUS_SUCCESS) 133 status = cpaDcGetNumIntermediateBuffers( 134 dc_inst_handles[i], &num_inter_buff_lists); 135 136 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) 137 status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], 138 num_inter_buff_lists * 139 sizeof (CpaBufferList *)); 140 141 for (buff_num = 0; buff_num < num_inter_buff_lists; 142 buff_num++) { 143 if (status == CPA_STATUS_SUCCESS) 144 status = QAT_PHYS_CONTIG_ALLOC( 145 &buffer_array[i][buff_num], 146 sizeof (CpaBufferList)); 147 148 if (status == CPA_STATUS_SUCCESS) 149 status = QAT_PHYS_CONTIG_ALLOC( 150 &buffer_array[i][buff_num]-> 151 pPrivateMetaData, 152 buff_meta_size); 153 154 if (status == CPA_STATUS_SUCCESS) 155 status = QAT_PHYS_CONTIG_ALLOC( 156 &buffer_array[i][buff_num]->pBuffers, 157 sizeof (CpaFlatBuffer)); 158 159 if (status == CPA_STATUS_SUCCESS) { 160 /* 161 * implementation requires an intermediate 162 * buffer approximately twice the size of 163 * output buffer, which is 2x max buffer 164 * size here. 165 */ 166 status = QAT_PHYS_CONTIG_ALLOC( 167 &buffer_array[i][buff_num]->pBuffers-> 168 pData, 2 * QAT_MAX_BUF_SIZE); 169 if (status != CPA_STATUS_SUCCESS) 170 goto fail; 171 172 buffer_array[i][buff_num]->numBuffers = 1; 173 buffer_array[i][buff_num]->pBuffers-> 174 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; 175 } 176 } 177 178 status = cpaDcStartInstance(dc_inst_handles[i], 179 num_inter_buff_lists, buffer_array[i]); 180 if (status != CPA_STATUS_SUCCESS) 181 goto fail; 182 183 sd.compLevel = CPA_DC_L1; 184 sd.compType = CPA_DC_DEFLATE; 185 sd.huffType = CPA_DC_HT_FULL_DYNAMIC; 186 sd.sessDirection = CPA_DC_DIR_COMBINED; 187 sd.sessState = CPA_DC_STATELESS; 188 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6) 189 sd.deflateWindowSize = 7; 190 #endif 191 sd.checksum = CPA_DC_ADLER32; 192 status = cpaDcGetSessionSize(dc_inst_handles[i], 193 &sd, &sess_size, &ctx_size); 194 if (status != CPA_STATUS_SUCCESS) 195 goto fail; 196 197 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); 198 if (session_handles[i] == NULL) 199 goto fail; 200 201 status = cpaDcInitSession(dc_inst_handles[i], 202 session_handles[i], 203 &sd, NULL, qat_dc_callback); 204 if (status != CPA_STATUS_SUCCESS) 205 goto fail; 206 } 207 208 qat_dc_init_done = B_TRUE; 209 return (0); 210 fail: 211 qat_dc_clean(); 212 return (-1); 213 } 214 215 void 216 qat_dc_fini(void) 217 { 218 if (!qat_dc_init_done) 219 return; 220 221 qat_dc_clean(); 222 } 223 224 /* 225 * The "add" parameter is an additional buffer which is passed 226 * to QAT as a scratch buffer alongside the destination buffer 227 * in case the "compressed" data ends up being larger than the 228 * original source data. This is necessary to prevent QAT from 229 * generating buffer overflow warnings for incompressible data. 230 */ 231 static int 232 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, 233 char *dst, int dst_len, char *add, int add_len, size_t *c_len) 234 { 235 CpaInstanceHandle dc_inst_handle; 236 CpaDcSessionHandle session_handle; 237 CpaBufferList *buf_list_src = NULL; 238 CpaBufferList *buf_list_dst = NULL; 239 CpaFlatBuffer *flat_buf_src = NULL; 240 CpaFlatBuffer *flat_buf_dst = NULL; 241 Cpa8U *buffer_meta_src = NULL; 242 Cpa8U *buffer_meta_dst = NULL; 243 Cpa32U buffer_meta_size = 0; 244 CpaDcRqResults dc_results = {.checksum = 1}; 245 CpaStatus status = CPA_STATUS_FAIL; 246 Cpa32U hdr_sz = 0; 247 Cpa32U compressed_sz; 248 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; 249 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; 250 Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; 251 Cpa32U bytes_left; 252 Cpa32U dst_pages = 0; 253 Cpa32U adler32 = 0; 254 char *data; 255 struct page *page; 256 struct page **in_pages = NULL; 257 struct page **out_pages = NULL; 258 struct page **add_pages = NULL; 259 Cpa32U page_off = 0; 260 struct completion complete; 261 Cpa32U page_num = 0; 262 Cpa16U i; 263 264 /* 265 * We increment num_src_buf and num_dst_buf by 2 to allow 266 * us to handle non page-aligned buffer addresses and buffers 267 * whose sizes are not divisible by PAGE_SIZE. 268 */ 269 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + 270 (num_src_buf * sizeof (CpaFlatBuffer)); 271 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + 272 ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); 273 274 status = QAT_PHYS_CONTIG_ALLOC(&in_pages, 275 num_src_buf * sizeof (struct page *)); 276 if (status != CPA_STATUS_SUCCESS) 277 goto fail; 278 279 status = QAT_PHYS_CONTIG_ALLOC(&out_pages, 280 num_dst_buf * sizeof (struct page *)); 281 if (status != CPA_STATUS_SUCCESS) 282 goto fail; 283 284 status = QAT_PHYS_CONTIG_ALLOC(&add_pages, 285 num_add_buf * sizeof (struct page *)); 286 if (status != CPA_STATUS_SUCCESS) 287 goto fail; 288 289 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 290 dc_inst_handle = dc_inst_handles[i]; 291 session_handle = session_handles[i]; 292 293 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, 294 &buffer_meta_size); 295 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size); 296 if (status != CPA_STATUS_SUCCESS) 297 goto fail; 298 299 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, 300 &buffer_meta_size); 301 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size); 302 if (status != CPA_STATUS_SUCCESS) 303 goto fail; 304 305 /* build source buffer list */ 306 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size); 307 if (status != CPA_STATUS_SUCCESS) 308 goto fail; 309 310 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); 311 312 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ 313 314 /* build destination buffer list */ 315 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size); 316 if (status != CPA_STATUS_SUCCESS) 317 goto fail; 318 319 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); 320 321 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ 322 323 buf_list_src->numBuffers = 0; 324 buf_list_src->pPrivateMetaData = buffer_meta_src; 325 bytes_left = src_len; 326 data = src; 327 page_num = 0; 328 while (bytes_left > 0) { 329 page_off = ((long)data & ~PAGE_MASK); 330 page = qat_mem_to_page(data); 331 in_pages[page_num] = page; 332 flat_buf_src->pData = kmap(page) + page_off; 333 flat_buf_src->dataLenInBytes = 334 min((long)PAGE_SIZE - page_off, (long)bytes_left); 335 336 bytes_left -= flat_buf_src->dataLenInBytes; 337 data += flat_buf_src->dataLenInBytes; 338 flat_buf_src++; 339 buf_list_src->numBuffers++; 340 page_num++; 341 } 342 343 buf_list_dst->numBuffers = 0; 344 buf_list_dst->pPrivateMetaData = buffer_meta_dst; 345 bytes_left = dst_len; 346 data = dst; 347 page_num = 0; 348 while (bytes_left > 0) { 349 page_off = ((long)data & ~PAGE_MASK); 350 page = qat_mem_to_page(data); 351 flat_buf_dst->pData = kmap(page) + page_off; 352 out_pages[page_num] = page; 353 flat_buf_dst->dataLenInBytes = 354 min((long)PAGE_SIZE - page_off, (long)bytes_left); 355 356 bytes_left -= flat_buf_dst->dataLenInBytes; 357 data += flat_buf_dst->dataLenInBytes; 358 flat_buf_dst++; 359 buf_list_dst->numBuffers++; 360 page_num++; 361 dst_pages++; 362 } 363 364 /* map additional scratch pages into the destination buffer list */ 365 bytes_left = add_len; 366 data = add; 367 page_num = 0; 368 while (bytes_left > 0) { 369 page_off = ((long)data & ~PAGE_MASK); 370 page = qat_mem_to_page(data); 371 flat_buf_dst->pData = kmap(page) + page_off; 372 add_pages[page_num] = page; 373 flat_buf_dst->dataLenInBytes = 374 min((long)PAGE_SIZE - page_off, (long)bytes_left); 375 376 bytes_left -= flat_buf_dst->dataLenInBytes; 377 data += flat_buf_dst->dataLenInBytes; 378 flat_buf_dst++; 379 buf_list_dst->numBuffers++; 380 page_num++; 381 } 382 383 init_completion(&complete); 384 385 if (dir == QAT_COMPRESS) { 386 QAT_STAT_BUMP(comp_requests); 387 QAT_STAT_INCR(comp_total_in_bytes, src_len); 388 389 cpaDcGenerateHeader(session_handle, 390 buf_list_dst->pBuffers, &hdr_sz); 391 buf_list_dst->pBuffers->pData += hdr_sz; 392 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; 393 status = cpaDcCompressData( 394 dc_inst_handle, session_handle, 395 buf_list_src, buf_list_dst, 396 &dc_results, CPA_DC_FLUSH_FINAL, 397 &complete); 398 if (status != CPA_STATUS_SUCCESS) { 399 goto fail; 400 } 401 402 /* we now wait until the completion of the operation. */ 403 wait_for_completion(&complete); 404 405 if (dc_results.status != CPA_STATUS_SUCCESS) { 406 status = CPA_STATUS_FAIL; 407 goto fail; 408 } 409 410 compressed_sz = dc_results.produced; 411 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { 412 status = CPA_STATUS_INCOMPRESSIBLE; 413 goto fail; 414 } 415 416 /* get adler32 checksum and append footer */ 417 *(Cpa32U*)(dst + hdr_sz + compressed_sz) = 418 BSWAP_32(dc_results.checksum); 419 420 *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; 421 QAT_STAT_INCR(comp_total_out_bytes, *c_len); 422 } else { 423 ASSERT3U(dir, ==, QAT_DECOMPRESS); 424 QAT_STAT_BUMP(decomp_requests); 425 QAT_STAT_INCR(decomp_total_in_bytes, src_len); 426 427 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; 428 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; 429 status = cpaDcDecompressData(dc_inst_handle, session_handle, 430 buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, 431 &complete); 432 433 if (CPA_STATUS_SUCCESS != status) { 434 status = CPA_STATUS_FAIL; 435 goto fail; 436 } 437 438 /* we now wait until the completion of the operation. */ 439 wait_for_completion(&complete); 440 441 if (dc_results.status != CPA_STATUS_SUCCESS) { 442 status = CPA_STATUS_FAIL; 443 goto fail; 444 } 445 446 /* verify adler checksum */ 447 adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ); 448 if (adler32 != BSWAP_32(dc_results.checksum)) { 449 status = CPA_STATUS_FAIL; 450 goto fail; 451 } 452 *c_len = dc_results.produced; 453 QAT_STAT_INCR(decomp_total_out_bytes, *c_len); 454 } 455 456 fail: 457 if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) 458 QAT_STAT_BUMP(dc_fails); 459 460 if (in_pages) { 461 for (page_num = 0; 462 page_num < buf_list_src->numBuffers; 463 page_num++) { 464 kunmap(in_pages[page_num]); 465 } 466 QAT_PHYS_CONTIG_FREE(in_pages); 467 } 468 469 if (out_pages) { 470 for (page_num = 0; page_num < dst_pages; page_num++) { 471 kunmap(out_pages[page_num]); 472 } 473 QAT_PHYS_CONTIG_FREE(out_pages); 474 } 475 476 if (add_pages) { 477 for (page_num = 0; 478 page_num < buf_list_dst->numBuffers - dst_pages; 479 page_num++) { 480 kunmap(add_pages[page_num]); 481 } 482 QAT_PHYS_CONTIG_FREE(add_pages); 483 } 484 485 QAT_PHYS_CONTIG_FREE(buffer_meta_src); 486 QAT_PHYS_CONTIG_FREE(buffer_meta_dst); 487 QAT_PHYS_CONTIG_FREE(buf_list_src); 488 QAT_PHYS_CONTIG_FREE(buf_list_dst); 489 490 return (status); 491 } 492 493 /* 494 * Entry point for QAT accelerated compression / decompression. 495 */ 496 int 497 qat_compress(qat_compress_dir_t dir, char *src, int src_len, 498 char *dst, int dst_len, size_t *c_len) 499 { 500 int ret; 501 size_t add_len = 0; 502 void *add = NULL; 503 504 if (dir == QAT_COMPRESS) { 505 add_len = dst_len; 506 add = zio_data_buf_alloc(add_len); 507 } 508 509 ret = qat_compress_impl(dir, src, src_len, dst, 510 dst_len, add, add_len, c_len); 511 512 if (dir == QAT_COMPRESS) 513 zio_data_buf_free(add, add_len); 514 515 return (ret); 516 } 517 518 static int 519 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) 520 { 521 int ret; 522 int *pvalue = kp->arg; 523 ret = param_set_int(val, kp); 524 if (ret) 525 return (ret); 526 /* 527 * zfs_qat_compress_disable = 0: enable qat compress 528 * try to initialize qat instance if it has not been done 529 */ 530 if (*pvalue == 0 && !qat_dc_init_done) { 531 ret = qat_dc_init(); 532 if (ret != 0) { 533 zfs_qat_compress_disable = 1; 534 return (ret); 535 } 536 } 537 return (ret); 538 } 539 540 module_param_call(zfs_qat_compress_disable, param_set_qat_compress, 541 param_get_int, &zfs_qat_compress_disable, 0644); 542 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); 543 544 #endif 545