1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 #if defined(_KERNEL) && defined(HAVE_QAT) 24 #include <linux/slab.h> 25 #include <linux/vmalloc.h> 26 #include <linux/pagemap.h> 27 #include <linux/completion.h> 28 #include <sys/zfs_context.h> 29 #include <sys/byteorder.h> 30 #include <sys/zio.h> 31 #include <sys/qat.h> 32 33 /* 34 * Max instances in a QAT device, each instance is a channel to submit 35 * jobs to QAT hardware, this is only for pre-allocating instance and 36 * session arrays; the actual number of instances are defined in the 37 * QAT driver's configuration file. 38 */ 39 #define QAT_DC_MAX_INSTANCES 48 40 41 /* 42 * ZLIB head and foot size 43 */ 44 #define ZLIB_HEAD_SZ 2 45 #define ZLIB_FOOT_SZ 4 46 47 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; 48 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; 49 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; 50 static Cpa16U num_inst = 0; 51 static Cpa32U inst_num = 0; 52 static boolean_t qat_dc_init_done = B_FALSE; 53 int zfs_qat_compress_disable = 0; 54 55 boolean_t 56 qat_dc_use_accel(size_t s_len) 57 { 58 return (!zfs_qat_compress_disable && 59 qat_dc_init_done && 60 s_len >= QAT_MIN_BUF_SIZE && 61 s_len <= QAT_MAX_BUF_SIZE); 62 } 63 64 static void 65 qat_dc_callback(void *p_callback, CpaStatus status) 66 { 67 if (p_callback != NULL) 68 complete((struct completion *)p_callback); 69 } 70 71 static void 72 qat_dc_clean(void) 73 { 74 Cpa16U buff_num = 0; 75 Cpa16U num_inter_buff_lists = 0; 76 77 for (Cpa16U i = 0; i < num_inst; i++) { 78 cpaDcStopInstance(dc_inst_handles[i]); 79 QAT_PHYS_CONTIG_FREE(session_handles[i]); 80 /* free intermediate buffers */ 81 if (buffer_array[i] != NULL) { 82 cpaDcGetNumIntermediateBuffers( 83 dc_inst_handles[i], &num_inter_buff_lists); 84 for (buff_num = 0; buff_num < num_inter_buff_lists; 85 buff_num++) { 86 CpaBufferList *buffer_inter = 87 buffer_array[i][buff_num]; 88 if (buffer_inter->pBuffers) { 89 QAT_PHYS_CONTIG_FREE( 90 buffer_inter->pBuffers->pData); 91 QAT_PHYS_CONTIG_FREE( 92 buffer_inter->pBuffers); 93 } 94 QAT_PHYS_CONTIG_FREE( 95 buffer_inter->pPrivateMetaData); 96 QAT_PHYS_CONTIG_FREE(buffer_inter); 97 } 98 } 99 } 100 101 num_inst = 0; 102 qat_dc_init_done = B_FALSE; 103 } 104 105 int 106 qat_dc_init(void) 107 { 108 CpaStatus status = CPA_STATUS_SUCCESS; 109 Cpa32U sess_size = 0; 110 Cpa32U ctx_size = 0; 111 Cpa16U num_inter_buff_lists = 0; 112 Cpa16U buff_num = 0; 113 Cpa32U buff_meta_size = 0; 114 CpaDcSessionSetupData sd = {0}; 115 116 if (qat_dc_init_done) 117 return (0); 118 119 status = cpaDcGetNumInstances(&num_inst); 120 if (status != CPA_STATUS_SUCCESS) 121 return (-1); 122 123 /* if the user has configured no QAT compression units just return */ 124 if (num_inst == 0) 125 return (0); 126 127 if (num_inst > QAT_DC_MAX_INSTANCES) 128 num_inst = QAT_DC_MAX_INSTANCES; 129 130 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); 131 if (status != CPA_STATUS_SUCCESS) 132 return (-1); 133 134 for (Cpa16U i = 0; i < num_inst; i++) { 135 cpaDcSetAddressTranslation(dc_inst_handles[i], 136 (void*)virt_to_phys); 137 138 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], 139 1, &buff_meta_size); 140 141 if (status == CPA_STATUS_SUCCESS) 142 status = cpaDcGetNumIntermediateBuffers( 143 dc_inst_handles[i], &num_inter_buff_lists); 144 145 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) 146 status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], 147 num_inter_buff_lists * 148 sizeof (CpaBufferList *)); 149 150 for (buff_num = 0; buff_num < num_inter_buff_lists; 151 buff_num++) { 152 if (status == CPA_STATUS_SUCCESS) 153 status = QAT_PHYS_CONTIG_ALLOC( 154 &buffer_array[i][buff_num], 155 sizeof (CpaBufferList)); 156 157 if (status == CPA_STATUS_SUCCESS) 158 status = QAT_PHYS_CONTIG_ALLOC( 159 &buffer_array[i][buff_num]-> 160 pPrivateMetaData, 161 buff_meta_size); 162 163 if (status == CPA_STATUS_SUCCESS) 164 status = QAT_PHYS_CONTIG_ALLOC( 165 &buffer_array[i][buff_num]->pBuffers, 166 sizeof (CpaFlatBuffer)); 167 168 if (status == CPA_STATUS_SUCCESS) { 169 /* 170 * implementation requires an intermediate 171 * buffer approximately twice the size of 172 * output buffer, which is 2x max buffer 173 * size here. 174 */ 175 status = QAT_PHYS_CONTIG_ALLOC( 176 &buffer_array[i][buff_num]->pBuffers-> 177 pData, 2 * QAT_MAX_BUF_SIZE); 178 if (status != CPA_STATUS_SUCCESS) 179 goto fail; 180 181 buffer_array[i][buff_num]->numBuffers = 1; 182 buffer_array[i][buff_num]->pBuffers-> 183 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; 184 } 185 } 186 187 status = cpaDcStartInstance(dc_inst_handles[i], 188 num_inter_buff_lists, buffer_array[i]); 189 if (status != CPA_STATUS_SUCCESS) 190 goto fail; 191 192 sd.compLevel = CPA_DC_L1; 193 sd.compType = CPA_DC_DEFLATE; 194 sd.huffType = CPA_DC_HT_FULL_DYNAMIC; 195 sd.sessDirection = CPA_DC_DIR_COMBINED; 196 sd.sessState = CPA_DC_STATELESS; 197 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6) 198 sd.deflateWindowSize = 7; 199 #endif 200 sd.checksum = CPA_DC_ADLER32; 201 status = cpaDcGetSessionSize(dc_inst_handles[i], 202 &sd, &sess_size, &ctx_size); 203 if (status != CPA_STATUS_SUCCESS) 204 goto fail; 205 206 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); 207 if (session_handles[i] == NULL) 208 goto fail; 209 210 status = cpaDcInitSession(dc_inst_handles[i], 211 session_handles[i], 212 &sd, NULL, qat_dc_callback); 213 if (status != CPA_STATUS_SUCCESS) 214 goto fail; 215 } 216 217 qat_dc_init_done = B_TRUE; 218 return (0); 219 fail: 220 qat_dc_clean(); 221 return (-1); 222 } 223 224 void 225 qat_dc_fini(void) 226 { 227 if (!qat_dc_init_done) 228 return; 229 230 qat_dc_clean(); 231 } 232 233 /* 234 * The "add" parameter is an additional buffer which is passed 235 * to QAT as a scratch buffer alongside the destination buffer 236 * in case the "compressed" data ends up being larger than the 237 * original source data. This is necessary to prevent QAT from 238 * generating buffer overflow warnings for incompressible data. 239 */ 240 static int 241 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, 242 char *dst, int dst_len, char *add, int add_len, size_t *c_len) 243 { 244 CpaInstanceHandle dc_inst_handle; 245 CpaDcSessionHandle session_handle; 246 CpaBufferList *buf_list_src = NULL; 247 CpaBufferList *buf_list_dst = NULL; 248 CpaFlatBuffer *flat_buf_src = NULL; 249 CpaFlatBuffer *flat_buf_dst = NULL; 250 Cpa8U *buffer_meta_src = NULL; 251 Cpa8U *buffer_meta_dst = NULL; 252 Cpa32U buffer_meta_size = 0; 253 CpaDcRqResults dc_results = {.checksum = 1}; 254 CpaStatus status = CPA_STATUS_FAIL; 255 Cpa32U hdr_sz = 0; 256 Cpa32U compressed_sz; 257 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; 258 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; 259 Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; 260 Cpa32U bytes_left; 261 Cpa32U dst_pages = 0; 262 Cpa32U adler32 = 0; 263 char *data; 264 struct page *page; 265 struct page **in_pages = NULL; 266 struct page **out_pages = NULL; 267 struct page **add_pages = NULL; 268 Cpa32U page_off = 0; 269 struct completion complete; 270 Cpa32U page_num = 0; 271 Cpa16U i; 272 273 /* 274 * We increment num_src_buf and num_dst_buf by 2 to allow 275 * us to handle non page-aligned buffer addresses and buffers 276 * whose sizes are not divisible by PAGE_SIZE. 277 */ 278 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + 279 (num_src_buf * sizeof (CpaFlatBuffer)); 280 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + 281 ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); 282 283 status = QAT_PHYS_CONTIG_ALLOC(&in_pages, 284 num_src_buf * sizeof (struct page *)); 285 if (status != CPA_STATUS_SUCCESS) 286 goto fail; 287 288 status = QAT_PHYS_CONTIG_ALLOC(&out_pages, 289 num_dst_buf * sizeof (struct page *)); 290 if (status != CPA_STATUS_SUCCESS) 291 goto fail; 292 293 status = QAT_PHYS_CONTIG_ALLOC(&add_pages, 294 num_add_buf * sizeof (struct page *)); 295 if (status != CPA_STATUS_SUCCESS) 296 goto fail; 297 298 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 299 dc_inst_handle = dc_inst_handles[i]; 300 session_handle = session_handles[i]; 301 302 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, 303 &buffer_meta_size); 304 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size); 305 if (status != CPA_STATUS_SUCCESS) 306 goto fail; 307 308 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, 309 &buffer_meta_size); 310 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size); 311 if (status != CPA_STATUS_SUCCESS) 312 goto fail; 313 314 /* build source buffer list */ 315 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size); 316 if (status != CPA_STATUS_SUCCESS) 317 goto fail; 318 319 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); 320 321 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ 322 323 /* build destination buffer list */ 324 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size); 325 if (status != CPA_STATUS_SUCCESS) 326 goto fail; 327 328 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); 329 330 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ 331 332 buf_list_src->numBuffers = 0; 333 buf_list_src->pPrivateMetaData = buffer_meta_src; 334 bytes_left = src_len; 335 data = src; 336 page_num = 0; 337 while (bytes_left > 0) { 338 page_off = ((long)data & ~PAGE_MASK); 339 page = qat_mem_to_page(data); 340 in_pages[page_num] = page; 341 flat_buf_src->pData = kmap(page) + page_off; 342 flat_buf_src->dataLenInBytes = 343 min((long)PAGE_SIZE - page_off, (long)bytes_left); 344 345 bytes_left -= flat_buf_src->dataLenInBytes; 346 data += flat_buf_src->dataLenInBytes; 347 flat_buf_src++; 348 buf_list_src->numBuffers++; 349 page_num++; 350 } 351 352 buf_list_dst->numBuffers = 0; 353 buf_list_dst->pPrivateMetaData = buffer_meta_dst; 354 bytes_left = dst_len; 355 data = dst; 356 page_num = 0; 357 while (bytes_left > 0) { 358 page_off = ((long)data & ~PAGE_MASK); 359 page = qat_mem_to_page(data); 360 flat_buf_dst->pData = kmap(page) + page_off; 361 out_pages[page_num] = page; 362 flat_buf_dst->dataLenInBytes = 363 min((long)PAGE_SIZE - page_off, (long)bytes_left); 364 365 bytes_left -= flat_buf_dst->dataLenInBytes; 366 data += flat_buf_dst->dataLenInBytes; 367 flat_buf_dst++; 368 buf_list_dst->numBuffers++; 369 page_num++; 370 dst_pages++; 371 } 372 373 /* map additional scratch pages into the destination buffer list */ 374 bytes_left = add_len; 375 data = add; 376 page_num = 0; 377 while (bytes_left > 0) { 378 page_off = ((long)data & ~PAGE_MASK); 379 page = qat_mem_to_page(data); 380 flat_buf_dst->pData = kmap(page) + page_off; 381 add_pages[page_num] = page; 382 flat_buf_dst->dataLenInBytes = 383 min((long)PAGE_SIZE - page_off, (long)bytes_left); 384 385 bytes_left -= flat_buf_dst->dataLenInBytes; 386 data += flat_buf_dst->dataLenInBytes; 387 flat_buf_dst++; 388 buf_list_dst->numBuffers++; 389 page_num++; 390 } 391 392 init_completion(&complete); 393 394 if (dir == QAT_COMPRESS) { 395 QAT_STAT_BUMP(comp_requests); 396 QAT_STAT_INCR(comp_total_in_bytes, src_len); 397 398 cpaDcGenerateHeader(session_handle, 399 buf_list_dst->pBuffers, &hdr_sz); 400 buf_list_dst->pBuffers->pData += hdr_sz; 401 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; 402 status = cpaDcCompressData( 403 dc_inst_handle, session_handle, 404 buf_list_src, buf_list_dst, 405 &dc_results, CPA_DC_FLUSH_FINAL, 406 &complete); 407 if (status != CPA_STATUS_SUCCESS) { 408 goto fail; 409 } 410 411 /* we now wait until the completion of the operation. */ 412 wait_for_completion(&complete); 413 414 if (dc_results.status != CPA_STATUS_SUCCESS) { 415 status = CPA_STATUS_FAIL; 416 goto fail; 417 } 418 419 compressed_sz = dc_results.produced; 420 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { 421 status = CPA_STATUS_INCOMPRESSIBLE; 422 goto fail; 423 } 424 425 /* get adler32 checksum and append footer */ 426 *(Cpa32U*)(dst + hdr_sz + compressed_sz) = 427 BSWAP_32(dc_results.checksum); 428 429 *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; 430 QAT_STAT_INCR(comp_total_out_bytes, *c_len); 431 } else { 432 ASSERT3U(dir, ==, QAT_DECOMPRESS); 433 QAT_STAT_BUMP(decomp_requests); 434 QAT_STAT_INCR(decomp_total_in_bytes, src_len); 435 436 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; 437 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; 438 status = cpaDcDecompressData(dc_inst_handle, session_handle, 439 buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, 440 &complete); 441 442 if (CPA_STATUS_SUCCESS != status) { 443 status = CPA_STATUS_FAIL; 444 goto fail; 445 } 446 447 /* we now wait until the completion of the operation. */ 448 wait_for_completion(&complete); 449 450 if (dc_results.status != CPA_STATUS_SUCCESS) { 451 status = CPA_STATUS_FAIL; 452 goto fail; 453 } 454 455 /* verify adler checksum */ 456 adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ); 457 if (adler32 != BSWAP_32(dc_results.checksum)) { 458 status = CPA_STATUS_FAIL; 459 goto fail; 460 } 461 *c_len = dc_results.produced; 462 QAT_STAT_INCR(decomp_total_out_bytes, *c_len); 463 } 464 465 fail: 466 if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) 467 QAT_STAT_BUMP(dc_fails); 468 469 if (in_pages) { 470 for (page_num = 0; 471 page_num < buf_list_src->numBuffers; 472 page_num++) { 473 kunmap(in_pages[page_num]); 474 } 475 QAT_PHYS_CONTIG_FREE(in_pages); 476 } 477 478 if (out_pages) { 479 for (page_num = 0; page_num < dst_pages; page_num++) { 480 kunmap(out_pages[page_num]); 481 } 482 QAT_PHYS_CONTIG_FREE(out_pages); 483 } 484 485 if (add_pages) { 486 for (page_num = 0; 487 page_num < buf_list_dst->numBuffers - dst_pages; 488 page_num++) { 489 kunmap(add_pages[page_num]); 490 } 491 QAT_PHYS_CONTIG_FREE(add_pages); 492 } 493 494 QAT_PHYS_CONTIG_FREE(buffer_meta_src); 495 QAT_PHYS_CONTIG_FREE(buffer_meta_dst); 496 QAT_PHYS_CONTIG_FREE(buf_list_src); 497 QAT_PHYS_CONTIG_FREE(buf_list_dst); 498 499 return (status); 500 } 501 502 /* 503 * Entry point for QAT accelerated compression / decompression. 504 */ 505 int 506 qat_compress(qat_compress_dir_t dir, char *src, int src_len, 507 char *dst, int dst_len, size_t *c_len) 508 { 509 int ret; 510 size_t add_len = 0; 511 void *add = NULL; 512 513 if (dir == QAT_COMPRESS) { 514 add_len = dst_len; 515 add = zio_data_buf_alloc(add_len); 516 } 517 518 ret = qat_compress_impl(dir, src, src_len, dst, 519 dst_len, add, add_len, c_len); 520 521 if (dir == QAT_COMPRESS) 522 zio_data_buf_free(add, add_len); 523 524 return (ret); 525 } 526 527 static int 528 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) 529 { 530 int ret; 531 int *pvalue = kp->arg; 532 ret = param_set_int(val, kp); 533 if (ret) 534 return (ret); 535 /* 536 * zfs_qat_compress_disable = 0: enable qat compress 537 * try to initialize qat instance if it has not been done 538 */ 539 if (*pvalue == 0 && !qat_dc_init_done) { 540 ret = qat_dc_init(); 541 if (ret != 0) { 542 zfs_qat_compress_disable = 1; 543 return (ret); 544 } 545 } 546 return (ret); 547 } 548 549 module_param_call(zfs_qat_compress_disable, param_set_qat_compress, 550 param_get_int, &zfs_qat_compress_disable, 0644); 551 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); 552 553 #endif 554