1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 #if defined(_KERNEL) && defined(HAVE_QAT) 23 #include <linux/slab.h> 24 #include <linux/vmalloc.h> 25 #include <linux/pagemap.h> 26 #include <linux/completion.h> 27 #include <sys/zfs_context.h> 28 #include <sys/byteorder.h> 29 #include <sys/zio.h> 30 #include <sys/qat.h> 31 32 /* 33 * Max instances in a QAT device, each instance is a channel to submit 34 * jobs to QAT hardware, this is only for pre-allocating instance and 35 * session arrays; the actual number of instances are defined in the 36 * QAT driver's configuration file. 37 */ 38 #define QAT_DC_MAX_INSTANCES 48 39 40 /* 41 * ZLIB head and foot size 42 */ 43 #define ZLIB_HEAD_SZ 2 44 #define ZLIB_FOOT_SZ 4 45 46 static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES]; 47 static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES]; 48 static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES]; 49 static Cpa16U num_inst = 0; 50 static Cpa32U inst_num = 0; 51 static boolean_t qat_dc_init_done = B_FALSE; 52 int zfs_qat_compress_disable = 0; 53 54 boolean_t 55 qat_dc_use_accel(size_t s_len) 56 { 57 return (!zfs_qat_compress_disable && 58 qat_dc_init_done && 59 s_len >= QAT_MIN_BUF_SIZE && 60 s_len <= QAT_MAX_BUF_SIZE); 61 } 62 63 static void 64 qat_dc_callback(void *p_callback, CpaStatus status) 65 { 66 if (p_callback != NULL) 67 complete((struct completion *)p_callback); 68 } 69 70 static void 71 qat_dc_clean(void) 72 { 73 Cpa16U buff_num = 0; 74 Cpa16U num_inter_buff_lists = 0; 75 76 for (Cpa16U i = 0; i < num_inst; i++) { 77 cpaDcStopInstance(dc_inst_handles[i]); 78 QAT_PHYS_CONTIG_FREE(session_handles[i]); 79 /* free intermediate buffers */ 80 if (buffer_array[i] != NULL) { 81 cpaDcGetNumIntermediateBuffers( 82 dc_inst_handles[i], &num_inter_buff_lists); 83 for (buff_num = 0; buff_num < num_inter_buff_lists; 84 buff_num++) { 85 CpaBufferList *buffer_inter = 86 buffer_array[i][buff_num]; 87 if (buffer_inter->pBuffers) { 88 QAT_PHYS_CONTIG_FREE( 89 buffer_inter->pBuffers->pData); 90 QAT_PHYS_CONTIG_FREE( 91 buffer_inter->pBuffers); 92 } 93 QAT_PHYS_CONTIG_FREE( 94 buffer_inter->pPrivateMetaData); 95 QAT_PHYS_CONTIG_FREE(buffer_inter); 96 } 97 } 98 } 99 100 num_inst = 0; 101 qat_dc_init_done = B_FALSE; 102 } 103 104 int 105 qat_dc_init(void) 106 { 107 CpaStatus status = CPA_STATUS_SUCCESS; 108 Cpa32U sess_size = 0; 109 Cpa32U ctx_size = 0; 110 Cpa16U num_inter_buff_lists = 0; 111 Cpa16U buff_num = 0; 112 Cpa32U buff_meta_size = 0; 113 CpaDcSessionSetupData sd = {0}; 114 115 if (qat_dc_init_done) 116 return (0); 117 118 status = cpaDcGetNumInstances(&num_inst); 119 if (status != CPA_STATUS_SUCCESS) 120 return (-1); 121 122 /* if the user has configured no QAT compression units just return */ 123 if (num_inst == 0) 124 return (0); 125 126 if (num_inst > QAT_DC_MAX_INSTANCES) 127 num_inst = QAT_DC_MAX_INSTANCES; 128 129 status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); 130 if (status != CPA_STATUS_SUCCESS) 131 return (-1); 132 133 for (Cpa16U i = 0; i < num_inst; i++) { 134 cpaDcSetAddressTranslation(dc_inst_handles[i], 135 (void*)virt_to_phys); 136 137 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], 138 1, &buff_meta_size); 139 140 if (status == CPA_STATUS_SUCCESS) 141 status = cpaDcGetNumIntermediateBuffers( 142 dc_inst_handles[i], &num_inter_buff_lists); 143 144 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) 145 status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i], 146 num_inter_buff_lists * 147 sizeof (CpaBufferList *)); 148 149 for (buff_num = 0; buff_num < num_inter_buff_lists; 150 buff_num++) { 151 if (status == CPA_STATUS_SUCCESS) 152 status = QAT_PHYS_CONTIG_ALLOC( 153 &buffer_array[i][buff_num], 154 sizeof (CpaBufferList)); 155 156 if (status == CPA_STATUS_SUCCESS) 157 status = QAT_PHYS_CONTIG_ALLOC( 158 &buffer_array[i][buff_num]-> 159 pPrivateMetaData, 160 buff_meta_size); 161 162 if (status == CPA_STATUS_SUCCESS) 163 status = QAT_PHYS_CONTIG_ALLOC( 164 &buffer_array[i][buff_num]->pBuffers, 165 sizeof (CpaFlatBuffer)); 166 167 if (status == CPA_STATUS_SUCCESS) { 168 /* 169 * implementation requires an intermediate 170 * buffer approximately twice the size of 171 * output buffer, which is 2x max buffer 172 * size here. 173 */ 174 status = QAT_PHYS_CONTIG_ALLOC( 175 &buffer_array[i][buff_num]->pBuffers-> 176 pData, 2 * QAT_MAX_BUF_SIZE); 177 if (status != CPA_STATUS_SUCCESS) 178 goto fail; 179 180 buffer_array[i][buff_num]->numBuffers = 1; 181 buffer_array[i][buff_num]->pBuffers-> 182 dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; 183 } 184 } 185 186 status = cpaDcStartInstance(dc_inst_handles[i], 187 num_inter_buff_lists, buffer_array[i]); 188 if (status != CPA_STATUS_SUCCESS) 189 goto fail; 190 191 sd.compLevel = CPA_DC_L1; 192 sd.compType = CPA_DC_DEFLATE; 193 sd.huffType = CPA_DC_HT_FULL_DYNAMIC; 194 sd.sessDirection = CPA_DC_DIR_COMBINED; 195 sd.sessState = CPA_DC_STATELESS; 196 #if (CPA_DC_API_VERSION_NUM_MAJOR == 1 && CPA_DC_API_VERSION_NUM_MINOR < 6) 197 sd.deflateWindowSize = 7; 198 #endif 199 sd.checksum = CPA_DC_ADLER32; 200 status = cpaDcGetSessionSize(dc_inst_handles[i], 201 &sd, &sess_size, &ctx_size); 202 if (status != CPA_STATUS_SUCCESS) 203 goto fail; 204 205 QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); 206 if (session_handles[i] == NULL) 207 goto fail; 208 209 status = cpaDcInitSession(dc_inst_handles[i], 210 session_handles[i], 211 &sd, NULL, qat_dc_callback); 212 if (status != CPA_STATUS_SUCCESS) 213 goto fail; 214 } 215 216 qat_dc_init_done = B_TRUE; 217 return (0); 218 fail: 219 qat_dc_clean(); 220 return (-1); 221 } 222 223 void 224 qat_dc_fini(void) 225 { 226 if (!qat_dc_init_done) 227 return; 228 229 qat_dc_clean(); 230 } 231 232 /* 233 * The "add" parameter is an additional buffer which is passed 234 * to QAT as a scratch buffer alongside the destination buffer 235 * in case the "compressed" data ends up being larger than the 236 * original source data. This is necessary to prevent QAT from 237 * generating buffer overflow warnings for incompressible data. 238 */ 239 static int 240 qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, 241 char *dst, int dst_len, char *add, int add_len, size_t *c_len) 242 { 243 CpaInstanceHandle dc_inst_handle; 244 CpaDcSessionHandle session_handle; 245 CpaBufferList *buf_list_src = NULL; 246 CpaBufferList *buf_list_dst = NULL; 247 CpaFlatBuffer *flat_buf_src = NULL; 248 CpaFlatBuffer *flat_buf_dst = NULL; 249 Cpa8U *buffer_meta_src = NULL; 250 Cpa8U *buffer_meta_dst = NULL; 251 Cpa32U buffer_meta_size = 0; 252 CpaDcRqResults dc_results = {.checksum = 1}; 253 CpaStatus status = CPA_STATUS_FAIL; 254 Cpa32U hdr_sz = 0; 255 Cpa32U compressed_sz; 256 Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; 257 Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; 258 Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; 259 Cpa32U bytes_left; 260 Cpa32U dst_pages = 0; 261 Cpa32U adler32 = 0; 262 char *data; 263 struct page *page; 264 struct page **in_pages = NULL; 265 struct page **out_pages = NULL; 266 struct page **add_pages = NULL; 267 Cpa32U page_off = 0; 268 struct completion complete; 269 Cpa32U page_num = 0; 270 Cpa16U i; 271 272 /* 273 * We increment num_src_buf and num_dst_buf by 2 to allow 274 * us to handle non page-aligned buffer addresses and buffers 275 * whose sizes are not divisible by PAGE_SIZE. 276 */ 277 Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + 278 (num_src_buf * sizeof (CpaFlatBuffer)); 279 Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + 280 ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer)); 281 282 status = QAT_PHYS_CONTIG_ALLOC(&in_pages, 283 num_src_buf * sizeof (struct page *)); 284 if (status != CPA_STATUS_SUCCESS) 285 goto fail; 286 287 status = QAT_PHYS_CONTIG_ALLOC(&out_pages, 288 num_dst_buf * sizeof (struct page *)); 289 if (status != CPA_STATUS_SUCCESS) 290 goto fail; 291 292 status = QAT_PHYS_CONTIG_ALLOC(&add_pages, 293 num_add_buf * sizeof (struct page *)); 294 if (status != CPA_STATUS_SUCCESS) 295 goto fail; 296 297 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst; 298 dc_inst_handle = dc_inst_handles[i]; 299 session_handle = session_handles[i]; 300 301 cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, 302 &buffer_meta_size); 303 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size); 304 if (status != CPA_STATUS_SUCCESS) 305 goto fail; 306 307 cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf, 308 &buffer_meta_size); 309 status = QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size); 310 if (status != CPA_STATUS_SUCCESS) 311 goto fail; 312 313 /* build source buffer list */ 314 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size); 315 if (status != CPA_STATUS_SUCCESS) 316 goto fail; 317 318 flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); 319 320 buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ 321 322 /* build destination buffer list */ 323 status = QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size); 324 if (status != CPA_STATUS_SUCCESS) 325 goto fail; 326 327 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); 328 329 buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ 330 331 buf_list_src->numBuffers = 0; 332 buf_list_src->pPrivateMetaData = buffer_meta_src; 333 bytes_left = src_len; 334 data = src; 335 page_num = 0; 336 while (bytes_left > 0) { 337 page_off = ((long)data & ~PAGE_MASK); 338 page = qat_mem_to_page(data); 339 in_pages[page_num] = page; 340 flat_buf_src->pData = kmap(page) + page_off; 341 flat_buf_src->dataLenInBytes = 342 min((long)PAGE_SIZE - page_off, (long)bytes_left); 343 344 bytes_left -= flat_buf_src->dataLenInBytes; 345 data += flat_buf_src->dataLenInBytes; 346 flat_buf_src++; 347 buf_list_src->numBuffers++; 348 page_num++; 349 } 350 351 buf_list_dst->numBuffers = 0; 352 buf_list_dst->pPrivateMetaData = buffer_meta_dst; 353 bytes_left = dst_len; 354 data = dst; 355 page_num = 0; 356 while (bytes_left > 0) { 357 page_off = ((long)data & ~PAGE_MASK); 358 page = qat_mem_to_page(data); 359 flat_buf_dst->pData = kmap(page) + page_off; 360 out_pages[page_num] = page; 361 flat_buf_dst->dataLenInBytes = 362 min((long)PAGE_SIZE - page_off, (long)bytes_left); 363 364 bytes_left -= flat_buf_dst->dataLenInBytes; 365 data += flat_buf_dst->dataLenInBytes; 366 flat_buf_dst++; 367 buf_list_dst->numBuffers++; 368 page_num++; 369 dst_pages++; 370 } 371 372 /* map additional scratch pages into the destination buffer list */ 373 bytes_left = add_len; 374 data = add; 375 page_num = 0; 376 while (bytes_left > 0) { 377 page_off = ((long)data & ~PAGE_MASK); 378 page = qat_mem_to_page(data); 379 flat_buf_dst->pData = kmap(page) + page_off; 380 add_pages[page_num] = page; 381 flat_buf_dst->dataLenInBytes = 382 min((long)PAGE_SIZE - page_off, (long)bytes_left); 383 384 bytes_left -= flat_buf_dst->dataLenInBytes; 385 data += flat_buf_dst->dataLenInBytes; 386 flat_buf_dst++; 387 buf_list_dst->numBuffers++; 388 page_num++; 389 } 390 391 init_completion(&complete); 392 393 if (dir == QAT_COMPRESS) { 394 QAT_STAT_BUMP(comp_requests); 395 QAT_STAT_INCR(comp_total_in_bytes, src_len); 396 397 cpaDcGenerateHeader(session_handle, 398 buf_list_dst->pBuffers, &hdr_sz); 399 buf_list_dst->pBuffers->pData += hdr_sz; 400 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; 401 status = cpaDcCompressData( 402 dc_inst_handle, session_handle, 403 buf_list_src, buf_list_dst, 404 &dc_results, CPA_DC_FLUSH_FINAL, 405 &complete); 406 if (status != CPA_STATUS_SUCCESS) { 407 goto fail; 408 } 409 410 /* we now wait until the completion of the operation. */ 411 wait_for_completion(&complete); 412 413 if (dc_results.status != CPA_STATUS_SUCCESS) { 414 status = CPA_STATUS_FAIL; 415 goto fail; 416 } 417 418 compressed_sz = dc_results.produced; 419 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { 420 status = CPA_STATUS_INCOMPRESSIBLE; 421 goto fail; 422 } 423 424 /* get adler32 checksum and append footer */ 425 *(Cpa32U*)(dst + hdr_sz + compressed_sz) = 426 BSWAP_32(dc_results.checksum); 427 428 *c_len = hdr_sz + compressed_sz + ZLIB_FOOT_SZ; 429 QAT_STAT_INCR(comp_total_out_bytes, *c_len); 430 } else { 431 ASSERT3U(dir, ==, QAT_DECOMPRESS); 432 QAT_STAT_BUMP(decomp_requests); 433 QAT_STAT_INCR(decomp_total_in_bytes, src_len); 434 435 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; 436 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; 437 status = cpaDcDecompressData(dc_inst_handle, session_handle, 438 buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL, 439 &complete); 440 441 if (CPA_STATUS_SUCCESS != status) { 442 status = CPA_STATUS_FAIL; 443 goto fail; 444 } 445 446 /* we now wait until the completion of the operation. */ 447 wait_for_completion(&complete); 448 449 if (dc_results.status != CPA_STATUS_SUCCESS) { 450 status = CPA_STATUS_FAIL; 451 goto fail; 452 } 453 454 /* verify adler checksum */ 455 adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ); 456 if (adler32 != BSWAP_32(dc_results.checksum)) { 457 status = CPA_STATUS_FAIL; 458 goto fail; 459 } 460 *c_len = dc_results.produced; 461 QAT_STAT_INCR(decomp_total_out_bytes, *c_len); 462 } 463 464 fail: 465 if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE) 466 QAT_STAT_BUMP(dc_fails); 467 468 if (in_pages) { 469 for (page_num = 0; 470 page_num < buf_list_src->numBuffers; 471 page_num++) { 472 kunmap(in_pages[page_num]); 473 } 474 QAT_PHYS_CONTIG_FREE(in_pages); 475 } 476 477 if (out_pages) { 478 for (page_num = 0; page_num < dst_pages; page_num++) { 479 kunmap(out_pages[page_num]); 480 } 481 QAT_PHYS_CONTIG_FREE(out_pages); 482 } 483 484 if (add_pages) { 485 for (page_num = 0; 486 page_num < buf_list_dst->numBuffers - dst_pages; 487 page_num++) { 488 kunmap(add_pages[page_num]); 489 } 490 QAT_PHYS_CONTIG_FREE(add_pages); 491 } 492 493 QAT_PHYS_CONTIG_FREE(buffer_meta_src); 494 QAT_PHYS_CONTIG_FREE(buffer_meta_dst); 495 QAT_PHYS_CONTIG_FREE(buf_list_src); 496 QAT_PHYS_CONTIG_FREE(buf_list_dst); 497 498 return (status); 499 } 500 501 /* 502 * Entry point for QAT accelerated compression / decompression. 503 */ 504 int 505 qat_compress(qat_compress_dir_t dir, char *src, int src_len, 506 char *dst, int dst_len, size_t *c_len) 507 { 508 int ret; 509 size_t add_len = 0; 510 void *add = NULL; 511 512 if (dir == QAT_COMPRESS) { 513 add_len = dst_len; 514 add = zio_data_buf_alloc(add_len); 515 } 516 517 ret = qat_compress_impl(dir, src, src_len, dst, 518 dst_len, add, add_len, c_len); 519 520 if (dir == QAT_COMPRESS) 521 zio_data_buf_free(add, add_len); 522 523 return (ret); 524 } 525 526 static int 527 param_set_qat_compress(const char *val, zfs_kernel_param_t *kp) 528 { 529 int ret; 530 int *pvalue = kp->arg; 531 ret = param_set_int(val, kp); 532 if (ret) 533 return (ret); 534 /* 535 * zfs_qat_compress_disable = 0: enable qat compress 536 * try to initialize qat instance if it has not been done 537 */ 538 if (*pvalue == 0 && !qat_dc_init_done) { 539 ret = qat_dc_init(); 540 if (ret != 0) { 541 zfs_qat_compress_disable = 1; 542 return (ret); 543 } 544 } 545 return (ret); 546 } 547 548 module_param_call(zfs_qat_compress_disable, param_set_qat_compress, 549 param_get_int, &zfs_qat_compress_disable, 0644); 550 MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); 551 552 #endif 553