1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 ***************************************************************************** 6 * @file dc_datapath.c 7 * 8 * @defgroup Dc_DataCompression DC Data Compression 9 * 10 * @ingroup Dc_DataCompression 11 * 12 * @description 13 * Implementation of the Data Compression datapath operations. 14 * 15 *****************************************************************************/ 16 17 /* 18 ******************************************************************************* 19 * Include public/global header files 20 ******************************************************************************* 21 */ 22 #include "cpa.h" 23 #include "cpa_dc.h" 24 #include "cpa_dc_dp.h" 25 26 /* 27 ******************************************************************************* 28 * Include private header files 29 ******************************************************************************* 30 */ 31 #include "dc_session.h" 32 #include "dc_datapath.h" 33 #include "sal_statistics.h" 34 #include "lac_common.h" 35 #include "lac_mem.h" 36 #include "lac_mem_pools.h" 37 #include "sal_types_compression.h" 38 #include "dc_stats.h" 39 #include "lac_buffer_desc.h" 40 #include "lac_sal.h" 41 #include "lac_log.h" 42 #include "lac_sync.h" 43 #include "sal_service_state.h" 44 #include "sal_qat_cmn_msg.h" 45 #include "dc_error_counter.h" 46 #define DC_COMP_MAX_BUFF_SIZE (1024 * 64) 47 48 static QatUtilsAtomic dcErrorCount[MAX_DC_ERROR_TYPE]; 49 50 void 51 dcErrorLog(CpaDcReqStatus dcError) 52 { 53 Cpa32U absError = 0; 54 55 absError = abs(dcError); 56 if ((dcError < CPA_DC_OK) && (absError < MAX_DC_ERROR_TYPE)) { 57 qatUtilsAtomicInc(&(dcErrorCount[absError])); 58 } 59 } 60 61 Cpa64U 62 getDcErrorCounter(CpaDcReqStatus dcError) 63 { 64 Cpa32U absError = 0; 65 66 absError = abs(dcError); 67 if (!(dcError >= CPA_DC_OK || dcError < CPA_DC_EMPTY_DYM_BLK)) { 68 return (Cpa64U)qatUtilsAtomicGet(&dcErrorCount[absError]); 69 } 70 71 return 0; 72 } 73 74 void 75 dcCompression_ProcessCallback(void *pRespMsg) 76 { 77 CpaStatus status = CPA_STATUS_SUCCESS; 78 icp_qat_fw_comp_resp_t *pCompRespMsg = NULL; 79 void *callbackTag = NULL; 80 Cpa64U *pReqData = NULL; 81 CpaDcDpOpData *pResponse = NULL; 82 CpaDcRqResults *pResults = NULL; 83 CpaDcCallbackFn pCbFunc = NULL; 84 dc_session_desc_t *pSessionDesc = NULL; 85 sal_compression_service_t *pService = NULL; 86 dc_compression_cookie_t *pCookie = NULL; 87 CpaDcOpData *pOpData = NULL; 88 CpaBoolean cmpPass = CPA_TRUE, xlatPass = CPA_TRUE; 89 CpaBoolean verifyHwIntegrityCrcs = CPA_FALSE; 90 Cpa8U cmpErr = ERR_CODE_NO_ERROR, xlatErr = ERR_CODE_NO_ERROR; 91 dc_request_dir_t compDecomp = DC_COMPRESSION_REQUEST; 92 Cpa8U opStatus = ICP_QAT_FW_COMN_STATUS_FLAG_OK; 93 Cpa8U hdrFlags = 0; 94 95 /* Cast response message to compression response message type */ 96 pCompRespMsg = (icp_qat_fw_comp_resp_t *)pRespMsg; 97 98 /* Extract request data pointer from the opaque data */ 99 LAC_MEM_SHARED_READ_TO_PTR(pCompRespMsg->opaque_data, pReqData); 100 101 /* Extract fields from the request data structure */ 102 pCookie = (dc_compression_cookie_t *)pReqData; 103 if (!pCookie) 104 return; 105 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle); 106 107 if (CPA_TRUE == pSessionDesc->isDcDp) { 108 pResponse = (CpaDcDpOpData *)pReqData; 109 pResults = &(pResponse->results); 110 111 if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) { 112 compDecomp = DC_DECOMPRESSION_REQUEST; 113 } 114 } else { 115 pSessionDesc = pCookie->pSessionDesc; 116 pResults = pCookie->pResults; 117 callbackTag = pCookie->callbackTag; 118 pCbFunc = pCookie->pSessionDesc->pCompressionCb; 119 compDecomp = pCookie->compDecomp; 120 pOpData = pCookie->pDcOpData; 121 } 122 123 pService = (sal_compression_service_t *)(pCookie->dcInstance); 124 125 opStatus = pCompRespMsg->comn_resp.comn_status; 126 127 if (NULL != pOpData) { 128 verifyHwIntegrityCrcs = pOpData->verifyHwIntegrityCrcs; 129 } 130 131 hdrFlags = pCompRespMsg->comn_resp.hdr_flags; 132 133 /* Get the cmp error code */ 134 cmpErr = pCompRespMsg->comn_resp.comn_error.s1.cmp_err_code; 135 if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(opStatus)) { 136 /* Compression not supported by firmware, set produced/consumed 137 to zero 138 and call the cb function with status CPA_STATUS_UNSUPPORTED 139 */ 140 QAT_UTILS_LOG("Compression feature not supported\n"); 141 status = CPA_STATUS_UNSUPPORTED; 142 pResults->status = (Cpa8S)cmpErr; 143 pResults->consumed = 0; 144 pResults->produced = 0; 145 if (CPA_TRUE == pSessionDesc->isDcDp) { 146 if (pResponse) 147 pResponse->responseStatus = 148 CPA_STATUS_UNSUPPORTED; 149 (pService->pDcDpCb)(pResponse); 150 } else { 151 /* Free the memory pool */ 152 Lac_MemPoolEntryFree(pCookie); 153 pCookie = NULL; 154 if (NULL != pCbFunc) { 155 pCbFunc(callbackTag, status); 156 } 157 } 158 if (DC_COMPRESSION_REQUEST == compDecomp) { 159 COMPRESSION_STAT_INC(numCompCompletedErrors, pService); 160 } else { 161 COMPRESSION_STAT_INC(numDecompCompletedErrors, 162 pService); 163 } 164 return; 165 } else { 166 /* Check compression response status */ 167 cmpPass = 168 (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK == 169 ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(opStatus)); 170 } 171 172 if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) { 173 cmpPass = CPA_TRUE; 174 cmpErr = ERR_CODE_NO_ERROR; 175 } 176 /* log the slice hang and endpoint push/pull error inside the response 177 */ 178 if (ERR_CODE_SSM_ERROR == (Cpa8S)cmpErr) { 179 QAT_UTILS_LOG( 180 "Slice hang detected on the compression slice.\n"); 181 } else if (ERR_CODE_ENDPOINT_ERROR == (Cpa8S)cmpErr) { 182 QAT_UTILS_LOG( 183 "PCIe End Point Push/Pull or TI/RI Parity error detected.\n"); 184 } 185 186 /* We return the compression error code for now. We would need to update 187 * the API if we decide to return both error codes */ 188 pResults->status = (Cpa8S)cmpErr; 189 190 /* Check the translator status */ 191 if ((DC_COMPRESSION_REQUEST == compDecomp) && 192 (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) { 193 /* Check translator response status */ 194 xlatPass = 195 (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK == 196 ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(opStatus)); 197 198 /* Get the translator error code */ 199 xlatErr = pCompRespMsg->comn_resp.comn_error.s1.xlat_err_code; 200 201 /* Return a fatal error or a potential error in the translator 202 * slice 203 * if the compression slice did not return any error */ 204 if ((CPA_DC_OK == pResults->status) || 205 (CPA_DC_FATALERR == (Cpa8S)xlatErr)) { 206 pResults->status = (Cpa8S)xlatErr; 207 } 208 } 209 /* Update dc error counter */ 210 dcErrorLog(pResults->status); 211 212 if (CPA_FALSE == pSessionDesc->isDcDp) { 213 /* In case of any error for an end of packet request, we need to 214 * update 215 * the request type for the following request */ 216 if (CPA_DC_FLUSH_FINAL == pCookie->flushFlag && cmpPass && 217 xlatPass) { 218 pSessionDesc->requestType = DC_REQUEST_FIRST; 219 } else { 220 pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT; 221 } 222 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) || 223 ((CPA_DC_STATELESS == pSessionDesc->sessState) && 224 (DC_COMPRESSION_REQUEST == compDecomp))) { 225 /* Overflow is a valid use case for Traditional API 226 * only. 227 * Stateful Overflow is supported in both compression 228 * and 229 * decompression direction. 230 * Stateless Overflow is supported only in compression 231 * direction. 232 */ 233 if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr) 234 cmpPass = CPA_TRUE; 235 236 if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) { 237 xlatPass = CPA_TRUE; 238 } 239 } 240 } else { 241 if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr) { 242 cmpPass = CPA_FALSE; 243 } 244 if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) { 245 xlatPass = CPA_FALSE; 246 } 247 } 248 249 if ((CPA_TRUE == cmpPass) && (CPA_TRUE == xlatPass)) { 250 /* Extract the response from the firmware */ 251 pResults->consumed = 252 pCompRespMsg->comp_resp_pars.input_byte_counter; 253 pResults->produced = 254 pCompRespMsg->comp_resp_pars.output_byte_counter; 255 pSessionDesc->cumulativeConsumedBytes += pResults->consumed; 256 257 if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) { 258 if (CPA_DC_CRC32 == pSessionDesc->checksumType) { 259 pResults->checksum = 260 pCompRespMsg->comp_resp_pars.crc.legacy 261 .curr_crc32; 262 } else if (CPA_DC_ADLER32 == 263 pSessionDesc->checksumType) { 264 pResults->checksum = 265 pCompRespMsg->comp_resp_pars.crc.legacy 266 .curr_adler_32; 267 } 268 pSessionDesc->previousChecksum = pResults->checksum; 269 } 270 271 if (DC_DECOMPRESSION_REQUEST == compDecomp) { 272 pResults->endOfLastBlock = 273 (ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET == 274 ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET( 275 opStatus)); 276 } 277 278 /* Save the checksum for the next request */ 279 if ((CPA_DC_OVERFLOW != (Cpa8S)xlatErr) && 280 (CPA_TRUE == verifyHwIntegrityCrcs)) { 281 pSessionDesc->previousChecksum = 282 pSessionDesc->seedSwCrc.swCrcI; 283 } 284 285 /* Check if a CNV recovery happened and 286 * increase stats counter 287 */ 288 if ((DC_COMPRESSION_REQUEST == compDecomp) && 289 ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdrFlags) && 290 ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdrFlags)) { 291 COMPRESSION_STAT_INC(numCompCnvErrorsRecovered, 292 pService); 293 } 294 295 if (CPA_TRUE == pSessionDesc->isDcDp) { 296 if (pResponse) 297 pResponse->responseStatus = CPA_STATUS_SUCCESS; 298 } else { 299 if (DC_COMPRESSION_REQUEST == compDecomp) { 300 COMPRESSION_STAT_INC(numCompCompleted, 301 pService); 302 } else { 303 COMPRESSION_STAT_INC(numDecompCompleted, 304 pService); 305 } 306 } 307 } else { 308 pResults->consumed = 0; 309 pResults->produced = 0; 310 if (CPA_DC_OVERFLOW == pResults->status && 311 CPA_DC_STATELESS == pSessionDesc->sessState) { 312 /* This error message will be returned by Data Plane API 313 * in both 314 * compression and decompression direction. With 315 * Traditional API 316 * this error message will be returned only in stateless 317 * decompression direction */ 318 QAT_UTILS_LOG( 319 "Unrecoverable error: stateless overflow. You may need to increase the size of your destination buffer.\n"); 320 } 321 322 if (CPA_TRUE == pSessionDesc->isDcDp) { 323 if (pResponse) 324 pResponse->responseStatus = CPA_STATUS_FAIL; 325 } else { 326 if (CPA_DC_OK != pResults->status && 327 CPA_DC_INCOMPLETE_FILE_ERR != pResults->status) { 328 status = CPA_STATUS_FAIL; 329 } 330 331 if (DC_COMPRESSION_REQUEST == compDecomp) { 332 COMPRESSION_STAT_INC(numCompCompletedErrors, 333 pService); 334 } else { 335 COMPRESSION_STAT_INC(numDecompCompletedErrors, 336 pService); 337 } 338 } 339 } 340 341 if (CPA_TRUE == pSessionDesc->isDcDp) { 342 /* Decrement number of stateless pending callbacks for session 343 */ 344 pSessionDesc->pendingDpStatelessCbCount--; 345 (pService->pDcDpCb)(pResponse); 346 } else { 347 /* Decrement number of pending callbacks for session */ 348 if (CPA_DC_STATELESS == pSessionDesc->sessState) { 349 qatUtilsAtomicDec( 350 &(pCookie->pSessionDesc->pendingStatelessCbCount)); 351 } else if (0 != 352 qatUtilsAtomicGet(&pCookie->pSessionDesc 353 ->pendingStatefulCbCount)) { 354 qatUtilsAtomicDec( 355 &(pCookie->pSessionDesc->pendingStatefulCbCount)); 356 } 357 358 /* Free the memory pool */ 359 if (NULL != pCookie) { 360 Lac_MemPoolEntryFree(pCookie); 361 pCookie = NULL; 362 } 363 364 if (NULL != pCbFunc) { 365 pCbFunc(callbackTag, status); 366 } 367 } 368 } 369 370 /** 371 ***************************************************************************** 372 * @ingroup Dc_DataCompression 373 * Check that all the parameters in the pOpData structure are valid 374 * 375 * @description 376 * Check that all the parameters in the pOpData structure are valid 377 * 378 * @param[in] pService Pointer to the compression service 379 * @param[in] pOpData Pointer to request information structure 380 * holding parameters for cpaDcCompress2 and 381 * CpaDcDecompressData2 382 * @retval CPA_STATUS_SUCCESS Function executed successfully 383 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 384 * 385 *****************************************************************************/ 386 static CpaStatus 387 dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData) 388 { 389 CpaDcSkipMode skipMode = 0; 390 391 if ((pOpData->flushFlag < CPA_DC_FLUSH_NONE) || 392 (pOpData->flushFlag > CPA_DC_FLUSH_FULL)) { 393 LAC_INVALID_PARAM_LOG("Invalid flushFlag value"); 394 return CPA_STATUS_INVALID_PARAM; 395 } 396 397 skipMode = pOpData->inputSkipData.skipMode; 398 if ((skipMode < CPA_DC_SKIP_DISABLED) || 399 (skipMode > CPA_DC_SKIP_STRIDE)) { 400 LAC_INVALID_PARAM_LOG("Invalid input skip mode value"); 401 return CPA_STATUS_INVALID_PARAM; 402 } 403 404 skipMode = pOpData->outputSkipData.skipMode; 405 if ((skipMode < CPA_DC_SKIP_DISABLED) || 406 (skipMode > CPA_DC_SKIP_STRIDE)) { 407 LAC_INVALID_PARAM_LOG("Invalid output skip mode value"); 408 return CPA_STATUS_INVALID_PARAM; 409 } 410 411 if (pOpData->integrityCrcCheck == CPA_FALSE && 412 pOpData->verifyHwIntegrityCrcs == CPA_TRUE) { 413 LAC_INVALID_PARAM_LOG( 414 "integrityCrcCheck must be set to true" 415 "in order to enable verifyHwIntegrityCrcs"); 416 return CPA_STATUS_INVALID_PARAM; 417 } 418 419 if (pOpData->integrityCrcCheck != CPA_TRUE && 420 pOpData->integrityCrcCheck != CPA_FALSE) { 421 LAC_INVALID_PARAM_LOG("Invalid integrityCrcCheck value"); 422 return CPA_STATUS_INVALID_PARAM; 423 } 424 425 if (pOpData->verifyHwIntegrityCrcs != CPA_TRUE && 426 pOpData->verifyHwIntegrityCrcs != CPA_FALSE) { 427 LAC_INVALID_PARAM_LOG("Invalid verifyHwIntegrityCrcs value"); 428 return CPA_STATUS_INVALID_PARAM; 429 } 430 431 if (pOpData->compressAndVerify != CPA_TRUE && 432 pOpData->compressAndVerify != CPA_FALSE) { 433 LAC_INVALID_PARAM_LOG("Invalid cnv decompress check value"); 434 return CPA_STATUS_INVALID_PARAM; 435 } 436 437 if (CPA_TRUE == pOpData->integrityCrcCheck && 438 CPA_FALSE == pService->generic_service_info.integrityCrcCheck) { 439 LAC_INVALID_PARAM_LOG("Integrity CRC check is not " 440 "supported on this device"); 441 return CPA_STATUS_INVALID_PARAM; 442 } 443 return CPA_STATUS_SUCCESS; 444 } 445 446 /** 447 ***************************************************************************** 448 * @ingroup Dc_DataCompression 449 * Check the compression source buffer for Batch and Pack API. 450 * 451 * @description 452 * Check that all the parameters used for Pack compression 453 * request are valid. This function essentially checks the source buffer 454 * parameters and results structure parameters. 455 * 456 * @param[in] pSessionHandle Session handle 457 * @param[in] pSrcBuff Pointer to data buffer for compression 458 * @param[in] pDestBuff Pointer to buffer space allocated for 459 * output data 460 * @param[in] pResults Pointer to results structure 461 * @param[in] flushFlag Indicates the type of flush to be 462 * performed 463 * @param[in] srcBuffSize Size of the source buffer 464 * 465 * @retval CPA_STATUS_SUCCESS Function executed successfully 466 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 467 * 468 *****************************************************************************/ 469 static CpaStatus 470 dcCheckSourceData(CpaDcSessionHandle pSessionHandle, 471 CpaBufferList *pSrcBuff, 472 CpaBufferList *pDestBuff, 473 CpaDcRqResults *pResults, 474 CpaDcFlush flushFlag, 475 Cpa64U srcBuffSize, 476 CpaDcSkipData *skipData) 477 { 478 dc_session_desc_t *pSessionDesc = NULL; 479 480 LAC_CHECK_NULL_PARAM(pSessionHandle); 481 LAC_CHECK_NULL_PARAM(pSrcBuff); 482 LAC_CHECK_NULL_PARAM(pDestBuff); 483 LAC_CHECK_NULL_PARAM(pResults); 484 485 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 486 if (NULL == pSessionDesc) { 487 LAC_INVALID_PARAM_LOG("Session handle not as expected"); 488 return CPA_STATUS_INVALID_PARAM; 489 } 490 491 if ((flushFlag < CPA_DC_FLUSH_NONE) || 492 (flushFlag > CPA_DC_FLUSH_FULL)) { 493 LAC_INVALID_PARAM_LOG("Invalid flushFlag value"); 494 return CPA_STATUS_INVALID_PARAM; 495 } 496 497 if (pSrcBuff == pDestBuff) { 498 LAC_INVALID_PARAM_LOG("In place operation not supported"); 499 return CPA_STATUS_INVALID_PARAM; 500 } 501 502 /* Compressing zero bytes is not supported for stateless sessions 503 * for non Batch and Pack requests */ 504 if ((CPA_DC_STATELESS == pSessionDesc->sessState) && 505 (0 == srcBuffSize) && (NULL == skipData)) { 506 LAC_INVALID_PARAM_LOG( 507 "The source buffer size needs to be greater than " 508 "zero bytes for stateless sessions"); 509 return CPA_STATUS_INVALID_PARAM; 510 } 511 512 if (srcBuffSize > DC_BUFFER_MAX_SIZE) { 513 LAC_INVALID_PARAM_LOG( 514 "The source buffer size needs to be less than or " 515 "equal to 2^32-1 bytes"); 516 return CPA_STATUS_INVALID_PARAM; 517 } 518 519 return CPA_STATUS_SUCCESS; 520 } 521 522 /** 523 ***************************************************************************** 524 * @ingroup Dc_DataCompression 525 * Check the compression or decompression function parameters. 526 * 527 * @description 528 * Check that all the parameters used for a Batch and Pack compression 529 * request are valid. This function essentially checks the destination 530 * buffer parameters and intermediate buffer parameters. 531 * 532 * @param[in] pService Pointer to the compression service 533 * @param[in] pSessionHandle Session handle 534 * @param[in] pDestBuff Pointer to buffer space allocated for 535 * output data 536 * @param[in] compDecomp Direction of the operation 537 * 538 * @retval CPA_STATUS_SUCCESS Function executed successfully 539 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 540 * 541 *****************************************************************************/ 542 static CpaStatus 543 dcCheckDestinationData(sal_compression_service_t *pService, 544 CpaDcSessionHandle pSessionHandle, 545 CpaBufferList *pDestBuff, 546 dc_request_dir_t compDecomp) 547 { 548 dc_session_desc_t *pSessionDesc = NULL; 549 Cpa64U destBuffSize = 0; 550 551 LAC_CHECK_NULL_PARAM(pSessionHandle); 552 LAC_CHECK_NULL_PARAM(pDestBuff); 553 554 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 555 if (NULL == pSessionDesc) { 556 LAC_INVALID_PARAM_LOG("Session handle not as expected"); 557 return CPA_STATUS_INVALID_PARAM; 558 } 559 560 if (LacBuffDesc_BufferListVerify(pDestBuff, 561 &destBuffSize, 562 LAC_NO_ALIGNMENT_SHIFT) != 563 CPA_STATUS_SUCCESS) { 564 LAC_INVALID_PARAM_LOG( 565 "Invalid destination buffer list parameter"); 566 return CPA_STATUS_INVALID_PARAM; 567 } 568 569 if (destBuffSize > DC_BUFFER_MAX_SIZE) { 570 LAC_INVALID_PARAM_LOG( 571 "The destination buffer size needs to be less " 572 "than or equal to 2^32-1 bytes"); 573 return CPA_STATUS_INVALID_PARAM; 574 } 575 576 if (CPA_TRUE == pSessionDesc->isDcDp) { 577 LAC_INVALID_PARAM_LOG( 578 "The session type should not be data plane"); 579 return CPA_STATUS_INVALID_PARAM; 580 } 581 582 if (DC_COMPRESSION_REQUEST == compDecomp) { 583 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { 584 585 /* Check if intermediate buffers are supported */ 586 if ((0 == pService->pInterBuffPtrsArrayPhyAddr) || 587 (NULL == pService->pInterBuffPtrsArray)) { 588 LAC_LOG_ERROR( 589 "No intermediate buffer defined for this instance " 590 "- see cpaDcStartInstance"); 591 return CPA_STATUS_INVALID_PARAM; 592 } 593 594 /* Ensure that the destination buffer size is greater or 595 * equal to 128B */ 596 if (destBuffSize < DC_DEST_BUFFER_DYN_MIN_SIZE) { 597 LAC_INVALID_PARAM_LOG( 598 "Destination buffer size should be " 599 "greater or equal to 128B"); 600 return CPA_STATUS_INVALID_PARAM; 601 } 602 } else 603 { 604 /* Ensure that the destination buffer size is greater or 605 * equal to devices min output buff size */ 606 if (destBuffSize < 607 pService->comp_device_data.minOutputBuffSize) { 608 LAC_INVALID_PARAM_LOG1( 609 "Destination buffer size should be " 610 "greater or equal to %d bytes", 611 pService->comp_device_data 612 .minOutputBuffSize); 613 return CPA_STATUS_INVALID_PARAM; 614 } 615 } 616 } else { 617 /* Ensure that the destination buffer size is greater than 618 * 0 bytes */ 619 if (destBuffSize < DC_DEST_BUFFER_DEC_MIN_SIZE) { 620 LAC_INVALID_PARAM_LOG( 621 "Destination buffer size should be " 622 "greater than 0 bytes"); 623 return CPA_STATUS_INVALID_PARAM; 624 } 625 } 626 return CPA_STATUS_SUCCESS; 627 } 628 629 /** 630 ***************************************************************************** 631 * @ingroup Dc_DataCompression 632 * Populate the compression request parameters 633 * 634 * @description 635 * This function will populate the compression request parameters 636 * 637 * @param[out] pCompReqParams Pointer to the compression request parameters 638 * @param[in] pCookie Pointer to the compression cookie 639 * 640 *****************************************************************************/ 641 static void 642 dcCompRequestParamsPopulate(icp_qat_fw_comp_req_params_t *pCompReqParams, 643 dc_compression_cookie_t *pCookie) 644 { 645 pCompReqParams->comp_len = pCookie->srcTotalDataLenInBytes; 646 pCompReqParams->out_buffer_sz = pCookie->dstTotalDataLenInBytes; 647 } 648 649 /** 650 ***************************************************************************** 651 * @ingroup Dc_DataCompression 652 * Create the requests for compression or decompression 653 * 654 * @description 655 * Create the requests for compression or decompression. This function 656 * will update the cookie will all required information. 657 * 658 * @param{out] pCookie Pointer to the compression cookie 659 * @param[in] pService Pointer to the compression service 660 * @param[in] pSessionDesc Pointer to the session descriptor 661 * @param[in pSessionHandle Session handle 662 * @param[in] pSrcBuff Pointer to data buffer for compression 663 * @param[in] pDestBuff Pointer to buffer space for data after 664 * compression 665 * @param[in] pResults Pointer to results structure 666 * @param[in] flushFlag Indicates the type of flush to be 667 * performed 668 * @param[in] pOpData Pointer to request information structure 669 * holding parameters for cpaDcCompress2 670 * and CpaDcDecompressData2 671 * @param[in] callbackTag Pointer to the callback tag 672 * @param[in] compDecomp Direction of the operation 673 * @param[in] compressAndVerify Compress and Verify 674 * 675 * @retval CPA_STATUS_SUCCESS Function executed successfully 676 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 677 * 678 *****************************************************************************/ 679 static CpaStatus 680 dcCreateRequest(dc_compression_cookie_t *pCookie, 681 sal_compression_service_t *pService, 682 dc_session_desc_t *pSessionDesc, 683 CpaDcSessionHandle pSessionHandle, 684 CpaBufferList *pSrcBuff, 685 CpaBufferList *pDestBuff, 686 CpaDcRqResults *pResults, 687 CpaDcFlush flushFlag, 688 CpaDcOpData *pOpData, 689 void *callbackTag, 690 dc_request_dir_t compDecomp, 691 dc_cnv_mode_t cnvMode) 692 { 693 icp_qat_fw_comp_req_t *pMsg = NULL; 694 icp_qat_fw_comp_req_params_t *pCompReqParams = NULL; 695 Cpa64U srcAddrPhys = 0, dstAddrPhys = 0; 696 Cpa64U srcTotalDataLenInBytes = 0, dstTotalDataLenInBytes = 0; 697 698 Cpa32U rpCmdFlags = 0; 699 Cpa8U sop = ICP_QAT_FW_COMP_SOP; 700 Cpa8U eop = ICP_QAT_FW_COMP_EOP; 701 Cpa8U bFinal = ICP_QAT_FW_COMP_NOT_BFINAL; 702 Cpa8U crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY; 703 Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 704 Cpa8U cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 705 CpaBoolean integrityCrcCheck = CPA_FALSE; 706 CpaStatus status = CPA_STATUS_SUCCESS; 707 CpaDcFlush flush = CPA_DC_FLUSH_NONE; 708 Cpa32U initial_adler = 1; 709 Cpa32U initial_crc32 = 0; 710 icp_qat_fw_comp_req_t *pReqCache = NULL; 711 712 /* Write the buffer descriptors */ 713 status = LacBuffDesc_BufferListDescWriteAndGetSize( 714 pSrcBuff, 715 &srcAddrPhys, 716 CPA_FALSE, 717 &srcTotalDataLenInBytes, 718 &(pService->generic_service_info)); 719 if (status != CPA_STATUS_SUCCESS) { 720 return status; 721 } 722 723 status = LacBuffDesc_BufferListDescWriteAndGetSize( 724 pDestBuff, 725 &dstAddrPhys, 726 CPA_FALSE, 727 &dstTotalDataLenInBytes, 728 &(pService->generic_service_info)); 729 if (status != CPA_STATUS_SUCCESS) { 730 return status; 731 } 732 733 /* Populate the compression cookie */ 734 pCookie->dcInstance = pService; 735 pCookie->pSessionHandle = pSessionHandle; 736 pCookie->callbackTag = callbackTag; 737 pCookie->pSessionDesc = pSessionDesc; 738 pCookie->pDcOpData = pOpData; 739 pCookie->pResults = pResults; 740 pCookie->compDecomp = compDecomp; 741 pCookie->pUserSrcBuff = NULL; 742 pCookie->pUserDestBuff = NULL; 743 744 /* Extract flush flag from either the opData or from the 745 * parameter. Opdata have been introduce with APIs 746 * cpaDcCompressData2 and cpaDcDecompressData2 */ 747 if (NULL != pOpData) { 748 flush = pOpData->flushFlag; 749 integrityCrcCheck = pOpData->integrityCrcCheck; 750 } else { 751 flush = flushFlag; 752 } 753 pCookie->flushFlag = flush; 754 755 /* The firmware expects the length in bytes for source and destination 756 * to be Cpa32U parameters. However the total data length could be 757 * bigger as allocated by the user. We ensure that this is not the case 758 * in dcCheckSourceData and cast the values to Cpa32U here */ 759 pCookie->srcTotalDataLenInBytes = (Cpa32U)srcTotalDataLenInBytes; 760 if ((DC_COMPRESSION_REQUEST == compDecomp) && 761 (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) { 762 if (pService->minInterBuffSizeInBytes < 763 (Cpa32U)dstTotalDataLenInBytes) { 764 pCookie->dstTotalDataLenInBytes = 765 (Cpa32U)(pService->minInterBuffSizeInBytes); 766 } else { 767 pCookie->dstTotalDataLenInBytes = 768 (Cpa32U)dstTotalDataLenInBytes; 769 } 770 } else 771 { 772 pCookie->dstTotalDataLenInBytes = 773 (Cpa32U)dstTotalDataLenInBytes; 774 } 775 776 /* Device can not decompress an odd byte decompression request 777 * if bFinal is not set 778 */ 779 if (CPA_TRUE != pService->comp_device_data.oddByteDecompNobFinal) { 780 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) && 781 (CPA_DC_FLUSH_FINAL != flushFlag) && 782 (DC_DECOMPRESSION_REQUEST == compDecomp) && 783 (pCookie->srcTotalDataLenInBytes & 0x1)) { 784 pCookie->srcTotalDataLenInBytes--; 785 } 786 } 787 /* Device can not decompress odd byte interim requests */ 788 if (CPA_TRUE != pService->comp_device_data.oddByteDecompInterim) { 789 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) && 790 (CPA_DC_FLUSH_FINAL != flushFlag) && 791 (CPA_DC_FLUSH_FULL != flushFlag) && 792 (DC_DECOMPRESSION_REQUEST == compDecomp) && 793 (pCookie->srcTotalDataLenInBytes & 0x1)) { 794 pCookie->srcTotalDataLenInBytes--; 795 } 796 } 797 798 pMsg = (icp_qat_fw_comp_req_t *)&pCookie->request; 799 800 if (DC_COMPRESSION_REQUEST == compDecomp) { 801 pReqCache = &(pSessionDesc->reqCacheComp); 802 } else { 803 pReqCache = &(pSessionDesc->reqCacheDecomp); 804 } 805 806 /* Fills the msg from the template cached in the session descriptor */ 807 memcpy((void *)pMsg, 808 (void *)(pReqCache), 809 LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES); 810 811 if (DC_REQUEST_FIRST == pSessionDesc->requestType) { 812 initial_adler = 1; 813 initial_crc32 = 0; 814 815 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) { 816 pSessionDesc->previousChecksum = 1; 817 } else { 818 pSessionDesc->previousChecksum = 0; 819 } 820 } else if (CPA_DC_STATELESS == pSessionDesc->sessState) { 821 pSessionDesc->previousChecksum = pResults->checksum; 822 823 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) { 824 initial_adler = pSessionDesc->previousChecksum; 825 } else { 826 initial_crc32 = pSessionDesc->previousChecksum; 827 } 828 } 829 830 /* Backup source and destination buffer addresses, 831 * CRC calculations both for CNV and translator overflow 832 * will be performed on them in the callback function. 833 */ 834 pCookie->pUserSrcBuff = pSrcBuff; 835 pCookie->pUserDestBuff = pDestBuff; 836 837 /* 838 * Due to implementation of CNV support and need for backwards 839 * compatibility certain fields in the request and response structs had 840 * been changed, moved or placed in unions cnvMode flag signifies fields 841 * to be selected from req/res 842 * 843 * Doing extended crc checks makes sense only when we want to do the 844 * actual CNV 845 */ 846 if (CPA_TRUE == pService->generic_service_info.integrityCrcCheck && 847 CPA_TRUE == integrityCrcCheck) { 848 pMsg->comp_pars.crc.crc_data_addr = 849 pSessionDesc->physDataIntegrityCrcs; 850 crcMode = ICP_QAT_FW_COMP_CRC_MODE_E2E; 851 } else { 852 /* Legacy request structure */ 853 pMsg->comp_pars.crc.legacy.initial_adler = initial_adler; 854 pMsg->comp_pars.crc.legacy.initial_crc32 = initial_crc32; 855 crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY; 856 } 857 858 /* Populate the cmdFlags */ 859 if (CPA_DC_STATEFUL == pSessionDesc->sessState) { 860 pSessionDesc->previousRequestType = pSessionDesc->requestType; 861 862 if (DC_REQUEST_FIRST == pSessionDesc->requestType) { 863 /* Update the request type for following requests */ 864 pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT; 865 866 /* Reinitialise the cumulative amount of consumed bytes 867 */ 868 pSessionDesc->cumulativeConsumedBytes = 0; 869 870 if (DC_COMPRESSION_REQUEST == compDecomp) { 871 pSessionDesc->isSopForCompressionProcessed = 872 CPA_TRUE; 873 } else if (DC_DECOMPRESSION_REQUEST == compDecomp) { 874 pSessionDesc->isSopForDecompressionProcessed = 875 CPA_TRUE; 876 } 877 } else { 878 if (DC_COMPRESSION_REQUEST == compDecomp) { 879 if (CPA_TRUE == 880 pSessionDesc 881 ->isSopForCompressionProcessed) { 882 sop = ICP_QAT_FW_COMP_NOT_SOP; 883 } else { 884 pSessionDesc 885 ->isSopForCompressionProcessed = 886 CPA_TRUE; 887 } 888 } else if (DC_DECOMPRESSION_REQUEST == compDecomp) { 889 if (CPA_TRUE == 890 pSessionDesc 891 ->isSopForDecompressionProcessed) { 892 sop = ICP_QAT_FW_COMP_NOT_SOP; 893 } else { 894 pSessionDesc 895 ->isSopForDecompressionProcessed = 896 CPA_TRUE; 897 } 898 } 899 } 900 901 if ((CPA_DC_FLUSH_FINAL == flush) || 902 (CPA_DC_FLUSH_FULL == flush)) { 903 /* Update the request type for following requests */ 904 pSessionDesc->requestType = DC_REQUEST_FIRST; 905 } else { 906 eop = ICP_QAT_FW_COMP_NOT_EOP; 907 } 908 } else { 909 910 if (DC_REQUEST_FIRST == pSessionDesc->requestType) { 911 /* Reinitialise the cumulative amount of consumed bytes 912 */ 913 pSessionDesc->cumulativeConsumedBytes = 0; 914 } 915 } 916 917 /* (LW 14 - 15) */ 918 pCompReqParams = &(pMsg->comp_pars); 919 dcCompRequestParamsPopulate(pCompReqParams, pCookie); 920 if (CPA_DC_FLUSH_FINAL == flush) { 921 bFinal = ICP_QAT_FW_COMP_BFINAL; 922 } 923 924 switch (cnvMode) { 925 case DC_CNVNR: 926 cnvRecovery = ICP_QAT_FW_COMP_CNV_RECOVERY; 927 /* Fall through is intended here, because for CNVNR 928 * cnvDecompReq also needs to be set */ 929 case DC_CNV: 930 cnvDecompReq = ICP_QAT_FW_COMP_CNV; 931 break; 932 case DC_NO_CNV: 933 cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 934 cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 935 break; 936 } 937 938 /* LW 18 */ 939 rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( 940 sop, eop, bFinal, cnvDecompReq, cnvRecovery, crcMode); 941 pMsg->comp_pars.req_par_flags = rpCmdFlags; 942 943 /* Populates the QAT common request middle part of the message 944 * (LW 6 to 11) */ 945 SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pMsg, 946 pCookie, 947 DC_DEFAULT_QAT_PTR_TYPE, 948 srcAddrPhys, 949 dstAddrPhys, 950 0, 951 0); 952 953 return CPA_STATUS_SUCCESS; 954 } 955 956 /** 957 ***************************************************************************** 958 * @ingroup Dc_DataCompression 959 * Send a compression request to QAT 960 * 961 * @description 962 * Send the requests for compression or decompression to QAT 963 * 964 * @param{in] pCookie Pointer to the compression cookie 965 * @param[in] pService Pointer to the compression service 966 * @param[in] pSessionDesc Pointer to the session descriptor 967 * @param[in] compDecomp Direction of the operation 968 * 969 * @retval CPA_STATUS_SUCCESS Function executed successfully 970 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 971 * 972 *****************************************************************************/ 973 static CpaStatus 974 dcSendRequest(dc_compression_cookie_t *pCookie, 975 sal_compression_service_t *pService, 976 dc_session_desc_t *pSessionDesc, 977 dc_request_dir_t compDecomp) 978 { 979 CpaStatus status = CPA_STATUS_SUCCESS; 980 981 /* Send to QAT */ 982 status = icp_adf_transPutMsg(pService->trans_handle_compression_tx, 983 (void *)&(pCookie->request), 984 LAC_QAT_DC_REQ_SZ_LW); 985 986 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) && 987 (CPA_STATUS_RETRY == status)) { 988 /* reset requestType after receiving an retry on 989 * the stateful request */ 990 pSessionDesc->requestType = pSessionDesc->previousRequestType; 991 } 992 993 return status; 994 } 995 996 /** 997 ***************************************************************************** 998 * @ingroup Dc_DataCompression 999 * Process the synchronous and asynchronous case for compression or 1000 * decompression 1001 * 1002 * @description 1003 * Process the synchronous and asynchronous case for compression or 1004 * decompression. This function will then create and send the request to 1005 * the firmware. 1006 * 1007 * @param[in] pService Pointer to the compression service 1008 * @param[in] pSessionDesc Pointer to the session descriptor 1009 * @param[in] dcInstance Instance handle derived from discovery 1010 * functions 1011 * @param[in] pSessionHandle Session handle 1012 * @param[in] numRequests Number of operations in the batch request 1013 * @param[in] pBatchOpData Address of the list of jobs to be processed 1014 * @param[in] pSrcBuff Pointer to data buffer for compression 1015 * @param[in] pDestBuff Pointer to buffer space for data after 1016 * compression 1017 * @param[in] pResults Pointer to results structure 1018 * @param[in] flushFlag Indicates the type of flush to be 1019 * performed 1020 * @param[in] pOpData Pointer to request information structure 1021 * holding parameters for cpaDcCompress2 and 1022 * CpaDcDecompressData2 1023 * @param[in] callbackTag Pointer to the callback tag 1024 * @param[in] compDecomp Direction of the operation 1025 * @param[in] isAsyncMode Used to know if synchronous or asynchronous 1026 * mode 1027 * @param[in] cnvMode CNV Mode 1028 * 1029 * @retval CPA_STATUS_SUCCESS Function executed successfully 1030 * @retval CPA_STATUS_RETRY Retry operation 1031 * @retval CPA_STATUS_FAIL Function failed 1032 * @retval CPA_STATUS_RESOURCE Resource error 1033 * 1034 *****************************************************************************/ 1035 static CpaStatus 1036 dcCompDecompData(sal_compression_service_t *pService, 1037 dc_session_desc_t *pSessionDesc, 1038 CpaInstanceHandle dcInstance, 1039 CpaDcSessionHandle pSessionHandle, 1040 CpaBufferList *pSrcBuff, 1041 CpaBufferList *pDestBuff, 1042 CpaDcRqResults *pResults, 1043 CpaDcFlush flushFlag, 1044 CpaDcOpData *pOpData, 1045 void *callbackTag, 1046 dc_request_dir_t compDecomp, 1047 CpaBoolean isAsyncMode, 1048 dc_cnv_mode_t cnvMode) 1049 { 1050 CpaStatus status = CPA_STATUS_SUCCESS; 1051 dc_compression_cookie_t *pCookie = NULL; 1052 1053 if ((LacSync_GenWakeupSyncCaller == pSessionDesc->pCompressionCb) && 1054 isAsyncMode == CPA_TRUE) { 1055 lac_sync_op_data_t *pSyncCallbackData = NULL; 1056 1057 status = LacSync_CreateSyncCookie(&pSyncCallbackData); 1058 1059 if (CPA_STATUS_SUCCESS == status) { 1060 status = dcCompDecompData(pService, 1061 pSessionDesc, 1062 dcInstance, 1063 pSessionHandle, 1064 pSrcBuff, 1065 pDestBuff, 1066 pResults, 1067 flushFlag, 1068 pOpData, 1069 pSyncCallbackData, 1070 compDecomp, 1071 CPA_FALSE, 1072 cnvMode); 1073 } else { 1074 return status; 1075 } 1076 1077 if (CPA_STATUS_SUCCESS == status) { 1078 CpaStatus syncStatus = CPA_STATUS_SUCCESS; 1079 1080 syncStatus = 1081 LacSync_WaitForCallback(pSyncCallbackData, 1082 DC_SYNC_CALLBACK_TIMEOUT, 1083 &status, 1084 NULL); 1085 1086 /* If callback doesn't come back */ 1087 if (CPA_STATUS_SUCCESS != syncStatus) { 1088 if (DC_COMPRESSION_REQUEST == compDecomp) { 1089 COMPRESSION_STAT_INC( 1090 numCompCompletedErrors, pService); 1091 } else { 1092 COMPRESSION_STAT_INC( 1093 numDecompCompletedErrors, pService); 1094 } 1095 LAC_LOG_ERROR("Callback timed out"); 1096 status = syncStatus; 1097 } 1098 } else { 1099 /* As the Request was not sent the Callback will never 1100 * be called, so need to indicate that we're finished 1101 * with cookie so it can be destroyed. */ 1102 LacSync_SetSyncCookieComplete(pSyncCallbackData); 1103 } 1104 1105 LacSync_DestroySyncCookie(&pSyncCallbackData); 1106 return status; 1107 } 1108 1109 /* Allocate the compression cookie 1110 * The memory is freed in callback or in sendRequest if an error occurs 1111 */ 1112 pCookie = (dc_compression_cookie_t *)Lac_MemPoolEntryAlloc( 1113 pService->compression_mem_pool); 1114 if (NULL == pCookie) { 1115 LAC_LOG_ERROR("Cannot get mem pool entry for compression"); 1116 status = CPA_STATUS_RESOURCE; 1117 } else if ((void *)CPA_STATUS_RETRY == pCookie) { 1118 pCookie = NULL; 1119 status = CPA_STATUS_RETRY; 1120 } 1121 1122 if (CPA_STATUS_SUCCESS == status) { 1123 status = dcCreateRequest(pCookie, 1124 pService, 1125 pSessionDesc, 1126 pSessionHandle, 1127 pSrcBuff, 1128 pDestBuff, 1129 pResults, 1130 flushFlag, 1131 pOpData, 1132 callbackTag, 1133 compDecomp, 1134 cnvMode); 1135 } 1136 1137 if (CPA_STATUS_SUCCESS == status) { 1138 /* Increment number of pending callbacks for session */ 1139 if (CPA_DC_STATELESS == pSessionDesc->sessState) { 1140 qatUtilsAtomicInc( 1141 &(pSessionDesc->pendingStatelessCbCount)); 1142 } 1143 status = 1144 dcSendRequest(pCookie, pService, pSessionDesc, compDecomp); 1145 } 1146 1147 if (CPA_STATUS_SUCCESS == status) { 1148 if (DC_COMPRESSION_REQUEST == compDecomp) { 1149 COMPRESSION_STAT_INC(numCompRequests, pService); 1150 } else { 1151 COMPRESSION_STAT_INC(numDecompRequests, pService); 1152 } 1153 } else { 1154 if (DC_COMPRESSION_REQUEST == compDecomp) { 1155 COMPRESSION_STAT_INC(numCompRequestsErrors, pService); 1156 } else { 1157 COMPRESSION_STAT_INC(numDecompRequestsErrors, pService); 1158 } 1159 1160 /* Decrement number of pending callbacks for session */ 1161 if (CPA_DC_STATELESS == pSessionDesc->sessState) { 1162 qatUtilsAtomicDec( 1163 &(pSessionDesc->pendingStatelessCbCount)); 1164 } else { 1165 qatUtilsAtomicDec( 1166 &(pSessionDesc->pendingStatefulCbCount)); 1167 } 1168 1169 /* Free the memory pool */ 1170 if (NULL != pCookie) { 1171 if (status != CPA_STATUS_UNSUPPORTED) { 1172 /* Free the memory pool */ 1173 Lac_MemPoolEntryFree(pCookie); 1174 pCookie = NULL; 1175 } 1176 } 1177 } 1178 1179 return status; 1180 } 1181 1182 /** 1183 ***************************************************************************** 1184 * @ingroup Dc_DataCompression 1185 * Handle zero length compression or decompression requests 1186 * 1187 * @description 1188 * Handle zero length compression or decompression requests 1189 * 1190 * @param[in] pService Pointer to the compression service 1191 * @param[in] pSessionDesc Pointer to the session descriptor 1192 * @param[in] pResults Pointer to results structure 1193 * @param[in] flushFlag Indicates the type of flush to be 1194 * performed 1195 * @param[in] callbackTag User supplied value to help correlate 1196 * the callback with its associated request 1197 * @param[in] compDecomp Direction of the operation 1198 * 1199 * @retval CPA_TRUE Zero length SOP or MOP processed 1200 * @retval CPA_FALSE Zero length EOP 1201 * 1202 *****************************************************************************/ 1203 static CpaStatus 1204 dcZeroLengthRequests(sal_compression_service_t *pService, 1205 dc_session_desc_t *pSessionDesc, 1206 CpaDcRqResults *pResults, 1207 CpaDcFlush flushFlag, 1208 void *callbackTag, 1209 dc_request_dir_t compDecomp) 1210 { 1211 CpaBoolean status = CPA_FALSE; 1212 CpaDcCallbackFn pCbFunc = pSessionDesc->pCompressionCb; 1213 1214 if (DC_REQUEST_FIRST == pSessionDesc->requestType) { 1215 /* Reinitialise the cumulative amount of consumed bytes */ 1216 pSessionDesc->cumulativeConsumedBytes = 0; 1217 1218 /* Zero length SOP */ 1219 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) { 1220 pResults->checksum = 1; 1221 } else { 1222 pResults->checksum = 0; 1223 } 1224 1225 status = CPA_TRUE; 1226 } else if ((CPA_DC_FLUSH_NONE == flushFlag) || 1227 (CPA_DC_FLUSH_SYNC == flushFlag)) { 1228 /* Zero length MOP */ 1229 pResults->checksum = pSessionDesc->previousChecksum; 1230 status = CPA_TRUE; 1231 } 1232 1233 if (CPA_TRUE == status) { 1234 pResults->status = CPA_DC_OK; 1235 pResults->produced = 0; 1236 pResults->consumed = 0; 1237 1238 /* Increment statistics */ 1239 if (DC_COMPRESSION_REQUEST == compDecomp) { 1240 COMPRESSION_STAT_INC(numCompRequests, pService); 1241 COMPRESSION_STAT_INC(numCompCompleted, pService); 1242 } else { 1243 COMPRESSION_STAT_INC(numDecompRequests, pService); 1244 COMPRESSION_STAT_INC(numDecompCompleted, pService); 1245 } 1246 1247 if (CPA_STATUS_SUCCESS != 1248 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { 1249 LAC_LOG_ERROR("Cannot unlock session lock"); 1250 } 1251 1252 if ((NULL != pCbFunc) && 1253 (LacSync_GenWakeupSyncCaller != pCbFunc)) { 1254 pCbFunc(callbackTag, CPA_STATUS_SUCCESS); 1255 } 1256 1257 return CPA_TRUE; 1258 } 1259 1260 return CPA_FALSE; 1261 } 1262 1263 static CpaStatus 1264 dcParamCheck(CpaInstanceHandle dcInstance, 1265 CpaDcSessionHandle pSessionHandle, 1266 sal_compression_service_t *pService, 1267 CpaBufferList *pSrcBuff, 1268 CpaBufferList *pDestBuff, 1269 CpaDcRqResults *pResults, 1270 dc_session_desc_t *pSessionDesc, 1271 CpaDcFlush flushFlag, 1272 Cpa64U srcBuffSize) 1273 { 1274 1275 if (dcCheckSourceData(pSessionHandle, 1276 pSrcBuff, 1277 pDestBuff, 1278 pResults, 1279 flushFlag, 1280 srcBuffSize, 1281 NULL) != CPA_STATUS_SUCCESS) { 1282 return CPA_STATUS_INVALID_PARAM; 1283 } 1284 if (dcCheckDestinationData( 1285 pService, pSessionHandle, pDestBuff, DC_COMPRESSION_REQUEST) != 1286 CPA_STATUS_SUCCESS) { 1287 return CPA_STATUS_INVALID_PARAM; 1288 } 1289 if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) { 1290 LAC_INVALID_PARAM_LOG("Invalid sessDirection value"); 1291 return CPA_STATUS_INVALID_PARAM; 1292 } 1293 return CPA_STATUS_SUCCESS; 1294 } 1295 1296 CpaStatus 1297 cpaDcCompressData(CpaInstanceHandle dcInstance, 1298 CpaDcSessionHandle pSessionHandle, 1299 CpaBufferList *pSrcBuff, 1300 CpaBufferList *pDestBuff, 1301 CpaDcRqResults *pResults, 1302 CpaDcFlush flushFlag, 1303 void *callbackTag) 1304 { 1305 sal_compression_service_t *pService = NULL; 1306 dc_session_desc_t *pSessionDesc = NULL; 1307 CpaInstanceHandle insHandle = NULL; 1308 Cpa64U srcBuffSize = 0; 1309 1310 1311 if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { 1312 insHandle = dcGetFirstHandle(); 1313 } else { 1314 insHandle = dcInstance; 1315 } 1316 1317 pService = (sal_compression_service_t *)insHandle; 1318 1319 LAC_CHECK_NULL_PARAM(insHandle); 1320 LAC_CHECK_NULL_PARAM(pSessionHandle); 1321 1322 /* Check if SAL is initialised otherwise return an error */ 1323 SAL_RUNNING_CHECK(insHandle); 1324 1325 /* This check is outside the parameter checking as it is needed to 1326 * manage zero length requests */ 1327 if (LacBuffDesc_BufferListVerifyNull(pSrcBuff, 1328 &srcBuffSize, 1329 LAC_NO_ALIGNMENT_SHIFT) != 1330 CPA_STATUS_SUCCESS) { 1331 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter"); 1332 return CPA_STATUS_INVALID_PARAM; 1333 } 1334 1335 /* Ensure this is a compression instance */ 1336 SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION); 1337 1338 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 1339 if (CPA_STATUS_SUCCESS != 1340 dcParamCheck(insHandle, 1341 pSessionHandle, 1342 pService, 1343 pSrcBuff, 1344 pDestBuff, 1345 pResults, 1346 pSessionDesc, 1347 flushFlag, 1348 srcBuffSize)) { 1349 return CPA_STATUS_INVALID_PARAM; 1350 } 1351 if (CPA_DC_STATEFUL == pSessionDesc->sessState) { 1352 LAC_INVALID_PARAM_LOG( 1353 "Invalid session state, stateful sessions " 1354 "are not supported"); 1355 return CPA_STATUS_UNSUPPORTED; 1356 } 1357 1358 if (!(pService->generic_service_info.dcExtendedFeatures & 1359 DC_CNV_EXTENDED_CAPABILITY)) { 1360 LAC_INVALID_PARAM_LOG( 1361 "CompressAndVerify feature not supported"); 1362 return CPA_STATUS_UNSUPPORTED; 1363 } 1364 1365 if (!(pService->generic_service_info.dcExtendedFeatures & 1366 DC_CNVNR_EXTENDED_CAPABILITY)) { 1367 LAC_INVALID_PARAM_LOG( 1368 "CompressAndVerifyAndRecovery feature not supported"); 1369 return CPA_STATUS_UNSUPPORTED; 1370 } 1371 1372 return dcCompDecompData(pService, 1373 pSessionDesc, 1374 dcInstance, 1375 pSessionHandle, 1376 pSrcBuff, 1377 pDestBuff, 1378 pResults, 1379 flushFlag, 1380 NULL, 1381 callbackTag, 1382 DC_COMPRESSION_REQUEST, 1383 CPA_TRUE, 1384 DC_CNVNR); 1385 } 1386 1387 CpaStatus 1388 cpaDcCompressData2(CpaInstanceHandle dcInstance, 1389 CpaDcSessionHandle pSessionHandle, 1390 CpaBufferList *pSrcBuff, 1391 CpaBufferList *pDestBuff, 1392 CpaDcOpData *pOpData, 1393 CpaDcRqResults *pResults, 1394 void *callbackTag) 1395 { 1396 sal_compression_service_t *pService = NULL; 1397 dc_session_desc_t *pSessionDesc = NULL; 1398 CpaInstanceHandle insHandle = NULL; 1399 Cpa64U srcBuffSize = 0; 1400 dc_cnv_mode_t cnvMode = DC_NO_CNV; 1401 1402 LAC_CHECK_NULL_PARAM(pOpData); 1403 1404 if (((CPA_TRUE != pOpData->compressAndVerify) && 1405 (CPA_FALSE != pOpData->compressAndVerify)) || 1406 ((CPA_FALSE != pOpData->compressAndVerifyAndRecover) && 1407 (CPA_TRUE != pOpData->compressAndVerifyAndRecover))) { 1408 return CPA_STATUS_INVALID_PARAM; 1409 } 1410 1411 if ((CPA_FALSE == pOpData->compressAndVerify) && 1412 (CPA_TRUE == pOpData->compressAndVerifyAndRecover)) { 1413 return CPA_STATUS_INVALID_PARAM; 1414 } 1415 1416 1417 if ((CPA_TRUE == pOpData->compressAndVerify) && 1418 (CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 1419 (CPA_FALSE == pOpData->integrityCrcCheck)) { 1420 return cpaDcCompressData(dcInstance, 1421 pSessionHandle, 1422 pSrcBuff, 1423 pDestBuff, 1424 pResults, 1425 pOpData->flushFlag, 1426 callbackTag); 1427 } 1428 1429 if (CPA_FALSE == pOpData->compressAndVerify) { 1430 LAC_INVALID_PARAM_LOG( 1431 "Data compression without verification not allowed"); 1432 return CPA_STATUS_UNSUPPORTED; 1433 } 1434 1435 1436 if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { 1437 insHandle = dcGetFirstHandle(); 1438 } else { 1439 insHandle = dcInstance; 1440 } 1441 1442 pService = (sal_compression_service_t *)insHandle; 1443 1444 LAC_CHECK_NULL_PARAM(insHandle); 1445 LAC_CHECK_NULL_PARAM(pSessionHandle); 1446 LAC_CHECK_NULL_PARAM(pOpData); 1447 1448 /* Check if SAL is initialised otherwise return an error */ 1449 SAL_RUNNING_CHECK(insHandle); 1450 1451 /* This check is outside the parameter checking as it is needed to 1452 * manage zero length requests */ 1453 if (LacBuffDesc_BufferListVerifyNull(pSrcBuff, 1454 &srcBuffSize, 1455 LAC_NO_ALIGNMENT_SHIFT) != 1456 CPA_STATUS_SUCCESS) { 1457 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter"); 1458 return CPA_STATUS_INVALID_PARAM; 1459 } 1460 1461 /* Ensure this is a compression instance */ 1462 SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION); 1463 1464 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 1465 1466 if (CPA_TRUE == pOpData->compressAndVerify && 1467 CPA_DC_STATEFUL == pSessionDesc->sessState) { 1468 LAC_INVALID_PARAM_LOG( 1469 "Invalid session state, stateful sessions " 1470 "not supported with CNV"); 1471 return CPA_STATUS_UNSUPPORTED; 1472 } 1473 1474 if (!(pService->generic_service_info.dcExtendedFeatures & 1475 DC_CNV_EXTENDED_CAPABILITY) && 1476 (CPA_TRUE == pOpData->compressAndVerify)) { 1477 LAC_INVALID_PARAM_LOG( 1478 "CompressAndVerify feature not supported"); 1479 return CPA_STATUS_UNSUPPORTED; 1480 } 1481 1482 if (CPA_STATUS_SUCCESS != 1483 dcParamCheck(insHandle, 1484 pSessionHandle, 1485 pService, 1486 pSrcBuff, 1487 pDestBuff, 1488 pResults, 1489 pSessionDesc, 1490 pOpData->flushFlag, 1491 srcBuffSize)) { 1492 return CPA_STATUS_INVALID_PARAM; 1493 } 1494 if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) { 1495 return CPA_STATUS_INVALID_PARAM; 1496 } 1497 if (CPA_TRUE != pOpData->compressAndVerify) { 1498 if (srcBuffSize > DC_COMP_MAX_BUFF_SIZE) { 1499 LAC_LOG_ERROR( 1500 "Compression payload greater than 64KB is " 1501 "unsupported, when CnV is disabled\n"); 1502 return CPA_STATUS_UNSUPPORTED; 1503 } 1504 } 1505 1506 if (CPA_DC_STATEFUL == pSessionDesc->sessState) { 1507 /* Lock the session to check if there are in-flight stateful 1508 * requests */ 1509 if (CPA_STATUS_SUCCESS != 1510 LAC_SPINLOCK(&(pSessionDesc->sessionLock))) { 1511 LAC_LOG_ERROR("Cannot unlock session lock"); 1512 } 1513 1514 /* Check if there is already one in-flight stateful request */ 1515 if (0 != 1516 qatUtilsAtomicGet( 1517 &(pSessionDesc->pendingStatefulCbCount))) { 1518 LAC_LOG_ERROR( 1519 "Only one in-flight stateful request supported"); 1520 if (CPA_STATUS_SUCCESS != 1521 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { 1522 LAC_LOG_ERROR("Cannot unlock session lock"); 1523 } 1524 return CPA_STATUS_RETRY; 1525 } 1526 1527 if (0 == srcBuffSize) { 1528 if (CPA_TRUE == 1529 dcZeroLengthRequests(pService, 1530 pSessionDesc, 1531 pResults, 1532 pOpData->flushFlag, 1533 callbackTag, 1534 DC_COMPRESSION_REQUEST)) { 1535 return CPA_STATUS_SUCCESS; 1536 } 1537 } 1538 1539 qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount)); 1540 if (CPA_STATUS_SUCCESS != 1541 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { 1542 LAC_LOG_ERROR("Cannot unlock session lock"); 1543 } 1544 } 1545 1546 if (CPA_TRUE == pOpData->compressAndVerify) { 1547 cnvMode = DC_CNV; 1548 } 1549 1550 return dcCompDecompData(pService, 1551 pSessionDesc, 1552 dcInstance, 1553 pSessionHandle, 1554 pSrcBuff, 1555 pDestBuff, 1556 pResults, 1557 pOpData->flushFlag, 1558 pOpData, 1559 callbackTag, 1560 DC_COMPRESSION_REQUEST, 1561 CPA_TRUE, 1562 cnvMode); 1563 } 1564 1565 static CpaStatus 1566 dcDecompressDataCheck(CpaInstanceHandle insHandle, 1567 CpaDcSessionHandle pSessionHandle, 1568 CpaBufferList *pSrcBuff, 1569 CpaBufferList *pDestBuff, 1570 CpaDcRqResults *pResults, 1571 CpaDcFlush flushFlag, 1572 Cpa64U *srcBufferSize) 1573 { 1574 sal_compression_service_t *pService = NULL; 1575 dc_session_desc_t *pSessionDesc = NULL; 1576 Cpa64U srcBuffSize = 0; 1577 1578 pService = (sal_compression_service_t *)insHandle; 1579 1580 LAC_CHECK_NULL_PARAM(insHandle); 1581 1582 /* Check if SAL is initialised otherwise return an error */ 1583 SAL_RUNNING_CHECK(insHandle); 1584 1585 /* This check is outside the parameter checking as it is needed to 1586 * manage zero length requests */ 1587 if (LacBuffDesc_BufferListVerifyNull(pSrcBuff, 1588 &srcBuffSize, 1589 LAC_NO_ALIGNMENT_SHIFT) != 1590 CPA_STATUS_SUCCESS) { 1591 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter"); 1592 return CPA_STATUS_INVALID_PARAM; 1593 } 1594 1595 /* Ensure this is a compression instance */ 1596 SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION); 1597 1598 if (dcCheckSourceData(pSessionHandle, 1599 pSrcBuff, 1600 pDestBuff, 1601 pResults, 1602 flushFlag, 1603 srcBuffSize, 1604 NULL) != CPA_STATUS_SUCCESS) { 1605 return CPA_STATUS_INVALID_PARAM; 1606 } 1607 if (dcCheckDestinationData(pService, 1608 pSessionHandle, 1609 pDestBuff, 1610 DC_DECOMPRESSION_REQUEST) != 1611 CPA_STATUS_SUCCESS) { 1612 return CPA_STATUS_INVALID_PARAM; 1613 } 1614 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 1615 1616 if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) { 1617 LAC_INVALID_PARAM_LOG("Invalid sessDirection value"); 1618 return CPA_STATUS_INVALID_PARAM; 1619 } 1620 1621 1622 *srcBufferSize = srcBuffSize; 1623 1624 return CPA_STATUS_SUCCESS; 1625 } 1626 1627 CpaStatus 1628 cpaDcDecompressData(CpaInstanceHandle dcInstance, 1629 CpaDcSessionHandle pSessionHandle, 1630 CpaBufferList *pSrcBuff, 1631 CpaBufferList *pDestBuff, 1632 CpaDcRqResults *pResults, 1633 CpaDcFlush flushFlag, 1634 void *callbackTag) 1635 { 1636 sal_compression_service_t *pService = NULL; 1637 dc_session_desc_t *pSessionDesc = NULL; 1638 CpaInstanceHandle insHandle = NULL; 1639 Cpa64U srcBuffSize = 0; 1640 CpaStatus status = CPA_STATUS_SUCCESS; 1641 1642 1643 if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { 1644 insHandle = dcGetFirstHandle(); 1645 } else { 1646 insHandle = dcInstance; 1647 } 1648 1649 status = dcDecompressDataCheck(insHandle, 1650 pSessionHandle, 1651 pSrcBuff, 1652 pDestBuff, 1653 pResults, 1654 flushFlag, 1655 &srcBuffSize); 1656 if (CPA_STATUS_SUCCESS != status) { 1657 return status; 1658 } 1659 1660 pService = (sal_compression_service_t *)insHandle; 1661 1662 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 1663 1664 if (CPA_DC_STATEFUL == pSessionDesc->sessState) { 1665 /* Lock the session to check if there are in-flight stateful 1666 * requests */ 1667 if (CPA_STATUS_SUCCESS != 1668 LAC_SPINLOCK(&(pSessionDesc->sessionLock))) { 1669 LAC_LOG_ERROR("Cannot lock session lock"); 1670 return CPA_STATUS_RESOURCE; 1671 } 1672 1673 /* Check if there is already one in-flight stateful request */ 1674 if (0 != 1675 qatUtilsAtomicGet( 1676 &(pSessionDesc->pendingStatefulCbCount))) { 1677 LAC_LOG_ERROR( 1678 "Only one in-flight stateful request supported"); 1679 if (CPA_STATUS_SUCCESS != 1680 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { 1681 LAC_LOG_ERROR("Cannot unlock session lock"); 1682 } 1683 return CPA_STATUS_RETRY; 1684 } 1685 1686 if ((0 == srcBuffSize) || 1687 ((1 == srcBuffSize) && (CPA_DC_FLUSH_FINAL != flushFlag) && 1688 (CPA_DC_FLUSH_FULL != flushFlag))) { 1689 if (CPA_TRUE == 1690 dcZeroLengthRequests(pService, 1691 pSessionDesc, 1692 pResults, 1693 flushFlag, 1694 callbackTag, 1695 DC_DECOMPRESSION_REQUEST)) { 1696 return CPA_STATUS_SUCCESS; 1697 } 1698 } 1699 1700 qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount)); 1701 if (CPA_STATUS_SUCCESS != 1702 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { 1703 LAC_LOG_ERROR("Cannot unlock session lock"); 1704 } 1705 } 1706 1707 return dcCompDecompData(pService, 1708 pSessionDesc, 1709 dcInstance, 1710 pSessionHandle, 1711 pSrcBuff, 1712 pDestBuff, 1713 pResults, 1714 flushFlag, 1715 NULL, 1716 callbackTag, 1717 DC_DECOMPRESSION_REQUEST, 1718 CPA_TRUE, 1719 DC_NO_CNV); 1720 } 1721 1722 CpaStatus 1723 cpaDcDecompressData2(CpaInstanceHandle dcInstance, 1724 CpaDcSessionHandle pSessionHandle, 1725 CpaBufferList *pSrcBuff, 1726 CpaBufferList *pDestBuff, 1727 CpaDcOpData *pOpData, 1728 CpaDcRqResults *pResults, 1729 void *callbackTag) 1730 { 1731 sal_compression_service_t *pService = NULL; 1732 dc_session_desc_t *pSessionDesc = NULL; 1733 CpaInstanceHandle insHandle = NULL; 1734 CpaStatus status = CPA_STATUS_SUCCESS; 1735 Cpa64U srcBuffSize = 0; 1736 LAC_CHECK_NULL_PARAM(pOpData); 1737 1738 if (CPA_FALSE == pOpData->integrityCrcCheck) { 1739 1740 return cpaDcDecompressData(dcInstance, 1741 pSessionHandle, 1742 pSrcBuff, 1743 pDestBuff, 1744 pResults, 1745 pOpData->flushFlag, 1746 callbackTag); 1747 } 1748 1749 1750 if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { 1751 insHandle = dcGetFirstHandle(); 1752 } else { 1753 insHandle = dcInstance; 1754 } 1755 1756 status = dcDecompressDataCheck(insHandle, 1757 pSessionHandle, 1758 pSrcBuff, 1759 pDestBuff, 1760 pResults, 1761 pOpData->flushFlag, 1762 &srcBuffSize); 1763 if (CPA_STATUS_SUCCESS != status) { 1764 return status; 1765 } 1766 1767 pService = (sal_compression_service_t *)insHandle; 1768 1769 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 1770 1771 if (CPA_DC_STATEFUL == pSessionDesc->sessState) { 1772 LAC_INVALID_PARAM_LOG("Invalid session: Stateful session is " 1773 "not supported"); 1774 return CPA_STATUS_INVALID_PARAM; 1775 } 1776 1777 return dcCompDecompData(pService, 1778 pSessionDesc, 1779 insHandle, 1780 pSessionHandle, 1781 pSrcBuff, 1782 pDestBuff, 1783 pResults, 1784 pOpData->flushFlag, 1785 pOpData, 1786 callbackTag, 1787 DC_DECOMPRESSION_REQUEST, 1788 CPA_TRUE, 1789 DC_NO_CNV); 1790 } 1791