1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 ***************************************************************************** 6 * @file dc_dp.c 7 * 8 * @defgroup cpaDcDp Data Compression Data Plane API 9 * 10 * @ingroup cpaDcDp 11 * 12 * @description 13 * Implementation of the Data Compression DP operations. 14 * 15 *****************************************************************************/ 16 17 /* 18 ******************************************************************************* 19 * Include public/global header files 20 ******************************************************************************* 21 */ 22 #include "cpa.h" 23 #include "cpa_dc.h" 24 #include "cpa_dc_dp.h" 25 26 #include "icp_qat_fw_comp.h" 27 28 /* 29 ******************************************************************************* 30 * Include private header files 31 ******************************************************************************* 32 */ 33 #include "dc_session.h" 34 #include "dc_datapath.h" 35 #include "lac_common.h" 36 #include "lac_mem.h" 37 #include "lac_mem_pools.h" 38 #include "sal_types_compression.h" 39 #include "lac_sal.h" 40 #include "lac_sync.h" 41 #include "sal_service_state.h" 42 #include "sal_qat_cmn_msg.h" 43 #include "icp_sal_poll.h" 44 45 /** 46 ***************************************************************************** 47 * @ingroup cpaDcDp 48 * Check that pOpData is valid 49 * 50 * @description 51 * Check that all the parameters defined in the pOpData are valid 52 * 53 * @param[in] pOpData Pointer to a structure containing the 54 * request parameters 55 * 56 * @retval CPA_STATUS_SUCCESS Function executed successfully 57 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 58 * 59 *****************************************************************************/ 60 static CpaStatus 61 dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) 62 { 63 sal_compression_service_t *pService = NULL; 64 dc_session_desc_t *pSessionDesc = NULL; 65 66 LAC_CHECK_NULL_PARAM(pOpData); 67 LAC_CHECK_NULL_PARAM(pOpData->dcInstance); 68 LAC_CHECK_NULL_PARAM(pOpData->pSessionHandle); 69 70 /* Ensure this is a compression instance */ 71 SAL_CHECK_INSTANCE_TYPE(pOpData->dcInstance, 72 SAL_SERVICE_TYPE_COMPRESSION); 73 74 pService = (sal_compression_service_t *)(pOpData->dcInstance); 75 76 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 77 if (NULL == pSessionDesc) { 78 QAT_UTILS_LOG("Session handle not as expected.\n"); 79 return CPA_STATUS_INVALID_PARAM; 80 } 81 82 if (CPA_FALSE == pSessionDesc->isDcDp) { 83 QAT_UTILS_LOG("The session type should be data plane.\n"); 84 return CPA_STATUS_INVALID_PARAM; 85 } 86 87 /* Compressing zero byte is not supported */ 88 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) && 89 (0 == pOpData->bufferLenToCompress)) { 90 QAT_UTILS_LOG( 91 "The source buffer length to compress needs to be greater than zero byte.\n"); 92 return CPA_STATUS_INVALID_PARAM; 93 } 94 95 if (pOpData->sessDirection > CPA_DC_DIR_DECOMPRESS) { 96 QAT_UTILS_LOG("Invalid direction of operation.\n"); 97 return CPA_STATUS_INVALID_PARAM; 98 } 99 100 if (0 == pOpData->srcBuffer) { 101 QAT_UTILS_LOG("Invalid srcBuffer\n"); 102 return CPA_STATUS_INVALID_PARAM; 103 } 104 if (0 == pOpData->destBuffer) { 105 QAT_UTILS_LOG("Invalid destBuffer\n"); 106 return CPA_STATUS_INVALID_PARAM; 107 } 108 if (pOpData->srcBuffer == pOpData->destBuffer) { 109 QAT_UTILS_LOG("In place operation is not supported.\n"); 110 return CPA_STATUS_INVALID_PARAM; 111 } 112 if (0 == pOpData->thisPhys) { 113 QAT_UTILS_LOG("Invalid thisPhys\n"); 114 return CPA_STATUS_INVALID_PARAM; 115 } 116 117 if ((CPA_TRUE != pOpData->compressAndVerify) && 118 (CPA_FALSE != pOpData->compressAndVerify)) { 119 QAT_UTILS_LOG("Invalid compressAndVerify\n"); 120 return CPA_STATUS_INVALID_PARAM; 121 } 122 if ((CPA_TRUE == pOpData->compressAndVerify) && 123 !(pService->generic_service_info.dcExtendedFeatures & 124 DC_CNV_EXTENDED_CAPABILITY)) { 125 QAT_UTILS_LOG("Invalid compressAndVerify, no CNV capability\n"); 126 return CPA_STATUS_UNSUPPORTED; 127 } 128 if ((CPA_TRUE != pOpData->compressAndVerifyAndRecover) && 129 (CPA_FALSE != pOpData->compressAndVerifyAndRecover)) { 130 QAT_UTILS_LOG("Invalid compressAndVerifyAndRecover\n"); 131 return CPA_STATUS_INVALID_PARAM; 132 } 133 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 134 (CPA_FALSE == pOpData->compressAndVerify)) { 135 QAT_UTILS_LOG("CnVnR option set without setting CnV\n"); 136 return CPA_STATUS_INVALID_PARAM; 137 } 138 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 139 !(pService->generic_service_info.dcExtendedFeatures & 140 DC_CNVNR_EXTENDED_CAPABILITY)) { 141 QAT_UTILS_LOG( 142 "Invalid CnVnR option set and no CnVnR capability.\n"); 143 return CPA_STATUS_UNSUPPORTED; 144 } 145 146 if ((CPA_DP_BUFLIST == pOpData->srcBufferLen) && 147 (CPA_DP_BUFLIST != pOpData->destBufferLen)) { 148 QAT_UTILS_LOG( 149 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 150 return CPA_STATUS_INVALID_PARAM; 151 } 152 if ((CPA_DP_BUFLIST != pOpData->srcBufferLen) && 153 (CPA_DP_BUFLIST == pOpData->destBufferLen)) { 154 QAT_UTILS_LOG( 155 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 156 return CPA_STATUS_INVALID_PARAM; 157 } 158 159 if (CPA_DP_BUFLIST != pOpData->srcBufferLen) { 160 if (pOpData->srcBufferLen < pOpData->bufferLenToCompress) { 161 QAT_UTILS_LOG( 162 "srcBufferLen is smaller than bufferLenToCompress.\n"); 163 return CPA_STATUS_INVALID_PARAM; 164 } 165 166 if (pOpData->destBufferLen < pOpData->bufferLenForData) { 167 QAT_UTILS_LOG( 168 "destBufferLen is smaller than bufferLenForData.\n"); 169 return CPA_STATUS_INVALID_PARAM; 170 } 171 } else { 172 /* We are assuming that there is enough memory in the source and 173 * destination buffer lists. We only receive physical addresses 174 * of the 175 * buffers so we are unable to test it here */ 176 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer); 177 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer); 178 } 179 180 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->thisPhys); 181 182 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) || 183 (CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) { 184 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { 185 /* Check if Intermediate Buffer Array pointer is NULL */ 186 if ((0 == pService->pInterBuffPtrsArrayPhyAddr) || 187 (NULL == pService->pInterBuffPtrsArray)) { 188 QAT_UTILS_LOG( 189 "No intermediate buffer defined for this instance - see cpaDcStartInstance.\n"); 190 return CPA_STATUS_INVALID_PARAM; 191 } 192 193 /* Ensure that the destination buffer length for data is 194 * greater 195 * or equal to 128B */ 196 if (pOpData->bufferLenForData < 197 DC_DEST_BUFFER_DYN_MIN_SIZE) { 198 QAT_UTILS_LOG( 199 "Destination buffer length for data should be greater or equal to 128B.\n"); 200 return CPA_STATUS_INVALID_PARAM; 201 } 202 } else { 203 /* Ensure that the destination buffer length for data is 204 * greater 205 * or equal to min output buffsize */ 206 if (pOpData->bufferLenForData < 207 pService->comp_device_data.minOutputBuffSize) { 208 QAT_UTILS_LOG( 209 "Destination buffer size should be greater or equal to %d bytes.\n", 210 pService->comp_device_data 211 .minOutputBuffSize); 212 return CPA_STATUS_INVALID_PARAM; 213 } 214 } 215 } 216 217 return CPA_STATUS_SUCCESS; 218 } 219 220 CpaStatus 221 cpaDcDpGetSessionSize(CpaInstanceHandle dcInstance, 222 CpaDcSessionSetupData *pSessionData, 223 Cpa32U *pSessionSize) 224 { 225 return dcGetSessionSize(dcInstance, pSessionData, pSessionSize, NULL); 226 } 227 228 CpaStatus 229 cpaDcDpInitSession(CpaInstanceHandle dcInstance, 230 CpaDcSessionHandle pSessionHandle, 231 CpaDcSessionSetupData *pSessionData) 232 { 233 CpaStatus status = CPA_STATUS_SUCCESS; 234 dc_session_desc_t *pSessionDesc = NULL; 235 sal_compression_service_t *pService = NULL; 236 237 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 238 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 239 240 pService = (sal_compression_service_t *)dcInstance; 241 242 /* Check if SAL is initialised otherwise return an error */ 243 SAL_RUNNING_CHECK(pService); 244 245 /* Stateful is not supported */ 246 if (CPA_DC_STATELESS != pSessionData->sessState) { 247 QAT_UTILS_LOG("Invalid sessState value\n"); 248 return CPA_STATUS_INVALID_PARAM; 249 } 250 251 status = 252 dcInitSession(dcInstance, pSessionHandle, pSessionData, NULL, NULL); 253 if (CPA_STATUS_SUCCESS == status) { 254 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 255 pSessionDesc->isDcDp = CPA_TRUE; 256 257 ICP_QAT_FW_COMN_PTR_TYPE_SET( 258 pSessionDesc->reqCacheDecomp.comn_hdr.comn_req_flags, 259 DC_DP_QAT_PTR_TYPE); 260 ICP_QAT_FW_COMN_PTR_TYPE_SET( 261 pSessionDesc->reqCacheComp.comn_hdr.comn_req_flags, 262 DC_DP_QAT_PTR_TYPE); 263 } 264 265 return status; 266 } 267 268 CpaStatus 269 cpaDcDpRemoveSession(const CpaInstanceHandle dcInstance, 270 CpaDcSessionHandle pSessionHandle) 271 { 272 return cpaDcRemoveSession(dcInstance, pSessionHandle); 273 } 274 275 CpaStatus 276 cpaDcDpRegCbFunc(const CpaInstanceHandle dcInstance, 277 const CpaDcDpCallbackFn pNewCb) 278 { 279 sal_compression_service_t *pService = NULL; 280 281 LAC_CHECK_NULL_PARAM(dcInstance); 282 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 283 LAC_CHECK_NULL_PARAM(pNewCb); 284 285 /* Check if SAL is initialised otherwise return an error */ 286 SAL_RUNNING_CHECK(dcInstance); 287 288 pService = (sal_compression_service_t *)dcInstance; 289 pService->pDcDpCb = pNewCb; 290 291 return CPA_STATUS_SUCCESS; 292 } 293 294 /** 295 ***************************************************************************** 296 * @ingroup cpaDcDp 297 * 298 * @description 299 * Writes the message to the ring 300 * 301 * @param[in] pOpData Pointer to a structure containing the 302 * request parameters 303 * @param[in] pCurrentQatMsg Pointer to current QAT message on the ring 304 * 305 *****************************************************************************/ 306 static void 307 dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) 308 { 309 icp_qat_fw_comp_req_t *pReqCache = NULL; 310 dc_session_desc_t *pSessionDesc = NULL; 311 Cpa8U bufferFormat; 312 313 Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 314 Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 315 316 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 317 318 if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) { 319 pReqCache = &(pSessionDesc->reqCacheComp); 320 /* CNV check */ 321 if (CPA_TRUE == pOpData->compressAndVerify) { 322 cnvDecompReq = ICP_QAT_FW_COMP_CNV; 323 /* CNVNR check */ 324 if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) { 325 cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY; 326 } 327 } 328 } else { 329 pReqCache = &(pSessionDesc->reqCacheDecomp); 330 } 331 332 /* Fills in the template DC ET ring message - cached from the 333 * session descriptor */ 334 memcpy((void *)pCurrentQatMsg, 335 (void *)(pReqCache), 336 (LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES)); 337 338 if (CPA_DP_BUFLIST == pOpData->srcBufferLen) { 339 bufferFormat = QAT_COMN_PTR_TYPE_SGL; 340 } else { 341 bufferFormat = QAT_COMN_PTR_TYPE_FLAT; 342 } 343 344 pCurrentQatMsg->comp_pars.req_par_flags |= 345 ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( 346 0, 0, 0, cnvDecompReq, cnvnrCompReq, 0); 347 348 SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg, 349 pOpData, 350 bufferFormat, 351 pOpData->srcBuffer, 352 pOpData->destBuffer, 353 pOpData->srcBufferLen, 354 pOpData->destBufferLen); 355 356 pCurrentQatMsg->comp_pars.comp_len = pOpData->bufferLenToCompress; 357 pCurrentQatMsg->comp_pars.out_buffer_sz = pOpData->bufferLenForData; 358 } 359 360 CpaStatus 361 cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) 362 { 363 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 364 icp_comms_trans_handle trans_handle = NULL; 365 dc_session_desc_t *pSessionDesc = NULL; 366 CpaStatus status = CPA_STATUS_SUCCESS; 367 368 status = dcDataPlaneParamCheck(pOpData); 369 if (CPA_STATUS_SUCCESS != status) { 370 return status; 371 } 372 373 if ((CPA_FALSE == pOpData->compressAndVerify) && 374 (CPA_DC_DIR_COMPRESS == pOpData->sessDirection)) { 375 return CPA_STATUS_UNSUPPORTED; 376 } 377 378 /* Check if SAL is initialised otherwise return an error */ 379 SAL_RUNNING_CHECK(pOpData->dcInstance); 380 381 trans_handle = ((sal_compression_service_t *)pOpData->dcInstance) 382 ->trans_handle_compression_tx; 383 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 384 385 if ((CPA_DC_DIR_COMPRESS == pOpData->sessDirection) && 386 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 387 QAT_UTILS_LOG( 388 "The session does not support this direction of operation.\n"); 389 return CPA_STATUS_INVALID_PARAM; 390 } else if ((CPA_DC_DIR_DECOMPRESS == pOpData->sessDirection) && 391 (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection)) { 392 QAT_UTILS_LOG( 393 "The session does not support this direction of operation.\n"); 394 return CPA_STATUS_INVALID_PARAM; 395 } 396 397 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 398 if (NULL == pCurrentQatMsg) { 399 return CPA_STATUS_RETRY; 400 } 401 402 dcDpWriteRingMsg(pOpData, pCurrentQatMsg); 403 pSessionDesc->pendingDpStatelessCbCount++; 404 405 if (CPA_TRUE == performOpNow) { 406 SalQatMsg_updateQueueTail(trans_handle); 407 } 408 409 return CPA_STATUS_SUCCESS; 410 } 411 412 CpaStatus 413 cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, 414 CpaDcDpOpData *pOpData[], 415 const CpaBoolean performOpNow) 416 { 417 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 418 icp_comms_trans_handle trans_handle = NULL; 419 dc_session_desc_t *pSessionDesc = NULL; 420 Cpa32U i = 0; 421 CpaStatus status = CPA_STATUS_SUCCESS; 422 sal_compression_service_t *pService = NULL; 423 424 LAC_CHECK_NULL_PARAM(pOpData); 425 LAC_CHECK_NULL_PARAM(pOpData[0]); 426 LAC_CHECK_NULL_PARAM(pOpData[0]->dcInstance); 427 428 pService = (sal_compression_service_t *)(pOpData[0]->dcInstance); 429 if ((numberRequests == 0) || 430 (numberRequests > pService->maxNumCompConcurrentReq)) { 431 QAT_UTILS_LOG( 432 "The number of requests needs to be between 1 and %d.\n", 433 pService->maxNumCompConcurrentReq); 434 return CPA_STATUS_INVALID_PARAM; 435 } 436 437 for (i = 0; i < numberRequests; i++) { 438 status = dcDataPlaneParamCheck(pOpData[i]); 439 if (CPA_STATUS_SUCCESS != status) { 440 return status; 441 } 442 443 /* Check that all instance handles and session handles are the 444 * same */ 445 if (pOpData[i]->dcInstance != pOpData[0]->dcInstance) { 446 QAT_UTILS_LOG( 447 "All instance handles should be the same in the pOpData.\n"); 448 return CPA_STATUS_INVALID_PARAM; 449 } 450 451 if (pOpData[i]->pSessionHandle != pOpData[0]->pSessionHandle) { 452 QAT_UTILS_LOG( 453 "All session handles should be the same in the pOpData.\n"); 454 return CPA_STATUS_INVALID_PARAM; 455 } 456 } 457 458 for (i = 0; i < numberRequests; i++) { 459 if ((CPA_FALSE == pOpData[i]->compressAndVerify) && 460 (CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection)) { 461 return CPA_STATUS_UNSUPPORTED; 462 } 463 } 464 465 /* Check if SAL is initialised otherwise return an error */ 466 SAL_RUNNING_CHECK(pOpData[0]->dcInstance); 467 468 trans_handle = ((sal_compression_service_t *)pOpData[0]->dcInstance) 469 ->trans_handle_compression_tx; 470 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData[0]->pSessionHandle); 471 472 for (i = 0; i < numberRequests; i++) { 473 if ((CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection) && 474 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 475 QAT_UTILS_LOG( 476 "The session does not support this direction of operation.\n"); 477 return CPA_STATUS_INVALID_PARAM; 478 } else if ((CPA_DC_DIR_DECOMPRESS == 479 pOpData[i]->sessDirection) && 480 (CPA_DC_DIR_COMPRESS == 481 pSessionDesc->sessDirection)) { 482 QAT_UTILS_LOG( 483 "The session does not support this direction of operation.\n"); 484 return CPA_STATUS_INVALID_PARAM; 485 } 486 } 487 488 icp_adf_getQueueMemory(trans_handle, 489 numberRequests, 490 (void **)&pCurrentQatMsg); 491 if (NULL == pCurrentQatMsg) { 492 return CPA_STATUS_RETRY; 493 } 494 495 for (i = 0; i < numberRequests; i++) { 496 dcDpWriteRingMsg(pOpData[i], pCurrentQatMsg); 497 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 498 } 499 500 pSessionDesc->pendingDpStatelessCbCount += numberRequests; 501 502 if (CPA_TRUE == performOpNow) { 503 SalQatMsg_updateQueueTail(trans_handle); 504 } 505 506 return CPA_STATUS_SUCCESS; 507 } 508 509 CpaStatus 510 icp_sal_DcPollDpInstance(CpaInstanceHandle dcInstance, Cpa32U responseQuota) 511 { 512 icp_comms_trans_handle trans_handle = NULL; 513 514 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 515 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 516 517 /* Check if SAL is initialised otherwise return an error */ 518 SAL_RUNNING_CHECK(dcInstance); 519 520 trans_handle = ((sal_compression_service_t *)dcInstance) 521 ->trans_handle_compression_rx; 522 523 return icp_adf_pollQueue(trans_handle, responseQuota); 524 } 525 526 CpaStatus 527 cpaDcDpPerformOpNow(CpaInstanceHandle dcInstance) 528 { 529 icp_comms_trans_handle trans_handle = NULL; 530 531 LAC_CHECK_NULL_PARAM(dcInstance); 532 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 533 534 /* Check if SAL is initialised otherwise return an error */ 535 SAL_RUNNING_CHECK(dcInstance); 536 537 trans_handle = ((sal_compression_service_t *)dcInstance) 538 ->trans_handle_compression_tx; 539 540 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 541 SalQatMsg_updateQueueTail(trans_handle); 542 } 543 544 return CPA_STATUS_SUCCESS; 545 } 546