1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3
4 /**
5 ***************************************************************************
6 * @file lac_sym_cb.c Callback handler functions for symmetric components
7 *
8 * @ingroup LacSym
9 *
10 ***************************************************************************/
11
12 /*
13 *******************************************************************************
14 * Include public/global header files
15 *******************************************************************************
16 */
17
18 #include "cpa.h"
19 #include "cpa_cy_sym.h"
20
21 #include "icp_accel_devices.h"
22 #include "icp_adf_init.h"
23 #include "icp_qat_fw_la.h"
24 #include "icp_adf_transport.h"
25 #include "icp_adf_debug.h"
26
27 #include "lac_sym.h"
28 #include "lac_sym_cipher.h"
29 #include "lac_common.h"
30 #include "lac_list.h"
31 #include "lac_sal_types_crypto.h"
32 #include "lac_sal.h"
33 #include "lac_sal_ctrl.h"
34 #include "lac_session.h"
35 #include "lac_sym_stats.h"
36 #include "lac_log.h"
37 #include "lac_sym_cb.h"
38 #include "lac_sym_hash.h"
39 #include "lac_sym_qat_cipher.h"
40 #include "lac_sym_qat.h"
41
42 #define DEQUEUE_MSGPUT_MAX_RETRIES 10000
43
44 /*
45 *******************************************************************************
46 * Define static function definitions
47 *******************************************************************************
48 */
49
50 /**
51 *****************************************************************************
52 * @ingroup LacSymCb
53 * Function to clean computed data.
54 *
55 * @description
56 * This function cleans GCM or CCM data in the case of a failure.
57 *
58 * @param[in] pSessionDesc pointer to the session descriptor
59 * @param[out] pBufferList pointer to the bufferlist to clean
60 * @param[in] pOpData pointer to operation data
61 * @param[in] isCCM is it a CCM operation boolean
62 *
63 * @return None
64 *****************************************************************************/
65 static void
LacSymCb_CleanUserData(const lac_session_desc_t * pSessionDesc,CpaBufferList * pBufferList,const CpaCySymOpData * pOpData,CpaBoolean isCCM)66 LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc,
67 CpaBufferList *pBufferList,
68 const CpaCySymOpData *pOpData,
69 CpaBoolean isCCM)
70 {
71 Cpa32U authTagLen = 0;
72
73 /* Retrieve authTagLen */
74 authTagLen = pSessionDesc->hashResultSize;
75
76 /* Cleaning */
77 if (isCCM) {
78 /* for CCM the digest is inside the buffer list */
79 LacBuffDesc_BufferListZeroFromOffset(
80 pBufferList,
81 pOpData->cryptoStartSrcOffsetInBytes,
82 pOpData->messageLenToCipherInBytes + authTagLen);
83 } else {
84 /* clean buffer list */
85 LacBuffDesc_BufferListZeroFromOffset(
86 pBufferList,
87 pOpData->cryptoStartSrcOffsetInBytes,
88 pOpData->messageLenToCipherInBytes);
89 }
90 if ((CPA_TRUE != pSessionDesc->digestIsAppended) &&
91 (NULL != pOpData->pDigestResult)) {
92 /* clean digest */
93 memset(pOpData->pDigestResult, 0, authTagLen);
94 }
95 }
96
97 /**
98 *****************************************************************************
99 * @ingroup LacSymCb
100 * Definition of callback function for processing symmetric responses
101 *
102 * @description
103 * This callback is invoked to process symmetric response messages from
104 * the QAT. It will extract some details from the message and invoke
105 * the user's callback to complete a symmetric operation.
106 *
107 * @param[in] pCookie Pointer to cookie associated with this request
108 * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT
109 * @param[in] status Status variable indicating an error occurred
110 * in sending the message (e.g. when dequeueing)
111 * @param[in] pSessionDesc Session descriptor
112 *
113 * @return None
114 *****************************************************************************/
115 static void
LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t * pCookie,CpaBoolean qatRespStatusOkFlag,CpaStatus status,lac_session_desc_t * pSessionDesc)116 LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie,
117 CpaBoolean qatRespStatusOkFlag,
118 CpaStatus status,
119 lac_session_desc_t *pSessionDesc)
120 {
121 CpaCySymCbFunc pSymCb = NULL;
122 void *pCallbackTag = NULL;
123 CpaCySymOpData *pOpData = NULL;
124 CpaBufferList *pDstBuffer = NULL;
125 CpaCySymOp operationType = CPA_CY_SYM_OP_NONE;
126 CpaStatus dequeueStatus = CPA_STATUS_SUCCESS;
127
128 CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE;
129 /* NOTE: cookie pointer validated in previous function */
130 instanceHandle = pCookie->instanceHandle;
131
132 pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData);
133 operationType = pSessionDesc->symOperation;
134
135 /* Set the destination pointer to the one supplied in the cookie. */
136 pDstBuffer = pCookie->pDstBuffer;
137
138 /* For a digest verify operation - for full packet and final partial
139 * only, perform a comparison with the digest generated and with the one
140 * supplied in the packet. In case of AES_GCM in SPC mode, destination
141 * buffer needs to be cleared if digest verify operation fails */
142
143 if (((SPC == pSessionDesc->singlePassState) ||
144 (CPA_CY_SYM_OP_CIPHER != operationType)) &&
145 (CPA_TRUE == pSessionDesc->digestVerify) &&
146 ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) ||
147 (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) {
148 if (CPA_FALSE == qatRespStatusOkFlag) {
149 LAC_SYM_STAT_INC(numSymOpVerifyFailures,
150 instanceHandle);
151
152 /* The comparison has failed at this point (status is
153 * fail), need to clean any sensitive calculated data up
154 * to this point. The data calculated is no longer
155 * useful to the end result and does not need to be
156 * returned to the user so setting buffers to zero.
157 */
158 if (pSessionDesc->cipherAlgorithm ==
159 CPA_CY_SYM_CIPHER_AES_CCM) {
160 LacSymCb_CleanUserData(pSessionDesc,
161 pDstBuffer,
162 pOpData,
163 CPA_TRUE);
164 } else if (pSessionDesc->cipherAlgorithm ==
165 CPA_CY_SYM_CIPHER_AES_GCM) {
166 LacSymCb_CleanUserData(pSessionDesc,
167 pDstBuffer,
168 pOpData,
169 CPA_FALSE);
170 }
171 }
172 } else {
173 /* Most commands have no point of failure and always return
174 * success. This is the default response from the QAT.
175 * If status is already set to an error value, don't overwrite
176 * it
177 */
178 if ((CPA_STATUS_SUCCESS == status) &&
179 (CPA_TRUE != qatRespStatusOkFlag)) {
180 LAC_LOG_ERROR("Response status value not as expected");
181 status = CPA_STATUS_FAIL;
182 }
183 }
184
185 pSymCb = pSessionDesc->pSymCb;
186 pCallbackTag = pCookie->pCallbackTag;
187
188 /* State returned to the client for intermediate partials packets
189 * for hash only and cipher only partial packets. Cipher update
190 * allow next partial through */
191 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) {
192 if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
193 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
194 if (CPA_TRUE == pCookie->updateUserIvOnRecieve) {
195 /* Update the user's IV buffer
196 * Very important to do this BEFORE dequeuing
197 * subsequent partial requests, as the state
198 * buffer may get overwritten
199 */
200 memcpy(pCookie->pOpData->pIv,
201 pSessionDesc->cipherPartialOpState,
202 pCookie->pOpData->ivLenInBytes);
203 }
204 if (CPA_TRUE == pCookie->updateKeySizeOnRecieve &&
205 LAC_CIPHER_IS_XTS_MODE(
206 pSessionDesc->cipherAlgorithm)) {
207 LacSymQat_CipherXTSModeUpdateKeyLen(
208 pSessionDesc,
209 pSessionDesc->cipherKeyLenInBytes / 2);
210 }
211 }
212 } else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) {
213 if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
214 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
215 if (CPA_TRUE ==
216 LAC_CIPHER_IS_XTS_MODE(
217 pSessionDesc->cipherAlgorithm)) {
218 /*
219 * For XTS mode, we replace the updated key with
220 * the original key - for subsequent partial
221 * requests
222 *
223 */
224 LacSymQat_CipherXTSModeUpdateKeyLen(
225 pSessionDesc,
226 pSessionDesc->cipherKeyLenInBytes);
227 }
228 }
229 }
230
231 if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) &&
232 (qatRespStatusOkFlag != CPA_FALSE)) {
233 /* There may be requests blocked pending the completion of this
234 * operation
235 */
236
237 dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc);
238 if (CPA_STATUS_SUCCESS != dequeueStatus) {
239 LAC_SYM_STAT_INC(numSymOpCompletedErrors,
240 instanceHandle);
241 qatRespStatusOkFlag = CPA_FALSE;
242 if (CPA_STATUS_SUCCESS == status) {
243 status = dequeueStatus;
244 }
245 }
246 }
247
248 if (CPA_STATUS_SUCCESS == status) {
249 /* update stats */
250 if (pSessionDesc->internalSession == CPA_FALSE) {
251 LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle);
252 if (CPA_STATUS_SUCCESS != status) {
253 LAC_SYM_STAT_INC(numSymOpCompletedErrors,
254 instanceHandle);
255 }
256 }
257 }
258
259 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
260
261 /* deallocate the memory for the internal callback cookie */
262 Lac_MemPoolEntryFree(pCookie);
263
264 /* user callback function is the last thing to be called */
265 pSymCb(pCallbackTag,
266 status,
267 operationType,
268 pOpData,
269 pDstBuffer,
270 qatRespStatusOkFlag);
271 }
272
273 /**
274 ******************************************************************************
275 * @ingroup LacSymCb
276 * Definition of callback function for processing symmetric Data Plane
277 * responses
278 *
279 * @description
280 * This callback checks the status, decrements the number of operations
281 * pending and calls the user callback
282 *
283 * @param[in/out] pResponse pointer to the response structure
284 * @param[in] qatRespStatusOkFlag status
285 * @param[in] pSessionDesc pointer to the session descriptor
286 *
287 * @return None
288 ******************************************************************************/
289 static void
LacSymCb_ProcessDpCallback(CpaCySymDpOpData * pResponse,CpaBoolean qatRespStatusOkFlag,CpaStatus status,lac_session_desc_t * pSessionDesc)290 LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse,
291 CpaBoolean qatRespStatusOkFlag,
292 CpaStatus status,
293 lac_session_desc_t *pSessionDesc)
294 {
295 CpaCySymDpCbFunc pSymDpCb = NULL;
296
297 /* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be
298 * cleaned as stated in RFC 3610; in DP mode, it is the user
299 * responsability to do so */
300
301 if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) &&
302 SPC != pSessionDesc->singlePassState) ||
303 (CPA_FALSE == pSessionDesc->digestVerify)) {
304 /* If not doing digest compare and qatRespStatusOkFlag !=
305 CPA_TRUE then there is something very wrong */
306 if ((CPA_FALSE == qatRespStatusOkFlag) &&
307 (status != CPA_STATUS_UNSUPPORTED)) {
308 LAC_LOG_ERROR("Response status value not as expected");
309 status = CPA_STATUS_FAIL;
310 }
311 }
312
313 pSymDpCb =
314 ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb;
315
316 pSymDpCb(pResponse, status, qatRespStatusOkFlag);
317
318 /*
319 * Decrement the number of pending CB.
320 *
321 * If the @pendingDpCbCount becomes zero, we may remove the session,
322 * please read more information in the cpaCySymRemoveSession().
323 *
324 * But there is a field in the @pResponse to store the session,
325 * the "sessionCtx". In another word, in the above @->pSymDpCb()
326 * callback, it may use the session again. If we decrease the
327 * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the
328 * @->pSymDpCb() may reference to a deleted session.
329 *
330 * So in order to avoid the risk, we decrease the @pendingDpCbCount
331 * after the @->pSymDpCb() callback.
332 */
333 qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount);
334 }
335
336 /**
337 ******************************************************************************
338 * @ingroup LacSymCb
339 * Definition of callback function for processing symmetric responses
340 *
341 * @description
342 * This callback, which is registered with the common symmetric response
343 * message handler, is invoked to process symmetric response messages from
344 * the QAT. It will extract the response status from the cmnRespFlags set
345 * by the QAT, and then will pass it to @ref
346 * LacSymCb_ProcessCallbackInternal to complete the response processing.
347 *
348 * @param[in] lacCmdId ID of the symmetric QAT command of the request
349 * message
350 * @param[in] pOpaqueData pointer to opaque data in the request message
351 * @param[in] cmnRespFlags Flags set by QAT to indicate response status
352 *
353 * @return None
354 ******************************************************************************/
355 static void
LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId,void * pOpaqueData,icp_qat_fw_comn_flags cmnRespFlags)356 LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId,
357 void *pOpaqueData,
358 icp_qat_fw_comn_flags cmnRespFlags)
359 {
360 CpaStatus status = CPA_STATUS_SUCCESS;
361 CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData;
362 lac_session_desc_t *pSessionDesc =
363 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx);
364 CpaBoolean qatRespStatusOkFlag =
365 (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
366 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags));
367
368 if (CPA_TRUE == pSessionDesc->isDPSession) {
369 /* DP session */
370 if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(
371 cmnRespFlags)) {
372 status = CPA_STATUS_UNSUPPORTED;
373 }
374 LacSymCb_ProcessDpCallback(pDpOpData,
375 qatRespStatusOkFlag,
376 status,
377 pSessionDesc);
378 } else {
379 /* Trad session */
380 LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *)
381 pOpaqueData,
382 qatRespStatusOkFlag,
383 CPA_STATUS_SUCCESS,
384 pSessionDesc);
385 }
386 }
387
388 /*
389 *******************************************************************************
390 * Define public/global function definitions
391 *******************************************************************************
392 */
393
394 /**
395 * @ingroup LacSymCb
396 *
397 * @return CpaStatus
398 * value returned will be the result of icp_adf_transPutMsg
399 */
400 CpaStatus
LacSymCb_PendingReqsDequeue(lac_session_desc_t * pSessionDesc)401 LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc)
402 {
403 CpaStatus status = CPA_STATUS_SUCCESS;
404 sal_crypto_service_t *pService = NULL;
405 Cpa32U retries = 0;
406
407 pService = (sal_crypto_service_t *)pSessionDesc->pInstance;
408
409 /* Need to protect access to queue head and tail pointers, which may
410 * be accessed by multiple contexts simultaneously for enqueue and
411 * dequeue operations
412 */
413 LAC_SPINLOCK(&pSessionDesc->requestQueueLock);
414
415 /* Clear the blocking flag in the session descriptor */
416 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
417
418 while ((NULL != pSessionDesc->pRequestQueueHead) &&
419 (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) {
420
421 /* If we send a partial packet request, set the
422 * blockingOpsInProgress flag for the session to indicate that
423 * subsequent requests must be queued up until this request
424 * completes
425 */
426 if (CPA_CY_SYM_PACKET_TYPE_FULL !=
427 pSessionDesc->pRequestQueueHead->pOpData->packetType) {
428 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
429 }
430
431 /* At this point, we're clear to send the request. For cipher
432 * requests, we need to check if the session IV needs to be
433 * updated. This can only be done when no other partials are in
434 * flight for this session, to ensure the cipherPartialOpState
435 * buffer in the session descriptor is not currently in use
436 */
437 if (CPA_TRUE ==
438 pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) {
439 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
440 memcpy(pSessionDesc->cipherPartialOpState,
441 pSessionDesc->cipherARC4InitialState,
442 LAC_CIPHER_ARC4_STATE_LEN_BYTES);
443 } else {
444 memcpy(pSessionDesc->cipherPartialOpState,
445 pSessionDesc->pRequestQueueHead->pOpData
446 ->pIv,
447 pSessionDesc->pRequestQueueHead->pOpData
448 ->ivLenInBytes);
449 }
450 }
451
452 /*
453 * Now we'll attempt to send the message directly to QAT. We'll
454 * keep looing until it succeeds (or at least a very high number
455 * of retries), as the failure only happens when the ring is
456 * full, and this is only a temporary situation. After a few
457 * retries, space will become availble, allowing the putMsg to
458 * succeed.
459 */
460 retries = 0;
461 do {
462 /* Send to QAT */
463 status = icp_adf_transPutMsg(
464 pService->trans_handle_sym_tx,
465 (void *)&(pSessionDesc->pRequestQueueHead->qatMsg),
466 LAC_QAT_SYM_REQ_SZ_LW);
467
468 retries++;
469 /*
470 * Yield to allow other threads that may be on this
471 * session to poll and make some space on the ring
472 */
473 if (CPA_STATUS_SUCCESS != status) {
474 qatUtilsYield();
475 }
476 } while ((CPA_STATUS_SUCCESS != status) &&
477 (retries < DEQUEUE_MSGPUT_MAX_RETRIES));
478
479 if ((CPA_STATUS_SUCCESS != status) ||
480 (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) {
481 LAC_LOG_ERROR(
482 "Failed to SalQatMsg_transPutMsg, maximum retries exceeded.");
483 goto cleanup;
484 }
485
486 pSessionDesc->pRequestQueueHead =
487 pSessionDesc->pRequestQueueHead->pNext;
488 }
489
490 /* If we've drained the queue, ensure the tail pointer is set to NULL */
491 if (NULL == pSessionDesc->pRequestQueueHead) {
492 pSessionDesc->pRequestQueueTail = NULL;
493 }
494
495 cleanup:
496 LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock);
497 return status;
498 }
499
500 /**
501 * @ingroup LacSymCb
502 */
503 void
LacSymCb_CallbacksRegister(void)504 LacSymCb_CallbacksRegister(void)
505 {
506 /*** HASH ***/
507 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH,
508 LacSymCb_ProcessCallback);
509
510 /*** ALGORITHM-CHAINING CIPHER_HASH***/
511 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH,
512 LacSymCb_ProcessCallback);
513
514 /*** ALGORITHM-CHAINING HASH_CIPHER***/
515 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER,
516 LacSymCb_ProcessCallback);
517
518 /*** CIPHER ***/
519 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER,
520 LacSymCb_ProcessCallback);
521
522 /* Call compile time param check function to ensure it is included
523 in the build by the compiler - this compile time check
524 ensures callbacks run as expected */
525 LacSym_CompileTimeAssertions();
526 }
527