1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3
4 /**
5 ***************************************************************************
6 * @file lac_sym_alg_chain.c Algorithm Chaining Perform
7 *
8 * @ingroup LacAlgChain
9 ***************************************************************************/
10
11 /*
12 *******************************************************************************
13 * Include public/global header files
14 *******************************************************************************
15 */
16
17 #include "cpa.h"
18 #include "cpa_cy_sym.h"
19
20 #include "icp_accel_devices.h"
21 #include "icp_adf_init.h"
22 #include "icp_adf_transport.h"
23 #include "icp_adf_debug.h"
24
25 /*
26 *******************************************************************************
27 * Include private header files
28 *******************************************************************************
29 */
30
31 #include "lac_mem.h"
32 #include "lac_log.h"
33 #include "lac_sym.h"
34 #include "lac_list.h"
35 #include "icp_qat_fw_la.h"
36 #include "lac_sal_types_crypto.h"
37 #include "lac_sal.h"
38 #include "lac_sal_ctrl.h"
39 #include "lac_sym_alg_chain.h"
40 #include "lac_sym_cipher.h"
41 #include "lac_sym_cipher_defs.h"
42 #include "lac_sym_hash.h"
43 #include "lac_sym_hash_defs.h"
44 #include "lac_sym_qat_cipher.h"
45 #include "lac_sym_qat_hash.h"
46 #include "lac_sym_stats.h"
47 #include "lac_sym_queue.h"
48 #include "lac_sym_cb.h"
49 #include "sal_string_parse.h"
50 #include "lac_sym_auth_enc.h"
51 #include "lac_sym_qat.h"
52 #include "sal_hw_gen.h"
53
54 /**
55 * @ingroup LacAlgChain
56 * This callback function will be invoked whenever a hash precompute
57 * operation completes. It will dequeue and send any QAT requests
58 * which were queued up while the precompute was in progress.
59 *
60 * @param[in] callbackTag Opaque value provided by user. This will
61 * be a pointer to the session descriptor.
62 *
63 * @retval
64 * None
65 *
66 */
67 static void
LacSymAlgChain_HashPrecomputeDoneCb(void * callbackTag)68 LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag)
69 {
70 LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag);
71 }
72
73 /**
74 * @ingroup LacAlgChain
75 * Walk the buffer list and find the address for the given offset within
76 * a buffer.
77 *
78 * @param[in] pBufferList Buffer List
79 * @param[in] packetOffset Offset in the buffer list for which address
80 * is to be found.
81 * @param[out] ppDataPtr This is where the sought pointer will be put
82 * @param[out] pSpaceLeft Pointer to a variable in which information about
83 * available space from the given offset to the end
84 * of the flat buffer it is located in will be returned
85 *
86 * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list
87 * @retval CPA_STATUS_FAIL Address with a given offset not found in the list.
88 *
89 */
90 static CpaStatus
LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList * pBufferList,const Cpa32U packetOffset,Cpa8U ** ppDataPtr)91 LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList,
92 const Cpa32U packetOffset,
93 Cpa8U **ppDataPtr)
94 {
95 Cpa32U currentOffset = 0;
96 Cpa32U i = 0;
97
98 for (i = 0; i < pBufferList->numBuffers; i++) {
99 Cpa8U *pCurrData = pBufferList->pBuffers[i].pData;
100 Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes;
101
102 /* If the offset is within the address space of the current
103 * buffer */
104 if ((packetOffset >= currentOffset) &&
105 (packetOffset < (currentOffset + currDataSize))) {
106 /* increment by offset of the address in the current
107 * buffer */
108 *ppDataPtr = pCurrData + (packetOffset - currentOffset);
109 return CPA_STATUS_SUCCESS;
110 }
111
112 /* Increment by the size of the buffer */
113 currentOffset += currDataSize;
114 }
115
116 return CPA_STATUS_FAIL;
117 }
118
119 /**
120 * @ingroup LacAlgChain
121 * Function which checks for support of partial packets for symmetric
122 * crypto operations
123 *
124 * @param[in] pService Pointer to service descriptor
125 * @param[in/out] pSessionDesc Pointer to session descriptor
126 *
127 */
128 static void
LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)129 LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,
130 lac_session_desc_t *pSessionDesc)
131 {
132 CpaBoolean isHashPartialSupported = CPA_FALSE;
133 CpaBoolean isCipherPartialSupported = CPA_FALSE;
134 CpaBoolean isPartialSupported = CPA_FALSE;
135
136 switch (pSessionDesc->cipherAlgorithm) {
137 /* Following ciphers don't support partial */
138 case CPA_CY_SYM_CIPHER_KASUMI_F8:
139 case CPA_CY_SYM_CIPHER_AES_F8:
140 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
141 case CPA_CY_SYM_CIPHER_CHACHA:
142 case CPA_CY_SYM_CIPHER_ZUC_EEA3:
143 break;
144 /* All others support partial */
145 default:
146 isCipherPartialSupported = CPA_TRUE;
147 break;
148 }
149 switch (pSessionDesc->hashAlgorithm) {
150 /* Following hash don't support partial */
151 case CPA_CY_SYM_HASH_KASUMI_F9:
152 case CPA_CY_SYM_HASH_SNOW3G_UIA2:
153 case CPA_CY_SYM_HASH_POLY:
154 case CPA_CY_SYM_HASH_ZUC_EIA3:
155 break;
156 /* Following hash may support partial based on device capabilities */
157 case CPA_CY_SYM_HASH_SHA3_256:
158 if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) {
159 isHashPartialSupported = CPA_TRUE;
160 }
161 break;
162 /* All others support partial */
163 default:
164 isHashPartialSupported = CPA_TRUE;
165 break;
166 }
167 switch (pSessionDesc->symOperation) {
168 case CPA_CY_SYM_OP_CIPHER:
169 isPartialSupported = isCipherPartialSupported;
170 break;
171 case CPA_CY_SYM_OP_HASH:
172 isPartialSupported = isHashPartialSupported;
173 break;
174 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
175 if (isCipherPartialSupported && isHashPartialSupported) {
176 isPartialSupported = CPA_TRUE;
177 }
178 break;
179 case CPA_CY_SYM_OP_NONE:
180 break;
181 default:
182 break;
183 }
184
185 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
186 /* UCS slice has no support for state flush and
187 * because of that is not able to do partial processing */
188 isPartialSupported = CPA_FALSE;
189 }
190
191 pSessionDesc->isPartialSupported = isPartialSupported;
192 }
193
194 static void
LacAlgChain_CipherCDBuild_ForOptimisedCD(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)195 LacAlgChain_CipherCDBuild_ForOptimisedCD(
196 const CpaCySymCipherSetupData *pCipherData,
197 lac_session_desc_t *pSessionDesc,
198 icp_qat_fw_slice_t nextSlice,
199 Cpa8U cipherOffsetInConstantsTable,
200 Cpa8U *pOptimisedHwBlockBaseInDRAM,
201 Cpa32U *pOptimisedHwBlockOffsetInDRAM)
202 {
203 Cpa8U *pCipherKeyField = NULL;
204 Cpa32U sizeInBytes = 0;
205 pCipherKeyField = pOptimisedHwBlockBaseInDRAM;
206
207 /* Need to build up the alternative CD for SHRAM Constants Table use
208 * with an optimised content desc of 64 bytes for this case. Cipher key
209 * will be in the Content desc in DRAM, The cipher config data
210 * is now in the SHRAM constants table. */
211
212 LacSymQat_CipherHwBlockPopulateKeySetup(
213 pSessionDesc,
214 pCipherData,
215 pCipherData->cipherKeyLenInBytes,
216 pSessionDesc->cipherSliceType,
217 pCipherKeyField,
218 &sizeInBytes);
219
220 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
221 pSessionDesc->cipherAlgorithm,
222 pSessionDesc->cipherKeyLenInBytes,
223 pSessionDesc->cipherSliceType,
224 nextSlice,
225 cipherOffsetInConstantsTable);
226
227 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
228 }
229
230 static void
LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable)231 LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData,
232 lac_session_desc_t *pSessionDesc,
233 icp_qat_fw_slice_t nextSlice,
234 Cpa8U cipherOffsetInConstantsTable)
235 {
236 Cpa32U sizeInBytes = 0;
237 Cpa8U *pCipherKeyField = NULL;
238 /* Need to build up the alternative CD for SHRAM Constants Table use
239 * Cipher key will be in the Request, The cipher config data is now in
240 * the SHRAM constants table. And nothing is now stored in the content
241 * desc */
242 pCipherKeyField = (Cpa8U *)&(
243 pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields);
244
245 LacSymQat_CipherHwBlockPopulateKeySetup(
246 pSessionDesc,
247 pCipherData,
248 pCipherData->cipherKeyLenInBytes,
249 pSessionDesc->cipherSliceType,
250 pCipherKeyField,
251 &sizeInBytes);
252
253 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
254 pSessionDesc->cipherAlgorithm,
255 pSessionDesc->cipherKeyLenInBytes,
256 pSessionDesc->cipherSliceType,
257 nextSlice,
258 cipherOffsetInConstantsTable);
259 }
260
261 static void
LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa32U capabilitiesMask)262 LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData,
263 lac_session_desc_t *pSessionDesc,
264 icp_qat_fw_slice_t nextSlice,
265 Cpa8U cipherOffsetInConstantsTable,
266 icp_qat_fw_comn_flags *pCmnRequestFlags,
267 icp_qat_fw_serv_specif_flags *pLaCmdFlags,
268 Cpa8U *pHwBlockBaseInDRAM,
269 Cpa32U *pHwBlockOffsetInDRAM,
270 Cpa32U capabilitiesMask)
271 {
272 Cpa8U *pCipherKeyField = NULL;
273 Cpa8U cipherOffsetInReqQW = 0;
274 Cpa32U sizeInBytes = 0;
275 void *pCfgData = NULL;
276 Cpa32U cfgOffset = 0;
277
278 /* Construct the ContentDescriptor in DRAM */
279 cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES);
280 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
281 *pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP);
282
283 /* construct cipherConfig in CD in DRAM */
284 cfgOffset = *pHwBlockOffsetInDRAM;
285 pCfgData = pHwBlockBaseInDRAM + cfgOffset;
286 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
287 pCfgData,
288 &sizeInBytes);
289
290 ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags,
291 pSessionDesc->cipherSliceType);
292
293 *pHwBlockOffsetInDRAM += sizeInBytes;
294
295 /* Cipher key will be in CD in DRAM.
296 * The Request contains a ptr to the CD.
297 * This ptr will be copied into the request later once the CD is
298 * fully constructed, but the flag is set here. */
299 pCipherKeyField = pHwBlockBaseInDRAM + *pHwBlockOffsetInDRAM;
300 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(*pCmnRequestFlags,
301 QAT_COMN_CD_FLD_TYPE_64BIT_ADR);
302
303 LacSymQat_CipherHwBlockPopulateKeySetup(
304 pSessionDesc,
305 pCipherData,
306 pCipherData->cipherKeyLenInBytes,
307 pSessionDesc->cipherSliceType,
308 pCipherKeyField,
309 &sizeInBytes);
310 /* update offset */
311 *pHwBlockOffsetInDRAM += sizeInBytes;
312
313 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr),
314 pSessionDesc->cipherAlgorithm,
315 pSessionDesc->cipherKeyLenInBytes,
316 pSessionDesc->cipherSliceType,
317 nextSlice,
318 cipherOffsetInReqQW);
319 if (NON_SPC != pSessionDesc->singlePassState) {
320 LacSymQat_CipherCtrlBlockWrite(
321 &(pSessionDesc->reqSpcCacheFtr),
322 pSessionDesc->cipherAlgorithm,
323 pSessionDesc->cipherKeyLenInBytes,
324 pSessionDesc->cipherSliceType,
325 ICP_QAT_FW_SLICE_DRAM_WR,
326 cipherOffsetInReqQW);
327 }
328 }
329
330 static void
LacAlgChain_HashCDBuild(const CpaCySymHashSetupData * pHashData,CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U hashOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,lac_sym_qat_hash_precompute_info_t * pPrecomputeData,lac_sym_qat_hash_precompute_info_t * pPrecomputeDataOptimisedCd,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)331 LacAlgChain_HashCDBuild(
332 const CpaCySymHashSetupData *pHashData,
333 CpaInstanceHandle instanceHandle,
334 lac_session_desc_t *pSessionDesc,
335 icp_qat_fw_slice_t nextSlice,
336 Cpa8U hashOffsetInConstantsTable,
337 icp_qat_fw_comn_flags *pCmnRequestFlags,
338 icp_qat_fw_serv_specif_flags *pLaCmdFlags,
339 lac_sym_qat_hash_precompute_info_t *pPrecomputeData,
340 lac_sym_qat_hash_precompute_info_t *pPrecomputeDataOptimisedCd,
341 Cpa8U *pHwBlockBaseInDRAM,
342 Cpa32U *pHwBlockOffsetInDRAM,
343 Cpa8U *pOptimisedHwBlockBaseInDRAM,
344 Cpa32U *pOptimisedHwBlockOffsetInDRAM)
345 {
346 Cpa32U sizeInBytes = 0;
347 Cpa32U hwBlockOffsetInQuadWords =
348 *pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES;
349
350 /* build:
351 * - the hash part of the ContentDescriptor in DRAM */
352 /* - the hash part of the CD control block in the Request template */
353 LacSymQat_HashContentDescInit(&(pSessionDesc->reqCacheFtr),
354 instanceHandle,
355 pHashData,
356 pHwBlockBaseInDRAM,
357 hwBlockOffsetInQuadWords,
358 nextSlice,
359 pSessionDesc->qatHashMode,
360 CPA_FALSE,
361 CPA_FALSE,
362 pSessionDesc->useStatefulSha3ContentDesc,
363 pPrecomputeData,
364 &sizeInBytes);
365
366 /* Using DRAM CD so update offset */
367 *pHwBlockOffsetInDRAM += sizeInBytes;
368
369 sizeInBytes = 0;
370
371 if (pSessionDesc->useOptimisedContentDesc) {
372 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
373 instanceHandle,
374 pHashData,
375 pOptimisedHwBlockBaseInDRAM,
376 hashOffsetInConstantsTable,
377 nextSlice,
378 pSessionDesc->qatHashMode,
379 CPA_TRUE,
380 CPA_TRUE,
381 CPA_FALSE,
382 pPrecomputeDataOptimisedCd,
383 &sizeInBytes);
384
385 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
386 } else if (pSessionDesc->useSymConstantsTable) {
387 /* Need to build up the alternative CD for SHRAM Constants Table
388 * use */
389 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
390 instanceHandle,
391 pHashData,
392 pHwBlockBaseInDRAM,
393 hashOffsetInConstantsTable,
394 nextSlice,
395 pSessionDesc->qatHashMode,
396 CPA_TRUE,
397 CPA_FALSE,
398 CPA_FALSE,
399 pPrecomputeData,
400 &sizeInBytes);
401 }
402 }
403 static Cpa16U
LacAlgChain_GetCipherConfigSize(lac_session_desc_t * pSessionDesc)404 LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc)
405 {
406 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
407 return sizeof(icp_qat_hw_ucs_cipher_config_t);
408 } else {
409 return sizeof(icp_qat_hw_cipher_config_t);
410 }
411 }
412
413 static Cpa16U
LacAlgChain_GetCipherConfigOffset(lac_session_desc_t * pSessionDesc)414 LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc)
415 {
416 Cpa16U offset = 0;
417
418 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation ||
419 SPC == pSessionDesc->singlePassState) {
420 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
421 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc
422 ->reqCacheFtr.cd_ctrl;
423 offset = cd_ctrl->cipher_cfg_offset;
424 } else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) {
425 icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl =
426 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc
427 ->reqCacheFtr.cd_ctrl;
428 offset = cd_ctrl->cipher_cfg_offset;
429 }
430
431 return offset * LAC_QUAD_WORD_IN_BYTES;
432 }
433
434 CpaStatus
LacAlgChain_SessionAADUpdate(lac_session_desc_t * pSessionDesc,Cpa32U newAADLength)435 LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc,
436 Cpa32U newAADLength)
437 {
438 icp_qat_la_bulk_req_ftr_t *req_ftr = &pSessionDesc->reqCacheFtr;
439 icp_qat_la_auth_req_params_t *req_params = &req_ftr->serv_specif_rqpars;
440
441 if (!pSessionDesc)
442 return CPA_STATUS_FAIL;
443
444 pSessionDesc->aadLenInBytes = newAADLength;
445 req_params->u2.aad_sz =
446 LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE);
447
448 if (SPC == pSessionDesc->singlePassState) {
449 Cpa8U *pHwBlockBaseInDRAM = NULL;
450 Cpa32U hwBlockOffsetInDRAM = 0;
451 Cpa32U pSizeInBytes = 0;
452 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
453
454 pHwBlockBaseInDRAM =
455 (Cpa8U *)pSessionDesc->contentDescInfo.pData;
456 if (pSessionDesc->cipherDirection ==
457 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) {
458 if (LAC_CIPHER_IS_GCM(cipher)) {
459 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
460 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
461 } else {
462 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
463 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
464 }
465 }
466 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
467 pHwBlockBaseInDRAM +
468 hwBlockOffsetInDRAM,
469 &pSizeInBytes);
470 }
471
472 return CPA_STATUS_SUCCESS;
473 }
474
475 CpaStatus
LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pCipherKey)476 LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc,
477 Cpa8U *pCipherKey)
478 {
479 CpaStatus status = CPA_STATUS_SUCCESS;
480
481 if (pSessionDesc == NULL || pCipherKey == NULL)
482 return CPA_STATUS_FAIL;
483
484 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
485 LacSymQat_CipherArc4StateInit(
486 pCipherKey,
487 pSessionDesc->cipherKeyLenInBytes,
488 pSessionDesc->cipherARC4InitialState);
489 } else {
490 CpaCySymCipherSetupData cipherSetupData = { 0 };
491 Cpa32U sizeInBytes;
492 Cpa8U *pCipherKeyField;
493 Cpa16U cipherConfigSize;
494 Cpa16U cipherConfigOffset;
495 sal_qat_content_desc_info_t *pCdInfo =
496 &(pSessionDesc->contentDescInfo);
497
498 cipherSetupData.cipherAlgorithm = pSessionDesc->cipherAlgorithm;
499 cipherSetupData.cipherKeyLenInBytes =
500 pSessionDesc->cipherKeyLenInBytes;
501 cipherSetupData.pCipherKey = pCipherKey;
502 cipherSetupData.cipherDirection = pSessionDesc->cipherDirection;
503
504 cipherConfigSize =
505 LacAlgChain_GetCipherConfigSize(pSessionDesc);
506 cipherConfigOffset =
507 LacAlgChain_GetCipherConfigOffset(pSessionDesc);
508
509 pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset +
510 cipherConfigSize;
511
512 switch (pSessionDesc->symOperation) {
513 case CPA_CY_SYM_OP_CIPHER: {
514 LacSymQat_CipherHwBlockPopulateKeySetup(
515 pSessionDesc,
516 &(cipherSetupData),
517 cipherSetupData.cipherKeyLenInBytes,
518 pSessionDesc->cipherSliceType,
519 pCipherKeyField,
520 &sizeInBytes);
521
522 if (pSessionDesc->useSymConstantsTable) {
523 pCipherKeyField = (Cpa8U *)&(
524 pSessionDesc->shramReqCacheHdr.cd_pars.s1
525 .serv_specif_fields);
526
527 LacSymQat_CipherHwBlockPopulateKeySetup(
528 pSessionDesc,
529 &(cipherSetupData),
530 cipherSetupData.cipherKeyLenInBytes,
531 pSessionDesc->cipherSliceType,
532 pCipherKeyField,
533 &sizeInBytes);
534 }
535 } break;
536
537 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
538 LacSymQat_CipherHwBlockPopulateKeySetup(
539 pSessionDesc,
540 &(cipherSetupData),
541 cipherSetupData.cipherKeyLenInBytes,
542 pSessionDesc->cipherSliceType,
543 pCipherKeyField,
544 &sizeInBytes);
545 } break;
546
547 default:
548 LAC_LOG_ERROR("Invalid sym operation\n");
549 status = CPA_STATUS_INVALID_PARAM;
550 break;
551 }
552 }
553 return status;
554 }
555
556 CpaStatus
LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pAuthKey)557 LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc,
558 Cpa8U *pAuthKey)
559 {
560 CpaStatus status = CPA_STATUS_SUCCESS;
561 Cpa8U *pHwBlockBaseInDRAM = NULL;
562 Cpa8U *pOutHashSetup = NULL;
563 Cpa8U *pInnerState1 = NULL;
564 Cpa8U *pInnerState2 = NULL;
565 CpaCySymSessionSetupData sessionSetup = { 0 };
566 Cpa16U cipherConfigSize;
567
568 if (pSessionDesc == NULL || pAuthKey == NULL)
569 return CPA_STATUS_FAIL;
570
571 cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc);
572
573 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
574 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
575 .cd_ctrl;
576
577 pHwBlockBaseInDRAM = (Cpa8U *)pSessionDesc->contentDescInfo.pData;
578
579 sessionSetup.hashSetupData.hashAlgorithm = pSessionDesc->hashAlgorithm;
580 sessionSetup.hashSetupData.hashMode = pSessionDesc->hashMode;
581 sessionSetup.hashSetupData.authModeSetupData.authKey = pAuthKey;
582 sessionSetup.hashSetupData.authModeSetupData.authKeyLenInBytes =
583 pSessionDesc->authKeyLenInBytes;
584 sessionSetup.hashSetupData.authModeSetupData.aadLenInBytes =
585 pSessionDesc->aadLenInBytes;
586 sessionSetup.hashSetupData.digestResultLenInBytes =
587 pSessionDesc->hashResultSize;
588
589 sessionSetup.cipherSetupData.cipherAlgorithm =
590 pSessionDesc->cipherAlgorithm;
591 sessionSetup.cipherSetupData.cipherKeyLenInBytes =
592 pSessionDesc->cipherKeyLenInBytes;
593
594 /* Calculate hash states offsets */
595 pInnerState1 = pHwBlockBaseInDRAM +
596 cd_ctrl->hash_cfg_offset * LAC_QUAD_WORD_IN_BYTES +
597 sizeof(icp_qat_hw_auth_setup_t);
598
599 pInnerState2 = pInnerState1 + cd_ctrl->inner_state1_sz;
600
601 pOutHashSetup = pInnerState2 + cd_ctrl->inner_state2_sz;
602
603 /* Calculate offset of cipher key */
604 if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
605 sessionSetup.cipherSetupData.pCipherKey =
606 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
607 } else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
608 sessionSetup.cipherSetupData.pCipherKey =
609 pOutHashSetup + cipherConfigSize;
610 } else if (SPC == pSessionDesc->singlePassState) {
611 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
612 Cpa32U hwBlockOffsetInDRAM = 0;
613
614 if (pSessionDesc->cipherDirection ==
615 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
616 sessionSetup.cipherSetupData.pCipherKey =
617 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
618 } else {
619 if (LAC_CIPHER_IS_GCM(cipher))
620 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
621 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
622 else
623 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
624 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
625 sessionSetup.cipherSetupData.pCipherKey =
626 (Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM +
627 cipherConfigSize;
628 }
629 }
630
631 if (!sessionSetup.cipherSetupData.pCipherKey)
632 return CPA_STATUS_FAIL;
633
634 if (CPA_CY_SYM_HASH_SHA3_256 == pSessionDesc->hashAlgorithm) {
635 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
636 lac_sym_qat_hash_state_buffer_info_t
637 *pHashStateBufferInfo =
638 &(pSessionDesc->hashStateBufferInfo);
639
640 sal_crypto_service_t *pService =
641 (sal_crypto_service_t *)pSessionDesc->pInstance;
642
643 status = LacHash_StatePrefixAadBufferInit(
644 &(pService->generic_service_info),
645 &(sessionSetup.hashSetupData),
646 &(pSessionDesc->reqCacheFtr),
647 pSessionDesc->qatHashMode,
648 pSessionDesc->hashStatePrefixBuffer,
649 pHashStateBufferInfo);
650 /* SHRAM Constants Table not used for Auth-Enc */
651 }
652 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) {
653 Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize;
654 memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes);
655 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm ||
656 CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) {
657 memcpy(pInnerState2, pAuthKey, pSessionDesc->authKeyLenInBytes);
658 } else if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm ||
659 CPA_CY_SYM_HASH_KASUMI_F9 == pSessionDesc->hashAlgorithm ||
660 IS_HASH_MODE_1(pSessionDesc->qatHashMode)) {
661 if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm) {
662 memset(pInnerState2, 0, cd_ctrl->inner_state2_sz);
663 }
664
665 /* Block messages until precompute is completed */
666 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
667
668 status = LacHash_PrecomputeDataCreate(
669 pSessionDesc->pInstance,
670 (CpaCySymSessionSetupData *)&(sessionSetup),
671 LacSymAlgChain_HashPrecomputeDoneCb,
672 pSessionDesc,
673 pSessionDesc->hashStatePrefixBuffer,
674 pInnerState1,
675 pInnerState2);
676 }
677
678 return status;
679 }
680
681 static void
buildCmdData(sal_crypto_service_t * pService,lac_session_desc_t * pSessionDesc,CpaCySymAlgChainOrder * chainOrder,Cpa16U * proto,icp_qat_fw_serv_specif_flags * laCmdFlags,icp_qat_fw_comn_flags * cmnRequestFlags)682 buildCmdData(sal_crypto_service_t *pService,
683 lac_session_desc_t *pSessionDesc,
684 CpaCySymAlgChainOrder *chainOrder,
685 Cpa16U *proto,
686 icp_qat_fw_serv_specif_flags *laCmdFlags,
687 icp_qat_fw_comn_flags *cmnRequestFlags)
688 {
689 /* LW 28 is used to set hash flags for AlgChaining. */
690 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
691 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
692 .cd_ctrl;
693
694 /* proto refers to Protocol Flags, which is legacy FW <=> IA interface
695 * for ZUC and Snow3G. Use extended protocol flags for AlgChaining.
696 */
697 *proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
698
699 switch (pSessionDesc->symOperation) {
700 case CPA_CY_SYM_OP_CIPHER:
701 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
702
703 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
704 pSessionDesc->cipherAlgorithm) {
705 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
706 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
707 pSessionDesc->cipherAlgorithm) {
708 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
709 }
710 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
711 *proto = ICP_QAT_FW_LA_CCM_PROTO;
712 }
713 break;
714
715 case CPA_CY_SYM_OP_HASH:
716 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH;
717 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
718 pSessionDesc->hashAlgorithm) {
719 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
720 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
721 pSessionDesc->hashAlgorithm) {
722 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
723 }
724 break;
725
726 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
727 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
728 *proto = ICP_QAT_FW_LA_CCM_PROTO;
729
730 /* Derive chainOrder from direction for isAuthEncryptOp
731 * cases */
732 /* For CCM & GCM modes: force digest verify flag _TRUE
733 for decrypt and _FALSE for encrypt. For all other
734 cases use user defined value */
735
736 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
737 pSessionDesc->cipherDirection) {
738 *chainOrder =
739 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
740 pSessionDesc->digestVerify = CPA_FALSE;
741 } else {
742 *chainOrder =
743 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
744 if (CPA_TRUE == pService->forceAEADMacVerify) {
745 pSessionDesc->digestVerify = CPA_TRUE;
746 }
747 }
748 } else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) {
749 *proto = ICP_QAT_FW_LA_GCM_PROTO;
750
751 /* Derive chainOrder from direction for isAuthEncryptOp
752 * cases */
753 /* For CCM & GCM modes: force digest verify flag _TRUE
754 for decrypt and _FALSE for encrypt. For all other
755 cases use user defined value */
756
757 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
758 pSessionDesc->cipherDirection) {
759 *chainOrder =
760 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
761 pSessionDesc->digestVerify = CPA_FALSE;
762 } else {
763 *chainOrder =
764 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
765 if (CPA_TRUE == pService->forceAEADMacVerify) {
766 pSessionDesc->digestVerify = CPA_TRUE;
767 }
768 }
769 } else if (LAC_CIPHER_IS_CHACHA(
770 pSessionDesc->cipherAlgorithm)) {
771 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
772 pSessionDesc->cipherDirection) {
773 *chainOrder =
774 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
775 } else {
776 *chainOrder =
777 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
778 }
779 } else {
780 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
781
782 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
783 pSessionDesc->cipherAlgorithm) {
784 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
785 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
786 pSessionDesc->cipherAlgorithm) {
787 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
788 }
789
790 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
791 pSessionDesc->hashAlgorithm) {
792 /* Need to set LW 28 hash flags as well. */
793 ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET(
794 cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2);
795 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
796 pSessionDesc->hashAlgorithm) {
797 /* Need to set LW 28 hash flags as well. */
798 ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET(
799 cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3);
800 }
801 }
802
803 if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH ==
804 *chainOrder) {
805 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
806 } else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
807 *chainOrder) {
808 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
809 }
810 break;
811
812 default:
813 break;
814 }
815
816 /*
817 * Build the header flags with the default settings for this session.
818 */
819 if (pSessionDesc->isDPSession == CPA_TRUE) {
820 *cmnRequestFlags =
821 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
822 LAC_SYM_DP_QAT_PTR_TYPE);
823 } else {
824 *cmnRequestFlags =
825 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
826 LAC_SYM_DEFAULT_QAT_PTR_TYPE);
827 }
828
829 LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation);
830
831 return;
832 }
833
834 static void
updateLaCmdFlags(lac_session_desc_t * pSessionDesc,Cpa16U proto,icp_qat_fw_serv_specif_flags * laCmdFlags)835 updateLaCmdFlags(lac_session_desc_t *pSessionDesc,
836 Cpa16U proto,
837 icp_qat_fw_serv_specif_flags *laCmdFlags)
838 {
839 if (pSessionDesc->isAuth) {
840 if (pSessionDesc->digestVerify) {
841 ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags,
842 ICP_QAT_FW_LA_CMP_AUTH_RES);
843 ICP_QAT_FW_LA_RET_AUTH_SET(
844 *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES);
845 } else {
846 ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags,
847 ICP_QAT_FW_LA_RET_AUTH_RES);
848 ICP_QAT_FW_LA_CMP_AUTH_SET(
849 *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
850 }
851 }
852
853 if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) ||
854 (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) {
855 /* New bit position (12) for ZUC. The FW provides a specific
856 * macro to use to set the ZUC proto flag. With the new FW I/F
857 * this needs to be set for both Cipher and Auth */
858 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto);
859 } else {
860 /* Configure the common header */
861 ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto);
862 }
863
864 /* set Append flag, if digest is appended */
865 if (pSessionDesc->digestIsAppended) {
866 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
867 *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
868 } else {
869 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
870 *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
871 }
872 }
873
874 static lac_single_pass_state_t
LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,CpaCySymHashAlgorithm hash,Cpa32U capabilitiesMask)875 LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,
876 CpaCySymHashAlgorithm hash,
877 Cpa32U capabilitiesMask)
878 {
879 lac_single_pass_state_t state = NON_SPC;
880 if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) {
881 switch (cipher) {
882 case CPA_CY_SYM_CIPHER_CHACHA: {
883 if (CPA_CY_SYM_HASH_POLY == hash)
884 state = SPC;
885 break;
886 }
887 case CPA_CY_SYM_CIPHER_AES_GCM: {
888 if ((CPA_CY_SYM_HASH_AES_GCM == hash) ||
889 (CPA_CY_SYM_HASH_AES_GMAC == hash))
890 state = LIKELY_SPC;
891 break;
892 }
893 case CPA_CY_SYM_CIPHER_AES_CCM: {
894 if (LAC_CIPHER_AES_V2(capabilitiesMask))
895 state = SPC;
896 }
897 default:
898 /* Do Nothing as it is NON_SPC */
899 break;
900 }
901 }
902 return state;
903 }
904
905 static CpaBoolean
LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)906 LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,
907 Cpa32U capabilitiesMask,
908 lac_session_desc_t *pSessionDesc)
909 {
910 CpaBoolean hasSha3Ext =
911 ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask;
912 CpaBoolean useStatefulSha3DescFlag = CPA_FALSE;
913
914 if (hasSha3Ext && !partialsNotRequired &&
915 (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) &&
916 LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) {
917 useStatefulSha3DescFlag = CPA_TRUE;
918 }
919 return useStatefulSha3DescFlag;
920 }
921
922 /** @ingroup LacAlgChain */
923 CpaStatus
LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,const CpaCySymSessionSetupData * pSessionSetupData,lac_session_desc_t * pSessionDesc)924 LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,
925 const CpaCySymSessionSetupData *pSessionSetupData,
926 lac_session_desc_t *pSessionDesc)
927 {
928 CpaStatus stat, status = CPA_STATUS_SUCCESS;
929 sal_qat_content_desc_info_t *pCdInfo = NULL;
930 sal_qat_content_desc_info_t *pCdInfoOptimised = NULL;
931 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
932 Cpa32U capabilitiesMask =
933 pService->generic_service_info.capabilitiesMask;
934 Cpa8U *pHwBlockBaseInDRAM = NULL;
935 Cpa8U *pOptimisedHwBlockBaseInDRAM = NULL;
936 Cpa32U hwBlockOffsetInDRAM = 0;
937 Cpa32U optimisedHwBlockOffsetInDRAM = 0;
938 Cpa8U cipherOffsetInConstantsTable = 0;
939 Cpa8U hashOffsetInConstantsTable = 0;
940 icp_qat_fw_comn_flags cmnRequestFlags = 0;
941 icp_qat_fw_comn_req_t *pMsg = NULL;
942 icp_qat_fw_comn_req_t *pMsgS = NULL;
943 const CpaCySymCipherSetupData *pCipherData;
944 const CpaCySymHashSetupData *pHashData;
945 Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
946 CpaCySymAlgChainOrder chainOrder = 0;
947 lac_sym_qat_hash_precompute_info_t precomputeData = { 0 };
948 lac_sym_qat_hash_precompute_info_t precomputeDataOptimisedCd = { 0 };
949
950 pCipherData = &(pSessionSetupData->cipherSetupData);
951 pHashData = &(pSessionSetupData->hashSetupData);
952
953 /*-------------------------------------------------------------------------
954 * Populate session data
955 *-----------------------------------------------------------------------*/
956
957 /* Initialise Request Queue */
958 stat = LAC_SPINLOCK_INIT(&pSessionDesc->requestQueueLock);
959 if (CPA_STATUS_SUCCESS != stat) {
960 LAC_LOG_ERROR("Spinlock init failed for sessionLock");
961 return CPA_STATUS_RESOURCE;
962 }
963
964 pSessionDesc->pRequestQueueHead = NULL;
965 pSessionDesc->pRequestQueueTail = NULL;
966 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
967 pSessionDesc->pInstance = instanceHandle;
968 pSessionDesc->digestIsAppended = pSessionSetupData->digestIsAppended;
969 pSessionDesc->digestVerify = pSessionSetupData->verifyDigest;
970
971 /* Reset the pending callback counter */
972 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingCbCount);
973 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingDpCbCount);
974
975 /* Partial state must be set to full, to indicate that next packet
976 * expected on the session is a full packet or the start of a
977 * partial packet. */
978 pSessionDesc->partialState = CPA_CY_SYM_PACKET_TYPE_FULL;
979
980 pSessionDesc->symOperation = pSessionSetupData->symOperation;
981 switch (pSessionDesc->symOperation) {
982 case CPA_CY_SYM_OP_CIPHER:
983 pSessionDesc->isCipher = CPA_TRUE;
984 pSessionDesc->isAuth = CPA_FALSE;
985 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
986 pSessionDesc->singlePassState = NON_SPC;
987 break;
988 case CPA_CY_SYM_OP_HASH:
989 pSessionDesc->isCipher = CPA_FALSE;
990 pSessionDesc->isAuth = CPA_TRUE;
991 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
992 pSessionDesc->singlePassState = NON_SPC;
993 break;
994 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
995 pSessionDesc->isCipher = CPA_TRUE;
996 pSessionDesc->isAuth = CPA_TRUE;
997 pSessionDesc->singlePassState =
998 LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm,
999 pHashData->hashAlgorithm,
1000 capabilitiesMask);
1001
1002 switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) {
1003 case CPA_CY_SYM_CIPHER_AES_CCM: {
1004 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1005 pSessionDesc->digestIsAppended = CPA_TRUE;
1006 } break;
1007 case CPA_CY_SYM_CIPHER_AES_GCM:
1008 case CPA_CY_SYM_CIPHER_CHACHA:
1009 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1010 break;
1011 default: {
1012 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1013 /* Use the chainOrder passed in */
1014 chainOrder = pSessionSetupData->algChainOrder;
1015 if ((chainOrder !=
1016 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) &&
1017 (chainOrder !=
1018 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) {
1019 LAC_INVALID_PARAM_LOG("algChainOrder");
1020 return CPA_STATUS_INVALID_PARAM;
1021 }
1022 } break;
1023 }
1024 } break;
1025 default:
1026 pSessionDesc->singlePassState = NON_SPC;
1027 break;
1028 }
1029
1030 if (pSessionDesc->isCipher) {
1031 /* Populate cipher specific session data */
1032
1033 status = LacCipher_SessionSetupDataCheck(pCipherData,
1034 capabilitiesMask);
1035
1036 if (CPA_STATUS_SUCCESS == status) {
1037 pSessionDesc->cipherAlgorithm =
1038 pCipherData->cipherAlgorithm;
1039 pSessionDesc->cipherKeyLenInBytes =
1040 pCipherData->cipherKeyLenInBytes;
1041 pSessionDesc->cipherDirection =
1042 pCipherData->cipherDirection;
1043
1044 /* ARC4 base key isn't added to the content descriptor,
1045 * because we don't need to pass it directly to the QAT
1046 * engine. Instead an initial cipher state & key matrix
1047 * is derived from the base key and provided to the QAT
1048 * through the state pointer in the request params.
1049 * We'll store this initial state in the session
1050 * descriptor. */
1051
1052 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
1053 LacSymQat_CipherArc4StateInit(
1054 pCipherData->pCipherKey,
1055 pSessionDesc->cipherKeyLenInBytes,
1056 pSessionDesc->cipherARC4InitialState);
1057
1058 pSessionDesc->cipherARC4InitialStatePhysAddr =
1059 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1060 pService->generic_service_info,
1061 pSessionDesc->cipherARC4InitialState);
1062
1063 if (0 ==
1064 pSessionDesc
1065 ->cipherARC4InitialStatePhysAddr) {
1066 LAC_LOG_ERROR(
1067 "Unable to get the physical address of "
1068 "the initial state for ARC4\n");
1069 status = CPA_STATUS_FAIL;
1070 }
1071 }
1072 }
1073 }
1074
1075 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1076 /* Populate auth-specific session data */
1077 const CpaCySymHashSetupData *pHashData =
1078 &pSessionSetupData->hashSetupData;
1079
1080 status = LacHash_HashContextCheck(instanceHandle, pHashData);
1081 if (CPA_STATUS_SUCCESS == status) {
1082 pSessionDesc->hashResultSize =
1083 pHashData->digestResultLenInBytes;
1084 pSessionDesc->hashMode = pHashData->hashMode;
1085 pSessionDesc->hashAlgorithm = pHashData->hashAlgorithm;
1086
1087 /* Save the authentication key length for further update
1088 */
1089 if (CPA_CY_SYM_HASH_MODE_AUTH == pHashData->hashMode) {
1090 pSessionDesc->authKeyLenInBytes =
1091 pHashData->authModeSetupData
1092 .authKeyLenInBytes;
1093 }
1094 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp ||
1095 (pHashData->hashAlgorithm ==
1096 CPA_CY_SYM_HASH_SNOW3G_UIA2 ||
1097 pHashData->hashAlgorithm ==
1098 CPA_CY_SYM_HASH_ZUC_EIA3)) {
1099 pSessionDesc->aadLenInBytes =
1100 pHashData->authModeSetupData.aadLenInBytes;
1101 }
1102
1103 /* Set the QAT hash mode */
1104 if ((pHashData->hashMode ==
1105 CPA_CY_SYM_HASH_MODE_NESTED) ||
1106 (pHashData->hashMode ==
1107 CPA_CY_SYM_HASH_MODE_PLAIN) ||
1108 (pHashData->hashMode == CPA_CY_SYM_HASH_MODE_AUTH &&
1109 pHashData->hashAlgorithm ==
1110 CPA_CY_SYM_HASH_AES_CBC_MAC)) {
1111 pSessionDesc->qatHashMode =
1112 ICP_QAT_HW_AUTH_MODE0;
1113 } else /* CPA_CY_SYM_HASH_MODE_AUTH
1114 && anything except CPA_CY_SYM_HASH_AES_CBC_MAC
1115 */
1116 {
1117 if (IS_HMAC_ALG(pHashData->hashAlgorithm)) {
1118 /* SHA3 HMAC and SM3 do not support
1119 * precompute, force MODE2 for AUTH */
1120 if (LAC_HASH_IS_SHA3(
1121 pHashData->hashAlgorithm) ||
1122 (CPA_CY_SYM_HASH_SM3 ==
1123 pHashData->hashAlgorithm)) {
1124 pSessionDesc->qatHashMode =
1125 ICP_QAT_HW_AUTH_MODE2;
1126 } else {
1127 pSessionDesc->qatHashMode =
1128 pService->qatHmacMode;
1129 }
1130 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
1131 pHashData->hashAlgorithm) {
1132 pSessionDesc->qatHashMode =
1133 ICP_QAT_HW_AUTH_MODE0;
1134 } else {
1135 pSessionDesc->qatHashMode =
1136 ICP_QAT_HW_AUTH_MODE1;
1137 }
1138 }
1139 }
1140 }
1141
1142 /*-------------------------------------------------------------------------
1143 * build the message templates
1144 * create two content descriptors in the case we can support using SHRAM
1145 * constants and an optimised content descriptor. we have to do this in
1146 * case of partials. 64 byte content descriptor is used in the SHRAM
1147 * case for AES-128-HMAC-SHA1
1148 *-----------------------------------------------------------------------*/
1149 if (CPA_STATUS_SUCCESS == status) {
1150 pSessionDesc->cipherSliceType =
1151 LacCipher_GetCipherSliceType(pService,
1152 pSessionDesc->cipherAlgorithm,
1153 pSessionDesc->hashAlgorithm);
1154
1155 LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc);
1156 pSessionDesc->useOptimisedContentDesc = CPA_FALSE;
1157 pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE;
1158
1159 /* Build configuration data */
1160 buildCmdData(pService,
1161 pSessionDesc,
1162 &chainOrder,
1163 &proto,
1164 &pSessionDesc->laCmdFlags,
1165 &cmnRequestFlags);
1166
1167 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE ==
1168 pSessionDesc->cipherSliceType)
1169 pSessionDesc->useSymConstantsTable = CPA_FALSE;
1170 else
1171 pSessionDesc->useSymConstantsTable =
1172 LacSymQat_UseSymConstantsTable(
1173 pSessionDesc,
1174 &cipherOffsetInConstantsTable,
1175 &hashOffsetInConstantsTable);
1176
1177 /* for a certain combination of Algorithm Chaining we want to
1178 use an optimised cd block */
1179
1180 if (pSessionDesc->symOperation ==
1181 CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
1182 pSessionDesc->useSymConstantsTable == CPA_TRUE) {
1183 pSessionDesc->useOptimisedContentDesc =
1184 LacSymQat_UseOptimisedContentDesc(pSessionDesc);
1185 }
1186
1187 /* check whether we need to construct content desc for stateful
1188 * SHA3 */
1189 pSessionDesc->useStatefulSha3ContentDesc =
1190 LacAlgChain_UseStatefulSha3ContentDesc(
1191 pSessionSetupData->partialsNotRequired,
1192 capabilitiesMask,
1193 pSessionDesc);
1194
1195 /* setup some convenience pointers */
1196 pCdInfo = &(pSessionDesc->contentDescInfo);
1197 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1198 hwBlockOffsetInDRAM = 0;
1199
1200 /* set up the pointer for the optimised content desc if this is
1201 * possible we still have to support both cd types in case of
1202 * partials so we construct both */
1203 if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) {
1204 pCdInfoOptimised =
1205 &(pSessionDesc->contentDescOptimisedInfo);
1206 pOptimisedHwBlockBaseInDRAM =
1207 (Cpa8U *)pCdInfoOptimised->pData;
1208 optimisedHwBlockOffsetInDRAM = 0;
1209 }
1210
1211 switch (pSessionDesc->symOperation) {
1212 case CPA_CY_SYM_OP_CIPHER: {
1213 LacAlgChain_CipherCDBuild(
1214 pCipherData,
1215 pSessionDesc,
1216 ICP_QAT_FW_SLICE_DRAM_WR,
1217 cipherOffsetInConstantsTable,
1218 &pSessionDesc->cmnRequestFlags,
1219 &pSessionDesc->laCmdFlags,
1220 pHwBlockBaseInDRAM,
1221 &hwBlockOffsetInDRAM,
1222 capabilitiesMask);
1223
1224 if (pSessionDesc->useSymConstantsTable) {
1225 LacAlgChain_CipherCDBuild_ForSHRAM(
1226 pCipherData,
1227 pSessionDesc,
1228 ICP_QAT_FW_SLICE_DRAM_WR,
1229 cipherOffsetInConstantsTable);
1230 }
1231 } break;
1232 case CPA_CY_SYM_OP_HASH:
1233 LacAlgChain_HashCDBuild(pHashData,
1234 instanceHandle,
1235 pSessionDesc,
1236 ICP_QAT_FW_SLICE_NULL,
1237 hashOffsetInConstantsTable,
1238 &pSessionDesc->cmnRequestFlags,
1239 &pSessionDesc->laCmdFlags,
1240 &precomputeData,
1241 &precomputeDataOptimisedCd,
1242 pHwBlockBaseInDRAM,
1243 &hwBlockOffsetInDRAM,
1244 NULL,
1245 NULL);
1246 break;
1247 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
1248 /* For CCM/GCM, CPM firmware currently expects the
1249 * cipher and hash h/w setup blocks to be arranged
1250 * according to the chain order (Except for GCM/CCM,
1251 * order doesn't actually matter as long as the config
1252 * offsets are set correctly in CD control blocks
1253 */
1254 if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
1255 chainOrder) {
1256 LacAlgChain_HashCDBuild(
1257 pHashData,
1258 instanceHandle,
1259 pSessionDesc,
1260 ICP_QAT_FW_SLICE_CIPHER,
1261 hashOffsetInConstantsTable,
1262 &pSessionDesc->cmnRequestFlags,
1263 &pSessionDesc->laCmdFlags,
1264 &precomputeData,
1265 &precomputeDataOptimisedCd,
1266 pHwBlockBaseInDRAM,
1267 &hwBlockOffsetInDRAM,
1268 pOptimisedHwBlockBaseInDRAM,
1269 &optimisedHwBlockOffsetInDRAM);
1270
1271 LacAlgChain_CipherCDBuild(
1272 pCipherData,
1273 pSessionDesc,
1274 ICP_QAT_FW_SLICE_DRAM_WR,
1275 cipherOffsetInConstantsTable,
1276 &pSessionDesc->cmnRequestFlags,
1277 &pSessionDesc->laCmdFlags,
1278 pHwBlockBaseInDRAM,
1279 &hwBlockOffsetInDRAM,
1280 capabilitiesMask);
1281
1282 if (pSessionDesc->useOptimisedContentDesc) {
1283 LacAlgChain_CipherCDBuild_ForOptimisedCD(
1284 pCipherData,
1285 pSessionDesc,
1286 ICP_QAT_FW_SLICE_DRAM_WR,
1287 cipherOffsetInConstantsTable,
1288 pOptimisedHwBlockBaseInDRAM,
1289 &optimisedHwBlockOffsetInDRAM);
1290 }
1291
1292 if (NON_SPC != pSessionDesc->singlePassState) {
1293 pCdInfo->hwBlkSzQuadWords =
1294 (LAC_BYTES_TO_QUADWORDS(
1295 hwBlockOffsetInDRAM));
1296 pMsg = (icp_qat_fw_comn_req_t *)&(
1297 pSessionDesc->reqSpcCacheHdr);
1298 SalQatMsg_ContentDescHdrWrite(
1299 (icp_qat_fw_comn_req_t *)pMsg,
1300 pCdInfo);
1301 }
1302 } else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */
1303 {
1304 LacAlgChain_CipherCDBuild(
1305 pCipherData,
1306 pSessionDesc,
1307 ICP_QAT_FW_SLICE_AUTH,
1308 cipherOffsetInConstantsTable,
1309 &pSessionDesc->cmnRequestFlags,
1310 &pSessionDesc->laCmdFlags,
1311 pHwBlockBaseInDRAM,
1312 &hwBlockOffsetInDRAM,
1313 capabilitiesMask);
1314
1315 if (pSessionDesc->useOptimisedContentDesc) {
1316 LacAlgChain_CipherCDBuild_ForOptimisedCD(
1317 pCipherData,
1318 pSessionDesc,
1319 ICP_QAT_FW_SLICE_AUTH,
1320 cipherOffsetInConstantsTable,
1321 pOptimisedHwBlockBaseInDRAM,
1322 &optimisedHwBlockOffsetInDRAM);
1323 }
1324
1325 if (NON_SPC != pSessionDesc->singlePassState) {
1326 pCdInfo->hwBlkSzQuadWords =
1327 LAC_BYTES_TO_QUADWORDS(
1328 hwBlockOffsetInDRAM);
1329 pMsg = (icp_qat_fw_comn_req_t *)&(
1330 pSessionDesc->reqSpcCacheHdr);
1331 SalQatMsg_ContentDescHdrWrite(
1332 (icp_qat_fw_comn_req_t *)pMsg,
1333 pCdInfo);
1334 }
1335 LacAlgChain_HashCDBuild(
1336 pHashData,
1337 instanceHandle,
1338 pSessionDesc,
1339 ICP_QAT_FW_SLICE_DRAM_WR,
1340 hashOffsetInConstantsTable,
1341 &pSessionDesc->cmnRequestFlags,
1342 &pSessionDesc->laCmdFlags,
1343 &precomputeData,
1344 &precomputeDataOptimisedCd,
1345 pHwBlockBaseInDRAM,
1346 &hwBlockOffsetInDRAM,
1347 pOptimisedHwBlockBaseInDRAM,
1348 &optimisedHwBlockOffsetInDRAM);
1349 }
1350 break;
1351 default:
1352 LAC_LOG_ERROR("Invalid sym operation\n");
1353 status = CPA_STATUS_INVALID_PARAM;
1354 }
1355 }
1356
1357 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1358 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo =
1359 &(pSessionDesc->hashStateBufferInfo);
1360 CpaBoolean hashStateBuffer = CPA_TRUE;
1361
1362 /* set up fields in both the cd_ctrl and reqParams which
1363 * describe the ReqParams block */
1364 LacSymQat_HashSetupReqParamsMetaData(
1365 &(pSessionDesc->reqCacheFtr),
1366 instanceHandle,
1367 pHashData,
1368 hashStateBuffer,
1369 pSessionDesc->qatHashMode,
1370 pSessionDesc->digestVerify);
1371
1372 if (pSessionDesc->useSymConstantsTable) {
1373 /* Need to set up for SHRAM Constants Table use also */
1374 LacSymQat_HashSetupReqParamsMetaData(
1375 &(pSessionDesc->shramReqCacheFtr),
1376 instanceHandle,
1377 pHashData,
1378 hashStateBuffer,
1379 pSessionDesc->qatHashMode,
1380 pSessionDesc->digestVerify);
1381 }
1382
1383 /* populate the hash state prefix buffer info structure
1384 * (part of user allocated session memory & the
1385 * buffer itself. For CCM/GCM the buffer is stored in the
1386 * cookie and is not initialised here) */
1387 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
1388 LAC_CHECK_64_BYTE_ALIGNMENT(
1389 &(pSessionDesc->hashStatePrefixBuffer[0]));
1390 status = LacHash_StatePrefixAadBufferInit(
1391 &(pService->generic_service_info),
1392 pHashData,
1393 &(pSessionDesc->reqCacheFtr),
1394 pSessionDesc->qatHashMode,
1395 pSessionDesc->hashStatePrefixBuffer,
1396 pHashStateBufferInfo);
1397 /* SHRAM Constants Table not used for Auth-Enc */
1398 }
1399
1400 if (CPA_STATUS_SUCCESS == status) {
1401 if (IS_HASH_MODE_1(pSessionDesc->qatHashMode) ||
1402 CPA_CY_SYM_HASH_ZUC_EIA3 ==
1403 pHashData->hashAlgorithm) {
1404 LAC_CHECK_64_BYTE_ALIGNMENT(
1405 &(pSessionDesc->hashStatePrefixBuffer[0]));
1406
1407 /* Block messages until precompute is completed
1408 */
1409 pSessionDesc->nonBlockingOpsInProgress =
1410 CPA_FALSE;
1411 status = LacHash_PrecomputeDataCreate(
1412 instanceHandle,
1413 (CpaCySymSessionSetupData *)
1414 pSessionSetupData,
1415 LacSymAlgChain_HashPrecomputeDoneCb,
1416 pSessionDesc,
1417 pSessionDesc->hashStatePrefixBuffer,
1418 precomputeData.pState1,
1419 precomputeData.pState2);
1420 if (pSessionDesc->useOptimisedContentDesc) {
1421 status = LacHash_PrecomputeDataCreate(
1422 instanceHandle,
1423 (CpaCySymSessionSetupData *)
1424 pSessionSetupData,
1425 LacSymAlgChain_HashPrecomputeDoneCb,
1426 pSessionDesc,
1427 pSessionDesc->hashStatePrefixBuffer,
1428 precomputeDataOptimisedCd.pState1,
1429 precomputeDataOptimisedCd.pState2);
1430 }
1431 } else if (pHashData->hashAlgorithm ==
1432 CPA_CY_SYM_HASH_AES_CBC_MAC) {
1433 if (NULL != precomputeData.pState2) {
1434 LAC_OS_BZERO(precomputeData.pState2,
1435 precomputeData.state2Size);
1436 memcpy(precomputeData.pState2,
1437 pHashData->authModeSetupData
1438 .authKey,
1439 pHashData->authModeSetupData
1440 .authKeyLenInBytes);
1441 }
1442 }
1443 }
1444 }
1445
1446 if (CPA_STATUS_SUCCESS == status) {
1447
1448 /* Configure the ContentDescriptor field
1449 in the request if not done already */
1450 pCdInfo->hwBlkSzQuadWords =
1451 LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM);
1452 pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr);
1453 SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1454 pCdInfo);
1455
1456 pMsgS =
1457 (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr);
1458 /*If we are using the optimised CD then
1459 we have to set this up correctly in the SHARM reqCache*/
1460 if (pSessionDesc->useOptimisedContentDesc) {
1461 pCdInfoOptimised->hwBlkSzQuadWords =
1462 LAC_BYTES_TO_QUADWORDS(
1463 optimisedHwBlockOffsetInDRAM);
1464 SalQatMsg_ContentDescHdrWrite(
1465 (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised);
1466 }
1467
1468 /* Updates command flags basing on configured alg */
1469 updateLaCmdFlags(pSessionDesc,
1470 proto,
1471 &pSessionDesc->laCmdFlags);
1472
1473 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1474 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1475 pSessionDesc->laCmdId,
1476 pSessionDesc->cmnRequestFlags,
1477 pSessionDesc->laCmdFlags);
1478
1479 /* Need to duplicate if SHRAM Constants Table used */
1480 if (pSessionDesc->useSymConstantsTable) {
1481 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
1482 pSessionDesc->laCmdFlags,
1483 ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP);
1484
1485 if (pSessionDesc->isCipher &&
1486 !pSessionDesc->useOptimisedContentDesc) {
1487 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(
1488 cmnRequestFlags,
1489 QAT_COMN_CD_FLD_TYPE_16BYTE_DATA);
1490 }
1491
1492 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS,
1493 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1494 pSessionDesc->laCmdId,
1495 cmnRequestFlags,
1496 pSessionDesc->laCmdFlags);
1497 }
1498 }
1499
1500 return status;
1501 }
1502
1503 static void
LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t * pMsg,Cpa32U packetType,icp_qat_hw_auth_mode_t qatHashMode)1504 LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg,
1505 Cpa32U packetType,
1506 icp_qat_hw_auth_mode_t qatHashMode)
1507 {
1508 icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL;
1509
1510 pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl);
1511
1512 if (IS_HASH_MODE_2(qatHashMode)) {
1513 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1514 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1515 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1516 pAuthCdCtrlHdr->hash_flags,
1517 QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1518 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1519 pAuthCdCtrlHdr->hash_flags,
1520 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1521 } else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) {
1522 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1523 pAuthCdCtrlHdr->hash_flags,
1524 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1525 }
1526 } else {
1527 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1528 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1529 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1530 pAuthCdCtrlHdr->hash_flags,
1531 QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1532 }
1533 }
1534 }
1535
1536 /** @ingroup LacAlgChain */
1537 CpaStatus
LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,void * pCallbackTag,const CpaCySymOpData * pOpData,const CpaBufferList * pSrcBuffer,CpaBufferList * pDstBuffer,CpaBoolean * pVerifyResult)1538 LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,
1539 lac_session_desc_t *pSessionDesc,
1540 void *pCallbackTag,
1541 const CpaCySymOpData *pOpData,
1542 const CpaBufferList *pSrcBuffer,
1543 CpaBufferList *pDstBuffer,
1544 CpaBoolean *pVerifyResult)
1545 {
1546 CpaStatus status = CPA_STATUS_SUCCESS;
1547 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
1548 Cpa32U capabilitiesMask =
1549 pService->generic_service_info.capabilitiesMask;
1550 lac_sym_bulk_cookie_t *pCookie = NULL;
1551 lac_sym_cookie_t *pSymCookie = NULL;
1552 icp_qat_fw_la_bulk_req_t *pMsg = NULL;
1553 Cpa8U *pMsgDummy = NULL;
1554 Cpa8U *pCacheDummyHdr = NULL;
1555 Cpa8U *pCacheDummyFtr = NULL;
1556 Cpa32U qatPacketType = 0;
1557 CpaBufferList *pBufferList = NULL;
1558 Cpa8U *pDigestResult = NULL;
1559 Cpa64U srcAddrPhys = 0;
1560 Cpa64U dstAddrPhys = 0;
1561 icp_qat_fw_la_cmd_id_t laCmdId;
1562 sal_qat_content_desc_info_t *pCdInfo = NULL;
1563 Cpa8U *pHwBlockBaseInDRAM = NULL;
1564 Cpa32U hwBlockOffsetInDRAM = 0;
1565 Cpa32U sizeInBytes = 0;
1566 icp_qat_fw_cipher_cd_ctrl_hdr_t *pSpcCdCtrlHdr = NULL;
1567 CpaCySymCipherAlgorithm cipher;
1568 CpaCySymHashAlgorithm hash;
1569 Cpa8U paddingLen = 0;
1570 Cpa8U blockLen = 0;
1571 CpaBoolean digestIsAppended = CPA_FALSE;
1572 Cpa32U aadLenInBytes = 0;
1573 Cpa64U srcPktSize = 0;
1574 Cpa64U dstPktSize = 0;
1575
1576 /* Set the command id */
1577 laCmdId = pSessionDesc->laCmdId;
1578
1579 cipher = pSessionDesc->cipherAlgorithm;
1580 hash = pSessionDesc->hashAlgorithm;
1581
1582 CpaBoolean isSpCcm =
1583 (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask));
1584
1585 if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1586 pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes;
1587 if (pOpData->messageLenToHashInBytes == 0 ||
1588 pOpData->pAdditionalAuthData != NULL) {
1589 LAC_INVALID_PARAM_LOG(
1590 "For AES_GMAC, AAD Length "
1591 "(messageLenToHashInBytes) must "
1592 "be non zero and pAdditionalAuthData "
1593 "must be NULL");
1594 return CPA_STATUS_INVALID_PARAM;
1595 }
1596 }
1597
1598 aadLenInBytes = pSessionDesc->aadLenInBytes;
1599
1600 /* Convert Alg Chain Request to Cipher Request for CCP and
1601 * AES_GCM single pass */
1602 if ((NON_SPC != pSessionDesc->singlePassState) &&
1603 (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) {
1604 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
1605 laCmdId = pSessionDesc->laCmdId;
1606 pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER;
1607 pSessionDesc->singlePassState = SPC;
1608 pSessionDesc->isCipher = CPA_TRUE;
1609 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1610 pSessionDesc->isAuth = CPA_FALSE;
1611
1612 if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1613 if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) {
1614 LAC_INVALID_PARAM_LOG(
1615 "aadLenInBytes for AES_GMAC");
1616 return CPA_STATUS_INVALID_PARAM;
1617 }
1618 }
1619 /* New bit position (13) for SINGLE PASS.
1620 * The FW provides a specific macro to use to set the proto flag
1621 */
1622 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1623 pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
1624 if (isCyGen2x(pService)) {
1625 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
1626 }
1627
1628 pCdInfo = &(pSessionDesc->contentDescInfo);
1629 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1630 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
1631 pSessionDesc->cipherDirection) {
1632 if (LAC_CIPHER_IS_GCM(cipher))
1633 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1634 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
1635 else if (LAC_CIPHER_IS_CHACHA(cipher))
1636 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1637 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
1638 } else if (isSpCcm) {
1639 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1640 LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM);
1641 }
1642
1643 /* Update cipher slice type */
1644 pSessionDesc->cipherSliceType =
1645 LacCipher_GetCipherSliceType(pService,
1646 pSessionDesc->cipherAlgorithm,
1647 pSessionDesc->hashAlgorithm);
1648
1649 ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags,
1650 pSessionDesc->cipherSliceType);
1651
1652 /* construct cipherConfig in CD in DRAM */
1653 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1654 pHwBlockBaseInDRAM +
1655 hwBlockOffsetInDRAM,
1656 &sizeInBytes);
1657 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1658 pSessionDesc->reqSpcCacheHdr),
1659 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1660 laCmdId,
1661 pSessionDesc->cmnRequestFlags,
1662 pSessionDesc->laCmdFlags);
1663 } else if ((SPC == pSessionDesc->singlePassState) &&
1664 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1665 pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
1666 pSessionDesc->singlePassState = LIKELY_SPC;
1667 pSessionDesc->isCipher = CPA_TRUE;
1668 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1669 pSessionDesc->isAuth = CPA_TRUE;
1670 pCdInfo = &(pSessionDesc->contentDescInfo);
1671 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1672
1673 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
1674 pSessionDesc->cipherDirection) {
1675 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
1676 } else {
1677 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
1678 }
1679
1680 laCmdId = pSessionDesc->laCmdId;
1681 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1682 pSessionDesc->laCmdFlags, 0);
1683 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
1684 ICP_QAT_FW_LA_GCM_PROTO);
1685
1686 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1687 pHwBlockBaseInDRAM +
1688 hwBlockOffsetInDRAM,
1689 &sizeInBytes);
1690
1691 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1692 pSessionDesc->reqCacheHdr),
1693 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1694 laCmdId,
1695 pSessionDesc->cmnRequestFlags,
1696 pSessionDesc->laCmdFlags);
1697 }
1698
1699 else if (LAC_CIPHER_IS_CHACHA(cipher) &&
1700 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1701 LAC_INVALID_PARAM_LOG("IV for CHACHA");
1702 return CPA_STATUS_INVALID_PARAM;
1703 }
1704
1705 if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) {
1706 if (CPA_CY_SYM_HASH_AES_CCM == hash) {
1707 status = LacSymAlgChain_CheckCCMData(
1708 pOpData->pAdditionalAuthData,
1709 pOpData->pIv,
1710 pOpData->messageLenToCipherInBytes,
1711 pOpData->ivLenInBytes);
1712 if (CPA_STATUS_SUCCESS == status) {
1713 LacSymAlgChain_PrepareCCMData(
1714 pSessionDesc,
1715 pOpData->pAdditionalAuthData,
1716 pOpData->pIv,
1717 pOpData->messageLenToCipherInBytes,
1718 pOpData->ivLenInBytes);
1719 }
1720 } else if (CPA_CY_SYM_HASH_AES_GCM == hash) {
1721 if (aadLenInBytes != 0 &&
1722 pOpData->pAdditionalAuthData == NULL) {
1723 LAC_INVALID_PARAM_LOG("pAdditionalAuthData");
1724 status = CPA_STATUS_INVALID_PARAM;
1725 }
1726 if (CPA_STATUS_SUCCESS == status) {
1727 LacSymAlgChain_PrepareGCMData(
1728 pSessionDesc, pOpData->pAdditionalAuthData);
1729 }
1730 }
1731 }
1732
1733 /* allocate cookie (used by callback function) */
1734 if (CPA_STATUS_SUCCESS == status) {
1735 pSymCookie = (lac_sym_cookie_t *)Lac_MemPoolEntryAlloc(
1736 pService->lac_sym_cookie_pool);
1737 if (pSymCookie == NULL) {
1738 LAC_LOG_ERROR("Cannot allocate cookie - NULL");
1739 status = CPA_STATUS_RESOURCE;
1740 } else if ((void *)CPA_STATUS_RETRY == pSymCookie) {
1741 pSymCookie = NULL;
1742 status = CPA_STATUS_RETRY;
1743 } else {
1744 pCookie = &(pSymCookie->u.bulkCookie);
1745 }
1746 }
1747
1748 if (CPA_STATUS_SUCCESS == status) {
1749 /* write the buffer descriptors */
1750 if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) {
1751 status =
1752 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1753 (CpaBufferList *)pSrcBuffer,
1754 &srcAddrPhys,
1755 CPA_FALSE,
1756 &(pService->generic_service_info));
1757 if (CPA_STATUS_SUCCESS != status) {
1758 LAC_LOG_ERROR(
1759 "Unable to write src buffer descriptors");
1760 }
1761 /* For out of place operations */
1762 if ((pSrcBuffer != pDstBuffer) &&
1763 (CPA_STATUS_SUCCESS == status)) {
1764 status =
1765 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1766 pDstBuffer,
1767 &dstAddrPhys,
1768 CPA_FALSE,
1769 &(pService->generic_service_info));
1770 if (CPA_STATUS_SUCCESS != status) {
1771 LAC_LOG_ERROR(
1772 "Unable to write dest buffer descriptors");
1773 }
1774 }
1775 } else {
1776 status = LacBuffDesc_BufferListDescWrite(
1777 (CpaBufferList *)pSrcBuffer,
1778 &srcAddrPhys,
1779 CPA_FALSE,
1780 &(pService->generic_service_info));
1781 if (CPA_STATUS_SUCCESS != status) {
1782 LAC_LOG_ERROR(
1783 "Unable to write src buffer descriptors in "
1784 "LacBuffDesc_BufferListDescWrite");
1785 }
1786 /* For out of place operations */
1787 if ((pSrcBuffer != pDstBuffer) &&
1788 (CPA_STATUS_SUCCESS == status)) {
1789 status = LacBuffDesc_BufferListDescWrite(
1790 pDstBuffer,
1791 &dstAddrPhys,
1792 CPA_FALSE,
1793 &(pService->generic_service_info));
1794 if (CPA_STATUS_SUCCESS != status) {
1795 LAC_LOG_ERROR(
1796 "Unable to write dest buffer descriptors in "
1797 "LacBuffDesc_BufferListDescWrite");
1798 }
1799 }
1800 }
1801 }
1802 if (CPA_STATUS_SUCCESS == status) {
1803 /* populate the cookie */
1804 pCookie->pCallbackTag = pCallbackTag;
1805 pCookie->sessionCtx = pOpData->sessionCtx;
1806 pCookie->pOpData = (const CpaCySymOpData *)pOpData;
1807 pCookie->pDstBuffer = pDstBuffer;
1808 pCookie->updateSessionIvOnSend = CPA_FALSE;
1809 pCookie->updateUserIvOnRecieve = CPA_FALSE;
1810 pCookie->updateKeySizeOnRecieve = CPA_FALSE;
1811 pCookie->pNext = NULL;
1812 pCookie->instanceHandle = pService;
1813
1814 /* get the qat packet type for LAC packet type */
1815 LacSymQat_packetTypeGet(pOpData->packetType,
1816 pSessionDesc->partialState,
1817 &qatPacketType);
1818 /*
1819 * For XTS mode, the key size must be updated after
1820 * the first partial has been sent. Set a flag here so the
1821 * response knows to do this.
1822 */
1823 if (LAC_CIPHER_IS_XTS_MODE(cipher) &&
1824 (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
1825 (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) &&
1826 (qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) {
1827 pCookie->updateKeySizeOnRecieve = CPA_TRUE;
1828 }
1829
1830 /*
1831 * Now create the Request.
1832 * Start by populating it from the cache in the session
1833 * descriptor.
1834 */
1835 pMsg = &(pCookie->qatMsg);
1836 pMsgDummy = (Cpa8U *)pMsg;
1837
1838 if (SPC == pSessionDesc->singlePassState) {
1839 pCacheDummyHdr =
1840 (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr);
1841 pCacheDummyFtr =
1842 (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr);
1843 } else {
1844 /* Normally, we want to use the SHRAM Constants Table if
1845 * possible for best performance (less DRAM accesses
1846 * incurred by CPM). But we can't use it for
1847 * partial-packet hash operations. This is why we build
1848 * 2 versions of the message template at sessionInit,
1849 * one for SHRAM Constants Table usage and the other
1850 * (default) for Content Descriptor h/w setup data in
1851 * DRAM. And we chose between them here on a
1852 * per-request basis, when we know the packetType
1853 */
1854 if ((!pSessionDesc->useSymConstantsTable) ||
1855 (pSessionDesc->isAuth &&
1856 (CPA_CY_SYM_PACKET_TYPE_FULL !=
1857 pOpData->packetType))) {
1858 pCacheDummyHdr =
1859 (Cpa8U *)&(pSessionDesc->reqCacheHdr);
1860 pCacheDummyFtr =
1861 (Cpa8U *)&(pSessionDesc->reqCacheFtr);
1862 } else {
1863 pCacheDummyHdr =
1864 (Cpa8U *)&(pSessionDesc->shramReqCacheHdr);
1865 pCacheDummyFtr =
1866 (Cpa8U *)&(pSessionDesc->shramReqCacheFtr);
1867 }
1868 }
1869 memcpy(pMsgDummy,
1870 pCacheDummyHdr,
1871 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW));
1872 memset((pMsgDummy +
1873 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)),
1874 0,
1875 (LAC_LONG_WORD_IN_BYTES *
1876 LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW));
1877 memcpy(pMsgDummy +
1878 (LAC_LONG_WORD_IN_BYTES *
1879 LAC_START_OF_CACHE_FTR_IN_LW),
1880 pCacheDummyFtr,
1881 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW));
1882 /*
1883 * Populate the comn_mid section
1884 */
1885 SalQatMsg_CmnMidWrite(pMsg,
1886 pCookie,
1887 LAC_SYM_DEFAULT_QAT_PTR_TYPE,
1888 srcAddrPhys,
1889 dstAddrPhys,
1890 0,
1891 0);
1892
1893 /*
1894 * Populate the serv_specif_flags field of the Request header
1895 * Some of the flags are set up here.
1896 * Others are set up later when the RequestParams are set up.
1897 */
1898
1899 LacSymQat_LaPacketCommandFlagSet(
1900 qatPacketType,
1901 laCmdId,
1902 cipher,
1903 &pMsg->comn_hdr.serv_specif_flags,
1904 pOpData->ivLenInBytes);
1905
1906 if (SPC == pSessionDesc->singlePassState) {
1907 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1908 pMsg->comn_hdr.serv_specif_flags,
1909 ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS);
1910
1911 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL ==
1912 pOpData->packetType) {
1913 ICP_QAT_FW_LA_RET_AUTH_SET(
1914 pMsg->comn_hdr.serv_specif_flags,
1915 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
1916
1917 ICP_QAT_FW_LA_CMP_AUTH_SET(
1918 pMsg->comn_hdr.serv_specif_flags,
1919 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
1920 }
1921 }
1922
1923 ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags,
1924 pSessionDesc->cipherSliceType);
1925
1926 LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize);
1927 LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize);
1928
1929 /*
1930 * Populate the CipherRequestParams section of the Request
1931 */
1932 if (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) {
1933
1934 Cpa8U *pIvBuffer = NULL;
1935
1936 status = LacCipher_PerformParamCheck(cipher,
1937 pOpData,
1938 srcPktSize);
1939 if (CPA_STATUS_SUCCESS != status) {
1940 /* free the cookie */
1941 Lac_MemPoolEntryFree(pCookie);
1942 return status;
1943 }
1944
1945 if (CPA_STATUS_SUCCESS == status) {
1946 /* align cipher IV */
1947 status = LacCipher_PerformIvCheck(
1948 &(pService->generic_service_info),
1949 pCookie,
1950 qatPacketType,
1951 &pIvBuffer);
1952 }
1953 if ((SPC == pSessionDesc->singlePassState) &&
1954 ((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) ||
1955 (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) {
1956 /* For SPC stateful cipher state size for mid
1957 * and end partial packet is 48 bytes
1958 */
1959 pSpcCdCtrlHdr =
1960 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&(
1961 pMsg->cd_ctrl);
1962 pSpcCdCtrlHdr->cipher_state_sz =
1963 LAC_BYTES_TO_QUADWORDS(
1964 LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE);
1965 }
1966 /*populate the cipher request parameters */
1967 if (CPA_STATUS_SUCCESS == status) {
1968 Cpa64U ivBufferPhysAddr = 0;
1969
1970 if (pIvBuffer != NULL) {
1971 /* User OpData memory being used for IV
1972 * buffer */
1973 /* get the physical address */
1974 ivBufferPhysAddr =
1975 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1976 pService->generic_service_info,
1977 pIvBuffer);
1978 if (0 == ivBufferPhysAddr) {
1979 LAC_LOG_ERROR(
1980 "Unable to get the physical address "
1981 "of the IV\n");
1982 status = CPA_STATUS_FAIL;
1983 }
1984 }
1985
1986 if (status == CPA_STATUS_SUCCESS) {
1987 status =
1988 LacSymQat_CipherRequestParamsPopulate(
1989 pSessionDesc,
1990 pMsg,
1991 pOpData
1992 ->cryptoStartSrcOffsetInBytes,
1993 pOpData
1994 ->messageLenToCipherInBytes,
1995 ivBufferPhysAddr,
1996 pIvBuffer);
1997 }
1998 }
1999
2000 if ((SPC == pSessionDesc->singlePassState) &&
2001 CPA_STATUS_SUCCESS == status) {
2002 Cpa64U aadBufferPhysAddr = 0;
2003
2004 /* For CHACHA and AES-GCM there is an AAD buffer
2005 * if aadLenInBytes is nonzero In case of
2006 * AES-GMAC, AAD buffer passed in the src
2007 * buffer.
2008 */
2009 if ((0 != aadLenInBytes &&
2010 CPA_CY_SYM_HASH_AES_GMAC != hash) ||
2011 isSpCcm) {
2012 LAC_CHECK_NULL_PARAM(
2013 pOpData->pAdditionalAuthData);
2014 Cpa32U aadDataLen =
2015 pSessionDesc->aadLenInBytes;
2016
2017 /* In case of AES_CCM, B0 block size and
2018 * 2 bytes of AAD len encoding need to
2019 * be added to total AAD data len */
2020 if (isSpCcm)
2021 aadDataLen +=
2022 LAC_CIPHER_CCM_AAD_OFFSET;
2023
2024 blockLen =
2025 LacSymQat_CipherBlockSizeBytesGet(
2026 cipher);
2027 if ((aadDataLen % blockLen) != 0) {
2028 paddingLen = blockLen -
2029 (aadDataLen % blockLen);
2030 memset(
2031 &pOpData
2032 ->pAdditionalAuthData
2033 [aadDataLen],
2034 0,
2035 paddingLen);
2036 }
2037
2038 /* User OpData memory being used for AAD
2039 * buffer */
2040 /* get the physical address */
2041 aadBufferPhysAddr =
2042 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2043 pService->generic_service_info,
2044 pOpData->pAdditionalAuthData);
2045 if (0 == aadBufferPhysAddr) {
2046 LAC_LOG_ERROR(
2047 "Unable to get the physical address "
2048 "of the aad\n");
2049 status = CPA_STATUS_FAIL;
2050 }
2051 }
2052
2053 if (CPA_STATUS_SUCCESS == status) {
2054 icp_qat_fw_la_cipher_req_params_t *pCipherReqParams =
2055 (icp_qat_fw_la_cipher_req_params_t
2056 *)((Cpa8U *)&(
2057 pMsg->serv_specif_rqpars) +
2058 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2059
2060 icp_qat_fw_la_cipher_20_req_params_t
2061 *pCipher20ReqParams =
2062 (void
2063 *)((Cpa8U *)&(
2064 pMsg->serv_specif_rqpars) +
2065 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2066
2067 if (isCyGen4x(pService)) {
2068 pCipher20ReqParams
2069 ->spc_aad_addr =
2070 aadBufferPhysAddr;
2071 pCipher20ReqParams->spc_aad_sz =
2072 pSessionDesc->aadLenInBytes;
2073 pCipher20ReqParams
2074 ->spc_aad_offset = 0;
2075 if (isSpCcm)
2076 pCipher20ReqParams
2077 ->spc_aad_sz +=
2078 LAC_CIPHER_CCM_AAD_OFFSET;
2079 } else {
2080 pCipherReqParams->spc_aad_addr =
2081 aadBufferPhysAddr;
2082 pCipherReqParams->spc_aad_sz =
2083 (Cpa16U)pSessionDesc
2084 ->aadLenInBytes;
2085 }
2086
2087 if (CPA_TRUE !=
2088 pSessionDesc->digestIsAppended) {
2089 Cpa64U digestBufferPhysAddr = 0;
2090 /* User OpData memory being used
2091 * for digest buffer */
2092 /* get the physical address */
2093 digestBufferPhysAddr =
2094 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2095 pService
2096 ->generic_service_info,
2097 pOpData->pDigestResult);
2098 if (0 != digestBufferPhysAddr) {
2099 if (isCyGen4x(
2100 pService)) {
2101 pCipher20ReqParams
2102 ->spc_auth_res_addr =
2103 digestBufferPhysAddr;
2104 pCipher20ReqParams
2105 ->spc_auth_res_sz =
2106 (Cpa8U)pSessionDesc
2107 ->hashResultSize;
2108 } else {
2109 pCipherReqParams
2110 ->spc_auth_res_addr =
2111 digestBufferPhysAddr;
2112 pCipherReqParams
2113 ->spc_auth_res_sz =
2114 (Cpa8U)pSessionDesc
2115 ->hashResultSize;
2116 }
2117 } else {
2118 LAC_LOG_ERROR(
2119 "Unable to get the physical address "
2120 "of the digest\n");
2121 status =
2122 CPA_STATUS_FAIL;
2123 }
2124 } else {
2125 /* Check if the dest buffer can
2126 * handle the digest, only for
2127 * last packet */
2128 if (((ICP_QAT_FW_LA_PARTIAL_NONE ==
2129 qatPacketType) ||
2130 (ICP_QAT_FW_LA_PARTIAL_END ==
2131 qatPacketType))) {
2132 if (dstPktSize <
2133 (pOpData
2134 ->cryptoStartSrcOffsetInBytes +
2135 pOpData
2136 ->messageLenToCipherInBytes +
2137 pSessionDesc
2138 ->hashResultSize))
2139 status =
2140 CPA_STATUS_INVALID_PARAM;
2141 }
2142 if (isCyGen4x(pService)) {
2143 pCipher20ReqParams
2144 ->spc_auth_res_sz =
2145 (Cpa8U)pSessionDesc
2146 ->hashResultSize;
2147 } else {
2148 pCipherReqParams
2149 ->spc_auth_res_sz =
2150 (Cpa8U)pSessionDesc
2151 ->hashResultSize;
2152 }
2153 }
2154 }
2155 }
2156 }
2157
2158 /*
2159 * Set up HashRequestParams part of Request
2160 */
2161 if ((status == CPA_STATUS_SUCCESS) &&
2162 (laCmdId != ICP_QAT_FW_LA_CMD_CIPHER)) {
2163 Cpa32U authOffsetInBytes =
2164 pOpData->hashStartSrcOffsetInBytes;
2165 Cpa32U authLenInBytes =
2166 pOpData->messageLenToHashInBytes;
2167
2168 status = LacHash_PerformParamCheck(instanceHandle,
2169 pSessionDesc,
2170 pOpData,
2171 srcPktSize,
2172 pVerifyResult);
2173 if (CPA_STATUS_SUCCESS != status) {
2174 /* free the cookie */
2175 Lac_MemPoolEntryFree(pCookie);
2176 return status;
2177 }
2178 if (CPA_STATUS_SUCCESS == status) {
2179 /* Info structure for CCM/GCM */
2180 lac_sym_qat_hash_state_buffer_info_t
2181 hashStateBufferInfo = { 0 };
2182 lac_sym_qat_hash_state_buffer_info_t
2183 *pHashStateBufferInfo =
2184 &(pSessionDesc->hashStateBufferInfo);
2185
2186 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
2187 icp_qat_fw_la_auth_req_params_t *pHashReqParams =
2188 (icp_qat_fw_la_auth_req_params_t
2189 *)((Cpa8U *)&(
2190 pMsg->serv_specif_rqpars) +
2191 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
2192
2193 hashStateBufferInfo.pData =
2194 pOpData->pAdditionalAuthData;
2195 if (pOpData->pAdditionalAuthData ==
2196 NULL) {
2197 hashStateBufferInfo.pDataPhys =
2198 0;
2199 } else {
2200 hashStateBufferInfo
2201 .pDataPhys = LAC_MEM_CAST_PTR_TO_UINT64(
2202 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2203 pService
2204 ->generic_service_info,
2205 pOpData
2206 ->pAdditionalAuthData));
2207 }
2208
2209 hashStateBufferInfo
2210 .stateStorageSzQuadWords = 0;
2211 hashStateBufferInfo
2212 .prefixAadSzQuadWords =
2213 LAC_BYTES_TO_QUADWORDS(
2214 pHashReqParams->u2.aad_sz);
2215
2216 /* Overwrite hash state buffer info
2217 * structure pointer with the one
2218 * created for CCM/GCM */
2219 pHashStateBufferInfo =
2220 &hashStateBufferInfo;
2221
2222 /* Aad buffer could be null in the GCM
2223 * case */
2224 if (0 ==
2225 hashStateBufferInfo.pDataPhys &&
2226 CPA_CY_SYM_HASH_AES_GCM != hash &&
2227 CPA_CY_SYM_HASH_AES_GMAC != hash) {
2228 LAC_LOG_ERROR(
2229 "Unable to get the physical address"
2230 "of the AAD\n");
2231 status = CPA_STATUS_FAIL;
2232 }
2233
2234 /* for CCM/GCM the hash and cipher data
2235 * regions are equal */
2236 authOffsetInBytes =
2237 pOpData
2238 ->cryptoStartSrcOffsetInBytes;
2239
2240 /* For authenticated encryption,
2241 * authentication length is determined
2242 * by messageLenToCipherInBytes for
2243 * AES-GCM and AES-CCM, and by
2244 * messageLenToHashInBytes for AES-GMAC.
2245 * You don't see the latter here, as
2246 * that is the initial value of
2247 * authLenInBytes. */
2248 if (hash != CPA_CY_SYM_HASH_AES_GMAC)
2249 authLenInBytes =
2250 pOpData
2251 ->messageLenToCipherInBytes;
2252 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
2253 hash ||
2254 CPA_CY_SYM_HASH_ZUC_EIA3 == hash) {
2255 hashStateBufferInfo.pData =
2256 pOpData->pAdditionalAuthData;
2257 hashStateBufferInfo.pDataPhys =
2258 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2259 pService->generic_service_info,
2260 hashStateBufferInfo.pData);
2261 hashStateBufferInfo
2262 .stateStorageSzQuadWords = 0;
2263 hashStateBufferInfo
2264 .prefixAadSzQuadWords =
2265 LAC_BYTES_TO_QUADWORDS(
2266 aadLenInBytes);
2267
2268 pHashStateBufferInfo =
2269 &hashStateBufferInfo;
2270
2271 if (0 ==
2272 hashStateBufferInfo.pDataPhys) {
2273 LAC_LOG_ERROR(
2274 "Unable to get the physical address"
2275 "of the AAD\n");
2276 status = CPA_STATUS_FAIL;
2277 }
2278 }
2279 if (CPA_CY_SYM_HASH_AES_CCM == hash) {
2280 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
2281 pSessionDesc->cipherDirection) {
2282 /* On a decrypt path pSrcBuffer
2283 * is used as this is where
2284 * encrypted digest is located.
2285 * Firmware uses encrypted
2286 * digest for
2287 * compare/verification*/
2288 pBufferList =
2289 (CpaBufferList *)pSrcBuffer;
2290 } else {
2291 /* On an encrypt path pDstBuffer
2292 * is used as this is where
2293 * encrypted digest will be
2294 * written */
2295 pBufferList =
2296 (CpaBufferList *)pDstBuffer;
2297 }
2298 status = LacSymAlgChain_PtrFromOffsetGet(
2299 pBufferList,
2300 pOpData->cryptoStartSrcOffsetInBytes +
2301 pOpData
2302 ->messageLenToCipherInBytes,
2303 &pDigestResult);
2304 if (CPA_STATUS_SUCCESS != status) {
2305 LAC_LOG_ERROR(
2306 "Cannot set digest pointer within the"
2307 " buffer list - offset out of bounds");
2308 }
2309 } else {
2310 pDigestResult = pOpData->pDigestResult;
2311 }
2312
2313 if (CPA_TRUE ==
2314 pSessionDesc->useStatefulSha3ContentDesc) {
2315 LacAlgChain_StatefulSha3_SkipStateLoadFlags(
2316 pMsg,
2317 qatPacketType,
2318 pSessionDesc->qatHashMode);
2319 }
2320
2321 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
2322 pSessionDesc->symOperation) {
2323 /* In alg chaining mode, packets are not
2324 * seen as partials for hash operations.
2325 * Override to NONE.
2326 */
2327 qatPacketType =
2328 ICP_QAT_FW_LA_PARTIAL_NONE;
2329 }
2330 digestIsAppended =
2331 pSessionDesc->digestIsAppended;
2332 if (CPA_TRUE == digestIsAppended) {
2333 /*Check if the destination buffer can
2334 * handle the digest if digestIsAppend
2335 * is true*/
2336 if (srcPktSize <
2337 (authOffsetInBytes +
2338 authLenInBytes +
2339 pSessionDesc->hashResultSize)) {
2340 status =
2341 CPA_STATUS_INVALID_PARAM;
2342 }
2343 }
2344 if (CPA_STATUS_SUCCESS == status) {
2345 /* populate the hash request parameters
2346 */
2347 status =
2348 LacSymQat_HashRequestParamsPopulate(
2349 pMsg,
2350 authOffsetInBytes,
2351 authLenInBytes,
2352 &(pService
2353 ->generic_service_info),
2354 pHashStateBufferInfo,
2355 qatPacketType,
2356 pSessionDesc->hashResultSize,
2357 pSessionDesc->digestVerify,
2358 digestIsAppended ?
2359 NULL :
2360 pDigestResult,
2361 hash,
2362 NULL);
2363 }
2364 }
2365 }
2366 }
2367
2368 /*
2369 * send the message to the QAT
2370 */
2371 if (CPA_STATUS_SUCCESS == status) {
2372 qatUtilsAtomicInc(&(pSessionDesc->u.pendingCbCount));
2373
2374 status = LacSymQueue_RequestSend(instanceHandle,
2375 pCookie,
2376 pSessionDesc);
2377
2378 if (CPA_STATUS_SUCCESS != status) {
2379 /* Decrease pending callback counter on send fail. */
2380 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
2381 }
2382 }
2383 /* Case that will catch all error status's for this function */
2384 if (CPA_STATUS_SUCCESS != status) {
2385 /* free the cookie */
2386 if (NULL != pSymCookie) {
2387 Lac_MemPoolEntryFree(pSymCookie);
2388 }
2389 }
2390 return status;
2391 }
2392