1 /***************************************************************************
2 *
3 * <COPYRIGHT_TAG>
4 *
5 ***************************************************************************/
6
7 /**
8 ***************************************************************************
9 * @file lac_sym_alg_chain.c Algorithm Chaining Perform
10 *
11 * @ingroup LacAlgChain
12 ***************************************************************************/
13
14 /*
15 *******************************************************************************
16 * Include public/global header files
17 *******************************************************************************
18 */
19
20 #include "cpa.h"
21 #include "cpa_cy_sym.h"
22
23 #include "icp_accel_devices.h"
24 #include "icp_adf_init.h"
25 #include "icp_adf_transport.h"
26 #include "icp_adf_debug.h"
27
28 /*
29 *******************************************************************************
30 * Include private header files
31 *******************************************************************************
32 */
33
34 #include "lac_mem.h"
35 #include "lac_log.h"
36 #include "lac_sym.h"
37 #include "lac_list.h"
38 #include "icp_qat_fw_la.h"
39 #include "lac_sal_types_crypto.h"
40 #include "lac_sal.h"
41 #include "lac_sal_ctrl.h"
42 #include "lac_sym_alg_chain.h"
43 #include "lac_sym_cipher.h"
44 #include "lac_sym_cipher_defs.h"
45 #include "lac_sym_hash.h"
46 #include "lac_sym_hash_defs.h"
47 #include "lac_sym_qat_cipher.h"
48 #include "lac_sym_qat_hash.h"
49 #include "lac_sym_stats.h"
50 #include "lac_sym_queue.h"
51 #include "lac_sym_cb.h"
52 #include "sal_string_parse.h"
53 #include "lac_sym_auth_enc.h"
54 #include "lac_sym_qat.h"
55 #include "sal_hw_gen.h"
56
57 /**
58 * @ingroup LacAlgChain
59 * This callback function will be invoked whenever a hash precompute
60 * operation completes. It will dequeue and send any QAT requests
61 * which were queued up while the precompute was in progress.
62 *
63 * @param[in] callbackTag Opaque value provided by user. This will
64 * be a pointer to the session descriptor.
65 *
66 * @retval
67 * None
68 *
69 */
70 static void
LacSymAlgChain_HashPrecomputeDoneCb(void * callbackTag)71 LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag)
72 {
73 LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag);
74 }
75
76 /**
77 * @ingroup LacAlgChain
78 * Walk the buffer list and find the address for the given offset within
79 * a buffer.
80 *
81 * @param[in] pBufferList Buffer List
82 * @param[in] packetOffset Offset in the buffer list for which address
83 * is to be found.
84 * @param[out] ppDataPtr This is where the sought pointer will be put
85 * @param[out] pSpaceLeft Pointer to a variable in which information about
86 * available space from the given offset to the end
87 * of the flat buffer it is located in will be returned
88 *
89 * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list
90 * @retval CPA_STATUS_FAIL Address with a given offset not found in the list.
91 *
92 */
93 static CpaStatus
LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList * pBufferList,const Cpa32U packetOffset,Cpa8U ** ppDataPtr)94 LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList,
95 const Cpa32U packetOffset,
96 Cpa8U **ppDataPtr)
97 {
98 Cpa32U currentOffset = 0;
99 Cpa32U i = 0;
100
101 for (i = 0; i < pBufferList->numBuffers; i++) {
102 Cpa8U *pCurrData = pBufferList->pBuffers[i].pData;
103 Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes;
104
105 /* If the offset is within the address space of the current
106 * buffer */
107 if ((packetOffset >= currentOffset) &&
108 (packetOffset < (currentOffset + currDataSize))) {
109 /* increment by offset of the address in the current
110 * buffer */
111 *ppDataPtr = pCurrData + (packetOffset - currentOffset);
112 return CPA_STATUS_SUCCESS;
113 }
114
115 /* Increment by the size of the buffer */
116 currentOffset += currDataSize;
117 }
118
119 return CPA_STATUS_FAIL;
120 }
121
122 /**
123 * @ingroup LacAlgChain
124 * Function which checks for support of partial packets for symmetric
125 * crypto operations
126 *
127 * @param[in] pService Pointer to service descriptor
128 * @param[in/out] pSessionDesc Pointer to session descriptor
129 *
130 */
131 static void
LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)132 LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,
133 lac_session_desc_t *pSessionDesc)
134 {
135 CpaBoolean isHashPartialSupported = CPA_FALSE;
136 CpaBoolean isCipherPartialSupported = CPA_FALSE;
137 CpaBoolean isPartialSupported = CPA_FALSE;
138
139 switch (pSessionDesc->cipherAlgorithm) {
140 /* Following ciphers don't support partial */
141 case CPA_CY_SYM_CIPHER_KASUMI_F8:
142 case CPA_CY_SYM_CIPHER_AES_F8:
143 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
144 case CPA_CY_SYM_CIPHER_CHACHA:
145 case CPA_CY_SYM_CIPHER_ZUC_EEA3:
146 break;
147 /* All others support partial */
148 default:
149 isCipherPartialSupported = CPA_TRUE;
150 break;
151 }
152 switch (pSessionDesc->hashAlgorithm) {
153 /* Following hash don't support partial */
154 case CPA_CY_SYM_HASH_KASUMI_F9:
155 case CPA_CY_SYM_HASH_SNOW3G_UIA2:
156 case CPA_CY_SYM_HASH_POLY:
157 case CPA_CY_SYM_HASH_ZUC_EIA3:
158 break;
159 /* Following hash may support partial based on device capabilities */
160 case CPA_CY_SYM_HASH_SHA3_256:
161 if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) {
162 isHashPartialSupported = CPA_TRUE;
163 }
164 break;
165 /* All others support partial */
166 default:
167 isHashPartialSupported = CPA_TRUE;
168 break;
169 }
170 switch (pSessionDesc->symOperation) {
171 case CPA_CY_SYM_OP_CIPHER:
172 isPartialSupported = isCipherPartialSupported;
173 break;
174 case CPA_CY_SYM_OP_HASH:
175 isPartialSupported = isHashPartialSupported;
176 break;
177 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
178 if (isCipherPartialSupported && isHashPartialSupported) {
179 isPartialSupported = CPA_TRUE;
180 }
181 break;
182 case CPA_CY_SYM_OP_NONE:
183 break;
184 default:
185 break;
186 }
187
188 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
189 /* UCS slice has no support for state flush and
190 * because of that is not able to do partial processing */
191 isPartialSupported = CPA_FALSE;
192 }
193
194 pSessionDesc->isPartialSupported = isPartialSupported;
195 }
196
197 static void
LacAlgChain_CipherCDBuild_ForOptimisedCD(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)198 LacAlgChain_CipherCDBuild_ForOptimisedCD(
199 const CpaCySymCipherSetupData *pCipherData,
200 lac_session_desc_t *pSessionDesc,
201 icp_qat_fw_slice_t nextSlice,
202 Cpa8U cipherOffsetInConstantsTable,
203 Cpa8U *pOptimisedHwBlockBaseInDRAM,
204 Cpa32U *pOptimisedHwBlockOffsetInDRAM)
205 {
206 Cpa8U *pCipherKeyField = NULL;
207 Cpa32U sizeInBytes = 0;
208 pCipherKeyField = pOptimisedHwBlockBaseInDRAM;
209
210 /* Need to build up the alternative CD for SHRAM Constants Table use
211 * with an optimised content desc of 64 bytes for this case. Cipher key
212 * will be in the Content desc in DRAM, The cipher config data
213 * is now in the SHRAM constants table. */
214
215 LacSymQat_CipherHwBlockPopulateKeySetup(
216 pSessionDesc,
217 pCipherData,
218 pCipherData->cipherKeyLenInBytes,
219 pSessionDesc->cipherSliceType,
220 pCipherKeyField,
221 &sizeInBytes);
222
223 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
224 pSessionDesc->cipherAlgorithm,
225 pSessionDesc->cipherKeyLenInBytes,
226 pSessionDesc->cipherSliceType,
227 nextSlice,
228 cipherOffsetInConstantsTable);
229
230 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
231 }
232
233 static void
LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable)234 LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData,
235 lac_session_desc_t *pSessionDesc,
236 icp_qat_fw_slice_t nextSlice,
237 Cpa8U cipherOffsetInConstantsTable)
238 {
239 Cpa32U sizeInBytes = 0;
240 Cpa8U *pCipherKeyField = NULL;
241 /* Need to build up the alternative CD for SHRAM Constants Table use
242 * Cipher key will be in the Request, The cipher config data is now in
243 * the SHRAM constants table. And nothing is now stored in the content
244 * desc */
245 pCipherKeyField = (Cpa8U *)&(
246 pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields);
247
248 LacSymQat_CipherHwBlockPopulateKeySetup(
249 pSessionDesc,
250 pCipherData,
251 pCipherData->cipherKeyLenInBytes,
252 pSessionDesc->cipherSliceType,
253 pCipherKeyField,
254 &sizeInBytes);
255
256 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
257 pSessionDesc->cipherAlgorithm,
258 pSessionDesc->cipherKeyLenInBytes,
259 pSessionDesc->cipherSliceType,
260 nextSlice,
261 cipherOffsetInConstantsTable);
262 }
263
264 static void
LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa32U capabilitiesMask)265 LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData,
266 lac_session_desc_t *pSessionDesc,
267 icp_qat_fw_slice_t nextSlice,
268 Cpa8U cipherOffsetInConstantsTable,
269 icp_qat_fw_comn_flags *pCmnRequestFlags,
270 icp_qat_fw_serv_specif_flags *pLaCmdFlags,
271 Cpa8U *pHwBlockBaseInDRAM,
272 Cpa32U *pHwBlockOffsetInDRAM,
273 Cpa32U capabilitiesMask)
274 {
275 Cpa8U *pCipherKeyField = NULL;
276 Cpa8U cipherOffsetInReqQW = 0;
277 Cpa32U sizeInBytes = 0;
278 void *pCfgData = NULL;
279 Cpa32U cfgOffset = 0;
280
281 /* Construct the ContentDescriptor in DRAM */
282 cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES);
283 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
284 *pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP);
285
286 /* construct cipherConfig in CD in DRAM */
287 cfgOffset = *pHwBlockOffsetInDRAM;
288 pCfgData = pHwBlockBaseInDRAM + cfgOffset;
289 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
290 pCfgData,
291 &sizeInBytes);
292
293 ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags,
294 pSessionDesc->cipherSliceType);
295
296 *pHwBlockOffsetInDRAM += sizeInBytes;
297
298 /* Cipher key will be in CD in DRAM.
299 * The Request contains a ptr to the CD.
300 * This ptr will be copied into the request later once the CD is
301 * fully constructed, but the flag is set here. */
302 pCipherKeyField = pHwBlockBaseInDRAM + *pHwBlockOffsetInDRAM;
303 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(*pCmnRequestFlags,
304 QAT_COMN_CD_FLD_TYPE_64BIT_ADR);
305
306 LacSymQat_CipherHwBlockPopulateKeySetup(
307 pSessionDesc,
308 pCipherData,
309 pCipherData->cipherKeyLenInBytes,
310 pSessionDesc->cipherSliceType,
311 pCipherKeyField,
312 &sizeInBytes);
313 /* update offset */
314 *pHwBlockOffsetInDRAM += sizeInBytes;
315
316 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr),
317 pSessionDesc->cipherAlgorithm,
318 pSessionDesc->cipherKeyLenInBytes,
319 pSessionDesc->cipherSliceType,
320 nextSlice,
321 cipherOffsetInReqQW);
322 if (NON_SPC != pSessionDesc->singlePassState) {
323 LacSymQat_CipherCtrlBlockWrite(
324 &(pSessionDesc->reqSpcCacheFtr),
325 pSessionDesc->cipherAlgorithm,
326 pSessionDesc->cipherKeyLenInBytes,
327 pSessionDesc->cipherSliceType,
328 ICP_QAT_FW_SLICE_DRAM_WR,
329 cipherOffsetInReqQW);
330 }
331 }
332
333 static void
LacAlgChain_HashCDBuild(const CpaCySymHashSetupData * pHashData,CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U hashOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,lac_sym_qat_hash_precompute_info_t * pPrecomputeData,lac_sym_qat_hash_precompute_info_t * pPrecomputeDataOptimisedCd,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)334 LacAlgChain_HashCDBuild(
335 const CpaCySymHashSetupData *pHashData,
336 CpaInstanceHandle instanceHandle,
337 lac_session_desc_t *pSessionDesc,
338 icp_qat_fw_slice_t nextSlice,
339 Cpa8U hashOffsetInConstantsTable,
340 icp_qat_fw_comn_flags *pCmnRequestFlags,
341 icp_qat_fw_serv_specif_flags *pLaCmdFlags,
342 lac_sym_qat_hash_precompute_info_t *pPrecomputeData,
343 lac_sym_qat_hash_precompute_info_t *pPrecomputeDataOptimisedCd,
344 Cpa8U *pHwBlockBaseInDRAM,
345 Cpa32U *pHwBlockOffsetInDRAM,
346 Cpa8U *pOptimisedHwBlockBaseInDRAM,
347 Cpa32U *pOptimisedHwBlockOffsetInDRAM)
348 {
349 Cpa32U sizeInBytes = 0;
350 Cpa32U hwBlockOffsetInQuadWords =
351 *pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES;
352
353 /* build:
354 * - the hash part of the ContentDescriptor in DRAM */
355 /* - the hash part of the CD control block in the Request template */
356 LacSymQat_HashContentDescInit(&(pSessionDesc->reqCacheFtr),
357 instanceHandle,
358 pHashData,
359 pHwBlockBaseInDRAM,
360 hwBlockOffsetInQuadWords,
361 nextSlice,
362 pSessionDesc->qatHashMode,
363 CPA_FALSE,
364 CPA_FALSE,
365 pSessionDesc->useStatefulSha3ContentDesc,
366 pPrecomputeData,
367 &sizeInBytes);
368
369 /* Using DRAM CD so update offset */
370 *pHwBlockOffsetInDRAM += sizeInBytes;
371
372 sizeInBytes = 0;
373
374 if (pSessionDesc->useOptimisedContentDesc) {
375 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
376 instanceHandle,
377 pHashData,
378 pOptimisedHwBlockBaseInDRAM,
379 hashOffsetInConstantsTable,
380 nextSlice,
381 pSessionDesc->qatHashMode,
382 CPA_TRUE,
383 CPA_TRUE,
384 CPA_FALSE,
385 pPrecomputeDataOptimisedCd,
386 &sizeInBytes);
387
388 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
389 } else if (pSessionDesc->useSymConstantsTable) {
390 /* Need to build up the alternative CD for SHRAM Constants Table
391 * use */
392 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
393 instanceHandle,
394 pHashData,
395 pHwBlockBaseInDRAM,
396 hashOffsetInConstantsTable,
397 nextSlice,
398 pSessionDesc->qatHashMode,
399 CPA_TRUE,
400 CPA_FALSE,
401 CPA_FALSE,
402 pPrecomputeData,
403 &sizeInBytes);
404 }
405 }
406
407 static Cpa16U
LacAlgChain_GetCipherConfigSize(lac_session_desc_t * pSessionDesc)408 LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc)
409 {
410 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
411 return sizeof(icp_qat_hw_ucs_cipher_config_t);
412 } else {
413 return sizeof(icp_qat_hw_cipher_config_t);
414 }
415 }
416
417 static Cpa16U
LacAlgChain_GetCipherConfigOffset(lac_session_desc_t * pSessionDesc)418 LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc)
419 {
420 Cpa16U offset = 0;
421
422 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation ||
423 SPC == pSessionDesc->singlePassState) {
424 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
425 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc
426 ->reqCacheFtr.cd_ctrl;
427 offset = cd_ctrl->cipher_cfg_offset;
428 } else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) {
429 icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl =
430 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc
431 ->reqCacheFtr.cd_ctrl;
432 offset = cd_ctrl->cipher_cfg_offset;
433 }
434
435 return offset * LAC_QUAD_WORD_IN_BYTES;
436 }
437
438 CpaStatus
LacAlgChain_SessionAADUpdate(lac_session_desc_t * pSessionDesc,Cpa32U newAADLength)439 LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc,
440 Cpa32U newAADLength)
441 {
442 icp_qat_la_bulk_req_ftr_t *req_ftr = &pSessionDesc->reqCacheFtr;
443 icp_qat_la_auth_req_params_t *req_params = &req_ftr->serv_specif_rqpars;
444
445 if (!pSessionDesc)
446 return CPA_STATUS_FAIL;
447
448 pSessionDesc->aadLenInBytes = newAADLength;
449 req_params->u2.aad_sz =
450 LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE);
451
452 if (SPC == pSessionDesc->singlePassState) {
453 Cpa8U *pHwBlockBaseInDRAM = NULL;
454 Cpa32U hwBlockOffsetInDRAM = 0;
455 Cpa32U pSizeInBytes = 0;
456 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
457
458 pHwBlockBaseInDRAM =
459 (Cpa8U *)pSessionDesc->contentDescInfo.pData;
460 if (pSessionDesc->cipherDirection ==
461 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) {
462 if (LAC_CIPHER_IS_GCM(cipher)) {
463 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
464 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
465 } else {
466 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
467 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
468 }
469 }
470 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
471 pHwBlockBaseInDRAM +
472 hwBlockOffsetInDRAM,
473 &pSizeInBytes);
474 }
475
476 return CPA_STATUS_SUCCESS;
477 }
478
479 CpaStatus
LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pCipherKey)480 LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc,
481 Cpa8U *pCipherKey)
482 {
483 CpaStatus status = CPA_STATUS_SUCCESS;
484
485 if (pSessionDesc == NULL || pCipherKey == NULL)
486 return CPA_STATUS_FAIL;
487
488 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
489 LacSymQat_CipherArc4StateInit(
490 pCipherKey,
491 pSessionDesc->cipherKeyLenInBytes,
492 pSessionDesc->cipherARC4InitialState);
493 } else {
494 CpaCySymCipherSetupData cipherSetupData = { 0 };
495 Cpa32U sizeInBytes;
496 Cpa8U *pCipherKeyField;
497 Cpa16U cipherConfigSize;
498 Cpa16U cipherConfigOffset;
499 sal_qat_content_desc_info_t *pCdInfo =
500 &(pSessionDesc->contentDescInfo);
501
502 cipherSetupData.cipherAlgorithm = pSessionDesc->cipherAlgorithm;
503 cipherSetupData.cipherKeyLenInBytes =
504 pSessionDesc->cipherKeyLenInBytes;
505 cipherSetupData.pCipherKey = pCipherKey;
506 cipherSetupData.cipherDirection = pSessionDesc->cipherDirection;
507
508 cipherConfigSize =
509 LacAlgChain_GetCipherConfigSize(pSessionDesc);
510 cipherConfigOffset =
511 LacAlgChain_GetCipherConfigOffset(pSessionDesc);
512
513 pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset +
514 cipherConfigSize;
515
516 switch (pSessionDesc->symOperation) {
517 case CPA_CY_SYM_OP_CIPHER: {
518 LacSymQat_CipherHwBlockPopulateKeySetup(
519 pSessionDesc,
520 &(cipherSetupData),
521 cipherSetupData.cipherKeyLenInBytes,
522 pSessionDesc->cipherSliceType,
523 pCipherKeyField,
524 &sizeInBytes);
525
526 if (pSessionDesc->useSymConstantsTable) {
527 pCipherKeyField = (Cpa8U *)&(
528 pSessionDesc->shramReqCacheHdr.cd_pars.s1
529 .serv_specif_fields);
530
531 LacSymQat_CipherHwBlockPopulateKeySetup(
532 pSessionDesc,
533 &(cipherSetupData),
534 cipherSetupData.cipherKeyLenInBytes,
535 pSessionDesc->cipherSliceType,
536 pCipherKeyField,
537 &sizeInBytes);
538 }
539 } break;
540
541 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
542 LacSymQat_CipherHwBlockPopulateKeySetup(
543 pSessionDesc,
544 &(cipherSetupData),
545 cipherSetupData.cipherKeyLenInBytes,
546 pSessionDesc->cipherSliceType,
547 pCipherKeyField,
548 &sizeInBytes);
549 } break;
550
551 default:
552 LAC_LOG_ERROR("Invalid sym operation\n");
553 status = CPA_STATUS_INVALID_PARAM;
554 break;
555 }
556 }
557 return status;
558 }
559
560 CpaStatus
LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pAuthKey)561 LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc,
562 Cpa8U *pAuthKey)
563 {
564 CpaStatus status = CPA_STATUS_SUCCESS;
565 Cpa8U *pHwBlockBaseInDRAM = NULL;
566 Cpa8U *pOutHashSetup = NULL;
567 Cpa8U *pInnerState1 = NULL;
568 Cpa8U *pInnerState2 = NULL;
569 CpaCySymSessionSetupData sessionSetup = { 0 };
570 Cpa16U cipherConfigSize;
571
572 if (pSessionDesc == NULL || pAuthKey == NULL)
573 return CPA_STATUS_FAIL;
574
575 cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc);
576
577 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
578 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
579 .cd_ctrl;
580
581 pHwBlockBaseInDRAM = (Cpa8U *)pSessionDesc->contentDescInfo.pData;
582
583 sessionSetup.hashSetupData.hashAlgorithm = pSessionDesc->hashAlgorithm;
584 sessionSetup.hashSetupData.hashMode = pSessionDesc->hashMode;
585 sessionSetup.hashSetupData.authModeSetupData.authKey = pAuthKey;
586 sessionSetup.hashSetupData.authModeSetupData.authKeyLenInBytes =
587 pSessionDesc->authKeyLenInBytes;
588 sessionSetup.hashSetupData.authModeSetupData.aadLenInBytes =
589 pSessionDesc->aadLenInBytes;
590 sessionSetup.hashSetupData.digestResultLenInBytes =
591 pSessionDesc->hashResultSize;
592
593 sessionSetup.cipherSetupData.cipherAlgorithm =
594 pSessionDesc->cipherAlgorithm;
595 sessionSetup.cipherSetupData.cipherKeyLenInBytes =
596 pSessionDesc->cipherKeyLenInBytes;
597
598 /* Calculate hash states offsets */
599 pInnerState1 = pHwBlockBaseInDRAM +
600 cd_ctrl->hash_cfg_offset * LAC_QUAD_WORD_IN_BYTES +
601 sizeof(icp_qat_hw_auth_setup_t);
602
603 pInnerState2 = pInnerState1 + cd_ctrl->inner_state1_sz;
604
605 pOutHashSetup = pInnerState2 + cd_ctrl->inner_state2_sz;
606
607 /* Calculate offset of cipher key */
608 if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
609 sessionSetup.cipherSetupData.pCipherKey =
610 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
611 } else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
612 sessionSetup.cipherSetupData.pCipherKey =
613 pOutHashSetup + cipherConfigSize;
614 } else if (SPC == pSessionDesc->singlePassState) {
615 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
616 Cpa32U hwBlockOffsetInDRAM = 0;
617
618 if (pSessionDesc->cipherDirection ==
619 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
620 sessionSetup.cipherSetupData.pCipherKey =
621 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
622 } else {
623 if (LAC_CIPHER_IS_GCM(cipher))
624 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
625 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
626 else
627 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
628 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
629 sessionSetup.cipherSetupData.pCipherKey =
630 (Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM +
631 cipherConfigSize;
632 }
633 }
634
635 if (!sessionSetup.cipherSetupData.pCipherKey)
636 return CPA_STATUS_FAIL;
637
638 if (CPA_CY_SYM_HASH_SHA3_256 == pSessionDesc->hashAlgorithm) {
639 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
640 lac_sym_qat_hash_state_buffer_info_t
641 *pHashStateBufferInfo =
642 &(pSessionDesc->hashStateBufferInfo);
643
644 sal_crypto_service_t *pService =
645 (sal_crypto_service_t *)pSessionDesc->pInstance;
646
647 status = LacHash_StatePrefixAadBufferInit(
648 &(pService->generic_service_info),
649 &(sessionSetup.hashSetupData),
650 &(pSessionDesc->reqCacheFtr),
651 pSessionDesc->qatHashMode,
652 pSessionDesc->hashStatePrefixBuffer,
653 pHashStateBufferInfo);
654 /* SHRAM Constants Table not used for Auth-Enc */
655 }
656 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) {
657 Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize;
658 memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes);
659 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm ||
660 CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) {
661 memcpy(pInnerState2, pAuthKey, pSessionDesc->authKeyLenInBytes);
662 } else if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm ||
663 CPA_CY_SYM_HASH_KASUMI_F9 == pSessionDesc->hashAlgorithm ||
664 IS_HASH_MODE_1(pSessionDesc->qatHashMode)) {
665 if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm) {
666 memset(pInnerState2, 0, cd_ctrl->inner_state2_sz);
667 }
668
669 /* Block messages until precompute is completed */
670 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
671
672 status = LacHash_PrecomputeDataCreate(
673 pSessionDesc->pInstance,
674 (CpaCySymSessionSetupData *)&(sessionSetup),
675 LacSymAlgChain_HashPrecomputeDoneCb,
676 pSessionDesc,
677 pSessionDesc->hashStatePrefixBuffer,
678 pInnerState1,
679 pInnerState2);
680 }
681
682 return status;
683 }
684
685 static void
buildCmdData(sal_crypto_service_t * pService,lac_session_desc_t * pSessionDesc,CpaCySymAlgChainOrder * chainOrder,Cpa16U * proto,icp_qat_fw_serv_specif_flags * laCmdFlags,icp_qat_fw_comn_flags * cmnRequestFlags)686 buildCmdData(sal_crypto_service_t *pService,
687 lac_session_desc_t *pSessionDesc,
688 CpaCySymAlgChainOrder *chainOrder,
689 Cpa16U *proto,
690 icp_qat_fw_serv_specif_flags *laCmdFlags,
691 icp_qat_fw_comn_flags *cmnRequestFlags)
692 {
693 /* LW 28 is used to set hash flags for AlgChaining. */
694 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
695 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
696 .cd_ctrl;
697
698 /* proto refers to Protocol Flags, which is legacy FW <=> IA interface
699 * for ZUC and Snow3G. Use extended protocol flags for AlgChaining.
700 */
701 *proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
702
703 switch (pSessionDesc->symOperation) {
704 case CPA_CY_SYM_OP_CIPHER:
705 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
706
707 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
708 pSessionDesc->cipherAlgorithm) {
709 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
710 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
711 pSessionDesc->cipherAlgorithm) {
712 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
713 }
714 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
715 *proto = ICP_QAT_FW_LA_CCM_PROTO;
716 }
717 break;
718
719 case CPA_CY_SYM_OP_HASH:
720 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH;
721 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
722 pSessionDesc->hashAlgorithm) {
723 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
724 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
725 pSessionDesc->hashAlgorithm) {
726 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
727 }
728 break;
729
730 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
731 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
732 *proto = ICP_QAT_FW_LA_CCM_PROTO;
733
734 /* Derive chainOrder from direction for isAuthEncryptOp
735 * cases */
736 /* For CCM & GCM modes: force digest verify flag _TRUE
737 for decrypt and _FALSE for encrypt. For all other
738 cases use user defined value */
739
740 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
741 pSessionDesc->cipherDirection) {
742 *chainOrder =
743 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
744 pSessionDesc->digestVerify = CPA_FALSE;
745 } else {
746 *chainOrder =
747 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
748 if (CPA_TRUE == pService->forceAEADMacVerify) {
749 pSessionDesc->digestVerify = CPA_TRUE;
750 }
751 }
752 } else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) {
753 *proto = ICP_QAT_FW_LA_GCM_PROTO;
754
755 /* Derive chainOrder from direction for isAuthEncryptOp
756 * cases */
757 /* For CCM & GCM modes: force digest verify flag _TRUE
758 for decrypt and _FALSE for encrypt. For all other
759 cases use user defined value */
760
761 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
762 pSessionDesc->cipherDirection) {
763 *chainOrder =
764 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
765 pSessionDesc->digestVerify = CPA_FALSE;
766 } else {
767 *chainOrder =
768 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
769 if (CPA_TRUE == pService->forceAEADMacVerify) {
770 pSessionDesc->digestVerify = CPA_TRUE;
771 }
772 }
773 } else if (LAC_CIPHER_IS_CHACHA(
774 pSessionDesc->cipherAlgorithm)) {
775 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
776 pSessionDesc->cipherDirection) {
777 *chainOrder =
778 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
779 } else {
780 *chainOrder =
781 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
782 }
783 } else {
784 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
785
786 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
787 pSessionDesc->cipherAlgorithm) {
788 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
789 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
790 pSessionDesc->cipherAlgorithm) {
791 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
792 }
793
794 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
795 pSessionDesc->hashAlgorithm) {
796 /* Need to set LW 28 hash flags as well. */
797 ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET(
798 cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2);
799 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
800 pSessionDesc->hashAlgorithm) {
801 /* Need to set LW 28 hash flags as well. */
802 ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET(
803 cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3);
804 }
805 }
806
807 if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH ==
808 *chainOrder) {
809 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
810 } else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
811 *chainOrder) {
812 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
813 }
814 break;
815
816 default:
817 break;
818 }
819
820 /*
821 * Build the header flags with the default settings for this session.
822 */
823 if (pSessionDesc->isDPSession == CPA_TRUE) {
824 *cmnRequestFlags =
825 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
826 LAC_SYM_DP_QAT_PTR_TYPE);
827 } else {
828 *cmnRequestFlags =
829 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
830 LAC_SYM_DEFAULT_QAT_PTR_TYPE);
831 }
832
833 LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation);
834
835 return;
836 }
837
838 static void
updateLaCmdFlags(lac_session_desc_t * pSessionDesc,Cpa16U proto,icp_qat_fw_serv_specif_flags * laCmdFlags)839 updateLaCmdFlags(lac_session_desc_t *pSessionDesc,
840 Cpa16U proto,
841 icp_qat_fw_serv_specif_flags *laCmdFlags)
842 {
843 if (pSessionDesc->isAuth) {
844 if (pSessionDesc->digestVerify) {
845 ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags,
846 ICP_QAT_FW_LA_CMP_AUTH_RES);
847 ICP_QAT_FW_LA_RET_AUTH_SET(
848 *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES);
849 } else {
850 ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags,
851 ICP_QAT_FW_LA_RET_AUTH_RES);
852 ICP_QAT_FW_LA_CMP_AUTH_SET(
853 *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
854 }
855 }
856
857 if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) ||
858 (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) {
859 /* New bit position (12) for ZUC. The FW provides a specific
860 * macro to use to set the ZUC proto flag. With the new FW I/F
861 * this needs to be set for both Cipher and Auth */
862 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto);
863 } else {
864 /* Configure the common header */
865 ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto);
866 }
867
868 /* set Append flag, if digest is appended */
869 if (pSessionDesc->digestIsAppended) {
870 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
871 *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
872 } else {
873 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
874 *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
875 }
876 }
877
878 static lac_single_pass_state_t
LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,CpaCySymHashAlgorithm hash,Cpa32U capabilitiesMask)879 LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,
880 CpaCySymHashAlgorithm hash,
881 Cpa32U capabilitiesMask)
882 {
883 lac_single_pass_state_t state = NON_SPC;
884 if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) {
885 switch (cipher) {
886 case CPA_CY_SYM_CIPHER_CHACHA: {
887 if (CPA_CY_SYM_HASH_POLY == hash)
888 state = SPC;
889 break;
890 }
891 case CPA_CY_SYM_CIPHER_AES_GCM: {
892 if ((CPA_CY_SYM_HASH_AES_GCM == hash) ||
893 (CPA_CY_SYM_HASH_AES_GMAC == hash))
894 state = LIKELY_SPC;
895 break;
896 }
897 case CPA_CY_SYM_CIPHER_AES_CCM: {
898 if (LAC_CIPHER_AES_V2(capabilitiesMask))
899 state = SPC;
900 }
901 default:
902 /* Do Nothing as it is NON_SPC */
903 break;
904 }
905 }
906 return state;
907 }
908
909 static CpaBoolean
LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)910 LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,
911 Cpa32U capabilitiesMask,
912 lac_session_desc_t *pSessionDesc)
913 {
914 CpaBoolean hasSha3Ext =
915 ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask;
916 CpaBoolean useStatefulSha3DescFlag = CPA_FALSE;
917
918 if (hasSha3Ext && !partialsNotRequired &&
919 (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) &&
920 LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) {
921 useStatefulSha3DescFlag = CPA_TRUE;
922 }
923 return useStatefulSha3DescFlag;
924 }
925
926 /** @ingroup LacAlgChain */
927 CpaStatus
LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,const CpaCySymSessionSetupData * pSessionSetupData,lac_session_desc_t * pSessionDesc)928 LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,
929 const CpaCySymSessionSetupData *pSessionSetupData,
930 lac_session_desc_t *pSessionDesc)
931 {
932 CpaStatus stat, status = CPA_STATUS_SUCCESS;
933 sal_qat_content_desc_info_t *pCdInfo = NULL;
934 sal_qat_content_desc_info_t *pCdInfoOptimised = NULL;
935 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
936 Cpa32U capabilitiesMask =
937 pService->generic_service_info.capabilitiesMask;
938 Cpa8U *pHwBlockBaseInDRAM = NULL;
939 Cpa8U *pOptimisedHwBlockBaseInDRAM = NULL;
940 Cpa32U hwBlockOffsetInDRAM = 0;
941 Cpa32U optimisedHwBlockOffsetInDRAM = 0;
942 Cpa8U cipherOffsetInConstantsTable = 0;
943 Cpa8U hashOffsetInConstantsTable = 0;
944 icp_qat_fw_comn_flags cmnRequestFlags = 0;
945 icp_qat_fw_comn_req_t *pMsg = NULL;
946 icp_qat_fw_comn_req_t *pMsgS = NULL;
947 const CpaCySymCipherSetupData *pCipherData;
948 const CpaCySymHashSetupData *pHashData;
949 Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
950 CpaCySymAlgChainOrder chainOrder = 0;
951 lac_sym_qat_hash_precompute_info_t precomputeData = { 0 };
952 lac_sym_qat_hash_precompute_info_t precomputeDataOptimisedCd = { 0 };
953
954 pCipherData = &(pSessionSetupData->cipherSetupData);
955 pHashData = &(pSessionSetupData->hashSetupData);
956
957 /*-------------------------------------------------------------------------
958 * Populate session data
959 *-----------------------------------------------------------------------*/
960
961 /* Initialise Request Queue */
962 stat = LAC_SPINLOCK_INIT(&pSessionDesc->requestQueueLock);
963 if (CPA_STATUS_SUCCESS != stat) {
964 LAC_LOG_ERROR("Spinlock init failed for sessionLock");
965 return CPA_STATUS_RESOURCE;
966 }
967
968 pSessionDesc->pRequestQueueHead = NULL;
969 pSessionDesc->pRequestQueueTail = NULL;
970 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
971 pSessionDesc->pInstance = instanceHandle;
972 pSessionDesc->digestIsAppended = pSessionSetupData->digestIsAppended;
973 pSessionDesc->digestVerify = pSessionSetupData->verifyDigest;
974
975 /* Reset the pending callback counter */
976 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingCbCount);
977 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingDpCbCount);
978
979 /* Partial state must be set to full, to indicate that next packet
980 * expected on the session is a full packet or the start of a
981 * partial packet. */
982 pSessionDesc->partialState = CPA_CY_SYM_PACKET_TYPE_FULL;
983
984 pSessionDesc->symOperation = pSessionSetupData->symOperation;
985 switch (pSessionDesc->symOperation) {
986 case CPA_CY_SYM_OP_CIPHER:
987 pSessionDesc->isCipher = CPA_TRUE;
988 pSessionDesc->isAuth = CPA_FALSE;
989 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
990 pSessionDesc->singlePassState = NON_SPC;
991 break;
992 case CPA_CY_SYM_OP_HASH:
993 pSessionDesc->isCipher = CPA_FALSE;
994 pSessionDesc->isAuth = CPA_TRUE;
995 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
996 pSessionDesc->singlePassState = NON_SPC;
997 break;
998 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
999 pSessionDesc->isCipher = CPA_TRUE;
1000 pSessionDesc->isAuth = CPA_TRUE;
1001 pSessionDesc->singlePassState =
1002 LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm,
1003 pHashData->hashAlgorithm,
1004 capabilitiesMask);
1005
1006 switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) {
1007 case CPA_CY_SYM_CIPHER_AES_CCM: {
1008 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1009 pSessionDesc->digestIsAppended = CPA_TRUE;
1010 } break;
1011 case CPA_CY_SYM_CIPHER_AES_GCM:
1012 case CPA_CY_SYM_CIPHER_CHACHA:
1013 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1014 break;
1015 default: {
1016 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1017 /* Use the chainOrder passed in */
1018 chainOrder = pSessionSetupData->algChainOrder;
1019 if ((chainOrder !=
1020 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) &&
1021 (chainOrder !=
1022 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) {
1023 LAC_INVALID_PARAM_LOG("algChainOrder");
1024 return CPA_STATUS_INVALID_PARAM;
1025 }
1026 } break;
1027 }
1028 } break;
1029 default:
1030 pSessionDesc->singlePassState = NON_SPC;
1031 break;
1032 }
1033
1034 if (pSessionDesc->isCipher) {
1035 /* Populate cipher specific session data */
1036
1037 status = LacCipher_SessionSetupDataCheck(pCipherData,
1038 capabilitiesMask);
1039
1040 if (CPA_STATUS_SUCCESS == status) {
1041 pSessionDesc->cipherAlgorithm =
1042 pCipherData->cipherAlgorithm;
1043 pSessionDesc->cipherKeyLenInBytes =
1044 pCipherData->cipherKeyLenInBytes;
1045 pSessionDesc->cipherDirection =
1046 pCipherData->cipherDirection;
1047
1048 /* ARC4 base key isn't added to the content descriptor,
1049 * because we don't need to pass it directly to the QAT
1050 * engine. Instead an initial cipher state & key matrix
1051 * is derived from the base key and provided to the QAT
1052 * through the state pointer in the request params.
1053 * We'll store this initial state in the session
1054 * descriptor. */
1055
1056 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
1057 LacSymQat_CipherArc4StateInit(
1058 pCipherData->pCipherKey,
1059 pSessionDesc->cipherKeyLenInBytes,
1060 pSessionDesc->cipherARC4InitialState);
1061
1062 pSessionDesc->cipherARC4InitialStatePhysAddr =
1063 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1064 pService->generic_service_info,
1065 pSessionDesc->cipherARC4InitialState);
1066
1067 if (0 ==
1068 pSessionDesc
1069 ->cipherARC4InitialStatePhysAddr) {
1070 LAC_LOG_ERROR(
1071 "Unable to get the physical address of "
1072 "the initial state for ARC4\n");
1073 status = CPA_STATUS_FAIL;
1074 }
1075 }
1076 }
1077 }
1078
1079 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1080 /* Populate auth-specific session data */
1081 const CpaCySymHashSetupData *pHashData =
1082 &pSessionSetupData->hashSetupData;
1083
1084 status = LacHash_HashContextCheck(instanceHandle, pHashData);
1085 if (CPA_STATUS_SUCCESS == status) {
1086 pSessionDesc->hashResultSize =
1087 pHashData->digestResultLenInBytes;
1088 pSessionDesc->hashMode = pHashData->hashMode;
1089 pSessionDesc->hashAlgorithm = pHashData->hashAlgorithm;
1090
1091 /* Save the authentication key length for further update
1092 */
1093 if (CPA_CY_SYM_HASH_MODE_AUTH == pHashData->hashMode) {
1094 pSessionDesc->authKeyLenInBytes =
1095 pHashData->authModeSetupData
1096 .authKeyLenInBytes;
1097 }
1098 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp ||
1099 (pHashData->hashAlgorithm ==
1100 CPA_CY_SYM_HASH_SNOW3G_UIA2 ||
1101 pHashData->hashAlgorithm ==
1102 CPA_CY_SYM_HASH_ZUC_EIA3)) {
1103 pSessionDesc->aadLenInBytes =
1104 pHashData->authModeSetupData.aadLenInBytes;
1105 }
1106
1107 /* Set the QAT hash mode */
1108 if ((pHashData->hashMode ==
1109 CPA_CY_SYM_HASH_MODE_NESTED) ||
1110 (pHashData->hashMode ==
1111 CPA_CY_SYM_HASH_MODE_PLAIN) ||
1112 (pHashData->hashMode == CPA_CY_SYM_HASH_MODE_AUTH &&
1113 pHashData->hashAlgorithm ==
1114 CPA_CY_SYM_HASH_AES_CBC_MAC)) {
1115 pSessionDesc->qatHashMode =
1116 ICP_QAT_HW_AUTH_MODE0;
1117 } else /* CPA_CY_SYM_HASH_MODE_AUTH
1118 && anything except CPA_CY_SYM_HASH_AES_CBC_MAC
1119 */
1120 {
1121 if (IS_HMAC_ALG(pHashData->hashAlgorithm)) {
1122 /* SHA3 HMAC and SM3 do not support
1123 * precompute, force MODE2 for AUTH */
1124 if (LAC_HASH_IS_SHA3(
1125 pHashData->hashAlgorithm) ||
1126 (CPA_CY_SYM_HASH_SM3 ==
1127 pHashData->hashAlgorithm)) {
1128 pSessionDesc->qatHashMode =
1129 ICP_QAT_HW_AUTH_MODE2;
1130 } else {
1131 pSessionDesc->qatHashMode =
1132 pService->qatHmacMode;
1133 }
1134 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
1135 pHashData->hashAlgorithm) {
1136 pSessionDesc->qatHashMode =
1137 ICP_QAT_HW_AUTH_MODE0;
1138 } else {
1139 pSessionDesc->qatHashMode =
1140 ICP_QAT_HW_AUTH_MODE1;
1141 }
1142 }
1143 }
1144 }
1145
1146 /*-------------------------------------------------------------------------
1147 * build the message templates
1148 * create two content descriptors in the case we can support using SHRAM
1149 * constants and an optimised content descriptor. we have to do this in
1150 * case of partials. 64 byte content descriptor is used in the SHRAM
1151 * case for AES-128-HMAC-SHA1
1152 *-----------------------------------------------------------------------*/
1153 if (CPA_STATUS_SUCCESS == status) {
1154 pSessionDesc->cipherSliceType =
1155 LacCipher_GetCipherSliceType(pService,
1156 pSessionDesc->cipherAlgorithm,
1157 pSessionDesc->hashAlgorithm);
1158
1159 LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc);
1160 pSessionDesc->useOptimisedContentDesc = CPA_FALSE;
1161 pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE;
1162
1163 /* Build configuration data */
1164 buildCmdData(pService,
1165 pSessionDesc,
1166 &chainOrder,
1167 &proto,
1168 &pSessionDesc->laCmdFlags,
1169 &cmnRequestFlags);
1170
1171 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE ==
1172 pSessionDesc->cipherSliceType)
1173 pSessionDesc->useSymConstantsTable = CPA_FALSE;
1174 else
1175 pSessionDesc->useSymConstantsTable =
1176 LacSymQat_UseSymConstantsTable(
1177 pSessionDesc,
1178 &cipherOffsetInConstantsTable,
1179 &hashOffsetInConstantsTable);
1180
1181 /* for a certain combination of Algorthm Chaining we want to
1182 use an optimised cd block */
1183
1184 if (pSessionDesc->symOperation ==
1185 CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
1186 pSessionDesc->useSymConstantsTable == CPA_TRUE) {
1187 pSessionDesc->useOptimisedContentDesc =
1188 LacSymQat_UseOptimisedContentDesc(pSessionDesc);
1189 }
1190
1191 /* check whether we need to construct content desc for stateful
1192 * SHA3 */
1193 pSessionDesc->useStatefulSha3ContentDesc =
1194 LacAlgChain_UseStatefulSha3ContentDesc(
1195 pSessionSetupData->partialsNotRequired,
1196 capabilitiesMask,
1197 pSessionDesc);
1198
1199 /* setup some convenience pointers */
1200 pCdInfo = &(pSessionDesc->contentDescInfo);
1201 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1202 hwBlockOffsetInDRAM = 0;
1203
1204 /* set up the pointer for the optimised content desc if this is
1205 * possible we still have to support both cd types in case of
1206 * partials so we construct both */
1207 if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) {
1208 pCdInfoOptimised =
1209 &(pSessionDesc->contentDescOptimisedInfo);
1210 pOptimisedHwBlockBaseInDRAM =
1211 (Cpa8U *)pCdInfoOptimised->pData;
1212 optimisedHwBlockOffsetInDRAM = 0;
1213 }
1214
1215 switch (pSessionDesc->symOperation) {
1216 case CPA_CY_SYM_OP_CIPHER: {
1217 LacAlgChain_CipherCDBuild(
1218 pCipherData,
1219 pSessionDesc,
1220 ICP_QAT_FW_SLICE_DRAM_WR,
1221 cipherOffsetInConstantsTable,
1222 &pSessionDesc->cmnRequestFlags,
1223 &pSessionDesc->laCmdFlags,
1224 pHwBlockBaseInDRAM,
1225 &hwBlockOffsetInDRAM,
1226 capabilitiesMask);
1227
1228 if (pSessionDesc->useSymConstantsTable) {
1229 LacAlgChain_CipherCDBuild_ForSHRAM(
1230 pCipherData,
1231 pSessionDesc,
1232 ICP_QAT_FW_SLICE_DRAM_WR,
1233 cipherOffsetInConstantsTable);
1234 }
1235 } break;
1236 case CPA_CY_SYM_OP_HASH:
1237 LacAlgChain_HashCDBuild(pHashData,
1238 instanceHandle,
1239 pSessionDesc,
1240 ICP_QAT_FW_SLICE_NULL,
1241 hashOffsetInConstantsTable,
1242 &pSessionDesc->cmnRequestFlags,
1243 &pSessionDesc->laCmdFlags,
1244 &precomputeData,
1245 &precomputeDataOptimisedCd,
1246 pHwBlockBaseInDRAM,
1247 &hwBlockOffsetInDRAM,
1248 NULL,
1249 NULL);
1250 break;
1251 case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
1252 /* For CCM/GCM, CPM firmware currently expects the
1253 * cipher and hash h/w setup blocks to be arranged
1254 * according to the chain order (Except for GCM/CCM,
1255 * order doesn't actually matter as long as the config
1256 * offsets are set correctly in CD control blocks
1257 */
1258 if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
1259 chainOrder) {
1260 LacAlgChain_HashCDBuild(
1261 pHashData,
1262 instanceHandle,
1263 pSessionDesc,
1264 ICP_QAT_FW_SLICE_CIPHER,
1265 hashOffsetInConstantsTable,
1266 &pSessionDesc->cmnRequestFlags,
1267 &pSessionDesc->laCmdFlags,
1268 &precomputeData,
1269 &precomputeDataOptimisedCd,
1270 pHwBlockBaseInDRAM,
1271 &hwBlockOffsetInDRAM,
1272 pOptimisedHwBlockBaseInDRAM,
1273 &optimisedHwBlockOffsetInDRAM);
1274
1275 LacAlgChain_CipherCDBuild(
1276 pCipherData,
1277 pSessionDesc,
1278 ICP_QAT_FW_SLICE_DRAM_WR,
1279 cipherOffsetInConstantsTable,
1280 &pSessionDesc->cmnRequestFlags,
1281 &pSessionDesc->laCmdFlags,
1282 pHwBlockBaseInDRAM,
1283 &hwBlockOffsetInDRAM,
1284 capabilitiesMask);
1285
1286 if (pSessionDesc->useOptimisedContentDesc) {
1287 LacAlgChain_CipherCDBuild_ForOptimisedCD(
1288 pCipherData,
1289 pSessionDesc,
1290 ICP_QAT_FW_SLICE_DRAM_WR,
1291 cipherOffsetInConstantsTable,
1292 pOptimisedHwBlockBaseInDRAM,
1293 &optimisedHwBlockOffsetInDRAM);
1294 }
1295
1296 if (NON_SPC != pSessionDesc->singlePassState) {
1297 pCdInfo->hwBlkSzQuadWords =
1298 (LAC_BYTES_TO_QUADWORDS(
1299 hwBlockOffsetInDRAM));
1300 pMsg = (icp_qat_fw_comn_req_t *)&(
1301 pSessionDesc->reqSpcCacheHdr);
1302 SalQatMsg_ContentDescHdrWrite(
1303 (icp_qat_fw_comn_req_t *)pMsg,
1304 pCdInfo);
1305 }
1306 } else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */
1307 {
1308 LacAlgChain_CipherCDBuild(
1309 pCipherData,
1310 pSessionDesc,
1311 ICP_QAT_FW_SLICE_AUTH,
1312 cipherOffsetInConstantsTable,
1313 &pSessionDesc->cmnRequestFlags,
1314 &pSessionDesc->laCmdFlags,
1315 pHwBlockBaseInDRAM,
1316 &hwBlockOffsetInDRAM,
1317 capabilitiesMask);
1318
1319 if (pSessionDesc->useOptimisedContentDesc) {
1320 LacAlgChain_CipherCDBuild_ForOptimisedCD(
1321 pCipherData,
1322 pSessionDesc,
1323 ICP_QAT_FW_SLICE_AUTH,
1324 cipherOffsetInConstantsTable,
1325 pOptimisedHwBlockBaseInDRAM,
1326 &optimisedHwBlockOffsetInDRAM);
1327 }
1328
1329 if (NON_SPC != pSessionDesc->singlePassState) {
1330 pCdInfo->hwBlkSzQuadWords =
1331 LAC_BYTES_TO_QUADWORDS(
1332 hwBlockOffsetInDRAM);
1333 pMsg = (icp_qat_fw_comn_req_t *)&(
1334 pSessionDesc->reqSpcCacheHdr);
1335 SalQatMsg_ContentDescHdrWrite(
1336 (icp_qat_fw_comn_req_t *)pMsg,
1337 pCdInfo);
1338 }
1339 LacAlgChain_HashCDBuild(
1340 pHashData,
1341 instanceHandle,
1342 pSessionDesc,
1343 ICP_QAT_FW_SLICE_DRAM_WR,
1344 hashOffsetInConstantsTable,
1345 &pSessionDesc->cmnRequestFlags,
1346 &pSessionDesc->laCmdFlags,
1347 &precomputeData,
1348 &precomputeDataOptimisedCd,
1349 pHwBlockBaseInDRAM,
1350 &hwBlockOffsetInDRAM,
1351 pOptimisedHwBlockBaseInDRAM,
1352 &optimisedHwBlockOffsetInDRAM);
1353 }
1354 break;
1355 default:
1356 LAC_LOG_ERROR("Invalid sym operation\n");
1357 status = CPA_STATUS_INVALID_PARAM;
1358 }
1359 }
1360
1361 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1362 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo =
1363 &(pSessionDesc->hashStateBufferInfo);
1364 CpaBoolean hashStateBuffer = CPA_TRUE;
1365
1366 /* set up fields in both the cd_ctrl and reqParams which
1367 * describe the ReqParams block */
1368 LacSymQat_HashSetupReqParamsMetaData(
1369 &(pSessionDesc->reqCacheFtr),
1370 instanceHandle,
1371 pHashData,
1372 hashStateBuffer,
1373 pSessionDesc->qatHashMode,
1374 pSessionDesc->digestVerify);
1375
1376 if (pSessionDesc->useSymConstantsTable) {
1377 /* Need to set up for SHRAM Constants Table use also */
1378 LacSymQat_HashSetupReqParamsMetaData(
1379 &(pSessionDesc->shramReqCacheFtr),
1380 instanceHandle,
1381 pHashData,
1382 hashStateBuffer,
1383 pSessionDesc->qatHashMode,
1384 pSessionDesc->digestVerify);
1385 }
1386
1387 /* populate the hash state prefix buffer info structure
1388 * (part of user allocated session memory & the
1389 * buffer itself. For CCM/GCM the buffer is stored in the
1390 * cookie and is not initialised here) */
1391 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
1392 LAC_CHECK_64_BYTE_ALIGNMENT(
1393 &(pSessionDesc->hashStatePrefixBuffer[0]));
1394 status = LacHash_StatePrefixAadBufferInit(
1395 &(pService->generic_service_info),
1396 pHashData,
1397 &(pSessionDesc->reqCacheFtr),
1398 pSessionDesc->qatHashMode,
1399 pSessionDesc->hashStatePrefixBuffer,
1400 pHashStateBufferInfo);
1401 /* SHRAM Constants Table not used for Auth-Enc */
1402 }
1403
1404 if (CPA_STATUS_SUCCESS == status) {
1405 if (IS_HASH_MODE_1(pSessionDesc->qatHashMode) ||
1406 CPA_CY_SYM_HASH_ZUC_EIA3 ==
1407 pHashData->hashAlgorithm) {
1408 LAC_CHECK_64_BYTE_ALIGNMENT(
1409 &(pSessionDesc->hashStatePrefixBuffer[0]));
1410
1411 /* Block messages until precompute is completed
1412 */
1413 pSessionDesc->nonBlockingOpsInProgress =
1414 CPA_FALSE;
1415 status = LacHash_PrecomputeDataCreate(
1416 instanceHandle,
1417 (CpaCySymSessionSetupData *)
1418 pSessionSetupData,
1419 LacSymAlgChain_HashPrecomputeDoneCb,
1420 pSessionDesc,
1421 pSessionDesc->hashStatePrefixBuffer,
1422 precomputeData.pState1,
1423 precomputeData.pState2);
1424 if (pSessionDesc->useOptimisedContentDesc) {
1425 status = LacHash_PrecomputeDataCreate(
1426 instanceHandle,
1427 (CpaCySymSessionSetupData *)
1428 pSessionSetupData,
1429 LacSymAlgChain_HashPrecomputeDoneCb,
1430 pSessionDesc,
1431 pSessionDesc->hashStatePrefixBuffer,
1432 precomputeDataOptimisedCd.pState1,
1433 precomputeDataOptimisedCd.pState2);
1434 }
1435 } else if (pHashData->hashAlgorithm ==
1436 CPA_CY_SYM_HASH_AES_CBC_MAC) {
1437 if (NULL != precomputeData.pState2) {
1438 LAC_OS_BZERO(precomputeData.pState2,
1439 precomputeData.state2Size);
1440 memcpy(precomputeData.pState2,
1441 pHashData->authModeSetupData
1442 .authKey,
1443 pHashData->authModeSetupData
1444 .authKeyLenInBytes);
1445 }
1446 }
1447 }
1448 }
1449
1450 if (CPA_STATUS_SUCCESS == status) {
1451
1452 /* Configure the ContentDescriptor field
1453 in the request if not done already */
1454 pCdInfo->hwBlkSzQuadWords =
1455 LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM);
1456 pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr);
1457 SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1458 pCdInfo);
1459
1460 pMsgS =
1461 (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr);
1462 /*If we are using the optimised CD then
1463 we have to set this up correctly in the SHARM reqCache*/
1464 if (pSessionDesc->useOptimisedContentDesc) {
1465 pCdInfoOptimised->hwBlkSzQuadWords =
1466 LAC_BYTES_TO_QUADWORDS(
1467 optimisedHwBlockOffsetInDRAM);
1468 SalQatMsg_ContentDescHdrWrite(
1469 (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised);
1470 }
1471
1472 /* Updates command flags basing on configured alg */
1473 updateLaCmdFlags(pSessionDesc,
1474 proto,
1475 &pSessionDesc->laCmdFlags);
1476
1477 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1478 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1479 pSessionDesc->laCmdId,
1480 pSessionDesc->cmnRequestFlags,
1481 pSessionDesc->laCmdFlags);
1482
1483 /* Need to duplicate if SHRAM Constants Table used */
1484 if (pSessionDesc->useSymConstantsTable) {
1485 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
1486 pSessionDesc->laCmdFlags,
1487 ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP);
1488
1489 if (pSessionDesc->isCipher &&
1490 !pSessionDesc->useOptimisedContentDesc) {
1491 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(
1492 cmnRequestFlags,
1493 QAT_COMN_CD_FLD_TYPE_16BYTE_DATA);
1494 }
1495
1496 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS,
1497 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1498 pSessionDesc->laCmdId,
1499 cmnRequestFlags,
1500 pSessionDesc->laCmdFlags);
1501 }
1502 }
1503
1504 return status;
1505 }
1506
1507 static void
LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t * pMsg,Cpa32U packetType,icp_qat_hw_auth_mode_t qatHashMode)1508 LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg,
1509 Cpa32U packetType,
1510 icp_qat_hw_auth_mode_t qatHashMode)
1511 {
1512 icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL;
1513
1514 pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl);
1515
1516 if (IS_HASH_MODE_2(qatHashMode)) {
1517 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1518 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1519 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1520 pAuthCdCtrlHdr->hash_flags,
1521 QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1522 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1523 pAuthCdCtrlHdr->hash_flags,
1524 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1525 } else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) {
1526 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1527 pAuthCdCtrlHdr->hash_flags,
1528 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1529 }
1530 } else {
1531 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1532 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1533 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1534 pAuthCdCtrlHdr->hash_flags,
1535 QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1536 }
1537 }
1538 }
1539
1540 /** @ingroup LacAlgChain */
1541 CpaStatus
LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,void * pCallbackTag,const CpaCySymOpData * pOpData,const CpaBufferList * pSrcBuffer,CpaBufferList * pDstBuffer,CpaBoolean * pVerifyResult)1542 LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,
1543 lac_session_desc_t *pSessionDesc,
1544 void *pCallbackTag,
1545 const CpaCySymOpData *pOpData,
1546 const CpaBufferList *pSrcBuffer,
1547 CpaBufferList *pDstBuffer,
1548 CpaBoolean *pVerifyResult)
1549 {
1550 CpaStatus status = CPA_STATUS_SUCCESS;
1551 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
1552 Cpa32U capabilitiesMask =
1553 pService->generic_service_info.capabilitiesMask;
1554 lac_sym_bulk_cookie_t *pCookie = NULL;
1555 lac_sym_cookie_t *pSymCookie = NULL;
1556 icp_qat_fw_la_bulk_req_t *pMsg = NULL;
1557 Cpa8U *pMsgDummy = NULL;
1558 Cpa8U *pCacheDummyHdr = NULL;
1559 Cpa8U *pCacheDummyFtr = NULL;
1560 Cpa32U qatPacketType = 0;
1561 CpaBufferList *pBufferList = NULL;
1562 Cpa8U *pDigestResult = NULL;
1563 Cpa64U srcAddrPhys = 0;
1564 Cpa64U dstAddrPhys = 0;
1565 icp_qat_fw_la_cmd_id_t laCmdId;
1566 sal_qat_content_desc_info_t *pCdInfo = NULL;
1567 Cpa8U *pHwBlockBaseInDRAM = NULL;
1568 Cpa32U hwBlockOffsetInDRAM = 0;
1569 Cpa32U sizeInBytes = 0;
1570 icp_qat_fw_cipher_cd_ctrl_hdr_t *pSpcCdCtrlHdr = NULL;
1571 CpaCySymCipherAlgorithm cipher;
1572 CpaCySymHashAlgorithm hash;
1573 Cpa8U paddingLen = 0;
1574 Cpa8U blockLen = 0;
1575 CpaBoolean digestIsAppended = CPA_FALSE;
1576 Cpa32U aadLenInBytes = 0;
1577 Cpa64U srcPktSize = 0;
1578 Cpa64U dstPktSize = 0;
1579
1580 /* Set the command id */
1581 laCmdId = pSessionDesc->laCmdId;
1582
1583 cipher = pSessionDesc->cipherAlgorithm;
1584 hash = pSessionDesc->hashAlgorithm;
1585
1586 CpaBoolean isSpCcm =
1587 (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask));
1588
1589 if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1590 pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes;
1591 if (pOpData->messageLenToHashInBytes == 0 ||
1592 pOpData->pAdditionalAuthData != NULL) {
1593 LAC_INVALID_PARAM_LOG(
1594 "For AES_GMAC, AAD Length "
1595 "(messageLenToHashInBytes) must "
1596 "be non zero and pAdditionalAuthData "
1597 "must be NULL");
1598 return CPA_STATUS_INVALID_PARAM;
1599 }
1600 }
1601
1602 aadLenInBytes = pSessionDesc->aadLenInBytes;
1603
1604 /* Convert Alg Chain Request to Cipher Request for CCP and
1605 * AES_GCM single pass */
1606 if ((NON_SPC != pSessionDesc->singlePassState) &&
1607 (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) {
1608 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
1609 laCmdId = pSessionDesc->laCmdId;
1610 pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER;
1611 pSessionDesc->singlePassState = SPC;
1612 pSessionDesc->isCipher = CPA_TRUE;
1613 pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1614 pSessionDesc->isAuth = CPA_FALSE;
1615
1616 if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1617 if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) {
1618 LAC_INVALID_PARAM_LOG(
1619 "aadLenInBytes for AES_GMAC");
1620 return CPA_STATUS_INVALID_PARAM;
1621 }
1622 }
1623 /* New bit position (13) for SINGLE PASS.
1624 * The FW provides a specific macro to use to set the proto flag
1625 */
1626 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1627 pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
1628 if (isCyGen2x(pService)) {
1629 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
1630 }
1631
1632 pCdInfo = &(pSessionDesc->contentDescInfo);
1633 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1634 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
1635 pSessionDesc->cipherDirection) {
1636 if (LAC_CIPHER_IS_GCM(cipher))
1637 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1638 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
1639 else if (LAC_CIPHER_IS_CHACHA(cipher))
1640 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1641 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
1642 } else if (isSpCcm) {
1643 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1644 LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM);
1645 }
1646
1647 /* Update cipher slice type */
1648 pSessionDesc->cipherSliceType =
1649 LacCipher_GetCipherSliceType(pService,
1650 pSessionDesc->cipherAlgorithm,
1651 pSessionDesc->hashAlgorithm);
1652
1653 ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags,
1654 pSessionDesc->cipherSliceType);
1655
1656 /* construct cipherConfig in CD in DRAM */
1657 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1658 pHwBlockBaseInDRAM +
1659 hwBlockOffsetInDRAM,
1660 &sizeInBytes);
1661 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1662 pSessionDesc->reqSpcCacheHdr),
1663 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1664 laCmdId,
1665 pSessionDesc->cmnRequestFlags,
1666 pSessionDesc->laCmdFlags);
1667 } else if ((SPC == pSessionDesc->singlePassState) &&
1668 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1669 pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
1670 pSessionDesc->singlePassState = LIKELY_SPC;
1671 pSessionDesc->isCipher = CPA_TRUE;
1672 pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1673 pSessionDesc->isAuth = CPA_TRUE;
1674 pCdInfo = &(pSessionDesc->contentDescInfo);
1675 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1676
1677 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
1678 pSessionDesc->cipherDirection) {
1679 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
1680 } else {
1681 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
1682 }
1683
1684 laCmdId = pSessionDesc->laCmdId;
1685 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1686 pSessionDesc->laCmdFlags, 0);
1687 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
1688 ICP_QAT_FW_LA_GCM_PROTO);
1689
1690 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1691 pHwBlockBaseInDRAM +
1692 hwBlockOffsetInDRAM,
1693 &sizeInBytes);
1694
1695 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1696 pSessionDesc->reqCacheHdr),
1697 ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1698 laCmdId,
1699 pSessionDesc->cmnRequestFlags,
1700 pSessionDesc->laCmdFlags);
1701 }
1702
1703 else if (LAC_CIPHER_IS_CHACHA(cipher) &&
1704 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1705 LAC_INVALID_PARAM_LOG("IV for CHACHA");
1706 return CPA_STATUS_INVALID_PARAM;
1707 }
1708
1709 if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) {
1710 if (CPA_CY_SYM_HASH_AES_CCM == hash) {
1711 status = LacSymAlgChain_CheckCCMData(
1712 pOpData->pAdditionalAuthData,
1713 pOpData->pIv,
1714 pOpData->messageLenToCipherInBytes,
1715 pOpData->ivLenInBytes);
1716 if (CPA_STATUS_SUCCESS == status) {
1717 LacSymAlgChain_PrepareCCMData(
1718 pSessionDesc,
1719 pOpData->pAdditionalAuthData,
1720 pOpData->pIv,
1721 pOpData->messageLenToCipherInBytes,
1722 pOpData->ivLenInBytes);
1723 }
1724 } else if (CPA_CY_SYM_HASH_AES_GCM == hash) {
1725 if (aadLenInBytes != 0 &&
1726 pOpData->pAdditionalAuthData == NULL) {
1727 LAC_INVALID_PARAM_LOG("pAdditionalAuthData");
1728 status = CPA_STATUS_INVALID_PARAM;
1729 }
1730 if (CPA_STATUS_SUCCESS == status) {
1731 LacSymAlgChain_PrepareGCMData(
1732 pSessionDesc, pOpData->pAdditionalAuthData);
1733 }
1734 }
1735 }
1736
1737 /* allocate cookie (used by callback function) */
1738 if (CPA_STATUS_SUCCESS == status) {
1739 pSymCookie = (lac_sym_cookie_t *)Lac_MemPoolEntryAlloc(
1740 pService->lac_sym_cookie_pool);
1741 if (pSymCookie == NULL) {
1742 LAC_LOG_ERROR("Cannot allocate cookie - NULL");
1743 status = CPA_STATUS_RESOURCE;
1744 } else if ((void *)CPA_STATUS_RETRY == pSymCookie) {
1745 pSymCookie = NULL;
1746 status = CPA_STATUS_RETRY;
1747 } else {
1748 pCookie = &(pSymCookie->u.bulkCookie);
1749 }
1750 }
1751
1752 if (CPA_STATUS_SUCCESS == status) {
1753 /* write the buffer descriptors */
1754 if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) {
1755 status =
1756 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1757 (CpaBufferList *)pSrcBuffer,
1758 &srcAddrPhys,
1759 CPA_FALSE,
1760 &(pService->generic_service_info));
1761 if (CPA_STATUS_SUCCESS != status) {
1762 LAC_LOG_ERROR(
1763 "Unable to write src buffer descriptors");
1764 }
1765 /* For out of place operations */
1766 if ((pSrcBuffer != pDstBuffer) &&
1767 (CPA_STATUS_SUCCESS == status)) {
1768 status =
1769 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1770 pDstBuffer,
1771 &dstAddrPhys,
1772 CPA_FALSE,
1773 &(pService->generic_service_info));
1774 if (CPA_STATUS_SUCCESS != status) {
1775 LAC_LOG_ERROR(
1776 "Unable to write dest buffer descriptors");
1777 }
1778 }
1779 } else {
1780 status = LacBuffDesc_BufferListDescWrite(
1781 (CpaBufferList *)pSrcBuffer,
1782 &srcAddrPhys,
1783 CPA_FALSE,
1784 &(pService->generic_service_info));
1785 if (CPA_STATUS_SUCCESS != status) {
1786 LAC_LOG_ERROR(
1787 "Unable to write src buffer descriptors in "
1788 "LacBuffDesc_BufferListDescWrite");
1789 }
1790 /* For out of place operations */
1791 if ((pSrcBuffer != pDstBuffer) &&
1792 (CPA_STATUS_SUCCESS == status)) {
1793 status = LacBuffDesc_BufferListDescWrite(
1794 pDstBuffer,
1795 &dstAddrPhys,
1796 CPA_FALSE,
1797 &(pService->generic_service_info));
1798 if (CPA_STATUS_SUCCESS != status) {
1799 LAC_LOG_ERROR(
1800 "Unable to write dest buffer descriptors in "
1801 "LacBuffDesc_BufferListDescWrite");
1802 }
1803 }
1804 }
1805 }
1806 if (CPA_STATUS_SUCCESS == status) {
1807 /* populate the cookie */
1808 pCookie->pCallbackTag = pCallbackTag;
1809 pCookie->sessionCtx = pOpData->sessionCtx;
1810 pCookie->pOpData = (const CpaCySymOpData *)pOpData;
1811 pCookie->pDstBuffer = pDstBuffer;
1812 pCookie->updateSessionIvOnSend = CPA_FALSE;
1813 pCookie->updateUserIvOnRecieve = CPA_FALSE;
1814 pCookie->updateKeySizeOnRecieve = CPA_FALSE;
1815 pCookie->pNext = NULL;
1816 pCookie->instanceHandle = pService;
1817
1818 /* get the qat packet type for LAC packet type */
1819 LacSymQat_packetTypeGet(pOpData->packetType,
1820 pSessionDesc->partialState,
1821 &qatPacketType);
1822 /*
1823 * For XTS mode, the key size must be updated after
1824 * the first partial has been sent. Set a flag here so the
1825 * response knows to do this.
1826 */
1827 if (LAC_CIPHER_IS_XTS_MODE(cipher) &&
1828 (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
1829 (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) &&
1830 (qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) {
1831 pCookie->updateKeySizeOnRecieve = CPA_TRUE;
1832 }
1833
1834 /*
1835 * Now create the Request.
1836 * Start by populating it from the cache in the session
1837 * descriptor.
1838 */
1839 pMsg = &(pCookie->qatMsg);
1840 pMsgDummy = (Cpa8U *)pMsg;
1841
1842 if (SPC == pSessionDesc->singlePassState) {
1843 pCacheDummyHdr =
1844 (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr);
1845 pCacheDummyFtr =
1846 (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr);
1847 } else {
1848 /* Normally, we want to use the SHRAM Constants Table if
1849 * possible for best performance (less DRAM accesses
1850 * incurred by CPM). But we can't use it for
1851 * partial-packet hash operations. This is why we build
1852 * 2 versions of the message template at sessionInit,
1853 * one for SHRAM Constants Table usage and the other
1854 * (default) for Content Descriptor h/w setup data in
1855 * DRAM. And we chose between them here on a
1856 * per-request basis, when we know the packetType
1857 */
1858 if ((!pSessionDesc->useSymConstantsTable) ||
1859 (pSessionDesc->isAuth &&
1860 (CPA_CY_SYM_PACKET_TYPE_FULL !=
1861 pOpData->packetType))) {
1862 pCacheDummyHdr =
1863 (Cpa8U *)&(pSessionDesc->reqCacheHdr);
1864 pCacheDummyFtr =
1865 (Cpa8U *)&(pSessionDesc->reqCacheFtr);
1866 } else {
1867 pCacheDummyHdr =
1868 (Cpa8U *)&(pSessionDesc->shramReqCacheHdr);
1869 pCacheDummyFtr =
1870 (Cpa8U *)&(pSessionDesc->shramReqCacheFtr);
1871 }
1872 }
1873 memcpy(pMsgDummy,
1874 pCacheDummyHdr,
1875 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW));
1876 memset((pMsgDummy +
1877 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)),
1878 0,
1879 (LAC_LONG_WORD_IN_BYTES *
1880 LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW));
1881 memcpy(pMsgDummy +
1882 (LAC_LONG_WORD_IN_BYTES *
1883 LAC_START_OF_CACHE_FTR_IN_LW),
1884 pCacheDummyFtr,
1885 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW));
1886 /*
1887 * Populate the comn_mid section
1888 */
1889 SalQatMsg_CmnMidWrite(pMsg,
1890 pCookie,
1891 LAC_SYM_DEFAULT_QAT_PTR_TYPE,
1892 srcAddrPhys,
1893 dstAddrPhys,
1894 0,
1895 0);
1896
1897 /*
1898 * Populate the serv_specif_flags field of the Request header
1899 * Some of the flags are set up here.
1900 * Others are set up later when the RequestParams are set up.
1901 */
1902
1903 LacSymQat_LaPacketCommandFlagSet(
1904 qatPacketType,
1905 laCmdId,
1906 cipher,
1907 &pMsg->comn_hdr.serv_specif_flags,
1908 pOpData->ivLenInBytes);
1909
1910 if (SPC == pSessionDesc->singlePassState) {
1911 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1912 pMsg->comn_hdr.serv_specif_flags,
1913 ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS);
1914
1915 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL ==
1916 pOpData->packetType) {
1917 ICP_QAT_FW_LA_RET_AUTH_SET(
1918 pMsg->comn_hdr.serv_specif_flags,
1919 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
1920
1921 ICP_QAT_FW_LA_CMP_AUTH_SET(
1922 pMsg->comn_hdr.serv_specif_flags,
1923 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
1924 }
1925 }
1926
1927 ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags,
1928 pSessionDesc->cipherSliceType);
1929
1930 LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize);
1931 LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize);
1932
1933 /*
1934 * Populate the CipherRequestParams section of the Request
1935 */
1936 if (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) {
1937
1938 Cpa8U *pIvBuffer = NULL;
1939
1940 status = LacCipher_PerformParamCheck(cipher,
1941 pOpData,
1942 srcPktSize);
1943 if (CPA_STATUS_SUCCESS != status) {
1944 /* free the cookie */
1945 Lac_MemPoolEntryFree(pCookie);
1946 return status;
1947 }
1948
1949 if (CPA_STATUS_SUCCESS == status) {
1950 /* align cipher IV */
1951 status = LacCipher_PerformIvCheck(
1952 &(pService->generic_service_info),
1953 pCookie,
1954 qatPacketType,
1955 &pIvBuffer);
1956 }
1957 if ((SPC == pSessionDesc->singlePassState) &&
1958 ((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) ||
1959 (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) {
1960 /* For SPC stateful cipher state size for mid
1961 * and end partial packet is 48 bytes
1962 */
1963 pSpcCdCtrlHdr =
1964 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&(
1965 pMsg->cd_ctrl);
1966 pSpcCdCtrlHdr->cipher_state_sz =
1967 LAC_BYTES_TO_QUADWORDS(
1968 LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE);
1969 }
1970 /*populate the cipher request parameters */
1971 if (CPA_STATUS_SUCCESS == status) {
1972 Cpa64U ivBufferPhysAddr = 0;
1973
1974 if (pIvBuffer != NULL) {
1975 /* User OpData memory being used for IV
1976 * buffer */
1977 /* get the physical address */
1978 ivBufferPhysAddr =
1979 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1980 pService->generic_service_info,
1981 pIvBuffer);
1982 if (0 == ivBufferPhysAddr) {
1983 LAC_LOG_ERROR(
1984 "Unable to get the physical address "
1985 "of the IV\n");
1986 status = CPA_STATUS_FAIL;
1987 }
1988 }
1989
1990 if (status == CPA_STATUS_SUCCESS) {
1991 status =
1992 LacSymQat_CipherRequestParamsPopulate(
1993 pSessionDesc,
1994 pMsg,
1995 pOpData
1996 ->cryptoStartSrcOffsetInBytes,
1997 pOpData
1998 ->messageLenToCipherInBytes,
1999 ivBufferPhysAddr,
2000 pIvBuffer);
2001 }
2002 }
2003
2004 if ((SPC == pSessionDesc->singlePassState) &&
2005 CPA_STATUS_SUCCESS == status) {
2006 Cpa64U aadBufferPhysAddr = 0;
2007
2008 /* For CHACHA and AES-GCM there is an AAD buffer
2009 * if aadLenInBytes is nonzero In case of
2010 * AES-GMAC, AAD buffer passed in the src
2011 * buffer.
2012 */
2013 if ((0 != aadLenInBytes &&
2014 CPA_CY_SYM_HASH_AES_GMAC != hash) ||
2015 isSpCcm) {
2016 LAC_CHECK_NULL_PARAM(
2017 pOpData->pAdditionalAuthData);
2018 Cpa32U aadDataLen =
2019 pSessionDesc->aadLenInBytes;
2020
2021 /* In case of AES_CCM, B0 block size and
2022 * 2 bytes of AAD len encoding need to
2023 * be added to total AAD data len */
2024 if (isSpCcm)
2025 aadDataLen +=
2026 LAC_CIPHER_CCM_AAD_OFFSET;
2027
2028 blockLen =
2029 LacSymQat_CipherBlockSizeBytesGet(
2030 cipher);
2031 if ((aadDataLen % blockLen) != 0) {
2032 paddingLen = blockLen -
2033 (aadDataLen % blockLen);
2034 memset(
2035 &pOpData
2036 ->pAdditionalAuthData
2037 [aadDataLen],
2038 0,
2039 paddingLen);
2040 }
2041
2042 /* User OpData memory being used for AAD
2043 * buffer */
2044 /* get the physical address */
2045 aadBufferPhysAddr =
2046 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2047 pService->generic_service_info,
2048 pOpData->pAdditionalAuthData);
2049 if (0 == aadBufferPhysAddr) {
2050 LAC_LOG_ERROR(
2051 "Unable to get the physical address "
2052 "of the aad\n");
2053 status = CPA_STATUS_FAIL;
2054 }
2055 }
2056
2057 if (CPA_STATUS_SUCCESS == status) {
2058 icp_qat_fw_la_cipher_req_params_t *pCipherReqParams =
2059 (icp_qat_fw_la_cipher_req_params_t
2060 *)((Cpa8U *)&(
2061 pMsg->serv_specif_rqpars) +
2062 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2063
2064 icp_qat_fw_la_cipher_20_req_params_t
2065 *pCipher20ReqParams =
2066 (void
2067 *)((Cpa8U *)&(
2068 pMsg->serv_specif_rqpars) +
2069 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2070
2071 if (isCyGen4x(pService)) {
2072 pCipher20ReqParams
2073 ->spc_aad_addr =
2074 aadBufferPhysAddr;
2075 pCipher20ReqParams->spc_aad_sz =
2076 pSessionDesc->aadLenInBytes;
2077 pCipher20ReqParams
2078 ->spc_aad_offset = 0;
2079 if (isSpCcm)
2080 pCipher20ReqParams
2081 ->spc_aad_sz +=
2082 LAC_CIPHER_CCM_AAD_OFFSET;
2083 } else {
2084 pCipherReqParams->spc_aad_addr =
2085 aadBufferPhysAddr;
2086 pCipherReqParams->spc_aad_sz =
2087 (Cpa16U)pSessionDesc
2088 ->aadLenInBytes;
2089 }
2090
2091 if (CPA_TRUE !=
2092 pSessionDesc->digestIsAppended) {
2093 Cpa64U digestBufferPhysAddr = 0;
2094 /* User OpData memory being used
2095 * for digest buffer */
2096 /* get the physical address */
2097 digestBufferPhysAddr =
2098 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2099 pService
2100 ->generic_service_info,
2101 pOpData->pDigestResult);
2102 if (0 != digestBufferPhysAddr) {
2103 if (isCyGen4x(
2104 pService)) {
2105 pCipher20ReqParams
2106 ->spc_auth_res_addr =
2107 digestBufferPhysAddr;
2108 pCipher20ReqParams
2109 ->spc_auth_res_sz =
2110 (Cpa8U)pSessionDesc
2111 ->hashResultSize;
2112 } else {
2113 pCipherReqParams
2114 ->spc_auth_res_addr =
2115 digestBufferPhysAddr;
2116 pCipherReqParams
2117 ->spc_auth_res_sz =
2118 (Cpa8U)pSessionDesc
2119 ->hashResultSize;
2120 }
2121 } else {
2122 LAC_LOG_ERROR(
2123 "Unable to get the physical address "
2124 "of the digest\n");
2125 status =
2126 CPA_STATUS_FAIL;
2127 }
2128 } else {
2129 /* Check if the dest buffer can
2130 * handle the digest, only for
2131 * last packet */
2132 if (((ICP_QAT_FW_LA_PARTIAL_NONE ==
2133 qatPacketType) ||
2134 (ICP_QAT_FW_LA_PARTIAL_END ==
2135 qatPacketType))) {
2136 if (dstPktSize <
2137 (pOpData
2138 ->cryptoStartSrcOffsetInBytes +
2139 pOpData
2140 ->messageLenToCipherInBytes +
2141 pSessionDesc
2142 ->hashResultSize))
2143 status =
2144 CPA_STATUS_INVALID_PARAM;
2145 }
2146 if (isCyGen4x(pService)) {
2147 pCipher20ReqParams
2148 ->spc_auth_res_sz =
2149 (Cpa8U)pSessionDesc
2150 ->hashResultSize;
2151 } else {
2152 pCipherReqParams
2153 ->spc_auth_res_sz =
2154 (Cpa8U)pSessionDesc
2155 ->hashResultSize;
2156 }
2157 }
2158 }
2159 }
2160 }
2161
2162 /*
2163 * Set up HashRequestParams part of Request
2164 */
2165 if ((status == CPA_STATUS_SUCCESS) &&
2166 (laCmdId != ICP_QAT_FW_LA_CMD_CIPHER)) {
2167 Cpa32U authOffsetInBytes =
2168 pOpData->hashStartSrcOffsetInBytes;
2169 Cpa32U authLenInBytes =
2170 pOpData->messageLenToHashInBytes;
2171
2172 status = LacHash_PerformParamCheck(instanceHandle,
2173 pSessionDesc,
2174 pOpData,
2175 srcPktSize,
2176 pVerifyResult);
2177 if (CPA_STATUS_SUCCESS != status) {
2178 /* free the cookie */
2179 Lac_MemPoolEntryFree(pCookie);
2180 return status;
2181 }
2182 if (CPA_STATUS_SUCCESS == status) {
2183 /* Info structure for CCM/GCM */
2184 lac_sym_qat_hash_state_buffer_info_t
2185 hashStateBufferInfo = { 0 };
2186 lac_sym_qat_hash_state_buffer_info_t
2187 *pHashStateBufferInfo =
2188 &(pSessionDesc->hashStateBufferInfo);
2189
2190 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
2191 icp_qat_fw_la_auth_req_params_t *pHashReqParams =
2192 (icp_qat_fw_la_auth_req_params_t
2193 *)((Cpa8U *)&(
2194 pMsg->serv_specif_rqpars) +
2195 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
2196
2197 hashStateBufferInfo.pData =
2198 pOpData->pAdditionalAuthData;
2199 if (pOpData->pAdditionalAuthData ==
2200 NULL) {
2201 hashStateBufferInfo.pDataPhys =
2202 0;
2203 } else {
2204 hashStateBufferInfo
2205 .pDataPhys = LAC_MEM_CAST_PTR_TO_UINT64(
2206 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2207 pService
2208 ->generic_service_info,
2209 pOpData
2210 ->pAdditionalAuthData));
2211 }
2212
2213 hashStateBufferInfo
2214 .stateStorageSzQuadWords = 0;
2215 hashStateBufferInfo
2216 .prefixAadSzQuadWords =
2217 LAC_BYTES_TO_QUADWORDS(
2218 pHashReqParams->u2.aad_sz);
2219
2220 /* Overwrite hash state buffer info
2221 * structure pointer with the one
2222 * created for CCM/GCM */
2223 pHashStateBufferInfo =
2224 &hashStateBufferInfo;
2225
2226 /* Aad buffer could be null in the GCM
2227 * case */
2228 if (0 ==
2229 hashStateBufferInfo.pDataPhys &&
2230 CPA_CY_SYM_HASH_AES_GCM != hash &&
2231 CPA_CY_SYM_HASH_AES_GMAC != hash) {
2232 LAC_LOG_ERROR(
2233 "Unable to get the physical address"
2234 "of the AAD\n");
2235 status = CPA_STATUS_FAIL;
2236 }
2237
2238 /* for CCM/GCM the hash and cipher data
2239 * regions are equal */
2240 authOffsetInBytes =
2241 pOpData
2242 ->cryptoStartSrcOffsetInBytes;
2243
2244 /* For authenticated encryption,
2245 * authentication length is determined
2246 * by messageLenToCipherInBytes for
2247 * AES-GCM and AES-CCM, and by
2248 * messageLenToHashInBytes for AES-GMAC.
2249 * You don't see the latter here, as
2250 * that is the initial value of
2251 * authLenInBytes. */
2252 if (hash != CPA_CY_SYM_HASH_AES_GMAC)
2253 authLenInBytes =
2254 pOpData
2255 ->messageLenToCipherInBytes;
2256 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
2257 hash ||
2258 CPA_CY_SYM_HASH_ZUC_EIA3 == hash) {
2259 hashStateBufferInfo.pData =
2260 pOpData->pAdditionalAuthData;
2261 hashStateBufferInfo.pDataPhys =
2262 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2263 pService->generic_service_info,
2264 hashStateBufferInfo.pData);
2265 hashStateBufferInfo
2266 .stateStorageSzQuadWords = 0;
2267 hashStateBufferInfo
2268 .prefixAadSzQuadWords =
2269 LAC_BYTES_TO_QUADWORDS(
2270 aadLenInBytes);
2271
2272 pHashStateBufferInfo =
2273 &hashStateBufferInfo;
2274
2275 if (0 ==
2276 hashStateBufferInfo.pDataPhys) {
2277 LAC_LOG_ERROR(
2278 "Unable to get the physical address"
2279 "of the AAD\n");
2280 status = CPA_STATUS_FAIL;
2281 }
2282 }
2283 if (CPA_CY_SYM_HASH_AES_CCM == hash) {
2284 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
2285 pSessionDesc->cipherDirection) {
2286 /* On a decrypt path pSrcBuffer
2287 * is used as this is where
2288 * encrypted digest is located.
2289 * Firmware uses encrypted
2290 * digest for
2291 * compare/verification*/
2292 pBufferList =
2293 (CpaBufferList *)pSrcBuffer;
2294 } else {
2295 /* On an encrypt path pDstBuffer
2296 * is used as this is where
2297 * encrypted digest will be
2298 * written */
2299 pBufferList =
2300 (CpaBufferList *)pDstBuffer;
2301 }
2302 status = LacSymAlgChain_PtrFromOffsetGet(
2303 pBufferList,
2304 pOpData->cryptoStartSrcOffsetInBytes +
2305 pOpData
2306 ->messageLenToCipherInBytes,
2307 &pDigestResult);
2308 if (CPA_STATUS_SUCCESS != status) {
2309 LAC_LOG_ERROR(
2310 "Cannot set digest pointer within the"
2311 " buffer list - offset out of bounds");
2312 }
2313 } else {
2314 pDigestResult = pOpData->pDigestResult;
2315 }
2316
2317 if (CPA_TRUE ==
2318 pSessionDesc->useStatefulSha3ContentDesc) {
2319 LacAlgChain_StatefulSha3_SkipStateLoadFlags(
2320 pMsg,
2321 qatPacketType,
2322 pSessionDesc->qatHashMode);
2323 }
2324
2325 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
2326 pSessionDesc->symOperation) {
2327 /* In alg chaining mode, packets are not
2328 * seen as partials for hash operations.
2329 * Override to NONE.
2330 */
2331 qatPacketType =
2332 ICP_QAT_FW_LA_PARTIAL_NONE;
2333 }
2334 digestIsAppended =
2335 pSessionDesc->digestIsAppended;
2336 if (CPA_TRUE == digestIsAppended) {
2337 /*Check if the destination buffer can
2338 * handle the digest if digestIsAppend
2339 * is true*/
2340 if (srcPktSize <
2341 (authOffsetInBytes +
2342 authLenInBytes +
2343 pSessionDesc->hashResultSize)) {
2344 status =
2345 CPA_STATUS_INVALID_PARAM;
2346 }
2347 }
2348 if (CPA_STATUS_SUCCESS == status) {
2349 /* populate the hash request parameters
2350 */
2351 status =
2352 LacSymQat_HashRequestParamsPopulate(
2353 pMsg,
2354 authOffsetInBytes,
2355 authLenInBytes,
2356 &(pService
2357 ->generic_service_info),
2358 pHashStateBufferInfo,
2359 qatPacketType,
2360 pSessionDesc->hashResultSize,
2361 pSessionDesc->digestVerify,
2362 digestIsAppended ?
2363 NULL :
2364 pDigestResult,
2365 hash,
2366 NULL);
2367 }
2368 }
2369 }
2370 }
2371
2372 /*
2373 * send the message to the QAT
2374 */
2375 if (CPA_STATUS_SUCCESS == status) {
2376 qatUtilsAtomicInc(&(pSessionDesc->u.pendingCbCount));
2377
2378 status = LacSymQueue_RequestSend(instanceHandle,
2379 pCookie,
2380 pSessionDesc);
2381
2382 if (CPA_STATUS_SUCCESS != status) {
2383 /* Decrease pending callback counter on send fail. */
2384 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
2385 }
2386 }
2387 /* Case that will catch all error status's for this function */
2388 if (CPA_STATUS_SUCCESS != status) {
2389 /* free the cookie */
2390 if (NULL != pSymCookie) {
2391 Lac_MemPoolEntryFree(pSymCookie);
2392 }
2393 }
2394 return status;
2395 }
2396