xref: /freebsd/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c (revision 25f09d4a9c358c5452435d299e00c1a1bdafff87)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3 
4 /**
5  ***************************************************************************
6  * @file lac_sym_alg_chain.c      Algorithm Chaining Perform
7  *
8  * @ingroup LacAlgChain
9  ***************************************************************************/
10 
11 /*
12 *******************************************************************************
13 * Include public/global header files
14 *******************************************************************************
15 */
16 
17 #include "cpa.h"
18 #include "cpa_cy_sym.h"
19 
20 #include "icp_accel_devices.h"
21 #include "icp_adf_init.h"
22 #include "icp_adf_transport.h"
23 #include "icp_adf_debug.h"
24 
25 /*
26 *******************************************************************************
27 * Include private header files
28 *******************************************************************************
29 */
30 
31 #include "lac_mem.h"
32 #include "lac_log.h"
33 #include "lac_sym.h"
34 #include "lac_list.h"
35 #include "icp_qat_fw_la.h"
36 #include "lac_sal_types_crypto.h"
37 #include "lac_sal.h"
38 #include "lac_sal_ctrl.h"
39 #include "lac_sym_alg_chain.h"
40 #include "lac_sym_cipher.h"
41 #include "lac_sym_cipher_defs.h"
42 #include "lac_sym_hash.h"
43 #include "lac_sym_hash_defs.h"
44 #include "lac_sym_qat_cipher.h"
45 #include "lac_sym_qat_hash.h"
46 #include "lac_sym_stats.h"
47 #include "lac_sym_queue.h"
48 #include "lac_sym_cb.h"
49 #include "sal_string_parse.h"
50 #include "lac_sym_auth_enc.h"
51 #include "lac_sym_qat.h"
52 #include "sal_hw_gen.h"
53 
54 /**
55  * @ingroup LacAlgChain
56  * This callback function will be invoked whenever a hash precompute
57  * operation completes.  It will dequeue and send any QAT requests
58  * which were queued up while the precompute was in progress.
59  *
60  * @param[in] callbackTag  Opaque value provided by user. This will
61  *                         be a pointer to the session descriptor.
62  *
63  * @retval
64  *     None
65  *
66  */
67 static void
LacSymAlgChain_HashPrecomputeDoneCb(void * callbackTag)68 LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag)
69 {
70 	LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag);
71 }
72 
73 /**
74  * @ingroup LacAlgChain
75  * Walk the buffer list and find the address for the given offset within
76  * a buffer.
77  *
78  * @param[in] pBufferList   Buffer List
79  * @param[in] packetOffset  Offset in the buffer list for which address
80  *                          is to be found.
81  * @param[out] ppDataPtr    This is where the sought pointer will be put
82  * @param[out] pSpaceLeft   Pointer to a variable in which information about
83  *                          available space from the given offset to the end
84  *                          of the flat buffer it is located in will be returned
85  *
86  * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list
87  * @retval CPA_STATUS_FAIL    Address with a given offset not found in the list.
88  *
89  */
90 static CpaStatus
LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList * pBufferList,const Cpa32U packetOffset,Cpa8U ** ppDataPtr)91 LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList,
92 				const Cpa32U packetOffset,
93 				Cpa8U **ppDataPtr)
94 {
95 	Cpa32U currentOffset = 0;
96 	Cpa32U i = 0;
97 
98 	for (i = 0; i < pBufferList->numBuffers; i++) {
99 		Cpa8U *pCurrData = pBufferList->pBuffers[i].pData;
100 		Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes;
101 
102 		/* If the offset is within the address space of the current
103 		 * buffer */
104 		if ((packetOffset >= currentOffset) &&
105 		    (packetOffset < (currentOffset + currDataSize))) {
106 			/* increment by offset of the address in the current
107 			 * buffer */
108 			*ppDataPtr = pCurrData + (packetOffset - currentOffset);
109 			return CPA_STATUS_SUCCESS;
110 		}
111 
112 		/* Increment by the size of the buffer */
113 		currentOffset += currDataSize;
114 	}
115 
116 	return CPA_STATUS_FAIL;
117 }
118 
119 /**
120  * @ingroup LacAlgChain
121  * Function which checks for support of partial packets for symmetric
122  * crypto operations
123  *
124  * @param[in] pService            Pointer to service descriptor
125  * @param[in/out] pSessionDesc    Pointer to session descriptor
126  *
127  */
128 static void
LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)129 LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask,
130 			       lac_session_desc_t *pSessionDesc)
131 {
132 	CpaBoolean isHashPartialSupported = CPA_FALSE;
133 	CpaBoolean isCipherPartialSupported = CPA_FALSE;
134 	CpaBoolean isPartialSupported = CPA_FALSE;
135 
136 	switch (pSessionDesc->cipherAlgorithm) {
137 	/* Following ciphers don't support partial */
138 	case CPA_CY_SYM_CIPHER_KASUMI_F8:
139 	case CPA_CY_SYM_CIPHER_AES_F8:
140 	case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
141 	case CPA_CY_SYM_CIPHER_CHACHA:
142 	case CPA_CY_SYM_CIPHER_ZUC_EEA3:
143 		break;
144 	/* All others support partial */
145 	default:
146 		isCipherPartialSupported = CPA_TRUE;
147 		break;
148 	}
149 	switch (pSessionDesc->hashAlgorithm) {
150 	/* Following hash don't support partial */
151 	case CPA_CY_SYM_HASH_KASUMI_F9:
152 	case CPA_CY_SYM_HASH_SNOW3G_UIA2:
153 	case CPA_CY_SYM_HASH_POLY:
154 	case CPA_CY_SYM_HASH_ZUC_EIA3:
155 		break;
156 	/* Following hash may support partial based on device capabilities */
157 	case CPA_CY_SYM_HASH_SHA3_256:
158 		if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) {
159 			isHashPartialSupported = CPA_TRUE;
160 		}
161 		break;
162 	/* All others support partial */
163 	default:
164 		isHashPartialSupported = CPA_TRUE;
165 		break;
166 	}
167 	switch (pSessionDesc->symOperation) {
168 	case CPA_CY_SYM_OP_CIPHER:
169 		isPartialSupported = isCipherPartialSupported;
170 		break;
171 	case CPA_CY_SYM_OP_HASH:
172 		isPartialSupported = isHashPartialSupported;
173 		break;
174 	case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
175 		if (isCipherPartialSupported && isHashPartialSupported) {
176 			isPartialSupported = CPA_TRUE;
177 		}
178 		break;
179 	case CPA_CY_SYM_OP_NONE:
180 		break;
181 	default:
182 		break;
183 	}
184 
185 	if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
186 		/* UCS slice has no support for state flush and
187 		 * because of that is not able to do partial processing */
188 		isPartialSupported = CPA_FALSE;
189 	}
190 
191 	pSessionDesc->isPartialSupported = isPartialSupported;
192 }
193 
194 static void
LacAlgChain_CipherCDBuild_ForOptimisedCD(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)195 LacAlgChain_CipherCDBuild_ForOptimisedCD(
196     const CpaCySymCipherSetupData *pCipherData,
197     lac_session_desc_t *pSessionDesc,
198     icp_qat_fw_slice_t nextSlice,
199     Cpa8U cipherOffsetInConstantsTable,
200     Cpa8U *pOptimisedHwBlockBaseInDRAM,
201     Cpa32U *pOptimisedHwBlockOffsetInDRAM)
202 {
203 	Cpa8U *pCipherKeyField = NULL;
204 	Cpa32U sizeInBytes = 0;
205 	pCipherKeyField = pOptimisedHwBlockBaseInDRAM;
206 
207 	/* Need to build up the alternative CD for SHRAM Constants Table use
208 	 * with an optimised content desc of 64 bytes for this case. Cipher key
209 	 * will be in the Content desc in DRAM, The cipher config data
210 	 * is now in the SHRAM constants table. */
211 
212 	LacSymQat_CipherHwBlockPopulateKeySetup(
213 	    pSessionDesc,
214 	    pCipherData,
215 	    pCipherData->cipherKeyLenInBytes,
216 	    pSessionDesc->cipherSliceType,
217 	    pCipherKeyField,
218 	    &sizeInBytes);
219 
220 	LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
221 				       pSessionDesc->cipherAlgorithm,
222 				       pSessionDesc->cipherKeyLenInBytes,
223 				       pSessionDesc->cipherSliceType,
224 				       nextSlice,
225 				       cipherOffsetInConstantsTable);
226 
227 	*pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
228 }
229 
230 static void
LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable)231 LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData,
232 				   lac_session_desc_t *pSessionDesc,
233 				   icp_qat_fw_slice_t nextSlice,
234 				   Cpa8U cipherOffsetInConstantsTable)
235 {
236 	Cpa32U sizeInBytes = 0;
237 	Cpa8U *pCipherKeyField = NULL;
238 	/* Need to build up the alternative CD for SHRAM Constants Table use
239 	 * Cipher key will be in the Request, The cipher config data is now in
240 	 * the SHRAM constants table. And nothing is now stored in the content
241 	 * desc */
242 	pCipherKeyField = (Cpa8U *)&(
243 	    pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields);
244 
245 	LacSymQat_CipherHwBlockPopulateKeySetup(
246 	    pSessionDesc,
247 	    pCipherData,
248 	    pCipherData->cipherKeyLenInBytes,
249 	    pSessionDesc->cipherSliceType,
250 	    pCipherKeyField,
251 	    &sizeInBytes);
252 
253 	LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
254 				       pSessionDesc->cipherAlgorithm,
255 				       pSessionDesc->cipherKeyLenInBytes,
256 				       pSessionDesc->cipherSliceType,
257 				       nextSlice,
258 				       cipherOffsetInConstantsTable);
259 }
260 
261 static void
LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData * pCipherData,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U cipherOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa32U capabilitiesMask)262 LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData,
263 			  lac_session_desc_t *pSessionDesc,
264 			  icp_qat_fw_slice_t nextSlice,
265 			  Cpa8U cipherOffsetInConstantsTable,
266 			  icp_qat_fw_comn_flags *pCmnRequestFlags,
267 			  icp_qat_fw_serv_specif_flags *pLaCmdFlags,
268 			  Cpa8U *pHwBlockBaseInDRAM,
269 			  Cpa32U *pHwBlockOffsetInDRAM,
270 			  Cpa32U capabilitiesMask)
271 {
272 	Cpa8U *pCipherKeyField = NULL;
273 	Cpa8U cipherOffsetInReqQW = 0;
274 	Cpa32U sizeInBytes = 0;
275 	void *pCfgData = NULL;
276 	Cpa32U cfgOffset = 0;
277 
278 	/* Construct the ContentDescriptor in DRAM */
279 	cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES);
280 	ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
281 	    *pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP);
282 
283 	/* construct cipherConfig in CD in DRAM */
284 	cfgOffset = *pHwBlockOffsetInDRAM;
285 	pCfgData = pHwBlockBaseInDRAM + cfgOffset;
286 	LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
287 					       pCfgData,
288 					       &sizeInBytes);
289 
290 	ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags,
291 				     pSessionDesc->cipherSliceType);
292 
293 	*pHwBlockOffsetInDRAM += sizeInBytes;
294 
295 	/* Cipher key will be in CD in DRAM.
296 	 * The Request contains a ptr to the CD.
297 	 * This ptr will be copied into the request later once the CD is
298 	 * fully constructed, but the flag is set here.  */
299 	pCipherKeyField = pHwBlockBaseInDRAM + *pHwBlockOffsetInDRAM;
300 	ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(*pCmnRequestFlags,
301 					QAT_COMN_CD_FLD_TYPE_64BIT_ADR);
302 
303 	LacSymQat_CipherHwBlockPopulateKeySetup(
304 	    pSessionDesc,
305 	    pCipherData,
306 	    pCipherData->cipherKeyLenInBytes,
307 	    pSessionDesc->cipherSliceType,
308 	    pCipherKeyField,
309 	    &sizeInBytes);
310 	/* update offset */
311 	*pHwBlockOffsetInDRAM += sizeInBytes;
312 
313 	LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr),
314 				       pSessionDesc->cipherAlgorithm,
315 				       pSessionDesc->cipherKeyLenInBytes,
316 				       pSessionDesc->cipherSliceType,
317 				       nextSlice,
318 				       cipherOffsetInReqQW);
319 	if (NON_SPC != pSessionDesc->singlePassState) {
320 		LacSymQat_CipherCtrlBlockWrite(
321 		    &(pSessionDesc->reqSpcCacheFtr),
322 		    pSessionDesc->cipherAlgorithm,
323 		    pSessionDesc->cipherKeyLenInBytes,
324 		    pSessionDesc->cipherSliceType,
325 		    ICP_QAT_FW_SLICE_DRAM_WR,
326 		    cipherOffsetInReqQW);
327 	}
328 }
329 
330 static void
LacAlgChain_HashCDBuild(const CpaCySymHashSetupData * pHashData,CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,icp_qat_fw_slice_t nextSlice,Cpa8U hashOffsetInConstantsTable,icp_qat_fw_comn_flags * pCmnRequestFlags,icp_qat_fw_serv_specif_flags * pLaCmdFlags,lac_sym_qat_hash_precompute_info_t * pPrecomputeData,lac_sym_qat_hash_precompute_info_t * pPrecomputeDataOptimisedCd,Cpa8U * pHwBlockBaseInDRAM,Cpa32U * pHwBlockOffsetInDRAM,Cpa8U * pOptimisedHwBlockBaseInDRAM,Cpa32U * pOptimisedHwBlockOffsetInDRAM)331 LacAlgChain_HashCDBuild(
332     const CpaCySymHashSetupData *pHashData,
333     CpaInstanceHandle instanceHandle,
334     lac_session_desc_t *pSessionDesc,
335     icp_qat_fw_slice_t nextSlice,
336     Cpa8U hashOffsetInConstantsTable,
337     icp_qat_fw_comn_flags *pCmnRequestFlags,
338     icp_qat_fw_serv_specif_flags *pLaCmdFlags,
339     lac_sym_qat_hash_precompute_info_t *pPrecomputeData,
340     lac_sym_qat_hash_precompute_info_t *pPrecomputeDataOptimisedCd,
341     Cpa8U *pHwBlockBaseInDRAM,
342     Cpa32U *pHwBlockOffsetInDRAM,
343     Cpa8U *pOptimisedHwBlockBaseInDRAM,
344     Cpa32U *pOptimisedHwBlockOffsetInDRAM)
345 {
346 	Cpa32U sizeInBytes = 0;
347 	Cpa32U hwBlockOffsetInQuadWords =
348 	    *pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES;
349 
350 	/* build:
351 	 * - the hash part of the ContentDescriptor in DRAM */
352 	/* - the hash part of the CD control block in the Request template */
353 	LacSymQat_HashContentDescInit(&(pSessionDesc->reqCacheFtr),
354 				      instanceHandle,
355 				      pHashData,
356 				      pHwBlockBaseInDRAM,
357 				      hwBlockOffsetInQuadWords,
358 				      nextSlice,
359 				      pSessionDesc->qatHashMode,
360 				      CPA_FALSE,
361 				      CPA_FALSE,
362 				      pSessionDesc->useStatefulSha3ContentDesc,
363 				      pPrecomputeData,
364 				      &sizeInBytes);
365 
366 	/* Using DRAM CD so update offset */
367 	*pHwBlockOffsetInDRAM += sizeInBytes;
368 
369 	sizeInBytes = 0;
370 
371 	if (pSessionDesc->useOptimisedContentDesc) {
372 		LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
373 					      instanceHandle,
374 					      pHashData,
375 					      pOptimisedHwBlockBaseInDRAM,
376 					      hashOffsetInConstantsTable,
377 					      nextSlice,
378 					      pSessionDesc->qatHashMode,
379 					      CPA_TRUE,
380 					      CPA_TRUE,
381 					      CPA_FALSE,
382 					      pPrecomputeDataOptimisedCd,
383 					      &sizeInBytes);
384 
385 		*pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
386 	} else if (pSessionDesc->useSymConstantsTable) {
387 		/* Need to build up the alternative CD for SHRAM Constants Table
388 		 * use */
389 		LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
390 					      instanceHandle,
391 					      pHashData,
392 					      pHwBlockBaseInDRAM,
393 					      hashOffsetInConstantsTable,
394 					      nextSlice,
395 					      pSessionDesc->qatHashMode,
396 					      CPA_TRUE,
397 					      CPA_FALSE,
398 					      CPA_FALSE,
399 					      pPrecomputeData,
400 					      &sizeInBytes);
401 	}
402 }
403 static Cpa16U
LacAlgChain_GetCipherConfigSize(lac_session_desc_t * pSessionDesc)404 LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc)
405 {
406 	if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
407 		return sizeof(icp_qat_hw_ucs_cipher_config_t);
408 	} else {
409 		return sizeof(icp_qat_hw_cipher_config_t);
410 	}
411 }
412 
413 static Cpa16U
LacAlgChain_GetCipherConfigOffset(lac_session_desc_t * pSessionDesc)414 LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc)
415 {
416 	Cpa16U offset = 0;
417 
418 	if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation ||
419 	    SPC == pSessionDesc->singlePassState) {
420 		icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
421 		    (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc
422 			->reqCacheFtr.cd_ctrl;
423 		offset = cd_ctrl->cipher_cfg_offset;
424 	} else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) {
425 		icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl =
426 		    (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc
427 			->reqCacheFtr.cd_ctrl;
428 		offset = cd_ctrl->cipher_cfg_offset;
429 	}
430 
431 	return offset * LAC_QUAD_WORD_IN_BYTES;
432 }
433 
434 CpaStatus
LacAlgChain_SessionAADUpdate(lac_session_desc_t * pSessionDesc,Cpa32U newAADLength)435 LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc,
436 			     Cpa32U newAADLength)
437 {
438 	icp_qat_la_bulk_req_ftr_t *req_ftr = &pSessionDesc->reqCacheFtr;
439 	icp_qat_la_auth_req_params_t *req_params = &req_ftr->serv_specif_rqpars;
440 
441 	if (!pSessionDesc)
442 		return CPA_STATUS_FAIL;
443 
444 	pSessionDesc->aadLenInBytes = newAADLength;
445 	req_params->u2.aad_sz =
446 	    LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE);
447 
448 	if (SPC == pSessionDesc->singlePassState) {
449 		Cpa8U *pHwBlockBaseInDRAM = NULL;
450 		Cpa32U hwBlockOffsetInDRAM = 0;
451 		Cpa32U pSizeInBytes = 0;
452 		CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
453 
454 		pHwBlockBaseInDRAM =
455 		    (Cpa8U *)pSessionDesc->contentDescInfo.pData;
456 		if (pSessionDesc->cipherDirection ==
457 		    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) {
458 			if (LAC_CIPHER_IS_GCM(cipher)) {
459 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
460 				    LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
461 			} else {
462 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
463 				    LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
464 			}
465 		}
466 		LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
467 						       pHwBlockBaseInDRAM +
468 							   hwBlockOffsetInDRAM,
469 						       &pSizeInBytes);
470 	}
471 
472 	return CPA_STATUS_SUCCESS;
473 }
474 
475 CpaStatus
LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pCipherKey)476 LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc,
477 				   Cpa8U *pCipherKey)
478 {
479 	CpaStatus status = CPA_STATUS_SUCCESS;
480 
481 	if (pSessionDesc == NULL || pCipherKey == NULL)
482 		return CPA_STATUS_FAIL;
483 
484 	if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
485 		LacSymQat_CipherArc4StateInit(
486 		    pCipherKey,
487 		    pSessionDesc->cipherKeyLenInBytes,
488 		    pSessionDesc->cipherARC4InitialState);
489 	} else {
490 		CpaCySymCipherSetupData cipherSetupData = { 0 };
491 		Cpa32U sizeInBytes;
492 		Cpa8U *pCipherKeyField;
493 		Cpa16U cipherConfigSize;
494 		Cpa16U cipherConfigOffset;
495 		sal_qat_content_desc_info_t *pCdInfo =
496 		    &(pSessionDesc->contentDescInfo);
497 
498 		cipherSetupData.cipherAlgorithm = pSessionDesc->cipherAlgorithm;
499 		cipherSetupData.cipherKeyLenInBytes =
500 		    pSessionDesc->cipherKeyLenInBytes;
501 		cipherSetupData.pCipherKey = pCipherKey;
502 		cipherSetupData.cipherDirection = pSessionDesc->cipherDirection;
503 
504 		cipherConfigSize =
505 		    LacAlgChain_GetCipherConfigSize(pSessionDesc);
506 		cipherConfigOffset =
507 		    LacAlgChain_GetCipherConfigOffset(pSessionDesc);
508 
509 		pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset +
510 		    cipherConfigSize;
511 
512 		switch (pSessionDesc->symOperation) {
513 		case CPA_CY_SYM_OP_CIPHER: {
514 			LacSymQat_CipherHwBlockPopulateKeySetup(
515 			    pSessionDesc,
516 			    &(cipherSetupData),
517 			    cipherSetupData.cipherKeyLenInBytes,
518 			    pSessionDesc->cipherSliceType,
519 			    pCipherKeyField,
520 			    &sizeInBytes);
521 
522 			if (pSessionDesc->useSymConstantsTable) {
523 				pCipherKeyField = (Cpa8U *)&(
524 				    pSessionDesc->shramReqCacheHdr.cd_pars.s1
525 					.serv_specif_fields);
526 
527 				LacSymQat_CipherHwBlockPopulateKeySetup(
528 				    pSessionDesc,
529 				    &(cipherSetupData),
530 				    cipherSetupData.cipherKeyLenInBytes,
531 				    pSessionDesc->cipherSliceType,
532 				    pCipherKeyField,
533 				    &sizeInBytes);
534 			}
535 		} break;
536 
537 		case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
538 			LacSymQat_CipherHwBlockPopulateKeySetup(
539 			    pSessionDesc,
540 			    &(cipherSetupData),
541 			    cipherSetupData.cipherKeyLenInBytes,
542 			    pSessionDesc->cipherSliceType,
543 			    pCipherKeyField,
544 			    &sizeInBytes);
545 		} break;
546 
547 		default:
548 			LAC_LOG_ERROR("Invalid sym operation\n");
549 			status = CPA_STATUS_INVALID_PARAM;
550 			break;
551 		}
552 	}
553 	return status;
554 }
555 
556 CpaStatus
LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t * pSessionDesc,Cpa8U * pAuthKey)557 LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc,
558 				 Cpa8U *pAuthKey)
559 {
560 	CpaStatus status = CPA_STATUS_SUCCESS;
561 	Cpa8U *pHwBlockBaseInDRAM = NULL;
562 	Cpa8U *pOutHashSetup = NULL;
563 	Cpa8U *pInnerState1 = NULL;
564 	Cpa8U *pInnerState2 = NULL;
565 	CpaCySymSessionSetupData sessionSetup = { 0 };
566 	Cpa16U cipherConfigSize;
567 
568 	if (pSessionDesc == NULL || pAuthKey == NULL)
569 		return CPA_STATUS_FAIL;
570 
571 	cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc);
572 
573 	icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
574 	    (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
575 		.cd_ctrl;
576 
577 	pHwBlockBaseInDRAM = (Cpa8U *)pSessionDesc->contentDescInfo.pData;
578 
579 	sessionSetup.hashSetupData.hashAlgorithm = pSessionDesc->hashAlgorithm;
580 	sessionSetup.hashSetupData.hashMode = pSessionDesc->hashMode;
581 	sessionSetup.hashSetupData.authModeSetupData.authKey = pAuthKey;
582 	sessionSetup.hashSetupData.authModeSetupData.authKeyLenInBytes =
583 	    pSessionDesc->authKeyLenInBytes;
584 	sessionSetup.hashSetupData.authModeSetupData.aadLenInBytes =
585 	    pSessionDesc->aadLenInBytes;
586 	sessionSetup.hashSetupData.digestResultLenInBytes =
587 	    pSessionDesc->hashResultSize;
588 
589 	sessionSetup.cipherSetupData.cipherAlgorithm =
590 	    pSessionDesc->cipherAlgorithm;
591 	sessionSetup.cipherSetupData.cipherKeyLenInBytes =
592 	    pSessionDesc->cipherKeyLenInBytes;
593 
594 	/* Calculate hash states offsets */
595 	pInnerState1 = pHwBlockBaseInDRAM +
596 	    cd_ctrl->hash_cfg_offset * LAC_QUAD_WORD_IN_BYTES +
597 	    sizeof(icp_qat_hw_auth_setup_t);
598 
599 	pInnerState2 = pInnerState1 + cd_ctrl->inner_state1_sz;
600 
601 	pOutHashSetup = pInnerState2 + cd_ctrl->inner_state2_sz;
602 
603 	/* Calculate offset of cipher key */
604 	if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
605 		sessionSetup.cipherSetupData.pCipherKey =
606 		    (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
607 	} else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
608 		sessionSetup.cipherSetupData.pCipherKey =
609 		    pOutHashSetup + cipherConfigSize;
610 	} else if (SPC == pSessionDesc->singlePassState) {
611 		CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
612 		Cpa32U hwBlockOffsetInDRAM = 0;
613 
614 		if (pSessionDesc->cipherDirection ==
615 		    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
616 			sessionSetup.cipherSetupData.pCipherKey =
617 			    (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
618 		} else {
619 			if (LAC_CIPHER_IS_GCM(cipher))
620 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
621 				    LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
622 			else
623 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
624 				    LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
625 			sessionSetup.cipherSetupData.pCipherKey =
626 			    (Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM +
627 			    cipherConfigSize;
628 		}
629 	}
630 
631 	if (!sessionSetup.cipherSetupData.pCipherKey)
632 		return CPA_STATUS_FAIL;
633 
634 	if (CPA_CY_SYM_HASH_SHA3_256 == pSessionDesc->hashAlgorithm) {
635 		if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
636 			lac_sym_qat_hash_state_buffer_info_t
637 			    *pHashStateBufferInfo =
638 				&(pSessionDesc->hashStateBufferInfo);
639 
640 			sal_crypto_service_t *pService =
641 			    (sal_crypto_service_t *)pSessionDesc->pInstance;
642 
643 			status = LacHash_StatePrefixAadBufferInit(
644 			    &(pService->generic_service_info),
645 			    &(sessionSetup.hashSetupData),
646 			    &(pSessionDesc->reqCacheFtr),
647 			    pSessionDesc->qatHashMode,
648 			    pSessionDesc->hashStatePrefixBuffer,
649 			    pHashStateBufferInfo);
650 			/* SHRAM Constants Table not used for Auth-Enc */
651 		}
652 	} else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) {
653 		Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize;
654 		memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes);
655 	} else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm ||
656 		   CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) {
657 		memcpy(pInnerState2, pAuthKey, pSessionDesc->authKeyLenInBytes);
658 	} else if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm ||
659 		   CPA_CY_SYM_HASH_KASUMI_F9 == pSessionDesc->hashAlgorithm ||
660 		   IS_HASH_MODE_1(pSessionDesc->qatHashMode)) {
661 		if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm) {
662 			memset(pInnerState2, 0, cd_ctrl->inner_state2_sz);
663 		}
664 
665 		/* Block messages until precompute is completed */
666 		pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
667 
668 		status = LacHash_PrecomputeDataCreate(
669 		    pSessionDesc->pInstance,
670 		    (CpaCySymSessionSetupData *)&(sessionSetup),
671 		    LacSymAlgChain_HashPrecomputeDoneCb,
672 		    pSessionDesc,
673 		    pSessionDesc->hashStatePrefixBuffer,
674 		    pInnerState1,
675 		    pInnerState2);
676 	}
677 
678 	return status;
679 }
680 
681 static void
buildCmdData(sal_crypto_service_t * pService,lac_session_desc_t * pSessionDesc,CpaCySymAlgChainOrder * chainOrder,Cpa16U * proto,icp_qat_fw_serv_specif_flags * laCmdFlags,icp_qat_fw_comn_flags * cmnRequestFlags)682 buildCmdData(sal_crypto_service_t *pService,
683 	     lac_session_desc_t *pSessionDesc,
684 	     CpaCySymAlgChainOrder *chainOrder,
685 	     Cpa16U *proto,
686 	     icp_qat_fw_serv_specif_flags *laCmdFlags,
687 	     icp_qat_fw_comn_flags *cmnRequestFlags)
688 {
689 	/* LW 28 is used to set hash flags for AlgChaining. */
690 	icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
691 	    (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
692 		.cd_ctrl;
693 
694 	/* proto refers to Protocol Flags, which is legacy FW <=> IA interface
695 	 * for ZUC and Snow3G. Use extended protocol flags for AlgChaining.
696 	 */
697 	*proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
698 
699 	switch (pSessionDesc->symOperation) {
700 	case CPA_CY_SYM_OP_CIPHER:
701 		pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
702 
703 		if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
704 		    pSessionDesc->cipherAlgorithm) {
705 			*proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
706 		} else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
707 			   pSessionDesc->cipherAlgorithm) {
708 			*proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
709 		}
710 		if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
711 			*proto = ICP_QAT_FW_LA_CCM_PROTO;
712 		}
713 		break;
714 
715 	case CPA_CY_SYM_OP_HASH:
716 		pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH;
717 		if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
718 		    pSessionDesc->hashAlgorithm) {
719 			*proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
720 		} else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
721 			   pSessionDesc->hashAlgorithm) {
722 			*proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
723 		}
724 		break;
725 
726 	case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
727 		if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
728 			*proto = ICP_QAT_FW_LA_CCM_PROTO;
729 
730 			/* Derive chainOrder from direction for isAuthEncryptOp
731 			 * cases */
732 			/* For CCM & GCM modes: force digest verify flag _TRUE
733 			   for decrypt and _FALSE for encrypt. For all other
734 			   cases use user defined value */
735 
736 			if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
737 			    pSessionDesc->cipherDirection) {
738 				*chainOrder =
739 				    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
740 				pSessionDesc->digestVerify = CPA_FALSE;
741 			} else {
742 				*chainOrder =
743 				    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
744 				if (CPA_TRUE == pService->forceAEADMacVerify) {
745 					pSessionDesc->digestVerify = CPA_TRUE;
746 				}
747 			}
748 		} else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) {
749 			*proto = ICP_QAT_FW_LA_GCM_PROTO;
750 
751 			/* Derive chainOrder from direction for isAuthEncryptOp
752 			 * cases */
753 			/* For CCM & GCM modes: force digest verify flag _TRUE
754 			   for decrypt and _FALSE for encrypt. For all other
755 			   cases use user defined value */
756 
757 			if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
758 			    pSessionDesc->cipherDirection) {
759 				*chainOrder =
760 				    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
761 				pSessionDesc->digestVerify = CPA_FALSE;
762 			} else {
763 				*chainOrder =
764 				    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
765 				if (CPA_TRUE == pService->forceAEADMacVerify) {
766 					pSessionDesc->digestVerify = CPA_TRUE;
767 				}
768 			}
769 		} else if (LAC_CIPHER_IS_CHACHA(
770 			       pSessionDesc->cipherAlgorithm)) {
771 			if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
772 			    pSessionDesc->cipherDirection) {
773 				*chainOrder =
774 				    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
775 			} else {
776 				*chainOrder =
777 				    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
778 			}
779 		} else {
780 			pSessionDesc->isAuthEncryptOp = CPA_FALSE;
781 
782 			if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
783 			    pSessionDesc->cipherAlgorithm) {
784 				*proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
785 			} else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
786 				   pSessionDesc->cipherAlgorithm) {
787 				*proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
788 			}
789 
790 			if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
791 			    pSessionDesc->hashAlgorithm) {
792 				/* Need to set LW 28 hash flags as well. */
793 				ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET(
794 				    cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2);
795 			} else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
796 				   pSessionDesc->hashAlgorithm) {
797 				/* Need to set LW 28 hash flags as well. */
798 				ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET(
799 				    cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3);
800 			}
801 		}
802 
803 		if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH ==
804 		    *chainOrder) {
805 			pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
806 		} else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
807 			   *chainOrder) {
808 			pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
809 		}
810 		break;
811 
812 	default:
813 		break;
814 	}
815 
816 	/*
817 	 * Build the header flags with the default settings for this session.
818 	 */
819 	if (pSessionDesc->isDPSession == CPA_TRUE) {
820 		*cmnRequestFlags =
821 		    ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
822 						LAC_SYM_DP_QAT_PTR_TYPE);
823 	} else {
824 		*cmnRequestFlags =
825 		    ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
826 						LAC_SYM_DEFAULT_QAT_PTR_TYPE);
827 	}
828 
829 	LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation);
830 
831 	return;
832 }
833 
834 static void
updateLaCmdFlags(lac_session_desc_t * pSessionDesc,Cpa16U proto,icp_qat_fw_serv_specif_flags * laCmdFlags)835 updateLaCmdFlags(lac_session_desc_t *pSessionDesc,
836 		 Cpa16U proto,
837 		 icp_qat_fw_serv_specif_flags *laCmdFlags)
838 {
839 	if (pSessionDesc->isAuth) {
840 		if (pSessionDesc->digestVerify) {
841 			ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags,
842 						   ICP_QAT_FW_LA_CMP_AUTH_RES);
843 			ICP_QAT_FW_LA_RET_AUTH_SET(
844 			    *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES);
845 		} else {
846 			ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags,
847 						   ICP_QAT_FW_LA_RET_AUTH_RES);
848 			ICP_QAT_FW_LA_CMP_AUTH_SET(
849 			    *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
850 		}
851 	}
852 
853 	if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) ||
854 	    (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) {
855 		/* New bit position (12) for ZUC. The FW provides a specific
856 		 * macro to use to set the ZUC proto flag. With the new FW I/F
857 		 * this needs to be set for both Cipher and Auth */
858 		ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto);
859 	} else {
860 		/* Configure the common header */
861 		ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto);
862 	}
863 
864 	/* set Append flag, if digest is appended */
865 	if (pSessionDesc->digestIsAppended) {
866 		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
867 		    *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
868 	} else {
869 		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
870 		    *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
871 	}
872 }
873 
874 static lac_single_pass_state_t
LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,CpaCySymHashAlgorithm hash,Cpa32U capabilitiesMask)875 LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,
876 			   CpaCySymHashAlgorithm hash,
877 			   Cpa32U capabilitiesMask)
878 {
879 	lac_single_pass_state_t state = NON_SPC;
880 	if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) {
881 		switch (cipher) {
882 		case CPA_CY_SYM_CIPHER_CHACHA: {
883 			if (CPA_CY_SYM_HASH_POLY == hash)
884 				state = SPC;
885 			break;
886 		}
887 		case CPA_CY_SYM_CIPHER_AES_GCM: {
888 			if ((CPA_CY_SYM_HASH_AES_GCM == hash) ||
889 			    (CPA_CY_SYM_HASH_AES_GMAC == hash))
890 				state = LIKELY_SPC;
891 			break;
892 		}
893 		case CPA_CY_SYM_CIPHER_AES_CCM: {
894 			if (LAC_CIPHER_AES_V2(capabilitiesMask))
895 				state = SPC;
896 		}
897 		default:
898 			/* Do Nothing as it is NON_SPC */
899 			break;
900 		}
901 	}
902 	return state;
903 }
904 
905 static CpaBoolean
LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,Cpa32U capabilitiesMask,lac_session_desc_t * pSessionDesc)906 LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,
907 				       Cpa32U capabilitiesMask,
908 				       lac_session_desc_t *pSessionDesc)
909 {
910 	CpaBoolean hasSha3Ext =
911 	    ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask;
912 	CpaBoolean useStatefulSha3DescFlag = CPA_FALSE;
913 
914 	if (hasSha3Ext && !partialsNotRequired &&
915 	    (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) &&
916 	    LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) {
917 		useStatefulSha3DescFlag = CPA_TRUE;
918 	}
919 	return useStatefulSha3DescFlag;
920 }
921 
922 /** @ingroup LacAlgChain */
923 CpaStatus
LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,const CpaCySymSessionSetupData * pSessionSetupData,lac_session_desc_t * pSessionDesc)924 LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,
925 			const CpaCySymSessionSetupData *pSessionSetupData,
926 			lac_session_desc_t *pSessionDesc)
927 {
928 	CpaStatus stat, status = CPA_STATUS_SUCCESS;
929 	sal_qat_content_desc_info_t *pCdInfo = NULL;
930 	sal_qat_content_desc_info_t *pCdInfoOptimised = NULL;
931 	sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
932 	Cpa32U capabilitiesMask =
933 	    pService->generic_service_info.capabilitiesMask;
934 	Cpa8U *pHwBlockBaseInDRAM = NULL;
935 	Cpa8U *pOptimisedHwBlockBaseInDRAM = NULL;
936 	Cpa32U hwBlockOffsetInDRAM = 0;
937 	Cpa32U optimisedHwBlockOffsetInDRAM = 0;
938 	Cpa8U cipherOffsetInConstantsTable = 0;
939 	Cpa8U hashOffsetInConstantsTable = 0;
940 	icp_qat_fw_comn_flags cmnRequestFlags = 0;
941 	icp_qat_fw_comn_req_t *pMsg = NULL;
942 	icp_qat_fw_comn_req_t *pMsgS = NULL;
943 	const CpaCySymCipherSetupData *pCipherData;
944 	const CpaCySymHashSetupData *pHashData;
945 	Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
946 	CpaCySymAlgChainOrder chainOrder = 0;
947 	lac_sym_qat_hash_precompute_info_t precomputeData = { 0 };
948 	lac_sym_qat_hash_precompute_info_t precomputeDataOptimisedCd = { 0 };
949 
950 	pCipherData = &(pSessionSetupData->cipherSetupData);
951 	pHashData = &(pSessionSetupData->hashSetupData);
952 
953 	/*-------------------------------------------------------------------------
954 	 * Populate session data
955 	 *-----------------------------------------------------------------------*/
956 
957 	/* Initialise Request Queue */
958 	stat = LAC_SPINLOCK_INIT(&pSessionDesc->requestQueueLock);
959 	if (CPA_STATUS_SUCCESS != stat) {
960 		LAC_LOG_ERROR("Spinlock init failed for sessionLock");
961 		return CPA_STATUS_RESOURCE;
962 	}
963 
964 	pSessionDesc->pRequestQueueHead = NULL;
965 	pSessionDesc->pRequestQueueTail = NULL;
966 	pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
967 	pSessionDesc->pInstance = instanceHandle;
968 	pSessionDesc->digestIsAppended = pSessionSetupData->digestIsAppended;
969 	pSessionDesc->digestVerify = pSessionSetupData->verifyDigest;
970 
971 	/* Reset the pending callback counter */
972 	qatUtilsAtomicSet(0, &pSessionDesc->u.pendingCbCount);
973 	qatUtilsAtomicSet(0, &pSessionDesc->u.pendingDpCbCount);
974 
975 	/* Partial state must be set to full, to indicate that next packet
976 	 * expected on the session is a full packet or the start of a
977 	 * partial packet. */
978 	pSessionDesc->partialState = CPA_CY_SYM_PACKET_TYPE_FULL;
979 
980 	pSessionDesc->symOperation = pSessionSetupData->symOperation;
981 	switch (pSessionDesc->symOperation) {
982 	case CPA_CY_SYM_OP_CIPHER:
983 		pSessionDesc->isCipher = CPA_TRUE;
984 		pSessionDesc->isAuth = CPA_FALSE;
985 		pSessionDesc->isAuthEncryptOp = CPA_FALSE;
986 		pSessionDesc->singlePassState = NON_SPC;
987 		break;
988 	case CPA_CY_SYM_OP_HASH:
989 		pSessionDesc->isCipher = CPA_FALSE;
990 		pSessionDesc->isAuth = CPA_TRUE;
991 		pSessionDesc->isAuthEncryptOp = CPA_FALSE;
992 		pSessionDesc->singlePassState = NON_SPC;
993 		break;
994 	case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
995 		pSessionDesc->isCipher = CPA_TRUE;
996 		pSessionDesc->isAuth = CPA_TRUE;
997 		pSessionDesc->singlePassState =
998 		    LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm,
999 					       pHashData->hashAlgorithm,
1000 					       capabilitiesMask);
1001 
1002 		switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) {
1003 		case CPA_CY_SYM_CIPHER_AES_CCM: {
1004 			pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1005 			pSessionDesc->digestIsAppended = CPA_TRUE;
1006 		} break;
1007 		case CPA_CY_SYM_CIPHER_AES_GCM:
1008 		case CPA_CY_SYM_CIPHER_CHACHA:
1009 			pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1010 			break;
1011 		default: {
1012 			pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1013 			/* Use the chainOrder passed in */
1014 			chainOrder = pSessionSetupData->algChainOrder;
1015 			if ((chainOrder !=
1016 			     CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) &&
1017 			    (chainOrder !=
1018 			     CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) {
1019 				LAC_INVALID_PARAM_LOG("algChainOrder");
1020 				return CPA_STATUS_INVALID_PARAM;
1021 			}
1022 		} break;
1023 		}
1024 	} break;
1025 	default:
1026 		pSessionDesc->singlePassState = NON_SPC;
1027 		break;
1028 	}
1029 
1030 	if (pSessionDesc->isCipher) {
1031 		/* Populate cipher specific session data */
1032 
1033 		status = LacCipher_SessionSetupDataCheck(pCipherData,
1034 							 capabilitiesMask);
1035 
1036 		if (CPA_STATUS_SUCCESS == status) {
1037 			pSessionDesc->cipherAlgorithm =
1038 			    pCipherData->cipherAlgorithm;
1039 			pSessionDesc->cipherKeyLenInBytes =
1040 			    pCipherData->cipherKeyLenInBytes;
1041 			pSessionDesc->cipherDirection =
1042 			    pCipherData->cipherDirection;
1043 
1044 			/* ARC4 base key isn't added to the content descriptor,
1045 			 * because we don't need to pass it directly to the QAT
1046 			 * engine. Instead an initial cipher state & key matrix
1047 			 * is derived from the base key and provided to the QAT
1048 			 * through the state pointer in the request params.
1049 			 * We'll store this initial state in the session
1050 			 * descriptor. */
1051 
1052 			if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
1053 				LacSymQat_CipherArc4StateInit(
1054 				    pCipherData->pCipherKey,
1055 				    pSessionDesc->cipherKeyLenInBytes,
1056 				    pSessionDesc->cipherARC4InitialState);
1057 
1058 				pSessionDesc->cipherARC4InitialStatePhysAddr =
1059 				    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1060 					pService->generic_service_info,
1061 					pSessionDesc->cipherARC4InitialState);
1062 
1063 				if (0 ==
1064 				    pSessionDesc
1065 					->cipherARC4InitialStatePhysAddr) {
1066 					LAC_LOG_ERROR(
1067 					    "Unable to get the physical address of "
1068 					    "the initial state for ARC4\n");
1069 					status = CPA_STATUS_FAIL;
1070 				}
1071 			}
1072 		}
1073 	}
1074 
1075 	if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1076 		/* Populate auth-specific session data */
1077 		const CpaCySymHashSetupData *pHashData =
1078 		    &pSessionSetupData->hashSetupData;
1079 
1080 		status = LacHash_HashContextCheck(instanceHandle, pHashData);
1081 		if (CPA_STATUS_SUCCESS == status) {
1082 			pSessionDesc->hashResultSize =
1083 			    pHashData->digestResultLenInBytes;
1084 			pSessionDesc->hashMode = pHashData->hashMode;
1085 			pSessionDesc->hashAlgorithm = pHashData->hashAlgorithm;
1086 
1087 			/* Save the authentication key length for further update
1088 			 */
1089 			if (CPA_CY_SYM_HASH_MODE_AUTH == pHashData->hashMode) {
1090 				pSessionDesc->authKeyLenInBytes =
1091 				    pHashData->authModeSetupData
1092 					.authKeyLenInBytes;
1093 			}
1094 			if (CPA_TRUE == pSessionDesc->isAuthEncryptOp ||
1095 			    (pHashData->hashAlgorithm ==
1096 				 CPA_CY_SYM_HASH_SNOW3G_UIA2 ||
1097 			     pHashData->hashAlgorithm ==
1098 				 CPA_CY_SYM_HASH_ZUC_EIA3)) {
1099 				pSessionDesc->aadLenInBytes =
1100 				    pHashData->authModeSetupData.aadLenInBytes;
1101 			}
1102 
1103 			/* Set the QAT hash mode */
1104 			if ((pHashData->hashMode ==
1105 			     CPA_CY_SYM_HASH_MODE_NESTED) ||
1106 			    (pHashData->hashMode ==
1107 			     CPA_CY_SYM_HASH_MODE_PLAIN) ||
1108 			    (pHashData->hashMode == CPA_CY_SYM_HASH_MODE_AUTH &&
1109 			     pHashData->hashAlgorithm ==
1110 				 CPA_CY_SYM_HASH_AES_CBC_MAC)) {
1111 				pSessionDesc->qatHashMode =
1112 				    ICP_QAT_HW_AUTH_MODE0;
1113 			} else /* CPA_CY_SYM_HASH_MODE_AUTH
1114 				  && anything except CPA_CY_SYM_HASH_AES_CBC_MAC
1115 				*/
1116 			{
1117 				if (IS_HMAC_ALG(pHashData->hashAlgorithm)) {
1118 					/* SHA3 HMAC and SM3  do not support
1119 					 * precompute, force MODE2 for AUTH */
1120 					if (LAC_HASH_IS_SHA3(
1121 						pHashData->hashAlgorithm) ||
1122 					    (CPA_CY_SYM_HASH_SM3 ==
1123 					     pHashData->hashAlgorithm)) {
1124 						pSessionDesc->qatHashMode =
1125 						    ICP_QAT_HW_AUTH_MODE2;
1126 					} else {
1127 						pSessionDesc->qatHashMode =
1128 						    pService->qatHmacMode;
1129 					}
1130 				} else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
1131 					   pHashData->hashAlgorithm) {
1132 					pSessionDesc->qatHashMode =
1133 					    ICP_QAT_HW_AUTH_MODE0;
1134 				} else {
1135 					pSessionDesc->qatHashMode =
1136 					    ICP_QAT_HW_AUTH_MODE1;
1137 				}
1138 			}
1139 		}
1140 	}
1141 
1142 	/*-------------------------------------------------------------------------
1143 	 * build the message templates
1144 	 * create two content descriptors in the case we can support using SHRAM
1145 	 * constants and an optimised content descriptor. we have to do this in
1146 	 * case of partials. 64 byte content descriptor is used in the SHRAM
1147 	 * case for AES-128-HMAC-SHA1
1148 	 *-----------------------------------------------------------------------*/
1149 	if (CPA_STATUS_SUCCESS == status) {
1150 		pSessionDesc->cipherSliceType =
1151 		    LacCipher_GetCipherSliceType(pService,
1152 						 pSessionDesc->cipherAlgorithm,
1153 						 pSessionDesc->hashAlgorithm);
1154 
1155 		LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc);
1156 		pSessionDesc->useOptimisedContentDesc = CPA_FALSE;
1157 		pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE;
1158 
1159 		/* Build configuration data */
1160 		buildCmdData(pService,
1161 			     pSessionDesc,
1162 			     &chainOrder,
1163 			     &proto,
1164 			     &pSessionDesc->laCmdFlags,
1165 			     &cmnRequestFlags);
1166 
1167 		if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE ==
1168 		    pSessionDesc->cipherSliceType)
1169 			pSessionDesc->useSymConstantsTable = CPA_FALSE;
1170 		else
1171 			pSessionDesc->useSymConstantsTable =
1172 			    LacSymQat_UseSymConstantsTable(
1173 				pSessionDesc,
1174 				&cipherOffsetInConstantsTable,
1175 				&hashOffsetInConstantsTable);
1176 
1177 		/* for a certain combination of Algorithm Chaining we want to
1178 		   use an optimised cd block */
1179 
1180 		if (pSessionDesc->symOperation ==
1181 			CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
1182 		    pSessionDesc->useSymConstantsTable == CPA_TRUE) {
1183 			pSessionDesc->useOptimisedContentDesc =
1184 			    LacSymQat_UseOptimisedContentDesc(pSessionDesc);
1185 		}
1186 
1187 		/* check whether we need to construct content desc for stateful
1188 		 * SHA3 */
1189 		pSessionDesc->useStatefulSha3ContentDesc =
1190 		    LacAlgChain_UseStatefulSha3ContentDesc(
1191 			pSessionSetupData->partialsNotRequired,
1192 			capabilitiesMask,
1193 			pSessionDesc);
1194 
1195 		/* setup some convenience pointers */
1196 		pCdInfo = &(pSessionDesc->contentDescInfo);
1197 		pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1198 		hwBlockOffsetInDRAM = 0;
1199 
1200 		/* set up the pointer for the optimised content desc if this is
1201 		 * possible we still have to support both cd types in case of
1202 		 * partials so we construct both */
1203 		if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) {
1204 			pCdInfoOptimised =
1205 			    &(pSessionDesc->contentDescOptimisedInfo);
1206 			pOptimisedHwBlockBaseInDRAM =
1207 			    (Cpa8U *)pCdInfoOptimised->pData;
1208 			optimisedHwBlockOffsetInDRAM = 0;
1209 		}
1210 
1211 		switch (pSessionDesc->symOperation) {
1212 		case CPA_CY_SYM_OP_CIPHER: {
1213 			LacAlgChain_CipherCDBuild(
1214 			    pCipherData,
1215 			    pSessionDesc,
1216 			    ICP_QAT_FW_SLICE_DRAM_WR,
1217 			    cipherOffsetInConstantsTable,
1218 			    &pSessionDesc->cmnRequestFlags,
1219 			    &pSessionDesc->laCmdFlags,
1220 			    pHwBlockBaseInDRAM,
1221 			    &hwBlockOffsetInDRAM,
1222 			    capabilitiesMask);
1223 
1224 			if (pSessionDesc->useSymConstantsTable) {
1225 				LacAlgChain_CipherCDBuild_ForSHRAM(
1226 				    pCipherData,
1227 				    pSessionDesc,
1228 				    ICP_QAT_FW_SLICE_DRAM_WR,
1229 				    cipherOffsetInConstantsTable);
1230 			}
1231 		} break;
1232 		case CPA_CY_SYM_OP_HASH:
1233 			LacAlgChain_HashCDBuild(pHashData,
1234 						instanceHandle,
1235 						pSessionDesc,
1236 						ICP_QAT_FW_SLICE_NULL,
1237 						hashOffsetInConstantsTable,
1238 						&pSessionDesc->cmnRequestFlags,
1239 						&pSessionDesc->laCmdFlags,
1240 						&precomputeData,
1241 						&precomputeDataOptimisedCd,
1242 						pHwBlockBaseInDRAM,
1243 						&hwBlockOffsetInDRAM,
1244 						NULL,
1245 						NULL);
1246 			break;
1247 		case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
1248 			/* For CCM/GCM, CPM firmware currently expects the
1249 			 * cipher and hash h/w setup blocks to be arranged
1250 			 * according to the chain order (Except for GCM/CCM,
1251 			 * order doesn't actually matter as long as the config
1252 			 * offsets are set correctly in CD control blocks
1253 			 */
1254 			if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
1255 			    chainOrder) {
1256 				LacAlgChain_HashCDBuild(
1257 				    pHashData,
1258 				    instanceHandle,
1259 				    pSessionDesc,
1260 				    ICP_QAT_FW_SLICE_CIPHER,
1261 				    hashOffsetInConstantsTable,
1262 				    &pSessionDesc->cmnRequestFlags,
1263 				    &pSessionDesc->laCmdFlags,
1264 				    &precomputeData,
1265 				    &precomputeDataOptimisedCd,
1266 				    pHwBlockBaseInDRAM,
1267 				    &hwBlockOffsetInDRAM,
1268 				    pOptimisedHwBlockBaseInDRAM,
1269 				    &optimisedHwBlockOffsetInDRAM);
1270 
1271 				LacAlgChain_CipherCDBuild(
1272 				    pCipherData,
1273 				    pSessionDesc,
1274 				    ICP_QAT_FW_SLICE_DRAM_WR,
1275 				    cipherOffsetInConstantsTable,
1276 				    &pSessionDesc->cmnRequestFlags,
1277 				    &pSessionDesc->laCmdFlags,
1278 				    pHwBlockBaseInDRAM,
1279 				    &hwBlockOffsetInDRAM,
1280 				    capabilitiesMask);
1281 
1282 				if (pSessionDesc->useOptimisedContentDesc) {
1283 					LacAlgChain_CipherCDBuild_ForOptimisedCD(
1284 					    pCipherData,
1285 					    pSessionDesc,
1286 					    ICP_QAT_FW_SLICE_DRAM_WR,
1287 					    cipherOffsetInConstantsTable,
1288 					    pOptimisedHwBlockBaseInDRAM,
1289 					    &optimisedHwBlockOffsetInDRAM);
1290 				}
1291 
1292 				if (NON_SPC != pSessionDesc->singlePassState) {
1293 					pCdInfo->hwBlkSzQuadWords =
1294 					    (LAC_BYTES_TO_QUADWORDS(
1295 						hwBlockOffsetInDRAM));
1296 					pMsg = (icp_qat_fw_comn_req_t *)&(
1297 					    pSessionDesc->reqSpcCacheHdr);
1298 					SalQatMsg_ContentDescHdrWrite(
1299 					    (icp_qat_fw_comn_req_t *)pMsg,
1300 					    pCdInfo);
1301 				}
1302 			} else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */
1303 			{
1304 				LacAlgChain_CipherCDBuild(
1305 				    pCipherData,
1306 				    pSessionDesc,
1307 				    ICP_QAT_FW_SLICE_AUTH,
1308 				    cipherOffsetInConstantsTable,
1309 				    &pSessionDesc->cmnRequestFlags,
1310 				    &pSessionDesc->laCmdFlags,
1311 				    pHwBlockBaseInDRAM,
1312 				    &hwBlockOffsetInDRAM,
1313 				    capabilitiesMask);
1314 
1315 				if (pSessionDesc->useOptimisedContentDesc) {
1316 					LacAlgChain_CipherCDBuild_ForOptimisedCD(
1317 					    pCipherData,
1318 					    pSessionDesc,
1319 					    ICP_QAT_FW_SLICE_AUTH,
1320 					    cipherOffsetInConstantsTable,
1321 					    pOptimisedHwBlockBaseInDRAM,
1322 					    &optimisedHwBlockOffsetInDRAM);
1323 				}
1324 
1325 				if (NON_SPC != pSessionDesc->singlePassState) {
1326 					pCdInfo->hwBlkSzQuadWords =
1327 					    LAC_BYTES_TO_QUADWORDS(
1328 						hwBlockOffsetInDRAM);
1329 					pMsg = (icp_qat_fw_comn_req_t *)&(
1330 					    pSessionDesc->reqSpcCacheHdr);
1331 					SalQatMsg_ContentDescHdrWrite(
1332 					    (icp_qat_fw_comn_req_t *)pMsg,
1333 					    pCdInfo);
1334 				}
1335 				LacAlgChain_HashCDBuild(
1336 				    pHashData,
1337 				    instanceHandle,
1338 				    pSessionDesc,
1339 				    ICP_QAT_FW_SLICE_DRAM_WR,
1340 				    hashOffsetInConstantsTable,
1341 				    &pSessionDesc->cmnRequestFlags,
1342 				    &pSessionDesc->laCmdFlags,
1343 				    &precomputeData,
1344 				    &precomputeDataOptimisedCd,
1345 				    pHwBlockBaseInDRAM,
1346 				    &hwBlockOffsetInDRAM,
1347 				    pOptimisedHwBlockBaseInDRAM,
1348 				    &optimisedHwBlockOffsetInDRAM);
1349 			}
1350 			break;
1351 		default:
1352 			LAC_LOG_ERROR("Invalid sym operation\n");
1353 			status = CPA_STATUS_INVALID_PARAM;
1354 		}
1355 	}
1356 
1357 	if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) {
1358 		lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo =
1359 		    &(pSessionDesc->hashStateBufferInfo);
1360 		CpaBoolean hashStateBuffer = CPA_TRUE;
1361 
1362 		/* set up fields in both the cd_ctrl and reqParams which
1363 		 * describe the ReqParams block */
1364 		LacSymQat_HashSetupReqParamsMetaData(
1365 		    &(pSessionDesc->reqCacheFtr),
1366 		    instanceHandle,
1367 		    pHashData,
1368 		    hashStateBuffer,
1369 		    pSessionDesc->qatHashMode,
1370 		    pSessionDesc->digestVerify);
1371 
1372 		if (pSessionDesc->useSymConstantsTable) {
1373 			/* Need to set up for SHRAM Constants Table use also */
1374 			LacSymQat_HashSetupReqParamsMetaData(
1375 			    &(pSessionDesc->shramReqCacheFtr),
1376 			    instanceHandle,
1377 			    pHashData,
1378 			    hashStateBuffer,
1379 			    pSessionDesc->qatHashMode,
1380 			    pSessionDesc->digestVerify);
1381 		}
1382 
1383 		/* populate the hash state prefix buffer info structure
1384 		 * (part of user allocated session memory & the
1385 		 * buffer itself. For CCM/GCM the buffer is stored in the
1386 		 * cookie and is not initialised here) */
1387 		if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) {
1388 			LAC_CHECK_64_BYTE_ALIGNMENT(
1389 			    &(pSessionDesc->hashStatePrefixBuffer[0]));
1390 			status = LacHash_StatePrefixAadBufferInit(
1391 			    &(pService->generic_service_info),
1392 			    pHashData,
1393 			    &(pSessionDesc->reqCacheFtr),
1394 			    pSessionDesc->qatHashMode,
1395 			    pSessionDesc->hashStatePrefixBuffer,
1396 			    pHashStateBufferInfo);
1397 			/* SHRAM Constants Table not used for Auth-Enc */
1398 		}
1399 
1400 		if (CPA_STATUS_SUCCESS == status) {
1401 			if (IS_HASH_MODE_1(pSessionDesc->qatHashMode) ||
1402 			    CPA_CY_SYM_HASH_ZUC_EIA3 ==
1403 				pHashData->hashAlgorithm) {
1404 				LAC_CHECK_64_BYTE_ALIGNMENT(
1405 				    &(pSessionDesc->hashStatePrefixBuffer[0]));
1406 
1407 				/* Block messages until precompute is completed
1408 				 */
1409 				pSessionDesc->nonBlockingOpsInProgress =
1410 				    CPA_FALSE;
1411 				status = LacHash_PrecomputeDataCreate(
1412 				    instanceHandle,
1413 				    (CpaCySymSessionSetupData *)
1414 					pSessionSetupData,
1415 				    LacSymAlgChain_HashPrecomputeDoneCb,
1416 				    pSessionDesc,
1417 				    pSessionDesc->hashStatePrefixBuffer,
1418 				    precomputeData.pState1,
1419 				    precomputeData.pState2);
1420 				if (pSessionDesc->useOptimisedContentDesc) {
1421 					status = LacHash_PrecomputeDataCreate(
1422 					    instanceHandle,
1423 					    (CpaCySymSessionSetupData *)
1424 						pSessionSetupData,
1425 					    LacSymAlgChain_HashPrecomputeDoneCb,
1426 					    pSessionDesc,
1427 					    pSessionDesc->hashStatePrefixBuffer,
1428 					    precomputeDataOptimisedCd.pState1,
1429 					    precomputeDataOptimisedCd.pState2);
1430 				}
1431 			} else if (pHashData->hashAlgorithm ==
1432 				   CPA_CY_SYM_HASH_AES_CBC_MAC) {
1433 				if (NULL != precomputeData.pState2) {
1434 					LAC_OS_BZERO(precomputeData.pState2,
1435 						     precomputeData.state2Size);
1436 					memcpy(precomputeData.pState2,
1437 					       pHashData->authModeSetupData
1438 						   .authKey,
1439 					       pHashData->authModeSetupData
1440 						   .authKeyLenInBytes);
1441 				}
1442 			}
1443 		}
1444 	}
1445 
1446 	if (CPA_STATUS_SUCCESS == status) {
1447 
1448 		/* Configure the ContentDescriptor field
1449 		    in the request if not done already */
1450 		pCdInfo->hwBlkSzQuadWords =
1451 		    LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM);
1452 		pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr);
1453 		SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1454 					      pCdInfo);
1455 
1456 		pMsgS =
1457 		    (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr);
1458 		/*If we are using the optimised CD then
1459 		  we have to set this up correctly in the SHARM reqCache*/
1460 		if (pSessionDesc->useOptimisedContentDesc) {
1461 			pCdInfoOptimised->hwBlkSzQuadWords =
1462 			    LAC_BYTES_TO_QUADWORDS(
1463 				optimisedHwBlockOffsetInDRAM);
1464 			SalQatMsg_ContentDescHdrWrite(
1465 			    (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised);
1466 		}
1467 
1468 		/* Updates command flags basing on configured alg */
1469 		updateLaCmdFlags(pSessionDesc,
1470 				 proto,
1471 				 &pSessionDesc->laCmdFlags);
1472 
1473 		SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
1474 				      ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1475 				      pSessionDesc->laCmdId,
1476 				      pSessionDesc->cmnRequestFlags,
1477 				      pSessionDesc->laCmdFlags);
1478 
1479 		/* Need to duplicate if SHRAM Constants Table used */
1480 		if (pSessionDesc->useSymConstantsTable) {
1481 			ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
1482 			    pSessionDesc->laCmdFlags,
1483 			    ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP);
1484 
1485 			if (pSessionDesc->isCipher &&
1486 			    !pSessionDesc->useOptimisedContentDesc) {
1487 				ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(
1488 				    cmnRequestFlags,
1489 				    QAT_COMN_CD_FLD_TYPE_16BYTE_DATA);
1490 			}
1491 
1492 			SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS,
1493 					      ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1494 					      pSessionDesc->laCmdId,
1495 					      cmnRequestFlags,
1496 					      pSessionDesc->laCmdFlags);
1497 		}
1498 	}
1499 
1500 	return status;
1501 }
1502 
1503 static void
LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t * pMsg,Cpa32U packetType,icp_qat_hw_auth_mode_t qatHashMode)1504 LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg,
1505 					    Cpa32U packetType,
1506 					    icp_qat_hw_auth_mode_t qatHashMode)
1507 {
1508 	icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL;
1509 
1510 	pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl);
1511 
1512 	if (IS_HASH_MODE_2(qatHashMode)) {
1513 		if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1514 		    (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1515 			ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1516 			    pAuthCdCtrlHdr->hash_flags,
1517 			    QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1518 			ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1519 			    pAuthCdCtrlHdr->hash_flags,
1520 			    QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1521 		} else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) {
1522 			ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
1523 			    pAuthCdCtrlHdr->hash_flags,
1524 			    QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
1525 		}
1526 	} else {
1527 		if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
1528 		    (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
1529 			ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
1530 			    pAuthCdCtrlHdr->hash_flags,
1531 			    QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
1532 		}
1533 	}
1534 }
1535 
1536 /** @ingroup LacAlgChain */
1537 CpaStatus
LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,lac_session_desc_t * pSessionDesc,void * pCallbackTag,const CpaCySymOpData * pOpData,const CpaBufferList * pSrcBuffer,CpaBufferList * pDstBuffer,CpaBoolean * pVerifyResult)1538 LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,
1539 		    lac_session_desc_t *pSessionDesc,
1540 		    void *pCallbackTag,
1541 		    const CpaCySymOpData *pOpData,
1542 		    const CpaBufferList *pSrcBuffer,
1543 		    CpaBufferList *pDstBuffer,
1544 		    CpaBoolean *pVerifyResult)
1545 {
1546 	CpaStatus status = CPA_STATUS_SUCCESS;
1547 	sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
1548 	Cpa32U capabilitiesMask =
1549 	    pService->generic_service_info.capabilitiesMask;
1550 	lac_sym_bulk_cookie_t *pCookie = NULL;
1551 	lac_sym_cookie_t *pSymCookie = NULL;
1552 	icp_qat_fw_la_bulk_req_t *pMsg = NULL;
1553 	Cpa8U *pMsgDummy = NULL;
1554 	Cpa8U *pCacheDummyHdr = NULL;
1555 	Cpa8U *pCacheDummyFtr = NULL;
1556 	Cpa32U qatPacketType = 0;
1557 	CpaBufferList *pBufferList = NULL;
1558 	Cpa8U *pDigestResult = NULL;
1559 	Cpa64U srcAddrPhys = 0;
1560 	Cpa64U dstAddrPhys = 0;
1561 	icp_qat_fw_la_cmd_id_t laCmdId;
1562 	sal_qat_content_desc_info_t *pCdInfo = NULL;
1563 	Cpa8U *pHwBlockBaseInDRAM = NULL;
1564 	Cpa32U hwBlockOffsetInDRAM = 0;
1565 	Cpa32U sizeInBytes = 0;
1566 	icp_qat_fw_cipher_cd_ctrl_hdr_t *pSpcCdCtrlHdr = NULL;
1567 	CpaCySymCipherAlgorithm cipher;
1568 	CpaCySymHashAlgorithm hash;
1569 	Cpa8U paddingLen = 0;
1570 	Cpa8U blockLen = 0;
1571 	CpaBoolean digestIsAppended = CPA_FALSE;
1572 	Cpa32U aadLenInBytes = 0;
1573 	Cpa64U srcPktSize = 0;
1574 	Cpa64U dstPktSize = 0;
1575 
1576 	/* Set the command id */
1577 	laCmdId = pSessionDesc->laCmdId;
1578 
1579 	cipher = pSessionDesc->cipherAlgorithm;
1580 	hash = pSessionDesc->hashAlgorithm;
1581 
1582 	CpaBoolean isSpCcm =
1583 	    (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask));
1584 
1585 	if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1586 		pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes;
1587 		if (pOpData->messageLenToHashInBytes == 0 ||
1588 		    pOpData->pAdditionalAuthData != NULL) {
1589 			LAC_INVALID_PARAM_LOG(
1590 			    "For AES_GMAC, AAD Length "
1591 			    "(messageLenToHashInBytes) must "
1592 			    "be non zero and pAdditionalAuthData "
1593 			    "must be NULL");
1594 			return CPA_STATUS_INVALID_PARAM;
1595 		}
1596 	}
1597 
1598 	aadLenInBytes = pSessionDesc->aadLenInBytes;
1599 
1600 	/* Convert Alg Chain Request to Cipher Request for CCP and
1601 	 * AES_GCM single pass */
1602 	if ((NON_SPC != pSessionDesc->singlePassState) &&
1603 	    (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) {
1604 		pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
1605 		laCmdId = pSessionDesc->laCmdId;
1606 		pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER;
1607 		pSessionDesc->singlePassState = SPC;
1608 		pSessionDesc->isCipher = CPA_TRUE;
1609 		pSessionDesc->isAuthEncryptOp = CPA_FALSE;
1610 		pSessionDesc->isAuth = CPA_FALSE;
1611 
1612 		if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
1613 			if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) {
1614 				LAC_INVALID_PARAM_LOG(
1615 				    "aadLenInBytes for AES_GMAC");
1616 				return CPA_STATUS_INVALID_PARAM;
1617 			}
1618 		}
1619 		/* New bit position (13) for SINGLE PASS.
1620 		 * The FW provides a specific macro to use to set the proto flag
1621 		 */
1622 		ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1623 		    pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
1624 		if (isCyGen2x(pService)) {
1625 			ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
1626 		}
1627 
1628 		pCdInfo = &(pSessionDesc->contentDescInfo);
1629 		pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1630 		if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
1631 		    pSessionDesc->cipherDirection) {
1632 			if (LAC_CIPHER_IS_GCM(cipher))
1633 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1634 				    LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
1635 			else if (LAC_CIPHER_IS_CHACHA(cipher))
1636 				hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1637 				    LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
1638 		} else if (isSpCcm) {
1639 			hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
1640 			    LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM);
1641 		}
1642 
1643 		/* Update cipher slice type */
1644 		pSessionDesc->cipherSliceType =
1645 		    LacCipher_GetCipherSliceType(pService,
1646 						 pSessionDesc->cipherAlgorithm,
1647 						 pSessionDesc->hashAlgorithm);
1648 
1649 		ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags,
1650 					     pSessionDesc->cipherSliceType);
1651 
1652 		/* construct cipherConfig in CD in DRAM */
1653 		LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1654 						       pHwBlockBaseInDRAM +
1655 							   hwBlockOffsetInDRAM,
1656 						       &sizeInBytes);
1657 		SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1658 					  pSessionDesc->reqSpcCacheHdr),
1659 				      ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1660 				      laCmdId,
1661 				      pSessionDesc->cmnRequestFlags,
1662 				      pSessionDesc->laCmdFlags);
1663 	} else if ((SPC == pSessionDesc->singlePassState) &&
1664 		   (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1665 		pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
1666 		pSessionDesc->singlePassState = LIKELY_SPC;
1667 		pSessionDesc->isCipher = CPA_TRUE;
1668 		pSessionDesc->isAuthEncryptOp = CPA_TRUE;
1669 		pSessionDesc->isAuth = CPA_TRUE;
1670 		pCdInfo = &(pSessionDesc->contentDescInfo);
1671 		pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
1672 
1673 		if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
1674 		    pSessionDesc->cipherDirection) {
1675 			pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
1676 		} else {
1677 			pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
1678 		}
1679 
1680 		laCmdId = pSessionDesc->laCmdId;
1681 		ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
1682 		    pSessionDesc->laCmdFlags, 0);
1683 		ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
1684 					ICP_QAT_FW_LA_GCM_PROTO);
1685 
1686 		LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
1687 						       pHwBlockBaseInDRAM +
1688 							   hwBlockOffsetInDRAM,
1689 						       &sizeInBytes);
1690 
1691 		SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
1692 					  pSessionDesc->reqCacheHdr),
1693 				      ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
1694 				      laCmdId,
1695 				      pSessionDesc->cmnRequestFlags,
1696 				      pSessionDesc->laCmdFlags);
1697 	}
1698 
1699 	else if (LAC_CIPHER_IS_CHACHA(cipher) &&
1700 		 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
1701 		LAC_INVALID_PARAM_LOG("IV for CHACHA");
1702 		return CPA_STATUS_INVALID_PARAM;
1703 	}
1704 
1705 	if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) {
1706 		if (CPA_CY_SYM_HASH_AES_CCM == hash) {
1707 			status = LacSymAlgChain_CheckCCMData(
1708 			    pOpData->pAdditionalAuthData,
1709 			    pOpData->pIv,
1710 			    pOpData->messageLenToCipherInBytes,
1711 			    pOpData->ivLenInBytes);
1712 			if (CPA_STATUS_SUCCESS == status) {
1713 				LacSymAlgChain_PrepareCCMData(
1714 				    pSessionDesc,
1715 				    pOpData->pAdditionalAuthData,
1716 				    pOpData->pIv,
1717 				    pOpData->messageLenToCipherInBytes,
1718 				    pOpData->ivLenInBytes);
1719 			}
1720 		} else if (CPA_CY_SYM_HASH_AES_GCM == hash) {
1721 			if (aadLenInBytes != 0 &&
1722 			    pOpData->pAdditionalAuthData == NULL) {
1723 				LAC_INVALID_PARAM_LOG("pAdditionalAuthData");
1724 				status = CPA_STATUS_INVALID_PARAM;
1725 			}
1726 			if (CPA_STATUS_SUCCESS == status) {
1727 				LacSymAlgChain_PrepareGCMData(
1728 				    pSessionDesc, pOpData->pAdditionalAuthData);
1729 			}
1730 		}
1731 	}
1732 
1733 	/* allocate cookie (used by callback function) */
1734 	if (CPA_STATUS_SUCCESS == status) {
1735 		pSymCookie = (lac_sym_cookie_t *)Lac_MemPoolEntryAlloc(
1736 		    pService->lac_sym_cookie_pool);
1737 		if (pSymCookie == NULL) {
1738 			LAC_LOG_ERROR("Cannot allocate cookie - NULL");
1739 			status = CPA_STATUS_RESOURCE;
1740 		} else if ((void *)CPA_STATUS_RETRY == pSymCookie) {
1741 			pSymCookie = NULL;
1742 			status = CPA_STATUS_RETRY;
1743 		} else {
1744 			pCookie = &(pSymCookie->u.bulkCookie);
1745 		}
1746 	}
1747 
1748 	if (CPA_STATUS_SUCCESS == status) {
1749 		/* write the buffer descriptors */
1750 		if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) {
1751 			status =
1752 			    LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1753 				(CpaBufferList *)pSrcBuffer,
1754 				&srcAddrPhys,
1755 				CPA_FALSE,
1756 				&(pService->generic_service_info));
1757 			if (CPA_STATUS_SUCCESS != status) {
1758 				LAC_LOG_ERROR(
1759 				    "Unable to write src buffer descriptors");
1760 			}
1761 			/* For out of place operations */
1762 			if ((pSrcBuffer != pDstBuffer) &&
1763 			    (CPA_STATUS_SUCCESS == status)) {
1764 				status =
1765 				    LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
1766 					pDstBuffer,
1767 					&dstAddrPhys,
1768 					CPA_FALSE,
1769 					&(pService->generic_service_info));
1770 				if (CPA_STATUS_SUCCESS != status) {
1771 					LAC_LOG_ERROR(
1772 					    "Unable to write dest buffer descriptors");
1773 				}
1774 			}
1775 		} else {
1776 			status = LacBuffDesc_BufferListDescWrite(
1777 			    (CpaBufferList *)pSrcBuffer,
1778 			    &srcAddrPhys,
1779 			    CPA_FALSE,
1780 			    &(pService->generic_service_info));
1781 			if (CPA_STATUS_SUCCESS != status) {
1782 				LAC_LOG_ERROR(
1783 				    "Unable to write src buffer descriptors in "
1784 				    "LacBuffDesc_BufferListDescWrite");
1785 			}
1786 			/* For out of place operations */
1787 			if ((pSrcBuffer != pDstBuffer) &&
1788 			    (CPA_STATUS_SUCCESS == status)) {
1789 				status = LacBuffDesc_BufferListDescWrite(
1790 				    pDstBuffer,
1791 				    &dstAddrPhys,
1792 				    CPA_FALSE,
1793 				    &(pService->generic_service_info));
1794 				if (CPA_STATUS_SUCCESS != status) {
1795 					LAC_LOG_ERROR(
1796 					    "Unable to write dest buffer descriptors in "
1797 					    "LacBuffDesc_BufferListDescWrite");
1798 				}
1799 			}
1800 		}
1801 	}
1802 	if (CPA_STATUS_SUCCESS == status) {
1803 		/* populate the cookie */
1804 		pCookie->pCallbackTag = pCallbackTag;
1805 		pCookie->sessionCtx = pOpData->sessionCtx;
1806 		pCookie->pOpData = (const CpaCySymOpData *)pOpData;
1807 		pCookie->pDstBuffer = pDstBuffer;
1808 		pCookie->updateSessionIvOnSend = CPA_FALSE;
1809 		pCookie->updateUserIvOnRecieve = CPA_FALSE;
1810 		pCookie->updateKeySizeOnRecieve = CPA_FALSE;
1811 		pCookie->pNext = NULL;
1812 		pCookie->instanceHandle = pService;
1813 
1814 		/* get the qat packet type for LAC packet type */
1815 		LacSymQat_packetTypeGet(pOpData->packetType,
1816 					pSessionDesc->partialState,
1817 					&qatPacketType);
1818 		/*
1819 		 * For XTS mode, the key size must be updated after
1820 		 * the first partial has been sent. Set a flag here so the
1821 		 * response knows to do this.
1822 		 */
1823 		if (LAC_CIPHER_IS_XTS_MODE(cipher) &&
1824 		    (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
1825 		    (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) &&
1826 		    (qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) {
1827 			pCookie->updateKeySizeOnRecieve = CPA_TRUE;
1828 		}
1829 
1830 		/*
1831 		 * Now create the Request.
1832 		 * Start by populating it from the cache in the session
1833 		 * descriptor.
1834 		 */
1835 		pMsg = &(pCookie->qatMsg);
1836 		pMsgDummy = (Cpa8U *)pMsg;
1837 
1838 		if (SPC == pSessionDesc->singlePassState) {
1839 			pCacheDummyHdr =
1840 			    (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr);
1841 			pCacheDummyFtr =
1842 			    (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr);
1843 		} else {
1844 			/* Normally, we want to use the SHRAM Constants Table if
1845 			 * possible for best performance (less DRAM accesses
1846 			 * incurred by CPM).  But we can't use it for
1847 			 * partial-packet hash operations.  This is why we build
1848 			 * 2 versions of the message template at sessionInit,
1849 			 * one for SHRAM Constants Table usage and the other
1850 			 * (default) for Content Descriptor h/w setup data in
1851 			 * DRAM.  And we chose between them here on a
1852 			 * per-request basis, when we know the packetType
1853 			 */
1854 			if ((!pSessionDesc->useSymConstantsTable) ||
1855 			    (pSessionDesc->isAuth &&
1856 			     (CPA_CY_SYM_PACKET_TYPE_FULL !=
1857 			      pOpData->packetType))) {
1858 				pCacheDummyHdr =
1859 				    (Cpa8U *)&(pSessionDesc->reqCacheHdr);
1860 				pCacheDummyFtr =
1861 				    (Cpa8U *)&(pSessionDesc->reqCacheFtr);
1862 			} else {
1863 				pCacheDummyHdr =
1864 				    (Cpa8U *)&(pSessionDesc->shramReqCacheHdr);
1865 				pCacheDummyFtr =
1866 				    (Cpa8U *)&(pSessionDesc->shramReqCacheFtr);
1867 			}
1868 		}
1869 		memcpy(pMsgDummy,
1870 		       pCacheDummyHdr,
1871 		       (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW));
1872 		memset((pMsgDummy +
1873 			(LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)),
1874 		       0,
1875 		       (LAC_LONG_WORD_IN_BYTES *
1876 			LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW));
1877 		memcpy(pMsgDummy +
1878 			   (LAC_LONG_WORD_IN_BYTES *
1879 			    LAC_START_OF_CACHE_FTR_IN_LW),
1880 		       pCacheDummyFtr,
1881 		       (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW));
1882 		/*
1883 		 * Populate the comn_mid section
1884 		 */
1885 		SalQatMsg_CmnMidWrite(pMsg,
1886 				      pCookie,
1887 				      LAC_SYM_DEFAULT_QAT_PTR_TYPE,
1888 				      srcAddrPhys,
1889 				      dstAddrPhys,
1890 				      0,
1891 				      0);
1892 
1893 		/*
1894 		 * Populate the serv_specif_flags field of the Request header
1895 		 * Some of the flags are set up here.
1896 		 * Others are set up later when the RequestParams are set up.
1897 		 */
1898 
1899 		LacSymQat_LaPacketCommandFlagSet(
1900 		    qatPacketType,
1901 		    laCmdId,
1902 		    cipher,
1903 		    &pMsg->comn_hdr.serv_specif_flags,
1904 		    pOpData->ivLenInBytes);
1905 
1906 		if (SPC == pSessionDesc->singlePassState) {
1907 			ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1908 			    pMsg->comn_hdr.serv_specif_flags,
1909 			    ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS);
1910 
1911 			if (CPA_CY_SYM_PACKET_TYPE_PARTIAL ==
1912 			    pOpData->packetType) {
1913 				ICP_QAT_FW_LA_RET_AUTH_SET(
1914 				    pMsg->comn_hdr.serv_specif_flags,
1915 				    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
1916 
1917 				ICP_QAT_FW_LA_CMP_AUTH_SET(
1918 				    pMsg->comn_hdr.serv_specif_flags,
1919 				    ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
1920 			}
1921 		}
1922 
1923 		ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags,
1924 					     pSessionDesc->cipherSliceType);
1925 
1926 		LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize);
1927 		LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize);
1928 
1929 		/*
1930 		 * Populate the CipherRequestParams section of the Request
1931 		 */
1932 		if (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) {
1933 
1934 			Cpa8U *pIvBuffer = NULL;
1935 
1936 			status = LacCipher_PerformParamCheck(cipher,
1937 							     pOpData,
1938 							     srcPktSize);
1939 			if (CPA_STATUS_SUCCESS != status) {
1940 				/* free the cookie */
1941 				Lac_MemPoolEntryFree(pCookie);
1942 				return status;
1943 			}
1944 
1945 			if (CPA_STATUS_SUCCESS == status) {
1946 				/* align cipher IV */
1947 				status = LacCipher_PerformIvCheck(
1948 				    &(pService->generic_service_info),
1949 				    pCookie,
1950 				    qatPacketType,
1951 				    &pIvBuffer);
1952 			}
1953 			if ((SPC == pSessionDesc->singlePassState) &&
1954 			    ((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) ||
1955 			     (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) {
1956 				/* For SPC stateful cipher state size for mid
1957 				 * and end partial packet is 48 bytes
1958 				 */
1959 				pSpcCdCtrlHdr =
1960 				    (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&(
1961 					pMsg->cd_ctrl);
1962 				pSpcCdCtrlHdr->cipher_state_sz =
1963 				    LAC_BYTES_TO_QUADWORDS(
1964 					LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE);
1965 			}
1966 			/*populate the cipher request parameters */
1967 			if (CPA_STATUS_SUCCESS == status) {
1968 				Cpa64U ivBufferPhysAddr = 0;
1969 
1970 				if (pIvBuffer != NULL) {
1971 					/* User OpData memory being used for IV
1972 					 * buffer */
1973 					/* get the physical address */
1974 					ivBufferPhysAddr =
1975 					    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1976 						pService->generic_service_info,
1977 						pIvBuffer);
1978 					if (0 == ivBufferPhysAddr) {
1979 						LAC_LOG_ERROR(
1980 						    "Unable to get the physical address "
1981 						    "of the IV\n");
1982 						status = CPA_STATUS_FAIL;
1983 					}
1984 				}
1985 
1986 				if (status == CPA_STATUS_SUCCESS) {
1987 					status =
1988 					    LacSymQat_CipherRequestParamsPopulate(
1989 						pSessionDesc,
1990 						pMsg,
1991 						pOpData
1992 						    ->cryptoStartSrcOffsetInBytes,
1993 						pOpData
1994 						    ->messageLenToCipherInBytes,
1995 						ivBufferPhysAddr,
1996 						pIvBuffer);
1997 				}
1998 			}
1999 
2000 			if ((SPC == pSessionDesc->singlePassState) &&
2001 			    CPA_STATUS_SUCCESS == status) {
2002 				Cpa64U aadBufferPhysAddr = 0;
2003 
2004 				/* For CHACHA and AES-GCM there is an AAD buffer
2005 				 * if aadLenInBytes is nonzero In case of
2006 				 * AES-GMAC, AAD buffer passed in the src
2007 				 * buffer.
2008 				 */
2009 				if ((0 != aadLenInBytes &&
2010 				     CPA_CY_SYM_HASH_AES_GMAC != hash) ||
2011 				    isSpCcm) {
2012 					LAC_CHECK_NULL_PARAM(
2013 					    pOpData->pAdditionalAuthData);
2014 					Cpa32U aadDataLen =
2015 					    pSessionDesc->aadLenInBytes;
2016 
2017 					/* In case of AES_CCM, B0 block size and
2018 					 * 2 bytes of AAD len encoding need to
2019 					 * be added to total AAD data len */
2020 					if (isSpCcm)
2021 						aadDataLen +=
2022 						    LAC_CIPHER_CCM_AAD_OFFSET;
2023 
2024 					blockLen =
2025 					    LacSymQat_CipherBlockSizeBytesGet(
2026 						cipher);
2027 					if ((aadDataLen % blockLen) != 0) {
2028 						paddingLen = blockLen -
2029 						    (aadDataLen % blockLen);
2030 						memset(
2031 						    &pOpData
2032 							 ->pAdditionalAuthData
2033 							     [aadDataLen],
2034 						    0,
2035 						    paddingLen);
2036 					}
2037 
2038 					/* User OpData memory being used for AAD
2039 					 * buffer */
2040 					/* get the physical address */
2041 					aadBufferPhysAddr =
2042 					    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2043 						pService->generic_service_info,
2044 						pOpData->pAdditionalAuthData);
2045 					if (0 == aadBufferPhysAddr) {
2046 						LAC_LOG_ERROR(
2047 						    "Unable to get the physical address "
2048 						    "of the aad\n");
2049 						status = CPA_STATUS_FAIL;
2050 					}
2051 				}
2052 
2053 				if (CPA_STATUS_SUCCESS == status) {
2054 					icp_qat_fw_la_cipher_req_params_t *pCipherReqParams =
2055 					    (icp_qat_fw_la_cipher_req_params_t
2056 						 *)((Cpa8U *)&(
2057 							pMsg->serv_specif_rqpars) +
2058 						    ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2059 
2060 					icp_qat_fw_la_cipher_20_req_params_t
2061 					    *pCipher20ReqParams =
2062 						(void
2063 						     *)((Cpa8U *)&(
2064 							    pMsg->serv_specif_rqpars) +
2065 							ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
2066 
2067 					if (isCyGen4x(pService)) {
2068 						pCipher20ReqParams
2069 						    ->spc_aad_addr =
2070 						    aadBufferPhysAddr;
2071 						pCipher20ReqParams->spc_aad_sz =
2072 						    pSessionDesc->aadLenInBytes;
2073 						pCipher20ReqParams
2074 						    ->spc_aad_offset = 0;
2075 						if (isSpCcm)
2076 							pCipher20ReqParams
2077 							    ->spc_aad_sz +=
2078 							    LAC_CIPHER_CCM_AAD_OFFSET;
2079 					} else {
2080 						pCipherReqParams->spc_aad_addr =
2081 						    aadBufferPhysAddr;
2082 						pCipherReqParams->spc_aad_sz =
2083 						    (Cpa16U)pSessionDesc
2084 							->aadLenInBytes;
2085 					}
2086 
2087 					if (CPA_TRUE !=
2088 					    pSessionDesc->digestIsAppended) {
2089 						Cpa64U digestBufferPhysAddr = 0;
2090 						/* User OpData memory being used
2091 						 * for digest buffer */
2092 						/* get the physical address */
2093 						digestBufferPhysAddr =
2094 						    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2095 							pService
2096 							    ->generic_service_info,
2097 							pOpData->pDigestResult);
2098 						if (0 != digestBufferPhysAddr) {
2099 							if (isCyGen4x(
2100 								pService)) {
2101 								pCipher20ReqParams
2102 								    ->spc_auth_res_addr =
2103 								    digestBufferPhysAddr;
2104 								pCipher20ReqParams
2105 								    ->spc_auth_res_sz =
2106 								    (Cpa8U)pSessionDesc
2107 									->hashResultSize;
2108 							} else {
2109 								pCipherReqParams
2110 								    ->spc_auth_res_addr =
2111 								    digestBufferPhysAddr;
2112 								pCipherReqParams
2113 								    ->spc_auth_res_sz =
2114 								    (Cpa8U)pSessionDesc
2115 									->hashResultSize;
2116 							}
2117 						} else {
2118 							LAC_LOG_ERROR(
2119 							    "Unable to get the physical address "
2120 							    "of the digest\n");
2121 							status =
2122 							    CPA_STATUS_FAIL;
2123 						}
2124 					} else {
2125 						/* Check if the dest buffer can
2126 						 * handle the digest, only for
2127 						 * last packet */
2128 						if (((ICP_QAT_FW_LA_PARTIAL_NONE ==
2129 						      qatPacketType) ||
2130 						     (ICP_QAT_FW_LA_PARTIAL_END ==
2131 						      qatPacketType))) {
2132 							if (dstPktSize <
2133 							    (pOpData
2134 								 ->cryptoStartSrcOffsetInBytes +
2135 							     pOpData
2136 								 ->messageLenToCipherInBytes +
2137 							     pSessionDesc
2138 								 ->hashResultSize))
2139 								status =
2140 								    CPA_STATUS_INVALID_PARAM;
2141 						}
2142 						if (isCyGen4x(pService)) {
2143 							pCipher20ReqParams
2144 							    ->spc_auth_res_sz =
2145 							    (Cpa8U)pSessionDesc
2146 								->hashResultSize;
2147 						} else {
2148 							pCipherReqParams
2149 							    ->spc_auth_res_sz =
2150 							    (Cpa8U)pSessionDesc
2151 								->hashResultSize;
2152 						}
2153 					}
2154 				}
2155 			}
2156 		}
2157 
2158 		/*
2159 		 * Set up HashRequestParams part of Request
2160 		 */
2161 		if ((status == CPA_STATUS_SUCCESS) &&
2162 		    (laCmdId != ICP_QAT_FW_LA_CMD_CIPHER)) {
2163 			Cpa32U authOffsetInBytes =
2164 			    pOpData->hashStartSrcOffsetInBytes;
2165 			Cpa32U authLenInBytes =
2166 			    pOpData->messageLenToHashInBytes;
2167 
2168 			status = LacHash_PerformParamCheck(instanceHandle,
2169 							   pSessionDesc,
2170 							   pOpData,
2171 							   srcPktSize,
2172 							   pVerifyResult);
2173 			if (CPA_STATUS_SUCCESS != status) {
2174 				/* free the cookie */
2175 				Lac_MemPoolEntryFree(pCookie);
2176 				return status;
2177 			}
2178 			if (CPA_STATUS_SUCCESS == status) {
2179 				/* Info structure for CCM/GCM */
2180 				lac_sym_qat_hash_state_buffer_info_t
2181 				    hashStateBufferInfo = { 0 };
2182 				lac_sym_qat_hash_state_buffer_info_t
2183 				    *pHashStateBufferInfo =
2184 					&(pSessionDesc->hashStateBufferInfo);
2185 
2186 				if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
2187 					icp_qat_fw_la_auth_req_params_t *pHashReqParams =
2188 					    (icp_qat_fw_la_auth_req_params_t
2189 						 *)((Cpa8U *)&(
2190 							pMsg->serv_specif_rqpars) +
2191 						    ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
2192 
2193 					hashStateBufferInfo.pData =
2194 					    pOpData->pAdditionalAuthData;
2195 					if (pOpData->pAdditionalAuthData ==
2196 					    NULL) {
2197 						hashStateBufferInfo.pDataPhys =
2198 						    0;
2199 					} else {
2200 						hashStateBufferInfo
2201 						    .pDataPhys = LAC_MEM_CAST_PTR_TO_UINT64(
2202 						    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2203 							pService
2204 							    ->generic_service_info,
2205 							pOpData
2206 							    ->pAdditionalAuthData));
2207 					}
2208 
2209 					hashStateBufferInfo
2210 					    .stateStorageSzQuadWords = 0;
2211 					hashStateBufferInfo
2212 					    .prefixAadSzQuadWords =
2213 					    LAC_BYTES_TO_QUADWORDS(
2214 						pHashReqParams->u2.aad_sz);
2215 
2216 					/* Overwrite hash state buffer info
2217 					 * structure pointer with the one
2218 					 * created for CCM/GCM */
2219 					pHashStateBufferInfo =
2220 					    &hashStateBufferInfo;
2221 
2222 					/* Aad buffer could be null in the GCM
2223 					 * case */
2224 					if (0 ==
2225 						hashStateBufferInfo.pDataPhys &&
2226 					    CPA_CY_SYM_HASH_AES_GCM != hash &&
2227 					    CPA_CY_SYM_HASH_AES_GMAC != hash) {
2228 						LAC_LOG_ERROR(
2229 						    "Unable to get the physical address"
2230 						    "of the AAD\n");
2231 						status = CPA_STATUS_FAIL;
2232 					}
2233 
2234 					/* for CCM/GCM the hash and cipher data
2235 					 * regions are equal */
2236 					authOffsetInBytes =
2237 					    pOpData
2238 						->cryptoStartSrcOffsetInBytes;
2239 
2240 					/* For authenticated encryption,
2241 					 * authentication length is determined
2242 					 * by messageLenToCipherInBytes for
2243 					 * AES-GCM and AES-CCM, and by
2244 					 * messageLenToHashInBytes for AES-GMAC.
2245 					 * You don't see the latter here, as
2246 					 * that is the initial value of
2247 					 * authLenInBytes. */
2248 					if (hash != CPA_CY_SYM_HASH_AES_GMAC)
2249 						authLenInBytes =
2250 						    pOpData
2251 							->messageLenToCipherInBytes;
2252 				} else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
2253 					       hash ||
2254 					   CPA_CY_SYM_HASH_ZUC_EIA3 == hash) {
2255 					hashStateBufferInfo.pData =
2256 					    pOpData->pAdditionalAuthData;
2257 					hashStateBufferInfo.pDataPhys =
2258 					    LAC_OS_VIRT_TO_PHYS_EXTERNAL(
2259 						pService->generic_service_info,
2260 						hashStateBufferInfo.pData);
2261 					hashStateBufferInfo
2262 					    .stateStorageSzQuadWords = 0;
2263 					hashStateBufferInfo
2264 					    .prefixAadSzQuadWords =
2265 					    LAC_BYTES_TO_QUADWORDS(
2266 						aadLenInBytes);
2267 
2268 					pHashStateBufferInfo =
2269 					    &hashStateBufferInfo;
2270 
2271 					if (0 ==
2272 					    hashStateBufferInfo.pDataPhys) {
2273 						LAC_LOG_ERROR(
2274 						    "Unable to get the physical address"
2275 						    "of the AAD\n");
2276 						status = CPA_STATUS_FAIL;
2277 					}
2278 				}
2279 				if (CPA_CY_SYM_HASH_AES_CCM == hash) {
2280 					if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
2281 					    pSessionDesc->cipherDirection) {
2282 						/* On a decrypt path pSrcBuffer
2283 						 * is used as this is where
2284 						 * encrypted digest is located.
2285 						 * Firmware uses encrypted
2286 						 * digest for
2287 						 * compare/verification*/
2288 						pBufferList =
2289 						    (CpaBufferList *)pSrcBuffer;
2290 					} else {
2291 						/* On an encrypt path pDstBuffer
2292 						 * is used as this is where
2293 						 * encrypted digest will be
2294 						 * written */
2295 						pBufferList =
2296 						    (CpaBufferList *)pDstBuffer;
2297 					}
2298 					status = LacSymAlgChain_PtrFromOffsetGet(
2299 					    pBufferList,
2300 					    pOpData->cryptoStartSrcOffsetInBytes +
2301 						pOpData
2302 						    ->messageLenToCipherInBytes,
2303 					    &pDigestResult);
2304 					if (CPA_STATUS_SUCCESS != status) {
2305 						LAC_LOG_ERROR(
2306 						    "Cannot set digest pointer within the"
2307 						    " buffer list - offset out of bounds");
2308 					}
2309 				} else {
2310 					pDigestResult = pOpData->pDigestResult;
2311 				}
2312 
2313 				if (CPA_TRUE ==
2314 				    pSessionDesc->useStatefulSha3ContentDesc) {
2315 					LacAlgChain_StatefulSha3_SkipStateLoadFlags(
2316 					    pMsg,
2317 					    qatPacketType,
2318 					    pSessionDesc->qatHashMode);
2319 				}
2320 
2321 				if (CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
2322 				    pSessionDesc->symOperation) {
2323 					/* In alg chaining mode, packets are not
2324 					 * seen as partials for hash operations.
2325 					 * Override to NONE.
2326 					 */
2327 					qatPacketType =
2328 					    ICP_QAT_FW_LA_PARTIAL_NONE;
2329 				}
2330 				digestIsAppended =
2331 				    pSessionDesc->digestIsAppended;
2332 				if (CPA_TRUE == digestIsAppended) {
2333 					/*Check if the destination buffer can
2334 					 * handle the digest if digestIsAppend
2335 					 * is true*/
2336 					if (srcPktSize <
2337 					    (authOffsetInBytes +
2338 					     authLenInBytes +
2339 					     pSessionDesc->hashResultSize)) {
2340 						status =
2341 						    CPA_STATUS_INVALID_PARAM;
2342 					}
2343 				}
2344 				if (CPA_STATUS_SUCCESS == status) {
2345 					/* populate the hash request parameters
2346 					 */
2347 					status =
2348 					    LacSymQat_HashRequestParamsPopulate(
2349 						pMsg,
2350 						authOffsetInBytes,
2351 						authLenInBytes,
2352 						&(pService
2353 						      ->generic_service_info),
2354 						pHashStateBufferInfo,
2355 						qatPacketType,
2356 						pSessionDesc->hashResultSize,
2357 						pSessionDesc->digestVerify,
2358 						digestIsAppended ?
2359 						    NULL :
2360 						    pDigestResult,
2361 						hash,
2362 						NULL);
2363 				}
2364 			}
2365 		}
2366 	}
2367 
2368 	/*
2369 	 * send the message to the QAT
2370 	 */
2371 	if (CPA_STATUS_SUCCESS == status) {
2372 		qatUtilsAtomicInc(&(pSessionDesc->u.pendingCbCount));
2373 
2374 		status = LacSymQueue_RequestSend(instanceHandle,
2375 						 pCookie,
2376 						 pSessionDesc);
2377 
2378 		if (CPA_STATUS_SUCCESS != status) {
2379 			/* Decrease pending callback counter on send fail. */
2380 			qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
2381 		}
2382 	}
2383 	/* Case that will catch all error status's for this function */
2384 	if (CPA_STATUS_SUCCESS != status) {
2385 		/* free the cookie */
2386 		if (NULL != pSymCookie) {
2387 			Lac_MemPoolEntryFree(pSymCookie);
2388 		}
2389 	}
2390 	return status;
2391 }
2392