xref: /freebsd/sys/dev/qat/qat_api/common/ctrl/sal_compression.c (revision 2a58b312b62f908ec92311d1bd8536dbaeb8e55b)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /* $FreeBSD$ */
4 /**
5  *****************************************************************************
6  * @file sal_compression.c
7  *
8  * @ingroup SalCtrl
9  *
10  * @description
11  *    This file contains the sal implementation for compression.
12  *
13  *****************************************************************************/
14 
15 /* QAT-API includes */
16 #include "cpa.h"
17 #include "cpa_dc.h"
18 
19 /* QAT utils includes */
20 #include "qat_utils.h"
21 
22 /* ADF includes */
23 #include "icp_adf_init.h"
24 #include "icp_adf_transport.h"
25 #include "icp_accel_devices.h"
26 #include "icp_adf_cfg.h"
27 #include "icp_adf_accel_mgr.h"
28 #include "icp_adf_poll.h"
29 #include "icp_adf_debug.h"
30 #include "icp_adf_esram.h"
31 #include "icp_qat_hw.h"
32 
33 /* SAL includes */
34 #include "lac_mem.h"
35 #include "lac_common.h"
36 #include "lac_mem_pools.h"
37 #include "sal_statistics.h"
38 #include "lac_list.h"
39 #include "icp_sal_poll.h"
40 #include "sal_types_compression.h"
41 #include "dc_session.h"
42 #include "dc_datapath.h"
43 #include "dc_stats.h"
44 #include "lac_sal.h"
45 #include "lac_sal_ctrl.h"
46 #include "sal_string_parse.h"
47 #include "sal_service_state.h"
48 #include "lac_buffer_desc.h"
49 #include "icp_qat_fw_comp.h"
50 #include "icp_qat_hw_20_comp_defs.h"
51 #include "icp_sal_versions.h"
52 
53 /* C string null terminator size */
54 #define SAL_NULL_TERM_SIZE 1
55 
56 /* Type to access extended features bit fields */
57 typedef struct dc_extended_features_s {
58 	unsigned is_cnv : 1; /* Bit<0> */
59 	unsigned padding : 7;
60 	unsigned is_cnvnr : 1; /* Bit<8> */
61 	unsigned not_used : 23;
62 } dc_extd_ftrs_t;
63 
64 /*
65  * Prints statistics for a compression instance
66  */
67 static int
68 SalCtrl_CompresionDebug(void *private_data, char *data, int size, int offset)
69 {
70 	sal_compression_service_t *pCompressionService =
71 	    (sal_compression_service_t *)private_data;
72 	CpaStatus status = CPA_STATUS_SUCCESS;
73 	CpaDcStats dcStats = { 0 };
74 	Cpa32S len = 0;
75 
76 	status = cpaDcGetStats(pCompressionService, &dcStats);
77 	if (status != CPA_STATUS_SUCCESS) {
78 		QAT_UTILS_LOG("cpaDcGetStats returned error.\n");
79 		return (-1);
80 	}
81 
82 	/* Engine Info */
83 	if (NULL != pCompressionService->debug_file) {
84 		len += snprintf(data + len,
85 				size - len,
86 				SEPARATOR BORDER
87 				" Statistics for Instance %24s | \n" SEPARATOR,
88 				pCompressionService->debug_file->name);
89 	}
90 
91 	/* Perform Info */
92 	len += snprintf(data + len,
93 			size - len,
94 			BORDER " DC comp Requests:               %16llu " BORDER
95 			       "\n" BORDER
96 			       " DC comp Request Errors:         %16llu " BORDER
97 			       "\n" BORDER
98 			       " DC comp Completed:              %16llu " BORDER
99 			       "\n" BORDER
100 			       " DC comp Completed Errors:       %16llu " BORDER
101 			       "\n" SEPARATOR,
102 			(long long unsigned int)dcStats.numCompRequests,
103 			(long long unsigned int)dcStats.numCompRequestsErrors,
104 			(long long unsigned int)dcStats.numCompCompleted,
105 			(long long unsigned int)dcStats.numCompCompletedErrors);
106 
107 	/* Perform Info */
108 	len += snprintf(
109 	    data + len,
110 	    size - len,
111 	    BORDER " DC decomp Requests:             %16llu " BORDER "\n" BORDER
112 		   " DC decomp Request Errors:       %16llu " BORDER "\n" BORDER
113 		   " DC decomp Completed:            %16llu " BORDER "\n" BORDER
114 		   " DC decomp Completed Errors:     %16llu " BORDER
115 		   "\n" SEPARATOR,
116 	    (long long unsigned int)dcStats.numDecompRequests,
117 	    (long long unsigned int)dcStats.numDecompRequestsErrors,
118 	    (long long unsigned int)dcStats.numDecompCompleted,
119 	    (long long unsigned int)dcStats.numDecompCompletedErrors);
120 	return 0;
121 }
122 
123 /* Initialise device specific information needed by compression service */
124 static CpaStatus
125 SalCtrl_CompressionInit_CompData(icp_accel_dev_t *device,
126 				 sal_compression_service_t *pCompService)
127 {
128 	int level = 0;
129 
130 	pCompService->comp_device_data.uniqueCompressionLevels[0] = CPA_FALSE;
131 
132 	switch (device->deviceType) {
133 	case DEVICE_DH895XCC:
134 	case DEVICE_DH895XCCVF:
135 		pCompService->generic_service_info.integrityCrcCheck =
136 		    CPA_FALSE;
137 		pCompService->numInterBuffs =
138 		    DC_QAT_MAX_NUM_INTER_BUFFERS_6COMP_SLICES;
139 		pCompService->comp_device_data.minOutputBuffSize =
140 		    DC_DEST_BUFFER_STA_MIN_SIZE;
141 		pCompService->comp_device_data.oddByteDecompNobFinal = CPA_TRUE;
142 		pCompService->comp_device_data.oddByteDecompInterim = CPA_FALSE;
143 		pCompService->comp_device_data.translatorOverflow = CPA_FALSE;
144 		pCompService->comp_device_data.useDevRam =
145 		    ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
146 		pCompService->comp_device_data.enableDmm =
147 		    ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED;
148 
149 		pCompService->comp_device_data.inflateContextSize =
150 		    DC_INFLATE_CONTEXT_SIZE;
151 		pCompService->comp_device_data.highestHwCompressionDepth =
152 		    ICP_QAT_HW_COMPRESSION_DEPTH_16;
153 
154 		pCompService->comp_device_data.windowSizeMask =
155 		    (1 << DC_8K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
156 		pCompService->comp_device_data.cnvnrSupported = CPA_FALSE;
157 		break;
158 	case DEVICE_C3XXX:
159 	case DEVICE_C3XXXVF:
160 	case DEVICE_200XX:
161 	case DEVICE_200XXVF:
162 		pCompService->generic_service_info.integrityCrcCheck =
163 		    CPA_FALSE;
164 		pCompService->numInterBuffs =
165 		    DC_QAT_MAX_NUM_INTER_BUFFERS_6COMP_SLICES;
166 		pCompService->comp_device_data.oddByteDecompNobFinal =
167 		    CPA_FALSE;
168 		pCompService->comp_device_data.oddByteDecompInterim = CPA_TRUE;
169 		pCompService->comp_device_data.translatorOverflow = CPA_FALSE;
170 		pCompService->comp_device_data.useDevRam =
171 		    ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF;
172 		pCompService->comp_device_data.inflateContextSize =
173 		    DC_INFLATE_EH_CONTEXT_SIZE;
174 		pCompService->comp_device_data.highestHwCompressionDepth =
175 		    ICP_QAT_HW_COMPRESSION_DEPTH_16;
176 		pCompService->comp_device_data.windowSizeMask =
177 		    (1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
178 		pCompService->comp_device_data.minOutputBuffSize =
179 		    DC_DEST_BUFFER_STA_MIN_SIZE;
180 		pCompService->comp_device_data.enableDmm =
181 		    ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
182 
183 		pCompService->comp_device_data.cnvnrSupported = CPA_TRUE;
184 		break;
185 	case DEVICE_C62X:
186 	case DEVICE_C62XVF:
187 		pCompService->generic_service_info.integrityCrcCheck =
188 		    CPA_FALSE;
189 		pCompService->numInterBuffs =
190 		    DC_QAT_MAX_NUM_INTER_BUFFERS_10COMP_SLICES;
191 		pCompService->comp_device_data.oddByteDecompNobFinal =
192 		    CPA_FALSE;
193 		pCompService->comp_device_data.oddByteDecompInterim = CPA_TRUE;
194 		pCompService->comp_device_data.translatorOverflow = CPA_FALSE;
195 		pCompService->comp_device_data.useDevRam =
196 		    ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
197 		pCompService->comp_device_data.inflateContextSize =
198 		    DC_INFLATE_EH_CONTEXT_SIZE;
199 		pCompService->comp_device_data.highestHwCompressionDepth =
200 		    ICP_QAT_HW_COMPRESSION_DEPTH_16;
201 		pCompService->comp_device_data.windowSizeMask =
202 		    (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE |
203 		     1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
204 		pCompService->comp_device_data.minOutputBuffSize =
205 		    DC_DEST_BUFFER_STA_MIN_SIZE;
206 		pCompService->comp_device_data.minOutputBuffSizeDynamic =
207 		    pCompService->comp_device_data.minOutputBuffSize;
208 		pCompService->comp_device_data.enableDmm =
209 		    ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
210 		pCompService->comp_device_data.cnvnrSupported = CPA_TRUE;
211 
212 		for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) {
213 			switch (level) {
214 			case CPA_DC_L1:
215 			case CPA_DC_L2:
216 			case CPA_DC_L3:
217 			case CPA_DC_L4:
218 				pCompService->comp_device_data
219 				    .uniqueCompressionLevels[level] = CPA_TRUE;
220 				break;
221 			default:
222 				pCompService->comp_device_data
223 				    .uniqueCompressionLevels[level] = CPA_FALSE;
224 				break;
225 			}
226 		}
227 		pCompService->comp_device_data.numCompressionLevels =
228 		    DC_NUM_COMPRESSION_LEVELS;
229 		break;
230 	case DEVICE_C4XXX:
231 	case DEVICE_C4XXXVF:
232 		pCompService->generic_service_info.integrityCrcCheck = CPA_TRUE;
233 		pCompService->numInterBuffs =
234 		    DC_QAT_MAX_NUM_INTER_BUFFERS_24COMP_SLICES;
235 		pCompService->comp_device_data.minOutputBuffSize =
236 		    DC_DEST_BUFFER_MIN_SIZE;
237 		pCompService->comp_device_data.oddByteDecompNobFinal = CPA_TRUE;
238 		pCompService->comp_device_data.oddByteDecompInterim = CPA_TRUE;
239 		pCompService->comp_device_data.translatorOverflow = CPA_TRUE;
240 		if (pCompService->generic_service_info.capabilitiesMask &
241 		    ICP_ACCEL_CAPABILITIES_INLINE) {
242 			pCompService->comp_device_data.useDevRam =
243 			    ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF;
244 		} else {
245 			pCompService->comp_device_data.useDevRam =
246 			    ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
247 		}
248 		pCompService->comp_device_data.enableDmm =
249 		    ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
250 		pCompService->comp_device_data.inflateContextSize =
251 		    DC_INFLATE_EH_CONTEXT_SIZE;
252 		pCompService->comp_device_data.highestHwCompressionDepth =
253 		    ICP_QAT_HW_COMPRESSION_DEPTH_128;
254 		pCompService->comp_device_data.windowSizeMask =
255 		    (1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
256 		pCompService->comp_device_data.cnvnrSupported = CPA_TRUE;
257 		break;
258 	case DEVICE_GEN4:
259 		pCompService->generic_service_info.integrityCrcCheck = CPA_TRUE;
260 		pCompService->numInterBuffs = 0;
261 		pCompService->comp_device_data.minOutputBuffSize =
262 		    DC_DEST_BUFFER_STA_MIN_SIZE_GEN4;
263 		pCompService->comp_device_data.minOutputBuffSizeDynamic =
264 		    DC_DEST_BUFFER_DYN_MIN_SIZE_GEN4;
265 		pCompService->comp_device_data.oddByteDecompNobFinal = CPA_TRUE;
266 		pCompService->comp_device_data.oddByteDecompInterim = CPA_FALSE;
267 		pCompService->comp_device_data.translatorOverflow = CPA_TRUE;
268 		pCompService->comp_device_data.useDevRam =
269 		    ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
270 		pCompService->comp_device_data.enableDmm =
271 		    ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
272 
273 		pCompService->comp_device_data.inflateContextSize =
274 		    DC_INFLATE_CONTEXT_SIZE;
275 		pCompService->comp_device_data.highestHwCompressionDepth =
276 		    ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9;
277 		pCompService->comp_device_data.windowSizeMask =
278 		    (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE |
279 		     1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
280 		for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) {
281 			switch (level) {
282 			case CPA_DC_L1:
283 			case CPA_DC_L6:
284 			case CPA_DC_L9:
285 				pCompService->comp_device_data
286 				    .uniqueCompressionLevels[level] = CPA_TRUE;
287 				break;
288 			default:
289 				pCompService->comp_device_data
290 				    .uniqueCompressionLevels[level] = CPA_FALSE;
291 				break;
292 			}
293 		}
294 		pCompService->comp_device_data.numCompressionLevels =
295 		    DC_NUM_COMPRESSION_LEVELS;
296 		break;
297 	default:
298 		QAT_UTILS_LOG("Unknown device type! - %d.\n",
299 			      device->deviceType);
300 		return CPA_STATUS_FAIL;
301 	}
302 	return CPA_STATUS_SUCCESS;
303 }
304 
305 CpaStatus
306 SalCtrl_CompressionInit(icp_accel_dev_t *device, sal_service_t *service)
307 {
308 	CpaStatus status = CPA_STATUS_SUCCESS;
309 	Cpa32U numCompConcurrentReq = 0;
310 	Cpa32U request_ring_id = 0;
311 	Cpa32U response_ring_id = 0;
312 
313 	char adfGetParam[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
314 	char compMemPool[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
315 	char temp_string[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
316 	char temp_string2[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
317 	char *instance_name = NULL;
318 	sal_statistics_collection_t *pStatsCollection =
319 	    (sal_statistics_collection_t *)device->pQatStats;
320 	icp_resp_deliv_method rx_resp_type = ICP_RESP_TYPE_IRQ;
321 	sal_compression_service_t *pCompressionService =
322 	    (sal_compression_service_t *)service;
323 	Cpa32U msgSize = 0;
324 	char *section = DYN_SEC;
325 
326 	SAL_SERVICE_GOOD_FOR_INIT(pCompressionService);
327 
328 	pCompressionService->generic_service_info.state =
329 	    SAL_SERVICE_STATE_INITIALIZING;
330 
331 	if (CPA_FALSE == pCompressionService->generic_service_info.is_dyn) {
332 		section = icpGetProcessName();
333 	}
334 
335 	if (pStatsCollection == NULL) {
336 		return CPA_STATUS_FAIL;
337 	}
338 
339 	/* Get Config Info: Accel Num, bank Num, packageID,
340 				    coreAffinity, nodeAffinity and response mode
341 	   */
342 
343 	pCompressionService->acceleratorNum = 0;
344 
345 	/* Initialise device specific compression data */
346 	SalCtrl_CompressionInit_CompData(device, pCompressionService);
347 
348 	status = Sal_StringParsing(
349 	    "Dc",
350 	    pCompressionService->generic_service_info.instance,
351 	    "BankNumber",
352 	    temp_string);
353 	LAC_CHECK_STATUS(status);
354 	status =
355 	    icp_adf_cfgGetParamValue(device, section, temp_string, adfGetParam);
356 	if (CPA_STATUS_SUCCESS != status) {
357 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
358 			      temp_string);
359 		return status;
360 	}
361 
362 	pCompressionService->bankNum =
363 	    Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
364 
365 	status = Sal_StringParsing(
366 	    "Dc",
367 	    pCompressionService->generic_service_info.instance,
368 	    "IsPolled",
369 	    temp_string);
370 	LAC_CHECK_STATUS(status);
371 	status =
372 	    icp_adf_cfgGetParamValue(device, section, temp_string, adfGetParam);
373 	if (CPA_STATUS_SUCCESS != status) {
374 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
375 			      temp_string);
376 		return status;
377 	}
378 	pCompressionService->isPolled =
379 	    (Cpa8U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
380 
381 	/* User instances only support poll and epoll mode */
382 	if (SAL_RESP_POLL_CFG_FILE != pCompressionService->isPolled) {
383 		QAT_UTILS_LOG(
384 		    "IsPolled %u is not supported for user instance %s.\n",
385 		    pCompressionService->isPolled,
386 		    temp_string);
387 		return CPA_STATUS_FAIL;
388 	}
389 
390 	if (SAL_RESP_POLL_CFG_FILE == pCompressionService->isPolled) {
391 		rx_resp_type = ICP_RESP_TYPE_POLL;
392 	}
393 
394 	status = icp_adf_cfgGetParamValue(device,
395 					  LAC_CFG_SECTION_GENERAL,
396 					  ADF_DEV_PKG_ID,
397 					  adfGetParam);
398 	if (CPA_STATUS_SUCCESS != status) {
399 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
400 			      ADF_DEV_PKG_ID);
401 		return status;
402 	}
403 	pCompressionService->pkgID =
404 	    (Cpa16U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
405 
406 	status = icp_adf_cfgGetParamValue(device,
407 					  LAC_CFG_SECTION_GENERAL,
408 					  ADF_DEV_NODE_ID,
409 					  adfGetParam);
410 	if (CPA_STATUS_SUCCESS != status) {
411 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
412 			      ADF_DEV_NODE_ID);
413 		return status;
414 	}
415 	pCompressionService->nodeAffinity =
416 	    (Cpa32U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
417 
418 	/* In case of interrupt instance, use the bank affinity set by adf_ctl
419 	 * Otherwise, use the instance affinity for backwards compatibility */
420 	if (SAL_RESP_POLL_CFG_FILE != pCompressionService->isPolled) {
421 		/* Next need to read the [AcceleratorX] section of the config
422 		 * file */
423 		status = Sal_StringParsing("Accelerator",
424 					   pCompressionService->acceleratorNum,
425 					   "",
426 					   temp_string2);
427 		LAC_CHECK_STATUS(status);
428 
429 		status = Sal_StringParsing("Bank",
430 					   pCompressionService->bankNum,
431 					   "CoreAffinity",
432 					   temp_string);
433 		LAC_CHECK_STATUS(status);
434 	} else {
435 		strncpy(temp_string2,
436 			section,
437 			sizeof(temp_string2) - SAL_NULL_TERM_SIZE);
438 		temp_string2[SAL_CFG_MAX_VAL_LEN_IN_BYTES -
439 			     SAL_NULL_TERM_SIZE] = '\0';
440 
441 		status = Sal_StringParsing(
442 		    "Dc",
443 		    pCompressionService->generic_service_info.instance,
444 		    "CoreAffinity",
445 		    temp_string);
446 		LAC_CHECK_STATUS(status);
447 	}
448 
449 	status = icp_adf_cfgGetParamValue(device,
450 					  temp_string2,
451 					  temp_string,
452 					  adfGetParam);
453 	if (CPA_STATUS_SUCCESS != status) {
454 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
455 			      temp_string);
456 		return status;
457 	}
458 	pCompressionService->coreAffinity =
459 	    (Cpa32U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
460 
461 	status = Sal_StringParsing(
462 	    "Dc",
463 	    pCompressionService->generic_service_info.instance,
464 	    "NumConcurrentRequests",
465 	    temp_string);
466 	LAC_CHECK_STATUS(status);
467 	status =
468 	    icp_adf_cfgGetParamValue(device, section, temp_string, adfGetParam);
469 	if (CPA_STATUS_SUCCESS != status) {
470 		QAT_UTILS_LOG("Failed to get %s from configuration.\n",
471 			      temp_string);
472 		return status;
473 	}
474 
475 	numCompConcurrentReq =
476 	    (Cpa32U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
477 	if (validateConcurrRequest(numCompConcurrentReq)) {
478 		QAT_UTILS_LOG(
479 		    "Invalid NumConcurrentRequests, valid values are: {64, 128, 256, ... 32768, 65536}.\n");
480 		return CPA_STATUS_FAIL;
481 	}
482 
483 	/* ADF does not allow us to completely fill the ring for batch requests
484 	 */
485 	pCompressionService->maxNumCompConcurrentReq =
486 	    (numCompConcurrentReq - SAL_BATCH_SUBMIT_FREE_SPACE);
487 
488 	/* 1. Create transport handles */
489 	status = Sal_StringParsing(
490 	    "Dc",
491 	    pCompressionService->generic_service_info.instance,
492 	    "RingTx",
493 	    temp_string);
494 	LAC_CHECK_STATUS(status);
495 
496 	msgSize = LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES;
497 	status = icp_adf_transCreateHandle(
498 	    device,
499 	    ICP_TRANS_TYPE_ETR,
500 	    section,
501 	    pCompressionService->acceleratorNum,
502 	    pCompressionService->bankNum,
503 	    temp_string,
504 	    lac_getRingType(SAL_RING_TYPE_DC),
505 	    NULL,
506 	    ICP_RESP_TYPE_NONE,
507 	    numCompConcurrentReq,
508 	    msgSize,
509 	    (icp_comms_trans_handle *)&(
510 		pCompressionService->trans_handle_compression_tx));
511 	LAC_CHECK_STATUS(status);
512 
513 	if (icp_adf_transGetRingNum(
514 		pCompressionService->trans_handle_compression_tx,
515 		&request_ring_id) != CPA_STATUS_SUCCESS) {
516 		icp_adf_transReleaseHandle(
517 		    pCompressionService->trans_handle_compression_tx);
518 
519 		QAT_UTILS_LOG("Failed to get DC TX ring number.\n");
520 		return CPA_STATUS_FAIL;
521 	}
522 
523 	status = Sal_StringParsing(
524 	    "Dc",
525 	    pCompressionService->generic_service_info.instance,
526 	    "RingRx",
527 	    temp_string);
528 	if (CPA_STATUS_SUCCESS != status) {
529 		icp_adf_transReleaseHandle(
530 		    pCompressionService->trans_handle_compression_tx);
531 		return status;
532 	}
533 
534 	msgSize = LAC_QAT_DC_RESP_SZ_LW * LAC_LONG_WORD_IN_BYTES;
535 	status = icp_adf_transCreateHandle(
536 	    device,
537 	    ICP_TRANS_TYPE_ETR,
538 	    section,
539 	    pCompressionService->acceleratorNum,
540 	    pCompressionService->bankNum,
541 	    temp_string,
542 	    lac_getRingType(SAL_RING_TYPE_NONE),
543 	    (icp_trans_callback)dcCompression_ProcessCallback,
544 	    rx_resp_type,
545 	    numCompConcurrentReq,
546 	    msgSize,
547 	    (icp_comms_trans_handle *)&(
548 		pCompressionService->trans_handle_compression_rx));
549 	if (CPA_STATUS_SUCCESS != status) {
550 		icp_adf_transReleaseHandle(
551 		    pCompressionService->trans_handle_compression_tx);
552 		return status;
553 	}
554 
555 	if (icp_adf_transGetRingNum(
556 		pCompressionService->trans_handle_compression_rx,
557 		&response_ring_id) != CPA_STATUS_SUCCESS) {
558 		icp_adf_transReleaseHandle(
559 		    pCompressionService->trans_handle_compression_tx);
560 
561 		icp_adf_transReleaseHandle(
562 		    pCompressionService->trans_handle_compression_rx);
563 
564 		QAT_UTILS_LOG("Failed to get DC RX ring number.\n");
565 		return CPA_STATUS_FAIL;
566 	}
567 
568 	/* 2. Allocates memory pools */
569 
570 	/* Valid initialisation value for a pool ID */
571 	pCompressionService->compression_mem_pool = LAC_MEM_POOL_INIT_POOL_ID;
572 
573 	status = Sal_StringParsing(
574 	    "Comp",
575 	    pCompressionService->generic_service_info.instance,
576 	    "_MemPool",
577 	    compMemPool);
578 	if (CPA_STATUS_SUCCESS != status) {
579 		icp_adf_transReleaseHandle(
580 		    pCompressionService->trans_handle_compression_tx);
581 
582 		icp_adf_transReleaseHandle(
583 		    pCompressionService->trans_handle_compression_rx);
584 
585 		return status;
586 	}
587 
588 	status = Lac_MemPoolCreate(&pCompressionService->compression_mem_pool,
589 				   compMemPool,
590 				   (numCompConcurrentReq + 1),
591 				   sizeof(dc_compression_cookie_t),
592 				   LAC_64BYTE_ALIGNMENT,
593 				   CPA_FALSE,
594 				   pCompressionService->nodeAffinity);
595 	if (CPA_STATUS_SUCCESS != status) {
596 		icp_adf_transReleaseHandle(
597 		    pCompressionService->trans_handle_compression_tx);
598 
599 		icp_adf_transReleaseHandle(
600 		    pCompressionService->trans_handle_compression_rx);
601 
602 		return status;
603 	}
604 
605 	/* Init compression statistics */
606 	status = dcStatsInit(pCompressionService);
607 	if (CPA_STATUS_SUCCESS != status) {
608 		Lac_MemPoolDestroy(pCompressionService->compression_mem_pool);
609 
610 		icp_adf_transReleaseHandle(
611 		    pCompressionService->trans_handle_compression_tx);
612 
613 		icp_adf_transReleaseHandle(
614 		    pCompressionService->trans_handle_compression_rx);
615 
616 		return status;
617 	}
618 	if (CPA_TRUE == pStatsCollection->bDcStatsEnabled) {
619 		/* Get instance name for stats */
620 		instance_name = LAC_OS_MALLOC(ADF_CFG_MAX_VAL_LEN_IN_BYTES);
621 		if (NULL == instance_name) {
622 			Lac_MemPoolDestroy(
623 			    pCompressionService->compression_mem_pool);
624 
625 			icp_adf_transReleaseHandle(
626 			    pCompressionService->trans_handle_compression_tx);
627 
628 			icp_adf_transReleaseHandle(
629 			    pCompressionService->trans_handle_compression_rx);
630 
631 			return CPA_STATUS_RESOURCE;
632 		}
633 
634 		status = Sal_StringParsing(
635 		    "Dc",
636 		    pCompressionService->generic_service_info.instance,
637 		    "Name",
638 		    temp_string);
639 		if (CPA_STATUS_SUCCESS != status) {
640 			Lac_MemPoolDestroy(
641 			    pCompressionService->compression_mem_pool);
642 
643 			icp_adf_transReleaseHandle(
644 			    pCompressionService->trans_handle_compression_tx);
645 
646 			icp_adf_transReleaseHandle(
647 			    pCompressionService->trans_handle_compression_rx);
648 			LAC_OS_FREE(instance_name);
649 			return status;
650 		}
651 		status = icp_adf_cfgGetParamValue(device,
652 						  section,
653 						  temp_string,
654 						  adfGetParam);
655 		if (CPA_STATUS_SUCCESS != status) {
656 			QAT_UTILS_LOG("Failed to get %s from configuration.\n",
657 				      temp_string);
658 
659 			Lac_MemPoolDestroy(
660 			    pCompressionService->compression_mem_pool);
661 
662 			icp_adf_transReleaseHandle(
663 			    pCompressionService->trans_handle_compression_tx);
664 
665 			icp_adf_transReleaseHandle(
666 			    pCompressionService->trans_handle_compression_rx);
667 			LAC_OS_FREE(instance_name);
668 			return status;
669 		}
670 
671 		snprintf(instance_name,
672 			 ADF_CFG_MAX_VAL_LEN_IN_BYTES,
673 			 "%s",
674 			 adfGetParam);
675 
676 		pCompressionService->debug_file =
677 		    LAC_OS_MALLOC(sizeof(debug_file_info_t));
678 		if (NULL == pCompressionService->debug_file) {
679 			Lac_MemPoolDestroy(
680 			    pCompressionService->compression_mem_pool);
681 
682 			icp_adf_transReleaseHandle(
683 			    pCompressionService->trans_handle_compression_tx);
684 
685 			icp_adf_transReleaseHandle(
686 			    pCompressionService->trans_handle_compression_rx);
687 			LAC_OS_FREE(instance_name);
688 			return CPA_STATUS_RESOURCE;
689 		}
690 
691 		memset(pCompressionService->debug_file,
692 		       0,
693 		       sizeof(debug_file_info_t));
694 		pCompressionService->debug_file->name = instance_name;
695 		pCompressionService->debug_file->seq_read =
696 		    SalCtrl_CompresionDebug;
697 		pCompressionService->debug_file->private_data =
698 		    pCompressionService;
699 		pCompressionService->debug_file->parent =
700 		    pCompressionService->generic_service_info.debug_parent_dir;
701 
702 		status = icp_adf_debugAddFile(device,
703 					      pCompressionService->debug_file);
704 		if (CPA_STATUS_SUCCESS != status) {
705 			Lac_MemPoolDestroy(
706 			    pCompressionService->compression_mem_pool);
707 
708 			icp_adf_transReleaseHandle(
709 			    pCompressionService->trans_handle_compression_tx);
710 
711 			icp_adf_transReleaseHandle(
712 			    pCompressionService->trans_handle_compression_rx);
713 			LAC_OS_FREE(instance_name);
714 			LAC_OS_FREE(pCompressionService->debug_file);
715 			return status;
716 		}
717 	}
718 	pCompressionService->generic_service_info.stats = pStatsCollection;
719 	pCompressionService->generic_service_info.state =
720 	    SAL_SERVICE_STATE_INITIALIZED;
721 
722 	return status;
723 }
724 
725 CpaStatus
726 SalCtrl_CompressionStart(icp_accel_dev_t *device, sal_service_t *service)
727 {
728 	CpaStatus status = CPA_STATUS_SUCCESS;
729 
730 	sal_compression_service_t *pCompressionService =
731 	    (sal_compression_service_t *)service;
732 
733 	if (SAL_SERVICE_STATE_INITIALIZED !=
734 	    pCompressionService->generic_service_info.state) {
735 		QAT_UTILS_LOG("Not in the correct state to call start.\n");
736 		return CPA_STATUS_FAIL;
737 	}
738 	/**************************************************************/
739 	/* Obtain Extended Features. I.e. Compress And Verify         */
740 	/**************************************************************/
741 	pCompressionService->generic_service_info.dcExtendedFeatures =
742 	    device->dcExtendedFeatures;
743 	pCompressionService->generic_service_info.state =
744 	    SAL_SERVICE_STATE_RUNNING;
745 
746 	return status;
747 }
748 
749 CpaStatus
750 SalCtrl_CompressionStop(icp_accel_dev_t *device, sal_service_t *service)
751 {
752 	sal_compression_service_t *pCompressionService =
753 	    (sal_compression_service_t *)service;
754 
755 	if (SAL_SERVICE_STATE_RUNNING !=
756 	    pCompressionService->generic_service_info.state) {
757 		QAT_UTILS_LOG("Not in the correct state to call stop.\n");
758 		return CPA_STATUS_FAIL;
759 	}
760 
761 	if (icp_adf_is_dev_in_reset(device)) {
762 		pCompressionService->generic_service_info.state =
763 		    SAL_SERVICE_STATE_RESTARTING;
764 		return CPA_STATUS_SUCCESS;
765 	}
766 
767 	pCompressionService->generic_service_info.state =
768 	    SAL_SERVICE_STATE_SHUTTING_DOWN;
769 	return CPA_STATUS_RETRY;
770 }
771 
772 CpaStatus
773 SalCtrl_CompressionShutdown(icp_accel_dev_t *device, sal_service_t *service)
774 {
775 	CpaStatus status = CPA_STATUS_SUCCESS;
776 
777 	sal_compression_service_t *pCompressionService =
778 	    (sal_compression_service_t *)service;
779 	sal_statistics_collection_t *pStatsCollection =
780 	    (sal_statistics_collection_t *)device->pQatStats;
781 
782 	if ((SAL_SERVICE_STATE_INITIALIZED !=
783 	     pCompressionService->generic_service_info.state) &&
784 	    (SAL_SERVICE_STATE_SHUTTING_DOWN !=
785 	     pCompressionService->generic_service_info.state) &&
786 	    (SAL_SERVICE_STATE_RESTARTING !=
787 	     pCompressionService->generic_service_info.state)) {
788 		QAT_UTILS_LOG("Not in the correct state to call shutdown.\n");
789 		return CPA_STATUS_FAIL;
790 	}
791 
792 	Lac_MemPoolDestroy(pCompressionService->compression_mem_pool);
793 
794 	status = icp_adf_transReleaseHandle(
795 	    pCompressionService->trans_handle_compression_tx);
796 	LAC_CHECK_STATUS(status);
797 
798 	status = icp_adf_transReleaseHandle(
799 	    pCompressionService->trans_handle_compression_rx);
800 	LAC_CHECK_STATUS(status);
801 
802 	if (CPA_TRUE == pStatsCollection->bDcStatsEnabled) {
803 		/* Clean stats */
804 		if (NULL != pCompressionService->debug_file) {
805 			icp_adf_debugRemoveFile(
806 			    pCompressionService->debug_file);
807 			LAC_OS_FREE(pCompressionService->debug_file->name);
808 			LAC_OS_FREE(pCompressionService->debug_file);
809 			pCompressionService->debug_file = NULL;
810 		}
811 	}
812 	pCompressionService->generic_service_info.stats = NULL;
813 	dcStatsFree(pCompressionService);
814 
815 	if (icp_adf_is_dev_in_reset(device)) {
816 		pCompressionService->generic_service_info.state =
817 		    SAL_SERVICE_STATE_RESTARTING;
818 		return CPA_STATUS_SUCCESS;
819 	}
820 	pCompressionService->generic_service_info.state =
821 	    SAL_SERVICE_STATE_SHUTDOWN;
822 	return status;
823 }
824 
825 CpaStatus
826 cpaDcGetStatusText(const CpaInstanceHandle dcInstance,
827 		   const CpaStatus errStatus,
828 		   Cpa8S *pStatusText)
829 {
830 	CpaStatus status = CPA_STATUS_SUCCESS;
831 
832 	LAC_CHECK_NULL_PARAM(pStatusText);
833 
834 	switch (errStatus) {
835 	case CPA_STATUS_SUCCESS:
836 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_SUCCESS);
837 		break;
838 	case CPA_STATUS_FAIL:
839 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_FAIL);
840 		break;
841 	case CPA_STATUS_RETRY:
842 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_RETRY);
843 		break;
844 	case CPA_STATUS_RESOURCE:
845 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_RESOURCE);
846 		break;
847 	case CPA_STATUS_INVALID_PARAM:
848 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_INVALID_PARAM);
849 		break;
850 	case CPA_STATUS_FATAL:
851 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_FATAL);
852 		break;
853 	case CPA_STATUS_UNSUPPORTED:
854 		LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_UNSUPPORTED);
855 		break;
856 	default:
857 		status = CPA_STATUS_INVALID_PARAM;
858 		break;
859 	}
860 
861 	return status;
862 }
863 
864 CpaStatus
865 cpaDcGetNumIntermediateBuffers(CpaInstanceHandle dcInstance,
866 			       Cpa16U *pNumBuffers)
867 {
868 	CpaInstanceHandle insHandle = NULL;
869 	sal_compression_service_t *pService = NULL;
870 
871 	if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
872 		insHandle = dcGetFirstHandle();
873 	} else {
874 		insHandle = dcInstance;
875 	}
876 
877 	LAC_CHECK_NULL_PARAM(insHandle);
878 	LAC_CHECK_NULL_PARAM(pNumBuffers);
879 
880 	pService = (sal_compression_service_t *)insHandle;
881 	*pNumBuffers = pService->numInterBuffs;
882 
883 	return CPA_STATUS_SUCCESS;
884 }
885 
886 CpaStatus
887 cpaDcStartInstance(CpaInstanceHandle instanceHandle,
888 		   Cpa16U numBuffers,
889 		   CpaBufferList **pIntermediateBufferPtrsArray)
890 {
891 	icp_qat_addr_width_t *pInterBuffPtrsArray = NULL;
892 	icp_qat_addr_width_t pArrayBufferListDescPhyAddr = 0;
893 	icp_qat_addr_width_t bufListDescPhyAddr;
894 	icp_qat_addr_width_t bufListAlignedPhyAddr;
895 	CpaFlatBuffer *pClientCurrFlatBuffer = NULL;
896 	icp_buffer_list_desc_t *pBufferListDesc = NULL;
897 	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
898 	CpaInstanceInfo2 info = { 0 };
899 	icp_accel_dev_t *dev = NULL;
900 	CpaStatus status = CPA_STATUS_SUCCESS;
901 	sal_compression_service_t *pService = NULL;
902 	CpaInstanceHandle insHandle = NULL;
903 	Cpa16U bufferIndex = 0;
904 	Cpa32U numFlatBuffers = 0;
905 	Cpa64U clientListSize = 0;
906 	CpaBufferList *pClientCurrentIntermediateBuffer = NULL;
907 	Cpa32U bufferIndex2 = 0;
908 	CpaBufferList **pTempIntermediateBufferPtrsArray;
909 	Cpa64U lastClientListSize = 0;
910 
911 	if (CPA_INSTANCE_HANDLE_SINGLE == instanceHandle) {
912 		insHandle = dcGetFirstHandle();
913 	} else {
914 		insHandle = instanceHandle;
915 	}
916 	LAC_CHECK_NULL_PARAM(insHandle);
917 
918 	status = cpaDcInstanceGetInfo2(insHandle, &info);
919 	if (CPA_STATUS_SUCCESS != status) {
920 		QAT_UTILS_LOG("Can not get instance info.\n");
921 		return status;
922 	}
923 
924 	dev = icp_adf_getAccelDevByAccelId(info.physInstId.packageId);
925 	if (NULL == dev) {
926 		QAT_UTILS_LOG("Can not find device for the instance\n");
927 		return CPA_STATUS_FAIL;
928 	}
929 
930 	if (NULL == pIntermediateBufferPtrsArray) {
931 		/* Increment dev ref counter and return - DRAM is not used */
932 		icp_qa_dev_get(dev);
933 		return CPA_STATUS_SUCCESS;
934 	}
935 
936 	if (0 == numBuffers) {
937 		/* Increment dev ref counter and return - DRAM is not used */
938 		icp_qa_dev_get(dev);
939 		return CPA_STATUS_SUCCESS;
940 	}
941 
942 	pService = (sal_compression_service_t *)insHandle;
943 
944 	LAC_CHECK_NULL_PARAM(insHandle);
945 
946 	if ((numBuffers > 0) && (NULL == pIntermediateBufferPtrsArray)) {
947 		QAT_UTILS_LOG("Invalid Intermediate Buffers Array pointer\n");
948 		return CPA_STATUS_INVALID_PARAM;
949 	}
950 
951 	/* Check number of intermediate buffers allocated by user */
952 	if ((pService->numInterBuffs != numBuffers)) {
953 		QAT_UTILS_LOG("Invalid number of buffers\n");
954 		return CPA_STATUS_INVALID_PARAM;
955 	}
956 
957 	pTempIntermediateBufferPtrsArray = pIntermediateBufferPtrsArray;
958 	for (bufferIndex = 0; bufferIndex < numBuffers; bufferIndex++) {
959 		if (NULL == *pTempIntermediateBufferPtrsArray) {
960 			QAT_UTILS_LOG(
961 			    "Intermediate Buffer - Invalid Buffer List pointer\n");
962 			return CPA_STATUS_INVALID_PARAM;
963 		}
964 
965 		if (NULL == (*pTempIntermediateBufferPtrsArray)->pBuffers) {
966 			QAT_UTILS_LOG(
967 			    "Intermediate Buffer - Invalid Flat Buffer descriptor pointer\n");
968 			return CPA_STATUS_INVALID_PARAM;
969 		}
970 
971 		if (NULL ==
972 		    (*pTempIntermediateBufferPtrsArray)->pPrivateMetaData) {
973 			QAT_UTILS_LOG(
974 			    "Intermediate Buffer - Invalid Private MetaData descriptor pointer\n");
975 			return CPA_STATUS_INVALID_PARAM;
976 		}
977 
978 		clientListSize = 0;
979 		for (bufferIndex2 = 0; bufferIndex2 <
980 		     (*pTempIntermediateBufferPtrsArray)->numBuffers;
981 		     bufferIndex2++) {
982 
983 			if ((0 !=
984 			     (*pTempIntermediateBufferPtrsArray)
985 				 ->pBuffers[bufferIndex2]
986 				 .dataLenInBytes) &&
987 			    NULL ==
988 				(*pTempIntermediateBufferPtrsArray)
989 				    ->pBuffers[bufferIndex2]
990 				    .pData) {
991 				QAT_UTILS_LOG(
992 				    "Intermediate Buffer - Invalid Flat Buffer pointer\n");
993 				return CPA_STATUS_INVALID_PARAM;
994 			}
995 
996 			clientListSize += (*pTempIntermediateBufferPtrsArray)
997 					      ->pBuffers[bufferIndex2]
998 					      .dataLenInBytes;
999 		}
1000 
1001 		if (bufferIndex != 0) {
1002 			if (lastClientListSize != clientListSize) {
1003 				QAT_UTILS_LOG(
1004 				    "SGLs have to be of the same size.\n");
1005 				return CPA_STATUS_INVALID_PARAM;
1006 			}
1007 		} else {
1008 			lastClientListSize = clientListSize;
1009 		}
1010 		pTempIntermediateBufferPtrsArray++;
1011 	}
1012 
1013 	/* Allocate array of physical pointers to icp_buffer_list_desc_t */
1014 	status = LAC_OS_CAMALLOC(&pInterBuffPtrsArray,
1015 				 (numBuffers * sizeof(icp_qat_addr_width_t)),
1016 				 LAC_64BYTE_ALIGNMENT,
1017 				 pService->nodeAffinity);
1018 	if (CPA_STATUS_SUCCESS != status) {
1019 		QAT_UTILS_LOG("Can not allocate Intermediate Buffers array.\n");
1020 		return status;
1021 	}
1022 
1023 	/* Get physical address of the intermediate buffer pointers array */
1024 	pArrayBufferListDescPhyAddr = LAC_MEM_CAST_PTR_TO_UINT64(
1025 	    LAC_OS_VIRT_TO_PHYS_INTERNAL(pInterBuffPtrsArray));
1026 
1027 	pService->pInterBuffPtrsArray = pInterBuffPtrsArray;
1028 	pService->pInterBuffPtrsArrayPhyAddr = pArrayBufferListDescPhyAddr;
1029 
1030 	/* Get the full size of the buffer list */
1031 	/* Assumption: all the SGLs allocated by the user have the same size */
1032 	clientListSize = 0;
1033 	for (bufferIndex = 0;
1034 	     bufferIndex < (*pIntermediateBufferPtrsArray)->numBuffers;
1035 	     bufferIndex++) {
1036 		clientListSize += ((*pIntermediateBufferPtrsArray)
1037 				       ->pBuffers[bufferIndex]
1038 				       .dataLenInBytes);
1039 	}
1040 	pService->minInterBuffSizeInBytes = clientListSize;
1041 
1042 	for (bufferIndex = 0; bufferIndex < numBuffers; bufferIndex++) {
1043 
1044 		/* Get pointer to the client Intermediate Buffer List
1045 		 * (CpaBufferList) */
1046 		pClientCurrentIntermediateBuffer =
1047 		    *pIntermediateBufferPtrsArray;
1048 
1049 		/* Get number of flat buffers in the buffer list */
1050 		numFlatBuffers = pClientCurrentIntermediateBuffer->numBuffers;
1051 
1052 		/* Get pointer to the client array of CpaFlatBuffers */
1053 		pClientCurrFlatBuffer =
1054 		    pClientCurrentIntermediateBuffer->pBuffers;
1055 
1056 		/* Calculate Physical address of current private SGL */
1057 		bufListDescPhyAddr = LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1058 		    (*pService),
1059 		    pClientCurrentIntermediateBuffer->pPrivateMetaData);
1060 		if (bufListDescPhyAddr == 0) {
1061 			QAT_UTILS_LOG(
1062 			    "Unable to get the physical address of the metadata.\n");
1063 			return CPA_STATUS_FAIL;
1064 		}
1065 
1066 		/* Align SGL physical address */
1067 		bufListAlignedPhyAddr =
1068 		    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
1069 					   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
1070 
1071 		/* Set physical address of the Intermediate Buffer SGL in the
1072 		 * SGLs array
1073 		 */
1074 		*pInterBuffPtrsArray =
1075 		    LAC_MEM_CAST_PTR_TO_UINT64(bufListAlignedPhyAddr);
1076 
1077 		/* Calculate (virtual) offset to the buffer list descriptor */
1078 		pBufferListDesc =
1079 		    (icp_buffer_list_desc_t
1080 			 *)((LAC_ARCH_UINT)pClientCurrentIntermediateBuffer
1081 				->pPrivateMetaData +
1082 			    (LAC_ARCH_UINT)(bufListAlignedPhyAddr -
1083 					    bufListDescPhyAddr));
1084 
1085 		/* Set number of flat buffers in the physical Buffer List
1086 		 * descriptor */
1087 		pBufferListDesc->numBuffers = numFlatBuffers;
1088 
1089 		/* Go past the Buffer List descriptor to the list of buffer
1090 		 * descriptors
1091 		 */
1092 		pCurrFlatBufDesc =
1093 		    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
1094 
1095 		/* Loop for each flat buffer in the SGL */
1096 		while (0 != numFlatBuffers) {
1097 			/* Set length of the current flat buffer */
1098 			pCurrFlatBufDesc->dataLenInBytes =
1099 			    pClientCurrFlatBuffer->dataLenInBytes;
1100 
1101 			/* Set physical address of the flat buffer */
1102 			pCurrFlatBufDesc->phyBuffer =
1103 			    LAC_MEM_CAST_PTR_TO_UINT64(
1104 				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
1105 				    (*pService), pClientCurrFlatBuffer->pData));
1106 
1107 			if (pCurrFlatBufDesc->phyBuffer == 0) {
1108 				QAT_UTILS_LOG(
1109 				    "Unable to get the physical address of the flat buffer.\n");
1110 				return CPA_STATUS_FAIL;
1111 			}
1112 
1113 			pCurrFlatBufDesc++;
1114 			pClientCurrFlatBuffer++;
1115 			numFlatBuffers--;
1116 		}
1117 		pIntermediateBufferPtrsArray++;
1118 		pInterBuffPtrsArray++;
1119 	}
1120 
1121 	pService->generic_service_info.isInstanceStarted = CPA_TRUE;
1122 
1123 	/* Increment dev ref counter */
1124 	icp_qa_dev_get(dev);
1125 	return CPA_STATUS_SUCCESS;
1126 }
1127 
1128 CpaStatus
1129 cpaDcStopInstance(CpaInstanceHandle instanceHandle)
1130 {
1131 	CpaInstanceHandle insHandle = NULL;
1132 	CpaInstanceInfo2 info = { 0 };
1133 	icp_accel_dev_t *dev = NULL;
1134 	CpaStatus status = CPA_STATUS_SUCCESS;
1135 	sal_compression_service_t *pService = NULL;
1136 
1137 	if (CPA_INSTANCE_HANDLE_SINGLE == instanceHandle) {
1138 		insHandle = dcGetFirstHandle();
1139 	} else {
1140 		insHandle = instanceHandle;
1141 	}
1142 
1143 	LAC_CHECK_NULL_PARAM(insHandle);
1144 	pService = (sal_compression_service_t *)insHandle;
1145 
1146 	/* Free Intermediate Buffer Pointers Array */
1147 	if (pService->pInterBuffPtrsArray != NULL) {
1148 		LAC_OS_CAFREE(pService->pInterBuffPtrsArray);
1149 		pService->pInterBuffPtrsArray = 0;
1150 	}
1151 
1152 	pService->pInterBuffPtrsArrayPhyAddr = 0;
1153 
1154 	status = cpaDcInstanceGetInfo2(insHandle, &info);
1155 	if (CPA_STATUS_SUCCESS != status) {
1156 		QAT_UTILS_LOG("Can not get instance info.\n");
1157 		return status;
1158 	}
1159 	dev = icp_adf_getAccelDevByAccelId(info.physInstId.packageId);
1160 	if (NULL == dev) {
1161 		QAT_UTILS_LOG("Can not find device for the instance.\n");
1162 		return CPA_STATUS_FAIL;
1163 	}
1164 
1165 	pService->generic_service_info.isInstanceStarted = CPA_FALSE;
1166 
1167 	/* Decrement dev ref counter */
1168 	icp_qa_dev_put(dev);
1169 	return CPA_STATUS_SUCCESS;
1170 }
1171 
1172 CpaStatus
1173 cpaDcGetNumInstances(Cpa16U *pNumInstances)
1174 {
1175 	CpaStatus status = CPA_STATUS_SUCCESS;
1176 	icp_accel_dev_t **pAdfInsts = NULL;
1177 	icp_accel_dev_t *dev_addr = NULL;
1178 	sal_t *base_addr = NULL;
1179 	sal_list_t *list_temp = NULL;
1180 	Cpa16U num_accel_dev = 0;
1181 	Cpa16U num = 0;
1182 	Cpa16U i = 0;
1183 
1184 	LAC_CHECK_NULL_PARAM(pNumInstances);
1185 
1186 	/* Get the number of accel_dev in the system */
1187 	status = icp_amgr_getNumInstances(&num_accel_dev);
1188 	LAC_CHECK_STATUS(status);
1189 
1190 	/* Allocate memory to store addr of accel_devs */
1191 	pAdfInsts =
1192 	    malloc(num_accel_dev * sizeof(icp_accel_dev_t *), M_QAT, M_WAITOK);
1193 	num_accel_dev = 0;
1194 
1195 	/* Get ADF to return accel_devs with dc enabled */
1196 	status = icp_amgr_getAllAccelDevByCapabilities(
1197 	    ICP_ACCEL_CAPABILITIES_COMPRESSION, pAdfInsts, &num_accel_dev);
1198 	if (CPA_STATUS_SUCCESS == status) {
1199 		for (i = 0; i < num_accel_dev; i++) {
1200 			dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
1201 			if (NULL != dev_addr) {
1202 				base_addr = dev_addr->pSalHandle;
1203 				if (NULL != base_addr) {
1204 					list_temp =
1205 					    base_addr->compression_services;
1206 					while (NULL != list_temp) {
1207 						num++;
1208 						list_temp =
1209 						    SalList_next(list_temp);
1210 					}
1211 				}
1212 			}
1213 		}
1214 
1215 		*pNumInstances = num;
1216 	}
1217 
1218 	free(pAdfInsts, M_QAT);
1219 
1220 	return status;
1221 }
1222 
1223 CpaStatus
1224 cpaDcGetInstances(Cpa16U numInstances, CpaInstanceHandle *dcInstances)
1225 {
1226 	CpaStatus status = CPA_STATUS_SUCCESS;
1227 	icp_accel_dev_t **pAdfInsts = NULL;
1228 	icp_accel_dev_t *dev_addr = NULL;
1229 	sal_t *base_addr = NULL;
1230 	sal_list_t *list_temp = NULL;
1231 	Cpa16U num_accel_dev = 0;
1232 	Cpa16U index = 0;
1233 	Cpa16U i = 0;
1234 
1235 	LAC_CHECK_NULL_PARAM(dcInstances);
1236 	if (0 == numInstances) {
1237 		QAT_UTILS_LOG("numInstances is 0.\n");
1238 		return CPA_STATUS_INVALID_PARAM;
1239 	}
1240 
1241 	/* Get the number of accel_dev in the system */
1242 	status = icp_amgr_getNumInstances(&num_accel_dev);
1243 	LAC_CHECK_STATUS(status);
1244 
1245 	/* Allocate memory to store addr of accel_devs */
1246 	pAdfInsts =
1247 	    malloc(num_accel_dev * sizeof(icp_accel_dev_t *), M_QAT, M_WAITOK);
1248 
1249 	num_accel_dev = 0;
1250 	/* Get ADF to return accel_devs with dc enabled */
1251 	status = icp_amgr_getAllAccelDevByCapabilities(
1252 	    ICP_ACCEL_CAPABILITIES_COMPRESSION, pAdfInsts, &num_accel_dev);
1253 
1254 	if (CPA_STATUS_SUCCESS == status) {
1255 		/* First check the number of instances in the system */
1256 		for (i = 0; i < num_accel_dev; i++) {
1257 			dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
1258 			if (NULL != dev_addr) {
1259 				base_addr = dev_addr->pSalHandle;
1260 				if (NULL != base_addr) {
1261 					list_temp =
1262 					    base_addr->compression_services;
1263 					while (NULL != list_temp) {
1264 						if (index >
1265 						    (numInstances - 1)) {
1266 							break;
1267 						}
1268 
1269 						dcInstances[index] =
1270 						    SalList_getObject(
1271 							list_temp);
1272 						list_temp =
1273 						    SalList_next(list_temp);
1274 						index++;
1275 					}
1276 				}
1277 			}
1278 		}
1279 
1280 		if (numInstances > index) {
1281 			QAT_UTILS_LOG("Only %d dc instances available.\n",
1282 				      index);
1283 			status = CPA_STATUS_RESOURCE;
1284 		}
1285 	}
1286 
1287 	if (CPA_STATUS_SUCCESS == status) {
1288 		index = 0;
1289 		for (i = 0; i < num_accel_dev; i++) {
1290 			dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
1291 			/* Note dev_addr cannot be NULL here as numInstances=0
1292 			   is not valid and if dev_addr=NULL then index=0 (which
1293 			   is less than numInstances and status is set to
1294 			   _RESOURCE
1295 			   above */
1296 			base_addr = dev_addr->pSalHandle;
1297 			if (NULL != base_addr) {
1298 				list_temp = base_addr->compression_services;
1299 				while (NULL != list_temp) {
1300 					if (index > (numInstances - 1)) {
1301 						break;
1302 					}
1303 
1304 					dcInstances[index] =
1305 					    SalList_getObject(list_temp);
1306 					list_temp = SalList_next(list_temp);
1307 					index++;
1308 				}
1309 			}
1310 		}
1311 	}
1312 
1313 	free(pAdfInsts, M_QAT);
1314 
1315 	return status;
1316 }
1317 
1318 CpaStatus
1319 cpaDcInstanceGetInfo2(const CpaInstanceHandle instanceHandle,
1320 		      CpaInstanceInfo2 *pInstanceInfo2)
1321 {
1322 	sal_compression_service_t *pCompressionService = NULL;
1323 	CpaInstanceHandle insHandle = NULL;
1324 	icp_accel_dev_t *dev = NULL;
1325 	CpaStatus status = CPA_STATUS_SUCCESS;
1326 	char keyStr[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 };
1327 	char valStr[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
1328 	char *section = DYN_SEC;
1329 
1330 	if (CPA_INSTANCE_HANDLE_SINGLE == instanceHandle) {
1331 		insHandle = dcGetFirstHandle();
1332 	} else {
1333 		insHandle = instanceHandle;
1334 	}
1335 
1336 	LAC_CHECK_NULL_PARAM(insHandle);
1337 	SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
1338 	LAC_CHECK_NULL_PARAM(pInstanceInfo2);
1339 
1340 	LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2));
1341 	pInstanceInfo2->accelerationServiceType =
1342 	    CPA_ACC_SVC_TYPE_DATA_COMPRESSION;
1343 
1344 	snprintf((char *)pInstanceInfo2->vendorName,
1345 		 CPA_INST_VENDOR_NAME_SIZE,
1346 		 "%s",
1347 		 SAL_INFO2_VENDOR_NAME);
1348 	pInstanceInfo2->vendorName[CPA_INST_VENDOR_NAME_SIZE - 1] = '\0';
1349 
1350 	snprintf((char *)pInstanceInfo2->swVersion,
1351 		 CPA_INST_SW_VERSION_SIZE,
1352 		 "Version %d.%d",
1353 		 SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER,
1354 		 SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER);
1355 	pInstanceInfo2->swVersion[CPA_INST_SW_VERSION_SIZE - 1] = '\0';
1356 
1357 	/* Note we can safely read the contents of the compression service
1358 	   instance
1359 	   here because icp_amgr_getAccelDevByCapabilities() only returns devs
1360 	   that have started */
1361 	pCompressionService = (sal_compression_service_t *)insHandle;
1362 	pInstanceInfo2->physInstId.packageId = pCompressionService->pkgID;
1363 	pInstanceInfo2->physInstId.acceleratorId =
1364 	    pCompressionService->acceleratorNum;
1365 	pInstanceInfo2->physInstId.executionEngineId = 0;
1366 	pInstanceInfo2->physInstId.busAddress =
1367 	    icp_adf_get_busAddress(pInstanceInfo2->physInstId.packageId);
1368 
1369 	/* set coreAffinity to zero before use */
1370 	LAC_OS_BZERO(pInstanceInfo2->coreAffinity,
1371 		     sizeof(pInstanceInfo2->coreAffinity));
1372 	CPA_BITMAP_BIT_SET(pInstanceInfo2->coreAffinity,
1373 			   pCompressionService->coreAffinity);
1374 
1375 	pInstanceInfo2->nodeAffinity = pCompressionService->nodeAffinity;
1376 
1377 	if (CPA_TRUE ==
1378 	    pCompressionService->generic_service_info.isInstanceStarted) {
1379 		pInstanceInfo2->operState = CPA_OPER_STATE_UP;
1380 	} else {
1381 		pInstanceInfo2->operState = CPA_OPER_STATE_DOWN;
1382 	}
1383 
1384 	pInstanceInfo2->requiresPhysicallyContiguousMemory = CPA_TRUE;
1385 
1386 	if (SAL_RESP_POLL_CFG_FILE == pCompressionService->isPolled) {
1387 		pInstanceInfo2->isPolled = CPA_TRUE;
1388 	} else {
1389 		pInstanceInfo2->isPolled = CPA_FALSE;
1390 	}
1391 
1392 	pInstanceInfo2->isOffloaded = CPA_TRUE;
1393 	/* Get the instance name and part name from the config file */
1394 	dev = icp_adf_getAccelDevByAccelId(pCompressionService->pkgID);
1395 	if (NULL == dev) {
1396 		QAT_UTILS_LOG("Can not find device for the instance.\n");
1397 		LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2));
1398 		return CPA_STATUS_FAIL;
1399 	}
1400 	snprintf((char *)pInstanceInfo2->partName,
1401 		 CPA_INST_PART_NAME_SIZE,
1402 		 SAL_INFO2_PART_NAME,
1403 		 dev->deviceName);
1404 	pInstanceInfo2->partName[CPA_INST_PART_NAME_SIZE - 1] = '\0';
1405 
1406 	if (CPA_FALSE == pCompressionService->generic_service_info.is_dyn) {
1407 		section = icpGetProcessName();
1408 	}
1409 
1410 	status = Sal_StringParsing(
1411 	    "Dc",
1412 	    pCompressionService->generic_service_info.instance,
1413 	    "Name",
1414 	    keyStr);
1415 	LAC_CHECK_STATUS(status);
1416 	status = icp_adf_cfgGetParamValue(dev, section, keyStr, valStr);
1417 	LAC_CHECK_STATUS(status);
1418 	strncpy((char *)pInstanceInfo2->instName,
1419 		valStr,
1420 		sizeof(pInstanceInfo2->instName) - 1);
1421 	pInstanceInfo2->instName[CPA_INST_NAME_SIZE - 1] = '\0';
1422 
1423 #if __GNUC__ >= 7
1424 #pragma GCC diagnostic push
1425 #pragma GCC diagnostic ignored "-Wformat-truncation"
1426 #endif
1427 	snprintf((char *)pInstanceInfo2->instID,
1428 		 CPA_INST_ID_SIZE,
1429 		 "%s_%s",
1430 		 section,
1431 		 valStr);
1432 #if __GNUC__ >= 7
1433 #pragma GCC diagnostic pop
1434 #endif
1435 
1436 	return CPA_STATUS_SUCCESS;
1437 }
1438 
1439 CpaStatus
1440 cpaDcQueryCapabilities(CpaInstanceHandle dcInstance,
1441 		       CpaDcInstanceCapabilities *pInstanceCapabilities)
1442 {
1443 	CpaInstanceHandle insHandle = NULL;
1444 	sal_compression_service_t *pService = NULL;
1445 	Cpa32U capabilitiesMask = 0;
1446 	dc_extd_ftrs_t *pExtendedFtrs = NULL;
1447 
1448 	if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
1449 		insHandle = dcGetFirstHandle();
1450 		if (NULL == insHandle) {
1451 			QAT_UTILS_LOG("Can not get the instance.\n");
1452 			return CPA_STATUS_FAIL;
1453 		}
1454 	} else {
1455 		insHandle = dcInstance;
1456 	}
1457 
1458 	pService = (sal_compression_service_t *)insHandle;
1459 
1460 	LAC_CHECK_NULL_PARAM(insHandle);
1461 	SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
1462 	LAC_CHECK_NULL_PARAM(pInstanceCapabilities);
1463 
1464 	memset(pInstanceCapabilities, 0, sizeof(CpaDcInstanceCapabilities));
1465 
1466 	capabilitiesMask = pService->generic_service_info.capabilitiesMask;
1467 
1468 	/* Set compression capabilities */
1469 	if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY) {
1470 		pInstanceCapabilities->integrityCrcs = CPA_TRUE;
1471 	}
1472 
1473 	pInstanceCapabilities->endOfLastBlock = CPA_TRUE;
1474 	pInstanceCapabilities->statefulDeflateCompression = CPA_FALSE;
1475 	pInstanceCapabilities->statefulDeflateDecompression = CPA_TRUE;
1476 	pInstanceCapabilities->statelessDeflateCompression = CPA_TRUE;
1477 	pInstanceCapabilities->statelessDeflateDecompression = CPA_TRUE;
1478 	pInstanceCapabilities->checksumCRC32 = CPA_TRUE;
1479 	pInstanceCapabilities->checksumAdler32 = CPA_TRUE;
1480 	pInstanceCapabilities->dynamicHuffman = CPA_TRUE;
1481 	pInstanceCapabilities->precompiledHuffman = CPA_FALSE;
1482 	pInstanceCapabilities->dynamicHuffmanBufferReq = CPA_TRUE;
1483 	pInstanceCapabilities->autoSelectBestHuffmanTree = CPA_TRUE;
1484 
1485 	pInstanceCapabilities->validWindowSizeMaskCompression =
1486 	    pService->comp_device_data.windowSizeMask;
1487 	pInstanceCapabilities->validWindowSizeMaskDecompression =
1488 	    pService->comp_device_data.windowSizeMask;
1489 	pExtendedFtrs = (dc_extd_ftrs_t *)&(
1490 	    ((sal_service_t *)insHandle)->dcExtendedFeatures);
1491 	pInstanceCapabilities->batchAndPack = CPA_FALSE;
1492 	pInstanceCapabilities->compressAndVerify =
1493 	    (CpaBoolean)pExtendedFtrs->is_cnv;
1494 	pInstanceCapabilities->compressAndVerifyStrict = CPA_TRUE;
1495 	pInstanceCapabilities->compressAndVerifyAndRecover =
1496 	    (CpaBoolean)pExtendedFtrs->is_cnvnr;
1497 	return CPA_STATUS_SUCCESS;
1498 }
1499 
1500 CpaStatus
1501 cpaDcSetAddressTranslation(const CpaInstanceHandle instanceHandle,
1502 			   CpaVirtualToPhysical virtual2Physical)
1503 {
1504 	sal_service_t *pService = NULL;
1505 	CpaInstanceHandle insHandle = NULL;
1506 
1507 	if (CPA_INSTANCE_HANDLE_SINGLE == instanceHandle) {
1508 		insHandle = dcGetFirstHandle();
1509 	} else {
1510 		insHandle = instanceHandle;
1511 	}
1512 
1513 	LAC_CHECK_NULL_PARAM(insHandle);
1514 	SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
1515 	LAC_CHECK_NULL_PARAM(virtual2Physical);
1516 
1517 	pService = (sal_service_t *)insHandle;
1518 
1519 	pService->virt2PhysClient = virtual2Physical;
1520 
1521 	return CPA_STATUS_SUCCESS;
1522 }
1523 
1524 /**
1525  ******************************************************************************
1526  * @ingroup cpaDcCommon
1527  * Data compression specific polling function which polls a DC instance.
1528  *****************************************************************************/
1529 
1530 CpaStatus
1531 icp_sal_DcPollInstance(CpaInstanceHandle instanceHandle_in,
1532 		       Cpa32U response_quota)
1533 {
1534 	CpaStatus status = CPA_STATUS_SUCCESS;
1535 	sal_compression_service_t *dc_handle = NULL;
1536 	sal_service_t *gen_handle = NULL;
1537 	icp_comms_trans_handle trans_hndTable[DC_NUM_RX_RINGS];
1538 
1539 	if (CPA_INSTANCE_HANDLE_SINGLE == instanceHandle_in) {
1540 		dc_handle = (sal_compression_service_t *)dcGetFirstHandle();
1541 	} else {
1542 		dc_handle = (sal_compression_service_t *)instanceHandle_in;
1543 	}
1544 
1545 	LAC_CHECK_NULL_PARAM(dc_handle);
1546 	SAL_RUNNING_CHECK(dc_handle);
1547 
1548 	gen_handle = &(dc_handle->generic_service_info);
1549 	if (SAL_SERVICE_TYPE_COMPRESSION != gen_handle->type) {
1550 		QAT_UTILS_LOG("Instance handle type is incorrect.\n");
1551 		return CPA_STATUS_FAIL;
1552 	}
1553 
1554 	/*
1555 	 * From the instanceHandle we must get the trans_handle and send
1556 	 * down to adf for polling.
1557 	 * Populate our trans handle table with the appropriate handles.
1558 	 */
1559 	trans_hndTable[0] = dc_handle->trans_handle_compression_rx;
1560 
1561 	/* Call adf to do the polling. */
1562 	status = icp_adf_pollInstance(trans_hndTable,
1563 				      DC_NUM_RX_RINGS,
1564 				      response_quota);
1565 	return status;
1566 }
1567 
1568 /**
1569  ******************************************************************************
1570  * @ingroup cpaDcCommon
1571  *****************************************************************************/
1572 CpaStatus
1573 cpaDcInstanceSetNotificationCb(
1574     const CpaInstanceHandle instanceHandle,
1575     const CpaDcInstanceNotificationCbFunc pInstanceNotificationCb,
1576     void *pCallbackTag)
1577 {
1578 	CpaStatus status = CPA_STATUS_SUCCESS;
1579 	sal_service_t *gen_handle = instanceHandle;
1580 
1581 	LAC_CHECK_NULL_PARAM(gen_handle);
1582 	gen_handle->notification_cb = pInstanceNotificationCb;
1583 	gen_handle->cb_tag = pCallbackTag;
1584 	return status;
1585 }
1586 
1587 CpaInstanceHandle
1588 dcGetFirstHandle(void)
1589 {
1590 	CpaStatus status = CPA_STATUS_SUCCESS;
1591 	static icp_accel_dev_t *adfInsts[ADF_MAX_DEVICES] = { 0 };
1592 	CpaInstanceHandle dcInst = NULL;
1593 	icp_accel_dev_t *dev_addr = NULL;
1594 	sal_t *base_addr = NULL;
1595 	sal_list_t *list_temp = NULL;
1596 	Cpa16U i, num_dc = 0;
1597 
1598 	/* Only need 1 dev with compression enabled - so check all devices */
1599 	status = icp_amgr_getAllAccelDevByCapabilities(
1600 	    ICP_ACCEL_CAPABILITIES_COMPRESSION, adfInsts, &num_dc);
1601 	if ((0 == num_dc) || (CPA_STATUS_SUCCESS != status)) {
1602 		QAT_UTILS_LOG(
1603 		    "No compression devices enabled in the system.\n");
1604 		return dcInst;
1605 	}
1606 
1607 	for (i = 0; i < num_dc; i++) {
1608 		dev_addr = (icp_accel_dev_t *)adfInsts[i];
1609 		if (NULL != dev_addr) {
1610 			base_addr = dev_addr->pSalHandle;
1611 			if (NULL != base_addr) {
1612 				list_temp = base_addr->compression_services;
1613 				if (NULL != list_temp) {
1614 					dcInst = SalList_getObject(list_temp);
1615 					break;
1616 				}
1617 			}
1618 		}
1619 	}
1620 	return dcInst;
1621 }
1622