1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2026 Intel Corporation */
3 #include "adf_cfg.h"
4 #include "cpa.h"
5 #include "icp_accel_devices.h"
6 #include "adf_common_drv.h"
7 #include "icp_adf_accel_mgr.h"
8 #include "icp_adf_cfg.h"
9 #include "icp_adf_debug.h"
10 #include "icp_adf_init.h"
11 #include "lac_sal_ctrl.h"
12
13 static subservice_registation_handle_t *salService = NULL;
14 static struct service_hndl adfService = { 0 };
15 static icp_accel_dev_t *adfDevices = NULL;
16 static icp_accel_dev_t *adfDevicesHead = NULL;
17 struct mtx *adfDevicesLock;
18
19 /*
20 * Need to keep track of what device is currently in reset state
21 */
22 static char accel_dev_reset_stat[ADF_MAX_DEVICES] = { 0 };
23
24 /*
25 * Need to keep track of what device is currently in error state
26 */
27 static char accel_dev_error_stat[ADF_MAX_DEVICES] = { 0 };
28
29 /*
30 * Need to preserve sal handle during restart
31 */
32 static void *accel_dev_sal_hdl_ptr[ADF_MAX_DEVICES] = { 0 };
33
34 static icp_accel_dev_t *
create_adf_dev_structure(struct adf_accel_dev * accel_dev)35 create_adf_dev_structure(struct adf_accel_dev *accel_dev)
36 {
37 icp_accel_dev_t *adf = NULL;
38
39 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
40
41 adf = malloc(sizeof(*adf), M_QAT, M_WAITOK);
42 memset(adf, 0, sizeof(*adf));
43 adf->accelId = accel_dev->accel_id;
44 adf->pAccelName = (char *)hw_data->dev_class->name;
45 adf->deviceType = (device_type_t)hw_data->dev_class->type;
46 strlcpy(adf->deviceName,
47 hw_data->dev_class->name,
48 sizeof(adf->deviceName));
49 adf->accelCapabilitiesMask = hw_data->accel_capabilities_mask;
50 adf->sku = hw_data->get_sku(hw_data);
51 adf->accel_dev = accel_dev;
52 adf->pciDevId = pci_get_device(accel_to_pci_dev(adf->accel_dev));
53 accel_dev->lac_dev = adf;
54
55 return adf;
56 }
57
58 /*
59 * adf_event_handler
60 * Handle device init/uninit/start/stop event
61 */
62 static CpaStatus
adf_event_handler(struct adf_accel_dev * accel_dev,enum adf_event event)63 adf_event_handler(struct adf_accel_dev *accel_dev, enum adf_event event)
64 {
65 CpaStatus status = CPA_STATUS_FAIL;
66 icp_accel_dev_t *adf = NULL;
67
68 if (!adf_cfg_sec_find(accel_dev, ADF_KERNEL_SAL_SEC)) {
69 return CPA_STATUS_SUCCESS;
70 }
71
72 if (event == ADF_EVENT_INIT) {
73 adf = create_adf_dev_structure(accel_dev);
74 if (NULL == adf) {
75 return CPA_STATUS_FAIL;
76 }
77 if (accel_dev_sal_hdl_ptr[accel_dev->accel_id]) {
78 adf->pSalHandle =
79 accel_dev_sal_hdl_ptr[accel_dev->accel_id];
80 accel_dev_sal_hdl_ptr[accel_dev->accel_id] = NULL;
81 }
82
83 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
84 ICP_ADD_ELEMENT_TO_END_OF_LIST(adf, adfDevices, adfDevicesHead);
85 qatUtilsMutexUnlock(&adfDevicesLock);
86 } else {
87 adf = accel_dev->lac_dev;
88 }
89
90 if (event == ADF_EVENT_START) {
91 adf->dcExtendedFeatures =
92 accel_dev->hw_device->extended_dc_capabilities;
93 }
94
95 if (event == ADF_EVENT_RESTARTING) {
96 accel_dev_reset_stat[accel_dev->accel_id] = 1;
97 accel_dev_sal_hdl_ptr[accel_dev->accel_id] = adf->pSalHandle;
98 }
99
100 if (event == ADF_EVENT_RESTARTED) {
101 accel_dev_reset_stat[accel_dev->accel_id] = 0;
102 accel_dev_error_stat[accel_dev->accel_id] = 0;
103 }
104
105 status =
106 salService->subserviceEventHandler(adf,
107 (icp_adf_subsystemEvent_t)event,
108 NULL);
109
110 if (event == ADF_EVENT_ERROR) {
111 accel_dev_error_stat[accel_dev->accel_id] = 1;
112 }
113
114 if ((status == CPA_STATUS_SUCCESS && event == ADF_EVENT_SHUTDOWN) ||
115 (status != CPA_STATUS_SUCCESS && event == ADF_EVENT_INIT)) {
116 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
117 ICP_REMOVE_ELEMENT_FROM_LIST(adf, adfDevices, adfDevicesHead);
118 qatUtilsMutexUnlock(&adfDevicesLock);
119 accel_dev->lac_dev = NULL;
120 free(adf, M_QAT);
121 }
122
123 if (status == CPA_STATUS_SUCCESS && event == ADF_EVENT_START) {
124 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
125 adf->adfSubsystemStatus = 1;
126 qatUtilsMutexUnlock(&adfDevicesLock);
127 }
128
129 if ((status == CPA_STATUS_SUCCESS && event == ADF_EVENT_STOP) ||
130 (status == CPA_STATUS_RETRY && event == ADF_EVENT_STOP)) {
131 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
132 adf->adfSubsystemStatus = 0;
133 qatUtilsMutexUnlock(&adfDevicesLock);
134 status = CPA_STATUS_SUCCESS;
135 }
136
137 return status;
138 }
139
140 /*
141 * icp_adf_subsystemRegister
142 * adapter function from SAL to adf driver
143 * call adf_service_register from adf driver directly with same
144 * parameters
145 */
146 CpaStatus
icp_adf_subsystemRegister(subservice_registation_handle_t * sal_service_reg_handle)147 icp_adf_subsystemRegister(
148 subservice_registation_handle_t *sal_service_reg_handle)
149 {
150 if (salService != NULL)
151 return CPA_STATUS_FAIL;
152
153 salService = sal_service_reg_handle;
154 adfService.name = sal_service_reg_handle->subsystem_name;
155 adfService.event_hld = adf_event_handler;
156
157 if (adf_service_register(&adfService) == 0) {
158 return CPA_STATUS_SUCCESS;
159 } else {
160 salService = NULL;
161 return CPA_STATUS_FAIL;
162 }
163 }
164
165 /*
166 * icp_adf_subsystemUnegister
167 * adapter function from SAL to adf driver
168 */
169 CpaStatus
icp_adf_subsystemUnregister(subservice_registation_handle_t * sal_service_reg_handle)170 icp_adf_subsystemUnregister(
171 subservice_registation_handle_t *sal_service_reg_handle)
172 {
173 if (adf_service_unregister(&adfService) == 0) {
174 salService = NULL;
175 return CPA_STATUS_SUCCESS;
176 } else {
177 return CPA_STATUS_FAIL;
178 }
179 }
180
181 /*
182 * icp_adf_cfgGetParamValue
183 * get parameter value from section @section with key @param
184 */
185 CpaStatus
icp_adf_cfgGetParamValue(icp_accel_dev_t * adf,const char * section,const char * param,char * value)186 icp_adf_cfgGetParamValue(icp_accel_dev_t *adf,
187 const char *section,
188 const char *param,
189 char *value)
190 {
191 if (adf_cfg_get_param_value(adf->accel_dev, section, param, value) ==
192 0) {
193 return CPA_STATUS_SUCCESS;
194 } else {
195 return CPA_STATUS_FAIL;
196 }
197 }
198
199 CpaBoolean
icp_adf_is_dev_in_reset(icp_accel_dev_t * accel_dev)200 icp_adf_is_dev_in_reset(icp_accel_dev_t *accel_dev)
201 {
202 return (CpaBoolean)accel_dev_reset_stat[accel_dev->accelId];
203 }
204
205 CpaStatus
icp_adf_debugAddDir(icp_accel_dev_t * adf,debug_dir_info_t * dir_info)206 icp_adf_debugAddDir(icp_accel_dev_t *adf, debug_dir_info_t *dir_info)
207 {
208 return CPA_STATUS_SUCCESS;
209 }
210
211 void
icp_adf_debugRemoveDir(debug_dir_info_t * dir_info)212 icp_adf_debugRemoveDir(debug_dir_info_t *dir_info)
213 {
214 }
215
216 CpaStatus
icp_adf_debugAddFile(icp_accel_dev_t * adf,debug_file_info_t * file_info)217 icp_adf_debugAddFile(icp_accel_dev_t *adf, debug_file_info_t *file_info)
218 {
219 return CPA_STATUS_SUCCESS;
220 }
221
222 void
icp_adf_debugRemoveFile(debug_file_info_t * file_info)223 icp_adf_debugRemoveFile(debug_file_info_t *file_info)
224 {
225 }
226
227 /*
228 * icp_adf_getAccelDevByAccelId
229 * return acceleration device with id @accelId
230 */
231 icp_accel_dev_t *
icp_adf_getAccelDevByAccelId(Cpa32U accelId)232 icp_adf_getAccelDevByAccelId(Cpa32U accelId)
233 {
234 icp_accel_dev_t *adf = NULL;
235
236 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
237 adf = adfDevicesHead;
238 while (adf != NULL && adf->accelId != accelId)
239 adf = adf->pNext;
240 qatUtilsMutexUnlock(&adfDevicesLock);
241 return adf;
242 }
243
244 /*
245 * icp_amgr_getNumInstances
246 * Return the number of acceleration devices it the system.
247 */
248 CpaStatus
icp_amgr_getNumInstances(Cpa16U * pNumInstances)249 icp_amgr_getNumInstances(Cpa16U *pNumInstances)
250 {
251 icp_accel_dev_t *adf = NULL;
252 Cpa16U count = 0;
253
254 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
255 for (adf = adfDevicesHead; adf != NULL; adf = adf->pNext)
256 count++;
257 qatUtilsMutexUnlock(&adfDevicesLock);
258 *pNumInstances = count;
259 return CPA_STATUS_SUCCESS;
260 }
261
262 /*
263 * icp_amgr_getAccelDevByCapabilities
264 * Returns a started accel device that implements
265 * the capabilities specified in capabilitiesMask.
266 */
267 CpaStatus
icp_amgr_getAccelDevByCapabilities(Cpa32U capabilitiesMask,icp_accel_dev_t ** pAccel_devs,Cpa16U * pNumInstances)268 icp_amgr_getAccelDevByCapabilities(Cpa32U capabilitiesMask,
269 icp_accel_dev_t **pAccel_devs,
270 Cpa16U *pNumInstances)
271 {
272 icp_accel_dev_t *adf = NULL;
273 *pNumInstances = 0;
274
275 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
276 for (adf = adfDevicesHead; adf != NULL; adf = adf->pNext) {
277 if (adf->accelCapabilitiesMask & capabilitiesMask) {
278 if (adf->adfSubsystemStatus) {
279 pAccel_devs[0] = adf;
280 *pNumInstances = 1;
281 qatUtilsMutexUnlock(&adfDevicesLock);
282 return CPA_STATUS_SUCCESS;
283 }
284 }
285 }
286 qatUtilsMutexUnlock(&adfDevicesLock);
287 return CPA_STATUS_FAIL;
288 }
289
290 /*
291 * icp_amgr_getAllAccelDevByEachCapabilities
292 * Returns table of accel devices that are started and that implement
293 * at least one of the capabilities specified in capabilitiesMask.
294 */
295 CpaStatus
icp_amgr_getAllAccelDevByEachCapability(Cpa32U capabilitiesMask,icp_accel_dev_t ** pAccel_devs,Cpa16U * pNumInstances)296 icp_amgr_getAllAccelDevByEachCapability(Cpa32U capabilitiesMask,
297 icp_accel_dev_t **pAccel_devs,
298 Cpa16U *pNumInstances)
299 {
300 icp_accel_dev_t *adf = NULL;
301 *pNumInstances = 0;
302 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
303 for (adf = adfDevicesHead; adf != NULL; adf = adf->pNext) {
304 Cpa32U enabled_caps =
305 adf->accelCapabilitiesMask & capabilitiesMask;
306 if (enabled_caps == capabilitiesMask) {
307 if (adf->adfSubsystemStatus) {
308 pAccel_devs[(*pNumInstances)++] =
309 (icp_accel_dev_t *)adf;
310 }
311 }
312 }
313 qatUtilsMutexUnlock(&adfDevicesLock);
314 return CPA_STATUS_SUCCESS;
315 }
316
317 /*
318 * icp_amgr_getAllAccelDevByCapabilities
319 * Fetches accel devices based on the capability
320 * and returns the count of the device
321 */
322 CpaStatus
icp_amgr_getAllAccelDevByCapabilities(Cpa32U capabilitiesMask,icp_accel_dev_t ** pAccel_devs,Cpa16U * pNumInstances)323 icp_amgr_getAllAccelDevByCapabilities(Cpa32U capabilitiesMask,
324 icp_accel_dev_t **pAccel_devs,
325 Cpa16U *pNumInstances)
326 {
327 icp_accel_dev_t *adf = NULL;
328 Cpa16U i = 0;
329
330 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
331 for (adf = adfDevicesHead; adf != NULL; adf = adf->pNext) {
332 if (adf->accelCapabilitiesMask & capabilitiesMask) {
333 if (adf->adfSubsystemStatus) {
334 pAccel_devs[i++] = adf;
335 }
336 }
337 }
338 qatUtilsMutexUnlock(&adfDevicesLock);
339 *pNumInstances = i;
340 return CPA_STATUS_SUCCESS;
341 }
342
343 /*
344 * icp_amgr_getAccelDevCapabilities
345 * Returns accel devices capabilities specified in capabilitiesMask.
346 *
347 * Returns:
348 * CPA_STATUS_SUCCESS on success
349 * CPA_STATUS_FAIL on failure
350 */
351 CpaStatus
icp_amgr_getAccelDevCapabilities(icp_accel_dev_t * accel_dev,Cpa32U * pCapabilitiesMask)352 icp_amgr_getAccelDevCapabilities(icp_accel_dev_t *accel_dev,
353 Cpa32U *pCapabilitiesMask)
354 {
355 ICP_CHECK_FOR_NULL_PARAM(accel_dev);
356 ICP_CHECK_FOR_NULL_PARAM(pCapabilitiesMask);
357
358 *pCapabilitiesMask = accel_dev->accelCapabilitiesMask;
359 return CPA_STATUS_SUCCESS;
360 }
361
362 /*
363 * icp_qa_dev_get
364 *
365 * Description:
366 * Function increments the device usage counter.
367 *
368 * Returns: void
369 */
370 void
icp_qa_dev_get(icp_accel_dev_t * pDev)371 icp_qa_dev_get(icp_accel_dev_t *pDev)
372 {
373 ICP_CHECK_FOR_NULL_PARAM_VOID(pDev);
374 adf_dev_get(pDev->accel_dev);
375 }
376
377 /*
378 * icp_qa_dev_put
379 *
380 * Description:
381 * Function decrements the device usage counter.
382 *
383 * Returns: void
384 */
385 void
icp_qa_dev_put(icp_accel_dev_t * pDev)386 icp_qa_dev_put(icp_accel_dev_t *pDev)
387 {
388 ICP_CHECK_FOR_NULL_PARAM_VOID(pDev);
389 adf_dev_put(pDev->accel_dev);
390 }
391
392 Cpa16U
icp_adf_get_busAddress(Cpa16U packageId)393 icp_adf_get_busAddress(Cpa16U packageId)
394 {
395 Cpa16U busAddr = 0xFFFF;
396 icp_accel_dev_t *adf = NULL;
397
398 qatUtilsMutexLock(&adfDevicesLock, QAT_UTILS_WAIT_FOREVER);
399 for (adf = adfDevicesHead; adf != NULL; adf = adf->pNext) {
400 if (adf->accelId == packageId) {
401 busAddr = pci_get_bus(accel_to_pci_dev(adf->accel_dev))
402 << 8 |
403 pci_get_slot(accel_to_pci_dev(adf->accel_dev))
404 << 3 |
405 pci_get_function(accel_to_pci_dev(adf->accel_dev));
406 break;
407 }
408 }
409 qatUtilsMutexUnlock(&adfDevicesLock);
410 return busAddr;
411 }
412
413 CpaBoolean
icp_adf_isSubsystemStarted(subservice_registation_handle_t * subsystem_hdl)414 icp_adf_isSubsystemStarted(subservice_registation_handle_t *subsystem_hdl)
415 {
416 if (subsystem_hdl == salService)
417 return CPA_TRUE;
418 else
419 return CPA_FALSE;
420 }
421
422 CpaBoolean
icp_adf_is_dev_in_error(icp_accel_dev_t * accel_dev)423 icp_adf_is_dev_in_error(icp_accel_dev_t *accel_dev)
424 {
425 return (CpaBoolean)accel_dev_error_stat[accel_dev->accelId];
426 }
427