1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 #ifndef ADF_DRV_H
4 #define ADF_DRV_H
5
6 #include <dev/pci/pcivar.h>
7 #include "adf_accel_devices.h"
8 #include "icp_qat_fw_loader_handle.h"
9 #include "icp_qat_hal.h"
10 #include "adf_cfg_user.h"
11 #include "adf_uio.h"
12 #include "adf_uio_control.h"
13
14 #define QAT_UIO_IOC_MAGIC 'b'
15 #define ADF_MAJOR_VERSION 0
16 #define ADF_MINOR_VERSION 6
17 #define ADF_BUILD_VERSION 0
18 #define ADF_DRV_VERSION \
19 __stringify(ADF_MAJOR_VERSION) "." __stringify( \
20 ADF_MINOR_VERSION) "." __stringify(ADF_BUILD_VERSION)
21
22 #define IOCTL_GET_BUNDLE_SIZE _IOR(QAT_UIO_IOC_MAGIC, 0, int32_t)
23 #define IOCTL_ALLOC_BUNDLE _IOW(QAT_UIO_IOC_MAGIC, 1, int)
24 #define IOCTL_GET_ACCEL_TYPE _IOR(QAT_UIO_IOC_MAGIC, 2, uint32_t)
25 #define IOCTL_ADD_MEM_FD _IOW(QAT_UIO_IOC_MAGIC, 3, int)
26 #define ADF_STATUS_RESTARTING 0
27 #define ADF_STATUS_STARTING 1
28 #define ADF_STATUS_CONFIGURED 2
29 #define ADF_STATUS_STARTED 3
30 #define ADF_STATUS_AE_INITIALISED 4
31 #define ADF_STATUS_AE_UCODE_LOADED 5
32 #define ADF_STATUS_AE_STARTED 6
33 #define ADF_STATUS_PF_RUNNING 7
34 #define ADF_STATUS_IRQ_ALLOCATED 8
35 #define ADF_PCIE_FLR_ATTEMPT 10
36 #define ADF_STATUS_SYSCTL_CTX_INITIALISED 9
37
38 #define PCI_EXP_AERUCS 0x104
39
40 /* PMISC BAR upper and lower offsets in PCIe config space */
41 #define ADF_PMISC_L_OFFSET 0x18
42 #define ADF_PMISC_U_OFFSET 0x1c
43
44 enum adf_dev_reset_mode { ADF_DEV_RESET_ASYNC = 0, ADF_DEV_RESET_SYNC };
45
46 enum adf_event {
47 ADF_EVENT_INIT = 0,
48 ADF_EVENT_START,
49 ADF_EVENT_STOP,
50 ADF_EVENT_SHUTDOWN,
51 ADF_EVENT_RESTARTING,
52 ADF_EVENT_RESTARTED,
53 ADF_EVENT_ERROR,
54 };
55
56 struct adf_state {
57 enum adf_event dev_state;
58 int dev_id;
59 };
60
61 struct service_hndl {
62 int (*event_hld)(struct adf_accel_dev *accel_dev, enum adf_event event);
63 unsigned long init_status[ADF_DEVS_ARRAY_SIZE];
64 unsigned long start_status[ADF_DEVS_ARRAY_SIZE];
65 char *name;
66 struct list_head list;
67 };
68
69 static inline int
get_current_node(void)70 get_current_node(void)
71 {
72 return PCPU_GET(domain);
73 }
74
75 int adf_service_register(struct service_hndl *service);
76 int adf_service_unregister(struct service_hndl *service);
77
78 int adf_dev_init(struct adf_accel_dev *accel_dev);
79 int adf_dev_start(struct adf_accel_dev *accel_dev);
80 int adf_dev_stop(struct adf_accel_dev *accel_dev);
81 void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
82 int adf_dev_autoreset(struct adf_accel_dev *accel_dev);
83 int adf_dev_reset(struct adf_accel_dev *accel_dev,
84 enum adf_dev_reset_mode mode);
85 int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev,
86 enum adf_dev_reset_mode mode);
87 void adf_error_notifier(uintptr_t arg);
88 int adf_init_fatal_error_wq(void);
89 void adf_exit_fatal_error_wq(void);
90 int adf_notify_fatal_error(struct adf_accel_dev *accel_dev);
91 void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data);
92 void adf_clean_vf_map(bool);
93 int adf_sysctl_add_fw_versions(struct adf_accel_dev *accel_dev);
94 int adf_sysctl_remove_fw_versions(struct adf_accel_dev *accel_dev);
95
96 int adf_ctl_dev_register(void);
97 void adf_ctl_dev_unregister(void);
98 int adf_register_ctl_device_driver(void);
99 void adf_unregister_ctl_device_driver(void);
100 int adf_processes_dev_register(void);
101 void adf_processes_dev_unregister(void);
102 void adf_state_init(void);
103 void adf_state_destroy(void);
104 int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev,
105 struct adf_accel_dev *pf);
106 void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev,
107 struct adf_accel_dev *pf);
108 struct list_head *adf_devmgr_get_head(void);
109 struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id);
110 struct adf_accel_dev *adf_devmgr_get_first(void);
111 struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(device_t pci_dev);
112 int adf_devmgr_verify_id(uint32_t *id);
113 void adf_devmgr_get_num_dev(uint32_t *num);
114 int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev);
115 int adf_dev_started(struct adf_accel_dev *accel_dev);
116 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev);
117 int adf_dev_restarting_notify_sync(struct adf_accel_dev *accel_dev);
118 int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev);
119 int adf_dev_stop_notify_sync(struct adf_accel_dev *accel_dev);
120 int adf_ae_init(struct adf_accel_dev *accel_dev);
121 int adf_ae_shutdown(struct adf_accel_dev *accel_dev);
122 int adf_ae_fw_load(struct adf_accel_dev *accel_dev);
123 void adf_ae_fw_release(struct adf_accel_dev *accel_dev);
124 int adf_ae_start(struct adf_accel_dev *accel_dev);
125 int adf_ae_stop(struct adf_accel_dev *accel_dev);
126
127 int adf_aer_store_ppaerucm_reg(device_t pdev,
128 struct adf_hw_device_data *hw_data);
129
130 int adf_enable_aer(struct adf_accel_dev *accel_dev, device_t *adf);
131 void adf_disable_aer(struct adf_accel_dev *accel_dev);
132 void adf_reset_sbr(struct adf_accel_dev *accel_dev);
133 void adf_reset_flr(struct adf_accel_dev *accel_dev);
134 void adf_dev_pre_reset(struct adf_accel_dev *accel_dev);
135 void adf_dev_post_reset(struct adf_accel_dev *accel_dev);
136 void adf_dev_restore(struct adf_accel_dev *accel_dev);
137 int adf_init_aer(void);
138 void adf_exit_aer(void);
139 int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev,
140 u32 ae,
141 void *in,
142 void *out);
143 struct icp_qat_fw_init_admin_req;
144 struct icp_qat_fw_init_admin_resp;
145 int adf_send_admin(struct adf_accel_dev *accel_dev,
146 struct icp_qat_fw_init_admin_req *req,
147 struct icp_qat_fw_init_admin_resp *resp,
148 u32 ae_mask);
149 int adf_config_device(struct adf_accel_dev *accel_dev);
150
151 int adf_init_admin_comms(struct adf_accel_dev *accel_dev);
152 void adf_exit_admin_comms(struct adf_accel_dev *accel_dev);
153 int adf_send_admin_init(struct adf_accel_dev *accel_dev);
154 int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp);
155 int adf_get_fw_pke_stats(struct adf_accel_dev *accel_dev,
156 u64 *suc_count,
157 u64 *unsuc_count);
158 int adf_dev_measure_clock(struct adf_accel_dev *accel_dev,
159 u32 *frequency,
160 u32 min,
161 u32 max);
162 int adf_clock_debugfs_add(struct adf_accel_dev *accel_dev);
163 u64 adf_clock_get_current_time(void);
164 int adf_init_arb(struct adf_accel_dev *accel_dev);
165 int adf_init_gen2_arb(struct adf_accel_dev *accel_dev);
166 void adf_exit_arb(struct adf_accel_dev *accel_dev);
167 void adf_disable_arb(struct adf_accel_dev *accel_dev);
168 void adf_update_ring_arb(struct adf_etr_ring_data *ring);
169 void adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
170 void *csr_addr,
171 unsigned int bank_nr,
172 unsigned int mask);
173 void adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
174 void *csr_addr,
175 unsigned int bank_nr,
176 unsigned int mask);
177 int adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
178 void adf_update_uio_ring_arb(struct adf_uio_control_bundle *bundle);
179 struct adf_accel_dev *adf_devmgr_get_dev_by_bdf(struct adf_pci_address *addr);
180 struct adf_accel_dev *adf_devmgr_get_dev_by_pci_bus(u8 bus);
181 int adf_get_vf_nr(struct adf_pci_address *vf_pci_addr, int *vf_nr);
182 u32 adf_get_slices_for_svc(struct adf_accel_dev *accel_dev,
183 enum adf_svc_type svc);
184 bool adf_is_bdf_equal(struct adf_pci_address *bdf1,
185 struct adf_pci_address *bdf2);
186 int adf_is_vf_nr_valid(struct adf_accel_dev *accel_dev, int vf_nr);
187 void adf_dev_get(struct adf_accel_dev *accel_dev);
188 void adf_dev_put(struct adf_accel_dev *accel_dev);
189 int adf_dev_in_use(struct adf_accel_dev *accel_dev);
190 int adf_init_etr_data(struct adf_accel_dev *accel_dev);
191 void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev);
192
193 struct qat_crypto_instance *qat_crypto_get_instance_node(int node);
194 void qat_crypto_put_instance(struct qat_crypto_instance *inst);
195 void qat_alg_callback(void *resp);
196 void qat_alg_asym_callback(void *resp);
197 int qat_algs_register(void);
198 void qat_algs_unregister(void);
199 int qat_asym_algs_register(void);
200 void qat_asym_algs_unregister(void);
201
202 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev);
203 void adf_isr_resource_free(struct adf_accel_dev *accel_dev);
204 int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev);
205 void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
206 int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev);
207 int qat_hal_init(struct adf_accel_dev *accel_dev);
208 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
209 int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
210 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle,
211 unsigned char ae,
212 unsigned int ctx_mask);
213 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle);
214 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle);
215 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
216 unsigned char ae,
217 unsigned int ctx_mask);
218 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
219 unsigned int ae);
220 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
221 unsigned char ae,
222 enum icp_qat_uof_regtype lm_type,
223 unsigned char mode);
224 void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
225 unsigned char ae,
226 unsigned char mode);
227 void qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle *handle,
228 unsigned char ae,
229 unsigned char mode);
230 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
231 unsigned char ae,
232 unsigned char mode);
233 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
234 unsigned char ae,
235 unsigned char mode);
236 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
237 unsigned char ae,
238 unsigned int ctx_mask,
239 unsigned int upc);
240 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
241 unsigned char ae,
242 unsigned int uaddr,
243 unsigned int words_num,
244 const uint64_t *uword);
245 void qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle *handle,
246 unsigned char ae,
247 unsigned int uaddr,
248 unsigned int words_num,
249 uint64_t *uword);
250
251 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
252 unsigned char ae,
253 unsigned int uword_addr,
254 unsigned int words_num,
255 unsigned int *data);
256 int qat_hal_get_ins_num(void);
257 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
258 unsigned char ae,
259 struct icp_qat_uof_batch_init *lm_init_header);
260 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
261 unsigned char ae,
262 unsigned long ctx_mask,
263 enum icp_qat_uof_regtype reg_type,
264 unsigned short reg_num,
265 unsigned int regdata);
266 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
267 unsigned char ae,
268 unsigned long ctx_mask,
269 enum icp_qat_uof_regtype reg_type,
270 unsigned short reg_num,
271 unsigned int regdata);
272 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
273 unsigned char ae,
274 unsigned long ctx_mask,
275 enum icp_qat_uof_regtype reg_type,
276 unsigned short reg_num,
277 unsigned int regdata);
278 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
279 unsigned char ae,
280 unsigned long ctx_mask,
281 unsigned short reg_num,
282 unsigned int regdata);
283 int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle,
284 unsigned char ae,
285 unsigned short lm_addr,
286 unsigned int value);
287 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle);
288 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle);
289 void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle);
290 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
291 const void *addr_ptr,
292 int mem_size);
293 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
294 const void *addr_ptr,
295 u32 mem_size,
296 const char *obj_name);
297
298 void qat_hal_get_scs_neigh_ae(unsigned char ae, unsigned char *ae_neigh);
299 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle,
300 unsigned int cfg_ae_mask);
301 int adf_init_vf_wq(void);
302 void adf_exit_vf_wq(void);
303 void adf_flush_vf_wq(struct adf_accel_dev *accel_dev);
304 int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev);
305 int adf_pf2vf_handle_pf_rp_reset(struct adf_accel_dev *accel_dev,
306 struct pfvf_message msg);
307 bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev);
308 static inline int
adf_sriov_configure(device_t * pdev,int numvfs)309 adf_sriov_configure(device_t *pdev, int numvfs)
310 {
311 return 0;
312 }
313
314 static inline void
adf_disable_sriov(struct adf_accel_dev * accel_dev)315 adf_disable_sriov(struct adf_accel_dev *accel_dev)
316 {
317 }
318
319 static inline void
adf_vf2pf_handler(struct adf_accel_vf_info * vf_info)320 adf_vf2pf_handler(struct adf_accel_vf_info *vf_info)
321 {
322 }
323
324 static inline int
adf_init_pf_wq(void)325 adf_init_pf_wq(void)
326 {
327 return 0;
328 }
329
330 static inline void
adf_exit_pf_wq(void)331 adf_exit_pf_wq(void)
332 {
333 }
334 #endif
335