1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2026 Intel Corporation */
3 #ifndef ADF_ACCEL_DEVICES_H_
4 #define ADF_ACCEL_DEVICES_H_
5
6 #include "qat_freebsd.h"
7 #include "adf_cfg_common.h"
8 #include "adf_pfvf_msg.h"
9
10 #include "opt_qat.h"
11
12 #define ADF_CFG_NUM_SERVICES 4
13
14 #define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
15 #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf"
16 #define ADF_C62X_DEVICE_NAME "c6xx"
17 #define ADF_C62XVF_DEVICE_NAME "c6xxvf"
18 #define ADF_C3XXX_DEVICE_NAME "c3xxx"
19 #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf"
20 #define ADF_200XX_DEVICE_NAME "200xx"
21 #define ADF_200XXVF_DEVICE_NAME "200xxvf"
22 #define ADF_C4XXX_DEVICE_NAME "c4xxx"
23 #define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf"
24 #define ADF_4XXX_DEVICE_NAME "4xxx"
25 #define ADF_4XXXVF_DEVICE_NAME "4xxxvf"
26 #define ADF_DH895XCC_PCI_DEVICE_ID 0x435
27 #define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443
28 #define ADF_C62X_PCI_DEVICE_ID 0x37c8
29 #define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9
30 #define ADF_C3XXX_PCI_DEVICE_ID 0x19e2
31 #define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3
32 #define ADF_200XX_PCI_DEVICE_ID 0x18ee
33 #define ADF_200XXIOV_PCI_DEVICE_ID 0x18ef
34 #define ADF_D15XX_PCI_DEVICE_ID 0x6f54
35 #define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55
36 #define ADF_C4XXX_PCI_DEVICE_ID 0x18a0
37 #define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1
38 #define ADF_4XXX_PCI_DEVICE_ID 0x4940
39 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941
40 #define ADF_401XX_PCI_DEVICE_ID 0x4942
41 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943
42 #define ADF_402XX_PCI_DEVICE_ID 0x4944
43 #define ADF_402XXIOV_PCI_DEVICE_ID 0x4945
44
45 #define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); })
46 static inline bool
IS_QAT_GEN4(const unsigned int id)47 IS_QAT_GEN4(const unsigned int id)
48 {
49 return (id == ADF_4XXX_PCI_DEVICE_ID || id == ADF_401XX_PCI_DEVICE_ID ||
50 id == ADF_402XX_PCI_DEVICE_ID ||
51 id == ADF_402XXIOV_PCI_DEVICE_ID ||
52 id == ADF_4XXXIOV_PCI_DEVICE_ID ||
53 id == ADF_401XXIOV_PCI_DEVICE_ID);
54 }
55
56 #define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID))
57 #define ADF_VF2PF_SET_SIZE 32
58 #define ADF_MAX_VF2PF_SET 4
59 #define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE)
60 #define ADF_VF2PF_VFNR_TO_SET(vf_nr) ((vf_nr) / ADF_VF2PF_SET_SIZE)
61 #define ADF_VF2PF_VFNR_TO_MASK(vf_nr) \
62 ({ \
63 u32 vf_nr_ = (vf_nr); \
64 BIT((vf_nr_)-ADF_VF2PF_SET_SIZE *ADF_VF2PF_VFNR_TO_SET( \
65 vf_nr_)); \
66 })
67
68 #define ADF_DEVICE_FUSECTL_OFFSET 0x40
69 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C
70 #define ADF_DEVICE_FUSECTL_MASK 0x80000000
71 #define ADF_PCI_MAX_BARS 3
72 #define ADF_DEVICE_NAME_LENGTH 32
73 #define ADF_ETR_MAX_RINGS_PER_BANK 16
74 #define ADF_MAX_MSIX_VECTOR_NAME 32
75 #define ADF_DEVICE_NAME_PREFIX "qat_"
76 #define ADF_STOP_RETRY 50
77 #define ADF_NUM_THREADS_PER_AE (8)
78 #define ADF_AE_ADMIN_THREAD (7)
79 #define ADF_NUM_PKE_STRAND (2)
80 #define ADF_AE_STRAND0_THREAD (8)
81 #define ADF_AE_STRAND1_THREAD (9)
82 #define ADF_CFG_NUM_SERVICES 4
83 #define ADF_SRV_TYPE_BIT_LEN 3
84 #define ADF_SRV_TYPE_MASK 0x7
85 #define ADF_RINGS_PER_SRV_TYPE 2
86 #define ADF_THRD_ABILITY_BIT_LEN 4
87 #define ADF_THRD_ABILITY_MASK 0xf
88 #define ADF_VF_OFFSET 0x8
89 #define ADF_MAX_FUNC_PER_DEV 0x7
90 #define ADF_PCI_DEV_OFFSET 0x3
91
92 #define ADF_SRV_TYPE_BIT_LEN 3
93 #define ADF_SRV_TYPE_MASK 0x7
94
95 #define GET_SRV_TYPE(ena_srv_mask, srv) \
96 (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK)
97
98 #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops)
99 #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.pfvf_ops)
100 #define ADF_DEFAULT_RING_TO_SRV_MAP \
101 (CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
102 NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
103 COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
104
105 enum adf_accel_capabilities {
106 ADF_ACCEL_CAPABILITIES_NULL = 0,
107 ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1,
108 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2,
109 ADF_ACCEL_CAPABILITIES_CIPHER = 4,
110 ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8,
111 ADF_ACCEL_CAPABILITIES_COMPRESSION = 32,
112 ADF_ACCEL_CAPABILITIES_DEPRECATED = 64,
113 ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128
114 };
115
116 struct adf_bar {
117 rman_res_t base_addr;
118 struct resource *virt_addr;
119 rman_res_t size;
120 } __packed;
121
122 struct adf_accel_msix {
123 struct msix_entry *entries;
124 u32 num_entries;
125 } __packed;
126
127 struct adf_accel_pci {
128 device_t pci_dev;
129 struct adf_accel_msix msix_entries;
130 struct adf_bar pci_bars[ADF_PCI_MAX_BARS];
131 uint8_t revid;
132 uint8_t sku;
133 int node;
134 } __packed;
135
136 enum dev_state { DEV_DOWN = 0, DEV_UP };
137
138 enum dev_sku_info {
139 DEV_SKU_1 = 0,
140 DEV_SKU_2,
141 DEV_SKU_3,
142 DEV_SKU_4,
143 DEV_SKU_VF,
144 DEV_SKU_1_CY,
145 DEV_SKU_2_CY,
146 DEV_SKU_3_CY,
147 DEV_SKU_UNKNOWN
148 };
149
150 static inline const char *
get_sku_info(enum dev_sku_info info)151 get_sku_info(enum dev_sku_info info)
152 {
153 switch (info) {
154 case DEV_SKU_1:
155 return "SKU1";
156 case DEV_SKU_1_CY:
157 return "SKU1CY";
158 case DEV_SKU_2:
159 return "SKU2";
160 case DEV_SKU_2_CY:
161 return "SKU2CY";
162 case DEV_SKU_3:
163 return "SKU3";
164 case DEV_SKU_3_CY:
165 return "SKU3CY";
166 case DEV_SKU_4:
167 return "SKU4";
168 case DEV_SKU_VF:
169 return "SKUVF";
170 case DEV_SKU_UNKNOWN:
171 default:
172 break;
173 }
174 return "Unknown SKU";
175 }
176
177 enum adf_accel_unit_services {
178 ADF_ACCEL_SERVICE_NULL = 0,
179 ADF_ACCEL_INLINE_CRYPTO = 1,
180 ADF_ACCEL_CRYPTO = 2,
181 ADF_ACCEL_COMPRESSION = 4,
182 ADF_ACCEL_ASYM = 8,
183 ADF_ACCEL_ADMIN = 16
184 };
185
186 struct adf_ae_info {
187 u32 num_asym_thd;
188 u32 num_sym_thd;
189 u32 num_dc_thd;
190 } __packed;
191
192 struct adf_accel_unit {
193 u8 au_mask;
194 u32 accel_mask;
195 u64 ae_mask;
196 u64 comp_ae_mask;
197 u32 num_ae;
198 enum adf_accel_unit_services services;
199 } __packed;
200
201 struct adf_accel_unit_info {
202 u64 inline_ingress_msk;
203 u64 inline_egress_msk;
204 u64 sym_ae_msk;
205 u64 asym_ae_msk;
206 u64 dc_ae_msk;
207 u8 num_cy_au;
208 u8 num_dc_au;
209 u8 num_asym_au;
210 u8 num_inline_au;
211 struct adf_accel_unit *au;
212 const struct adf_ae_info *ae_info;
213 } __packed;
214
215 struct adf_hw_aram_info {
216 /* Inline Egress mask. "1" = AE is working with egress traffic */
217 u32 inline_direction_egress_mask;
218 /* Inline congestion managmenet profiles set in config file */
219 u32 inline_congest_mngt_profile;
220 /* Initialise CY AE mask, "1" = AE is used for CY operations */
221 u32 cy_ae_mask;
222 /* Initialise DC AE mask, "1" = AE is used for DC operations */
223 u32 dc_ae_mask;
224 /* Number of long words used to define the ARAM regions */
225 u32 num_aram_lw_entries;
226 /* ARAM region definitions */
227 u32 mmp_region_size;
228 u32 mmp_region_offset;
229 u32 skm_region_size;
230 u32 skm_region_offset;
231 /*
232 * Defines size and offset of compression intermediate buffers stored
233 * in ARAM (device's on-chip memory).
234 */
235 u32 inter_buff_aram_region_size;
236 u32 inter_buff_aram_region_offset;
237 u32 sadb_region_size;
238 u32 sadb_region_offset;
239 } __packed;
240
241 struct adf_hw_device_class {
242 const char *name;
243 const enum adf_device_type type;
244 uint32_t instances;
245 } __packed;
246
247 struct arb_info {
248 u32 arbiter_offset;
249 u32 wrk_thd_2_srv_arb_map;
250 u32 wrk_cfg_offset;
251 } __packed;
252
253 struct admin_info {
254 u32 admin_msg_ur;
255 u32 admin_msg_lr;
256 u32 mailbox_offset;
257 } __packed;
258
259 struct adf_hw_csr_ops {
260 u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size);
261 u32 (*read_csr_ring_head)(struct resource *csr_base_addr,
262 u32 bank,
263 u32 ring);
264 void (*write_csr_ring_head)(struct resource *csr_base_addr,
265 u32 bank,
266 u32 ring,
267 u32 value);
268 u32 (*read_csr_ring_tail)(struct resource *csr_base_addr,
269 u32 bank,
270 u32 ring);
271 void (*write_csr_ring_tail)(struct resource *csr_base_addr,
272 u32 bank,
273 u32 ring,
274 u32 value);
275 u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank);
276 void (*write_csr_ring_config)(struct resource *csr_base_addr,
277 u32 bank,
278 u32 ring,
279 u32 value);
280 bus_addr_t (*read_csr_ring_base)(struct resource *csr_base_addr,
281 u32 bank,
282 u32 ring);
283 void (*write_csr_ring_base)(struct resource *csr_base_addr,
284 u32 bank,
285 u32 ring,
286 bus_addr_t addr);
287 void (*write_csr_int_flag)(struct resource *csr_base_addr,
288 u32 bank,
289 u32 value);
290 void (*write_csr_int_srcsel)(struct resource *csr_base_addr,
291 u32 bank,
292 u32 idx,
293 u32 value);
294 void (*write_csr_int_col_en)(struct resource *csr_base_addr,
295 u32 bank,
296 u32 value);
297 void (*write_csr_int_col_ctl)(struct resource *csr_base_addr,
298 u32 bank,
299 u32 value);
300 void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr,
301 u32 bank,
302 u32 value);
303 u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
304 u32 bank);
305 void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
306 u32 bank,
307 u32 value);
308 u32 (*get_src_sel_mask)(void);
309 u32 (*get_int_col_ctl_enable_mask)(void);
310 u32 (*get_bank_irq_mask)(u32 irq_mask);
311 };
312
313 struct adf_cfg_device_data;
314 struct adf_accel_dev;
315 struct adf_etr_data;
316 struct adf_etr_ring_data;
317
318 struct adf_pfvf_ops {
319 int (*enable_comms)(struct adf_accel_dev *accel_dev);
320 u32 (*get_pf2vf_offset)(u32 i);
321 u32 (*get_vf2pf_offset)(u32 i);
322 void (*enable_vf2pf_interrupts)(struct resource *pmisc_addr,
323 u32 vf_mask);
324 void (*disable_all_vf2pf_interrupts)(struct resource *pmisc_addr);
325 u32 (*disable_pending_vf2pf_interrupts)(struct resource *pmisc_addr);
326 int (*send_msg)(struct adf_accel_dev *accel_dev,
327 struct pfvf_message msg,
328 u32 pfvf_offset,
329 struct mutex *csr_lock);
330 struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,
331 u32 pfvf_offset,
332 u8 compat_ver);
333 };
334
335 struct adf_hw_csr_info {
336 struct adf_hw_csr_ops csr_ops;
337 struct adf_pfvf_ops pfvf_ops;
338 u32 csr_addr_offset;
339 u32 ring_bundle_size;
340 u32 bank_int_flag_clear_mask;
341 u32 num_rings_per_int_srcsel;
342 u32 arb_enable_mask;
343 };
344
345 struct adf_hw_device_data {
346 struct adf_hw_device_class *dev_class;
347 uint32_t (*get_accel_mask)(struct adf_accel_dev *accel_dev);
348 uint64_t (*get_ae_mask)(struct adf_accel_dev *accel_dev);
349 uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self);
350 uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self);
351 uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self);
352 uint32_t (*get_num_aes)(struct adf_hw_device_data *self);
353 uint32_t (*get_num_accels)(struct adf_hw_device_data *self);
354 void (*notify_and_wait_ethernet)(struct adf_accel_dev *accel_dev);
355 bool (*get_eth_doorbell_msg)(struct adf_accel_dev *accel_dev);
356 void (*get_arb_info)(struct arb_info *arb_csrs_info);
357 void (*get_admin_info)(struct admin_info *admin_csrs_info);
358 void (*get_errsou_offset)(u32 *errsou3, u32 *errsou5);
359 uint32_t (*get_num_accel_units)(struct adf_hw_device_data *self);
360 int (*init_accel_units)(struct adf_accel_dev *accel_dev);
361 void (*exit_accel_units)(struct adf_accel_dev *accel_dev);
362 uint32_t (*get_clock_speed)(struct adf_hw_device_data *self);
363 enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self);
364 bool (*check_prod_sku)(struct adf_accel_dev *accel_dev);
365 int (*alloc_irq)(struct adf_accel_dev *accel_dev);
366 void (*free_irq)(struct adf_accel_dev *accel_dev);
367 void (*enable_error_correction)(struct adf_accel_dev *accel_dev);
368 int (*check_uncorrectable_error)(struct adf_accel_dev *accel_dev);
369 void (*print_err_registers)(struct adf_accel_dev *accel_dev);
370 void (*disable_error_interrupts)(struct adf_accel_dev *accel_dev);
371 int (*init_ras)(struct adf_accel_dev *accel_dev);
372 void (*exit_ras)(struct adf_accel_dev *accel_dev);
373 void (*disable_arb)(struct adf_accel_dev *accel_dev);
374 void (*update_ras_errors)(struct adf_accel_dev *accel_dev, int error);
375 bool (*ras_interrupts)(struct adf_accel_dev *accel_dev,
376 bool *reset_required);
377 int (*init_admin_comms)(struct adf_accel_dev *accel_dev);
378 void (*exit_admin_comms)(struct adf_accel_dev *accel_dev);
379 int (*send_admin_init)(struct adf_accel_dev *accel_dev);
380 void (*set_asym_rings_mask)(struct adf_accel_dev *accel_dev);
381 int (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev,
382 u16 *ring_to_svc_map);
383 uint32_t (*get_accel_cap)(struct adf_accel_dev *accel_dev);
384 int (*init_arb)(struct adf_accel_dev *accel_dev);
385 void (*exit_arb)(struct adf_accel_dev *accel_dev);
386 void (*get_arb_mapping)(struct adf_accel_dev *accel_dev,
387 const uint32_t **cfg);
388 int (*init_device)(struct adf_accel_dev *accel_dev);
389 int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev);
390 int (*int_timer_init)(struct adf_accel_dev *accel_dev);
391 void (*int_timer_exit)(struct adf_accel_dev *accel_dev);
392 uint32_t (*get_ae_clock)(struct adf_hw_device_data *self);
393 uint32_t (*get_hb_clock)(struct adf_hw_device_data *self);
394 void (*disable_iov)(struct adf_accel_dev *accel_dev);
395 void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,
396 bool enable);
397 void (*enable_ints)(struct adf_accel_dev *accel_dev);
398 bool (*check_slice_hang)(struct adf_accel_dev *accel_dev);
399 int (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
400 void (*enable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev);
401 void (*disable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev);
402 int (*interrupt_active_pf2vf)(struct adf_accel_dev *accel_dev);
403 int (*get_int_active_bundles)(struct adf_accel_dev *accel_dev);
404 void (*reset_device)(struct adf_accel_dev *accel_dev);
405 void (*reset_hw_units)(struct adf_accel_dev *accel_dev);
406 int (*measure_clock)(struct adf_accel_dev *accel_dev);
407 void (*restore_device)(struct adf_accel_dev *accel_dev);
408 uint64_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev,
409 enum adf_accel_unit_services services);
410 enum adf_accel_unit_services (
411 *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num);
412 int (*add_pke_stats)(struct adf_accel_dev *accel_dev);
413 void (*remove_pke_stats)(struct adf_accel_dev *accel_dev);
414 int (*add_misc_error)(struct adf_accel_dev *accel_dev);
415 int (*count_ras_event)(struct adf_accel_dev *accel_dev,
416 u32 *ras_event,
417 char *aeidstr);
418 void (*remove_misc_error)(struct adf_accel_dev *accel_dev);
419 int (*configure_accel_units)(struct adf_accel_dev *accel_dev);
420 int (*ring_pair_reset)(struct adf_accel_dev *accel_dev,
421 u32 bank_number);
422 void (*config_ring_irq)(struct adf_accel_dev *accel_dev,
423 u32 bank_number,
424 u16 ring_mask);
425 uint32_t (*get_objs_num)(struct adf_accel_dev *accel_dev);
426 const char *(*get_obj_name)(struct adf_accel_dev *accel_dev,
427 enum adf_accel_unit_services services);
428 void (*pre_reset)(struct adf_accel_dev *accel_dev);
429 void (*post_reset)(struct adf_accel_dev *accel_dev);
430 void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
431 void (*get_ring_svc_map_data)(int ring_pair_index,
432 u16 ring_to_svc_map,
433 u8 *serv_type,
434 int *ring_index,
435 int *num_rings_per_srv,
436 int bundle_num);
437 struct adf_hw_csr_info csr_info;
438 const char *fw_name;
439 const char *fw_mmp_name;
440 bool reset_ack;
441 uint32_t fuses;
442 uint32_t accel_capabilities_mask;
443 uint32_t instance_id;
444 uint16_t accel_mask;
445 u32 aerucm_mask;
446 u64 ae_mask;
447 u64 admin_ae_mask;
448 u32 service_mask;
449 u32 service_to_load_mask;
450 u32 heartbeat_ctr_num;
451 uint16_t tx_rings_mask;
452 uint8_t tx_rx_gap;
453 uint16_t num_banks;
454 u8 num_rings_per_bank;
455 uint8_t num_accel;
456 uint8_t num_logical_accel;
457 uint8_t num_engines;
458 bool get_ring_to_svc_done;
459 int (*get_storage_enabled)(struct adf_accel_dev *accel_dev,
460 uint32_t *storage_enabled);
461 u8 query_storage_cap;
462 u32 clock_frequency;
463 u8 storage_enable;
464 u32 extended_dc_capabilities;
465 int (*config_device)(struct adf_accel_dev *accel_dev);
466 u32 asym_ae_active_thd_mask;
467 u16 asym_rings_mask;
468 int (*get_fw_image_type)(struct adf_accel_dev *accel_dev,
469 enum adf_cfg_fw_image_type *fw_image_type);
470 u16 ring_to_svc_map;
471 } __packed;
472
473 /* helper enum for performing CSR operations */
474 enum operation {
475 AND,
476 OR,
477 };
478
479 /* 32-bit CSR write macro */
480 #define ADF_CSR_WR(csr_base, csr_offset, val) \
481 bus_write_4(csr_base, csr_offset, val)
482
483 /* 64-bit CSR write macro */
484 #ifdef __x86_64__
485 #define ADF_CSR_WR64(csr_base, csr_offset, val) \
486 bus_write_8(csr_base, csr_offset, val)
487 #else
488 static __inline void
adf_csr_wr64(struct resource * csr_base,bus_size_t offset,uint64_t value)489 adf_csr_wr64(struct resource *csr_base, bus_size_t offset, uint64_t value)
490 {
491 bus_write_4(csr_base, offset, (uint32_t)value);
492 bus_write_4(csr_base, offset + 4, (uint32_t)(value >> 32));
493 }
494 #define ADF_CSR_WR64(csr_base, csr_offset, val) \
495 adf_csr_wr64(csr_base, csr_offset, val)
496 #endif
497
498 /* 32-bit CSR read macro */
499 #define ADF_CSR_RD(csr_base, csr_offset) bus_read_4(csr_base, csr_offset)
500
501 /* 64-bit CSR read macro */
502 #ifdef __x86_64__
503 #define ADF_CSR_RD64(csr_base, csr_offset) bus_read_8(csr_base, csr_offset)
504 #else
505 static __inline uint64_t
adf_csr_rd64(struct resource * csr_base,bus_size_t offset)506 adf_csr_rd64(struct resource *csr_base, bus_size_t offset)
507 {
508 return (((uint64_t)bus_read_4(csr_base, offset)) |
509 (((uint64_t)bus_read_4(csr_base, offset + 4)) << 32));
510 }
511 #define ADF_CSR_RD64(csr_base, csr_offset) adf_csr_rd64(csr_base, csr_offset)
512 #endif
513
514 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev)
515 #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars)
516 #define GET_HW_DATA(accel_dev) (accel_dev->hw_device)
517 #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks)
518 #define GET_DEV_SKU(accel_dev) (accel_dev->accel_pci_dev.sku)
519 #define GET_NUM_RINGS_PER_BANK(accel_dev) \
520 (GET_HW_DATA(accel_dev)->num_rings_per_bank)
521 #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines)
522 #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev
523 #define GET_SRV_TYPE(ena_srv_mask, srv) \
524 (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK)
525 #define SET_ASYM_MASK(asym_mask, srv) \
526 ({ \
527 typeof(srv) srv_ = (srv); \
528 (asym_mask) |= ((1 << (srv_)*ADF_RINGS_PER_SRV_TYPE) | \
529 (1 << ((srv_)*ADF_RINGS_PER_SRV_TYPE + 1))); \
530 })
531
532 #define GET_NUM_RINGS_PER_BANK(accel_dev) \
533 (GET_HW_DATA(accel_dev)->num_rings_per_bank)
534 #define GET_MAX_PROCESSES(accel_dev) \
535 ({ \
536 typeof(accel_dev) dev = (accel_dev); \
537 (GET_MAX_BANKS(dev) * (GET_NUM_RINGS_PER_BANK(dev) / 2)); \
538 })
539 #define GET_DU_TABLE(accel_dev) (accel_dev->du_table)
540
541 static inline void
adf_csr_fetch_and_and(struct resource * csr,size_t offs,unsigned long mask)542 adf_csr_fetch_and_and(struct resource *csr, size_t offs, unsigned long mask)
543 {
544 unsigned int val = ADF_CSR_RD(csr, offs);
545
546 val &= mask;
547 ADF_CSR_WR(csr, offs, val);
548 }
549
550 static inline void
adf_csr_fetch_and_or(struct resource * csr,size_t offs,unsigned long mask)551 adf_csr_fetch_and_or(struct resource *csr, size_t offs, unsigned long mask)
552 {
553 unsigned int val = ADF_CSR_RD(csr, offs);
554
555 val |= mask;
556 ADF_CSR_WR(csr, offs, val);
557 }
558
559 static inline void
adf_csr_fetch_and_update(enum operation op,struct resource * csr,size_t offs,unsigned long mask)560 adf_csr_fetch_and_update(enum operation op,
561 struct resource *csr,
562 size_t offs,
563 unsigned long mask)
564 {
565 switch (op) {
566 case AND:
567 adf_csr_fetch_and_and(csr, offs, mask);
568 break;
569 case OR:
570 adf_csr_fetch_and_or(csr, offs, mask);
571 break;
572 }
573 }
574
575 struct pfvf_stats {
576 struct dentry *stats_file;
577 /* Messages put in CSR */
578 unsigned int tx;
579 /* Messages read from CSR */
580 unsigned int rx;
581 /* Interrupt fired but int bit was clear */
582 unsigned int spurious;
583 /* Block messages sent */
584 unsigned int blk_tx;
585 /* Block messages received */
586 unsigned int blk_rx;
587 /* Blocks received with CRC errors */
588 unsigned int crc_err;
589 /* CSR in use by other side */
590 unsigned int busy;
591 /* Receiver did not acknowledge */
592 unsigned int no_ack;
593 /* Collision detected */
594 unsigned int collision;
595 /* Couldn't send a response */
596 unsigned int tx_timeout;
597 /* Didn't receive a response */
598 unsigned int rx_timeout;
599 /* Responses received */
600 unsigned int rx_rsp;
601 /* Messages re-transmitted */
602 unsigned int retry;
603 /* Event put timeout */
604 unsigned int event_timeout;
605 };
606
607 #define NUM_PFVF_COUNTERS 14
608
609 void adf_get_admin_info(struct admin_info *admin_csrs_info);
610 struct adf_admin_comms {
611 bus_addr_t phy_addr;
612 bus_addr_t const_tbl_addr;
613 bus_addr_t aram_map_phys_addr;
614 bus_addr_t phy_hb_addr;
615 bus_dmamap_t aram_map;
616 bus_dmamap_t const_tbl_map;
617 bus_dmamap_t hb_map;
618 char *virt_addr;
619 char *virt_hb_addr;
620 uint32_t mailbox_offset;
621 struct resource *mailbox_addr;
622 struct sx lock;
623 struct bus_dmamem dma_mem;
624 struct bus_dmamem dma_hb;
625 };
626
627 struct icp_qat_fw_loader_handle;
628 struct adf_fw_loader_data {
629 struct icp_qat_fw_loader_handle *fw_loader;
630 const struct firmware *uof_fw;
631 const struct firmware *mmp_fw;
632 };
633
634 struct adf_accel_vf_info {
635 struct adf_accel_dev *accel_dev;
636 struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */
637 u32 vf_nr;
638 bool init;
639 u8 compat_ver;
640 struct pfvf_stats pfvf_counters;
641 };
642
643 struct adf_fw_versions {
644 u16 fw_version_major;
645 u16 fw_version_minor;
646 u32 fw_version_patch;
647 u8 mmp_version_major;
648 u8 mmp_version_minor;
649 u8 mmp_version_patch;
650 };
651
652 struct adf_int_timer {
653 struct adf_accel_dev *accel_dev;
654 struct workqueue_struct *timer_irq_wq;
655 struct timer_list timer;
656 u32 timeout_val;
657 u32 int_cnt;
658 bool enabled;
659 };
660
661 #define ADF_COMPAT_CHECKER_MAX 8
662 typedef int (*adf_iov_compat_checker_t)(struct adf_accel_dev *accel_dev,
663 u8 vf_compat_ver);
664 struct adf_accel_compat_manager {
665 u8 num_chker;
666 adf_iov_compat_checker_t iov_compat_checkers[ADF_COMPAT_CHECKER_MAX];
667 };
668
669 struct adf_heartbeat;
670 struct adf_accel_dev {
671 struct adf_hw_aram_info *aram_info;
672 struct adf_accel_unit_info *au_info;
673 struct adf_etr_data *transport;
674 struct adf_hw_device_data *hw_device;
675 struct adf_cfg_device_data *cfg;
676 struct adf_fw_loader_data *fw_loader;
677 struct adf_admin_comms *admin;
678 struct adf_uio_control_accel *accel;
679 struct adf_heartbeat *heartbeat;
680 struct adf_int_timer *int_timer;
681 struct adf_fw_versions fw_versions;
682 unsigned int autoreset_on_error;
683 struct adf_fw_counters_data *fw_counters_data;
684 struct sysctl_oid *debugfs_ae_config;
685 struct list_head crypto_list;
686 atomic_t *ras_counters;
687 unsigned long status;
688 atomic_t ref_count;
689 bus_dma_tag_t dma_tag;
690 struct sysctl_ctx_list sysctl_ctx;
691 struct sysctl_oid *ras_correctable;
692 struct sysctl_oid *ras_uncorrectable;
693 struct sysctl_oid *ras_fatal;
694 struct sysctl_oid *ras_reset;
695 struct sysctl_oid *pke_replay_dbgfile;
696 struct sysctl_oid *misc_error_dbgfile;
697 struct sysctl_oid *fw_version_oid;
698 struct sysctl_oid *mmp_version_oid;
699 struct sysctl_oid *hw_version_oid;
700 struct sysctl_oid *cnv_error_oid;
701 struct list_head list;
702 struct adf_accel_pci accel_pci_dev;
703 struct adf_accel_compat_manager *cm;
704 u8 compat_ver;
705 #ifdef QAT_DISABLE_SAFE_DC_MODE
706 struct sysctl_oid *safe_dc_mode;
707 u8 disable_safe_dc_mode;
708 #endif /* QAT_DISABLE_SAFE_DC_MODE */
709 union {
710 struct {
711 /* vf_info is non-zero when SR-IOV is init'ed */
712 struct adf_accel_vf_info *vf_info;
713 int num_vfs;
714 } pf;
715 struct {
716 bool irq_enabled;
717 struct resource *irq;
718 void *cookie;
719 struct task pf2vf_bh_tasklet;
720 struct mutex vf2pf_lock; /* protect CSR access */
721 struct completion msg_received;
722 struct pfvf_message
723 response; /* temp field holding pf2vf response */
724 enum ring_reset_result rpreset_sts;
725 struct mutex rpreset_lock; /* protect rpreset_sts */
726 struct pfvf_stats pfvf_counters;
727 u8 pf_compat_ver;
728 } vf;
729 } u1;
730 bool is_vf;
731 u32 accel_id;
732 void *lac_dev;
733 struct mutex lock; /* protect accel_dev during start/stop e.t.c */
734 };
735 #endif
736