1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2026 Intel Corporation */ 3 #ifndef ADF_ACCEL_DEVICES_H_ 4 #define ADF_ACCEL_DEVICES_H_ 5 6 #include "qat_freebsd.h" 7 #include "adf_cfg_common.h" 8 #include "adf_pfvf_msg.h" 9 10 #include "opt_qat.h" 11 12 #define ADF_CFG_NUM_SERVICES 4 13 14 #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" 15 #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" 16 #define ADF_C62X_DEVICE_NAME "c6xx" 17 #define ADF_C62XVF_DEVICE_NAME "c6xxvf" 18 #define ADF_C3XXX_DEVICE_NAME "c3xxx" 19 #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" 20 #define ADF_200XX_DEVICE_NAME "200xx" 21 #define ADF_200XXVF_DEVICE_NAME "200xxvf" 22 #define ADF_C4XXX_DEVICE_NAME "c4xxx" 23 #define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf" 24 #define ADF_4XXX_DEVICE_NAME "4xxx" 25 #define ADF_4XXXVF_DEVICE_NAME "4xxxvf" 26 #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 27 #define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 28 #define ADF_C62X_PCI_DEVICE_ID 0x37c8 29 #define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9 30 #define ADF_C3XXX_PCI_DEVICE_ID 0x19e2 31 #define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3 32 #define ADF_200XX_PCI_DEVICE_ID 0x18ee 33 #define ADF_200XXIOV_PCI_DEVICE_ID 0x18ef 34 #define ADF_D15XX_PCI_DEVICE_ID 0x6f54 35 #define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55 36 #define ADF_C4XXX_PCI_DEVICE_ID 0x18a0 37 #define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1 38 #define ADF_4XXX_PCI_DEVICE_ID 0x4940 39 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 40 #define ADF_401XX_PCI_DEVICE_ID 0x4942 41 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 42 #define ADF_402XX_PCI_DEVICE_ID 0x4944 43 #define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 44 45 #define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); }) 46 static inline bool 47 IS_QAT_GEN4(const unsigned int id) 48 { 49 return (id == ADF_4XXX_PCI_DEVICE_ID || id == ADF_401XX_PCI_DEVICE_ID || 50 id == ADF_402XX_PCI_DEVICE_ID || 51 id == ADF_402XXIOV_PCI_DEVICE_ID || 52 id == ADF_4XXXIOV_PCI_DEVICE_ID || 53 id == ADF_401XXIOV_PCI_DEVICE_ID); 54 } 55 56 #define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID)) 57 #define ADF_VF2PF_SET_SIZE 32 58 #define ADF_MAX_VF2PF_SET 4 59 #define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE) 60 #define ADF_VF2PF_VFNR_TO_SET(vf_nr) ((vf_nr) / ADF_VF2PF_SET_SIZE) 61 #define ADF_VF2PF_VFNR_TO_MASK(vf_nr) \ 62 ({ \ 63 u32 vf_nr_ = (vf_nr); \ 64 BIT((vf_nr_)-ADF_VF2PF_SET_SIZE *ADF_VF2PF_VFNR_TO_SET( \ 65 vf_nr_)); \ 66 }) 67 68 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 69 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C 70 #define ADF_DEVICE_FUSECTL_MASK 0x80000000 71 #define ADF_PCI_MAX_BARS 3 72 #define ADF_DEVICE_NAME_LENGTH 32 73 #define ADF_ETR_MAX_RINGS_PER_BANK 16 74 #define ADF_MAX_MSIX_VECTOR_NAME 32 75 #define ADF_DEVICE_NAME_PREFIX "qat_" 76 #define ADF_STOP_RETRY 50 77 #define ADF_NUM_THREADS_PER_AE (8) 78 #define ADF_AE_ADMIN_THREAD (7) 79 #define ADF_NUM_PKE_STRAND (2) 80 #define ADF_AE_STRAND0_THREAD (8) 81 #define ADF_AE_STRAND1_THREAD (9) 82 #define ADF_CFG_NUM_SERVICES 4 83 #define ADF_SRV_TYPE_BIT_LEN 3 84 #define ADF_SRV_TYPE_MASK 0x7 85 #define ADF_RINGS_PER_SRV_TYPE 2 86 #define ADF_THRD_ABILITY_BIT_LEN 4 87 #define ADF_THRD_ABILITY_MASK 0xf 88 #define ADF_VF_OFFSET 0x8 89 #define ADF_MAX_FUNC_PER_DEV 0x7 90 #define ADF_PCI_DEV_OFFSET 0x3 91 92 #define ADF_SRV_TYPE_BIT_LEN 3 93 #define ADF_SRV_TYPE_MASK 0x7 94 95 #define GET_SRV_TYPE(ena_srv_mask, srv) \ 96 (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) 97 98 #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops) 99 #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.pfvf_ops) 100 #define ADF_DEFAULT_RING_TO_SRV_MAP \ 101 (CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ 102 NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ 103 COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) 104 105 enum adf_accel_capabilities { 106 ADF_ACCEL_CAPABILITIES_NULL = 0, 107 ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, 108 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, 109 ADF_ACCEL_CAPABILITIES_CIPHER = 4, 110 ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, 111 ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, 112 ADF_ACCEL_CAPABILITIES_DEPRECATED = 64, 113 ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 114 }; 115 116 struct adf_bar { 117 rman_res_t base_addr; 118 struct resource *virt_addr; 119 rman_res_t size; 120 } __packed; 121 122 struct adf_accel_msix { 123 struct msix_entry *entries; 124 u32 num_entries; 125 } __packed; 126 127 struct adf_accel_pci { 128 device_t pci_dev; 129 struct adf_accel_msix msix_entries; 130 struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; 131 uint8_t revid; 132 uint8_t sku; 133 int node; 134 } __packed; 135 136 enum dev_state { DEV_DOWN = 0, DEV_UP }; 137 138 enum dev_sku_info { 139 DEV_SKU_1 = 0, 140 DEV_SKU_2, 141 DEV_SKU_3, 142 DEV_SKU_4, 143 DEV_SKU_VF, 144 DEV_SKU_1_CY, 145 DEV_SKU_2_CY, 146 DEV_SKU_3_CY, 147 DEV_SKU_UNKNOWN 148 }; 149 150 static inline const char * 151 get_sku_info(enum dev_sku_info info) 152 { 153 switch (info) { 154 case DEV_SKU_1: 155 return "SKU1"; 156 case DEV_SKU_1_CY: 157 return "SKU1CY"; 158 case DEV_SKU_2: 159 return "SKU2"; 160 case DEV_SKU_2_CY: 161 return "SKU2CY"; 162 case DEV_SKU_3: 163 return "SKU3"; 164 case DEV_SKU_3_CY: 165 return "SKU3CY"; 166 case DEV_SKU_4: 167 return "SKU4"; 168 case DEV_SKU_VF: 169 return "SKUVF"; 170 case DEV_SKU_UNKNOWN: 171 default: 172 break; 173 } 174 return "Unknown SKU"; 175 } 176 177 enum adf_accel_unit_services { 178 ADF_ACCEL_SERVICE_NULL = 0, 179 ADF_ACCEL_INLINE_CRYPTO = 1, 180 ADF_ACCEL_CRYPTO = 2, 181 ADF_ACCEL_COMPRESSION = 4, 182 ADF_ACCEL_ASYM = 8, 183 ADF_ACCEL_ADMIN = 16 184 }; 185 186 struct adf_ae_info { 187 u32 num_asym_thd; 188 u32 num_sym_thd; 189 u32 num_dc_thd; 190 } __packed; 191 192 struct adf_accel_unit { 193 u8 au_mask; 194 u32 accel_mask; 195 u64 ae_mask; 196 u64 comp_ae_mask; 197 u32 num_ae; 198 enum adf_accel_unit_services services; 199 } __packed; 200 201 struct adf_accel_unit_info { 202 u64 inline_ingress_msk; 203 u64 inline_egress_msk; 204 u64 sym_ae_msk; 205 u64 asym_ae_msk; 206 u64 dc_ae_msk; 207 u8 num_cy_au; 208 u8 num_dc_au; 209 u8 num_asym_au; 210 u8 num_inline_au; 211 struct adf_accel_unit *au; 212 const struct adf_ae_info *ae_info; 213 } __packed; 214 215 struct adf_hw_aram_info { 216 /* Inline Egress mask. "1" = AE is working with egress traffic */ 217 u32 inline_direction_egress_mask; 218 /* Inline congestion managmenet profiles set in config file */ 219 u32 inline_congest_mngt_profile; 220 /* Initialise CY AE mask, "1" = AE is used for CY operations */ 221 u32 cy_ae_mask; 222 /* Initialise DC AE mask, "1" = AE is used for DC operations */ 223 u32 dc_ae_mask; 224 /* Number of long words used to define the ARAM regions */ 225 u32 num_aram_lw_entries; 226 /* ARAM region definitions */ 227 u32 mmp_region_size; 228 u32 mmp_region_offset; 229 u32 skm_region_size; 230 u32 skm_region_offset; 231 /* 232 * Defines size and offset of compression intermediate buffers stored 233 * in ARAM (device's on-chip memory). 234 */ 235 u32 inter_buff_aram_region_size; 236 u32 inter_buff_aram_region_offset; 237 u32 sadb_region_size; 238 u32 sadb_region_offset; 239 } __packed; 240 241 struct adf_hw_device_class { 242 const char *name; 243 const enum adf_device_type type; 244 uint32_t instances; 245 } __packed; 246 247 struct arb_info { 248 u32 arbiter_offset; 249 u32 wrk_thd_2_srv_arb_map; 250 u32 wrk_cfg_offset; 251 } __packed; 252 253 struct admin_info { 254 u32 admin_msg_ur; 255 u32 admin_msg_lr; 256 u32 mailbox_offset; 257 } __packed; 258 259 struct adf_hw_csr_ops { 260 u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size); 261 u32 (*read_csr_ring_head)(struct resource *csr_base_addr, 262 u32 bank, 263 u32 ring); 264 void (*write_csr_ring_head)(struct resource *csr_base_addr, 265 u32 bank, 266 u32 ring, 267 u32 value); 268 u32 (*read_csr_ring_tail)(struct resource *csr_base_addr, 269 u32 bank, 270 u32 ring); 271 void (*write_csr_ring_tail)(struct resource *csr_base_addr, 272 u32 bank, 273 u32 ring, 274 u32 value); 275 u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank); 276 void (*write_csr_ring_config)(struct resource *csr_base_addr, 277 u32 bank, 278 u32 ring, 279 u32 value); 280 bus_addr_t (*read_csr_ring_base)(struct resource *csr_base_addr, 281 u32 bank, 282 u32 ring); 283 void (*write_csr_ring_base)(struct resource *csr_base_addr, 284 u32 bank, 285 u32 ring, 286 bus_addr_t addr); 287 void (*write_csr_int_flag)(struct resource *csr_base_addr, 288 u32 bank, 289 u32 value); 290 void (*write_csr_int_srcsel)(struct resource *csr_base_addr, 291 u32 bank, 292 u32 idx, 293 u32 value); 294 void (*write_csr_int_col_en)(struct resource *csr_base_addr, 295 u32 bank, 296 u32 value); 297 void (*write_csr_int_col_ctl)(struct resource *csr_base_addr, 298 u32 bank, 299 u32 value); 300 void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr, 301 u32 bank, 302 u32 value); 303 u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr, 304 u32 bank); 305 void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr, 306 u32 bank, 307 u32 value); 308 u32 (*get_src_sel_mask)(void); 309 u32 (*get_int_col_ctl_enable_mask)(void); 310 u32 (*get_bank_irq_mask)(u32 irq_mask); 311 }; 312 313 struct adf_cfg_device_data; 314 struct adf_accel_dev; 315 struct adf_etr_data; 316 struct adf_etr_ring_data; 317 318 struct adf_pfvf_ops { 319 int (*enable_comms)(struct adf_accel_dev *accel_dev); 320 u32 (*get_pf2vf_offset)(u32 i); 321 u32 (*get_vf2pf_offset)(u32 i); 322 void (*enable_vf2pf_interrupts)(struct resource *pmisc_addr, 323 u32 vf_mask); 324 void (*disable_all_vf2pf_interrupts)(struct resource *pmisc_addr); 325 u32 (*disable_pending_vf2pf_interrupts)(struct resource *pmisc_addr); 326 int (*send_msg)(struct adf_accel_dev *accel_dev, 327 struct pfvf_message msg, 328 u32 pfvf_offset, 329 struct mutex *csr_lock); 330 struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, 331 u32 pfvf_offset, 332 u8 compat_ver); 333 }; 334 335 struct adf_hw_csr_info { 336 struct adf_hw_csr_ops csr_ops; 337 struct adf_pfvf_ops pfvf_ops; 338 u32 csr_addr_offset; 339 u32 ring_bundle_size; 340 u32 bank_int_flag_clear_mask; 341 u32 num_rings_per_int_srcsel; 342 u32 arb_enable_mask; 343 }; 344 345 struct adf_hw_device_data { 346 struct adf_hw_device_class *dev_class; 347 uint32_t (*get_accel_mask)(struct adf_accel_dev *accel_dev); 348 uint64_t (*get_ae_mask)(struct adf_accel_dev *accel_dev); 349 uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self); 350 uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); 351 uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); 352 uint32_t (*get_num_aes)(struct adf_hw_device_data *self); 353 uint32_t (*get_num_accels)(struct adf_hw_device_data *self); 354 void (*notify_and_wait_ethernet)(struct adf_accel_dev *accel_dev); 355 bool (*get_eth_doorbell_msg)(struct adf_accel_dev *accel_dev); 356 void (*get_arb_info)(struct arb_info *arb_csrs_info); 357 void (*get_admin_info)(struct admin_info *admin_csrs_info); 358 void (*get_errsou_offset)(u32 *errsou3, u32 *errsou5); 359 uint32_t (*get_num_accel_units)(struct adf_hw_device_data *self); 360 int (*init_accel_units)(struct adf_accel_dev *accel_dev); 361 void (*exit_accel_units)(struct adf_accel_dev *accel_dev); 362 uint32_t (*get_clock_speed)(struct adf_hw_device_data *self); 363 enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); 364 bool (*check_prod_sku)(struct adf_accel_dev *accel_dev); 365 int (*alloc_irq)(struct adf_accel_dev *accel_dev); 366 void (*free_irq)(struct adf_accel_dev *accel_dev); 367 void (*enable_error_correction)(struct adf_accel_dev *accel_dev); 368 int (*check_uncorrectable_error)(struct adf_accel_dev *accel_dev); 369 void (*print_err_registers)(struct adf_accel_dev *accel_dev); 370 void (*disable_error_interrupts)(struct adf_accel_dev *accel_dev); 371 int (*init_ras)(struct adf_accel_dev *accel_dev); 372 void (*exit_ras)(struct adf_accel_dev *accel_dev); 373 void (*disable_arb)(struct adf_accel_dev *accel_dev); 374 void (*update_ras_errors)(struct adf_accel_dev *accel_dev, int error); 375 bool (*ras_interrupts)(struct adf_accel_dev *accel_dev, 376 bool *reset_required); 377 int (*init_admin_comms)(struct adf_accel_dev *accel_dev); 378 void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); 379 int (*send_admin_init)(struct adf_accel_dev *accel_dev); 380 void (*set_asym_rings_mask)(struct adf_accel_dev *accel_dev); 381 int (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev, 382 u16 *ring_to_svc_map); 383 uint32_t (*get_accel_cap)(struct adf_accel_dev *accel_dev); 384 int (*init_arb)(struct adf_accel_dev *accel_dev); 385 void (*exit_arb)(struct adf_accel_dev *accel_dev); 386 void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, 387 const uint32_t **cfg); 388 int (*init_device)(struct adf_accel_dev *accel_dev); 389 int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev); 390 int (*int_timer_init)(struct adf_accel_dev *accel_dev); 391 void (*int_timer_exit)(struct adf_accel_dev *accel_dev); 392 uint32_t (*get_ae_clock)(struct adf_hw_device_data *self); 393 uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); 394 void (*disable_iov)(struct adf_accel_dev *accel_dev); 395 void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, 396 bool enable); 397 void (*enable_ints)(struct adf_accel_dev *accel_dev); 398 bool (*check_slice_hang)(struct adf_accel_dev *accel_dev); 399 int (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); 400 void (*enable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev); 401 void (*disable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev); 402 int (*interrupt_active_pf2vf)(struct adf_accel_dev *accel_dev); 403 int (*get_int_active_bundles)(struct adf_accel_dev *accel_dev); 404 void (*reset_device)(struct adf_accel_dev *accel_dev); 405 void (*reset_hw_units)(struct adf_accel_dev *accel_dev); 406 int (*measure_clock)(struct adf_accel_dev *accel_dev); 407 void (*restore_device)(struct adf_accel_dev *accel_dev); 408 uint64_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev, 409 enum adf_accel_unit_services services); 410 enum adf_accel_unit_services ( 411 *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num); 412 int (*add_pke_stats)(struct adf_accel_dev *accel_dev); 413 void (*remove_pke_stats)(struct adf_accel_dev *accel_dev); 414 int (*add_misc_error)(struct adf_accel_dev *accel_dev); 415 int (*count_ras_event)(struct adf_accel_dev *accel_dev, 416 u32 *ras_event, 417 char *aeidstr); 418 void (*remove_misc_error)(struct adf_accel_dev *accel_dev); 419 int (*configure_accel_units)(struct adf_accel_dev *accel_dev); 420 int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, 421 u32 bank_number); 422 void (*config_ring_irq)(struct adf_accel_dev *accel_dev, 423 u32 bank_number, 424 u16 ring_mask); 425 uint32_t (*get_objs_num)(struct adf_accel_dev *accel_dev); 426 const char *(*get_obj_name)(struct adf_accel_dev *accel_dev, 427 enum adf_accel_unit_services services); 428 void (*pre_reset)(struct adf_accel_dev *accel_dev); 429 void (*post_reset)(struct adf_accel_dev *accel_dev); 430 void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); 431 void (*get_ring_svc_map_data)(int ring_pair_index, 432 u16 ring_to_svc_map, 433 u8 *serv_type, 434 int *ring_index, 435 int *num_rings_per_srv, 436 int bundle_num); 437 struct adf_hw_csr_info csr_info; 438 const char *fw_name; 439 const char *fw_mmp_name; 440 bool reset_ack; 441 uint32_t fuses; 442 uint32_t accel_capabilities_mask; 443 uint32_t instance_id; 444 uint16_t accel_mask; 445 u32 aerucm_mask; 446 u64 ae_mask; 447 u64 admin_ae_mask; 448 u32 service_mask; 449 u32 service_to_load_mask; 450 u32 heartbeat_ctr_num; 451 uint16_t tx_rings_mask; 452 uint8_t tx_rx_gap; 453 uint16_t num_banks; 454 u8 num_rings_per_bank; 455 uint8_t num_accel; 456 uint8_t num_logical_accel; 457 uint8_t num_engines; 458 bool get_ring_to_svc_done; 459 int (*get_storage_enabled)(struct adf_accel_dev *accel_dev, 460 uint32_t *storage_enabled); 461 u8 query_storage_cap; 462 u32 clock_frequency; 463 u8 storage_enable; 464 u32 extended_dc_capabilities; 465 int (*config_device)(struct adf_accel_dev *accel_dev); 466 u32 asym_ae_active_thd_mask; 467 u16 asym_rings_mask; 468 int (*get_fw_image_type)(struct adf_accel_dev *accel_dev, 469 enum adf_cfg_fw_image_type *fw_image_type); 470 u16 ring_to_svc_map; 471 } __packed; 472 473 /* helper enum for performing CSR operations */ 474 enum operation { 475 AND, 476 OR, 477 }; 478 479 /* 32-bit CSR write macro */ 480 #define ADF_CSR_WR(csr_base, csr_offset, val) \ 481 bus_write_4(csr_base, csr_offset, val) 482 483 /* 64-bit CSR write macro */ 484 #ifdef __x86_64__ 485 #define ADF_CSR_WR64(csr_base, csr_offset, val) \ 486 bus_write_8(csr_base, csr_offset, val) 487 #else 488 static __inline void 489 adf_csr_wr64(struct resource *csr_base, bus_size_t offset, uint64_t value) 490 { 491 bus_write_4(csr_base, offset, (uint32_t)value); 492 bus_write_4(csr_base, offset + 4, (uint32_t)(value >> 32)); 493 } 494 #define ADF_CSR_WR64(csr_base, csr_offset, val) \ 495 adf_csr_wr64(csr_base, csr_offset, val) 496 #endif 497 498 /* 32-bit CSR read macro */ 499 #define ADF_CSR_RD(csr_base, csr_offset) bus_read_4(csr_base, csr_offset) 500 501 /* 64-bit CSR read macro */ 502 #ifdef __x86_64__ 503 #define ADF_CSR_RD64(csr_base, csr_offset) bus_read_8(csr_base, csr_offset) 504 #else 505 static __inline uint64_t 506 adf_csr_rd64(struct resource *csr_base, bus_size_t offset) 507 { 508 return (((uint64_t)bus_read_4(csr_base, offset)) | 509 (((uint64_t)bus_read_4(csr_base, offset + 4)) << 32)); 510 } 511 #define ADF_CSR_RD64(csr_base, csr_offset) adf_csr_rd64(csr_base, csr_offset) 512 #endif 513 514 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev) 515 #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) 516 #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) 517 #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) 518 #define GET_DEV_SKU(accel_dev) (accel_dev->accel_pci_dev.sku) 519 #define GET_NUM_RINGS_PER_BANK(accel_dev) \ 520 (GET_HW_DATA(accel_dev)->num_rings_per_bank) 521 #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) 522 #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev 523 #define GET_SRV_TYPE(ena_srv_mask, srv) \ 524 (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) 525 #define SET_ASYM_MASK(asym_mask, srv) \ 526 ({ \ 527 typeof(srv) srv_ = (srv); \ 528 (asym_mask) |= ((1 << (srv_)*ADF_RINGS_PER_SRV_TYPE) | \ 529 (1 << ((srv_)*ADF_RINGS_PER_SRV_TYPE + 1))); \ 530 }) 531 532 #define GET_NUM_RINGS_PER_BANK(accel_dev) \ 533 (GET_HW_DATA(accel_dev)->num_rings_per_bank) 534 #define GET_MAX_PROCESSES(accel_dev) \ 535 ({ \ 536 typeof(accel_dev) dev = (accel_dev); \ 537 (GET_MAX_BANKS(dev) * (GET_NUM_RINGS_PER_BANK(dev) / 2)); \ 538 }) 539 #define GET_DU_TABLE(accel_dev) (accel_dev->du_table) 540 541 static inline void 542 adf_csr_fetch_and_and(struct resource *csr, size_t offs, unsigned long mask) 543 { 544 unsigned int val = ADF_CSR_RD(csr, offs); 545 546 val &= mask; 547 ADF_CSR_WR(csr, offs, val); 548 } 549 550 static inline void 551 adf_csr_fetch_and_or(struct resource *csr, size_t offs, unsigned long mask) 552 { 553 unsigned int val = ADF_CSR_RD(csr, offs); 554 555 val |= mask; 556 ADF_CSR_WR(csr, offs, val); 557 } 558 559 static inline void 560 adf_csr_fetch_and_update(enum operation op, 561 struct resource *csr, 562 size_t offs, 563 unsigned long mask) 564 { 565 switch (op) { 566 case AND: 567 adf_csr_fetch_and_and(csr, offs, mask); 568 break; 569 case OR: 570 adf_csr_fetch_and_or(csr, offs, mask); 571 break; 572 } 573 } 574 575 struct pfvf_stats { 576 struct dentry *stats_file; 577 /* Messages put in CSR */ 578 unsigned int tx; 579 /* Messages read from CSR */ 580 unsigned int rx; 581 /* Interrupt fired but int bit was clear */ 582 unsigned int spurious; 583 /* Block messages sent */ 584 unsigned int blk_tx; 585 /* Block messages received */ 586 unsigned int blk_rx; 587 /* Blocks received with CRC errors */ 588 unsigned int crc_err; 589 /* CSR in use by other side */ 590 unsigned int busy; 591 /* Receiver did not acknowledge */ 592 unsigned int no_ack; 593 /* Collision detected */ 594 unsigned int collision; 595 /* Couldn't send a response */ 596 unsigned int tx_timeout; 597 /* Didn't receive a response */ 598 unsigned int rx_timeout; 599 /* Responses received */ 600 unsigned int rx_rsp; 601 /* Messages re-transmitted */ 602 unsigned int retry; 603 /* Event put timeout */ 604 unsigned int event_timeout; 605 }; 606 607 #define NUM_PFVF_COUNTERS 14 608 609 void adf_get_admin_info(struct admin_info *admin_csrs_info); 610 struct adf_admin_comms { 611 bus_addr_t phy_addr; 612 bus_addr_t const_tbl_addr; 613 bus_addr_t aram_map_phys_addr; 614 bus_addr_t phy_hb_addr; 615 bus_dmamap_t aram_map; 616 bus_dmamap_t const_tbl_map; 617 bus_dmamap_t hb_map; 618 char *virt_addr; 619 char *virt_hb_addr; 620 uint32_t mailbox_offset; 621 struct resource *mailbox_addr; 622 struct sx lock; 623 struct bus_dmamem dma_mem; 624 struct bus_dmamem dma_hb; 625 }; 626 627 struct icp_qat_fw_loader_handle; 628 struct adf_fw_loader_data { 629 struct icp_qat_fw_loader_handle *fw_loader; 630 const struct firmware *uof_fw; 631 const struct firmware *mmp_fw; 632 }; 633 634 struct adf_accel_vf_info { 635 struct adf_accel_dev *accel_dev; 636 struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ 637 u32 vf_nr; 638 bool init; 639 u8 compat_ver; 640 struct pfvf_stats pfvf_counters; 641 }; 642 643 struct adf_fw_versions { 644 u16 fw_version_major; 645 u16 fw_version_minor; 646 u32 fw_version_patch; 647 u8 mmp_version_major; 648 u8 mmp_version_minor; 649 u8 mmp_version_patch; 650 }; 651 652 struct adf_int_timer { 653 struct adf_accel_dev *accel_dev; 654 struct workqueue_struct *timer_irq_wq; 655 struct timer_list timer; 656 u32 timeout_val; 657 u32 int_cnt; 658 bool enabled; 659 }; 660 661 #define ADF_COMPAT_CHECKER_MAX 8 662 typedef int (*adf_iov_compat_checker_t)(struct adf_accel_dev *accel_dev, 663 u8 vf_compat_ver); 664 struct adf_accel_compat_manager { 665 u8 num_chker; 666 adf_iov_compat_checker_t iov_compat_checkers[ADF_COMPAT_CHECKER_MAX]; 667 }; 668 669 struct adf_heartbeat; 670 struct adf_accel_dev { 671 struct adf_hw_aram_info *aram_info; 672 struct adf_accel_unit_info *au_info; 673 struct adf_etr_data *transport; 674 struct adf_hw_device_data *hw_device; 675 struct adf_cfg_device_data *cfg; 676 struct adf_fw_loader_data *fw_loader; 677 struct adf_admin_comms *admin; 678 struct adf_uio_control_accel *accel; 679 struct adf_heartbeat *heartbeat; 680 struct adf_int_timer *int_timer; 681 struct adf_fw_versions fw_versions; 682 unsigned int autoreset_on_error; 683 struct adf_fw_counters_data *fw_counters_data; 684 struct sysctl_oid *debugfs_ae_config; 685 struct list_head crypto_list; 686 atomic_t *ras_counters; 687 unsigned long status; 688 atomic_t ref_count; 689 bus_dma_tag_t dma_tag; 690 struct sysctl_ctx_list sysctl_ctx; 691 struct sysctl_oid *ras_correctable; 692 struct sysctl_oid *ras_uncorrectable; 693 struct sysctl_oid *ras_fatal; 694 struct sysctl_oid *ras_reset; 695 struct sysctl_oid *pke_replay_dbgfile; 696 struct sysctl_oid *misc_error_dbgfile; 697 struct sysctl_oid *fw_version_oid; 698 struct sysctl_oid *mmp_version_oid; 699 struct sysctl_oid *hw_version_oid; 700 struct sysctl_oid *cnv_error_oid; 701 struct list_head list; 702 struct adf_accel_pci accel_pci_dev; 703 struct adf_accel_compat_manager *cm; 704 u8 compat_ver; 705 #ifdef QAT_DISABLE_SAFE_DC_MODE 706 struct sysctl_oid *safe_dc_mode; 707 u8 disable_safe_dc_mode; 708 #endif /* QAT_DISABLE_SAFE_DC_MODE */ 709 union { 710 struct { 711 /* vf_info is non-zero when SR-IOV is init'ed */ 712 struct adf_accel_vf_info *vf_info; 713 int num_vfs; 714 } pf; 715 struct { 716 bool irq_enabled; 717 struct resource *irq; 718 void *cookie; 719 struct task pf2vf_bh_tasklet; 720 struct mutex vf2pf_lock; /* protect CSR access */ 721 struct completion msg_received; 722 struct pfvf_message 723 response; /* temp field holding pf2vf response */ 724 enum ring_reset_result rpreset_sts; 725 struct mutex rpreset_lock; /* protect rpreset_sts */ 726 struct pfvf_stats pfvf_counters; 727 u8 pf_compat_ver; 728 } vf; 729 } u1; 730 bool is_vf; 731 u32 accel_id; 732 void *lac_dev; 733 struct mutex lock; /* protect accel_dev during start/stop e.t.c */ 734 }; 735 #endif 736