1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #ifndef ADF_ACCEL_DEVICES_H_ 4 #define ADF_ACCEL_DEVICES_H_ 5 #include <linux/interrupt.h> 6 #include <linux/module.h> 7 #include <linux/list.h> 8 #include <linux/io.h> 9 #include <linux/pci.h> 10 #include <linux/ratelimit.h> 11 #include <linux/types.h> 12 #include <linux/qat/qat_mig_dev.h> 13 #include <linux/wordpart.h> 14 #include "adf_anti_rb.h" 15 #include "adf_cfg_common.h" 16 #include "adf_dc.h" 17 #include "adf_kpt.h" 18 #include "adf_rl.h" 19 #include "adf_telemetry.h" 20 #include "adf_pfvf_msg.h" 21 #include "icp_qat_hw.h" 22 23 #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" 24 #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" 25 #define ADF_C62X_DEVICE_NAME "c6xx" 26 #define ADF_C62XVF_DEVICE_NAME "c6xxvf" 27 #define ADF_C3XXX_DEVICE_NAME "c3xxx" 28 #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" 29 #define ADF_4XXX_DEVICE_NAME "4xxx" 30 #define ADF_420XX_DEVICE_NAME "420xx" 31 #define ADF_6XXX_DEVICE_NAME "6xxx" 32 #define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 33 #define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 34 #define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 35 #define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 36 #define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949 37 38 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 39 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C 40 #define ADF_DEVICE_FUSECTL_MASK 0x80000000 41 #define ADF_PCI_MAX_BARS 3 42 #define ADF_DEVICE_NAME_LENGTH 32 43 #define ADF_ETR_MAX_RINGS_PER_BANK 16 44 #define ADF_MAX_MSIX_VECTOR_NAME 48 45 #define ADF_DEVICE_NAME_PREFIX "qat_" 46 47 enum adf_accel_capabilities { 48 ADF_ACCEL_CAPABILITIES_NULL = 0, 49 ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, 50 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, 51 ADF_ACCEL_CAPABILITIES_CIPHER = 4, 52 ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, 53 ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, 54 ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64, 55 ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 56 }; 57 58 enum adf_accel_capabilities_ext { 59 ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S = BIT(0), 60 ADF_ACCEL_CAPABILITIES_EXT_ZSTD = BIT(1), 61 }; 62 63 enum adf_fuses { 64 ADF_FUSECTL0, 65 ADF_FUSECTL1, 66 ADF_FUSECTL2, 67 ADF_FUSECTL3, 68 ADF_FUSECTL4, 69 ADF_FUSECTL5, 70 ADF_MAX_FUSES 71 }; 72 73 struct adf_bar { 74 resource_size_t base_addr; 75 void __iomem *virt_addr; 76 resource_size_t size; 77 }; 78 79 struct adf_irq { 80 bool enabled; 81 char name[ADF_MAX_MSIX_VECTOR_NAME]; 82 }; 83 84 struct adf_accel_msix { 85 struct adf_irq *irqs; 86 u32 num_entries; 87 }; 88 89 struct adf_accel_pci { 90 struct pci_dev *pci_dev; 91 struct adf_accel_msix msix_entries; 92 struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; 93 u8 revid; 94 u8 sku; 95 }; 96 97 enum dev_state { 98 DEV_DOWN = 0, 99 DEV_UP 100 }; 101 102 enum dev_sku_info { 103 DEV_SKU_1 = 0, 104 DEV_SKU_2, 105 DEV_SKU_3, 106 DEV_SKU_4, 107 DEV_SKU_VF, 108 DEV_SKU_UNKNOWN, 109 }; 110 111 enum ras_errors { 112 ADF_RAS_CORR, 113 ADF_RAS_UNCORR, 114 ADF_RAS_FATAL, 115 ADF_RAS_ERRORS, 116 }; 117 118 struct adf_error_counters { 119 atomic_t counter[ADF_RAS_ERRORS]; 120 bool sysfs_added; 121 bool enabled; 122 }; 123 124 static inline const char *get_sku_info(enum dev_sku_info info) 125 { 126 switch (info) { 127 case DEV_SKU_1: 128 return "SKU1"; 129 case DEV_SKU_2: 130 return "SKU2"; 131 case DEV_SKU_3: 132 return "SKU3"; 133 case DEV_SKU_4: 134 return "SKU4"; 135 case DEV_SKU_VF: 136 return "SKUVF"; 137 case DEV_SKU_UNKNOWN: 138 default: 139 break; 140 } 141 return "Unknown SKU"; 142 } 143 144 struct adf_hw_device_class { 145 const char *name; 146 const enum adf_device_type type; 147 u32 instances; 148 }; 149 150 struct arb_info { 151 u32 arb_cfg; 152 u32 arb_offset; 153 u32 wt2sam_offset; 154 }; 155 156 struct admin_info { 157 u32 admin_msg_ur; 158 u32 admin_msg_lr; 159 u32 mailbox_offset; 160 }; 161 162 struct adf_bank_state; 163 164 struct adf_hw_csr_ops { 165 u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); 166 u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, 167 u32 ring); 168 void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, 169 u32 ring, u32 value); 170 u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, 171 u32 ring); 172 void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, 173 u32 ring, u32 value); 174 u32 (*read_csr_stat)(void __iomem *csr_base_addr, u32 bank); 175 u32 (*read_csr_uo_stat)(void __iomem *csr_base_addr, u32 bank); 176 u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); 177 u32 (*read_csr_ne_stat)(void __iomem *csr_base_addr, u32 bank); 178 u32 (*read_csr_nf_stat)(void __iomem *csr_base_addr, u32 bank); 179 u32 (*read_csr_f_stat)(void __iomem *csr_base_addr, u32 bank); 180 u32 (*read_csr_c_stat)(void __iomem *csr_base_addr, u32 bank); 181 u32 (*read_csr_exp_stat)(void __iomem *csr_base_addr, u32 bank); 182 u32 (*read_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank); 183 void (*write_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank, 184 u32 value); 185 u32 (*read_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, 186 u32 ring); 187 void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, 188 u32 ring, u32 value); 189 dma_addr_t (*read_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, 190 u32 ring); 191 void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, 192 u32 ring, dma_addr_t addr); 193 u32 (*read_csr_int_en)(void __iomem *csr_base_addr, u32 bank); 194 void (*write_csr_int_en)(void __iomem *csr_base_addr, u32 bank, 195 u32 value); 196 u32 (*read_csr_int_flag)(void __iomem *csr_base_addr, u32 bank); 197 void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, 198 u32 value); 199 u32 (*read_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); 200 void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); 201 void (*write_csr_int_srcsel_w_val)(void __iomem *csr_base_addr, 202 u32 bank, u32 value); 203 u32 (*read_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank); 204 void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, 205 u32 value); 206 u32 (*read_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank); 207 void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, 208 u32 value); 209 u32 (*read_csr_int_flag_and_col)(void __iomem *csr_base_addr, 210 u32 bank); 211 void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, 212 u32 bank, u32 value); 213 u32 (*read_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank); 214 void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, 215 u32 value); 216 u32 (*get_int_col_ctl_enable_mask)(void); 217 }; 218 219 struct adf_cfg_device_data; 220 struct adf_accel_dev; 221 struct adf_etr_data; 222 struct adf_etr_ring_data; 223 224 struct adf_ras_ops { 225 void (*enable_ras_errors)(struct adf_accel_dev *accel_dev); 226 void (*disable_ras_errors)(struct adf_accel_dev *accel_dev); 227 bool (*handle_interrupt)(struct adf_accel_dev *accel_dev, 228 bool *reset_required); 229 }; 230 231 struct adf_pfvf_ops { 232 int (*enable_comms)(struct adf_accel_dev *accel_dev); 233 u32 (*get_pf2vf_offset)(u32 i); 234 u32 (*get_vf2pf_offset)(u32 i); 235 void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); 236 void (*disable_all_vf2pf_interrupts)(void __iomem *pmisc_addr); 237 u32 (*disable_pending_vf2pf_interrupts)(void __iomem *pmisc_addr); 238 int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 239 u32 pfvf_offset, struct mutex *csr_lock); 240 struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, 241 u32 pfvf_offset, u8 compat_ver); 242 }; 243 244 struct adf_dc_ops { 245 int (*build_comp_block)(void *ctx, enum adf_dc_algo algo); 246 int (*build_decomp_block)(void *ctx, enum adf_dc_algo algo); 247 }; 248 249 struct qat_migdev_ops { 250 int (*init)(struct qat_mig_dev *mdev); 251 void (*cleanup)(struct qat_mig_dev *mdev); 252 void (*reset)(struct qat_mig_dev *mdev); 253 int (*open)(struct qat_mig_dev *mdev); 254 void (*close)(struct qat_mig_dev *mdev); 255 int (*suspend)(struct qat_mig_dev *mdev); 256 int (*resume)(struct qat_mig_dev *mdev); 257 int (*save_state)(struct qat_mig_dev *mdev); 258 int (*save_setup)(struct qat_mig_dev *mdev); 259 int (*load_state)(struct qat_mig_dev *mdev); 260 int (*load_setup)(struct qat_mig_dev *mdev, int size); 261 }; 262 263 struct adf_dev_err_mask { 264 u32 cppagentcmdpar_mask; 265 u32 parerr_ath_cph_mask; 266 u32 parerr_cpr_xlt_mask; 267 u32 parerr_dcpr_ucs_mask; 268 u32 parerr_pke_mask; 269 u32 parerr_wat_wcp_mask; 270 u32 ssmfeatren_mask; 271 }; 272 273 struct adf_hw_device_data { 274 struct adf_hw_device_class *dev_class; 275 u32 (*get_accel_mask)(struct adf_hw_device_data *self); 276 u32 (*get_ae_mask)(struct adf_hw_device_data *self); 277 u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); 278 u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); 279 u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); 280 u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); 281 u32 (*get_num_aes)(struct adf_hw_device_data *self); 282 void (*get_arb_info)(struct arb_info *arb_csrs_info); 283 void (*get_admin_info)(struct admin_info *admin_csrs_info); 284 enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); 285 u16 (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev); 286 int (*alloc_irq)(struct adf_accel_dev *accel_dev); 287 void (*free_irq)(struct adf_accel_dev *accel_dev); 288 void (*enable_error_correction)(struct adf_accel_dev *accel_dev); 289 int (*init_admin_comms)(struct adf_accel_dev *accel_dev); 290 void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); 291 int (*send_admin_init)(struct adf_accel_dev *accel_dev); 292 int (*start_timer)(struct adf_accel_dev *accel_dev); 293 void (*stop_timer)(struct adf_accel_dev *accel_dev); 294 void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev); 295 uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); 296 int (*measure_clock)(struct adf_accel_dev *accel_dev); 297 int (*init_arb)(struct adf_accel_dev *accel_dev); 298 void (*exit_arb)(struct adf_accel_dev *accel_dev); 299 const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); 300 int (*init_device)(struct adf_accel_dev *accel_dev); 301 int (*enable_pm)(struct adf_accel_dev *accel_dev); 302 bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev); 303 void (*disable_iov)(struct adf_accel_dev *accel_dev); 304 void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, 305 bool enable); 306 void (*enable_ints)(struct adf_accel_dev *accel_dev); 307 void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); 308 int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); 309 int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number, 310 struct adf_bank_state *state); 311 int (*bank_state_restore)(struct adf_accel_dev *accel_dev, 312 u32 bank_number, struct adf_bank_state *state); 313 void (*reset_device)(struct adf_accel_dev *accel_dev); 314 void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); 315 const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); 316 u32 (*uof_get_num_objs)(struct adf_accel_dev *accel_dev); 317 int (*uof_get_obj_type)(struct adf_accel_dev *accel_dev, u32 obj_num); 318 u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); 319 int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask); 320 u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); 321 int (*dev_config)(struct adf_accel_dev *accel_dev); 322 bool (*services_supported)(unsigned long mask); 323 u32 (*get_svc_slice_cnt)(struct adf_accel_dev *accel_dev, 324 enum adf_base_services svc); 325 struct adf_pfvf_ops pfvf_ops; 326 struct adf_hw_csr_ops csr_ops; 327 struct adf_dc_ops dc_ops; 328 struct adf_ras_ops ras_ops; 329 struct adf_dev_err_mask dev_err_mask; 330 struct adf_rl_hw_data rl_data; 331 struct adf_tl_hw_data tl_data; 332 struct adf_anti_rb_hw_data anti_rb_data; 333 struct adf_kpt_hw_data kpt_data; 334 struct qat_migdev_ops vfmig_ops; 335 const char *fw_name; 336 const char *fw_mmp_name; 337 u32 fuses[ADF_MAX_FUSES]; 338 u32 straps; 339 u32 accel_capabilities_mask; 340 u32 accel_capabilities_ext_mask; 341 u32 extended_dc_capabilities; 342 u16 fw_capabilities; 343 u32 clock_frequency; 344 u32 instance_id; 345 u16 accel_mask; 346 u32 ae_mask; 347 u32 admin_ae_mask; 348 u16 tx_rings_mask; 349 u16 ring_to_svc_map; 350 u32 thd_to_arb_map[ICP_QAT_HW_AE_DELIMITER]; 351 u8 tx_rx_gap; 352 u8 num_banks; 353 u16 num_banks_per_vf; 354 u8 num_rings_per_bank; 355 u8 num_accel; 356 u8 num_logical_accel; 357 u8 num_engines; 358 u32 num_hb_ctrs; 359 u8 num_rps; 360 }; 361 362 /* CSR write macro */ 363 #define ADF_CSR_WR(csr_base, csr_offset, val) \ 364 __raw_writel(val, csr_base + csr_offset) 365 /* 366 * CSR write macro to handle cases where the high and low 367 * offsets are sparsely located. 368 */ 369 #define ADF_CSR_WR64_LO_HI(csr_base, csr_low_offset, csr_high_offset, val) \ 370 do { \ 371 ADF_CSR_WR(csr_base, csr_low_offset, lower_32_bits(val)); \ 372 ADF_CSR_WR(csr_base, csr_high_offset, upper_32_bits(val)); \ 373 } while (0) 374 375 /* CSR read macro */ 376 #define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) 377 378 #define ADF_CFG_NUM_SERVICES 4 379 #define ADF_SRV_TYPE_BIT_LEN 3 380 #define ADF_SRV_TYPE_MASK 0x7 381 #define ADF_AE_ADMIN_THREAD 7 382 #define ADF_NUM_THREADS_PER_AE 8 383 #define ADF_NUM_PKE_STRAND 2 384 #define ADF_AE_STRAND0_THREAD 8 385 #define ADF_AE_STRAND1_THREAD 9 386 387 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) 388 #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) 389 #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) 390 #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) 391 #define GET_NUM_RINGS_PER_BANK(accel_dev) \ 392 GET_HW_DATA(accel_dev)->num_rings_per_bank 393 #define GET_SRV_TYPE(accel_dev, idx) \ 394 (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \ 395 & ADF_SRV_TYPE_MASK) 396 #define GET_ERR_MASK(accel_dev) (&GET_HW_DATA(accel_dev)->dev_err_mask) 397 #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) 398 #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) 399 #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) 400 #define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) 401 #define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops) 402 #define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data 403 #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev 404 405 struct adf_admin_comms; 406 struct icp_qat_fw_loader_handle; 407 struct adf_fw_loader_data { 408 struct icp_qat_fw_loader_handle *fw_loader; 409 const struct firmware *uof_fw; 410 const struct firmware *mmp_fw; 411 }; 412 413 struct adf_accel_vf_info { 414 struct adf_accel_dev *accel_dev; 415 struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ 416 struct mutex pfvf_mig_lock; /* protects PFVF state for migration */ 417 struct ratelimit_state vf2pf_ratelimit; 418 u32 vf_nr; 419 bool init; 420 bool restarting; 421 u8 vf_compat_ver; 422 /* 423 * Private area used for device migration. 424 * Memory allocation and free is managed by migration driver. 425 */ 426 void *mig_priv; 427 }; 428 429 struct adf_dc_data { 430 u8 *ovf_buff; 431 size_t ovf_buff_sz; 432 dma_addr_t ovf_buff_p; 433 }; 434 435 struct adf_pm { 436 struct dentry *debugfs_pm_status; 437 bool present; 438 int idle_irq_counters; 439 int throttle_irq_counters; 440 int fw_irq_counters; 441 int host_ack_counter; 442 int host_nack_counter; 443 ssize_t (*print_pm_status)(struct adf_accel_dev *accel_dev, 444 char __user *buf, size_t count, loff_t *pos); 445 }; 446 447 struct adf_sysfs { 448 int ring_num; 449 struct rw_semaphore lock; /* protects access to the fields in this struct */ 450 }; 451 452 struct adf_accel_dev { 453 struct adf_etr_data *transport; 454 struct adf_hw_device_data *hw_device; 455 struct adf_cfg_device_data *cfg; 456 struct adf_fw_loader_data *fw_loader; 457 struct adf_admin_comms *admin; 458 struct adf_telemetry *telemetry; 459 struct adf_dc_data *dc_data; 460 struct adf_pm power_management; 461 struct list_head crypto_list; 462 struct list_head compression_list; 463 unsigned long status; 464 atomic_t ref_count; 465 struct dentry *debugfs_dir; 466 struct dentry *fw_cntr_dbgfile; 467 struct dentry *cnv_dbgfile; 468 struct list_head list; 469 struct module *owner; 470 struct adf_accel_pci accel_pci_dev; 471 struct adf_timer *timer; 472 struct adf_heartbeat *heartbeat; 473 struct adf_rl *rate_limiting; 474 struct adf_sysfs sysfs; 475 union { 476 struct { 477 /* protects VF2PF interrupts access */ 478 spinlock_t vf2pf_ints_lock; 479 /* prevents VF2PF handling from racing with VF state teardown */ 480 bool vf2pf_disabled; 481 /* vf_info is non-zero when SR-IOV is init'ed */ 482 struct adf_accel_vf_info *vf_info; 483 } pf; 484 struct { 485 bool irq_enabled; 486 char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; 487 struct tasklet_struct pf2vf_bh_tasklet; 488 struct mutex vf2pf_lock; /* protect CSR access */ 489 struct completion msg_received; 490 struct pfvf_message response; /* temp field holding pf2vf response */ 491 u8 pf_compat_ver; 492 } vf; 493 }; 494 struct adf_error_counters ras_errors; 495 struct mutex state_lock; /* protect state of the device */ 496 bool is_vf; 497 bool autoreset_on_error; 498 u32 accel_id; 499 }; 500 #endif 501