Lines Matching refs:handle
44 #define AE(handle, ae) ((handle)->hal_handle->aes[ae]) argument
101 qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_live_ctx() argument
105 AE(handle, ae).live_ctx_mask = ctx_mask; in qat_hal_set_live_ctx()
110 qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, in qat_hal_rd_ae_csr() argument
118 *value = GET_AE_CSR(handle, ae, csr); in qat_hal_rd_ae_csr()
119 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) in qat_hal_rd_ae_csr()
128 qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_ae_csr() argument
136 SET_AE_CSR(handle, ae, csr, value); in qat_hal_wr_ae_csr()
137 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) in qat_hal_wr_ae_csr()
146 qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, in qat_hal_get_wakeup_event() argument
153 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); in qat_hal_get_wakeup_event()
154 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); in qat_hal_get_wakeup_event()
155 qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events); in qat_hal_get_wakeup_event()
156 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); in qat_hal_get_wakeup_event()
160 qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, in qat_hal_wait_cycles() argument
170 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt); in qat_hal_wait_cycles()
174 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr); in qat_hal_wait_cycles()
176 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt); in qat_hal_wait_cycles()
204 qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_ae_ctx_mode() argument
216 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); in qat_hal_set_ae_ctx_mode()
220 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); in qat_hal_set_ae_ctx_mode()
225 qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_ae_nn_mode() argument
231 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_set_ae_nn_mode()
236 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); in qat_hal_set_ae_nn_mode()
243 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); in qat_hal_set_ae_nn_mode()
249 qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_ae_lm_mode() argument
256 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); in qat_hal_set_ae_lm_mode()
281 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); in qat_hal_set_ae_lm_mode()
286 qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_ae_tindex_mode() argument
292 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); in qat_hal_set_ae_tindex_mode()
297 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); in qat_hal_set_ae_tindex_mode()
301 qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_ae_scs_mode() argument
307 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr); in qat_hal_set_ae_scs_mode()
311 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, new_csr); in qat_hal_set_ae_scs_mode()
370 qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle) in qat_hal_get_ae_mask_gen4() argument
375 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { in qat_hal_get_ae_mask_gen4()
376 if (handle->hal_handle->ae_mask & (1 << ae)) { in qat_hal_get_ae_mask_gen4()
385 qat_hal_reset(struct icp_qat_fw_loader_handle *handle) in qat_hal_reset() argument
393 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_reset()
396 if (handle->hal_handle->ae_mask > 0xffff) in qat_hal_reset()
398 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_reset()
407 pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_reset()
409 qat_hal_get_ae_mask_gen4(handle); in qat_hal_reset()
411 handle->hal_handle->slice_mask; in qat_hal_reset()
414 handle->hal_handle->ae_mask & 0xFFFF; in qat_hal_reset()
416 handle->hal_handle->slice_mask & 0x3F; in qat_hal_reset()
420 (handle->hal_handle->ae_mask >> AES_PER_CPP) & in qat_hal_reset()
423 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & in qat_hal_reset()
427 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); in qat_hal_reset()
430 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); in qat_hal_reset()
435 qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_indr_csr() argument
443 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); in qat_hal_wr_indr_csr()
448 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); in qat_hal_wr_indr_csr()
449 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); in qat_hal_wr_indr_csr()
452 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); in qat_hal_wr_indr_csr()
456 qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, in qat_hal_rd_indr_csr() argument
464 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); in qat_hal_rd_indr_csr()
465 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); in qat_hal_rd_indr_csr()
466 qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val); in qat_hal_rd_indr_csr()
467 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); in qat_hal_rd_indr_csr()
471 qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, in qat_hal_put_sig_event() argument
478 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); in qat_hal_put_sig_event()
482 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); in qat_hal_put_sig_event()
483 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); in qat_hal_put_sig_event()
485 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); in qat_hal_put_sig_event()
489 qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, in qat_hal_put_wakeup_event() argument
496 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); in qat_hal_put_wakeup_event()
500 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); in qat_hal_put_wakeup_event()
501 qat_hal_wr_ae_csr(handle, in qat_hal_put_wakeup_event()
506 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); in qat_hal_put_wakeup_event()
510 qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) in qat_hal_check_ae_alive() argument
514 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_check_ae_alive()
517 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_check_ae_alive()
519 qat_hal_rd_ae_csr(handle, in qat_hal_check_ae_alive()
526 qat_hal_rd_ae_csr(handle, in qat_hal_check_ae_alive()
543 qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, in qat_hal_check_ae_active() argument
548 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable); in qat_hal_check_ae_active()
549 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active); in qat_hal_check_ae_active()
558 qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) in qat_hal_reset_timestamp() argument
562 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_reset_timestamp()
565 (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ? in qat_hal_reset_timestamp()
569 misc_ctl = GET_GLB_CSR(handle, misc_ctl_csr); in qat_hal_reset_timestamp()
571 SET_GLB_CSR(handle, in qat_hal_reset_timestamp()
575 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_reset_timestamp()
577 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); in qat_hal_reset_timestamp()
578 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); in qat_hal_reset_timestamp()
581 SET_GLB_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); in qat_hal_reset_timestamp()
590 qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) in qat_hal_init_esram() argument
593 ((uintptr_t)handle->hal_ep_csr_addr_v + ESRAM_AUTO_INIT_CSR_OFFSET); in qat_hal_init_esram()
597 if (pci_get_device(GET_DEV(handle->accel_dev)) != in qat_hal_init_esram()
601 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); in qat_hal_init_esram()
604 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); in qat_hal_init_esram()
607 ADF_CSR_WR(handle->hal_misc_addr_v, csr_addr, csr_val); in qat_hal_init_esram()
609 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); in qat_hal_init_esram()
610 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); in qat_hal_init_esram()
622 qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) in qat_hal_clr_reset() argument
633 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_clr_reset()
635 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_clr_reset()
640 if (handle->hal_handle->ae_mask > 0xffff) in qat_hal_clr_reset()
642 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_clr_reset()
653 pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_clr_reset()
655 qat_hal_get_ae_mask_gen4(handle); in qat_hal_clr_reset()
657 handle->hal_handle->slice_mask; in qat_hal_clr_reset()
660 handle->hal_handle->ae_mask & 0xFFFF; in qat_hal_clr_reset()
662 handle->hal_handle->slice_mask & 0x3F; in qat_hal_clr_reset()
666 (handle->hal_handle->ae_mask >> AES_PER_CPP) & in qat_hal_clr_reset()
669 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & in qat_hal_clr_reset()
673 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); in qat_hal_clr_reset()
677 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); in qat_hal_clr_reset()
680 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); in qat_hal_clr_reset()
685 clk_val[i] = GET_GLB_CSR(handle, clk_csr[i]); in qat_hal_clr_reset()
688 SET_GLB_CSR(handle, clk_csr[i], clk_val[i]); in qat_hal_clr_reset()
690 if (qat_hal_check_ae_alive(handle)) in qat_hal_clr_reset()
694 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_clr_reset()
696 qat_hal_wr_ae_csr(handle, in qat_hal_clr_reset()
700 qat_hal_wr_indr_csr(handle, in qat_hal_clr_reset()
704 handle->hal_handle->upc_mask & in qat_hal_clr_reset()
706 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); in qat_hal_clr_reset()
707 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); in qat_hal_clr_reset()
708 qat_hal_put_wakeup_event(handle, in qat_hal_clr_reset()
712 qat_hal_put_sig_event(handle, in qat_hal_clr_reset()
717 if (qat_hal_init_esram(handle)) in qat_hal_clr_reset()
719 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) in qat_hal_clr_reset()
721 qat_hal_reset_timestamp(handle); in qat_hal_clr_reset()
730 qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, in qat_hal_disable_ctx() argument
736 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); in qat_hal_disable_ctx()
739 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); in qat_hal_disable_ctx()
775 qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_uwords() argument
784 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_wr_uwords()
790 if (ae_in_group + i >= handle->hal_handle->ae_max_num) in qat_hal_wr_uwords()
792 if (qat_hal_check_ae_active(handle, ae_in_group + i)) { in qat_hal_wr_uwords()
801 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); in qat_hal_wr_uwords()
803 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); in qat_hal_wr_uwords()
811 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); in qat_hal_wr_uwords()
812 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); in qat_hal_wr_uwords()
814 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); in qat_hal_wr_uwords()
818 qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_coalesce_uwords() argument
847 qat_hal_wr_uwords(handle, in qat_hal_wr_coalesce_uwords()
854 handle, odd_ae, uaddr / 2, odd_cpy_cnt, odd_uwords); in qat_hal_wr_coalesce_uwords()
860 qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, in qat_hal_enable_ctx() argument
866 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); in qat_hal_enable_ctx()
870 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); in qat_hal_enable_ctx()
874 qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) in qat_hal_clear_xfer() argument
878 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_clear_xfer()
880 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_clear_xfer()
884 handle, ae, 0, ICP_SR_RD_ABS, reg, 0); in qat_hal_clear_xfer()
886 handle, ae, 0, ICP_DR_RD_ABS, reg, 0); in qat_hal_clear_xfer()
892 qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) in qat_hal_clear_gpr() argument
900 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_clear_gpr()
903 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_clear_gpr()
905 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_clear_gpr()
908 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); in qat_hal_clear_gpr()
909 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); in qat_hal_clear_gpr()
911 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_clear_gpr()
914 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); in qat_hal_clear_gpr()
916 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_clear_gpr()
918 qat_hal_wr_uwords(handle, in qat_hal_clear_gpr()
924 qat_hal_wr_uwords(handle, in qat_hal_clear_gpr()
930 qat_hal_wr_indr_csr(handle, in qat_hal_clear_gpr()
934 handle->hal_handle->upc_mask & in qat_hal_clear_gpr()
936 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); in qat_hal_clear_gpr()
937 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); in qat_hal_clear_gpr()
938 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); in qat_hal_clear_gpr()
940 handle, ae, ctx_mask, CTX_SIG_EVENTS_INDIRECT, 0); in qat_hal_clear_gpr()
941 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); in qat_hal_clear_gpr()
942 qat_hal_enable_ctx(handle, ae, ctx_mask); in qat_hal_clear_gpr()
945 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_clear_gpr()
949 ret = qat_hal_wait_cycles(handle, ae, 20, 1); in qat_hal_clear_gpr()
956 qat_hal_disable_ctx(handle, ae, ctx_mask); in qat_hal_clear_gpr()
957 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_clear_gpr()
960 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); in qat_hal_clear_gpr()
961 qat_hal_wr_ae_csr(handle, in qat_hal_clear_gpr()
965 qat_hal_wr_ae_csr(handle, in qat_hal_clear_gpr()
969 qat_hal_wr_indr_csr(handle, in qat_hal_clear_gpr()
973 handle->hal_handle->upc_mask & in qat_hal_clear_gpr()
975 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); in qat_hal_clear_gpr()
976 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); in qat_hal_clear_gpr()
977 qat_hal_put_wakeup_event(handle, in qat_hal_clear_gpr()
981 qat_hal_put_sig_event(handle, in qat_hal_clear_gpr()
990 qat_hal_check_imr(struct icp_qat_fw_loader_handle *handle) in qat_hal_check_imr() argument
992 device_t dev = accel_to_pci_dev(handle->accel_dev); in qat_hal_check_imr()
995 if (pci_get_device(GET_DEV(handle->accel_dev)) != in qat_hal_check_imr()
997 pci_get_device(GET_DEV(handle->accel_dev)) != in qat_hal_check_imr()
1006 if ((reg_val & 0x2) && GET_FCU_CSR(handle, FCU_RAMBASE_ADDR_LO)) in qat_hal_check_imr()
1021 struct icp_qat_fw_loader_handle *handle; in qat_hal_init() local
1031 handle = malloc(sizeof(*handle), M_QAT, M_WAITOK | M_ZERO); in qat_hal_init()
1033 handle->hal_misc_addr_v = misc_bar->virt_addr; in qat_hal_init()
1034 handle->accel_dev = accel_dev; in qat_hal_init()
1035 if (pci_get_device(GET_DEV(handle->accel_dev)) == in qat_hal_init()
1037 IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_init()
1040 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) in qat_hal_init()
1043 handle->hal_sram_addr_v = sram_bar->virt_addr; in qat_hal_init()
1044 handle->hal_sram_offset = sram_offset; in qat_hal_init()
1045 handle->hal_sram_size = sram_bar->size; in qat_hal_init()
1047 GET_CSR_OFFSET(pci_get_device(GET_DEV(handle->accel_dev)), in qat_hal_init()
1051 handle->hal_cap_g_ctl_csr_addr_v = cap_offset; in qat_hal_init()
1052 handle->hal_cap_ae_xfer_csr_addr_v = ae_offset; in qat_hal_init()
1053 handle->hal_ep_csr_addr_v = ep_offset; in qat_hal_init()
1054 handle->hal_cap_ae_local_csr_addr_v = in qat_hal_init()
1055 ((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + in qat_hal_init()
1057 handle->fw_auth = (pci_get_device(GET_DEV(handle->accel_dev)) == in qat_hal_init()
1061 if (handle->fw_auth && qat_hal_check_imr(handle)) { in qat_hal_init()
1067 handle->hal_handle = in qat_hal_init()
1068 malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO); in qat_hal_init()
1069 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; in qat_hal_init()
1070 handle->hal_handle->ae_mask = hw_data->ae_mask; in qat_hal_init()
1071 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; in qat_hal_init()
1072 handle->hal_handle->slice_mask = hw_data->accel_mask; in qat_hal_init()
1073 handle->cfg_ae_mask = 0xFFFFFFFF; in qat_hal_init()
1075 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_init()
1076 handle->hal_handle->upc_mask = 0xffff; in qat_hal_init()
1077 handle->hal_handle->max_ustore = 0x2000; in qat_hal_init()
1079 handle->hal_handle->upc_mask = 0x1ffff; in qat_hal_init()
1080 handle->hal_handle->max_ustore = 0x4000; in qat_hal_init()
1087 handle->hal_handle->aes[ae].free_addr = 0; in qat_hal_init()
1088 handle->hal_handle->aes[ae].free_size = in qat_hal_init()
1089 handle->hal_handle->max_ustore; in qat_hal_init()
1090 handle->hal_handle->aes[ae].ustore_size = in qat_hal_init()
1091 handle->hal_handle->max_ustore; in qat_hal_init()
1092 handle->hal_handle->aes[ae].live_ctx_mask = in qat_hal_init()
1096 handle->hal_handle->ae_max_num = max_en_ae_id + 1; in qat_hal_init()
1098 if (qat_hal_clr_reset(handle)) { in qat_hal_init()
1103 qat_hal_clear_xfer(handle); in qat_hal_init()
1104 if (!handle->fw_auth) { in qat_hal_init()
1105 if (qat_hal_clear_gpr(handle)) { in qat_hal_init()
1112 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_init()
1116 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); in qat_hal_init()
1118 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); in qat_hal_init()
1120 accel_dev->fw_loader->fw_loader = handle; in qat_hal_init()
1124 free(handle->hal_handle, M_QAT); in qat_hal_init()
1126 free(handle, M_QAT); in qat_hal_init()
1131 qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) in qat_hal_deinit() argument
1133 if (!handle) in qat_hal_deinit()
1135 free(handle->hal_handle, M_QAT); in qat_hal_deinit()
1136 free(handle, M_QAT); in qat_hal_deinit()
1140 qat_hal_start(struct icp_qat_fw_loader_handle *handle) in qat_hal_start() argument
1146 unsigned long ae_mask = handle->hal_handle->ae_mask; in qat_hal_start()
1149 if (handle->fw_auth) { in qat_hal_start()
1150 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_start()
1155 pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_start()
1163 SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); in qat_hal_start()
1166 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); in qat_hal_start()
1175 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) in qat_hal_start()
1177 qat_hal_put_wakeup_event(handle, in qat_hal_start()
1182 handle->accel_dev))) ? in qat_hal_start()
1185 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); in qat_hal_start()
1193 qat_hal_stop(struct icp_qat_fw_loader_handle *handle, in qat_hal_stop() argument
1197 if (!handle->fw_auth) in qat_hal_stop()
1198 qat_hal_disable_ctx(handle, ae, ctx_mask); in qat_hal_stop()
1202 qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, in qat_hal_set_pc() argument
1207 qat_hal_wr_indr_csr(handle, in qat_hal_set_pc()
1211 handle->hal_handle->upc_mask & upc); in qat_hal_set_pc()
1215 qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, in qat_hal_get_uwords() argument
1225 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control); in qat_hal_get_uwords()
1228 qat_hal_wr_ae_csr(handle, in qat_hal_get_uwords()
1232 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); in qat_hal_get_uwords()
1235 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); in qat_hal_get_uwords()
1237 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo); in qat_hal_get_uwords()
1238 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi); in qat_hal_get_uwords()
1244 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); in qat_hal_get_uwords()
1245 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); in qat_hal_get_uwords()
1249 qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_umem() argument
1257 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); in qat_hal_wr_umem()
1259 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); in qat_hal_wr_umem()
1269 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); in qat_hal_wr_umem()
1270 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); in qat_hal_wr_umem()
1272 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); in qat_hal_wr_umem()
1277 qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, in qat_hal_exec_micro_inst() argument
1299 if (inst_num > handle->hal_handle->max_ustore || !micro_inst) { in qat_hal_exec_micro_inst()
1304 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0); in qat_hal_exec_micro_inst()
1305 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1); in qat_hal_exec_micro_inst()
1307 handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0); in qat_hal_exec_micro_inst()
1309 handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1); in qat_hal_exec_micro_inst()
1310 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_exec_micro_inst()
1312 handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2); in qat_hal_exec_micro_inst()
1314 handle, ae, ctx, LM_ADDR_3_INDIRECT, &ind_lm_addr3); in qat_hal_exec_micro_inst()
1315 qat_hal_rd_indr_csr(handle, in qat_hal_exec_micro_inst()
1320 qat_hal_rd_indr_csr(handle, in qat_hal_exec_micro_inst()
1326 handle, ae, ctx, INDIRECT_T_INDEX, &ind_t_index); in qat_hal_exec_micro_inst()
1327 qat_hal_rd_indr_csr(handle, in qat_hal_exec_micro_inst()
1333 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_exec_micro_inst()
1336 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); in qat_hal_exec_micro_inst()
1338 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); in qat_hal_exec_micro_inst()
1339 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); in qat_hal_exec_micro_inst()
1340 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc); in qat_hal_exec_micro_inst()
1341 savpc = (savpc & handle->hal_handle->upc_mask) >> 0; in qat_hal_exec_micro_inst()
1342 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_exec_micro_inst()
1344 qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc); in qat_hal_exec_micro_inst()
1345 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); in qat_hal_exec_micro_inst()
1346 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl); in qat_hal_exec_micro_inst()
1348 handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, &ind_cnt_sig); in qat_hal_exec_micro_inst()
1349 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig); in qat_hal_exec_micro_inst()
1350 qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig); in qat_hal_exec_micro_inst()
1352 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); in qat_hal_exec_micro_inst()
1353 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); in qat_hal_exec_micro_inst()
1354 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); in qat_hal_exec_micro_inst()
1355 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); in qat_hal_exec_micro_inst()
1357 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); in qat_hal_exec_micro_inst()
1358 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); in qat_hal_exec_micro_inst()
1359 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); in qat_hal_exec_micro_inst()
1360 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); in qat_hal_exec_micro_inst()
1361 qat_hal_enable_ctx(handle, ae, (1 << ctx)); in qat_hal_exec_micro_inst()
1363 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) in qat_hal_exec_micro_inst()
1369 handle, ae, ctx, CTX_STS_INDIRECT, &ctx_status); in qat_hal_exec_micro_inst()
1370 *endpc = ctx_status & handle->hal_handle->upc_mask; in qat_hal_exec_micro_inst()
1373 qat_hal_disable_ctx(handle, ae, (1 << ctx)); in qat_hal_exec_micro_inst()
1375 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); in qat_hal_exec_micro_inst()
1376 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); in qat_hal_exec_micro_inst()
1377 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1381 handle->hal_handle->upc_mask & savpc); in qat_hal_exec_micro_inst()
1382 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_exec_micro_inst()
1385 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); in qat_hal_exec_micro_inst()
1386 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); in qat_hal_exec_micro_inst()
1387 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); in qat_hal_exec_micro_inst()
1388 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); in qat_hal_exec_micro_inst()
1390 handle, ae, (1 << ctx), LM_ADDR_0_INDIRECT, ind_lm_addr0); in qat_hal_exec_micro_inst()
1392 handle, ae, (1 << ctx), LM_ADDR_1_INDIRECT, ind_lm_addr1); in qat_hal_exec_micro_inst()
1393 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1398 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1403 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_exec_micro_inst()
1405 handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2); in qat_hal_exec_micro_inst()
1407 handle, ae, (1 << ctx), LM_ADDR_3_INDIRECT, ind_lm_addr3); in qat_hal_exec_micro_inst()
1408 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1413 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1419 handle, ae, (1 << ctx), INDIRECT_T_INDEX, ind_t_index); in qat_hal_exec_micro_inst()
1420 qat_hal_wr_indr_csr(handle, in qat_hal_exec_micro_inst()
1427 handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); in qat_hal_exec_micro_inst()
1429 handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, ind_sig); in qat_hal_exec_micro_inst()
1430 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); in qat_hal_exec_micro_inst()
1431 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); in qat_hal_exec_micro_inst()
1437 qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, in qat_hal_rd_rel_reg() argument
1465 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_rd_rel_reg()
1468 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); in qat_hal_rd_rel_reg()
1469 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); in qat_hal_rd_rel_reg()
1470 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl); in qat_hal_rd_rel_reg()
1471 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_rd_rel_reg()
1474 qat_hal_wr_ae_csr(handle, in qat_hal_rd_rel_reg()
1478 qat_hal_get_uwords(handle, ae, 0, 1, &savuword); in qat_hal_rd_rel_reg()
1479 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); in qat_hal_rd_rel_reg()
1480 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); in qat_hal_rd_rel_reg()
1482 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); in qat_hal_rd_rel_reg()
1486 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); in qat_hal_rd_rel_reg()
1487 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); in qat_hal_rd_rel_reg()
1488 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); in qat_hal_rd_rel_reg()
1490 qat_hal_wait_cycles(handle, ae, 0x8, 0); in qat_hal_rd_rel_reg()
1496 qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data); in qat_hal_rd_rel_reg()
1497 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); in qat_hal_rd_rel_reg()
1498 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); in qat_hal_rd_rel_reg()
1500 qat_hal_wr_ae_csr(handle, in qat_hal_rd_rel_reg()
1504 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); in qat_hal_rd_rel_reg()
1505 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); in qat_hal_rd_rel_reg()
1508 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); in qat_hal_rd_rel_reg()
1509 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); in qat_hal_rd_rel_reg()
1515 qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, in qat_hal_wr_rel_reg() argument
1559 handle, ae, ctx, insts, num_inst, code_off, num_inst * 0x5, NULL); in qat_hal_wr_rel_reg()
1604 qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, in qat_hal_exec_micro_init_lm() argument
1616 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); in qat_hal_exec_micro_init_lm()
1617 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); in qat_hal_exec_micro_init_lm()
1618 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); in qat_hal_exec_micro_init_lm()
1619 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); in qat_hal_exec_micro_init_lm()
1620 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); in qat_hal_exec_micro_init_lm()
1624 handle, ae, ctx, micro_inst, inst_num, 1, inst_num * 0x5, NULL); in qat_hal_exec_micro_init_lm()
1627 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); in qat_hal_exec_micro_init_lm()
1628 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); in qat_hal_exec_micro_init_lm()
1629 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); in qat_hal_exec_micro_init_lm()
1630 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); in qat_hal_exec_micro_init_lm()
1631 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); in qat_hal_exec_micro_init_lm()
1637 qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, in qat_hal_batch_wr_lm() argument
1652 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) in qat_hal_batch_wr_lm()
1653 alloc_inst_size = handle->hal_handle->max_ustore; in qat_hal_batch_wr_lm()
1672 stat = qat_hal_exec_micro_init_lm(handle, in qat_hal_batch_wr_lm()
1684 qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, in qat_hal_put_rel_rd_xfer() argument
1697 status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_put_rel_rd_xfer()
1714 SET_AE_XFER(handle, ae, reg_addr, val); in qat_hal_put_rel_rd_xfer()
1718 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); in qat_hal_put_rel_rd_xfer()
1728 qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, in qat_hal_put_rel_wr_xfer() argument
1748 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_put_rel_wr_xfer()
1765 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); in qat_hal_put_rel_wr_xfer()
1780 handle, ae, ctx, micro_inst, num_inst, code_off, dly, NULL); in qat_hal_put_rel_wr_xfer()
1781 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); in qat_hal_put_rel_wr_xfer()
1786 qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, in qat_hal_put_rel_nn() argument
1795 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_put_rel_nn()
1797 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); in qat_hal_put_rel_nn()
1799 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); in qat_hal_put_rel_nn()
1800 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); in qat_hal_put_rel_nn()
1805 qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle *handle, in qat_hal_convert_abs_to_rel() argument
1813 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); in qat_hal_convert_abs_to_rel()
1827 qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, in qat_hal_init_gpr() argument
1845 handle, ae, reg_num, ®, &ctx); in qat_hal_init_gpr()
1853 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); in qat_hal_init_gpr()
1864 qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, in qat_hal_init_wr_xfer() argument
1882 handle, ae, reg_num, ®, &ctx); in qat_hal_init_wr_xfer()
1891 handle, ae, ctx, type, reg, regdata); in qat_hal_init_wr_xfer()
1902 qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, in qat_hal_init_rd_xfer() argument
1920 handle, ae, reg_num, ®, &ctx); in qat_hal_init_rd_xfer()
1929 handle, ae, ctx, type, reg, regdata); in qat_hal_init_rd_xfer()
1940 qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, in qat_hal_init_nn() argument
1949 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { in qat_hal_init_nn()
1959 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); in qat_hal_init_nn()