194 static inline u16 iwl_get_closed_rb_stts(struct iwl_trans *trans,  in iwl_get_closed_rb_stts()  argument
197 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {  in iwl_get_closed_rb_stts()
439 	struct iwl_trans *trans;  member
524 IWL_TRANS_GET_PCIE_TRANS(struct iwl_trans *trans)  in IWL_TRANS_GET_PCIE_TRANS()  argument
526 	return (void *)trans->trans_specific;  in IWL_TRANS_GET_PCIE_TRANS()
529 static inline void iwl_pcie_clear_irq(struct iwl_trans *trans, int queue)  in iwl_pcie_clear_irq()  argument
539 	iwl_write32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(queue));  in iwl_pcie_clear_irq()
557 void iwl_trans_pcie_free(struct iwl_trans *trans);
561 bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
562 #define _iwl_trans_pcie_grab_nic_access(trans)			\  argument
564 		    likely(__iwl_trans_pcie_grab_nic_access(trans)))
572 int iwl_pcie_rx_init(struct iwl_trans *trans);
573 int iwl_pcie_gen2_rx_init(struct iwl_trans *trans);
578 int iwl_pcie_rx_stop(struct iwl_trans *trans);
579 void iwl_pcie_rx_free(struct iwl_trans *trans);
580 void iwl_pcie_free_rbs_pool(struct iwl_trans *trans);
582 void iwl_pcie_rx_napi_sync(struct iwl_trans *trans);
583 void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority,
590 int iwl_pcie_alloc_ict(struct iwl_trans *trans);
591 void iwl_pcie_free_ict(struct iwl_trans *trans);
592 void iwl_pcie_reset_ict(struct iwl_trans *trans);
593 void iwl_pcie_disable_ict(struct iwl_trans *trans);
625 int iwl_pcie_tx_init(struct iwl_trans *trans);
626 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr);
627 int iwl_pcie_tx_stop(struct iwl_trans *trans);
628 void iwl_pcie_tx_free(struct iwl_trans *trans);
629 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int queue, u16 ssn,
632 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int queue,
634 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
636 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
638 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans);
639 void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
641 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans);
642 int iwl_pcie_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq,
647 struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
652 void iwl_pcie_free_tso_pages(struct iwl_trans *trans, struct sk_buff *skb,
677 static inline void *iwl_txq_get_tfd(struct iwl_trans *trans,  in iwl_txq_get_tfd()  argument
680 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_txq_get_tfd()
682 	if (trans->trans_cfg->gen2)  in iwl_txq_get_tfd()
698 int iwl_txq_space(struct iwl_trans *trans, const struct iwl_txq *q);
700 static inline void iwl_txq_stop(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_txq_stop()  argument
702 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_txq_stop()
705 		iwl_op_mode_queue_full(trans->op_mode, txq->id);  in iwl_txq_stop()
706 		IWL_DEBUG_TX_QUEUES(trans, "Stop hwq %d\n", txq->id);  in iwl_txq_stop()
708 		IWL_DEBUG_TX_QUEUES(trans, "hwq %d already stopped\n",  in iwl_txq_stop()
718 static inline int iwl_txq_inc_wrap(struct iwl_trans *trans, int index)  in iwl_txq_inc_wrap()  argument
721 		(trans->trans_cfg->base_params->max_tfd_queue_size - 1);  in iwl_txq_inc_wrap()
729 static inline int iwl_txq_dec_wrap(struct iwl_trans *trans, int index)  in iwl_txq_dec_wrap()  argument
732 		(trans->trans_cfg->base_params->max_tfd_queue_size - 1);  in iwl_txq_dec_wrap()
735 void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq);
738 iwl_trans_pcie_wake_queue(struct iwl_trans *trans, struct iwl_txq *txq)  in iwl_trans_pcie_wake_queue()  argument
740 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_wake_queue()
743 		IWL_DEBUG_TX_QUEUES(trans, "Wake hwq %d\n", txq->id);  in iwl_trans_pcie_wake_queue()
744 		iwl_op_mode_queue_not_full(trans->op_mode, txq->id);  in iwl_trans_pcie_wake_queue()
748 int iwl_txq_gen2_set_tb(struct iwl_trans *trans,
752 static inline void iwl_txq_set_tfd_invalid_gen2(struct iwl_trans *trans,  in iwl_txq_set_tfd_invalid_gen2()  argument
757 	iwl_txq_gen2_set_tb(trans, tfd, trans->invalid_tx_cmd.dma,  in iwl_txq_set_tfd_invalid_gen2()
758 			    trans->invalid_tx_cmd.size);  in iwl_txq_set_tfd_invalid_gen2()
761 void iwl_txq_gen2_tfd_unmap(struct iwl_trans *trans,
765 int iwl_txq_dyn_alloc(struct iwl_trans *trans, u32 flags,
769 int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
772 void iwl_txq_dyn_free(struct iwl_trans *trans, int queue);
773 void iwl_txq_gen2_tx_free(struct iwl_trans *trans);
774 int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
776 int iwl_txq_gen2_init(struct iwl_trans *trans, int txq_id,
779 static inline u16 iwl_txq_gen1_tfd_tb_get_len(struct iwl_trans *trans,  in iwl_txq_gen1_tfd_tb_get_len()  argument
785 	if (trans->trans_cfg->gen2) {  in iwl_txq_gen1_tfd_tb_get_len()
798 void iwl_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
800 void iwl_pcie_set_q_ptrs(struct iwl_trans *trans, int txq_id, int ptr);
801 void iwl_pcie_freeze_txq_timer(struct iwl_trans *trans,
803 int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx);
804 int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm);
809 void iwl_pcie_dump_csr(struct iwl_trans *trans);
814 static inline void _iwl_disable_interrupts(struct iwl_trans *trans)  in _iwl_disable_interrupts()  argument
816 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in _iwl_disable_interrupts()
818 	clear_bit(STATUS_INT_ENABLED, &trans->status);  in _iwl_disable_interrupts()
821 		iwl_write32(trans, CSR_INT_MASK, 0x00000000);  in _iwl_disable_interrupts()
825 		iwl_write32(trans, CSR_INT, 0xffffffff);  in _iwl_disable_interrupts()
826 		iwl_write32(trans, CSR_FH_INT_STATUS, 0xffffffff);  in _iwl_disable_interrupts()
829 		iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,  in _iwl_disable_interrupts()
831 		iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,  in _iwl_disable_interrupts()
834 	IWL_DEBUG_ISR(trans, "Disabled interrupts\n");  in _iwl_disable_interrupts()
852 static inline void iwl_pcie_ctxt_info_free_fw_img(struct iwl_trans *trans)  in iwl_pcie_ctxt_info_free_fw_img()  argument
854 	struct iwl_self_init_dram *dram = &trans->init_dram;  in iwl_pcie_ctxt_info_free_fw_img()
863 		dma_free_coherent(trans->dev, dram->fw[i].size,  in iwl_pcie_ctxt_info_free_fw_img()
871 static inline void iwl_disable_interrupts(struct iwl_trans *trans)  in iwl_disable_interrupts()  argument
873 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_disable_interrupts()
876 	_iwl_disable_interrupts(trans);  in iwl_disable_interrupts()
880 static inline void _iwl_enable_interrupts(struct iwl_trans *trans)  in _iwl_enable_interrupts()  argument
882 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in _iwl_enable_interrupts()
884 	IWL_DEBUG_ISR(trans, "Enabling interrupts\n");  in _iwl_enable_interrupts()
885 	set_bit(STATUS_INT_ENABLED, &trans->status);  in _iwl_enable_interrupts()
888 		iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);  in _iwl_enable_interrupts()
896 		iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,  in _iwl_enable_interrupts()
898 		iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,  in _iwl_enable_interrupts()
903 static inline void iwl_enable_interrupts(struct iwl_trans *trans)  in iwl_enable_interrupts()  argument
905 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_interrupts()
908 	_iwl_enable_interrupts(trans);  in iwl_enable_interrupts()
911 static inline void iwl_enable_hw_int_msk_msix(struct iwl_trans *trans, u32 msk)  in iwl_enable_hw_int_msk_msix()  argument
913 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_hw_int_msk_msix()
915 	iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD, ~msk);  in iwl_enable_hw_int_msk_msix()
919 static inline void iwl_enable_fh_int_msk_msix(struct iwl_trans *trans, u32 msk)  in iwl_enable_fh_int_msk_msix()  argument
921 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_fh_int_msk_msix()
923 	iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~msk);  in iwl_enable_fh_int_msk_msix()
927 static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)  in iwl_enable_fw_load_int()  argument
929 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_fw_load_int()
931 	IWL_DEBUG_ISR(trans, "Enabling FW load interrupt\n");  in iwl_enable_fw_load_int()
934 		iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);  in iwl_enable_fw_load_int()
936 		iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,  in iwl_enable_fw_load_int()
938 		iwl_enable_fh_int_msk_msix(trans,  in iwl_enable_fw_load_int()
943 static inline void iwl_enable_fw_load_int_ctx_info(struct iwl_trans *trans)  in iwl_enable_fw_load_int_ctx_info()  argument
945 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_fw_load_int_ctx_info()
947 	IWL_DEBUG_ISR(trans, "Enabling ALIVE interrupt only\n");  in iwl_enable_fw_load_int_ctx_info()
958 		iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);  in iwl_enable_fw_load_int_ctx_info()
960 		iwl_enable_hw_int_msk_msix(trans,  in iwl_enable_fw_load_int_ctx_info()
966 		iwl_enable_fh_int_msk_msix(trans, trans_pcie->fh_init_mask);  in iwl_enable_fw_load_int_ctx_info()
993 static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)  in iwl_enable_rfkill_int()  argument
995 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_enable_rfkill_int()
997 	IWL_DEBUG_ISR(trans, "Enabling rfkill interrupt\n");  in iwl_enable_rfkill_int()
1000 		iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);  in iwl_enable_rfkill_int()
1002 		iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,  in iwl_enable_rfkill_int()
1004 		iwl_enable_hw_int_msk_msix(trans,  in iwl_enable_rfkill_int()
1008 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_9000) {  in iwl_enable_rfkill_int()
1014 		iwl_set_bit(trans, CSR_GP_CNTRL,  in iwl_enable_rfkill_int()
1019 void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans, bool from_irq);
1021 static inline bool iwl_is_rfkill_set(struct iwl_trans *trans)  in iwl_is_rfkill_set()  argument
1023 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_is_rfkill_set()
1030 	return !(iwl_read32(trans, CSR_GP_CNTRL) &  in iwl_is_rfkill_set()
1034 static inline void __iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans,  in __iwl_trans_pcie_set_bits_mask()  argument
1043 	v = iwl_read32(trans, reg);  in __iwl_trans_pcie_set_bits_mask()
1046 	iwl_write32(trans, reg, v);  in __iwl_trans_pcie_set_bits_mask()
1049 static inline void __iwl_trans_pcie_clear_bit(struct iwl_trans *trans,  in __iwl_trans_pcie_clear_bit()  argument
1052 	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, 0);  in __iwl_trans_pcie_clear_bit()
1055 static inline void __iwl_trans_pcie_set_bit(struct iwl_trans *trans,  in __iwl_trans_pcie_set_bit()  argument
1058 	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, mask);  in __iwl_trans_pcie_set_bit()
1061 static inline bool iwl_pcie_dbg_on(struct iwl_trans *trans)  in iwl_pcie_dbg_on()  argument
1063 	return (trans->dbg.dest_tlv || iwl_trans_dbg_ini_valid(trans));  in iwl_pcie_dbg_on()
1066 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq);
1067 void iwl_trans_pcie_dump_regs(struct iwl_trans *trans);
1070 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans);
1071 void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans);
1073 static inline void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) { }  in iwl_trans_pcie_dbgfs_register()  argument
1079 void iwl_trans_pcie_configure(struct iwl_trans *trans,
1081 int iwl_trans_pcie_start_hw(struct iwl_trans *trans);
1082 void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans);
1083 void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val);
1084 void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val);
1085 u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs);
1086 u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg);
1087 void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val);
1088 int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
1090 int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
1092 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership);
1094 iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
1097 int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1100 int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, bool reset);
1101 void iwl_trans_pci_interrupts(struct iwl_trans *trans, bool enable);
1102 void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans);
1103 void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
1105 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
1107 bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
1108 void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans);
1111 void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr);
1112 int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
1114 void iwl_trans_pcie_stop_device(struct iwl_trans *trans);
1117 int iwl_pcie_gen2_apm_init(struct iwl_trans *trans);
1118 void iwl_pcie_apm_config(struct iwl_trans *trans);
1119 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans);
1120 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans);
1121 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans);
1122 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
1124 void iwl_pcie_apm_stop_master(struct iwl_trans *trans);
1126 int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
1128 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr);
1129 void iwl_pcie_apply_destination(struct iwl_trans *trans);
1132 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power);
1135 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
1137 void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans);
1138 int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans,
1140 void iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans);
1141 int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
1143 int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
1145 void iwl_trans_pcie_copy_imr_fh(struct iwl_trans *trans,
1147 int iwl_trans_pcie_copy_imr(struct iwl_trans *trans,
1149 int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,