Lines Matching refs:rtwdev
29 static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev,
32 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
47 rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val);
54 static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
59 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
63 rtwdev, R_AX_PCIE_INIT_CFG1);
68 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
72 const struct rtw89_pci_info *info = rtwdev->pci_info;
94 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
101 idx = rtw89_read32(rtwdev, addr_idx);
102 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
107 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
118 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
131 rtw89_err(rtwdev, "failed to release fwcmd\n");
141 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
147 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
150 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
153 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
160 idx = rtw89_read32(rtwdev, addr_idx);
161 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
166 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
174 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
178 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
186 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
190 static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
206 static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev,
211 const struct rtw89_pci_info *info = rtwdev->pci_info;
224 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n",
233 int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
242 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
243 rtw89_pci_rxbd_info_update(rtwdev, skb);
245 ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb);
256 static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable)
258 const struct rtw89_pci_info *info = rtwdev->pci_info;
263 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
265 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
267 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
269 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
273 static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable)
275 const struct rtw89_pci_info *info = rtwdev->pci_info;
279 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
281 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
285 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
294 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
297 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
303 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
313 static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
316 const struct rtw89_pci_info *info = rtwdev->pci_info;
328 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
333 const struct rtw89_pci_info *info = rtwdev->pci_info;
344 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
347 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
349 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
359 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
365 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
370 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
374 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
376 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
387 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
391 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
393 rtw89_pci_sync_skb_for_device(rtwdev, skb);
397 rtw89_warn(rtwdev, "no rx desc information\n");
401 rtw89_core_rx(rtwdev, desc_info, new);
409 rtw89_pci_sync_skb_for_device(rtwdev, skb);
420 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
427 while (cnt && rtwdev->napi_budget_countdown > 0) {
428 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
430 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
440 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
443 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
447 int countdown = rtwdev->napi_budget_countdown;
452 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
458 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
461 if (rtwdev->napi_budget_countdown <= 0)
467 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
474 rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
486 rtw89_debug(rtwdev, RTW89_DBG_FW,
499 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
504 ieee80211_tx_status_ni(rtwdev->hw, skb);
507 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
512 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
516 rtw89_warn(rtwdev, "No busy txwd pages available\n");
528 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
544 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
549 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
555 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
560 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
571 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
578 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
581 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
591 txch = rtw89_core_get_ch_dma(rtwdev, qsel);
594 rtw89_warn(rtwdev, "should no fwcmd release report\n");
602 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
605 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
618 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
622 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
638 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
641 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
643 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
650 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
654 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
660 rtw89_pci_release_rpp(rtwdev, rpp);
663 rtw89_pci_sync_skb_for_device(rtwdev, skb);
670 rtw89_pci_sync_skb_for_device(rtwdev, skb);
674 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
682 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
684 rtw89_err(rtwdev, "failed to release TX skbs\n");
694 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
697 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
708 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
712 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
719 rtwdev->napi_budget_countdown -= work_done;
724 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
737 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
743 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
745 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
751 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
755 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
756 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
757 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
759 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
760 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
761 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
765 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
769 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
771 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
773 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
775 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
778 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
780 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
782 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
786 void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
790 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
792 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
794 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
795 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
798 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
800 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
802 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
803 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
807 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
809 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
810 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
811 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
815 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
817 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
818 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
819 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
823 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
825 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
826 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
827 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
828 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
832 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
834 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
838 void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
840 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
841 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
842 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
843 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
847 void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
849 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
850 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
854 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
856 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
860 rtw89_chip_disable_intr(rtwdev, rtwpci);
861 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
862 rtw89_chip_enable_intr(rtwdev, rtwpci);
866 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
868 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
872 rtw89_chip_disable_intr(rtwdev, rtwpci);
873 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
874 rtw89_chip_enable_intr(rtwdev, rtwpci);
878 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
880 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
884 rtwdev->napi_budget_countdown = budget;
886 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
887 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
892 struct rtw89_dev *rtwdev = dev;
893 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
894 const struct rtw89_pci_info *info = rtwdev->pci_info;
900 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
904 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
907 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
910 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
916 rtw89_pci_low_power_interrupt_handler(rtwdev);
922 napi_schedule(&rtwdev->napi);
931 rtw89_chip_enable_intr(rtwdev, rtwpci);
938 struct rtw89_dev *rtwdev = dev;
939 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
953 rtw89_chip_disable_intr(rtwdev, rtwpci);
1068 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
1072 const struct rtw89_pci_info *info = rtwdev->pci_info;
1082 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
1086 const struct rtw89_pci_info *info = rtwdev->pci_info;
1108 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1110 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1115 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1123 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1126 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1140 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1143 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1146 const struct rtw89_chip_info *chip = rtwdev->chip;
1159 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1161 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1167 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1183 rtw89_debug(rtwdev, debug_mask,
1194 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1197 if (rtwdev->hci.paused)
1198 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1201 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1203 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1206 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1208 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1216 rtw89_write16(rtwdev, addr, host_idx);
1221 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1234 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1236 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1239 if (rtwdev->hci.paused) {
1244 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1247 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1249 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1258 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1262 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1264 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1276 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1285 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1288 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1291 const struct rtw89_pci_info *info = rtwdev->pci_info;
1302 __pci_flush_txch(rtwdev, i, drop);
1306 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1309 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1312 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1331 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1368 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1373 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1374 const struct rtw89_chip_info *chip = rtwdev->chip;
1391 rtw89_err(rtwdev, "failed to map skb dma data\n");
1420 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1425 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1435 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1440 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1441 const struct rtw89_chip_info *chip = rtwdev->chip;
1453 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1457 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1469 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1474 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1488 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1492 rtw89_err(rtwdev, "no available TXWD\n");
1497 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1499 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1511 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1521 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1524 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1535 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1544 rtw89_err(rtwdev, "no available TXBD\n");
1550 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1552 rtw89_err(rtwdev, "failed to submit TXBD\n");
1564 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1569 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1571 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1606 static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev)
1608 const struct rtw89_pci_info *info = rtwdev->pci_info;
1621 rtw89_write32(rtwdev, addr + i, val);
1625 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1627 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1628 const struct rtw89_pci_info *info = rtwdev->pci_info;
1654 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1660 rtw89_write32(rtwdev, addr_bdram, val32);
1662 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1663 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1681 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1682 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1683 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1686 rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
1689 rtw89_pci_init_wp_16sel(rtwdev);
1692 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1695 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1696 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1699 void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1701 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1702 const struct rtw89_pci_info *info = rtwdev->pci_info;
1705 rtw89_pci_reset_trx_rings(rtwdev);
1712 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1716 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1721 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1723 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1728 rtw89_chip_enable_intr(rtwdev, rtwpci);
1732 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1734 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1739 rtw89_chip_disable_intr(rtwdev, rtwpci);
1743 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1745 rtw89_core_napi_start(rtwdev);
1746 rtw89_pci_enable_intr_lock(rtwdev);
1751 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1753 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1756 rtw89_pci_disable_intr_lock(rtwdev);
1758 rtw89_core_napi_stop(rtwdev);
1761 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1763 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1767 rtw89_pci_disable_intr_lock(rtwdev);
1769 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1770 napi_synchronize(&rtwdev->napi);
1772 rtw89_pci_enable_intr_lock(rtwdev);
1773 rtw89_pci_tx_kick_off_pending(rtwdev);
1778 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1780 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1781 const struct rtw89_pci_info *info = rtwdev->pci_info;
1806 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1810 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1813 rtw89_chip_config_intr_mask(rtwdev, cfg);
1814 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1817 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1819 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1821 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1828 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1836 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1839 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1844 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1851 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1853 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1864 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val);
1871 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1875 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1877 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1888 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val);
1895 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1899 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1901 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1911 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1916 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1919 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1921 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1926 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, data);
1931 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1933 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1938 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, data);
1943 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1945 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1950 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, data);
1955 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1957 const struct rtw89_pci_info *info = rtwdev->pci_info;
1960 rtw89_write32_set(rtwdev, info->init_cfg_reg,
1963 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1967 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1969 const struct rtw89_pci_info *info = rtwdev->pci_info;
1973 rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
1975 rtw89_write32_set(rtwdev, reg->addr, reg->mask);
1978 void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1980 rtw89_pci_ctrl_dma_io(rtwdev, enable);
1981 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1984 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1988 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1990 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
2005 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
2008 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
2009 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
2012 false, rtwdev, R_AX_MDIO_CFG);
2016 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
2020 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
2022 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
2025 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
2031 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
2035 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
2036 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
2038 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
2046 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
2052 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2060 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
2067 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2072 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2075 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
2082 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2087 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2090 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
2097 static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data)
2106 rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data);
2107 rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr);
2108 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16);
2112 rtwdev, R_AX_DBI_FLAG + 2);
2114 rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n",
2120 static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value)
2126 rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr);
2127 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16);
2131 rtwdev, R_AX_DBI_FLAG + 2);
2133 rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n",
2139 *value = rtw89_read8(rtwdev, read_addr);
2144 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2147 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2148 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2156 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2157 ret = rtw89_dbi_write8(rtwdev, addr, data);
2162 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2165 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2166 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2174 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2175 ret = rtw89_dbi_read8(rtwdev, addr, value);
2180 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
2186 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2191 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2196 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
2202 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2207 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2213 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
2219 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
2222 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2226 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
2233 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
2236 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2243 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
2252 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2256 if (!rtw89_is_rtl885xb(rtwdev))
2259 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2264 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2272 if (!rtw89_is_rtl885xb(rtwdev))
2275 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
2277 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2287 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2291 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
2293 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2298 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2301 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2308 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2310 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2315 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2318 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2326 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
2328 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2333 ret = __get_target(rtwdev, &tar, phy_rate);
2335 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2354 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2360 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2362 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2368 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2370 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2374 ret = __get_target(rtwdev, &tar, phy_rate);
2376 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2380 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2382 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2385 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2390 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2392 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2397 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2403 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2406 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2415 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2417 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2421 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2425 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2430 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2432 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2439 static void rtw89_pci_disable_eq_ax(struct rtw89_dev *rtwdev)
2449 if (rtwdev->chip->chip_id != RTL8852C)
2452 g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2454 g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 +
2459 backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1);
2460 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
2462 ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset);
2466 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN);
2467 rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL);
2468 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL);
2470 oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT,
2473 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT,
2475 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT,
2478 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT,
2480 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT,
2488 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
2492 offset_cal = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2499 rtw89_write16_mask(rtwdev, phy_offset + RAC_ANA0B * RAC_MULT,
2501 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT,
2506 rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm);
2509 static void rtw89_pci_ber(struct rtw89_dev *rtwdev)
2513 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks))
2517 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL);
2518 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2521 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL);
2522 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2525 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2527 if (rtwdev->chip->chip_id != RTL8852A)
2530 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2533 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2535 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2537 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2540 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2543 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2547 if (rtwdev->chip->chip_id != RTL8852A)
2550 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2555 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2563 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2565 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2567 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2570 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2573 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2575 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2577 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2578 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2580 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2582 } else if (rtwdev->chip->chip_id == RTL8852C) {
2583 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2588 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2590 if (!rtw89_is_rtl885xb(rtwdev))
2593 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2597 static void rtw89_pci_power_wake_ax(struct rtw89_dev *rtwdev, bool pwr_up)
2600 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2602 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2605 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2607 if (rtwdev->chip->chip_id != RTL8852C)
2610 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2611 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2614 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2616 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2619 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2622 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2624 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2627 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2629 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2630 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2634 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2636 if (rtwdev->chip->chip_id != RTL8852C)
2639 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2642 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2644 if (rtwdev->chip->chip_id != RTL8852C)
2647 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2650 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2652 if (rtwdev->chip->chip_id == RTL8852C)
2655 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2659 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2661 const struct rtw89_pci_info *info = rtwdev->pci_info;
2664 if (rtwdev->chip->chip_id == RTL8852C)
2667 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2671 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2675 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2678 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2680 const struct rtw89_pci_info *info = rtwdev->pci_info;
2683 if (rtwdev->chip->chip_id != RTL8852C)
2689 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2690 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2691 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2693 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2694 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2695 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2697 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2698 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2699 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2702 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2705 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2707 if (rtwdev->chip->chip_id == RTL8852C)
2710 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2713 if (rtwdev->chip->chip_id == RTL8852A)
2714 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2718 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2720 if (rtwdev->chip->chip_id == RTL8852C)
2723 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2727 static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2729 const struct rtw89_pci_info *info = rtwdev->pci_info;
2730 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2741 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2743 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2745 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2749 static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2751 const struct rtw89_pci_info *info = rtwdev->pci_info;
2760 10, 100, false, rtwdev, dma_busy1);
2770 10, 100, false, rtwdev, dma_busy2);
2777 static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2779 const struct rtw89_pci_info *info = rtwdev->pci_info;
2787 10, 100, false, rtwdev, dma_busy3);
2794 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2798 ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev);
2800 rtw89_err(rtwdev, "txdma ch busy\n");
2804 ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev);
2806 rtw89_err(rtwdev, "rxdma ch busy\n");
2813 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2815 const struct rtw89_pci_info *info = rtwdev->pci_info;
2824 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2825 u8 cv = rtwdev->hal.cv;
2830 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2833 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2838 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2841 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2845 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2847 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2850 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2854 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2855 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2856 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2858 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2859 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2862 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2864 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2866 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2868 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2870 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2874 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2877 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2878 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2880 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2883 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2885 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2890 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2892 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2894 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2896 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2902 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2904 const struct rtw89_pci_info *info = rtwdev->pci_info;
2906 rtw89_pci_power_wake(rtwdev, false);
2908 if (rtwdev->chip->chip_id == RTL8852A) {
2910 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2912 info->ltr_set(rtwdev, false);
2913 rtw89_pci_ctrl_dma_all(rtwdev, false);
2914 rtw89_pci_clr_idx_all(rtwdev);
2919 static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
2921 const struct rtw89_pci_info *info = rtwdev->pci_info;
2924 rtw89_pci_ber(rtwdev);
2925 rtw89_pci_rxdma_prefth(rtwdev);
2926 rtw89_pci_l1off_pwroff(rtwdev);
2927 rtw89_pci_deglitch_setting(rtwdev);
2928 ret = rtw89_pci_l2_rxen_lat(rtwdev);
2930 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2934 rtw89_pci_aphy_pwrcut(rtwdev);
2935 rtw89_pci_hci_ldo(rtwdev);
2936 rtw89_pci_dphy_delay(rtwdev);
2938 ret = rtw89_pci_autok_x(rtwdev);
2940 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2944 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2946 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2950 rtw89_pci_power_wake_ax(rtwdev, true);
2951 rtw89_pci_autoload_hang(rtwdev);
2952 rtw89_pci_l12_vmain(rtwdev);
2953 rtw89_pci_gen2_force_ib(rtwdev);
2954 rtw89_pci_l1_ent_lat(rtwdev);
2955 rtw89_pci_wd_exit_l1(rtwdev);
2956 rtw89_pci_set_sic(rtwdev);
2957 rtw89_pci_set_lbc(rtwdev);
2958 rtw89_pci_set_io_rcy(rtwdev);
2959 rtw89_pci_set_dbg(rtwdev);
2960 rtw89_pci_set_keep_reg(rtwdev);
2962 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2965 rtw89_pci_ctrl_dma_all(rtwdev, false);
2967 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2969 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2973 rtw89_pci_clr_idx_all(rtwdev);
2974 rtw89_pci_mode_op(rtwdev);
2977 rtw89_pci_ops_reset(rtwdev);
2979 ret = rtw89_pci_rst_bdram_ax(rtwdev);
2981 rtw89_warn(rtwdev, "reset bdram busy\n");
2986 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false);
2987 rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true);
2990 rtw89_pci_ctrl_dma_all(rtwdev, true);
2995 static int rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev *rtwdev)
2997 rtw89_pci_power_wake_ax(rtwdev, false);
3002 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
3009 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
3012 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
3015 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
3018 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
3022 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
3024 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
3026 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3028 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
3029 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
3030 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
3031 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
3037 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
3042 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
3045 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
3048 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
3051 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
3054 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
3070 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
3072 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3074 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
3075 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
3076 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
3077 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
3078 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
3084 static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
3086 const struct rtw89_pci_info *info = rtwdev->pci_info;
3087 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3090 ret = info->ltr_set(rtwdev, true);
3092 rtw89_err(rtwdev, "pci ltr set fail\n");
3097 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
3099 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3101 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3103 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3107 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true);
3110 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
3116 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
3119 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3124 rtw89_err(rtwdev, "failed to enable pci device\n");
3129 pci_set_drvdata(pdev, rtwdev->hw);
3136 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
3142 static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev)
3144 const struct rtw89_chip_info *chip = rtwdev->chip;
3157 static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev)
3159 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3162 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3180 static void rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev)
3182 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3187 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3190 rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, RTW89_PCIE_BIT_EN_64BITS);
3193 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
3196 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3203 rtw89_err(rtwdev, "failed to request pci regions\n");
3207 if (!rtw89_pci_is_dac_compatible_bridge(rtwdev))
3213 rtw89_pci_cfg_dac(rtwdev);
3217 rtw89_err(rtwdev,
3230 rtw89_err(rtwdev, "failed to map pci io\n");
3243 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
3246 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3254 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
3269 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
3285 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
3288 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3289 const struct rtw89_pci_info *info = rtwdev->pci_info;
3297 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3298 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3302 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
3334 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
3337 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3343 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3347 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
3350 rtw89_pci_free_rx_rings(rtwdev, pdev);
3351 rtw89_pci_free_tx_rings(rtwdev, pdev);
3354 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
3381 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
3432 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3444 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3446 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3450 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
3452 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3475 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3480 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3483 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3484 const struct rtw89_pci_info *info = rtwdev->pci_info;
3497 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3501 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3503 rtw89_err(rtwdev, "failed to alloc tx ring %d: ret=%d\n", i, ret);
3515 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3521 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3526 const struct rtw89_pci_info *info = rtwdev->pci_info;
3536 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3538 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3572 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3576 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3578 rtw89_err(rtwdev, "failed to init rx buf %d ret=%d\n", i, ret);
3609 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3612 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3623 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3626 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3637 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3643 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3648 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3650 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3654 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3656 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3663 rtw89_pci_free_tx_rings(rtwdev, pdev);
3668 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3675 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3678 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3681 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3683 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3687 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3689 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3693 rtw89_pci_h2c_init(rtwdev, rtwpci);
3701 rtw89_pci_clear_mapping(rtwdev, pdev);
3706 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3709 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3711 rtw89_pci_free_trx_rings(rtwdev, pdev);
3712 rtw89_pci_clear_mapping(rtwdev, pdev);
3713 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3717 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3719 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3720 const struct rtw89_chip_info *chip = rtwdev->chip;
3746 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3748 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3756 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3758 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3774 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3776 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3785 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3787 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3790 rtw89_pci_recovery_intr_mask_v1(rtwdev);
3792 rtw89_pci_low_power_intr_mask_v1(rtwdev);
3794 rtw89_pci_default_intr_mask_v1(rtwdev);
3798 static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
3800 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3808 static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
3810 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3821 static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
3823 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3833 void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
3835 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3838 rtw89_pci_recovery_intr_mask_v2(rtwdev);
3840 rtw89_pci_low_power_intr_mask_v2(rtwdev);
3842 rtw89_pci_default_intr_mask_v2(rtwdev);
3846 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3855 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3859 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3862 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3864 rtw89_err(rtwdev, "failed to request threaded irq\n");
3868 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3878 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3881 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3897 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3899 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3905 if (rtwdev->chip->chip_id != RTL8852C)
3908 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3921 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3922 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3924 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3927 val16 = rtw89_read16_mask(rtwdev,
3931 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3936 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3938 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3940 rtw89_write16_set(rtwdev,
3946 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3952 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3954 const struct rtw89_pci_info *info = rtwdev->pci_info;
3960 gen_def->clkreq_set(rtwdev, enable);
3963 static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable)
3965 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3968 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3971 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3973 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3975 ret = rtw89_pci_config_byte_set(rtwdev,
3979 ret = rtw89_pci_config_byte_clr(rtwdev,
3983 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3986 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3989 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3992 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3997 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3999 const struct rtw89_pci_info *info = rtwdev->pci_info;
4005 gen_def->aspm_set(rtwdev, enable);
4008 static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable)
4010 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4014 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
4016 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
4021 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
4023 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
4025 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4027 ret = rtw89_pci_config_byte_set(rtwdev,
4031 ret = rtw89_pci_config_byte_clr(rtwdev,
4036 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4039 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4043 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
4047 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
4049 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
4050 const struct rtw89_pci_info *info = rtwdev->pci_info;
4051 struct rtw89_traffic_stats *stats = &rtwdev->stats;
4056 if (rtwdev->scanning ||
4069 rtw89_write32(rtwdev, info->mit_addr, val);
4072 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
4074 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4097 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
4102 rtw89_pci_clkreq_set(rtwdev, true);
4105 rtw89_pci_aspm_set(rtwdev, true);
4108 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
4110 const struct rtw89_pci_info *info = rtwdev->pci_info;
4116 gen_def->l1ss_set(rtwdev, enable);
4119 static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable)
4121 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4124 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4126 ret = rtw89_pci_config_byte_set(rtwdev,
4130 ret = rtw89_pci_config_byte_clr(rtwdev,
4134 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
4137 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
4141 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
4143 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4146 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4151 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
4153 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4167 rtw89_pci_l1ss_set(rtwdev, true);
4170 static void rtw89_pci_cpl_timeout_cfg(struct rtw89_dev *rtwdev)
4172 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4179 static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
4186 10, 1000, false, rtwdev,
4189 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
4190 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
4196 static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
4201 if (rtwdev->chip->chip_id == RTL8852C)
4204 rtw89_pci_ctrl_dma_all(rtwdev, false);
4205 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4207 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4208 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4212 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
4214 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
4215 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4216 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4217 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4218 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4226 static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
4230 if (rtwdev->chip->chip_id == RTL8852C)
4233 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
4234 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4235 rtw89_pci_clr_idx_all(rtwdev);
4237 ret = rtw89_pci_rst_bdram_ax(rtwdev);
4241 rtw89_pci_ctrl_dma_all(rtwdev, true);
4245 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
4248 const struct rtw89_pci_info *info = rtwdev->pci_info;
4254 ret = gen_def->lv1rst_stop_dma(rtwdev);
4256 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
4261 ret = gen_def->lv1rst_start_dma(rtwdev);
4263 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
4273 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
4275 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
4278 if (rtwdev->chip->chip_id == RTL8852C) {
4279 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4280 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
4281 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4282 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
4284 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
4285 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
4286 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4287 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
4288 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4289 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
4295 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
4296 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4297 const struct rtw89_pci_info *info = rtwdev->pci_info;
4302 rtwdev->napi_budget_countdown = budget;
4304 rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data);
4305 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4309 rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data);
4310 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4314 rtw89_chip_enable_intr(rtwdev, rtwpci);
4322 void rtw89_check_pci_ssid_quirks(struct rtw89_dev *rtwdev,
4341 bitmap_or(rtwdev->quirks, rtwdev->quirks, &ssid_quirks->bitmap,
4343 rtwdev->custid = ssid_quirks->custid;
4347 rtw89_debug(rtwdev, RTW89_DBG_HCI, "quirks=%*ph custid=%d\n",
4348 (int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid);
4354 struct rtw89_dev *rtwdev = hw->priv;
4355 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4357 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4358 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4359 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4360 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4361 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
4363 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
4366 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4373 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
4375 if (rtwdev->chip->chip_id == RTL8852C)
4379 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4381 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4385 void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume)
4388 rtw89_pci_cfg_dac(rtwdev);
4390 rtw89_pci_disable_eq(rtwdev);
4391 rtw89_pci_filter_out(rtwdev);
4392 rtw89_pci_cpl_timeout_cfg(rtwdev);
4393 rtw89_pci_link_cfg(rtwdev);
4394 rtw89_pci_l1ss_cfg(rtwdev);
4400 struct rtw89_dev *rtwdev = hw->priv;
4401 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4403 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4404 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4405 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4406 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4407 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
4409 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
4412 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4414 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4417 rtw89_pci_l2_hci_ldo(rtwdev);
4419 rtw89_pci_basic_cfg(rtwdev, true);
4503 struct rtw89_dev *rtwdev;
4510 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
4513 if (!rtwdev) {
4520 rtwdev->pci_info = info->bus.pci;
4521 rtwdev->hci.ops = &rtw89_pci_ops;
4522 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4523 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4524 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4526 rtw89_check_quirks(rtwdev, info->quirks);
4527 rtw89_check_pci_ssid_quirks(rtwdev, pdev, pci_info->ssid_quirks);
4529 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
4531 ret = rtw89_core_init(rtwdev);
4533 rtw89_err(rtwdev, "failed to initialise core\n");
4537 ret = rtw89_pci_claim_device(rtwdev, pdev);
4539 rtw89_err(rtwdev, "failed to claim pci device\n");
4543 ret = rtw89_pci_setup_resource(rtwdev, pdev);
4545 rtw89_err(rtwdev, "failed to setup pci resource\n");
4549 ret = rtw89_chip_info_setup(rtwdev);
4551 rtw89_err(rtwdev, "failed to setup chip information\n");
4555 rtw89_pci_basic_cfg(rtwdev, false);
4557 ret = rtw89_core_napi_init(rtwdev);
4559 rtw89_err(rtwdev, "failed to init napi\n");
4563 ret = rtw89_pci_request_irq(rtwdev, pdev);
4565 rtw89_err(rtwdev, "failed to request pci irq\n");
4569 ret = rtw89_core_register(rtwdev);
4571 rtw89_err(rtwdev, "failed to register core\n");
4575 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags);
4580 rtw89_pci_free_irq(rtwdev, pdev);
4582 rtw89_core_napi_deinit(rtwdev);
4584 rtw89_pci_clear_resource(rtwdev, pdev);
4586 rtw89_pci_declaim_device(rtwdev, pdev);
4588 rtw89_core_deinit(rtwdev);
4590 rtw89_free_ieee80211_hw(rtwdev);
4599 struct rtw89_dev *rtwdev;
4601 rtwdev = hw->priv;
4603 rtw89_pci_free_irq(rtwdev, pdev);
4604 rtw89_core_napi_deinit(rtwdev);
4605 rtw89_core_unregister(rtwdev);
4606 rtw89_pci_clear_resource(rtwdev, pdev);
4607 rtw89_pci_declaim_device(rtwdev, pdev);
4608 rtw89_core_deinit(rtwdev);
4609 rtw89_free_ieee80211_hw(rtwdev);