Lines Matching defs:rtwdev
29 static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev,
32 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
47 rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val);
54 static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
59 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
63 rtwdev, R_AX_PCIE_INIT_CFG1);
68 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
72 const struct rtw89_pci_info *info = rtwdev->pci_info;
94 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
101 idx = rtw89_read32(rtwdev, addr_idx);
102 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
107 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
118 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
131 rtw89_err(rtwdev, "failed to release fwcmd\n");
141 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
147 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
150 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
153 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
160 idx = rtw89_read32(rtwdev, addr_idx);
161 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
166 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
174 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
178 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
186 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
190 static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
206 static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev,
211 const struct rtw89_pci_info *info = rtwdev->pci_info;
224 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n",
233 int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
242 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
243 rtw89_pci_rxbd_info_update(rtwdev, skb);
245 ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb);
256 static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable)
258 const struct rtw89_pci_info *info = rtwdev->pci_info;
263 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
265 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
267 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
269 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
273 static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable)
275 const struct rtw89_pci_info *info = rtwdev->pci_info;
279 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
281 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
285 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
294 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
297 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
303 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
313 static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
316 const struct rtw89_pci_info *info = rtwdev->pci_info;
328 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
343 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
346 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
348 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
359 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
364 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
368 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
370 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
381 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
385 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
387 rtw89_pci_sync_skb_for_device(rtwdev, skb);
391 rtw89_warn(rtwdev, "no rx desc information\n");
395 rtw89_core_rx(rtwdev, desc_info, new);
403 rtw89_pci_sync_skb_for_device(rtwdev, skb);
414 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
421 while (cnt && rtwdev->napi_budget_countdown > 0) {
422 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
424 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
434 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
437 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
441 int countdown = rtwdev->napi_budget_countdown;
446 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
452 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
455 if (rtwdev->napi_budget_countdown <= 0)
461 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
468 rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
480 rtw89_debug(rtwdev, RTW89_DBG_FW,
493 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
498 ieee80211_tx_status_ni(rtwdev->hw, skb);
501 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
506 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
510 rtw89_warn(rtwdev, "No busy txwd pages available\n");
522 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
538 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
543 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
549 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
554 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
565 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
572 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
575 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
585 txch = rtw89_core_get_ch_dma(rtwdev, qsel);
588 rtw89_warn(rtwdev, "should no fwcmd release report\n");
596 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
599 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
612 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
616 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
632 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
635 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
637 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
644 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
648 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
654 rtw89_pci_release_rpp(rtwdev, rpp);
657 rtw89_pci_sync_skb_for_device(rtwdev, skb);
664 rtw89_pci_sync_skb_for_device(rtwdev, skb);
668 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
676 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
678 rtw89_err(rtwdev, "failed to release TX skbs\n");
688 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
691 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
702 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
706 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
713 rtwdev->napi_budget_countdown -= work_done;
718 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
731 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
737 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
739 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
745 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
749 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
750 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
751 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
753 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
754 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
755 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
759 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
763 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
765 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
767 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
769 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
772 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
774 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
776 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
780 void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
784 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
786 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
788 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
789 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
792 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
794 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
796 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
797 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
801 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
803 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
804 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
805 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
809 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
811 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
812 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
813 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
817 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
819 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
820 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
821 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
822 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
826 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
828 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
832 void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
834 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
835 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
836 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
837 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
841 void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
843 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
844 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
848 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
850 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
854 rtw89_chip_disable_intr(rtwdev, rtwpci);
855 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
856 rtw89_chip_enable_intr(rtwdev, rtwpci);
860 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
862 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
866 rtw89_chip_disable_intr(rtwdev, rtwpci);
867 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
868 rtw89_chip_enable_intr(rtwdev, rtwpci);
872 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
874 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
878 rtwdev->napi_budget_countdown = budget;
880 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
881 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
886 struct rtw89_dev *rtwdev = dev;
887 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
888 const struct rtw89_pci_info *info = rtwdev->pci_info;
894 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
898 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
901 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
904 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
910 rtw89_pci_low_power_interrupt_handler(rtwdev);
916 napi_schedule(&rtwdev->napi);
925 rtw89_chip_enable_intr(rtwdev, rtwpci);
932 struct rtw89_dev *rtwdev = dev;
933 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
947 rtw89_chip_disable_intr(rtwdev, rtwpci);
1062 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
1066 const struct rtw89_pci_info *info = rtwdev->pci_info;
1076 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
1080 const struct rtw89_pci_info *info = rtwdev->pci_info;
1102 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1104 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1109 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1117 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1120 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1134 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1137 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1140 const struct rtw89_chip_info *chip = rtwdev->chip;
1153 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1155 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1161 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1177 rtw89_debug(rtwdev, debug_mask,
1188 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1191 if (rtwdev->hci.paused)
1192 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1195 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1197 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1200 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1202 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1210 rtw89_write16(rtwdev, addr, host_idx);
1215 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1228 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1230 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1233 if (rtwdev->hci.paused) {
1238 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1241 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1243 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1252 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1256 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1258 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1270 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1279 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1282 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1285 const struct rtw89_pci_info *info = rtwdev->pci_info;
1296 __pci_flush_txch(rtwdev, i, drop);
1300 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1303 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1306 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1325 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1362 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1367 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1368 const struct rtw89_chip_info *chip = rtwdev->chip;
1385 rtw89_err(rtwdev, "failed to map skb dma data\n");
1414 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1419 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1429 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1434 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1435 const struct rtw89_chip_info *chip = rtwdev->chip;
1447 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1451 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1463 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1468 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1482 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1486 rtw89_err(rtwdev, "no available TXWD\n");
1491 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1493 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1505 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1515 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1518 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1529 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1538 rtw89_err(rtwdev, "no available TXBD\n");
1544 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1546 rtw89_err(rtwdev, "failed to submit TXBD\n");
1558 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1563 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1565 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1600 static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev)
1602 const struct rtw89_pci_info *info = rtwdev->pci_info;
1615 rtw89_write32(rtwdev, addr + i, val);
1619 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1621 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1622 const struct rtw89_pci_info *info = rtwdev->pci_info;
1648 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1654 rtw89_write32(rtwdev, addr_bdram, val32);
1656 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1657 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1675 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1676 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1677 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1680 rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
1683 rtw89_pci_init_wp_16sel(rtwdev);
1686 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1689 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1690 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1693 void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1695 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1696 const struct rtw89_pci_info *info = rtwdev->pci_info;
1699 rtw89_pci_reset_trx_rings(rtwdev);
1706 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1710 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1715 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1717 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1722 rtw89_chip_enable_intr(rtwdev, rtwpci);
1726 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1728 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1733 rtw89_chip_disable_intr(rtwdev, rtwpci);
1737 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1739 rtw89_core_napi_start(rtwdev);
1740 rtw89_pci_enable_intr_lock(rtwdev);
1745 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1747 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1750 rtw89_pci_disable_intr_lock(rtwdev);
1752 rtw89_core_napi_stop(rtwdev);
1755 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1757 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1761 rtw89_pci_disable_intr_lock(rtwdev);
1763 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1764 napi_synchronize(&rtwdev->napi);
1766 rtw89_pci_enable_intr_lock(rtwdev);
1767 rtw89_pci_tx_kick_off_pending(rtwdev);
1772 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1774 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1775 const struct rtw89_pci_info *info = rtwdev->pci_info;
1800 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1804 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1807 rtw89_chip_config_intr_mask(rtwdev, cfg);
1808 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1811 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1813 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1815 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1822 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1830 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1833 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1838 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1845 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1847 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1858 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val);
1865 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1869 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1871 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1882 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val);
1889 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1893 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1895 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1905 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
1910 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1913 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1915 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1920 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, data);
1925 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1927 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1932 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, data);
1937 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1939 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1944 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, data);
1949 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1951 const struct rtw89_pci_info *info = rtwdev->pci_info;
1954 rtw89_write32_set(rtwdev, info->init_cfg_reg,
1957 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1961 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1963 const struct rtw89_pci_info *info = rtwdev->pci_info;
1967 rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
1969 rtw89_write32_set(rtwdev, reg->addr, reg->mask);
1972 void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1974 rtw89_pci_ctrl_dma_io(rtwdev, enable);
1975 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1978 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1982 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1984 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1999 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
2002 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
2003 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
2006 false, rtwdev, R_AX_MDIO_CFG);
2010 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
2014 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
2016 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
2019 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
2025 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
2029 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
2030 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
2032 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
2040 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
2046 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2054 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
2061 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2066 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2069 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
2076 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2081 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2084 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
2091 static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data)
2100 rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data);
2101 rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr);
2102 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16);
2106 rtwdev, R_AX_DBI_FLAG + 2);
2108 rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n",
2114 static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value)
2120 rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr);
2121 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16);
2125 rtwdev, R_AX_DBI_FLAG + 2);
2127 rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n",
2133 *value = rtw89_read8(rtwdev, read_addr);
2138 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2141 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2142 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2150 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2151 ret = rtw89_dbi_write8(rtwdev, addr, data);
2156 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2159 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2160 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2168 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2169 ret = rtw89_dbi_read8(rtwdev, addr, value);
2174 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
2180 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2185 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2190 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
2196 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2201 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2207 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
2213 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
2216 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2220 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
2227 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
2230 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2237 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
2246 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2250 if (!rtw89_is_rtl885xb(rtwdev))
2253 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2258 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2266 if (!rtw89_is_rtl885xb(rtwdev))
2269 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
2271 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2281 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2285 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
2287 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2292 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2295 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2302 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2304 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2309 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2312 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2320 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
2322 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2327 ret = __get_target(rtwdev, &tar, phy_rate);
2329 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2348 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2354 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2356 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2362 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2364 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2368 ret = __get_target(rtwdev, &tar, phy_rate);
2370 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2374 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2376 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2379 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2384 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2386 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2391 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2397 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2400 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2409 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2411 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2415 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2419 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2424 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2426 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2433 static void rtw89_pci_disable_eq(struct rtw89_dev *rtwdev)
2441 if (rtwdev->chip->chip_id != RTL8852C)
2444 g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2446 g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 +
2451 backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1);
2452 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
2454 ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset);
2458 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN);
2459 rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL);
2460 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL);
2462 oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT,
2465 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT,
2467 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT,
2470 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT,
2472 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT,
2476 rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm);
2479 static void rtw89_pci_ber(struct rtw89_dev *rtwdev)
2483 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks))
2487 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL);
2488 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2491 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL);
2492 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2495 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2497 if (rtwdev->chip->chip_id != RTL8852A)
2500 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2503 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2505 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2507 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2510 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2513 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2517 if (rtwdev->chip->chip_id != RTL8852A)
2520 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2525 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2533 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2535 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2537 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2540 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2543 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2545 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2547 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2548 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2550 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2552 } else if (rtwdev->chip->chip_id == RTL8852C) {
2553 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2558 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2560 if (!rtw89_is_rtl885xb(rtwdev))
2563 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2567 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2570 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2572 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2575 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2577 if (rtwdev->chip->chip_id != RTL8852C)
2580 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2581 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2584 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2586 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2589 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2592 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2594 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2597 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2599 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2600 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2604 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2606 if (rtwdev->chip->chip_id != RTL8852C)
2609 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2612 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2614 if (rtwdev->chip->chip_id != RTL8852C)
2617 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2620 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2622 if (rtwdev->chip->chip_id == RTL8852C)
2625 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2629 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2631 const struct rtw89_pci_info *info = rtwdev->pci_info;
2634 if (rtwdev->chip->chip_id == RTL8852C)
2637 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2641 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2645 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2648 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2650 const struct rtw89_pci_info *info = rtwdev->pci_info;
2653 if (rtwdev->chip->chip_id != RTL8852C)
2659 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2660 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2661 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2663 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2664 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2665 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2667 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2668 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2669 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2672 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2675 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2677 if (rtwdev->chip->chip_id == RTL8852C)
2680 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2683 if (rtwdev->chip->chip_id == RTL8852A)
2684 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2688 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2690 if (rtwdev->chip->chip_id == RTL8852C)
2693 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2697 static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2699 const struct rtw89_pci_info *info = rtwdev->pci_info;
2700 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2711 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2713 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2715 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2719 static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2721 const struct rtw89_pci_info *info = rtwdev->pci_info;
2729 10, 100, false, rtwdev, dma_busy1);
2739 10, 100, false, rtwdev, dma_busy2);
2746 static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2748 const struct rtw89_pci_info *info = rtwdev->pci_info;
2755 10, 100, false, rtwdev, dma_busy3);
2762 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2766 ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev);
2768 rtw89_err(rtwdev, "txdma ch busy\n");
2772 ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev);
2774 rtw89_err(rtwdev, "rxdma ch busy\n");
2781 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2783 const struct rtw89_pci_info *info = rtwdev->pci_info;
2792 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2793 u8 cv = rtwdev->hal.cv;
2798 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2801 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2806 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2809 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2813 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2815 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2818 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2822 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2823 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2824 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2826 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2827 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2830 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2832 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2834 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2836 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2838 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2842 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2845 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2846 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2848 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2851 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2853 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2858 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2860 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2862 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2864 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2870 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2872 const struct rtw89_pci_info *info = rtwdev->pci_info;
2874 if (rtwdev->chip->chip_id == RTL8852A) {
2876 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2878 info->ltr_set(rtwdev, false);
2879 rtw89_pci_ctrl_dma_all(rtwdev, false);
2880 rtw89_pci_clr_idx_all(rtwdev);
2885 static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
2887 const struct rtw89_pci_info *info = rtwdev->pci_info;
2890 rtw89_pci_ber(rtwdev);
2891 rtw89_pci_rxdma_prefth(rtwdev);
2892 rtw89_pci_l1off_pwroff(rtwdev);
2893 rtw89_pci_deglitch_setting(rtwdev);
2894 ret = rtw89_pci_l2_rxen_lat(rtwdev);
2896 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2900 rtw89_pci_aphy_pwrcut(rtwdev);
2901 rtw89_pci_hci_ldo(rtwdev);
2902 rtw89_pci_dphy_delay(rtwdev);
2904 ret = rtw89_pci_autok_x(rtwdev);
2906 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2910 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2912 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2916 rtw89_pci_power_wake(rtwdev, true);
2917 rtw89_pci_autoload_hang(rtwdev);
2918 rtw89_pci_l12_vmain(rtwdev);
2919 rtw89_pci_gen2_force_ib(rtwdev);
2920 rtw89_pci_l1_ent_lat(rtwdev);
2921 rtw89_pci_wd_exit_l1(rtwdev);
2922 rtw89_pci_set_sic(rtwdev);
2923 rtw89_pci_set_lbc(rtwdev);
2924 rtw89_pci_set_io_rcy(rtwdev);
2925 rtw89_pci_set_dbg(rtwdev);
2926 rtw89_pci_set_keep_reg(rtwdev);
2928 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2931 rtw89_pci_ctrl_dma_all(rtwdev, false);
2933 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2935 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2939 rtw89_pci_clr_idx_all(rtwdev);
2940 rtw89_pci_mode_op(rtwdev);
2943 rtw89_pci_ops_reset(rtwdev);
2945 ret = rtw89_pci_rst_bdram_ax(rtwdev);
2947 rtw89_warn(rtwdev, "reset bdram busy\n");
2952 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false);
2953 rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true);
2956 rtw89_pci_ctrl_dma_all(rtwdev, true);
2961 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2968 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2971 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2974 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2977 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2981 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2983 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2985 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2987 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2988 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2989 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2990 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2996 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
3001 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
3004 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
3007 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
3010 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
3013 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
3029 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
3031 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3033 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
3034 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
3035 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
3036 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
3037 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
3043 static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
3045 const struct rtw89_pci_info *info = rtwdev->pci_info;
3046 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3049 ret = info->ltr_set(rtwdev, true);
3051 rtw89_err(rtwdev, "pci ltr set fail\n");
3056 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
3058 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3060 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3062 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3066 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true);
3069 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
3075 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
3078 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3083 rtw89_err(rtwdev, "failed to enable pci device\n");
3088 pci_set_drvdata(pdev, rtwdev->hw);
3095 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
3101 static void rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev)
3103 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3104 const struct rtw89_chip_info *chip = rtwdev->chip;
3119 rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, RTW89_PCIE_BIT_EN_64BITS);
3122 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
3125 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3132 rtw89_err(rtwdev, "failed to request pci regions\n");
3139 rtw89_pci_cfg_dac(rtwdev);
3143 rtw89_err(rtwdev,
3155 rtw89_err(rtwdev, "failed to map pci io\n");
3168 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
3171 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3179 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
3194 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
3210 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
3213 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3214 const struct rtw89_pci_info *info = rtwdev->pci_info;
3222 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3223 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3227 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
3259 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
3262 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3268 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3272 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
3275 rtw89_pci_free_rx_rings(rtwdev, pdev);
3276 rtw89_pci_free_tx_rings(rtwdev, pdev);
3279 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
3306 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
3357 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3369 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3371 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3375 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
3377 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3400 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3405 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3408 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3409 const struct rtw89_pci_info *info = rtwdev->pci_info;
3422 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3426 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3428 rtw89_err(rtwdev, "failed to alloc tx ring %d: ret=%d\n", i, ret);
3440 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3446 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3451 const struct rtw89_pci_info *info = rtwdev->pci_info;
3461 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3463 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3497 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3501 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3503 rtw89_err(rtwdev, "failed to init rx buf %d ret=%d\n", i, ret);
3534 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3537 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3548 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3551 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3562 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3568 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3573 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3575 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3579 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3581 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3588 rtw89_pci_free_tx_rings(rtwdev, pdev);
3593 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3600 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3603 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3606 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3608 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3612 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3614 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3618 rtw89_pci_h2c_init(rtwdev, rtwpci);
3626 rtw89_pci_clear_mapping(rtwdev, pdev);
3631 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3634 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3636 rtw89_pci_free_trx_rings(rtwdev, pdev);
3637 rtw89_pci_clear_mapping(rtwdev, pdev);
3638 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3642 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3644 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3645 const struct rtw89_chip_info *chip = rtwdev->chip;
3671 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3673 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3681 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3683 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3699 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3701 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3710 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3712 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3715 rtw89_pci_recovery_intr_mask_v1(rtwdev);
3717 rtw89_pci_low_power_intr_mask_v1(rtwdev);
3719 rtw89_pci_default_intr_mask_v1(rtwdev);
3723 static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
3725 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3733 static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
3735 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3746 static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
3748 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3758 void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
3760 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3763 rtw89_pci_recovery_intr_mask_v2(rtwdev);
3765 rtw89_pci_low_power_intr_mask_v2(rtwdev);
3767 rtw89_pci_default_intr_mask_v2(rtwdev);
3771 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3780 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3784 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3787 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3789 rtw89_err(rtwdev, "failed to request threaded irq\n");
3793 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3803 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3806 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3825 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3827 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3833 if (rtwdev->chip->chip_id != RTL8852C)
3836 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3849 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3850 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3852 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3855 val16 = rtw89_read16_mask(rtwdev,
3859 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3864 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3866 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3868 rtw89_write16_set(rtwdev,
3874 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3880 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3882 const struct rtw89_pci_info *info = rtwdev->pci_info;
3888 gen_def->clkreq_set(rtwdev, enable);
3891 static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable)
3893 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3896 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3899 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3901 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3903 ret = rtw89_pci_config_byte_set(rtwdev,
3907 ret = rtw89_pci_config_byte_clr(rtwdev,
3911 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3914 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3917 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3920 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3925 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3927 const struct rtw89_pci_info *info = rtwdev->pci_info;
3933 gen_def->aspm_set(rtwdev, enable);
3936 static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable)
3938 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3942 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3944 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3949 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3951 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3953 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3955 ret = rtw89_pci_config_byte_set(rtwdev,
3959 ret = rtw89_pci_config_byte_clr(rtwdev,
3964 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3967 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3971 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3975 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3977 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
3978 const struct rtw89_pci_info *info = rtwdev->pci_info;
3979 struct rtw89_traffic_stats *stats = &rtwdev->stats;
3984 if (rtwdev->scanning ||
3997 rtw89_write32(rtwdev, info->mit_addr, val);
4000 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
4002 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4025 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
4030 rtw89_pci_clkreq_set(rtwdev, true);
4033 rtw89_pci_aspm_set(rtwdev, true);
4036 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
4038 const struct rtw89_pci_info *info = rtwdev->pci_info;
4044 gen_def->l1ss_set(rtwdev, enable);
4047 static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable)
4049 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4052 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4054 ret = rtw89_pci_config_byte_set(rtwdev,
4058 ret = rtw89_pci_config_byte_clr(rtwdev,
4062 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
4065 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
4069 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
4071 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4074 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4079 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
4081 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4095 rtw89_pci_l1ss_set(rtwdev, true);
4098 static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
4105 10, 1000, false, rtwdev,
4108 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
4109 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
4115 static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
4120 if (rtwdev->chip->chip_id == RTL8852C)
4123 rtw89_pci_ctrl_dma_all(rtwdev, false);
4124 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4126 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4127 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4131 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
4133 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
4134 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4135 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4136 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4137 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4145 static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
4149 if (rtwdev->chip->chip_id == RTL8852C)
4152 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
4153 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4154 rtw89_pci_clr_idx_all(rtwdev);
4156 ret = rtw89_pci_rst_bdram_ax(rtwdev);
4160 rtw89_pci_ctrl_dma_all(rtwdev, true);
4164 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
4167 const struct rtw89_pci_info *info = rtwdev->pci_info;
4173 ret = gen_def->lv1rst_stop_dma(rtwdev);
4175 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
4180 ret = gen_def->lv1rst_start_dma(rtwdev);
4182 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
4192 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
4194 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
4197 if (rtwdev->chip->chip_id == RTL8852C) {
4198 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4199 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
4200 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4201 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
4203 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
4204 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
4205 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4206 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
4207 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4208 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
4214 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
4215 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4216 const struct rtw89_pci_info *info = rtwdev->pci_info;
4221 rtwdev->napi_budget_countdown = budget;
4223 rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data);
4224 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4228 rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data);
4229 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4233 rtw89_chip_enable_intr(rtwdev, rtwpci);
4243 struct rtw89_dev *rtwdev = hw->priv;
4244 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4246 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4247 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4248 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4249 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4250 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
4252 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
4255 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4262 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
4264 if (rtwdev->chip->chip_id == RTL8852C)
4268 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4270 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4277 struct rtw89_dev *rtwdev = hw->priv;
4278 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4280 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4281 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4282 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4283 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4284 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
4286 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
4289 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4291 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4294 rtw89_pci_l2_hci_ldo(rtwdev);
4295 rtw89_pci_disable_eq(rtwdev);
4296 rtw89_pci_cfg_dac(rtwdev);
4297 rtw89_pci_filter_out(rtwdev);
4298 rtw89_pci_link_cfg(rtwdev);
4299 rtw89_pci_l1ss_cfg(rtwdev);
4380 struct rtw89_dev *rtwdev;
4387 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
4390 if (!rtwdev) {
4397 rtwdev->pci_info = info->bus.pci;
4398 rtwdev->hci.ops = &rtw89_pci_ops;
4399 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4400 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4401 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4403 rtw89_check_quirks(rtwdev, info->quirks);
4405 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
4407 ret = rtw89_core_init(rtwdev);
4409 rtw89_err(rtwdev, "failed to initialise core\n");
4413 ret = rtw89_pci_claim_device(rtwdev, pdev);
4415 rtw89_err(rtwdev, "failed to claim pci device\n");
4419 ret = rtw89_pci_setup_resource(rtwdev, pdev);
4421 rtw89_err(rtwdev, "failed to setup pci resource\n");
4425 ret = rtw89_chip_info_setup(rtwdev);
4427 rtw89_err(rtwdev, "failed to setup chip information\n");
4431 rtw89_pci_disable_eq(rtwdev);
4432 rtw89_pci_filter_out(rtwdev);
4433 rtw89_pci_link_cfg(rtwdev);
4434 rtw89_pci_l1ss_cfg(rtwdev);
4436 ret = rtw89_core_napi_init(rtwdev);
4438 rtw89_err(rtwdev, "failed to init napi\n");
4442 ret = rtw89_pci_request_irq(rtwdev, pdev);
4444 rtw89_err(rtwdev, "failed to request pci irq\n");
4448 ret = rtw89_core_register(rtwdev);
4450 rtw89_err(rtwdev, "failed to register core\n");
4454 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags);
4459 rtw89_pci_free_irq(rtwdev, pdev);
4461 rtw89_core_napi_deinit(rtwdev);
4463 rtw89_pci_clear_resource(rtwdev, pdev);
4465 rtw89_pci_declaim_device(rtwdev, pdev);
4467 rtw89_core_deinit(rtwdev);
4469 rtw89_free_ieee80211_hw(rtwdev);
4478 struct rtw89_dev *rtwdev;
4480 rtwdev = hw->priv;
4482 rtw89_pci_free_irq(rtwdev, pdev);
4483 rtw89_core_napi_deinit(rtwdev);
4484 rtw89_core_unregister(rtwdev);
4485 rtw89_pci_clear_resource(rtwdev, pdev);
4486 rtw89_pci_declaim_device(rtwdev, pdev);
4487 rtw89_core_deinit(rtwdev);
4488 rtw89_free_ieee80211_hw(rtwdev);