Lines Matching +full:no +full:- +full:l1ss
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
27 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
32 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_get_phy_offset_by_link_speed()
33 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_get_phy_offset_by_link_speed()
48 return -EFAULT; in rtw89_pci_get_phy_offset_by_link_speed()
72 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_dma_recalc()
75 rp = bd_ring->rp; in rtw89_pci_dma_recalc()
76 wp = bd_ring->wp; in rtw89_pci_dma_recalc()
77 len = bd_ring->len; in rtw89_pci_dma_recalc()
81 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp); in rtw89_pci_dma_recalc()
83 if (info->rx_ring_eq_is_full) in rtw89_pci_dma_recalc()
86 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp); in rtw89_pci_dma_recalc()
89 bd_ring->rp = cur_rp; in rtw89_pci_dma_recalc()
97 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; in rtw89_pci_txbd_recalc()
98 u32 addr_idx = bd_ring->addr.idx; in rtw89_pci_txbd_recalc()
115 while (cnt--) { in rtw89_pci_release_fwcmd()
116 skb = skb_dequeue(&rtwpci->h2c_queue); in rtw89_pci_release_fwcmd()
118 rtw89_err(rtwdev, "failed to pre-release fwcmd\n"); in rtw89_pci_release_fwcmd()
121 skb_queue_tail(&rtwpci->h2c_release_queue, skb); in rtw89_pci_release_fwcmd()
124 qlen = skb_queue_len(&rtwpci->h2c_release_queue); in rtw89_pci_release_fwcmd()
126 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0; in rtw89_pci_release_fwcmd()
128 while (qlen--) { in rtw89_pci_release_fwcmd()
129 skb = skb_dequeue(&rtwpci->h2c_release_queue); in rtw89_pci_release_fwcmd()
135 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, in rtw89_pci_release_fwcmd()
144 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12]; in rtw89_pci_reclaim_tx_fwcmd()
156 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; in rtw89_pci_rxbd_recalc()
157 u32 addr_idx = bd_ring->addr.idx; in rtw89_pci_rxbd_recalc()
173 dma = rx_info->dma; in rtw89_pci_sync_skb_for_cpu()
174 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE, in rtw89_pci_sync_skb_for_cpu()
185 dma = rx_info->dma; in rtw89_pci_sync_skb_for_device()
186 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE, in rtw89_pci_sync_skb_for_device()
197 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data; in rtw89_pci_rxbd_info_update()
198 info = rxbd_info->dword; in rtw89_pci_rxbd_info_update()
200 rx_info->fs = le32_get_bits(info, RTW89_PCI_RXBD_FS); in rtw89_pci_rxbd_info_update()
201 rx_info->ls = le32_get_bits(info, RTW89_PCI_RXBD_LS); in rtw89_pci_rxbd_info_update()
202 rx_info->len = le32_get_bits(info, RTW89_PCI_RXBD_WRITE_SIZE); in rtw89_pci_rxbd_info_update()
203 rx_info->tag = le32_get_bits(info, RTW89_PCI_RXBD_TAG); in rtw89_pci_rxbd_info_update()
211 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_validate_rx_tag()
214 if (!info->check_rx_tag) in rtw89_pci_validate_rx_tag()
218 if (rx_ring->target_rx_tag == 0) in rtw89_pci_validate_rx_tag()
221 target_rx_tag = rx_ring->target_rx_tag; in rtw89_pci_validate_rx_tag()
223 if (rx_info->tag != target_rx_tag) { in rtw89_pci_validate_rx_tag()
225 rx_info->tag, target_rx_tag); in rtw89_pci_validate_rx_tag()
226 return -EAGAIN; in rtw89_pci_validate_rx_tag()
246 if (ret != -EAGAIN) in rtw89_pci_sync_skb_for_device_and_validate_rx_info()
248 } while (rx_tag_retry--); in rtw89_pci_sync_skb_for_device_and_validate_rx_info()
251 rx_ring->target_rx_tag = rx_info->tag + 1; in rtw89_pci_sync_skb_for_device_and_validate_rx_info()
258 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ctrl_txdma_ch_ax()
259 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; in rtw89_pci_ctrl_txdma_ch_ax()
260 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2; in rtw89_pci_ctrl_txdma_ch_ax()
263 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask); in rtw89_pci_ctrl_txdma_ch_ax()
264 if (dma_stop2->addr) in rtw89_pci_ctrl_txdma_ch_ax()
265 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask); in rtw89_pci_ctrl_txdma_ch_ax()
267 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask); in rtw89_pci_ctrl_txdma_ch_ax()
268 if (dma_stop2->addr) in rtw89_pci_ctrl_txdma_ch_ax()
269 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask); in rtw89_pci_ctrl_txdma_ch_ax()
275 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ctrl_txdma_fw_ch_ax()
276 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; in rtw89_pci_ctrl_txdma_fw_ch_ax()
279 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12); in rtw89_pci_ctrl_txdma_fw_ch_ax()
281 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12); in rtw89_pci_ctrl_txdma_fw_ch_ax()
291 u32 copy_len = rx_info->len - offset; in rtw89_skb_put_rx_data()
296 rx_info->len, desc_info->pkt_size, offset, fs, ls); in rtw89_skb_put_rx_data()
298 skb->data, rx_info->len); in rtw89_skb_put_rx_data()
299 /* length of a single segment skb is desc_info->pkt_size */ in rtw89_skb_put_rx_data()
301 copy_len = desc_info->pkt_size; in rtw89_skb_put_rx_data()
308 skb_put_data(new, skb->data + offset, copy_len); in rtw89_skb_put_rx_data()
316 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_get_rx_skb_idx()
317 u32 wp = bd_ring->wp; in rtw89_pci_get_rx_skb_idx()
319 if (!info->rx_ring_eq_is_full) in rtw89_pci_get_rx_skb_idx()
322 if (++wp >= bd_ring->len) in rtw89_pci_get_rx_skb_idx()
331 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc; in rtw89_pci_rxbd_deliver_skbs()
332 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; in rtw89_pci_rxbd_deliver_skbs()
333 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_rxbd_deliver_skbs()
334 struct sk_buff *new = rx_ring->diliver_skb; in rtw89_pci_rxbd_deliver_skbs()
345 skb = rx_ring->buf[skb_idx]; in rtw89_pci_rxbd_deliver_skbs()
350 bd_ring->wp, ret); in rtw89_pci_rxbd_deliver_skbs()
355 fs = info->no_rxbd_fs ? !new : rx_info->fs; in rtw89_pci_rxbd_deliver_skbs()
356 ls = rx_info->ls; in rtw89_pci_rxbd_deliver_skbs()
360 "unexpected fs/ls=%d/%d tag=%u len=%u new->len=%u\n", in rtw89_pci_rxbd_deliver_skbs()
361 fs, ls, rx_info->tag, rx_info->len, new ? new->len : 0); in rtw89_pci_rxbd_deliver_skbs()
369 if (desc_info->ready) { in rtw89_pci_rxbd_deliver_skbs()
374 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size); in rtw89_pci_rxbd_deliver_skbs()
376 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size); in rtw89_pci_rxbd_deliver_skbs()
380 rx_ring->diliver_skb = new; in rtw89_pci_rxbd_deliver_skbs()
383 offset = desc_info->offset + desc_info->rxd_len; in rtw89_pci_rxbd_deliver_skbs()
387 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n"); in rtw89_pci_rxbd_deliver_skbs()
396 if (!desc_info->ready) { in rtw89_pci_rxbd_deliver_skbs()
397 rtw89_warn(rtwdev, "no rx desc information\n"); in rtw89_pci_rxbd_deliver_skbs()
402 rx_ring->diliver_skb = NULL; in rtw89_pci_rxbd_deliver_skbs()
403 desc_info->ready = false; in rtw89_pci_rxbd_deliver_skbs()
414 rx_ring->diliver_skb = NULL; in rtw89_pci_rxbd_deliver_skbs()
415 desc_info->ready = false; in rtw89_pci_rxbd_deliver_skbs()
424 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; in rtw89_pci_rxbd_deliver()
427 while (cnt && rtwdev->napi_budget_countdown > 0) { in rtw89_pci_rxbd_deliver()
437 cnt -= rx_cnt; in rtw89_pci_rxbd_deliver()
440 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp); in rtw89_pci_rxbd_deliver()
447 int countdown = rtwdev->napi_budget_countdown; in rtw89_pci_poll_rxq_dma()
450 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ]; in rtw89_pci_poll_rxq_dma()
461 if (rtwdev->napi_budget_countdown <= 0) in rtw89_pci_poll_rxq_dma()
464 return budget - countdown; in rtw89_pci_poll_rxq_dma()
479 if (info->flags & IEEE80211_TX_CTL_NO_ACK) in rtw89_pci_tx_status()
480 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; in rtw89_pci_tx_status()
482 info->flags |= IEEE80211_TX_STAT_ACK; in rtw89_pci_tx_status()
483 tx_ring->tx_acked++; in rtw89_pci_tx_status()
485 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) in rtw89_pci_tx_status()
490 tx_ring->tx_retry_lmt++; in rtw89_pci_tx_status()
493 tx_ring->tx_life_time++; in rtw89_pci_tx_status()
496 tx_ring->tx_mac_id_drop++; in rtw89_pci_tx_status()
504 ieee80211_tx_status_ni(rtwdev->hw, skb); in rtw89_pci_tx_status()
513 while (cnt--) { in rtw89_pci_reclaim_txbd()
514 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list); in rtw89_pci_reclaim_txbd()
516 rtw89_warn(rtwdev, "No busy txwd pages available\n"); in rtw89_pci_reclaim_txbd()
520 list_del_init(&txwd->list); in rtw89_pci_reclaim_txbd()
523 if (skb_queue_len(&txwd->queue) == 0) in rtw89_pci_reclaim_txbd()
531 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in rtw89_pci_release_busy_txwd()
535 for (i = 0; i < wd_ring->page_num; i++) { in rtw89_pci_release_busy_txwd()
536 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list); in rtw89_pci_release_busy_txwd()
540 list_del_init(&txwd->list); in rtw89_pci_release_busy_txwd()
549 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_release_txwd_skb()
552 u8 txch = tx_ring->txch; in rtw89_pci_release_txwd_skb()
554 if (!list_empty(&txwd->list)) { in rtw89_pci_release_txwd_skb()
559 if (!rtwpci->low_power && !list_empty(&txwd->list)) in rtw89_pci_release_txwd_skb()
564 skb_queue_walk_safe(&txwd->queue, skb, tmp) { in rtw89_pci_release_txwd_skb()
565 skb_unlink(skb, &txwd->queue); in rtw89_pci_release_txwd_skb()
568 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, in rtw89_pci_release_txwd_skb()
574 if (list_empty(&txwd->list)) in rtw89_pci_release_txwd_skb()
581 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_release_rpp()
588 seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ); in rtw89_pci_release_rpp()
589 qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL); in rtw89_pci_release_rpp()
590 tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS); in rtw89_pci_release_rpp()
594 rtw89_warn(rtwdev, "should no fwcmd release report\n"); in rtw89_pci_release_rpp()
598 tx_ring = &rtwpci->tx_rings[txch]; in rtw89_pci_release_rpp()
599 wd_ring = &tx_ring->wd_ring; in rtw89_pci_release_rpp()
600 txwd = &wd_ring->pages[seq]; in rtw89_pci_release_rpp()
608 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in rtw89_pci_release_pending_txwd_skb()
612 for (i = 0; i < wd_ring->page_num; i++) { in rtw89_pci_release_pending_txwd_skb()
613 txwd = &wd_ring->pages[i]; in rtw89_pci_release_pending_txwd_skb()
615 if (!list_empty(&txwd->list)) in rtw89_pci_release_pending_txwd_skb()
626 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; in rtw89_pci_release_tx_skbs()
639 skb = rx_ring->buf[skb_idx]; in rtw89_pci_release_tx_skbs()
644 bd_ring->wp, ret); in rtw89_pci_release_tx_skbs()
649 if (!rx_info->fs || !rx_info->ls) { in rtw89_pci_release_tx_skbs()
654 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size); in rtw89_pci_release_tx_skbs()
658 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) { in rtw89_pci_release_tx_skbs()
659 rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset); in rtw89_pci_release_tx_skbs()
678 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; in rtw89_pci_release_tx()
691 cnt -= release_cnt; in rtw89_pci_release_tx()
694 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp); in rtw89_pci_release_tx()
704 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ]; in rtw89_pci_poll_rpq_dma()
706 spin_lock_bh(&rtwpci->trx_lock); in rtw89_pci_poll_rpq_dma()
715 spin_unlock_bh(&rtwpci->trx_lock); in rtw89_pci_poll_rpq_dma()
719 rtwdev->napi_budget_countdown -= work_done; in rtw89_pci_poll_rpq_dma()
734 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_isr_rxd_unavail()
735 bd_ring = &rx_ring->bd_ring; in rtw89_pci_isr_rxd_unavail()
737 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx); in rtw89_pci_isr_rxd_unavail()
740 hw_idx_next = (hw_idx + 1) % bd_ring->len; in rtw89_pci_isr_rxd_unavail()
747 i, reg_idx, bd_ring->len); in rtw89_pci_isr_rxd_unavail()
755 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs; in rtw89_pci_recognize_intrs()
756 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0]; in rtw89_pci_recognize_intrs()
757 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1]; in rtw89_pci_recognize_intrs()
759 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs); in rtw89_pci_recognize_intrs()
760 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]); in rtw89_pci_recognize_intrs()
761 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]); in rtw89_pci_recognize_intrs()
769 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs; in rtw89_pci_recognize_intrs_v1()
770 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ? in rtw89_pci_recognize_intrs_v1()
771 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0; in rtw89_pci_recognize_intrs_v1()
772 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ? in rtw89_pci_recognize_intrs_v1()
773 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0; in rtw89_pci_recognize_intrs_v1()
774 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ? in rtw89_pci_recognize_intrs_v1()
775 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0; in rtw89_pci_recognize_intrs_v1()
777 if (isrs->halt_c2h_isrs) in rtw89_pci_recognize_intrs_v1()
778 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs); in rtw89_pci_recognize_intrs_v1()
779 if (isrs->isrs[0]) in rtw89_pci_recognize_intrs_v1()
780 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]); in rtw89_pci_recognize_intrs_v1()
781 if (isrs->isrs[1]) in rtw89_pci_recognize_intrs_v1()
782 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]); in rtw89_pci_recognize_intrs_v1()
790 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs; in rtw89_pci_recognize_intrs_v2()
791 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ? in rtw89_pci_recognize_intrs_v2()
792 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0; in rtw89_pci_recognize_intrs_v2()
793 isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ? in rtw89_pci_recognize_intrs_v2()
794 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0; in rtw89_pci_recognize_intrs_v2()
795 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1]; in rtw89_pci_recognize_intrs_v2()
797 if (isrs->halt_c2h_isrs) in rtw89_pci_recognize_intrs_v2()
798 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs); in rtw89_pci_recognize_intrs_v2()
799 if (isrs->isrs[0]) in rtw89_pci_recognize_intrs_v2()
800 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]); in rtw89_pci_recognize_intrs_v2()
801 if (isrs->isrs[1]) in rtw89_pci_recognize_intrs_v2()
802 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]); in rtw89_pci_recognize_intrs_v2()
803 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs); in rtw89_pci_recognize_intrs_v2()
809 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs); in rtw89_pci_enable_intr()
810 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]); in rtw89_pci_enable_intr()
811 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]); in rtw89_pci_enable_intr()
825 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs); in rtw89_pci_enable_intr_v1()
826 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs); in rtw89_pci_enable_intr_v1()
827 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]); in rtw89_pci_enable_intr_v1()
828 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]); in rtw89_pci_enable_intr_v1()
840 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs); in rtw89_pci_enable_intr_v2()
841 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]); in rtw89_pci_enable_intr_v2()
842 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]); in rtw89_pci_enable_intr_v2()
843 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs); in rtw89_pci_enable_intr_v2()
856 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_recovery_start()
859 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_ops_recovery_start()
863 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_ops_recovery_start()
868 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_recovery_complete()
871 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_ops_recovery_complete()
875 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_ops_recovery_complete()
880 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_low_power_interrupt_handler()
884 rtwdev->napi_budget_countdown = budget; in rtw89_pci_low_power_interrupt_handler()
893 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_interrupt_threadfn()
894 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_interrupt_threadfn()
895 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_interrupt_threadfn()
899 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_threadfn()
901 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_threadfn()
903 if (unlikely(isrs.isrs[0] & gen_def->isr_rdu)) in rtw89_pci_interrupt_threadfn()
906 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_halt_c2h)) in rtw89_pci_interrupt_threadfn()
909 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_wdt_timeout)) in rtw89_pci_interrupt_threadfn()
912 if (unlikely(rtwpci->under_recovery)) in rtw89_pci_interrupt_threadfn()
915 if (unlikely(rtwpci->low_power)) { in rtw89_pci_interrupt_threadfn()
920 if (likely(rtwpci->running)) { in rtw89_pci_interrupt_threadfn()
922 napi_schedule(&rtwdev->napi); in rtw89_pci_interrupt_threadfn()
929 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_threadfn()
930 if (likely(rtwpci->running)) in rtw89_pci_interrupt_threadfn()
932 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_threadfn()
939 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_interrupt_handler()
943 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_handler()
948 if (unlikely(!rtwpci->running)) { in rtw89_pci_interrupt_handler()
955 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_interrupt_handler()
1072 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_get_txch_addrs()
1075 return -EINVAL; in rtw89_pci_get_txch_addrs()
1077 *addr = &info->dma_addr_set->tx[txch]; in rtw89_pci_get_txch_addrs()
1086 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_get_rxch_addrs()
1089 return -EINVAL; in rtw89_pci_get_rxch_addrs()
1091 *addr = &info->dma_addr_set->rx[rxch]; in rtw89_pci_get_rxch_addrs()
1098 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring; in rtw89_pci_get_avail_txbd_num()
1101 if (bd_ring->rp > bd_ring->wp) in rtw89_pci_get_avail_txbd_num()
1102 return bd_ring->rp - bd_ring->wp - 1; in rtw89_pci_get_avail_txbd_num()
1104 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1; in rtw89_pci_get_avail_txbd_num()
1110 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in __rtw89_pci_check_and_reclaim_tx_fwcmd_resource()
1111 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12]; in __rtw89_pci_check_and_reclaim_tx_fwcmd_resource()
1114 spin_lock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_fwcmd_resource()
1117 spin_unlock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_fwcmd_resource()
1126 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1127 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1128 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1131 spin_lock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1134 cnt = min(cnt, wd_ring->curr_num); in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1135 spin_unlock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_resource_noio()
1143 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in __rtw89_pci_check_and_reclaim_tx_resource()
1144 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; in __rtw89_pci_check_and_reclaim_tx_resource()
1145 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in __rtw89_pci_check_and_reclaim_tx_resource()
1146 const struct rtw89_chip_info *chip = rtwdev->chip; in __rtw89_pci_check_and_reclaim_tx_resource()
1152 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ]; in __rtw89_pci_check_and_reclaim_tx_resource()
1154 spin_lock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_resource()
1156 wd_cnt = wd_ring->curr_num; in __rtw89_pci_check_and_reclaim_tx_resource()
1171 wd_cnt = wd_ring->curr_num; in __rtw89_pci_check_and_reclaim_tx_resource()
1178 if (rtwpci->low_power || chip->small_fifo_size) in __rtw89_pci_check_and_reclaim_tx_resource()
1184 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n", in __rtw89_pci_check_and_reclaim_tx_resource()
1189 spin_unlock_bh(&rtwpci->trx_lock); in __rtw89_pci_check_and_reclaim_tx_resource()
1197 if (rtwdev->hci.paused) in rtw89_pci_check_and_reclaim_tx_resource()
1208 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in __rtw89_pci_tx_kick_off()
1209 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; in __rtw89_pci_tx_kick_off()
1212 spin_lock_bh(&rtwpci->trx_lock); in __rtw89_pci_tx_kick_off()
1214 addr = bd_ring->addr.idx; in __rtw89_pci_tx_kick_off()
1215 host_idx = bd_ring->wp; in __rtw89_pci_tx_kick_off()
1218 spin_unlock_bh(&rtwpci->trx_lock); in __rtw89_pci_tx_kick_off()
1224 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; in rtw89_pci_tx_bd_ring_update()
1227 len = bd_ring->len; in rtw89_pci_tx_bd_ring_update()
1228 host_idx = bd_ring->wp + n_txbd; in rtw89_pci_tx_bd_ring_update()
1229 host_idx = host_idx < len ? host_idx : host_idx - len; in rtw89_pci_tx_bd_ring_update()
1231 bd_ring->wp = host_idx; in rtw89_pci_tx_bd_ring_update()
1236 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_tx_kick_off()
1237 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; in rtw89_pci_ops_tx_kick_off()
1239 if (rtwdev->hci.paused) { in rtw89_pci_ops_tx_kick_off()
1240 set_bit(txch, rtwpci->kick_map); in rtw89_pci_ops_tx_kick_off()
1249 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_tx_kick_off_pending()
1254 if (!test_and_clear_bit(txch, rtwpci->kick_map)) in rtw89_pci_tx_kick_off_pending()
1257 tx_ring = &rtwpci->tx_rings[txch]; in rtw89_pci_tx_kick_off_pending()
1264 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in __pci_flush_txch()
1265 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; in __pci_flush_txch()
1266 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; in __pci_flush_txch()
1276 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx); in __pci_flush_txch()
1278 if (cur_rp == bd_ring->wp) in __pci_flush_txch()
1291 const struct rtw89_pci_info *info = rtwdev->pci_info; in __rtw89_pci_ops_flush_txchs()
1298 if (info->tx_dma_ch_mask & BIT(i)) in __rtw89_pci_ops_flush_txchs()
1309 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop); in rtw89_pci_ops_flush_queues()
1319 txaddr_info->length = cpu_to_le16(total_len); in rtw89_pci_fill_txaddr_info()
1322 txaddr_info->option = option; in rtw89_pci_fill_txaddr_info()
1323 txaddr_info->dma = cpu_to_le32(dma); in rtw89_pci_fill_txaddr_info()
1344 remain -= len; in rtw89_pci_fill_txaddr_info_v1()
1351 txaddr_info->length_opt = cpu_to_le16(length_option); in rtw89_pci_fill_txaddr_info_v1()
1352 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma)); in rtw89_pci_fill_txaddr_info_v1()
1353 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma)); in rtw89_pci_fill_txaddr_info_v1()
1373 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_txwd_submit()
1374 const struct rtw89_chip_info *chip = rtwdev->chip; in rtw89_pci_txwd_submit()
1375 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; in rtw89_pci_txwd_submit()
1378 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_txwd_submit()
1379 struct sk_buff *skb = tx_req->skb; in rtw89_pci_txwd_submit()
1382 bool en_wd_info = desc_info->en_wd_info; in rtw89_pci_txwd_submit()
1389 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); in rtw89_pci_txwd_submit()
1390 if (dma_mapping_error(&pdev->dev, dma)) { in rtw89_pci_txwd_submit()
1392 ret = -EBUSY; in rtw89_pci_txwd_submit()
1396 tx_data->dma = dma; in rtw89_pci_txwd_submit()
1397 rcu_assign_pointer(skb_data->wait, NULL); in rtw89_pci_txwd_submit()
1400 txwd_len = chip->txwd_body_size; in rtw89_pci_txwd_submit()
1401 txwd_len += en_wd_info ? chip->txwd_info_size : 0; in rtw89_pci_txwd_submit()
1404 txwp_info = txwd->vaddr + txwd_len; in rtw89_pci_txwd_submit()
1406 txwp_info = (struct rtw89_pci_tx_wp_info *)((u8 *)txwd->vaddr + txwd_len); in rtw89_pci_txwd_submit()
1408 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID); in rtw89_pci_txwd_submit()
1409 txwp_info->seq1 = 0; in rtw89_pci_txwd_submit()
1410 txwp_info->seq2 = 0; in rtw89_pci_txwd_submit()
1411 txwp_info->seq3 = 0; in rtw89_pci_txwd_submit()
1413 tx_ring->tx_cnt++; in rtw89_pci_txwd_submit()
1415 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len; in rtw89_pci_txwd_submit()
1417 txaddr_info_addr = (u8 *)txwd->vaddr + txwd_len + txwp_len; in rtw89_pci_txwd_submit()
1420 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len, in rtw89_pci_txwd_submit()
1421 dma, &desc_info->addr_info_nr); in rtw89_pci_txwd_submit()
1423 txwd->len = txwd_len + txwp_len + txaddr_info_len; in rtw89_pci_txwd_submit()
1425 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr); in rtw89_pci_txwd_submit()
1427 skb_queue_tail(&txwd->queue, skb); in rtw89_pci_txwd_submit()
1440 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_fwcmd_submit()
1441 const struct rtw89_chip_info *chip = rtwdev->chip; in rtw89_pci_fwcmd_submit()
1442 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; in rtw89_pci_fwcmd_submit()
1444 int txdesc_size = chip->h2c_desc_size; in rtw89_pci_fwcmd_submit()
1445 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_fwcmd_submit()
1446 struct sk_buff *skb = tx_req->skb; in rtw89_pci_fwcmd_submit()
1455 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); in rtw89_pci_fwcmd_submit()
1456 if (dma_mapping_error(&pdev->dev, dma)) { in rtw89_pci_fwcmd_submit()
1458 return -EBUSY; in rtw89_pci_fwcmd_submit()
1461 tx_data->dma = dma; in rtw89_pci_fwcmd_submit()
1464 txbd->opt = opt; in rtw89_pci_fwcmd_submit()
1465 txbd->length = cpu_to_le16(skb->len); in rtw89_pci_fwcmd_submit()
1466 txbd->dma = cpu_to_le32(tx_data->dma); in rtw89_pci_fwcmd_submit()
1467 skb_queue_tail(&rtwpci->h2c_queue, skb); in rtw89_pci_fwcmd_submit()
1487 if (tx_ring->txch == RTW89_TXCH_CH12) in rtw89_pci_txbd_submit()
1492 rtw89_err(rtwdev, "no available TXWD\n"); in rtw89_pci_txbd_submit()
1493 ret = -ENOSPC; in rtw89_pci_txbd_submit()
1499 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq); in rtw89_pci_txbd_submit()
1503 list_add_tail(&txwd->list, &tx_ring->busy_pages); in rtw89_pci_txbd_submit()
1506 opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI); in rtw89_pci_txbd_submit()
1507 txbd->opt = opt; in rtw89_pci_txbd_submit()
1508 txbd->length = cpu_to_le16(txwd->len); in rtw89_pci_txbd_submit()
1509 txbd->dma = cpu_to_le32(txwd->paddr); in rtw89_pci_txbd_submit()
1524 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_tx_write()
1532 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) && in rtw89_pci_tx_write()
1534 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) { in rtw89_pci_tx_write()
1536 return -EINVAL; in rtw89_pci_tx_write()
1539 tx_ring = &rtwpci->tx_rings[txch]; in rtw89_pci_tx_write()
1540 spin_lock_bh(&rtwpci->trx_lock); in rtw89_pci_tx_write()
1544 rtw89_err(rtwdev, "no available TXBD\n"); in rtw89_pci_tx_write()
1545 ret = -ENOSPC; in rtw89_pci_tx_write()
1556 spin_unlock_bh(&rtwpci->trx_lock); in rtw89_pci_tx_write()
1560 spin_unlock_bh(&rtwpci->trx_lock); in rtw89_pci_tx_write()
1566 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; in rtw89_pci_ops_tx_write()
1569 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma); in rtw89_pci_ops_tx_write()
1571 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma); in rtw89_pci_ops_tx_write()
1608 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_init_wp_16sel()
1609 u32 addr = info->wp_sel_addr; in rtw89_pci_init_wp_16sel()
1613 if (!info->wp_sel_addr) in rtw89_pci_init_wp_16sel()
1627 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_reset_trx_rings()
1628 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_reset_trx_rings()
1629 const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table; in rtw89_pci_reset_trx_rings()
1642 if (info->tx_dma_ch_mask & BIT(i)) in rtw89_pci_reset_trx_rings()
1645 tx_ring = &rtwpci->tx_rings[i]; in rtw89_pci_reset_trx_rings()
1646 bd_ring = &tx_ring->bd_ring; in rtw89_pci_reset_trx_rings()
1648 addr_num = bd_ring->addr.num; in rtw89_pci_reset_trx_rings()
1649 addr_bdram = bd_ring->addr.bdram; in rtw89_pci_reset_trx_rings()
1650 addr_desa_l = bd_ring->addr.desa_l; in rtw89_pci_reset_trx_rings()
1651 bd_ring->wp = 0; in rtw89_pci_reset_trx_rings()
1652 bd_ring->rp = 0; in rtw89_pci_reset_trx_rings()
1654 rtw89_write16(rtwdev, addr_num, bd_ring->len); in rtw89_pci_reset_trx_rings()
1656 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) | in rtw89_pci_reset_trx_rings()
1657 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) | in rtw89_pci_reset_trx_rings()
1658 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num); in rtw89_pci_reset_trx_rings()
1662 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); in rtw89_pci_reset_trx_rings()
1663 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); in rtw89_pci_reset_trx_rings()
1667 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_reset_trx_rings()
1668 bd_ring = &rx_ring->bd_ring; in rtw89_pci_reset_trx_rings()
1669 addr_num = bd_ring->addr.num; in rtw89_pci_reset_trx_rings()
1670 addr_idx = bd_ring->addr.idx; in rtw89_pci_reset_trx_rings()
1671 addr_desa_l = bd_ring->addr.desa_l; in rtw89_pci_reset_trx_rings()
1672 if (info->rx_ring_eq_is_full) in rtw89_pci_reset_trx_rings()
1673 bd_ring->wp = bd_ring->len - 1; in rtw89_pci_reset_trx_rings()
1675 bd_ring->wp = 0; in rtw89_pci_reset_trx_rings()
1676 bd_ring->rp = 0; in rtw89_pci_reset_trx_rings()
1677 rx_ring->diliver_skb = NULL; in rtw89_pci_reset_trx_rings()
1678 rx_ring->diliver_desc.ready = false; in rtw89_pci_reset_trx_rings()
1679 rx_ring->target_rx_tag = 0; in rtw89_pci_reset_trx_rings()
1681 rtw89_write16(rtwdev, addr_num, bd_ring->len); in rtw89_pci_reset_trx_rings()
1682 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); in rtw89_pci_reset_trx_rings()
1683 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); in rtw89_pci_reset_trx_rings()
1685 if (info->rx_ring_eq_is_full) in rtw89_pci_reset_trx_rings()
1686 rtw89_write16(rtwdev, addr_idx, bd_ring->wp); in rtw89_pci_reset_trx_rings()
1701 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_reset()
1702 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ops_reset()
1707 spin_lock_bh(&rtwpci->trx_lock); in rtw89_pci_ops_reset()
1709 if (info->tx_dma_ch_mask & BIT(txch)) in rtw89_pci_ops_reset()
1713 skb_queue_len(&rtwpci->h2c_queue), true); in rtw89_pci_ops_reset()
1716 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]); in rtw89_pci_ops_reset()
1718 spin_unlock_bh(&rtwpci->trx_lock); in rtw89_pci_ops_reset()
1723 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_enable_intr_lock()
1726 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_enable_intr_lock()
1727 rtwpci->running = true; in rtw89_pci_enable_intr_lock()
1729 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_enable_intr_lock()
1734 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_disable_intr_lock()
1737 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_disable_intr_lock()
1738 rtwpci->running = false; in rtw89_pci_disable_intr_lock()
1740 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_disable_intr_lock()
1753 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_stop()
1754 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_ops_stop()
1757 synchronize_irq(pdev->irq); in rtw89_pci_ops_stop()
1763 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_pause()
1764 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_ops_pause()
1768 synchronize_irq(pdev->irq); in rtw89_pci_ops_pause()
1769 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags)) in rtw89_pci_ops_pause()
1770 napi_synchronize(&rtwdev->napi); in rtw89_pci_ops_pause()
1780 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_switch_bd_idx_addr()
1781 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_switch_bd_idx_addr()
1782 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power; in rtw89_pci_switch_bd_idx_addr()
1783 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set; in rtw89_pci_switch_bd_idx_addr()
1792 tx_ring = &rtwpci->tx_rings[i]; in rtw89_pci_switch_bd_idx_addr()
1793 tx_ring->bd_ring.addr.idx = low_power ? in rtw89_pci_switch_bd_idx_addr()
1794 bd_idx_addr->tx_bd_addrs[i] : in rtw89_pci_switch_bd_idx_addr()
1795 dma_addr_set->tx[i].idx; in rtw89_pci_switch_bd_idx_addr()
1799 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_switch_bd_idx_addr()
1800 rx_ring->bd_ring.addr.idx = low_power ? in rtw89_pci_switch_bd_idx_addr()
1801 bd_idx_addr->rx_bd_addrs[i] : in rtw89_pci_switch_bd_idx_addr()
1802 dma_addr_set->rx[i].idx; in rtw89_pci_switch_bd_idx_addr()
1810 WARN(!rtwdev->hci.paused, "HCI isn't paused\n"); in rtw89_pci_ops_switch_mode()
1821 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_read32_cmac()
1823 u32 val = readl(rtwpci->mmap + addr); in rtw89_pci_ops_read32_cmac()
1827 val = bus_read_4((struct resource *)rtwpci->mmap, addr); in rtw89_pci_ops_read32_cmac()
1828 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); in rtw89_pci_ops_read32_cmac()
1841 val = readl(rtwpci->mmap + addr); in rtw89_pci_ops_read32_cmac()
1843 val = bus_read_4((struct resource *)rtwpci->mmap, addr); in rtw89_pci_ops_read32_cmac()
1844 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); in rtw89_pci_ops_read32_cmac()
1853 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_read8()
1858 return readb(rtwpci->mmap + addr); in rtw89_pci_ops_read8()
1863 val = bus_read_1((struct resource *)rtwpci->mmap, addr); in rtw89_pci_ops_read8()
1864 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val); in rtw89_pci_ops_read8()
1877 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_read16()
1882 return readw(rtwpci->mmap + addr); in rtw89_pci_ops_read16()
1887 val = bus_read_2((struct resource *)rtwpci->mmap, addr); in rtw89_pci_ops_read16()
1888 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val); in rtw89_pci_ops_read16()
1901 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_read32()
1905 return readl(rtwpci->mmap + addr); in rtw89_pci_ops_read32()
1910 val = bus_read_4((struct resource *)rtwpci->mmap, addr); in rtw89_pci_ops_read32()
1911 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); in rtw89_pci_ops_read32()
1921 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_write8()
1924 writeb(data, rtwpci->mmap + addr); in rtw89_pci_ops_write8()
1926 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, data); in rtw89_pci_ops_write8()
1927 return (bus_write_1((struct resource *)rtwpci->mmap, addr, data)); in rtw89_pci_ops_write8()
1933 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_write16()
1936 writew(data, rtwpci->mmap + addr); in rtw89_pci_ops_write16()
1938 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, data); in rtw89_pci_ops_write16()
1939 return (bus_write_2((struct resource *)rtwpci->mmap, addr, data)); in rtw89_pci_ops_write16()
1945 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_ops_write32()
1948 writel(data, rtwpci->mmap + addr); in rtw89_pci_ops_write32()
1950 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, data); in rtw89_pci_ops_write32()
1951 return (bus_write_4((struct resource *)rtwpci->mmap, addr, data)); in rtw89_pci_ops_write32()
1957 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ctrl_dma_trx()
1960 rtw89_write32_set(rtwdev, info->init_cfg_reg, in rtw89_pci_ctrl_dma_trx()
1961 info->rxhci_en_bit | info->txhci_en_bit); in rtw89_pci_ctrl_dma_trx()
1963 rtw89_write32_clr(rtwdev, info->init_cfg_reg, in rtw89_pci_ctrl_dma_trx()
1964 info->rxhci_en_bit | info->txhci_en_bit); in rtw89_pci_ctrl_dma_trx()
1969 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ctrl_dma_io()
1970 const struct rtw89_reg_def *reg = &info->dma_io_stop; in rtw89_pci_ctrl_dma_io()
1973 rtw89_write32_clr(rtwdev, reg->addr, reg->mask); in rtw89_pci_ctrl_dma_io()
1975 rtw89_write32_set(rtwdev, reg->addr, reg->mask); in rtw89_pci_ctrl_dma_io()
2006 return -EINVAL; in rtw89_pci_check_mdio()
2147 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_write_config_byte()
2148 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_write_config_byte()
2149 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_write_config_byte()
2165 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_read_config_byte()
2166 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_read_config_byte()
2167 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_read_config_byte()
2244 return -EINVAL; in __get_target()
2288 return -EOPNOTSUPP; in rtw89_pci_auto_refclk_cal()
2339 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar; in rtw89_pci_auto_refclk_cal()
2417 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_deglitch_setting()
2449 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_disable_eq_ax()
2513 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks)) in rtw89_pci_ber()
2527 if (rtwdev->chip->chip_id != RTL8852A) in rtw89_pci_rxdma_prefth()
2535 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_l1off_pwroff()
2547 if (rtwdev->chip->chip_id != RTL8852A) in rtw89_pci_l2_rxen_lat()
2565 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_aphy_pwrcut()
2575 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_hci_ldo()
2582 } else if (rtwdev->chip->chip_id == RTL8852C) { in rtw89_pci_hci_ldo()
2607 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_autoload_hang()
2616 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV)) in rtw89_pci_l12_vmain()
2624 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV)) in rtw89_pci_gen2_force_ib()
2636 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_l1_ent_lat()
2644 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_wd_exit_l1()
2652 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_set_sic()
2661 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_set_lbc()
2664 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_set_lbc()
2668 if (info->lbc_en == MAC_AX_PCIE_ENABLE) { in rtw89_pci_set_lbc()
2669 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER); in rtw89_pci_set_lbc()
2680 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_set_io_rcy()
2683 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_set_io_rcy()
2686 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) { in rtw89_pci_set_io_rcy()
2688 info->io_rcy_tmr); in rtw89_pci_set_io_rcy()
2707 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_set_dbg()
2717 if (rtwdev->chip->chip_id == RTL8852A) in rtw89_pci_set_dbg()
2724 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_set_keep_reg()
2733 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_clr_idx_all_ax()
2734 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_clr_idx_all_ax()
2738 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg; in rtw89_pci_clr_idx_all_ax()
2739 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg; in rtw89_pci_clr_idx_all_ax()
2755 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_poll_txdma_ch_idle_ax()
2756 u32 dma_busy1 = info->dma_busy1.addr; in rtw89_pci_poll_txdma_ch_idle_ax()
2757 u32 dma_busy2 = info->dma_busy2_reg; in rtw89_pci_poll_txdma_ch_idle_ax()
2761 check = info->dma_busy1.mask; in rtw89_pci_poll_txdma_ch_idle_ax()
2783 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_poll_rxdma_ch_idle_ax()
2784 u32 dma_busy3 = info->dma_busy3_reg; in rtw89_pci_poll_rxdma_ch_idle_ax()
2819 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_mode_op()
2820 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode; in rtw89_pci_mode_op()
2821 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode; in rtw89_pci_mode_op()
2822 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode; in rtw89_pci_mode_op()
2823 enum mac_ax_tag_mode tag_mode = info->tag_mode; in rtw89_pci_mode_op()
2824 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl; in rtw89_pci_mode_op()
2825 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl; in rtw89_pci_mode_op()
2826 enum mac_ax_tx_burst tx_burst = info->tx_burst; in rtw89_pci_mode_op()
2827 enum mac_ax_rx_burst rx_burst = info->rx_burst; in rtw89_pci_mode_op()
2828 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_mode_op()
2829 u8 cv = rtwdev->hal.cv; in rtw89_pci_mode_op()
2849 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit); in rtw89_pci_mode_op()
2851 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit); in rtw89_pci_mode_op()
2878 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask, in rtw89_pci_mode_op()
2879 info->multi_tag_num); in rtw89_pci_mode_op()
2908 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ops_deinit()
2912 if (rtwdev->chip->chip_id == RTL8852A) { in rtw89_pci_ops_deinit()
2916 info->ltr_set(rtwdev, false); in rtw89_pci_ops_deinit()
2925 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ops_mac_pre_init_ax()
2966 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA); in rtw89_pci_ops_mac_pre_init_ax()
3015 return -EINVAL; in rtw89_pci_ltr_set()
3018 return -EINVAL; in rtw89_pci_ltr_set()
3021 return -EINVAL; in rtw89_pci_ltr_set()
3024 return -EINVAL; in rtw89_pci_ltr_set()
3048 return -EINVAL; in rtw89_pci_ltr_set_v1()
3051 return -EINVAL; in rtw89_pci_ltr_set_v1()
3054 return -EINVAL; in rtw89_pci_ltr_set_v1()
3057 return -EINVAL; in rtw89_pci_ltr_set_v1()
3060 return -EINVAL; in rtw89_pci_ltr_set_v1()
3090 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ops_mac_post_init_ax()
3091 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_ops_mac_post_init_ax()
3094 ret = info->ltr_set(rtwdev, true); in rtw89_pci_ops_mac_post_init_ax()
3104 /* ADDR info 8-byte mode */ in rtw89_pci_ops_mac_post_init_ax()
3114 rtw89_write32_clr(rtwdev, info->dma_stop1.addr, in rtw89_pci_ops_mac_post_init_ax()
3123 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_claim_device()
3133 pci_set_drvdata(pdev, rtwdev->hw); in rtw89_pci_claim_device()
3135 rtwpci->pdev = pdev; in rtw89_pci_claim_device()
3148 const struct rtw89_chip_info *chip = rtwdev->chip; in rtw89_pci_chip_is_manual_dac()
3150 switch (chip->chip_id) { in rtw89_pci_chip_is_manual_dac()
3163 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_is_dac_compatible_bridge()
3164 struct pci_dev *bridge = pci_upstream_bridge(rtwpci->pdev); in rtw89_pci_is_dac_compatible_bridge()
3172 switch (bridge->vendor) { in rtw89_pci_is_dac_compatible_bridge()
3176 if (bridge->device == 0x2806) in rtw89_pci_is_dac_compatible_bridge()
3186 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_cfg_dac()
3187 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_cfg_dac()
3191 if (!rtwpci->enable_dac && !force) in rtw89_pci_cfg_dac()
3209 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_setup_mapping()
3223 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); in rtw89_pci_setup_mapping()
3227 rtwpci->enable_dac = true; in rtw89_pci_setup_mapping()
3231 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); in rtw89_pci_setup_mapping()
3234 "failed to set dma and consistent mask to 32/36-bit\n"); in rtw89_pci_setup_mapping()
3244 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len); in rtw89_pci_setup_mapping()
3245 if (!rtwpci->mmap) { in rtw89_pci_setup_mapping()
3247 ret = -EIO; in rtw89_pci_setup_mapping()
3262 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_clear_mapping()
3264 if (rtwpci->mmap) { in rtw89_pci_clear_mapping()
3265 pci_iounmap(pdev, rtwpci->mmap); in rtw89_pci_clear_mapping()
3274 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in rtw89_pci_free_tx_wd_ring()
3275 u8 *head = wd_ring->head; in rtw89_pci_free_tx_wd_ring()
3276 dma_addr_t dma = wd_ring->dma; in rtw89_pci_free_tx_wd_ring()
3277 u32 page_size = wd_ring->page_size; in rtw89_pci_free_tx_wd_ring()
3278 u32 page_num = wd_ring->page_num; in rtw89_pci_free_tx_wd_ring()
3281 dma_free_coherent(&pdev->dev, ring_sz, head, dma); in rtw89_pci_free_tx_wd_ring()
3282 wd_ring->head = NULL; in rtw89_pci_free_tx_wd_ring()
3293 head = tx_ring->bd_ring.head; in rtw89_pci_free_tx_ring()
3294 dma = tx_ring->bd_ring.dma; in rtw89_pci_free_tx_ring()
3295 ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len; in rtw89_pci_free_tx_ring()
3296 dma_free_coherent(&pdev->dev, ring_sz, head, dma); in rtw89_pci_free_tx_ring()
3298 tx_ring->bd_ring.head = NULL; in rtw89_pci_free_tx_ring()
3304 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_free_tx_rings()
3305 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_free_tx_rings()
3310 if (info->tx_dma_ch_mask & BIT(i)) in rtw89_pci_free_tx_rings()
3312 tx_ring = &rtwpci->tx_rings[i]; in rtw89_pci_free_tx_rings()
3327 int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len; in rtw89_pci_free_rx_ring()
3330 buf_sz = rx_ring->buf_sz; in rtw89_pci_free_rx_ring()
3331 for (i = 0; i < rx_ring->bd_ring.len; i++) { in rtw89_pci_free_rx_ring()
3332 skb = rx_ring->buf[i]; in rtw89_pci_free_rx_ring()
3337 dma = rx_info->dma; in rtw89_pci_free_rx_ring()
3338 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); in rtw89_pci_free_rx_ring()
3340 rx_ring->buf[i] = NULL; in rtw89_pci_free_rx_ring()
3343 head = rx_ring->bd_ring.head; in rtw89_pci_free_rx_ring()
3344 dma = rx_ring->bd_ring.dma; in rtw89_pci_free_rx_ring()
3345 dma_free_coherent(&pdev->dev, ring_sz, head, dma); in rtw89_pci_free_rx_ring()
3347 rx_ring->bd_ring.head = NULL; in rtw89_pci_free_rx_ring()
3353 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_free_rx_rings()
3358 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_free_rx_rings()
3379 return -EINVAL; in rtw89_pci_init_rx_bd()
3381 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); in rtw89_pci_init_rx_bd()
3382 if (dma_mapping_error(&pdev->dev, dma)) in rtw89_pci_init_rx_bd()
3383 return -EBUSY; in rtw89_pci_init_rx_bd()
3389 rx_bd->buf_size = cpu_to_le16(buf_sz); in rtw89_pci_init_rx_bd()
3390 rx_bd->dma = cpu_to_le32(dma); in rtw89_pci_init_rx_bd()
3391 rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI); in rtw89_pci_init_rx_bd()
3392 rx_info->dma = dma; in rtw89_pci_init_rx_bd()
3402 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; in rtw89_pci_alloc_tx_wd_ring()
3418 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); in rtw89_pci_alloc_tx_wd_ring()
3420 return -ENOMEM; in rtw89_pci_alloc_tx_wd_ring()
3422 INIT_LIST_HEAD(&wd_ring->free_pages); in rtw89_pci_alloc_tx_wd_ring()
3423 wd_ring->head = head; in rtw89_pci_alloc_tx_wd_ring()
3424 wd_ring->dma = dma; in rtw89_pci_alloc_tx_wd_ring()
3425 wd_ring->page_size = page_size; in rtw89_pci_alloc_tx_wd_ring()
3426 wd_ring->page_num = page_num; in rtw89_pci_alloc_tx_wd_ring()
3430 txwd = &wd_ring->pages[i]; in rtw89_pci_alloc_tx_wd_ring()
3434 skb_queue_head_init(&txwd->queue); in rtw89_pci_alloc_tx_wd_ring()
3435 INIT_LIST_HEAD(&txwd->list); in rtw89_pci_alloc_tx_wd_ring()
3436 txwd->paddr = cur_paddr; in rtw89_pci_alloc_tx_wd_ring()
3437 txwd->vaddr = cur_vaddr; in rtw89_pci_alloc_tx_wd_ring()
3438 txwd->len = page_size; in rtw89_pci_alloc_tx_wd_ring()
3439 txwd->seq = i; in rtw89_pci_alloc_tx_wd_ring()
3472 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); in rtw89_pci_alloc_tx_ring()
3474 ret = -ENOMEM; in rtw89_pci_alloc_tx_ring()
3478 INIT_LIST_HEAD(&tx_ring->busy_pages); in rtw89_pci_alloc_tx_ring()
3479 tx_ring->bd_ring.head = head; in rtw89_pci_alloc_tx_ring()
3480 tx_ring->bd_ring.dma = dma; in rtw89_pci_alloc_tx_ring()
3481 tx_ring->bd_ring.len = len; in rtw89_pci_alloc_tx_ring()
3482 tx_ring->bd_ring.desc_size = desc_size; in rtw89_pci_alloc_tx_ring()
3483 tx_ring->bd_ring.addr = *txch_addr; in rtw89_pci_alloc_tx_ring()
3484 tx_ring->bd_ring.wp = 0; in rtw89_pci_alloc_tx_ring()
3485 tx_ring->bd_ring.rp = 0; in rtw89_pci_alloc_tx_ring()
3486 tx_ring->txch = txch; in rtw89_pci_alloc_tx_ring()
3499 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_alloc_tx_rings()
3500 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_alloc_tx_rings()
3508 if (info->tx_dma_ch_mask & BIT(i)) in rtw89_pci_alloc_tx_rings()
3510 tx_ring = &rtwpci->tx_rings[i]; in rtw89_pci_alloc_tx_rings()
3530 tx_ring = &rtwpci->tx_rings[i]; in rtw89_pci_alloc_tx_rings()
3542 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_alloc_rx_ring()
3558 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); in rtw89_pci_alloc_rx_ring()
3560 ret = -ENOMEM; in rtw89_pci_alloc_rx_ring()
3564 rx_ring->bd_ring.head = head; in rtw89_pci_alloc_rx_ring()
3565 rx_ring->bd_ring.dma = dma; in rtw89_pci_alloc_rx_ring()
3566 rx_ring->bd_ring.len = len; in rtw89_pci_alloc_rx_ring()
3567 rx_ring->bd_ring.desc_size = desc_size; in rtw89_pci_alloc_rx_ring()
3568 rx_ring->bd_ring.addr = *rxch_addr; in rtw89_pci_alloc_rx_ring()
3569 if (info->rx_ring_eq_is_full) in rtw89_pci_alloc_rx_ring()
3570 rx_ring->bd_ring.wp = len - 1; in rtw89_pci_alloc_rx_ring()
3572 rx_ring->bd_ring.wp = 0; in rtw89_pci_alloc_rx_ring()
3573 rx_ring->bd_ring.rp = 0; in rtw89_pci_alloc_rx_ring()
3574 rx_ring->buf_sz = buf_sz; in rtw89_pci_alloc_rx_ring()
3575 rx_ring->diliver_skb = NULL; in rtw89_pci_alloc_rx_ring()
3576 rx_ring->diliver_desc.ready = false; in rtw89_pci_alloc_rx_ring()
3577 rx_ring->target_rx_tag = 0; in rtw89_pci_alloc_rx_ring()
3582 ret = -ENOMEM; in rtw89_pci_alloc_rx_ring()
3586 memset(skb->data, 0, buf_sz); in rtw89_pci_alloc_rx_ring()
3587 rx_ring->buf[i] = skb; in rtw89_pci_alloc_rx_ring()
3597 rx_ring->buf[i] = NULL; in rtw89_pci_alloc_rx_ring()
3607 skb = rx_ring->buf[i]; in rtw89_pci_alloc_rx_ring()
3610 dma = *((dma_addr_t *)skb->cb); in rtw89_pci_alloc_rx_ring()
3611 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); in rtw89_pci_alloc_rx_ring()
3613 rx_ring->buf[i] = NULL; in rtw89_pci_alloc_rx_ring()
3616 head = rx_ring->bd_ring.head; in rtw89_pci_alloc_rx_ring()
3617 dma = rx_ring->bd_ring.dma; in rtw89_pci_alloc_rx_ring()
3618 dma_free_coherent(&pdev->dev, ring_sz, head, dma); in rtw89_pci_alloc_rx_ring()
3620 rx_ring->bd_ring.head = NULL; in rtw89_pci_alloc_rx_ring()
3628 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_alloc_rx_rings()
3636 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_alloc_rx_rings()
3652 rx_ring = &rtwpci->rx_rings[i]; in rtw89_pci_alloc_rx_rings()
3687 skb_queue_head_init(&rtwpci->h2c_queue); in rtw89_pci_h2c_init()
3688 skb_queue_head_init(&rtwpci->h2c_release_queue); in rtw89_pci_h2c_init()
3694 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_setup_resource()
3711 spin_lock_init(&rtwpci->irq_lock); in rtw89_pci_setup_resource()
3712 spin_lock_init(&rtwpci->trx_lock); in rtw89_pci_setup_resource()
3725 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_clear_resource()
3730 skb_queue_len(&rtwpci->h2c_queue), true); in rtw89_pci_clear_resource()
3735 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_config_intr_mask()
3736 const struct rtw89_chip_info *chip = rtwdev->chip; in rtw89_pci_config_intr_mask()
3739 if (chip->chip_id == RTL8851B) in rtw89_pci_config_intr_mask()
3742 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0; in rtw89_pci_config_intr_mask()
3744 if (rtwpci->under_recovery) { in rtw89_pci_config_intr_mask()
3745 rtwpci->intrs[0] = hs0isr_ind_int_en; in rtw89_pci_config_intr_mask()
3746 rtwpci->intrs[1] = 0; in rtw89_pci_config_intr_mask()
3748 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN | in rtw89_pci_config_intr_mask()
3757 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN; in rtw89_pci_config_intr_mask()
3764 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_recovery_intr_mask_v1()
3766 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN; in rtw89_pci_recovery_intr_mask_v1()
3767 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; in rtw89_pci_recovery_intr_mask_v1()
3768 rtwpci->intrs[0] = 0; in rtw89_pci_recovery_intr_mask_v1()
3769 rtwpci->intrs[1] = 0; in rtw89_pci_recovery_intr_mask_v1()
3774 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_default_intr_mask_v1()
3776 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN | in rtw89_pci_default_intr_mask_v1()
3779 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; in rtw89_pci_default_intr_mask_v1()
3780 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN | in rtw89_pci_default_intr_mask_v1()
3787 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN; in rtw89_pci_default_intr_mask_v1()
3792 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_low_power_intr_mask_v1()
3794 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN | in rtw89_pci_low_power_intr_mask_v1()
3796 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; in rtw89_pci_low_power_intr_mask_v1()
3797 rtwpci->intrs[0] = 0; in rtw89_pci_low_power_intr_mask_v1()
3798 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN; in rtw89_pci_low_power_intr_mask_v1()
3803 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_config_intr_mask_v1()
3805 if (rtwpci->under_recovery) in rtw89_pci_config_intr_mask_v1()
3807 else if (rtwpci->low_power) in rtw89_pci_config_intr_mask_v1()
3816 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_recovery_intr_mask_v2()
3818 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; in rtw89_pci_recovery_intr_mask_v2()
3819 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; in rtw89_pci_recovery_intr_mask_v2()
3820 rtwpci->intrs[0] = 0; in rtw89_pci_recovery_intr_mask_v2()
3821 rtwpci->intrs[1] = 0; in rtw89_pci_recovery_intr_mask_v2()
3826 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_default_intr_mask_v2()
3828 rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 | in rtw89_pci_default_intr_mask_v2()
3830 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; in rtw89_pci_default_intr_mask_v2()
3831 rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 | in rtw89_pci_default_intr_mask_v2()
3833 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | in rtw89_pci_default_intr_mask_v2()
3839 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_low_power_intr_mask_v2()
3841 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 | in rtw89_pci_low_power_intr_mask_v2()
3843 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; in rtw89_pci_low_power_intr_mask_v2()
3844 rtwpci->intrs[0] = 0; in rtw89_pci_low_power_intr_mask_v2()
3845 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | in rtw89_pci_low_power_intr_mask_v2()
3851 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_config_intr_mask_v2()
3853 if (rtwpci->under_recovery) in rtw89_pci_config_intr_mask_v2()
3855 else if (rtwpci->low_power) in rtw89_pci_config_intr_mask_v2()
3875 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, in rtw89_pci_request_irq()
3897 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); in rtw89_pci_free_irq()
3915 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_filter_out()
3916 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_filter_out()
3921 if (rtwdev->chip->chip_id != RTL8852C) in rtw89_pci_filter_out()
3960 return -EOPNOTSUPP; in rtw89_pci_filter_out()
3970 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_clkreq_set()
3971 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_clkreq_set()
3976 gen_def->clkreq_set(rtwdev, enable); in rtw89_pci_clkreq_set()
3981 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_clkreq_set_ax()
4015 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_aspm_set()
4016 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_aspm_set()
4021 gen_def->aspm_set(rtwdev, enable); in rtw89_pci_aspm_set()
4026 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_aspm_set_ax()
4065 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen; in rtw89_pci_recalc_int_mit()
4066 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_recalc_int_mit()
4067 struct rtw89_traffic_stats *stats = &rtwdev->stats; in rtw89_pci_recalc_int_mit()
4068 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv; in rtw89_pci_recalc_int_mit()
4069 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv; in rtw89_pci_recalc_int_mit()
4072 if (rtwdev->scanning || in rtw89_pci_recalc_int_mit()
4085 rtw89_write32(rtwdev, info->mit_addr, val); in rtw89_pci_recalc_int_mit()
4090 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_link_cfg()
4091 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_link_cfg()
4104 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device in rtw89_pci_link_cfg()
4126 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_l1ss_set()
4127 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_l1ss_set()
4132 gen_def->l1ss_set(rtwdev, enable); in rtw89_pci_l1ss_set()
4137 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_l1ss_set_ax()
4150 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d", in rtw89_pci_l1ss_set_ax()
4169 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_l1ss_cfg()
4170 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_l1ss_cfg()
4188 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_cpl_timeout_cfg()
4189 struct pci_dev *pdev = rtwpci->pdev; in rtw89_pci_cpl_timeout_cfg()
4207 return -EINVAL; in rtw89_pci_poll_io_idle_ax()
4217 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_lv1rst_stop_dma_ax()
4246 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_lv1rst_start_dma_ax()
4264 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_ops_mac_lv1_recovery()
4265 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_ops_mac_lv1_recovery()
4270 ret = gen_def->lv1rst_stop_dma(rtwdev); in rtw89_pci_ops_mac_lv1_recovery()
4277 ret = gen_def->lv1rst_start_dma(rtwdev); in rtw89_pci_ops_mac_lv1_recovery()
4283 return -EINVAL; in rtw89_pci_ops_mac_lv1_recovery()
4291 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) in rtw89_pci_ops_dump_err_status()
4294 if (rtwdev->chip->chip_id == RTL8852C) { in rtw89_pci_ops_dump_err_status()
4312 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; in rtw89_pci_napi_poll()
4313 const struct rtw89_pci_info *info = rtwdev->pci_info; in rtw89_pci_napi_poll()
4314 const struct rtw89_pci_gen_def *gen_def = info->gen_def; in rtw89_pci_napi_poll()
4318 rtwdev->napi_budget_countdown = budget; in rtw89_pci_napi_poll()
4320 rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data); in rtw89_pci_napi_poll()
4321 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); in rtw89_pci_napi_poll()
4325 rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data); in rtw89_pci_napi_poll()
4326 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); in rtw89_pci_napi_poll()
4328 spin_lock_irqsave(&rtwpci->irq_lock, flags); in rtw89_pci_napi_poll()
4329 if (likely(rtwpci->running)) in rtw89_pci_napi_poll()
4331 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); in rtw89_pci_napi_poll()
4348 if (ssid_quirks->vendor == 0 && ssid_quirks->device == 0) in rtw89_check_pci_ssid_quirks()
4351 if (ssid_quirks->vendor != pdev->vendor || in rtw89_check_pci_ssid_quirks()
4352 ssid_quirks->device != pdev->device || in rtw89_check_pci_ssid_quirks()
4353 ssid_quirks->subsystem_vendor != pdev->subsystem_vendor || in rtw89_check_pci_ssid_quirks()
4354 ssid_quirks->subsystem_device != pdev->subsystem_device) in rtw89_check_pci_ssid_quirks()
4357 bitmap_or(rtwdev->quirks, rtwdev->quirks, &ssid_quirks->bitmap, in rtw89_check_pci_ssid_quirks()
4359 rtwdev->custid = ssid_quirks->custid; in rtw89_check_pci_ssid_quirks()
4364 (int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid); in rtw89_check_pci_ssid_quirks()
4370 struct rtw89_dev *rtwdev = hw->priv; in rtw89_pci_suspend()
4371 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_suspend()
4391 if (rtwdev->chip->chip_id == RTL8852C) in rtw89_pci_l2_hci_ldo()
4416 struct rtw89_dev *rtwdev = hw->priv; in rtw89_pci_resume()
4417 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; in rtw89_pci_resume()
4456 struct rtw89_dev *rtwdev = hw->priv; in rtw89_pci_io_slot_reset()
4561 info = (const struct rtw89_driver_info *)id->driver_data; in rtw89_pci_probe()
4563 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev, in rtw89_pci_probe()
4565 info->chip, info->variant); in rtw89_pci_probe()
4567 dev_err(&pdev->dev, "failed to allocate hw\n"); in rtw89_pci_probe()
4568 return -ENOMEM; in rtw89_pci_probe()
4571 pci_info = info->bus.pci; in rtw89_pci_probe()
4573 rtwdev->pci_info = info->bus.pci; in rtw89_pci_probe()
4574 rtwdev->hci.ops = &rtw89_pci_ops; in rtw89_pci_probe()
4575 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE; in rtw89_pci_probe()
4576 rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_PCIE; in rtw89_pci_probe()
4577 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr; in rtw89_pci_probe()
4578 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr; in rtw89_pci_probe()
4580 rtw89_check_quirks(rtwdev, info->quirks); in rtw89_pci_probe()
4581 rtw89_check_pci_ssid_quirks(rtwdev, pdev, pci_info->ssid_quirks); in rtw89_pci_probe()
4583 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); in rtw89_pci_probe()
4629 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags); in rtw89_pci_probe()
4655 rtwdev = hw->priv; in rtw89_pci_remove()