53 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read8()  local
55 	return readb(rtwpci->mmap + addr);  in rtw_pci_read8()
60 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read16()  local
62 	return readw(rtwpci->mmap + addr);  in rtw_pci_read16()
67 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_read32()  local
69 	return readl(rtwpci->mmap + addr);  in rtw_pci_read32()
74 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write8()  local
76 	writeb(val, rtwpci->mmap + addr);  in rtw_pci_write8()
81 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write16()  local
83 	writew(val, rtwpci->mmap + addr);  in rtw_pci_write16()
88 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_write32()  local
90 	writel(val, rtwpci->mmap + addr);  in rtw_pci_write32()
162 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_free_trx_ring()  local
168 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_free_trx_ring()
173 		rx_ring = &rtwpci->rx_rings[i];  in rtw_pci_free_trx_ring()
317 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init_trx_ring()  local
329 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
339 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
351 		tx_ring = &rtwpci->tx_rings[i];  in rtw_pci_init_trx_ring()
357 		rx_ring = &rtwpci->rx_rings[j];  in rtw_pci_init_trx_ring()
371 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_init()  local
374 	rtwpci->irq_mask[0] = IMR_HIGHDOK |  in rtw_pci_init()
384 	rtwpci->irq_mask[1] = IMR_TXFOVW |  in rtw_pci_init()
386 	rtwpci->irq_mask[3] = IMR_H2CDOK |  in rtw_pci_init()
388 	spin_lock_init(&rtwpci->irq_lock);  in rtw_pci_init()
389 	spin_lock_init(&rtwpci->hwirq_lock);  in rtw_pci_init()
397 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_reset_buf_desc()  local
405 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;  in rtw_pci_reset_buf_desc()
409 		len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;  in rtw_pci_reset_buf_desc()
410 		dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;  in rtw_pci_reset_buf_desc()
411 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;  in rtw_pci_reset_buf_desc()
412 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;  in rtw_pci_reset_buf_desc()
417 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;  in rtw_pci_reset_buf_desc()
418 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;  in rtw_pci_reset_buf_desc()
419 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;  in rtw_pci_reset_buf_desc()
420 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;  in rtw_pci_reset_buf_desc()
424 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;  in rtw_pci_reset_buf_desc()
425 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;  in rtw_pci_reset_buf_desc()
426 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;  in rtw_pci_reset_buf_desc()
427 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;  in rtw_pci_reset_buf_desc()
431 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;  in rtw_pci_reset_buf_desc()
432 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;  in rtw_pci_reset_buf_desc()
433 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;  in rtw_pci_reset_buf_desc()
434 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;  in rtw_pci_reset_buf_desc()
438 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;  in rtw_pci_reset_buf_desc()
439 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;  in rtw_pci_reset_buf_desc()
440 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;  in rtw_pci_reset_buf_desc()
441 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;  in rtw_pci_reset_buf_desc()
445 	len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;  in rtw_pci_reset_buf_desc()
446 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;  in rtw_pci_reset_buf_desc()
447 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;  in rtw_pci_reset_buf_desc()
448 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;  in rtw_pci_reset_buf_desc()
452 	len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;  in rtw_pci_reset_buf_desc()
453 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;  in rtw_pci_reset_buf_desc()
454 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;  in rtw_pci_reset_buf_desc()
455 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;  in rtw_pci_reset_buf_desc()
459 	len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;  in rtw_pci_reset_buf_desc()
460 	dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;  in rtw_pci_reset_buf_desc()
461 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;  in rtw_pci_reset_buf_desc()
462 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;  in rtw_pci_reset_buf_desc()
481 				     struct rtw_pci *rtwpci, bool exclude_rx)  in rtw_pci_enable_interrupt()  argument
486 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
488 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);  in rtw_pci_enable_interrupt()
489 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);  in rtw_pci_enable_interrupt()
491 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);  in rtw_pci_enable_interrupt()
493 	rtwpci->irq_enabled = true;  in rtw_pci_enable_interrupt()
495 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_enable_interrupt()
499 				      struct rtw_pci *rtwpci)  in rtw_pci_disable_interrupt()  argument
503 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
505 	if (!rtwpci->irq_enabled)  in rtw_pci_disable_interrupt()
513 	rtwpci->irq_enabled = false;  in rtw_pci_disable_interrupt()
516 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_disable_interrupt()
519 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_reset()  argument
524 	rtwpci->rx_tag = 0;  in rtw_pci_dma_reset()
529 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup()  local
532 	rtw_pci_dma_reset(rtwdev, rtwpci);  in rtw_pci_setup()
537 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)  in rtw_pci_dma_release()  argument
544 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_dma_release()
551 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_start()  local
553 	if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_start()
556 	napi_enable(&rtwpci->napi);  in rtw_pci_napi_start()
561 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_stop()  local
563 	if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))  in rtw_pci_napi_stop()
566 	napi_synchronize(&rtwpci->napi);  in rtw_pci_napi_stop()
567 	napi_disable(&rtwpci->napi);  in rtw_pci_napi_stop()
572 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_start()  local
576 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
577 	rtwpci->running = true;  in rtw_pci_start()
578 	rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_start()
579 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_start()
586 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_stop()  local
587 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_stop()
589 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
590 	rtwpci->running = false;  in rtw_pci_stop()
591 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_stop()
592 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
597 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
598 	rtw_pci_dma_release(rtwdev, rtwpci);  in rtw_pci_stop()
599 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_stop()
604 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_enter()  local
612 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_enter()
623 		tx_ring = &rtwpci->tx_rings[queue];  in rtw_pci_deep_ps_enter()
644 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps_leave()  local
646 	lockdep_assert_held(&rtwpci->irq_lock);  in rtw_pci_deep_ps_leave()
654 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_deep_ps()  local
656 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
664 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_deep_ps()
667 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,  in rtw_pci_release_rsvd_page()  argument
679 	dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);  in rtw_pci_release_rsvd_page()
687 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_dma_check()  local
698 	if (total_pkt_size != rtwpci->rx_tag)  in rtw_pci_dma_check()
701 	rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;  in rtw_pci_dma_check()
714 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in __pci_flush_queue()  local
715 	struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];  in __pci_flush_queue()
775 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off_queue()  local
779 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_kick_off_queue()
782 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
786 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_kick_off_queue()
791 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_kick_off()  local
795 		if (test_and_clear_bit(queue, rtwpci->tx_queued))  in rtw_pci_tx_kick_off()
804 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write_data()  local
816 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write_data()
821 		rtw_pci_release_rsvd_page(rtwpci, ring);  in rtw_pci_tx_write_data()
829 	dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,  in rtw_pci_tx_write_data()
831 	if (dma_mapping_error(&rtwpci->pdev->dev, dma))  in rtw_pci_tx_write_data()
851 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
859 	set_bit(queue, rtwpci->tx_queued);  in rtw_pci_tx_write_data()
864 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write_data()
921 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_tx_write()  local
929 	ring = &rtwpci->tx_rings[queue];  in rtw_pci_tx_write()
930 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
935 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_tx_write()
940 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_tx_isr()  argument
953 	ring = &rtwpci->tx_rings[hw_queue];  in rtw_pci_tx_isr()
973 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,  in rtw_pci_tx_isr()
1017 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_rx_isr()  local
1018 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_isr()
1024 				     struct rtw_pci *rtwpci)  in rtw_pci_get_hw_rx_ring_nr()  argument
1030 	ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_get_hw_rx_ring_nr()
1041 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,  in rtw_pci_rx_napi()  argument
1045 	struct napi_struct *napi = &rtwpci->napi;  in rtw_pci_rx_napi()
1046 	struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];  in rtw_pci_rx_napi()
1059 	count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);  in rtw_pci_rx_napi()
1120 				   struct rtw_pci *rtwpci, u32 *irq_status)  in rtw_pci_irq_recognized()  argument
1124 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1132 	irq_status[0] &= rtwpci->irq_mask[0];  in rtw_pci_irq_recognized()
1133 	irq_status[1] &= rtwpci->irq_mask[1];  in rtw_pci_irq_recognized()
1134 	irq_status[3] &= rtwpci->irq_mask[3];  in rtw_pci_irq_recognized()
1140 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);  in rtw_pci_irq_recognized()
1146 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_handler()  local
1156 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_interrupt_handler()
1164 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_interrupt_threadfn()  local
1168 	spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1169 	rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);  in rtw_pci_interrupt_threadfn()
1172 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);  in rtw_pci_interrupt_threadfn()
1174 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);  in rtw_pci_interrupt_threadfn()
1176 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);  in rtw_pci_interrupt_threadfn()
1178 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);  in rtw_pci_interrupt_threadfn()
1180 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);  in rtw_pci_interrupt_threadfn()
1182 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);  in rtw_pci_interrupt_threadfn()
1184 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);  in rtw_pci_interrupt_threadfn()
1193 	if (rtwpci->running)  in rtw_pci_interrupt_threadfn()
1194 		rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);  in rtw_pci_interrupt_threadfn()
1195 	spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_interrupt_threadfn()
1203 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_mapping()  local
1215 	rtwpci->mmap = pci_iomap(pdev, bar_id, len);  in rtw_pci_io_mapping()
1216 	if (!rtwpci->mmap) {  in rtw_pci_io_mapping()
1228 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_io_unmapping()  local
1230 	if (rtwpci->mmap) {  in rtw_pci_io_unmapping()
1231 		pci_iounmap(pdev, rtwpci->mmap);  in rtw_pci_io_unmapping()
1375 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_ps()  local
1387 	if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))  in rtw_pci_link_ps()
1390 	if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||  in rtw_pci_link_ps()
1391 	    (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))  in rtw_pci_link_ps()
1398 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_link_cfg()  local
1399 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_link_cfg()
1434 	rtwpci->link_ctrl = link_ctrl;  in rtw_pci_link_cfg()
1454 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_phy_cfg()  local
1457 	struct pci_dev *pdev = rtwpci->pdev;  in rtw_pci_phy_cfg()
1561 	struct rtw_pci *rtwpci;  in rtw_pci_setup_resource()  local
1564 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_setup_resource()
1565 	rtwpci->pdev = pdev;  in rtw_pci_setup_resource()
1652 	struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);  in rtw_pci_napi_poll()  local
1653 	struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,  in rtw_pci_napi_poll()
1657 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1663 		work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,  in rtw_pci_napi_poll()
1671 		spin_lock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1672 		if (rtwpci->running)  in rtw_pci_napi_poll()
1673 			rtw_pci_enable_interrupt(rtwdev, rtwpci, false);  in rtw_pci_napi_poll()
1674 		spin_unlock_bh(&rtwpci->irq_lock);  in rtw_pci_napi_poll()
1680 		if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))  in rtw_pci_napi_poll()
1683 	if (rtwpci->rx_no_aspm)  in rtw_pci_napi_poll()
1691 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_init()  local
1693 	rtwpci->netdev = alloc_netdev_dummy(0);  in rtw_pci_napi_init()
1694 	if (!rtwpci->netdev)  in rtw_pci_napi_init()
1697 	netif_napi_add(rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);  in rtw_pci_napi_init()
1703 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_napi_deinit()  local
1706 	netif_napi_del(&rtwpci->napi);  in rtw_pci_napi_deinit()
1707 	free_netdev(rtwpci->netdev);  in rtw_pci_napi_deinit()
1753 	struct rtw_pci *rtwpci;  in rtw_pci_probe()  local
1771 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_probe()
1772 	atomic_set(&rtwpci->link_usage, 1);  in rtw_pci_probe()
1808 		rtwpci->rx_no_aspm = true;  in rtw_pci_probe()
1847 	struct rtw_pci *rtwpci;  in rtw_pci_remove()  local
1853 	rtwpci = (struct rtw_pci *)rtwdev->priv;  in rtw_pci_remove()
1856 	rtw_pci_disable_interrupt(rtwdev, rtwpci);  in rtw_pci_remove()