290 /* QDMA */
733 	struct airoha_qdma *qdma;  member
752 	struct airoha_qdma *qdma;  member
800 	/* descriptor and packet buffers for qdma hw forward */
808 	struct airoha_qdma *qdma;  member
826 	struct airoha_qdma qdma[AIROHA_MAX_NUM_QDMA];  member
859 #define airoha_qdma_rr(qdma, offset)				\  argument
860 	airoha_rr((qdma)->regs, (offset))
861 #define airoha_qdma_wr(qdma, offset, val)			\  argument
862 	airoha_wr((qdma)->regs, (offset), (val))
863 #define airoha_qdma_rmw(qdma, offset, mask, val)		\  argument
864 	airoha_rmw((qdma)->regs, (offset), (mask), (val))
865 #define airoha_qdma_set(qdma, offset, val)			\  argument
866 	airoha_rmw((qdma)->regs, (offset), 0, (val))
867 #define airoha_qdma_clear(qdma, offset, val)			\  argument
868 	airoha_rmw((qdma)->regs, (offset), (val), 0)
870 static void airoha_qdma_set_irqmask(struct airoha_qdma *qdma, int index,  in airoha_qdma_set_irqmask()  argument
875 	if (WARN_ON_ONCE(index >= ARRAY_SIZE(qdma->irqmask)))  in airoha_qdma_set_irqmask()
878 	spin_lock_irqsave(&qdma->irq_lock, flags);  in airoha_qdma_set_irqmask()
880 	qdma->irqmask[index] &= ~clear;  in airoha_qdma_set_irqmask()
881 	qdma->irqmask[index] |= set;  in airoha_qdma_set_irqmask()
882 	airoha_qdma_wr(qdma, REG_INT_ENABLE(index), qdma->irqmask[index]);  in airoha_qdma_set_irqmask()
886 	airoha_qdma_rr(qdma, REG_INT_ENABLE(index));  in airoha_qdma_set_irqmask()
888 	spin_unlock_irqrestore(&qdma->irq_lock, flags);  in airoha_qdma_set_irqmask()
891 static void airoha_qdma_irq_enable(struct airoha_qdma *qdma, int index,  in airoha_qdma_irq_enable()  argument
894 	airoha_qdma_set_irqmask(qdma, index, 0, mask);  in airoha_qdma_irq_enable()
897 static void airoha_qdma_irq_disable(struct airoha_qdma *qdma, int index,  in airoha_qdma_irq_disable()  argument
900 	airoha_qdma_set_irqmask(qdma, index, mask, 0);  in airoha_qdma_irq_disable()
914 	struct airoha_eth *eth = port->qdma->eth;  in airoha_set_macaddr()
1381 	/* QDMA LAN, RX Ring-22 */  in airoha_fe_init()
1413 	struct airoha_qdma *qdma = q->qdma;  in airoha_qdma_fill_rx_queue()  local
1414 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_fill_rx_queue()
1415 	int qid = q - &qdma->q_rx[0];  in airoha_qdma_fill_rx_queue()
1451 		airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),  in airoha_qdma_fill_rx_queue()
1482 	struct airoha_qdma *qdma = q->qdma;  in airoha_qdma_rx_process()  local
1483 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_rx_process()
1484 	int qid = q - &qdma->q_rx[0];  in airoha_qdma_rx_process()
1554 		airoha_qdma_irq_enable(q->qdma, QDMA_INT_REG_IDX1,  in airoha_qdma_rx_napi_poll()
1561 				     struct airoha_qdma *qdma, int ndesc)  in airoha_qdma_init_rx_queue()  argument
1570 		.dev = qdma->eth->dev,  in airoha_qdma_init_rx_queue()
1573 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_init_rx_queue()
1574 	int qid = q - &qdma->q_rx[0], thr;  in airoha_qdma_init_rx_queue()
1579 	q->qdma = qdma;  in airoha_qdma_init_rx_queue()
1601 	airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);  in airoha_qdma_init_rx_queue()
1602 	airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),  in airoha_qdma_init_rx_queue()
1607 	airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,  in airoha_qdma_init_rx_queue()
1609 	airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,  in airoha_qdma_init_rx_queue()
1619 	struct airoha_eth *eth = q->qdma->eth;  in airoha_qdma_cleanup_rx_queue()
1633 static int airoha_qdma_init_rx(struct airoha_qdma *qdma)  in airoha_qdma_init_rx()  argument
1637 	for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {  in airoha_qdma_init_rx()
1645 		err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,  in airoha_qdma_init_rx()
1658 	struct airoha_qdma *qdma;  in airoha_qdma_tx_napi_poll()  local
1663 	qdma = irq_q->qdma;  in airoha_qdma_tx_napi_poll()
1664 	id = irq_q - &qdma->q_tx_irq[0];  in airoha_qdma_tx_napi_poll()
1665 	eth = qdma->eth;  in airoha_qdma_tx_napi_poll()
1667 	status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));  in airoha_qdma_tx_napi_poll()
1689 		if (qid >= ARRAY_SIZE(qdma->q_tx))  in airoha_qdma_tx_napi_poll()
1692 		q = &qdma->q_tx[qid];  in airoha_qdma_tx_napi_poll()
1750 			airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),  in airoha_qdma_tx_napi_poll()
1752 		airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),  in airoha_qdma_tx_napi_poll()
1757 		airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0,  in airoha_qdma_tx_napi_poll()
1764 				     struct airoha_qdma *qdma, int size)  in airoha_qdma_init_tx_queue()  argument
1766 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_init_tx_queue()
1767 	int i, qid = q - &qdma->q_tx[0];  in airoha_qdma_init_tx_queue()
1772 	q->qdma = qdma;  in airoha_qdma_init_tx_queue()
1792 	airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);  in airoha_qdma_init_tx_queue()
1793 	airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,  in airoha_qdma_init_tx_queue()
1795 	airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,  in airoha_qdma_init_tx_queue()
1802 				   struct airoha_qdma *qdma, int size)  in airoha_qdma_tx_irq_init()  argument
1804 	int id = irq_q - &qdma->q_tx_irq[0];  in airoha_qdma_tx_irq_init()
1805 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_tx_irq_init()
1817 	irq_q->qdma = qdma;  in airoha_qdma_tx_irq_init()
1819 	airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);  in airoha_qdma_tx_irq_init()
1820 	airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,  in airoha_qdma_tx_irq_init()
1822 	airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,  in airoha_qdma_tx_irq_init()
1828 static int airoha_qdma_init_tx(struct airoha_qdma *qdma)  in airoha_qdma_init_tx()  argument
1832 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {  in airoha_qdma_init_tx()
1833 		err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,  in airoha_qdma_init_tx()
1839 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {  in airoha_qdma_init_tx()
1840 		err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,  in airoha_qdma_init_tx()
1851 	struct airoha_eth *eth = q->qdma->eth;  in airoha_qdma_cleanup_tx_queue()
1868 static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma)  in airoha_qdma_init_hfwd_queues()  argument
1870 	struct airoha_eth *eth = qdma->eth;  in airoha_qdma_init_hfwd_queues()
1876 	qdma->hfwd.desc = dmam_alloc_coherent(eth->dev, size, &dma_addr,  in airoha_qdma_init_hfwd_queues()
1878 	if (!qdma->hfwd.desc)  in airoha_qdma_init_hfwd_queues()
1881 	airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);  in airoha_qdma_init_hfwd_queues()
1884 	qdma->hfwd.q = dmam_alloc_coherent(eth->dev, size, &dma_addr,  in airoha_qdma_init_hfwd_queues()
1886 	if (!qdma->hfwd.q)  in airoha_qdma_init_hfwd_queues()
1889 	airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);  in airoha_qdma_init_hfwd_queues()
1891 	airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,  in airoha_qdma_init_hfwd_queues()
1894 	airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,  in airoha_qdma_init_hfwd_queues()
1896 	airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,  in airoha_qdma_init_hfwd_queues()
1904 				 30 * USEC_PER_MSEC, true, qdma,  in airoha_qdma_init_hfwd_queues()
1908 static void airoha_qdma_init_qos(struct airoha_qdma *qdma)  in airoha_qdma_init_qos()  argument
1910 	airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);  in airoha_qdma_init_qos()
1911 	airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);  in airoha_qdma_init_qos()
1913 	airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,  in airoha_qdma_init_qos()
1916 	airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG,  in airoha_qdma_init_qos()
1920 	airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,  in airoha_qdma_init_qos()
1923 	airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,  in airoha_qdma_init_qos()
1928 	airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);  in airoha_qdma_init_qos()
1930 	airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,  in airoha_qdma_init_qos()
1932 	airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,  in airoha_qdma_init_qos()
1935 	airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);  in airoha_qdma_init_qos()
1936 	airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,  in airoha_qdma_init_qos()
1938 	airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,  in airoha_qdma_init_qos()
1942 	airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);  in airoha_qdma_init_qos()
1943 	airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,  in airoha_qdma_init_qos()
1945 	airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,  in airoha_qdma_init_qos()
1947 	airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,  in airoha_qdma_init_qos()
1951 	airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);  in airoha_qdma_init_qos()
1952 	airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,  in airoha_qdma_init_qos()
1954 	airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,  in airoha_qdma_init_qos()
1958 static int airoha_qdma_hw_init(struct airoha_qdma *qdma)  in airoha_qdma_hw_init()  argument
1963 	for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++)  in airoha_qdma_hw_init()
1964 		airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);  in airoha_qdma_hw_init()
1967 	airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0, INT_IDX0_MASK);  in airoha_qdma_hw_init()
1968 	airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX1, INT_IDX1_MASK);  in airoha_qdma_hw_init()
1969 	airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX4, INT_IDX4_MASK);  in airoha_qdma_hw_init()
1972 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {  in airoha_qdma_hw_init()
1973 		if (!qdma->q_tx[i].ndesc)  in airoha_qdma_hw_init()
1977 			airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),  in airoha_qdma_hw_init()
1980 			airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),  in airoha_qdma_hw_init()
1984 	airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,  in airoha_qdma_hw_init()
1995 	airoha_qdma_init_qos(qdma);  in airoha_qdma_hw_init()
1997 	/* disable qdma rx delay interrupt */  in airoha_qdma_hw_init()
1998 	for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {  in airoha_qdma_hw_init()
1999 		if (!qdma->q_rx[i].ndesc)  in airoha_qdma_hw_init()
2002 		airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),  in airoha_qdma_hw_init()
2006 	airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,  in airoha_qdma_hw_init()
2014 	struct airoha_qdma *qdma = dev_instance;  in airoha_irq_handler()  local
2015 	u32 intr[ARRAY_SIZE(qdma->irqmask)];  in airoha_irq_handler()
2018 	for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++) {  in airoha_irq_handler()
2019 		intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));  in airoha_irq_handler()
2020 		intr[i] &= qdma->irqmask[i];  in airoha_irq_handler()
2021 		airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);  in airoha_irq_handler()
2024 	if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))  in airoha_irq_handler()
2028 		airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX1,  in airoha_irq_handler()
2031 		for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {  in airoha_irq_handler()
2032 			if (!qdma->q_rx[i].ndesc)  in airoha_irq_handler()
2036 				napi_schedule(&qdma->q_rx[i].napi);  in airoha_irq_handler()
2041 		for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {  in airoha_irq_handler()
2045 			airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX0,  in airoha_irq_handler()
2047 			napi_schedule(&qdma->q_tx_irq[i].napi);  in airoha_irq_handler()
2056 			    struct airoha_qdma *qdma)  in airoha_qdma_init()  argument
2058 	int err, id = qdma - &eth->qdma[0];  in airoha_qdma_init()
2061 	spin_lock_init(&qdma->irq_lock);  in airoha_qdma_init()
2062 	qdma->eth = eth;  in airoha_qdma_init()
2064 	res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);  in airoha_qdma_init()
2068 	qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);  in airoha_qdma_init()
2069 	if (IS_ERR(qdma->regs))  in airoha_qdma_init()
2070 		return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),  in airoha_qdma_init()
2071 				     "failed to iomap qdma%d regs\n", id);  in airoha_qdma_init()
2073 	qdma->irq = platform_get_irq(pdev, 4 * id);  in airoha_qdma_init()
2074 	if (qdma->irq < 0)  in airoha_qdma_init()
2075 		return qdma->irq;  in airoha_qdma_init()
2077 	err = devm_request_irq(eth->dev, qdma->irq, airoha_irq_handler,  in airoha_qdma_init()
2078 			       IRQF_SHARED, KBUILD_MODNAME, qdma);  in airoha_qdma_init()
2082 	err = airoha_qdma_init_rx(qdma);  in airoha_qdma_init()
2086 	err = airoha_qdma_init_tx(qdma);  in airoha_qdma_init()
2090 	err = airoha_qdma_init_hfwd_queues(qdma);  in airoha_qdma_init()
2094 	return airoha_qdma_hw_init(qdma);  in airoha_qdma_init()
2122 	for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {  in airoha_hw_init()
2123 		err = airoha_qdma_init(pdev, eth, &eth->qdma[i]);  in airoha_hw_init()
2133 static void airoha_hw_cleanup(struct airoha_qdma *qdma)  in airoha_hw_cleanup()  argument
2137 	for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {  in airoha_hw_cleanup()
2138 		if (!qdma->q_rx[i].ndesc)  in airoha_hw_cleanup()
2141 		napi_disable(&qdma->q_rx[i].napi);  in airoha_hw_cleanup()
2142 		netif_napi_del(&qdma->q_rx[i].napi);  in airoha_hw_cleanup()
2143 		airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);  in airoha_hw_cleanup()
2144 		if (qdma->q_rx[i].page_pool)  in airoha_hw_cleanup()
2145 			page_pool_destroy(qdma->q_rx[i].page_pool);  in airoha_hw_cleanup()
2148 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {  in airoha_hw_cleanup()
2149 		napi_disable(&qdma->q_tx_irq[i].napi);  in airoha_hw_cleanup()
2150 		netif_napi_del(&qdma->q_tx_irq[i].napi);  in airoha_hw_cleanup()
2153 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {  in airoha_hw_cleanup()
2154 		if (!qdma->q_tx[i].ndesc)  in airoha_hw_cleanup()
2157 		airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);  in airoha_hw_cleanup()
2161 static void airoha_qdma_start_napi(struct airoha_qdma *qdma)  in airoha_qdma_start_napi()  argument
2165 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)  in airoha_qdma_start_napi()
2166 		napi_enable(&qdma->q_tx_irq[i].napi);  in airoha_qdma_start_napi()
2168 	for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {  in airoha_qdma_start_napi()
2169 		if (!qdma->q_rx[i].ndesc)  in airoha_qdma_start_napi()
2172 		napi_enable(&qdma->q_rx[i].napi);  in airoha_qdma_start_napi()
2178 	struct airoha_eth *eth = port->qdma->eth;  in airoha_update_hw_stats()
2323 	struct airoha_qdma *qdma = port->qdma;  in airoha_dev_open()  local
2327 	err = airoha_set_gdm_ports(qdma->eth, true);  in airoha_dev_open()
2332 		airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id),  in airoha_dev_open()
2335 		airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id),  in airoha_dev_open()
2338 	airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG,  in airoha_dev_open()
2348 	struct airoha_qdma *qdma = port->qdma;  in airoha_dev_stop()  local
2352 	err = airoha_set_gdm_ports(qdma->eth, false);  in airoha_dev_stop()
2356 	airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,  in airoha_dev_stop()
2360 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {  in airoha_dev_stop()
2361 		if (!qdma->q_tx[i].ndesc)  in airoha_dev_stop()
2364 		airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);  in airoha_dev_stop()
2423 	struct airoha_qdma *qdma = port->qdma;  in airoha_dev_xmit()  local
2453 	q = &qdma->q_tx[qid];  in airoha_dev_xmit()
2508 		airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),  in airoha_dev_xmit()
2538 	struct airoha_eth *eth = port->qdma->eth;  in airoha_ethtool_get_drvinfo()
2619 	struct airoha_qdma *qdma;  in airoha_alloc_gdm_port()  local
2649 	qdma = &eth->qdma[index % AIROHA_MAX_NUM_QDMA];  in airoha_alloc_gdm_port()
2659 	dev->irq = qdma->irq;  in airoha_alloc_gdm_port()
2675 	port->qdma = qdma;  in airoha_alloc_gdm_port()
2708 	eth->rsts[2].id = "qdma";  in airoha_probe()
2743 	for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)  in airoha_probe()
2744 		airoha_qdma_start_napi(&eth->qdma[i]);  in airoha_probe()
2763 	for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)  in airoha_probe()
2764 		airoha_hw_cleanup(&eth->qdma[i]);  in airoha_probe()
2783 	for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)  in airoha_remove()
2784 		airoha_hw_cleanup(&eth->qdma[i]);  in airoha_remove()