38 #define GPD_RX_BUF_LEN(mtu, x)	\  argument
41 	((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \
46 #define GPD_DATA_LEN(mtu, x)	\  argument
49 	((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \
57 #define GPD_EXT_NGP(mtu, x)	\  argument
60 	((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \
63 #define GPD_EXT_BUF(mtu, x)	\  argument
66 	((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \
171 	gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);  in mtu3_gpd_ring_alloc()
184 	dma_pool_free(mep->mtu->qmu_gpd_pool,  in mtu3_gpd_ring_free()
191 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_resume()  local
192 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_resume()
250 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_tx_gpd()  local
256 	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));  in mtu3_prepare_tx_gpd()
257 	gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length));  in mtu3_prepare_tx_gpd()
262 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_tx_gpd()
267 	ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));  in mtu3_prepare_tx_gpd()
271 		if (mtu->gen2cp)  in mtu3_prepare_tx_gpd()
293 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_rx_gpd()  local
299 	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));  in mtu3_prepare_rx_gpd()
300 	gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length));  in mtu3_prepare_rx_gpd()
305 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_rx_gpd()
310 	ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));  in mtu3_prepare_rx_gpd()
333 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_start()  local
334 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_start()
348 			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);  in mtu3_qmu_start()
365 			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);  in mtu3_qmu_start()
377 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_stop()  local
378 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_stop()
387 		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);  in mtu3_qmu_stop()
398 		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);  in mtu3_qmu_stop()
406 	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);  in mtu3_qmu_stop()
412 	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,  in mtu3_qmu_flush()
425 static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)  in qmu_tx_zlp_error_handler()  argument
427 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()
429 	void __iomem *mbase = mtu->mac_base;  in qmu_tx_zlp_error_handler()
443 	if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) {  in qmu_tx_zlp_error_handler()
444 		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);  in qmu_tx_zlp_error_handler()
448 	dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);  in qmu_tx_zlp_error_handler()
456 		dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);  in qmu_tx_zlp_error_handler()
474 static void qmu_error_rx(struct mtu3 *mtu, u8 epnum)  in qmu_error_rx()  argument
476 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_error_rx()
482 	cur_gpd_dma = read_rxq_cur_addr(mtu->mac_base, epnum);  in qmu_error_rx()
487 		dev_err(mtu->dev, "no correct RX req is found\n");  in qmu_error_rx()
497 	dev_dbg(mtu->dev, "%s EP%d, current=%p, req=%p\n",  in qmu_error_rx()
508 static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)  in qmu_done_tx()  argument
510 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()
512 	void __iomem *mbase = mtu->mac_base;  in qmu_done_tx()
523 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",  in qmu_done_tx()
531 			dev_err(mtu->dev, "no correct TX req is found\n");  in qmu_done_tx()
536 		request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));  in qmu_done_tx()
543 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",  in qmu_done_tx()
548 static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)  in qmu_done_rx()  argument
550 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()
552 	void __iomem *mbase = mtu->mac_base;  in qmu_done_rx()
562 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",  in qmu_done_rx()
570 			dev_err(mtu->dev, "no correct RX req is found\n");  in qmu_done_rx()
575 		req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));  in qmu_done_rx()
582 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",  in qmu_done_rx()
586 static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)  in qmu_done_isr()  argument
590 	for (i = 1; i < mtu->num_eps; i++) {  in qmu_done_isr()
592 			qmu_done_rx(mtu, i);  in qmu_done_isr()
594 			qmu_done_tx(mtu, i);  in qmu_done_isr()
598 static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)  in qmu_exception_isr()  argument
600 	void __iomem *mbase = mtu->mac_base;  in qmu_exception_isr()
608 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
610 				dev_err(mtu->dev, "Rx %d CS error!\n", i);  in qmu_exception_isr()
613 				dev_err(mtu->dev, "RX %d Length error\n", i);  in qmu_exception_isr()
616 				qmu_error_rx(mtu, i);  in qmu_exception_isr()
622 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
624 				dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);  in qmu_exception_isr()
631 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
633 				dev_err(mtu->dev, "Tx %d checksum error!\n", i);  in qmu_exception_isr()
636 				qmu_tx_zlp_error_handler(mtu, i);  in qmu_exception_isr()
642 irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)  in mtu3_qmu_isr()  argument
644 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_isr()
655 	dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",  in mtu3_qmu_isr()
661 		qmu_done_isr(mtu, qmu_done_status);  in mtu3_qmu_isr()
664 		qmu_exception_isr(mtu, qmu_status);  in mtu3_qmu_isr()
669 int mtu3_qmu_init(struct mtu3 *mtu)  in mtu3_qmu_init()  argument
674 	mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,  in mtu3_qmu_init()
677 	if (!mtu->qmu_gpd_pool)  in mtu3_qmu_init()
683 void mtu3_qmu_exit(struct mtu3 *mtu)  in mtu3_qmu_exit()  argument
685 	dma_pool_destroy(mtu->qmu_gpd_pool);  in mtu3_qmu_exit()