31 	*dataptr = lan966x->tx.dcbs_buf[dcb].dma_addr;  in lan966x_fdma_tx_dataptr_cb()
41 	*dataptr = lan966x->tx.dcbs_buf[dcb].dma_addr + XDP_PACKET_HEADROOM;  in lan966x_fdma_xdp_tx_dataptr_cb()
197 static int lan966x_fdma_tx_alloc(struct lan966x_tx *tx)  in lan966x_fdma_tx_alloc()  argument
199 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_alloc()
200 	struct fdma *fdma = &tx->fdma;  in lan966x_fdma_tx_alloc()
203 	tx->dcbs_buf = kcalloc(fdma->n_dcbs, sizeof(struct lan966x_tx_dcb_buf),  in lan966x_fdma_tx_alloc()
205 	if (!tx->dcbs_buf)  in lan966x_fdma_tx_alloc()
217 	kfree(tx->dcbs_buf);  in lan966x_fdma_tx_alloc()
221 static void lan966x_fdma_tx_free(struct lan966x_tx *tx)  in lan966x_fdma_tx_free()  argument
223 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_free()
225 	kfree(tx->dcbs_buf);  in lan966x_fdma_tx_free()
226 	fdma_free_coherent(lan966x->dev, &tx->fdma);  in lan966x_fdma_tx_free()
229 static void lan966x_fdma_tx_activate(struct lan966x_tx *tx)  in lan966x_fdma_tx_activate()  argument
231 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_activate()
232 	struct fdma *fdma = &tx->fdma;  in lan966x_fdma_tx_activate()
268 static void lan966x_fdma_tx_disable(struct lan966x_tx *tx)  in lan966x_fdma_tx_disable()  argument
270 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_disable()
271 	struct fdma *fdma = &tx->fdma;  in lan966x_fdma_tx_disable()
287 	tx->activated = false;  in lan966x_fdma_tx_disable()
290 static void lan966x_fdma_tx_reload(struct lan966x_tx *tx)  in lan966x_fdma_tx_reload()  argument
292 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_reload()
295 	lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(tx->fdma.channel_id)),  in lan966x_fdma_tx_reload()
331 	struct lan966x_tx *tx = &lan966x->tx;  in lan966x_fdma_tx_clear_buf()  local
334 	struct fdma *fdma = &tx->fdma;  in lan966x_fdma_tx_clear_buf()
345 		dcb_buf = &tx->dcbs_buf[i];  in lan966x_fdma_tx_clear_buf()
571 static int lan966x_fdma_get_next_dcb(struct lan966x_tx *tx)  in lan966x_fdma_get_next_dcb()  argument
574 	struct fdma *fdma = &tx->fdma;  in lan966x_fdma_get_next_dcb()
578 		dcb_buf = &tx->dcbs_buf[i];  in lan966x_fdma_get_next_dcb()
580 		    !fdma_is_last(&tx->fdma, &tx->fdma.dcbs[i]))  in lan966x_fdma_get_next_dcb()
587 static void lan966x_fdma_tx_start(struct lan966x_tx *tx)  in lan966x_fdma_tx_start()  argument
589 	struct lan966x *lan966x = tx->lan966x;  in lan966x_fdma_tx_start()
591 	if (likely(lan966x->tx.activated)) {  in lan966x_fdma_tx_start()
592 		lan966x_fdma_tx_reload(tx);  in lan966x_fdma_tx_start()
595 		lan966x->tx.activated = true;  in lan966x_fdma_tx_start()
596 		lan966x_fdma_tx_activate(tx);  in lan966x_fdma_tx_start()
604 	struct lan966x_tx *tx = &lan966x->tx;  in lan966x_fdma_xmit_xdpf()  local
615 	next_to_use = lan966x_fdma_get_next_dcb(tx);  in lan966x_fdma_xmit_xdpf()
623 	next_dcb_buf = &tx->dcbs_buf[next_to_use];  in lan966x_fdma_xmit_xdpf()
676 	__fdma_dcb_add(&tx->fdma,  in lan966x_fdma_xmit_xdpf()
688 	lan966x_fdma_tx_start(tx);  in lan966x_fdma_xmit_xdpf()
701 	struct lan966x_tx *tx = &lan966x->tx;  in lan966x_fdma_xmit()  local
709 	next_to_use = lan966x_fdma_get_next_dcb(tx);  in lan966x_fdma_xmit()
747 	next_dcb_buf = &tx->dcbs_buf[next_to_use];  in lan966x_fdma_xmit()
757 	fdma_dcb_add(&tx->fdma,  in lan966x_fdma_xmit()
771 	lan966x_fdma_tx_start(tx);  in lan966x_fdma_xmit()
942 	lan966x->tx.lan966x = lan966x;  in lan966x_fdma_init()
943 	lan966x->tx.fdma.channel_id = FDMA_INJ_CHANNEL;  in lan966x_fdma_init()
944 	lan966x->tx.fdma.n_dcbs = FDMA_DCB_MAX;  in lan966x_fdma_init()
945 	lan966x->tx.fdma.n_dbs = FDMA_TX_DCB_MAX_DBS;  in lan966x_fdma_init()
946 	lan966x->tx.fdma.priv = lan966x;  in lan966x_fdma_init()
947 	lan966x->tx.fdma.size = fdma_get_size(&lan966x->tx.fdma);  in lan966x_fdma_init()
948 	lan966x->tx.fdma.db_size = PAGE_SIZE << lan966x->rx.page_order;  in lan966x_fdma_init()
949 	lan966x->tx.fdma.ops.nextptr_cb = &fdma_nextptr_cb;  in lan966x_fdma_init()
950 	lan966x->tx.fdma.ops.dataptr_cb = &lan966x_fdma_tx_dataptr_cb;  in lan966x_fdma_init()
956 	err = lan966x_fdma_tx_alloc(&lan966x->tx);  in lan966x_fdma_init()
973 	lan966x_fdma_tx_disable(&lan966x->tx);  in lan966x_fdma_deinit()
981 	lan966x_fdma_tx_free(&lan966x->tx);  in lan966x_fdma_deinit()