44 void prueth_cleanup_rx_chns(struct prueth_emac *emac,  in prueth_cleanup_rx_chns()  argument
61 void prueth_cleanup_tx_chns(struct prueth_emac *emac)  in prueth_cleanup_tx_chns()  argument
65 	for (i = 0; i < emac->tx_ch_num; i++) {  in prueth_cleanup_tx_chns()
66 		struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];  in prueth_cleanup_tx_chns()
82 void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num)  in prueth_ndev_del_tx_napi()  argument
87 		struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];  in prueth_ndev_del_tx_napi()
133 int emac_tx_complete_packets(struct prueth_emac *emac, int chn,  in emac_tx_complete_packets()  argument
136 	struct net_device *ndev = emac->ndev;  in emac_tx_complete_packets()
147 	tx_chn = &emac->tx_chns[chn];  in emac_tx_complete_packets()
156 			if (atomic_dec_and_test(&emac->tdown_cnt))  in emac_tx_complete_packets()
157 				complete(&emac->tdown_complete);  in emac_tx_complete_packets()
222 	struct prueth_emac *emac = tx_chn->emac;  in emac_napi_tx_poll()  local
226 	num_tx_packets = emac_tx_complete_packets(emac, tx_chn->id, budget,  in emac_napi_tx_poll()
255 int prueth_ndev_add_tx_napi(struct prueth_emac *emac)  in prueth_ndev_add_tx_napi()  argument
257 	struct prueth *prueth = emac->prueth;  in prueth_ndev_add_tx_napi()
260 	for (i = 0; i < emac->tx_ch_num; i++) {  in prueth_ndev_add_tx_napi()
261 		struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];  in prueth_ndev_add_tx_napi()
263 		netif_napi_add_tx(emac->ndev, &tx_chn->napi_tx, emac_napi_tx_poll);  in prueth_ndev_add_tx_napi()
279 	prueth_ndev_del_tx_napi(emac, i);  in prueth_ndev_add_tx_napi()
284 int prueth_init_tx_chns(struct prueth_emac *emac)  in prueth_init_tx_chns()  argument
293 	struct device *dev = emac->prueth->dev;  in prueth_init_tx_chns()
294 	struct net_device *ndev = emac->ndev;  in prueth_init_tx_chns()
298 	slice = prueth_emac_slice(emac);  in prueth_init_tx_chns()
302 	init_completion(&emac->tdown_complete);  in prueth_init_tx_chns()
311 	for (i = 0; i < emac->tx_ch_num; i++) {  in prueth_init_tx_chns()
312 		struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];  in prueth_init_tx_chns()
318 		tx_chn->emac = emac;  in prueth_init_tx_chns()
360 	prueth_cleanup_tx_chns(emac);  in prueth_init_tx_chns()
365 int prueth_init_rx_chns(struct prueth_emac *emac,  in prueth_init_rx_chns()  argument
371 	struct device *dev = emac->prueth->dev;  in prueth_init_rx_chns()
372 	struct net_device *ndev = emac->ndev;  in prueth_init_rx_chns()
377 	slice = prueth_emac_slice(emac);  in prueth_init_rx_chns()
417 	if (emac->is_sr1 && !strcmp(name, "rxmgm")) {  in prueth_init_rx_chns()
418 		emac->rx_mgm_flow_id_base = flow_id_base;  in prueth_init_rx_chns()
421 		emac->rx_flow_id_base = flow_id_base;  in prueth_init_rx_chns()
447 		rx_flow_cfg.rxfdq_cfg.mode = emac->prueth->pdata.fdqring_mode;  in prueth_init_rx_chns()
470 	prueth_cleanup_rx_chns(emac, rx_chn, max_rflows);  in prueth_init_rx_chns()
475 int prueth_dma_rx_push_mapped(struct prueth_emac *emac,  in prueth_dma_rx_push_mapped()  argument
479 	struct net_device *ndev = emac->ndev;  in prueth_dma_rx_push_mapped()
523 void emac_rx_timestamp(struct prueth_emac *emac,  in emac_rx_timestamp()  argument
529 	if (emac->is_sr1) {  in emac_rx_timestamp()
532 		u32 hi_sw = readl(emac->prueth->shram.va +  in emac_rx_timestamp()
545  * @emac: emac device
552 u32 emac_xmit_xdp_frame(struct prueth_emac *emac,  in emac_xmit_xdp_frame()  argument
558 	struct net_device *ndev = emac->ndev;  in emac_xmit_xdp_frame()
571 	tx_chn = &emac->tx_chns[q_idx];  in emac_xmit_xdp_frame()
601 	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8)));  in emac_xmit_xdp_frame()
632  * @emac: emac device
639 static u32 emac_run_xdp(struct prueth_emac *emac, struct xdp_buff *xdp,  in emac_run_xdp()  argument
642 	struct net_device *ndev = emac->ndev;  in emac_run_xdp()
651 	xdp_prog = READ_ONCE(emac->xdp_prog);  in emac_run_xdp()
664 		q_idx = cpu % emac->tx_ch_num;  in emac_run_xdp()
667 		result = emac_xmit_xdp_frame(emac, xdpf, page, q_idx);  in emac_run_xdp()
677 		err = xdp_do_redirect(emac->ndev, xdp, xdp_prog);  in emac_run_xdp()
684 		bpf_warn_invalid_xdp_action(emac->ndev, xdp_prog, act);  in emac_run_xdp()
688 		trace_xdp_exception(emac->ndev, xdp_prog, act);  in emac_run_xdp()
692 		page_pool_recycle_direct(emac->rx_chns.pg_pool, page);  in emac_run_xdp()
697 static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state)  in emac_rx_packet()  argument
699 	struct prueth_rx_chn *rx_chn = &emac->rx_chns;  in emac_rx_packet()
701 	struct net_device *ndev = emac->ndev;  in emac_rx_packet()
755 	if (emac->xdp_prog) {  in emac_rx_packet()
759 		*xdp_state = emac_run_xdp(emac, &xdp, page, &pkt_len);  in emac_rx_packet()
782 	if (emac->rx_ts_enabled)  in emac_rx_packet()
783 		emac_rx_timestamp(emac, skb, psdata);  in emac_rx_packet()
785 	if (emac->prueth->is_switch_mode)  in emac_rx_packet()
786 		skb->offload_fwd_mark = emac->offload_fwd_mark;  in emac_rx_packet()
790 	napi_gro_receive(&emac->napi_rx, skb);  in emac_rx_packet()
796 	ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page,  in emac_rx_packet()
826 static int prueth_tx_ts_cookie_get(struct prueth_emac *emac)  in prueth_tx_ts_cookie_get()  argument
832 		if (!emac->tx_ts_skb[i]) {  in prueth_tx_ts_cookie_get()
833 			emac->tx_ts_skb[i] = ERR_PTR(-EBUSY); /* reserve slot */  in prueth_tx_ts_cookie_get()
842  * icssg_ndo_start_xmit - EMAC Transmit function
844  * @ndev: EMAC network adapter
847  * EMAC hardware transmit queue
856 	struct prueth_emac *emac = netdev_priv(ndev);  in icssg_ndo_start_xmit()  local
857 	struct prueth *prueth = emac->prueth;  in icssg_ndo_start_xmit()
871 	tx_chn = &emac->tx_chns[q_idx];  in icssg_ndo_start_xmit()
896 	    emac->tx_ts_enabled) {  in icssg_ndo_start_xmit()
897 		tx_ts_cookie = prueth_tx_ts_cookie_get(emac);  in icssg_ndo_start_xmit()
903 			emac->tx_ts_skb[tx_ts_cookie] = skb_get(skb);  in icssg_ndo_start_xmit()
913 	dst_tag_id = emac->port_id | (q_idx << 8);  in icssg_ndo_start_xmit()
983 		atomic_inc(&emac->tx_ts_pending);  in icssg_ndo_start_xmit()
999 		dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);  in icssg_ndo_start_xmit()
1000 		emac->tx_ts_skb[tx_ts_cookie] = NULL;  in icssg_ndo_start_xmit()
1017 		dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);  in icssg_ndo_start_xmit()
1018 		emac->tx_ts_skb[tx_ts_cookie] = NULL;  in icssg_ndo_start_xmit()
1057 	struct prueth_emac *emac = dev_id;  in prueth_rx_irq()  local
1060 	napi_schedule(&emac->napi_rx);  in prueth_rx_irq()
1066 void prueth_cleanup_tx_ts(struct prueth_emac *emac)  in prueth_cleanup_tx_ts()  argument
1071 		if (emac->tx_ts_skb[i]) {  in prueth_cleanup_tx_ts()
1072 			dev_kfree_skb_any(emac->tx_ts_skb[i]);  in prueth_cleanup_tx_ts()
1073 			emac->tx_ts_skb[i] = NULL;  in prueth_cleanup_tx_ts()
1081 	struct prueth_emac *emac = prueth_napi_to_emac(napi_rx);  in icssg_napi_rx_poll()  local
1082 	int rx_flow = emac->is_sr1 ?  in icssg_napi_rx_poll()
1084 	int flow = emac->is_sr1 ?  in icssg_napi_rx_poll()
1096 			ret = emac_rx_packet(emac, flow, &xdp_state);  in icssg_napi_rx_poll()
1111 		if (unlikely(emac->rx_pace_timeout_ns)) {  in icssg_napi_rx_poll()
1112 			hrtimer_start(&emac->rx_hrtimer,  in icssg_napi_rx_poll()
1113 				      ns_to_ktime(emac->rx_pace_timeout_ns),  in icssg_napi_rx_poll()
1116 			enable_irq(emac->rx_chns.irq[rx_flow]);  in icssg_napi_rx_poll()
1124 static struct page_pool *prueth_create_page_pool(struct prueth_emac *emac,  in prueth_create_page_pool()  argument
1134 	pp_params.nid = dev_to_node(emac->prueth->dev);  in prueth_create_page_pool()
1137 	pp_params.napi = &emac->napi_rx;  in prueth_create_page_pool()
1142 		netdev_err(emac->ndev, "cannot create rx page pool\n");  in prueth_create_page_pool()
1147 int prueth_prepare_rx_chan(struct prueth_emac *emac,  in prueth_prepare_rx_chan()  argument
1155 	pool = prueth_create_page_pool(emac, chn->dma_dev, chn->descs_num);  in prueth_prepare_rx_chan()
1168 			netdev_err(emac->ndev, "couldn't allocate rx page\n");  in prueth_prepare_rx_chan()
1173 		ret = prueth_dma_rx_push_mapped(emac, chn, page, buf_size);  in prueth_prepare_rx_chan()
1175 			netdev_err(emac->ndev,  in prueth_prepare_rx_chan()
1186 	prueth_reset_rx_chan(&emac->rx_chns, PRUETH_MAX_RX_FLOWS, false);  in prueth_prepare_rx_chan()
1192 void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num,  in prueth_reset_tx_chan()  argument
1199 			k3_udma_glue_reset_tx_chn(emac->tx_chns[i].tx_chn,  in prueth_reset_tx_chan()
1200 						  &emac->tx_chns[i],  in prueth_reset_tx_chan()
1202 		k3_udma_glue_disable_tx_chn(emac->tx_chns[i].tx_chn);  in prueth_reset_tx_chan()
1228 	struct prueth_emac *emac = netdev_priv(ndev);  in emac_set_ts_config()  local
1236 		emac->tx_ts_enabled = 0;  in emac_set_ts_config()
1239 		emac->tx_ts_enabled = 1;  in emac_set_ts_config()
1247 		emac->rx_ts_enabled = 0;  in emac_set_ts_config()
1264 		emac->rx_ts_enabled = 1;  in emac_set_ts_config()
1277 	struct prueth_emac *emac = netdev_priv(ndev);  in emac_get_ts_config()  local
1281 	config.tx_type = emac->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;  in emac_get_ts_config()
1282 	config.rx_filter = emac->rx_ts_enabled ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;  in emac_get_ts_config()
1306 	struct prueth_emac *emac = netdev_priv(ndev);  in icssg_ndo_get_stats64()  local
1308 	emac_update_hardware_stats(emac);  in icssg_ndo_get_stats64()
1310 	stats->rx_packets     = emac_get_stat_by_name(emac, "rx_packets");  in icssg_ndo_get_stats64()
1311 	stats->rx_bytes       = emac_get_stat_by_name(emac, "rx_bytes");  in icssg_ndo_get_stats64()
1312 	stats->tx_packets     = emac_get_stat_by_name(emac, "tx_packets");  in icssg_ndo_get_stats64()
1313 	stats->tx_bytes       = emac_get_stat_by_name(emac, "tx_bytes");  in icssg_ndo_get_stats64()
1314 	stats->rx_crc_errors  = emac_get_stat_by_name(emac, "rx_crc_errors");  in icssg_ndo_get_stats64()
1315 	stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors");  in icssg_ndo_get_stats64()
1316 	stats->multicast      = emac_get_stat_by_name(emac, "rx_multicast_frames");  in icssg_ndo_get_stats64()
1319 			    emac_get_stat_by_name(emac, "FW_RX_ERROR") +  in icssg_ndo_get_stats64()
1320 			    emac_get_stat_by_name(emac, "FW_RX_EOF_SHORT_FRMERR") +  in icssg_ndo_get_stats64()
1321 			    emac_get_stat_by_name(emac, "FW_RX_B0_DROP_EARLY_EOF") +  in icssg_ndo_get_stats64()
1322 			    emac_get_stat_by_name(emac, "FW_RX_EXP_FRAG_Q_DROP") +  in icssg_ndo_get_stats64()
1323 			    emac_get_stat_by_name(emac, "FW_RX_FIFO_OVERRUN");  in icssg_ndo_get_stats64()
1325 			    emac_get_stat_by_name(emac, "FW_DROPPED_PKT") +  in icssg_ndo_get_stats64()
1326 			    emac_get_stat_by_name(emac, "FW_INF_PORT_DISABLED") +  in icssg_ndo_get_stats64()
1327 			    emac_get_stat_by_name(emac, "FW_INF_SAV") +  in icssg_ndo_get_stats64()
1328 			    emac_get_stat_by_name(emac, "FW_INF_SA_DL") +  in icssg_ndo_get_stats64()
1329 			    emac_get_stat_by_name(emac, "FW_INF_PORT_BLOCKED") +  in icssg_ndo_get_stats64()
1330 			    emac_get_stat_by_name(emac, "FW_INF_DROP_TAGGED") +  in icssg_ndo_get_stats64()
1331 			    emac_get_stat_by_name(emac, "FW_INF_DROP_PRIOTAGGED") +  in icssg_ndo_get_stats64()
1332 			    emac_get_stat_by_name(emac, "FW_INF_DROP_NOTAG") +  in icssg_ndo_get_stats64()
1333 			    emac_get_stat_by_name(emac, "FW_INF_DROP_NOTMEMBER");  in icssg_ndo_get_stats64()
1336 			    emac_get_stat_by_name(emac, "FW_RTU_PKT_DROP") +  in icssg_ndo_get_stats64()
1337 			    emac_get_stat_by_name(emac, "FW_TX_DROPPED_PACKET") +  in icssg_ndo_get_stats64()
1338 			    emac_get_stat_by_name(emac, "FW_TX_TS_DROPPED_PACKET") +  in icssg_ndo_get_stats64()
1339 			    emac_get_stat_by_name(emac, "FW_TX_JUMBO_FRM_CUTOFF");  in icssg_ndo_get_stats64()
1346 	struct prueth_emac *emac = netdev_priv(ndev);  in icssg_ndo_get_phys_port_name()  local
1349 	ret = snprintf(name, len, "p%d", emac->port_id);  in icssg_ndo_get_phys_port_name()
1398 	struct prueth_emac *emac;  in prueth_netdev_exit()  local
1405 	emac = prueth->emac[mac];  in prueth_netdev_exit()
1406 	if (!emac)  in prueth_netdev_exit()
1409 	if (of_phy_is_fixed_link(emac->phy_node))  in prueth_netdev_exit()
1410 		of_phy_deregister_fixed_link(emac->phy_node);  in prueth_netdev_exit()
1412 	netif_napi_del(&emac->napi_rx);  in prueth_netdev_exit()
1414 	pruss_release_mem_region(prueth->pruss, &emac->dram);  in prueth_netdev_exit()
1415 	destroy_workqueue(emac->cmd_wq);  in prueth_netdev_exit()
1416 	free_netdev(emac->ndev);  in prueth_netdev_exit()
1417 	prueth->emac[mac] = NULL;  in prueth_netdev_exit()