438 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_rx_push()  local
445 	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);  in am65_cpsw_nuss_rx_push()
450 	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
452 	buf_dma = dma_map_single(rx_chn->dma_dev,  in am65_cpsw_nuss_rx_push()
455 	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_rx_push()
456 		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
463 	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_push()
470 	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx,  in am65_cpsw_nuss_rx_push()
510 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_destroy_rxq()  local
515 	flow = &rx_chn->flows[id];  in am65_cpsw_destroy_rxq()
518 	k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, id, rx_chn,  in am65_cpsw_destroy_rxq()
539 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_destroy_rxqs()  local
543 	k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true);  in am65_cpsw_destroy_rxqs()
554 	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_destroy_rxqs()
559 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_create_rxq()  local
575 	flow = &rx_chn->flows[id];  in am65_cpsw_create_rxq()
646 	ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_create_rxqs()
775 	struct am65_cpsw_rx_chn *rx_chn = data;  in am65_cpsw_nuss_rx_cleanup()  local
783 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_cleanup()
788 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_cleanup()
789 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_cleanup()
790 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_cleanup()
791 	am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false);  in am65_cpsw_nuss_rx_cleanup()
1271 	struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns;  in am65_cpsw_nuss_rx_packets()  local
1290 	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);  in am65_cpsw_nuss_rx_packets()
1304 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_packets()
1312 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);  in am65_cpsw_nuss_rx_packets()
1322 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_packets()
1323 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_packets()
2342 	struct am65_cpsw_rx_chn *rx_chn;  in am65_cpsw_nuss_free_rx_chns()  local
2344 	rx_chn = &common->rx_chns;  in am65_cpsw_nuss_free_rx_chns()
2346 	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))  in am65_cpsw_nuss_free_rx_chns()
2347 		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);  in am65_cpsw_nuss_free_rx_chns()
2349 	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))  in am65_cpsw_nuss_free_rx_chns()
2350 		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);  in am65_cpsw_nuss_free_rx_chns()
2356 	struct am65_cpsw_rx_chn *rx_chn;  in am65_cpsw_nuss_remove_rx_chns()  local
2360 	rx_chn = &common->rx_chns;  in am65_cpsw_nuss_remove_rx_chns()
2361 	flows = rx_chn->flows;  in am65_cpsw_nuss_remove_rx_chns()
2376 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_init_rx_chns()  local
2393 	rx_chn->dev = dev;  in am65_cpsw_nuss_init_rx_chns()
2394 	rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num;  in am65_cpsw_nuss_init_rx_chns()
2397 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
2401 	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);  in am65_cpsw_nuss_init_rx_chns()
2402 	if (IS_ERR(rx_chn->rx_chn)) {  in am65_cpsw_nuss_init_rx_chns()
2403 		ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),  in am65_cpsw_nuss_init_rx_chns()
2407 	rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
2409 	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,  in am65_cpsw_nuss_init_rx_chns()
2410 							  rx_chn->descs_num,  in am65_cpsw_nuss_init_rx_chns()
2412 	if (IS_ERR(rx_chn->desc_pool)) {  in am65_cpsw_nuss_init_rx_chns()
2413 		ret = PTR_ERR(rx_chn->desc_pool);  in am65_cpsw_nuss_init_rx_chns()
2418 	hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool);  in am65_cpsw_nuss_init_rx_chns()
2419 	rx_chn->dsize_log2 = __fls(hdesc_size_out);  in am65_cpsw_nuss_init_rx_chns()
2420 	WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2));  in am65_cpsw_nuss_init_rx_chns()
2423 			k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
2445 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
2456 		ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
2464 				k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
2467 		flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);  in am65_cpsw_nuss_init_rx_chns()
2506 		flow = &rx_chn->flows[i];  in am65_cpsw_nuss_init_rx_chns()
3367 		k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i,  in am65_cpsw_nuss_register_ndevs()
3371 	k3_udma_glue_disable_rx_chn(rx_chan->rx_chn);  in am65_cpsw_nuss_register_ndevs()