13 	struct lan966x_rx *rx = &lan966x->rx;  in lan966x_fdma_rx_dataptr_cb()  local
16 	page = page_pool_dev_alloc_pages(rx->page_pool);  in lan966x_fdma_rx_dataptr_cb()
20 	rx->page[dcb][db] = page;  in lan966x_fdma_rx_dataptr_cb()
51 static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx)  in lan966x_fdma_rx_free_pages()  argument
53 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_free_pages()
58 			page_pool_put_full_page(rx->page_pool,  in lan966x_fdma_rx_free_pages()
59 						rx->page[i][j], false);  in lan966x_fdma_rx_free_pages()
63 static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx)  in lan966x_fdma_rx_free_page()  argument
65 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_free_page()
68 	page = rx->page[fdma->dcb_index][fdma->db_index];  in lan966x_fdma_rx_free_page()
72 	page_pool_recycle_direct(rx->page_pool, page);  in lan966x_fdma_rx_free_page()
75 static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx)  in lan966x_fdma_rx_alloc_page_pool()  argument
77 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_alloc_page_pool()
79 		.order = rx->page_order,  in lan966x_fdma_rx_alloc_page_pool()
81 		.pool_size = rx->fdma.n_dcbs,  in lan966x_fdma_rx_alloc_page_pool()
86 		.max_len = rx->max_mtu -  in lan966x_fdma_rx_alloc_page_pool()
93 	rx->page_pool = page_pool_create(&pp_params);  in lan966x_fdma_rx_alloc_page_pool()
104 					   rx->page_pool);  in lan966x_fdma_rx_alloc_page_pool()
107 	return PTR_ERR_OR_ZERO(rx->page_pool);  in lan966x_fdma_rx_alloc_page_pool()
110 static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx)  in lan966x_fdma_rx_alloc()  argument
112 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_alloc()
113 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_alloc()
116 	if (lan966x_fdma_rx_alloc_page_pool(rx))  in lan966x_fdma_rx_alloc()
117 		return PTR_ERR(rx->page_pool);  in lan966x_fdma_rx_alloc()
129 static void lan966x_fdma_rx_start(struct lan966x_rx *rx)  in lan966x_fdma_rx_start()  argument
131 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_start()
132 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_start()
168 static void lan966x_fdma_rx_disable(struct lan966x_rx *rx)  in lan966x_fdma_rx_disable()  argument
170 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_disable()
171 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_disable()
188 static void lan966x_fdma_rx_reload(struct lan966x_rx *rx)  in lan966x_fdma_rx_reload()  argument
190 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_reload()
192 	lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(rx->fdma.channel_id)),  in lan966x_fdma_rx_reload()
332 	struct lan966x_rx *rx = &lan966x->rx;  in lan966x_fdma_tx_clear_buf()  local
376 				page_pool_recycle_direct(rx->page_pool,  in lan966x_fdma_tx_clear_buf()
391 static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port)  in lan966x_fdma_rx_check_frame()  argument
393 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_check_frame()
394 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_check_frame()
400 	page = rx->page[fdma->dcb_index][fdma->db_index];  in lan966x_fdma_rx_check_frame()
421 static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx,  in lan966x_fdma_rx_get_frame()  argument
424 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_get_frame()
425 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_rx_get_frame()
433 	page = rx->page[fdma->dcb_index][fdma->db_index];  in lan966x_fdma_rx_get_frame()
469 	page_pool_recycle_direct(rx->page_pool, page);  in lan966x_fdma_rx_get_frame()
477 	struct lan966x_rx *rx = &lan966x->rx;  in lan966x_fdma_napi_poll()  local
479 	struct fdma *fdma = &rx->fdma;  in lan966x_fdma_napi_poll()
495 		switch (lan966x_fdma_rx_check_frame(rx, &src_port)) {  in lan966x_fdma_napi_poll()
499 			lan966x_fdma_rx_free_page(rx);  in lan966x_fdma_napi_poll()
509 			lan966x_fdma_rx_free_page(rx);  in lan966x_fdma_napi_poll()
514 		skb = lan966x_fdma_rx_get_frame(rx, src_port);  in lan966x_fdma_napi_poll()
532 		lan966x_fdma_rx_reload(rx);  in lan966x_fdma_napi_poll()
817 	memcpy(&fdma_rx_old, &lan966x->rx.fdma, sizeof(struct fdma));  in lan966x_fdma_reload()
818 	page_pool = lan966x->rx.page_pool;  in lan966x_fdma_reload()
824 	lan966x_fdma_rx_disable(&lan966x->rx);  in lan966x_fdma_reload()
825 	lan966x_fdma_rx_free_pages(&lan966x->rx);  in lan966x_fdma_reload()
826 	lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1;  in lan966x_fdma_reload()
827 	lan966x->rx.max_mtu = new_mtu;  in lan966x_fdma_reload()
828 	err = lan966x_fdma_rx_alloc(&lan966x->rx);  in lan966x_fdma_reload()
831 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_reload()
842 	lan966x->rx.page_pool = page_pool;  in lan966x_fdma_reload()
843 	memcpy(&lan966x->rx.fdma, &fdma_rx_old, sizeof(struct fdma));  in lan966x_fdma_reload()
844 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_reload()
893 	if (max_mtu == lan966x->rx.max_mtu)  in lan966x_fdma_change_mtu()
932 	lan966x->rx.lan966x = lan966x;  in lan966x_fdma_init()
933 	lan966x->rx.fdma.channel_id = FDMA_XTR_CHANNEL;  in lan966x_fdma_init()
934 	lan966x->rx.fdma.n_dcbs = FDMA_DCB_MAX;  in lan966x_fdma_init()
935 	lan966x->rx.fdma.n_dbs = FDMA_RX_DCB_MAX_DBS;  in lan966x_fdma_init()
936 	lan966x->rx.fdma.priv = lan966x;  in lan966x_fdma_init()
937 	lan966x->rx.fdma.size = fdma_get_size(&lan966x->rx.fdma);  in lan966x_fdma_init()
938 	lan966x->rx.fdma.db_size = PAGE_SIZE << lan966x->rx.page_order;  in lan966x_fdma_init()
939 	lan966x->rx.fdma.ops.nextptr_cb = &fdma_nextptr_cb;  in lan966x_fdma_init()
940 	lan966x->rx.fdma.ops.dataptr_cb = &lan966x_fdma_rx_dataptr_cb;  in lan966x_fdma_init()
941 	lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x);  in lan966x_fdma_init()
948 	lan966x->tx.fdma.db_size = PAGE_SIZE << lan966x->rx.page_order;  in lan966x_fdma_init()
952 	err = lan966x_fdma_rx_alloc(&lan966x->rx);  in lan966x_fdma_init()
958 		fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma);  in lan966x_fdma_init()
962 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_init()
972 	lan966x_fdma_rx_disable(&lan966x->rx);  in lan966x_fdma_deinit()
978 	lan966x_fdma_rx_free_pages(&lan966x->rx);  in lan966x_fdma_deinit()
979 	fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma);  in lan966x_fdma_deinit()
980 	page_pool_destroy(lan966x->rx.page_pool);  in lan966x_fdma_deinit()