83 	struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];  in t7xx_dpmaif_update_bat_wr_idx()  local
84 	struct dpmaif_bat_request *bat_req = rxq->bat_req;  in t7xx_dpmaif_update_bat_wr_idx()
87 	if (!rxq->que_started) {  in t7xx_dpmaif_update_bat_wr_idx()
88 		dev_err(dpmaif_ctrl->dev, "RX queue %d has not been started\n", rxq->index);  in t7xx_dpmaif_update_bat_wr_idx()
235 static int t7xx_dpmaifq_release_pit_entry(struct dpmaif_rx_queue *rxq,  in t7xx_dpmaifq_release_pit_entry()  argument
238 	struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;  in t7xx_dpmaifq_release_pit_entry()
242 	if (!rxq->que_started)  in t7xx_dpmaifq_release_pit_entry()
245 	if (rel_entry_num >= rxq->pit_size_cnt) {  in t7xx_dpmaifq_release_pit_entry()
246 		dev_err(rxq->dpmaif_ctrl->dev, "Invalid PIT release index\n");  in t7xx_dpmaifq_release_pit_entry()
250 	old_rel_idx = rxq->pit_release_rd_idx;  in t7xx_dpmaifq_release_pit_entry()
252 	hw_wr_idx = rxq->pit_wr_idx;  in t7xx_dpmaifq_release_pit_entry()
253 	if (hw_wr_idx < old_rel_idx && new_rel_idx >= rxq->pit_size_cnt)  in t7xx_dpmaifq_release_pit_entry()
254 		new_rel_idx -= rxq->pit_size_cnt;  in t7xx_dpmaifq_release_pit_entry()
256 	ret = t7xx_dpmaif_dlq_add_pit_remain_cnt(hw_info, rxq->index, rel_entry_num);  in t7xx_dpmaifq_release_pit_entry()
258 		dev_err(rxq->dpmaif_ctrl->dev, "PIT release failure: %d\n", ret);  in t7xx_dpmaifq_release_pit_entry()
262 	rxq->pit_release_rd_idx = new_rel_idx;  in t7xx_dpmaifq_release_pit_entry()
275 static int t7xx_frag_bat_cur_bid_check(struct dpmaif_rx_queue *rxq,  in t7xx_frag_bat_cur_bid_check()  argument
278 	struct dpmaif_bat_request *bat_frag = rxq->bat_frag;  in t7xx_frag_bat_cur_bid_check()
392 static int t7xx_dpmaif_set_frag_to_skb(const struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_set_frag_to_skb()  argument
397 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_set_frag_to_skb()
402 	page_info = rxq->bat_frag->bat_skb;  in t7xx_dpmaif_set_frag_to_skb()
424 static int t7xx_dpmaif_get_frag(struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_get_frag()  argument
431 	ret = t7xx_frag_bat_cur_bid_check(rxq, cur_bid);  in t7xx_dpmaif_get_frag()
435 	ret = t7xx_dpmaif_set_frag_to_skb(rxq, pkt_info, skb_info->cur_skb);  in t7xx_dpmaif_get_frag()
437 		dev_err(rxq->dpmaif_ctrl->dev, "Failed to set frag data to skb: %d\n", ret);  in t7xx_dpmaif_get_frag()
441 	t7xx_dpmaif_set_bat_mask(rxq->bat_frag, cur_bid);  in t7xx_dpmaif_get_frag()
445 static int t7xx_bat_cur_bid_check(struct dpmaif_rx_queue *rxq, const unsigned int cur_bid)  in t7xx_bat_cur_bid_check()  argument
447 	struct dpmaif_bat_skb *bat_skb = rxq->bat_req->bat_skb;  in t7xx_bat_cur_bid_check()
461 static int t7xx_dpmaif_check_pit_seq(struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_check_pit_seq()  argument
464 	unsigned int cur_pit_seq, expect_pit_seq = rxq->expect_pit_seq;  in t7xx_dpmaif_check_pit_seq()
471 	rxq->expect_pit_seq++;  in t7xx_dpmaif_check_pit_seq()
472 	if (rxq->expect_pit_seq >= DPMAIF_DL_PIT_SEQ_VALUE)  in t7xx_dpmaif_check_pit_seq()
473 		rxq->expect_pit_seq = 0;  in t7xx_dpmaif_check_pit_seq()
499 static int t7xx_dpmaif_release_bat_entry(const struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_release_bat_entry()  argument
503 	struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;  in t7xx_dpmaif_release_bat_entry()
508 	if (!rxq->que_started || !rel_entry_num)  in t7xx_dpmaif_release_bat_entry()
512 		bat = rxq->bat_frag;  in t7xx_dpmaif_release_bat_entry()
513 		hw_rd_idx = t7xx_dpmaif_dl_get_frg_rd_idx(hw_info, rxq->index);  in t7xx_dpmaif_release_bat_entry()
515 		bat = rxq->bat_req;  in t7xx_dpmaif_release_bat_entry()
516 		hw_rd_idx = t7xx_dpmaif_dl_get_bat_rd_idx(hw_info, rxq->index);  in t7xx_dpmaif_release_bat_entry()
555 static int t7xx_dpmaif_pit_release_and_add(struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_pit_release_and_add()  argument
559 	if (rxq->pit_remain_release_cnt < DPMAIF_PIT_CNT_THRESHOLD)  in t7xx_dpmaif_pit_release_and_add()
562 	ret = t7xx_dpmaifq_release_pit_entry(rxq, rxq->pit_remain_release_cnt);  in t7xx_dpmaif_pit_release_and_add()
566 	rxq->pit_remain_release_cnt = 0;  in t7xx_dpmaif_pit_release_and_add()
570 static int t7xx_dpmaif_bat_release_and_add(const struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_bat_release_and_add()  argument
575 	bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(rxq->bat_req);  in t7xx_dpmaif_bat_release_and_add()
579 	ret = t7xx_dpmaif_release_bat_entry(rxq, bid_cnt, BAT_TYPE_NORMAL);  in t7xx_dpmaif_bat_release_and_add()
581 		dev_err(rxq->dpmaif_ctrl->dev, "Release PKT BAT failed: %d\n", ret);  in t7xx_dpmaif_bat_release_and_add()
585 	ret = t7xx_dpmaif_rx_buf_alloc(rxq->dpmaif_ctrl, rxq->bat_req, rxq->index, bid_cnt, false);  in t7xx_dpmaif_bat_release_and_add()
587 		dev_err(rxq->dpmaif_ctrl->dev, "Allocate new RX buffer failed: %d\n", ret);  in t7xx_dpmaif_bat_release_and_add()
592 static int t7xx_dpmaif_frag_bat_release_and_add(const struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_frag_bat_release_and_add()  argument
597 	bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(rxq->bat_frag);  in t7xx_dpmaif_frag_bat_release_and_add()
601 	ret = t7xx_dpmaif_release_bat_entry(rxq, bid_cnt, BAT_TYPE_FRAG);  in t7xx_dpmaif_frag_bat_release_and_add()
603 		dev_err(rxq->dpmaif_ctrl->dev, "Release BAT entry failed: %d\n", ret);  in t7xx_dpmaif_frag_bat_release_and_add()
607 	return t7xx_dpmaif_rx_frag_alloc(rxq->dpmaif_ctrl, rxq->bat_frag, bid_cnt, false);  in t7xx_dpmaif_frag_bat_release_and_add()
610 static void t7xx_dpmaif_parse_msg_pit(const struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_parse_msg_pit()  argument
622 static int t7xx_dpmaif_set_data_to_skb(const struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_set_data_to_skb()  argument
627 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_set_data_to_skb()
633 	bat_skb = rxq->bat_req->bat_skb;  in t7xx_dpmaif_set_data_to_skb()
658 static int t7xx_dpmaif_get_rx_pkt(struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_get_rx_pkt()  argument
665 	ret = t7xx_bat_cur_bid_check(rxq, cur_bid);  in t7xx_dpmaif_get_rx_pkt()
669 	ret = t7xx_dpmaif_set_data_to_skb(rxq, pkt_info, skb_info);  in t7xx_dpmaif_get_rx_pkt()
671 		dev_err(rxq->dpmaif_ctrl->dev, "RX set data to skb failed: %d\n", ret);  in t7xx_dpmaif_get_rx_pkt()
675 	t7xx_dpmaif_set_bat_mask(rxq->bat_req, cur_bid);  in t7xx_dpmaif_get_rx_pkt()
679 static int t7xx_dpmaifq_rx_notify_hw(struct dpmaif_rx_queue *rxq)  in t7xx_dpmaifq_rx_notify_hw()  argument
681 	struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaifq_rx_notify_hw()
686 	ret = t7xx_dpmaif_pit_release_and_add(rxq);  in t7xx_dpmaifq_rx_notify_hw()
688 		dev_err(dpmaif_ctrl->dev, "RXQ%u update PIT failed: %d\n", rxq->index, ret);  in t7xx_dpmaifq_rx_notify_hw()
693 static void t7xx_dpmaif_rx_skb(struct dpmaif_rx_queue *rxq,  in t7xx_dpmaif_rx_skb()  argument
696 	struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaif_rx_skb()
714 	dpmaif_ctrl->callbacks->recv_skb(dpmaif_ctrl->t7xx_dev->ccmni_ctlb, skb, &rxq->napi);  in t7xx_dpmaif_rx_skb()
717 static int t7xx_dpmaif_rx_start(struct dpmaif_rx_queue *rxq, const unsigned int pit_cnt,  in t7xx_dpmaif_rx_start()  argument
721 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_rx_start()
725 	pit_len = rxq->pit_size_cnt;  in t7xx_dpmaif_rx_start()
726 	skb_info = &rxq->rx_data_info;  in t7xx_dpmaif_rx_start()
727 	cur_pit = rxq->pit_rd_idx;  in t7xx_dpmaif_rx_start()
736 		pkt_info = (struct dpmaif_pit *)rxq->pit_base + cur_pit;  in t7xx_dpmaif_rx_start()
737 		if (t7xx_dpmaif_check_pit_seq(rxq, pkt_info)) {  in t7xx_dpmaif_rx_start()
738 			dev_err_ratelimited(dev, "RXQ%u checks PIT SEQ fail\n", rxq->index);  in t7xx_dpmaif_rx_start()
746 				dev_err(dev, "RXQ%u received repeated PIT\n", rxq->index);  in t7xx_dpmaif_rx_start()
749 			t7xx_dpmaif_parse_msg_pit(rxq, pkt_info, skb_info);  in t7xx_dpmaif_rx_start()
753 				ret = t7xx_dpmaif_get_rx_pkt(rxq, pkt_info, skb_info);  in t7xx_dpmaif_rx_start()
757 				ret = t7xx_dpmaif_get_frag(rxq, pkt_info, skb_info);  in t7xx_dpmaif_rx_start()
761 				dev_err_ratelimited(dev, "RXQ%u error payload\n", rxq->index);  in t7xx_dpmaif_rx_start()
767 					t7xx_dpmaif_rx_skb(rxq, skb_info);  in t7xx_dpmaif_rx_start()
779 		rxq->pit_rd_idx = cur_pit;  in t7xx_dpmaif_rx_start()
780 		rxq->pit_remain_release_cnt++;  in t7xx_dpmaif_rx_start()
783 			ret = t7xx_dpmaifq_rx_notify_hw(rxq);  in t7xx_dpmaif_rx_start()
790 		ret = t7xx_dpmaifq_rx_notify_hw(rxq);  in t7xx_dpmaif_rx_start()
798 static unsigned int t7xx_dpmaifq_poll_pit(struct dpmaif_rx_queue *rxq)  in t7xx_dpmaifq_poll_pit()  argument
802 	if (!rxq->que_started)  in t7xx_dpmaifq_poll_pit()
805 	hw_wr_idx = t7xx_dpmaif_dl_dlq_pit_get_wr_idx(&rxq->dpmaif_ctrl->hw_info, rxq->index);  in t7xx_dpmaifq_poll_pit()
806 	pit_cnt = t7xx_ring_buf_rd_wr_count(rxq->pit_size_cnt, rxq->pit_rd_idx, hw_wr_idx,  in t7xx_dpmaifq_poll_pit()
808 	rxq->pit_wr_idx = hw_wr_idx;  in t7xx_dpmaifq_poll_pit()
816 	struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];  in t7xx_dpmaif_napi_rx_data_collect()  local
820 	cnt = t7xx_dpmaifq_poll_pit(rxq);  in t7xx_dpmaif_napi_rx_data_collect()
824 	ret = t7xx_dpmaif_rx_start(rxq, cnt, budget, once_more);  in t7xx_dpmaif_napi_rx_data_collect()
826 		dev_err(dpmaif_ctrl->dev, "dlq%u rx ERR:%d\n", rxq->index, ret);  in t7xx_dpmaif_napi_rx_data_collect()
833 	struct dpmaif_rx_queue *rxq = container_of(napi, struct dpmaif_rx_queue, napi);  in t7xx_dpmaif_napi_rx_poll()  local
834 	struct t7xx_pci_dev *t7xx_dev = rxq->dpmaif_ctrl->t7xx_dev;  in t7xx_dpmaif_napi_rx_poll()
837 	atomic_set(&rxq->rx_processing, 1);  in t7xx_dpmaif_napi_rx_poll()
841 	if (!rxq->que_started) {  in t7xx_dpmaif_napi_rx_poll()
842 		atomic_set(&rxq->rx_processing, 0);  in t7xx_dpmaif_napi_rx_poll()
843 		pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
844 		dev_err(rxq->dpmaif_ctrl->dev, "Work RXQ: %d has not been started\n", rxq->index);  in t7xx_dpmaif_napi_rx_poll()
848 	if (!rxq->sleep_lock_pending)  in t7xx_dpmaif_napi_rx_poll()
854 		rxq->sleep_lock_pending = true;  in t7xx_dpmaif_napi_rx_poll()
859 	rxq->sleep_lock_pending = false;  in t7xx_dpmaif_napi_rx_poll()
862 		int rx_cnt = t7xx_dpmaif_napi_rx_data_collect(rxq->dpmaif_ctrl, rxq->index,  in t7xx_dpmaif_napi_rx_poll()
873 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
876 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
877 		t7xx_dpmaif_dlq_unmask_rx_done(&rxq->dpmaif_ctrl->hw_info, rxq->index);  in t7xx_dpmaif_napi_rx_poll()
878 		t7xx_pci_enable_sleep(rxq->dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_napi_rx_poll()
879 		pm_runtime_mark_last_busy(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
880 		pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
881 		atomic_set(&rxq->rx_processing, 0);  in t7xx_dpmaif_napi_rx_poll()
883 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
891 	struct dpmaif_rx_queue *rxq;  in t7xx_dpmaif_irq_rx_done()  local
901 	rxq = &dpmaif_ctrl->rxq[qno];  in t7xx_dpmaif_irq_rx_done()
902 	ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaif_irq_rx_done()
914 	napi_schedule(&rxq->napi);  in t7xx_dpmaif_irq_rx_done()
1006 static int t7xx_dpmaif_rx_alloc(struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_rx_alloc()  argument
1008 	rxq->pit_size_cnt = DPMAIF_PIT_COUNT;  in t7xx_dpmaif_rx_alloc()
1009 	rxq->pit_rd_idx = 0;  in t7xx_dpmaif_rx_alloc()
1010 	rxq->pit_wr_idx = 0;  in t7xx_dpmaif_rx_alloc()
1011 	rxq->pit_release_rd_idx = 0;  in t7xx_dpmaif_rx_alloc()
1012 	rxq->expect_pit_seq = 0;  in t7xx_dpmaif_rx_alloc()
1013 	rxq->pit_remain_release_cnt = 0;  in t7xx_dpmaif_rx_alloc()
1014 	memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info));  in t7xx_dpmaif_rx_alloc()
1016 	rxq->pit_base = dma_alloc_coherent(rxq->dpmaif_ctrl->dev,  in t7xx_dpmaif_rx_alloc()
1017 					   rxq->pit_size_cnt * sizeof(struct dpmaif_pit),  in t7xx_dpmaif_rx_alloc()
1018 					   &rxq->pit_bus_addr, GFP_KERNEL | __GFP_ZERO);  in t7xx_dpmaif_rx_alloc()
1019 	if (!rxq->pit_base)  in t7xx_dpmaif_rx_alloc()
1022 	rxq->bat_req = &rxq->dpmaif_ctrl->bat_req;  in t7xx_dpmaif_rx_alloc()
1023 	atomic_inc(&rxq->bat_req->refcnt);  in t7xx_dpmaif_rx_alloc()
1025 	rxq->bat_frag = &rxq->dpmaif_ctrl->bat_frag;  in t7xx_dpmaif_rx_alloc()
1026 	atomic_inc(&rxq->bat_frag->refcnt);  in t7xx_dpmaif_rx_alloc()
1030 static void t7xx_dpmaif_rx_buf_free(const struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_rx_buf_free()  argument
1032 	if (!rxq->dpmaif_ctrl)  in t7xx_dpmaif_rx_buf_free()
1035 	t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_req);  in t7xx_dpmaif_rx_buf_free()
1036 	t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_frag);  in t7xx_dpmaif_rx_buf_free()
1038 	if (rxq->pit_base)  in t7xx_dpmaif_rx_buf_free()
1039 		dma_free_coherent(rxq->dpmaif_ctrl->dev,  in t7xx_dpmaif_rx_buf_free()
1040 				  rxq->pit_size_cnt * sizeof(struct dpmaif_pit),  in t7xx_dpmaif_rx_buf_free()
1041 				  rxq->pit_base, rxq->pit_bus_addr);  in t7xx_dpmaif_rx_buf_free()
1063 	struct dpmaif_rx_queue *rxq;  in t7xx_dpmaif_bat_release_work()  local
1073 	rxq = &dpmaif_ctrl->rxq[DPF_RX_QNO_DFT];  in t7xx_dpmaif_bat_release_work()
1075 		t7xx_dpmaif_bat_release_and_add(rxq);  in t7xx_dpmaif_bat_release_work()
1076 		t7xx_dpmaif_frag_bat_release_and_add(rxq);  in t7xx_dpmaif_bat_release_work()
1116 		struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[i];  in t7xx_dpmaif_rx_stop()  local
1119 		timeout = readx_poll_timeout_atomic(atomic_read, &rxq->rx_processing, value,  in t7xx_dpmaif_rx_stop()
1126 		rxq->que_started = false;  in t7xx_dpmaif_rx_stop()
1130 static void t7xx_dpmaif_stop_rxq(struct dpmaif_rx_queue *rxq)  in t7xx_dpmaif_stop_rxq()  argument
1134 	rxq->que_started = false;  in t7xx_dpmaif_stop_rxq()
1137 		cnt = t7xx_ring_buf_rd_wr_count(rxq->pit_size_cnt, rxq->pit_rd_idx,  in t7xx_dpmaif_stop_rxq()
1138 						rxq->pit_wr_idx, DPMAIF_READ);  in t7xx_dpmaif_stop_rxq()
1141 			dev_err(rxq->dpmaif_ctrl->dev, "Stop RX SW failed, %d\n", cnt);  in t7xx_dpmaif_stop_rxq()
1146 	memset(rxq->pit_base, 0, rxq->pit_size_cnt * sizeof(struct dpmaif_pit));  in t7xx_dpmaif_stop_rxq()
1147 	memset(rxq->bat_req->bat_base, 0, rxq->bat_req->bat_size_cnt * sizeof(struct dpmaif_bat));  in t7xx_dpmaif_stop_rxq()
1148 	bitmap_zero(rxq->bat_req->bat_bitmap, rxq->bat_req->bat_size_cnt);  in t7xx_dpmaif_stop_rxq()
1149 	memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info));  in t7xx_dpmaif_stop_rxq()
1151 	rxq->pit_rd_idx = 0;  in t7xx_dpmaif_stop_rxq()
1152 	rxq->pit_wr_idx = 0;  in t7xx_dpmaif_stop_rxq()
1153 	rxq->pit_release_rd_idx = 0;  in t7xx_dpmaif_stop_rxq()
1154 	rxq->expect_pit_seq = 0;  in t7xx_dpmaif_stop_rxq()
1155 	rxq->pit_remain_release_cnt = 0;  in t7xx_dpmaif_stop_rxq()
1156 	rxq->bat_req->bat_release_rd_idx = 0;  in t7xx_dpmaif_stop_rxq()
1157 	rxq->bat_req->bat_wr_idx = 0;  in t7xx_dpmaif_stop_rxq()
1158 	rxq->bat_frag->bat_release_rd_idx = 0;  in t7xx_dpmaif_stop_rxq()
1159 	rxq->bat_frag->bat_wr_idx = 0;  in t7xx_dpmaif_stop_rxq()
1167 		t7xx_dpmaif_stop_rxq(&dpmaif_ctrl->rxq[i]);  in t7xx_dpmaif_rx_clear()