53 static void hinic_rxq_clean_stats(struct hinic_rxq *rxq)  in hinic_rxq_clean_stats()  argument
55 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_clean_stats()
71 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)  in hinic_rxq_get_stats()  argument
73 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_get_stats()
91 static void rxq_stats_init(struct hinic_rxq *rxq)  in rxq_stats_init()  argument
93 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in rxq_stats_init()
96 	hinic_rxq_clean_stats(rxq);  in rxq_stats_init()
99 static void rx_csum(struct hinic_rxq *rxq, u32 status,  in rx_csum()  argument
102 	struct net_device *netdev = rxq->netdev;  in rx_csum()
115 			rxq->rxq_stats.csum_errors++;  in rx_csum()
127 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,  in rx_alloc_skb()  argument
130 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_skb()
138 	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);  in rx_alloc_skb()
142 	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,  in rx_alloc_skb()
163 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)  in rx_unmap_skb()  argument
165 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_unmap_skb()
170 	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,  in rx_unmap_skb()
180 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,  in rx_free_skb()  argument
183 	rx_unmap_skb(rxq, dma_addr);  in rx_free_skb()
193 static int rx_alloc_pkts(struct hinic_rxq *rxq)  in rx_alloc_pkts()  argument
195 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_pkts()
204 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rx_alloc_pkts()
211 		skb = rx_alloc_skb(rxq, &dma_addr);  in rx_alloc_pkts()
217 		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_alloc_pkts()
220 			rx_free_skb(rxq, skb, dma_addr);  in rx_alloc_pkts()
224 		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);  in rx_alloc_pkts()
226 		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);  in rx_alloc_pkts()
233 		hinic_rq_update(rxq->rq, prod_idx);  in rx_alloc_pkts()
243 static void free_all_rx_skbs(struct hinic_rxq *rxq)  in free_all_rx_skbs()  argument
245 	struct hinic_rq *rq = rxq->rq;  in free_all_rx_skbs()
258 		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));  in free_all_rx_skbs()
271 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,  in rx_recv_jumbo_pkt()  argument
281 		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_recv_jumbo_pkt()
286 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rx_recv_jumbo_pkt()
288 		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));  in rx_recv_jumbo_pkt()
354 static int rxq_recv(struct hinic_rxq *rxq, int budget)  in rxq_recv()  argument
356 	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);  in rxq_recv()
357 	struct net_device *netdev = rxq->netdev;  in rxq_recv()
359 	struct hinic_rq *rq = rxq->rq;  in rxq_recv()
379 		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,  in rxq_recv()
389 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rxq_recv()
391 		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));  in rxq_recv()
393 		rx_csum(rxq, status, skb);  in rxq_recv()
403 			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -  in rxq_recv()
422 		skb->protocol = eth_type_trans(skb, rxq->netdev);  in rxq_recv()
424 		napi_gro_receive(&rxq->napi, skb);  in rxq_recv()
435 			(u16)(pkt_len >> rxq->rx_buff_shift) +  in rxq_recv()
436 			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);  in rxq_recv()
445 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rxq_recv()
447 		rx_alloc_pkts(rxq);  in rxq_recv()
449 	u64_stats_update_begin(&rxq->rxq_stats.syncp);  in rxq_recv()
450 	rxq->rxq_stats.pkts += pkts;  in rxq_recv()
451 	rxq->rxq_stats.bytes += rx_bytes;  in rxq_recv()
452 	u64_stats_update_end(&rxq->rxq_stats.syncp);  in rxq_recv()
459 	struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);  in rx_poll()  local
460 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_poll()
461 	struct hinic_rq *rq = rxq->rq;  in rx_poll()
464 	pkts = rxq_recv(rxq, budget);  in rx_poll()
478 static void rx_add_napi(struct hinic_rxq *rxq)  in rx_add_napi()  argument
480 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_add_napi()
482 	netif_napi_add_weight(rxq->netdev, &rxq->napi, rx_poll,  in rx_add_napi()
484 	napi_enable(&rxq->napi);  in rx_add_napi()
487 static void rx_del_napi(struct hinic_rxq *rxq)  in rx_del_napi()  argument
489 	napi_disable(&rxq->napi);  in rx_del_napi()
490 	netif_napi_del(&rxq->napi);  in rx_del_napi()
495 	struct hinic_rxq *rxq = (struct hinic_rxq *)data;  in rx_irq()  local
496 	struct hinic_rq *rq = rxq->rq;  in rx_irq()
500 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
506 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
509 	napi_schedule(&rxq->napi);  in rx_irq()
513 static int rx_request_irq(struct hinic_rxq *rxq)  in rx_request_irq()  argument
515 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_request_irq()
519 	struct hinic_rq *rq = rxq->rq;  in rx_request_irq()
525 	rx_add_napi(rxq);  in rx_request_irq()
540 		netif_err(nic_dev, drv, rxq->netdev,  in rx_request_irq()
545 	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);  in rx_request_irq()
557 	free_irq(rq->irq, rxq);  in rx_request_irq()
559 	rx_del_napi(rxq);  in rx_request_irq()
563 static void rx_free_irq(struct hinic_rxq *rxq)  in rx_free_irq()  argument
565 	struct hinic_rq *rq = rxq->rq;  in rx_free_irq()
568 	free_irq(rq->irq, rxq);  in rx_free_irq()
569 	rx_del_napi(rxq);  in rx_free_irq()
580 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,  in hinic_init_rxq()  argument
586 	rxq->netdev = netdev;  in hinic_init_rxq()
587 	rxq->rq = rq;  in hinic_init_rxq()
588 	rxq->buf_len = HINIC_RX_BUF_SZ;  in hinic_init_rxq()
589 	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);  in hinic_init_rxq()
591 	rxq_stats_init(rxq);  in hinic_init_rxq()
593 	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,  in hinic_init_rxq()
595 	if (!rxq->irq_name)  in hinic_init_rxq()
598 	pkts = rx_alloc_pkts(rxq);  in hinic_init_rxq()
604 	err = rx_request_irq(rxq);  in hinic_init_rxq()
614 	free_all_rx_skbs(rxq);  in hinic_init_rxq()
615 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_init_rxq()
623 void hinic_clean_rxq(struct hinic_rxq *rxq)  in hinic_clean_rxq()  argument
625 	struct net_device *netdev = rxq->netdev;  in hinic_clean_rxq()
627 	rx_free_irq(rxq);  in hinic_clean_rxq()
629 	free_all_rx_skbs(rxq);  in hinic_clean_rxq()
630 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_clean_rxq()