Lines Matching full:nic
16 #include "nic.h"
22 static void nicvf_get_page(struct nicvf *nic) in nicvf_get_page() argument
24 if (!nic->rb_pageref || !nic->rb_page) in nicvf_get_page()
27 page_ref_add(nic->rb_page, nic->rb_pageref); in nicvf_get_page()
28 nic->rb_pageref = 0; in nicvf_get_page()
32 static int nicvf_poll_reg(struct nicvf *nic, int qidx, in nicvf_poll_reg() argument
43 reg_val = nicvf_queue_reg_read(nic, reg, qidx); in nicvf_poll_reg()
49 netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg); in nicvf_poll_reg()
54 static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, in nicvf_alloc_q_desc_mem() argument
60 dmem->unalign_base = dma_alloc_coherent(&nic->pdev->dev, dmem->size, in nicvf_alloc_q_desc_mem()
72 static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) in nicvf_free_q_desc_mem() argument
77 dma_free_coherent(&nic->pdev->dev, dmem->size, in nicvf_free_q_desc_mem()
93 static inline struct pgcache *nicvf_alloc_page(struct nicvf *nic, in nicvf_alloc_page() argument
126 this_cpu_inc(nic->pnicvf->drv_stats->page_alloc); in nicvf_alloc_page()
131 nic->rb_page = page; in nicvf_alloc_page()
176 static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr, in nicvf_alloc_rcv_buffer() argument
184 if (!rbdr->is_xdp && nic->rb_page && in nicvf_alloc_rcv_buffer()
185 ((nic->rb_page_offset + buf_len) <= PAGE_SIZE)) { in nicvf_alloc_rcv_buffer()
186 nic->rb_pageref++; in nicvf_alloc_rcv_buffer()
190 nicvf_get_page(nic); in nicvf_alloc_rcv_buffer()
191 nic->rb_page = NULL; in nicvf_alloc_rcv_buffer()
194 pgcache = nicvf_alloc_page(nic, rbdr, gfp); in nicvf_alloc_rcv_buffer()
195 if (!pgcache && !nic->rb_page) { in nicvf_alloc_rcv_buffer()
196 this_cpu_inc(nic->pnicvf->drv_stats->rcv_buffer_alloc_failures); in nicvf_alloc_rcv_buffer()
200 nic->rb_page_offset = 0; in nicvf_alloc_rcv_buffer()
208 nic->rb_page = pgcache->page; in nicvf_alloc_rcv_buffer()
214 *rbuf = (u64)dma_map_page_attrs(&nic->pdev->dev, nic->rb_page, in nicvf_alloc_rcv_buffer()
215 nic->rb_page_offset, buf_len, in nicvf_alloc_rcv_buffer()
218 if (dma_mapping_error(&nic->pdev->dev, (dma_addr_t)*rbuf)) { in nicvf_alloc_rcv_buffer()
219 if (!nic->rb_page_offset) in nicvf_alloc_rcv_buffer()
220 __free_pages(nic->rb_page, 0); in nicvf_alloc_rcv_buffer()
221 nic->rb_page = NULL; in nicvf_alloc_rcv_buffer()
226 nic->rb_page_offset += buf_len; in nicvf_alloc_rcv_buffer()
233 static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic, in nicvf_rb_ptr_to_skb() argument
253 static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, in nicvf_init_rbdr() argument
261 err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len, in nicvf_init_rbdr()
284 if (!nic->pnicvf->xdp_prog) { in nicvf_init_rbdr()
299 nic->rb_page = NULL; in nicvf_init_rbdr()
301 err = nicvf_alloc_rcv_buffer(nic, rbdr, GFP_KERNEL, in nicvf_init_rbdr()
313 nicvf_get_page(nic); in nicvf_init_rbdr()
319 static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) in nicvf_free_rbdr() argument
340 phys_addr = nicvf_iova_to_phys(nic, buf_addr); in nicvf_free_rbdr()
341 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN, in nicvf_free_rbdr()
351 phys_addr = nicvf_iova_to_phys(nic, buf_addr); in nicvf_free_rbdr()
352 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN, in nicvf_free_rbdr()
375 nicvf_free_q_desc_mem(nic, &rbdr->dmem); in nicvf_free_rbdr()
380 static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp) in nicvf_refill_rbdr() argument
382 struct queue_set *qs = nic->qs; in nicvf_refill_rbdr()
401 qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx); in nicvf_refill_rbdr()
413 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; in nicvf_refill_rbdr()
418 if (nicvf_alloc_rcv_buffer(nic, rbdr, gfp, RCV_FRAG_LEN, &rbuf)) in nicvf_refill_rbdr()
427 nicvf_get_page(nic); in nicvf_refill_rbdr()
434 nic->rb_alloc_fail = true; in nicvf_refill_rbdr()
436 nic->rb_alloc_fail = false; in nicvf_refill_rbdr()
439 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, in nicvf_refill_rbdr()
443 if (!nic->rb_alloc_fail && rbdr->enable && in nicvf_refill_rbdr()
444 netif_running(nic->pnicvf->netdev)) in nicvf_refill_rbdr()
445 nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx); in nicvf_refill_rbdr()
454 struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work); in nicvf_rbdr_work() local
456 nicvf_refill_rbdr(nic, GFP_KERNEL); in nicvf_rbdr_work()
457 if (nic->rb_alloc_fail) in nicvf_rbdr_work()
458 schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); in nicvf_rbdr_work()
460 nic->rb_work_scheduled = false; in nicvf_rbdr_work()
466 struct nicvf *nic = from_tasklet(nic, t, rbdr_task); in nicvf_rbdr_task() local
468 nicvf_refill_rbdr(nic, GFP_ATOMIC); in nicvf_rbdr_task()
469 if (nic->rb_alloc_fail) { in nicvf_rbdr_task()
470 nic->rb_work_scheduled = true; in nicvf_rbdr_task()
471 schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); in nicvf_rbdr_task()
476 static int nicvf_init_cmp_queue(struct nicvf *nic, in nicvf_init_cmp_queue() argument
481 err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE, in nicvf_init_cmp_queue()
487 cq->thresh = pass1_silicon(nic->pdev) ? 0 : CMP_QUEUE_CQE_THRESH; in nicvf_init_cmp_queue()
488 nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1; in nicvf_init_cmp_queue()
493 static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) in nicvf_free_cmp_queue() argument
500 nicvf_free_q_desc_mem(nic, &cq->dmem); in nicvf_free_cmp_queue()
504 static int nicvf_init_snd_queue(struct nicvf *nic, in nicvf_init_snd_queue() argument
509 err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE, in nicvf_init_snd_queue()
524 if (nic->sqs_mode) in nicvf_init_snd_queue()
525 qidx += ((nic->sqs_id + 1) * MAX_SND_QUEUES_PER_QS); in nicvf_init_snd_queue()
526 if (qidx < nic->pnicvf->xdp_tx_queues) { in nicvf_init_snd_queue()
543 sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev, in nicvf_init_snd_queue()
554 void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq, in nicvf_unmap_sndq_buffers() argument
566 dma_unmap_page_attrs(&nic->pdev->dev, gather->addr, in nicvf_unmap_sndq_buffers()
572 static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) in nicvf_free_snd_queue() argument
585 dma_free_coherent(&nic->pdev->dev, in nicvf_free_snd_queue()
610 nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2, in nicvf_free_snd_queue()
613 nicvf_unmap_sndq_buffers(nic, sq, sq->head, in nicvf_free_snd_queue()
624 nicvf_free_q_desc_mem(nic, &sq->dmem); in nicvf_free_snd_queue()
627 static void nicvf_reclaim_snd_queue(struct nicvf *nic, in nicvf_reclaim_snd_queue() argument
631 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0); in nicvf_reclaim_snd_queue()
633 if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01)) in nicvf_reclaim_snd_queue()
636 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); in nicvf_reclaim_snd_queue()
639 static void nicvf_reclaim_rcv_queue(struct nicvf *nic, in nicvf_reclaim_rcv_queue() argument
646 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_reclaim_rcv_queue()
649 static void nicvf_reclaim_cmp_queue(struct nicvf *nic, in nicvf_reclaim_cmp_queue() argument
653 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0); in nicvf_reclaim_cmp_queue()
655 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0); in nicvf_reclaim_cmp_queue()
657 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); in nicvf_reclaim_cmp_queue()
660 static void nicvf_reclaim_rbdr(struct nicvf *nic, in nicvf_reclaim_rbdr() argument
667 rbdr->head = nicvf_queue_reg_read(nic, in nicvf_reclaim_rbdr()
670 rbdr->tail = nicvf_queue_reg_read(nic, in nicvf_reclaim_rbdr()
677 fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx); in nicvf_reclaim_rbdr()
679 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, in nicvf_reclaim_rbdr()
683 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0); in nicvf_reclaim_rbdr()
684 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) in nicvf_reclaim_rbdr()
687 tmp = nicvf_queue_reg_read(nic, in nicvf_reclaim_rbdr()
695 netdev_err(nic->netdev, in nicvf_reclaim_rbdr()
700 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, in nicvf_reclaim_rbdr()
703 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02)) in nicvf_reclaim_rbdr()
705 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00); in nicvf_reclaim_rbdr()
706 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) in nicvf_reclaim_rbdr()
710 void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features) in nicvf_config_vlan_stripping() argument
715 rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0); in nicvf_config_vlan_stripping()
722 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg); in nicvf_config_vlan_stripping()
725 for (sqs = 0; sqs < nic->sqs_count; sqs++) in nicvf_config_vlan_stripping()
726 if (nic->snicvf[sqs]) in nicvf_config_vlan_stripping()
727 nicvf_queue_reg_write(nic->snicvf[sqs], in nicvf_config_vlan_stripping()
731 static void nicvf_reset_rcv_queue_stats(struct nicvf *nic) in nicvf_reset_rcv_queue_stats() argument
741 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_reset_rcv_queue_stats()
745 static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, in nicvf_rcv_queue_config() argument
756 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); in nicvf_rcv_queue_config()
759 nicvf_reclaim_rcv_queue(nic, qs, qidx); in nicvf_rcv_queue_config()
774 WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx, 0) < 0); in nicvf_rcv_queue_config()
784 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_rcv_queue_config()
790 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_rcv_queue_config()
799 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_rcv_queue_config()
801 if (!nic->sqs_mode && (qidx == 0)) { in nicvf_rcv_queue_config()
805 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, in nicvf_rcv_queue_config()
807 nicvf_config_vlan_stripping(nic, nic->netdev->features); in nicvf_rcv_queue_config()
814 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg); in nicvf_rcv_queue_config()
818 void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, in nicvf_cmp_queue_config() argument
828 nicvf_reclaim_cmp_queue(nic, qs, qidx); in nicvf_cmp_queue_config()
833 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); in nicvf_cmp_queue_config()
840 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, in nicvf_cmp_queue_config()
850 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg); in nicvf_cmp_queue_config()
853 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh); in nicvf_cmp_queue_config()
854 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, in nicvf_cmp_queue_config()
859 static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, in nicvf_snd_queue_config() argument
870 nicvf_reclaim_snd_queue(nic, qs, qidx); in nicvf_snd_queue_config()
875 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); in nicvf_snd_queue_config()
884 mbx.sq.sqs_mode = nic->sqs_mode; in nicvf_snd_queue_config()
886 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_snd_queue_config()
889 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, in nicvf_snd_queue_config()
903 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg); in nicvf_snd_queue_config()
906 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh); in nicvf_snd_queue_config()
911 netif_set_xps_queue(nic->netdev, in nicvf_snd_queue_config()
917 static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, in nicvf_rbdr_config() argument
924 nicvf_reclaim_rbdr(nic, rbdr, qidx); in nicvf_rbdr_config()
929 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, in nicvf_rbdr_config()
941 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, in nicvf_rbdr_config()
945 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, in nicvf_rbdr_config()
949 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, in nicvf_rbdr_config()
954 void nicvf_qset_config(struct nicvf *nic, bool enable) in nicvf_qset_config() argument
957 struct queue_set *qs = nic->qs; in nicvf_qset_config()
961 netdev_warn(nic->netdev, in nicvf_qset_config()
967 qs->vnic_id = nic->vf_id; in nicvf_qset_config()
972 mbx.qs.sqs_count = nic->sqs_count; in nicvf_qset_config()
983 if (nic->ptp_clock) in nicvf_qset_config()
986 nicvf_send_msg_to_pf(nic, &mbx); in nicvf_qset_config()
989 static void nicvf_free_resources(struct nicvf *nic) in nicvf_free_resources() argument
992 struct queue_set *qs = nic->qs; in nicvf_free_resources()
996 nicvf_free_rbdr(nic, &qs->rbdr[qidx]); in nicvf_free_resources()
1000 nicvf_free_cmp_queue(nic, &qs->cq[qidx]); in nicvf_free_resources()
1004 nicvf_free_snd_queue(nic, &qs->sq[qidx]); in nicvf_free_resources()
1007 static int nicvf_alloc_resources(struct nicvf *nic) in nicvf_alloc_resources() argument
1010 struct queue_set *qs = nic->qs; in nicvf_alloc_resources()
1014 if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len, in nicvf_alloc_resources()
1021 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx)) in nicvf_alloc_resources()
1027 if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len)) in nicvf_alloc_resources()
1033 nicvf_free_resources(nic); in nicvf_alloc_resources()
1037 int nicvf_set_qset_resources(struct nicvf *nic) in nicvf_set_qset_resources() argument
1041 qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL); in nicvf_set_qset_resources()
1044 nic->qs = qs; in nicvf_set_qset_resources()
1057 nic->rx_queues = qs->rq_cnt; in nicvf_set_qset_resources()
1058 nic->tx_queues = qs->sq_cnt; in nicvf_set_qset_resources()
1059 nic->xdp_tx_queues = 0; in nicvf_set_qset_resources()
1064 int nicvf_config_data_transfer(struct nicvf *nic, bool enable) in nicvf_config_data_transfer() argument
1067 struct queue_set *qs = nic->qs; in nicvf_config_data_transfer()
1068 struct queue_set *pqs = nic->pnicvf->qs; in nicvf_config_data_transfer()
1078 if (nic->sqs_mode && pqs) { in nicvf_config_data_transfer()
1084 if (nicvf_alloc_resources(nic)) in nicvf_config_data_transfer()
1088 nicvf_snd_queue_config(nic, qs, qidx, enable); in nicvf_config_data_transfer()
1090 nicvf_cmp_queue_config(nic, qs, qidx, enable); in nicvf_config_data_transfer()
1092 nicvf_rbdr_config(nic, qs, qidx, enable); in nicvf_config_data_transfer()
1094 nicvf_rcv_queue_config(nic, qs, qidx, enable); in nicvf_config_data_transfer()
1097 nicvf_rcv_queue_config(nic, qs, qidx, disable); in nicvf_config_data_transfer()
1099 nicvf_rbdr_config(nic, qs, qidx, disable); in nicvf_config_data_transfer()
1101 nicvf_snd_queue_config(nic, qs, qidx, disable); in nicvf_config_data_transfer()
1103 nicvf_cmp_queue_config(nic, qs, qidx, disable); in nicvf_config_data_transfer()
1105 nicvf_free_resources(nic); in nicvf_config_data_transfer()
1111 nicvf_reset_rcv_queue_stats(nic); in nicvf_config_data_transfer()
1160 void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) in nicvf_sq_enable() argument
1164 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); in nicvf_sq_enable()
1166 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); in nicvf_sq_enable()
1168 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0); in nicvf_sq_enable()
1171 void nicvf_sq_disable(struct nicvf *nic, int qidx) in nicvf_sq_disable() argument
1175 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); in nicvf_sq_disable()
1177 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); in nicvf_sq_disable()
1185 struct nicvf *nic = netdev_priv(netdev); in nicvf_sq_free_used_descs() local
1188 head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4; in nicvf_sq_free_used_descs()
1206 void nicvf_xdp_sq_doorbell(struct nicvf *nic, in nicvf_xdp_sq_doorbell() argument
1216 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, in nicvf_xdp_sq_doorbell()
1236 int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq, in nicvf_xdp_sq_append_pkt() argument
1307 static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb) in nicvf_sq_subdesc_required() argument
1311 if (skb_shinfo(skb)->gso_size && !nic->hw_tso) { in nicvf_sq_subdesc_required()
1317 if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) in nicvf_sq_subdesc_required()
1330 nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry, in nicvf_sq_add_hdr_subdesc() argument
1346 if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) { in nicvf_sq_add_hdr_subdesc()
1383 if (nic->hw_tso && skb_shinfo(skb)->gso_size) { in nicvf_sq_add_hdr_subdesc()
1389 this_cpu_inc(nic->pnicvf->drv_stats->tx_tso); in nicvf_sq_add_hdr_subdesc()
1403 if (!atomic_add_unless(&nic->pnicvf->tx_ptp_skbs, 1, 1)) in nicvf_sq_add_hdr_subdesc()
1465 static inline void nicvf_sq_doorbell(struct nicvf *nic, struct sk_buff *skb, in nicvf_sq_doorbell() argument
1470 txq = netdev_get_tx_queue(nic->pnicvf->netdev, in nicvf_sq_doorbell()
1479 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, in nicvf_sq_doorbell()
1486 static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, in nicvf_sq_append_tso() argument
1533 nicvf_sq_add_hdr_subdesc(nic, sq, hdr_qentry, in nicvf_sq_append_tso()
1543 nicvf_sq_doorbell(nic, skb, sq_num, desc_cnt); in nicvf_sq_append_tso()
1545 this_cpu_inc(nic->pnicvf->drv_stats->tx_tso); in nicvf_sq_append_tso()
1550 int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq, in nicvf_sq_append_skb() argument
1558 subdesc_cnt = nicvf_sq_subdesc_required(nic, skb); in nicvf_sq_append_skb()
1565 if (skb_shinfo(skb)->gso_size && !nic->hw_tso) in nicvf_sq_append_skb()
1566 return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb); in nicvf_sq_append_skb()
1569 nicvf_sq_add_hdr_subdesc(nic, sq, qentry, subdesc_cnt - 1, in nicvf_sq_append_skb()
1577 dma_addr = dma_map_page_attrs(&nic->pdev->dev, virt_to_page(skb->data), in nicvf_sq_append_skb()
1580 if (dma_mapping_error(&nic->pdev->dev, dma_addr)) { in nicvf_sq_append_skb()
1596 dma_addr = dma_map_page_attrs(&nic->pdev->dev, in nicvf_sq_append_skb()
1601 if (dma_mapping_error(&nic->pdev->dev, dma_addr)) { in nicvf_sq_append_skb()
1605 nicvf_unmap_sndq_buffers(nic, sq, hdr_sqe, i); in nicvf_sq_append_skb()
1613 if (nic->t88 && skb_shinfo(skb)->gso_size) { in nicvf_sq_append_skb()
1618 nicvf_sq_doorbell(nic, skb, sq_num, subdesc_cnt); in nicvf_sq_append_skb()
1624 nic = nic->pnicvf; in nicvf_sq_append_skb()
1625 netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n"); in nicvf_sq_append_skb()
1638 static void nicvf_unmap_rcv_buffer(struct nicvf *nic, u64 dma_addr, in nicvf_unmap_rcv_buffer() argument
1658 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, len, in nicvf_unmap_rcv_buffer()
1663 struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, in nicvf_get_rcv_skb() argument
1683 if (!nic->hw_tso) in nicvf_get_rcv_skb()
1690 phys_addr = nicvf_iova_to_phys(nic, *rb_ptrs); in nicvf_get_rcv_skb()
1699 nicvf_unmap_rcv_buffer(nic, in nicvf_get_rcv_skb()
1702 skb = nicvf_rb_ptr_to_skb(nic, in nicvf_get_rcv_skb()
1711 nicvf_unmap_rcv_buffer(nic, *rb_ptrs, phys_addr, xdp); in nicvf_get_rcv_skb()
1757 void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) in nicvf_enable_intr() argument
1762 netdev_dbg(nic->netdev, in nicvf_enable_intr()
1766 nicvf_reg_write(nic, NIC_VF_ENA_W1S, in nicvf_enable_intr()
1767 nicvf_reg_read(nic, NIC_VF_ENA_W1S) | mask); in nicvf_enable_intr()
1771 void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) in nicvf_disable_intr() argument
1776 netdev_dbg(nic->netdev, in nicvf_disable_intr()
1781 nicvf_reg_write(nic, NIC_VF_ENA_W1C, mask); in nicvf_disable_intr()
1785 void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) in nicvf_clear_intr() argument
1790 netdev_dbg(nic->netdev, in nicvf_clear_intr()
1795 nicvf_reg_write(nic, NIC_VF_INT, mask); in nicvf_clear_intr()
1799 int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) in nicvf_is_intr_enabled() argument
1804 netdev_dbg(nic->netdev, in nicvf_is_intr_enabled()
1809 return mask & nicvf_reg_read(nic, NIC_VF_ENA_W1S); in nicvf_is_intr_enabled()
1812 void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) in nicvf_update_rq_stats() argument
1817 nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\ in nicvf_update_rq_stats()
1820 rq = &nic->qs->rq[rq_idx]; in nicvf_update_rq_stats()
1825 void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) in nicvf_update_sq_stats() argument
1830 nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\ in nicvf_update_sq_stats()
1833 sq = &nic->qs->sq[sq_idx]; in nicvf_update_sq_stats()
1839 int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx) in nicvf_check_cqe_rx_errs() argument
1841 netif_err(nic, rx_err, nic->netdev, in nicvf_check_cqe_rx_errs()
1847 this_cpu_inc(nic->drv_stats->rx_bgx_truncated_pkts); in nicvf_check_cqe_rx_errs()
1850 this_cpu_inc(nic->drv_stats->rx_jabber_errs); in nicvf_check_cqe_rx_errs()
1853 this_cpu_inc(nic->drv_stats->rx_fcs_errs); in nicvf_check_cqe_rx_errs()
1856 this_cpu_inc(nic->drv_stats->rx_bgx_errs); in nicvf_check_cqe_rx_errs()
1859 this_cpu_inc(nic->drv_stats->rx_prel2_errs); in nicvf_check_cqe_rx_errs()
1862 this_cpu_inc(nic->drv_stats->rx_l2_hdr_malformed); in nicvf_check_cqe_rx_errs()
1865 this_cpu_inc(nic->drv_stats->rx_oversize); in nicvf_check_cqe_rx_errs()
1868 this_cpu_inc(nic->drv_stats->rx_undersize); in nicvf_check_cqe_rx_errs()
1871 this_cpu_inc(nic->drv_stats->rx_l2_len_mismatch); in nicvf_check_cqe_rx_errs()
1874 this_cpu_inc(nic->drv_stats->rx_l2_pclp); in nicvf_check_cqe_rx_errs()
1877 this_cpu_inc(nic->drv_stats->rx_ip_ver_errs); in nicvf_check_cqe_rx_errs()
1880 this_cpu_inc(nic->drv_stats->rx_ip_csum_errs); in nicvf_check_cqe_rx_errs()
1883 this_cpu_inc(nic->drv_stats->rx_ip_hdr_malformed); in nicvf_check_cqe_rx_errs()
1886 this_cpu_inc(nic->drv_stats->rx_ip_payload_malformed); in nicvf_check_cqe_rx_errs()
1889 this_cpu_inc(nic->drv_stats->rx_ip_ttl_errs); in nicvf_check_cqe_rx_errs()
1892 this_cpu_inc(nic->drv_stats->rx_l3_pclp); in nicvf_check_cqe_rx_errs()
1895 this_cpu_inc(nic->drv_stats->rx_l4_malformed); in nicvf_check_cqe_rx_errs()
1898 this_cpu_inc(nic->drv_stats->rx_l4_csum_errs); in nicvf_check_cqe_rx_errs()
1901 this_cpu_inc(nic->drv_stats->rx_udp_len_errs); in nicvf_check_cqe_rx_errs()
1904 this_cpu_inc(nic->drv_stats->rx_l4_port_errs); in nicvf_check_cqe_rx_errs()
1907 this_cpu_inc(nic->drv_stats->rx_tcp_flag_errs); in nicvf_check_cqe_rx_errs()
1910 this_cpu_inc(nic->drv_stats->rx_tcp_offset_errs); in nicvf_check_cqe_rx_errs()
1913 this_cpu_inc(nic->drv_stats->rx_l4_pclp); in nicvf_check_cqe_rx_errs()
1916 this_cpu_inc(nic->drv_stats->rx_truncated_pkts); in nicvf_check_cqe_rx_errs()
1924 int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx) in nicvf_check_cqe_tx_errs() argument
1928 this_cpu_inc(nic->drv_stats->tx_desc_fault); in nicvf_check_cqe_tx_errs()
1931 this_cpu_inc(nic->drv_stats->tx_hdr_cons_err); in nicvf_check_cqe_tx_errs()
1934 this_cpu_inc(nic->drv_stats->tx_subdesc_err); in nicvf_check_cqe_tx_errs()
1937 this_cpu_inc(nic->drv_stats->tx_max_size_exceeded); in nicvf_check_cqe_tx_errs()
1940 this_cpu_inc(nic->drv_stats->tx_imm_size_oflow); in nicvf_check_cqe_tx_errs()
1943 this_cpu_inc(nic->drv_stats->tx_data_seq_err); in nicvf_check_cqe_tx_errs()
1946 this_cpu_inc(nic->drv_stats->tx_mem_seq_err); in nicvf_check_cqe_tx_errs()
1949 this_cpu_inc(nic->drv_stats->tx_lock_viol); in nicvf_check_cqe_tx_errs()
1952 this_cpu_inc(nic->drv_stats->tx_data_fault); in nicvf_check_cqe_tx_errs()
1955 this_cpu_inc(nic->drv_stats->tx_tstmp_conflict); in nicvf_check_cqe_tx_errs()
1958 this_cpu_inc(nic->drv_stats->tx_tstmp_timeout); in nicvf_check_cqe_tx_errs()
1961 this_cpu_inc(nic->drv_stats->tx_mem_fault); in nicvf_check_cqe_tx_errs()
1964 this_cpu_inc(nic->drv_stats->tx_csum_overlap); in nicvf_check_cqe_tx_errs()
1967 this_cpu_inc(nic->drv_stats->tx_csum_overflow); in nicvf_check_cqe_tx_errs()