Lines Matching full:vi
512 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
520 static void free_unused_bufs(struct virtnet_info *vi);
521 static void virtnet_del_vqs(struct virtnet_info *vi);
530 static size_t virtnet_rss_hdr_size(const struct virtnet_info *vi) in virtnet_rss_hdr_size() argument
532 u16 indir_table_size = vi->has_rss ? vi->rss_indir_table_size : 1; in virtnet_rss_hdr_size()
534 return struct_size(vi->rss_hdr, indirection_table, indir_table_size); in virtnet_rss_hdr_size()
537 static size_t virtnet_rss_trailer_size(const struct virtnet_info *vi) in virtnet_rss_trailer_size() argument
539 return struct_size(&vi->rss_trailer, hash_key_data, vi->rss_key_size); in virtnet_rss_trailer_size()
658 static int vq_type(struct virtnet_info *vi, int qid) in vq_type() argument
660 if (qid == vi->max_queue_pairs * 2) in vq_type()
683 /* Find end of list, sew whole thing into vi->rq.pages. */ in give_pages()
702 static void virtnet_rq_free_buf(struct virtnet_info *vi, in virtnet_rq_free_buf() argument
711 static void enable_rx_mode_work(struct virtnet_info *vi) in enable_rx_mode_work() argument
714 vi->rx_mode_work_enabled = true; in enable_rx_mode_work()
718 static void disable_rx_mode_work(struct virtnet_info *vi) in disable_rx_mode_work() argument
721 vi->rx_mode_work_enabled = false; in disable_rx_mode_work()
752 static void virtnet_tx_wake_queue(struct virtnet_info *vi, in virtnet_tx_wake_queue() argument
756 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); in virtnet_tx_wake_queue()
768 struct virtnet_info *vi = vq->vdev->priv; in skb_xmit_done() local
770 struct send_queue *sq = &vi->sq[index]; in skb_xmit_done()
779 virtnet_tx_wake_queue(vi, sq); in skb_xmit_done()
836 static struct sk_buff *page_to_skb(struct virtnet_info *vi, in page_to_skb() argument
852 hdr_len = vi->hdr_len; in page_to_skb()
853 if (vi->mergeable_rx_bufs) in page_to_skb()
900 if (vi->mergeable_rx_bufs) { in page_to_skb()
939 struct virtnet_info *vi = vq->vdev->priv; in virtnet_rq_unmap_free_buf() local
943 rq = &vi->rq[i]; in virtnet_rq_unmap_free_buf()
950 virtnet_rq_free_buf(vi, rq, buf); in virtnet_rq_unmap_free_buf()
972 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) in is_xdp_raw_buffer_queue() argument
974 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) in is_xdp_raw_buffer_queue()
976 else if (q < vi->curr_queue_pairs) in is_xdp_raw_buffer_queue()
982 static bool tx_may_stop(struct virtnet_info *vi, in tx_may_stop() argument
988 qnum = sq - vi->sq; in tx_may_stop()
1014 static void check_sq_full_and_disable(struct virtnet_info *vi, in check_sq_full_and_disable() argument
1021 qnum = sq - vi->sq; in check_sq_full_and_disable()
1023 if (tx_may_stop(vi, dev, sq)) { in check_sq_full_and_disable()
1044 static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi, in buf_to_xdp() argument
1055 * hard_start + XDP_PACKET_HEADROOM - vi->hdr_len in buf_to_xdp()
1061 * xsk_pool_get_rx_frame_size() + vi->hdr_len in buf_to_xdp()
1065 bufsize += vi->hdr_len; in buf_to_xdp()
1069 vi->dev->name, len, bufsize); in buf_to_xdp()
1070 DEV_STATS_INC(vi->dev, rx_length_errors); in buf_to_xdp()
1079 XDP_PACKET_HEADROOM - vi->hdr_len, len, 1); in buf_to_xdp()
1117 static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi, in virtnet_receive_xsk_small() argument
1169 static int xsk_append_merge_buffer(struct virtnet_info *vi, in xsk_append_merge_buffer() argument
1188 vi->dev->name, num_buf, in xsk_append_merge_buffer()
1189 virtio16_to_cpu(vi->vdev, in xsk_append_merge_buffer()
1191 DEV_STATS_INC(vi->dev, rx_length_errors); in xsk_append_merge_buffer()
1197 xdp = buf_to_xdp(vi, rq, buf, len, false); in xsk_append_merge_buffer()
1226 xsk_drop_follow_bufs(vi->dev, rq, num_buf, stats); in xsk_append_merge_buffer()
1230 static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi, in virtnet_receive_xsk_merge() argument
1240 hdr = xdp->data - vi->hdr_len; in virtnet_receive_xsk_merge()
1241 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); in virtnet_receive_xsk_merge()
1262 if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) { in virtnet_receive_xsk_merge()
1286 static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq, in virtnet_receive_xsk_buf() argument
1291 struct net_device *dev = vi->dev; in virtnet_receive_xsk_buf()
1296 len -= vi->hdr_len; in virtnet_receive_xsk_buf()
1300 xdp = buf_to_xdp(vi, rq, buf, len, true); in virtnet_receive_xsk_buf()
1311 flags = ((struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags; in virtnet_receive_xsk_buf()
1313 if (!vi->mergeable_rx_bufs) in virtnet_receive_xsk_buf()
1314 skb = virtnet_receive_xsk_small(dev, vi, rq, xdp, xdp_xmit, stats); in virtnet_receive_xsk_buf()
1316 skb = virtnet_receive_xsk_merge(dev, vi, rq, xdp, xdp_xmit, stats); in virtnet_receive_xsk_buf()
1319 virtnet_receive_done(vi, rq, skb, flags); in virtnet_receive_xsk_buf()
1322 static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq, in virtnet_add_recvbuf_xsk() argument
1348 len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len; in virtnet_add_recvbuf_xsk()
1355 addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len; in virtnet_add_recvbuf_xsk()
1388 struct virtnet_info *vi; in virtnet_xsk_xmit_one() local
1391 vi = sq->vq->vdev->priv; in virtnet_xsk_xmit_one()
1397 sg_fill_dma(sq->sg, sq->xsk_hdr_dma_addr, vi->hdr_len); in virtnet_xsk_xmit_one()
1440 struct virtnet_info *vi = sq->vq->vdev->priv; in virtnet_xsk_xmit() local
1442 struct net_device *dev = vi->dev; in virtnet_xsk_xmit()
1449 __free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), true, &stats); in virtnet_xsk_xmit()
1456 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) in virtnet_xsk_xmit()
1457 check_sq_full_and_disable(vi, vi->dev, sq); in virtnet_xsk_xmit()
1462 txq = netdev_get_tx_queue(vi->dev, sq - vi->sq); in virtnet_xsk_xmit()
1491 struct virtnet_info *vi = netdev_priv(dev); in virtnet_xsk_wakeup() local
1496 if (qid >= vi->curr_queue_pairs) in virtnet_xsk_wakeup()
1500 struct send_queue *sq = &vi->sq[qid]; in virtnet_xsk_wakeup()
1506 struct receive_queue *rq = &vi->rq[qid]; in virtnet_xsk_wakeup()
1525 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, in __virtnet_xdp_xmit_one() argument
1534 if (unlikely(xdpf->headroom < vi->hdr_len)) in __virtnet_xdp_xmit_one()
1549 xdpf->headroom -= vi->hdr_len; in __virtnet_xdp_xmit_one()
1550 xdpf->data -= vi->hdr_len; in __virtnet_xdp_xmit_one()
1553 memset(hdr, 0, vi->hdr_len); in __virtnet_xdp_xmit_one()
1554 xdpf->len += vi->hdr_len; in __virtnet_xdp_xmit_one()
1572 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
1580 #define virtnet_xdp_get_sq(vi) ({ \ argument
1583 typeof(vi) v = (vi); \
1599 #define virtnet_xdp_put_sq(vi, q) { \ argument
1601 typeof(vi) v = (vi); \
1613 struct virtnet_info *vi = netdev_priv(dev); in virtnet_xdp_xmit() local
1615 struct receive_queue *rq = vi->rq; in virtnet_xdp_xmit()
1630 sq = virtnet_xdp_get_sq(vi); in virtnet_xdp_xmit()
1638 virtnet_free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), in virtnet_xdp_xmit()
1644 if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) in virtnet_xdp_xmit()
1650 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) in virtnet_xdp_xmit()
1651 check_sq_full_and_disable(vi, dev, sq); in virtnet_xdp_xmit()
1666 virtnet_xdp_put_sq(vi, sq); in virtnet_xdp_xmit()
1739 static unsigned int virtnet_get_headroom(struct virtnet_info *vi) in virtnet_get_headroom() argument
1741 return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0; in virtnet_get_headroom()
1826 static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi, in receive_small_build_skb() argument
1837 headroom = vi->hdr_len + header_offset; in receive_small_build_skb()
1844 memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len); in receive_small_build_skb()
1850 struct virtnet_info *vi, in receive_small_xdp() argument
1861 unsigned int headroom = vi->hdr_len + header_offset; in receive_small_xdp()
1877 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { in receive_small_xdp()
1879 unsigned int tlen = len + vi->hdr_len; in receive_small_xdp()
1882 xdp_headroom = virtnet_get_headroom(vi); in receive_small_xdp()
1884 headroom = vi->hdr_len + header_offset; in receive_small_xdp()
1899 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, in receive_small_xdp()
1940 struct virtnet_info *vi, in receive_small() argument
1957 len -= vi->hdr_len; in receive_small()
1967 if (unlikely(vi->xdp_enabled)) { in receive_small()
1973 skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf, in receive_small()
1982 skb = receive_small_build_skb(vi, xdp_headroom, buf, len, buflen); in receive_small()
1995 struct virtnet_info *vi, in receive_big() argument
2007 if (unlikely(len > (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE)) { in receive_big()
2010 (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE); in receive_big()
2014 skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); in receive_big()
2015 u64_stats_add(&stats->bytes, len - vi->hdr_len); in receive_big()
2057 struct virtnet_info *vi, in build_skb_from_xdp_buff() argument
2099 struct virtnet_info *vi, in virtnet_build_xdp_buff_mrg() argument
2120 XDP_PACKET_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); in virtnet_build_xdp_buff_mrg()
2146 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); in virtnet_build_xdp_buff_mrg()
2183 static void *mergeable_xdp_get_buf(struct virtnet_info *vi, in mergeable_xdp_get_buf() argument
2217 if (likely(headroom >= virtnet_get_headroom(vi) && in mergeable_xdp_get_buf()
2232 xdp_page = xdp_linearize_page(vi->dev, rq, num_buf, in mergeable_xdp_get_buf()
2262 struct virtnet_info *vi, in receive_mergeable_xdp() argument
2272 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); in receive_mergeable_xdp()
2283 data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page, in receive_mergeable_xdp()
2288 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz, in receive_mergeable_xdp()
2297 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz); in receive_mergeable_xdp()
2374 struct virtnet_info *vi, in receive_mergeable() argument
2383 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); in receive_mergeable()
2392 u64_stats_add(&stats->bytes, len - vi->hdr_len); in receive_mergeable()
2397 if (unlikely(vi->xdp_enabled)) { in receive_mergeable()
2403 head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx, in receive_mergeable()
2411 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom); in receive_mergeable()
2423 virtio16_to_cpu(vi->vdev, in receive_mergeable()
2497 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq, in virtnet_receive_done() argument
2501 struct net_device *dev = vi->dev; in virtnet_receive_done()
2504 if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) in virtnet_receive_done()
2508 if (virtio_net_handle_csum_offload(skb, &hdr->hdr, vi->rx_tnl_csum)) { in virtnet_receive_done()
2511 hdr->hdr.gso_type, vi->rx_tnl_csum); in virtnet_receive_done()
2515 if (virtio_net_hdr_tnl_to_skb(skb, &hdr->tnl_hdr, vi->rx_tnl, in virtnet_receive_done()
2516 vi->rx_tnl_csum, in virtnet_receive_done()
2517 virtio_is_little_endian(vi->vdev))) { in virtnet_receive_done()
2521 vi->rx_tnl, vi->rx_tnl_csum); in virtnet_receive_done()
2538 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, in receive_buf() argument
2543 struct net_device *dev = vi->dev; in receive_buf()
2547 if (unlikely(len < vi->hdr_len + ETH_HLEN)) { in receive_buf()
2550 virtnet_rq_free_buf(vi, rq, buf); in receive_buf()
2573 if (vi->mergeable_rx_bufs) { in receive_buf()
2575 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, in receive_buf()
2577 } else if (vi->big_packets) { in receive_buf()
2581 skb = receive_big(dev, vi, rq, buf, len, stats); in receive_buf()
2584 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); in receive_buf()
2590 virtnet_receive_done(vi, rq, skb, flags); in receive_buf()
2616 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, in add_recvbuf_small() argument
2619 unsigned int xdp_headroom = virtnet_get_headroom(vi); in add_recvbuf_small()
2620 unsigned int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; in add_recvbuf_small()
2637 err = virtnet_rq_submit(rq, buf, vi->hdr_len + GOOD_PACKET_LEN, ctx, gfp); in add_recvbuf_small()
2644 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, in add_recvbuf_big() argument
2651 sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); in add_recvbuf_big()
2653 /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ in add_recvbuf_big()
2654 for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { in add_recvbuf_big()
2677 sg_set_buf(&rq->sg[0], p, vi->hdr_len); in add_recvbuf_big()
2685 err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, in add_recvbuf_big()
2697 struct virtnet_info *vi = rq->vq->vdev->priv; in get_mergeable_buf_len() local
2698 const size_t hdr_len = vi->hdr_len; in get_mergeable_buf_len()
2710 static int add_recvbuf_mergeable(struct virtnet_info *vi, in add_recvbuf_mergeable() argument
2713 unsigned int headroom = virtnet_get_headroom(vi); in add_recvbuf_mergeable()
2755 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, in try_fill_recv() argument
2761 err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk_pool, gfp); in try_fill_recv()
2766 if (vi->mergeable_rx_bufs) in try_fill_recv()
2767 err = add_recvbuf_mergeable(vi, rq, gfp); in try_fill_recv()
2768 else if (vi->big_packets) in try_fill_recv()
2769 err = add_recvbuf_big(vi, rq, gfp); in try_fill_recv()
2771 err = add_recvbuf_small(vi, rq, gfp); in try_fill_recv()
2791 struct virtnet_info *vi = rvq->vdev->priv; in skb_recv_done() local
2792 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; in skb_recv_done()
2814 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_napi_enable() local
2818 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, &rq->napi); in virtnet_napi_enable()
2823 struct virtnet_info *vi = sq->vq->vdev->priv; in virtnet_napi_tx_enable() local
2833 if (!vi->affinity_hint_set) { in virtnet_napi_tx_enable()
2839 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, napi); in virtnet_napi_tx_enable()
2844 struct virtnet_info *vi = sq->vq->vdev->priv; in virtnet_napi_tx_disable() local
2849 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, NULL); in virtnet_napi_tx_disable()
2856 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_napi_disable() local
2860 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, NULL); in virtnet_napi_disable()
2864 static int virtnet_receive_xsk_bufs(struct virtnet_info *vi, in virtnet_receive_xsk_bufs() argument
2879 virtnet_receive_xsk_buf(vi, rq, buf, len, xdp_xmit, stats); in virtnet_receive_xsk_bufs()
2886 static int virtnet_receive_packets(struct virtnet_info *vi, in virtnet_receive_packets() argument
2900 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, stats); in virtnet_receive_packets()
2906 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, stats); in virtnet_receive_packets()
2917 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_receive() local
2922 packets = virtnet_receive_xsk_bufs(vi, rq, budget, xdp_xmit, &stats); in virtnet_receive()
2924 packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats); in virtnet_receive()
2928 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) in virtnet_receive()
2956 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_poll_cleantx() local
2958 struct send_queue *sq = &vi->sq[index]; in virtnet_poll_cleantx()
2959 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); in virtnet_poll_cleantx()
2961 if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) in virtnet_poll_cleantx()
2976 virtnet_tx_wake_queue(vi, sq); in virtnet_poll_cleantx()
2982 static void virtnet_rx_dim_update(struct virtnet_info *vi, struct receive_queue *rq) in virtnet_rx_dim_update() argument
3005 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_poll() local
3027 virtnet_rx_dim_update(vi, rq); in virtnet_poll()
3031 sq = virtnet_xdp_get_sq(vi); in virtnet_poll()
3037 virtnet_xdp_put_sq(vi, sq); in virtnet_poll()
3043 static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index) in virtnet_disable_queue_pair() argument
3045 virtnet_napi_tx_disable(&vi->sq[qp_index]); in virtnet_disable_queue_pair()
3046 virtnet_napi_disable(&vi->rq[qp_index]); in virtnet_disable_queue_pair()
3047 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); in virtnet_disable_queue_pair()
3050 static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index) in virtnet_enable_queue_pair() argument
3052 struct net_device *dev = vi->dev; in virtnet_enable_queue_pair()
3055 err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index, in virtnet_enable_queue_pair()
3056 vi->rq[qp_index].napi.napi_id); in virtnet_enable_queue_pair()
3060 err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq, in virtnet_enable_queue_pair()
3061 vi->rq[qp_index].page_pool ? in virtnet_enable_queue_pair()
3064 vi->rq[qp_index].page_pool); in virtnet_enable_queue_pair()
3068 virtnet_napi_enable(&vi->rq[qp_index]); in virtnet_enable_queue_pair()
3069 virtnet_napi_tx_enable(&vi->sq[qp_index]); in virtnet_enable_queue_pair()
3074 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); in virtnet_enable_queue_pair()
3078 static void virtnet_cancel_dim(struct virtnet_info *vi, struct dim *dim) in virtnet_cancel_dim() argument
3080 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_cancel_dim()
3085 static void virtnet_update_settings(struct virtnet_info *vi) in virtnet_update_settings() argument
3090 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) in virtnet_update_settings()
3093 virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); in virtnet_update_settings()
3096 vi->speed = speed; in virtnet_update_settings()
3098 virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); in virtnet_update_settings()
3101 vi->duplex = duplex; in virtnet_update_settings()
3104 static int virtnet_create_page_pools(struct virtnet_info *vi) in virtnet_create_page_pools() argument
3108 if (vi->big_packets && !vi->mergeable_rx_bufs) in virtnet_create_page_pools()
3111 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_create_page_pools()
3112 struct receive_queue *rq = &vi->rq[i]; in virtnet_create_page_pools()
3124 pp_params.nid = dev_to_node(vi->vdev->dev.parent); in virtnet_create_page_pools()
3125 pp_params.netdev = vi->dev; in virtnet_create_page_pools()
3156 struct receive_queue *rq = &vi->rq[i]; in virtnet_create_page_pools()
3166 static void virtnet_destroy_page_pools(struct virtnet_info *vi) in virtnet_destroy_page_pools() argument
3170 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_destroy_page_pools()
3171 struct receive_queue *rq = &vi->rq[i]; in virtnet_destroy_page_pools()
3182 struct virtnet_info *vi = netdev_priv(dev); in virtnet_open() local
3185 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_open()
3186 if (i < vi->curr_queue_pairs) in virtnet_open()
3190 try_fill_recv(vi, &vi->rq[i], GFP_KERNEL); in virtnet_open()
3192 err = virtnet_enable_queue_pair(vi, i); in virtnet_open()
3197 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { in virtnet_open()
3198 if (vi->status & VIRTIO_NET_S_LINK_UP) in virtnet_open()
3199 netif_carrier_on(vi->dev); in virtnet_open()
3200 virtio_config_driver_enable(vi->vdev); in virtnet_open()
3202 vi->status = VIRTIO_NET_S_LINK_UP; in virtnet_open()
3210 virtnet_disable_queue_pair(vi, i); in virtnet_open()
3211 virtnet_cancel_dim(vi, &vi->rq[i].dim); in virtnet_open()
3220 struct virtnet_info *vi = sq->vq->vdev->priv; in virtnet_poll_tx() local
3226 if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { in virtnet_poll_tx()
3232 txq = netdev_get_tx_queue(vi->dev, index); in virtnet_poll_tx()
3242 virtnet_tx_wake_queue(vi, sq); in virtnet_poll_tx()
3275 struct virtnet_info *vi = sq->vq->vdev->priv; in xmit_skb() local
3278 unsigned hdr_len = vi->hdr_len; in xmit_skb()
3282 feature_hdrlen = virtio_has_feature(vi->vdev, in xmit_skb()
3285 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); in xmit_skb()
3291 can_push = vi->any_header_sg && in xmit_skb()
3302 if (virtio_net_hdr_tnl_from_skb(skb, hdr, vi->tx_tnl, in xmit_skb()
3303 virtio_is_little_endian(vi->vdev), 0, in xmit_skb()
3307 if (vi->mergeable_rx_bufs) in xmit_skb()
3332 struct virtnet_info *vi = netdev_priv(dev); in start_xmit() local
3334 struct send_queue *sq = &vi->sq[qnum]; in start_xmit()
3372 tx_may_stop(vi, dev, sq); in start_xmit()
3374 check_sq_full_and_disable(vi, dev,sq); in start_xmit()
3392 static void virtnet_rx_pause(struct virtnet_info *vi, in virtnet_rx_pause() argument
3395 bool running = netif_running(vi->dev); in virtnet_rx_pause()
3399 virtnet_cancel_dim(vi, &rq->dim); in virtnet_rx_pause()
3403 static void virtnet_rx_pause_all(struct virtnet_info *vi) in virtnet_rx_pause_all() argument
3407 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_rx_pause_all()
3408 virtnet_rx_pause(vi, &vi->rq[i]); in virtnet_rx_pause_all()
3411 static void virtnet_rx_resume(struct virtnet_info *vi, in virtnet_rx_resume() argument
3415 if (netif_running(vi->dev)) { in virtnet_rx_resume()
3420 try_fill_recv(vi, rq, GFP_KERNEL); in virtnet_rx_resume()
3426 static void virtnet_rx_resume_all(struct virtnet_info *vi) in virtnet_rx_resume_all() argument
3430 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_rx_resume_all()
3431 if (i < vi->curr_queue_pairs) in virtnet_rx_resume_all()
3432 virtnet_rx_resume(vi, &vi->rq[i], true); in virtnet_rx_resume_all()
3434 virtnet_rx_resume(vi, &vi->rq[i], false); in virtnet_rx_resume_all()
3438 static int virtnet_rx_resize(struct virtnet_info *vi, in virtnet_rx_resize() argument
3443 qindex = rq - vi->rq; in virtnet_rx_resize()
3445 virtnet_rx_pause(vi, rq); in virtnet_rx_resize()
3449 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); in virtnet_rx_resize()
3451 virtnet_rx_resume(vi, rq, true); in virtnet_rx_resize()
3455 static void virtnet_tx_pause(struct virtnet_info *vi, struct send_queue *sq) in virtnet_tx_pause() argument
3457 bool running = netif_running(vi->dev); in virtnet_tx_pause()
3461 qindex = sq - vi->sq; in virtnet_tx_pause()
3466 txq = netdev_get_tx_queue(vi->dev, qindex); in virtnet_tx_pause()
3477 netif_stop_subqueue(vi->dev, qindex); in virtnet_tx_pause()
3485 static void virtnet_tx_resume(struct virtnet_info *vi, struct send_queue *sq) in virtnet_tx_resume() argument
3487 bool running = netif_running(vi->dev); in virtnet_tx_resume()
3491 qindex = sq - vi->sq; in virtnet_tx_resume()
3493 txq = netdev_get_tx_queue(vi->dev, qindex); in virtnet_tx_resume()
3497 virtnet_tx_wake_queue(vi, sq); in virtnet_tx_resume()
3504 static int virtnet_tx_resize(struct virtnet_info *vi, struct send_queue *sq, in virtnet_tx_resize() argument
3510 netdev_err(vi->dev, "tx size (%d) cannot be smaller than %d\n", in virtnet_tx_resize()
3515 qindex = sq - vi->sq; in virtnet_tx_resize()
3517 virtnet_tx_pause(vi, sq); in virtnet_tx_resize()
3522 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); in virtnet_tx_resize()
3524 virtnet_tx_resume(vi, sq); in virtnet_tx_resize()
3534 static bool virtnet_send_command_reply(struct virtnet_info *vi, u8 class, u8 cmd, in virtnet_send_command_reply() argument
3544 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); in virtnet_send_command_reply()
3546 mutex_lock(&vi->cvq_lock); in virtnet_send_command_reply()
3547 vi->ctrl->status = ~0; in virtnet_send_command_reply()
3548 vi->ctrl->hdr.class = class; in virtnet_send_command_reply()
3549 vi->ctrl->hdr.cmd = cmd; in virtnet_send_command_reply()
3551 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); in virtnet_send_command_reply()
3558 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); in virtnet_send_command_reply()
3565 ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC); in virtnet_send_command_reply()
3567 dev_warn(&vi->vdev->dev, in virtnet_send_command_reply()
3569 mutex_unlock(&vi->cvq_lock); in virtnet_send_command_reply()
3573 if (unlikely(!virtqueue_kick(vi->cvq))) in virtnet_send_command_reply()
3579 while (!virtqueue_get_buf(vi->cvq, &tmp) && in virtnet_send_command_reply()
3580 !virtqueue_is_broken(vi->cvq)) { in virtnet_send_command_reply()
3586 ok = vi->ctrl->status == VIRTIO_NET_OK; in virtnet_send_command_reply()
3587 mutex_unlock(&vi->cvq_lock); in virtnet_send_command_reply()
3591 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, in virtnet_send_command() argument
3594 return virtnet_send_command_reply(vi, class, cmd, out, NULL); in virtnet_send_command()
3599 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_mac_address() local
3600 struct virtio_device *vdev = vi->vdev; in virtnet_set_mac_address()
3605 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) in virtnet_set_mac_address()
3618 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, in virtnet_set_mac_address()
3647 struct virtnet_info *vi = netdev_priv(dev); in virtnet_stats() local
3651 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_stats()
3653 struct receive_queue *rq = &vi->rq[i]; in virtnet_stats()
3654 struct send_queue *sq = &vi->sq[i]; in virtnet_stats()
3684 static void virtnet_ack_link_announce(struct virtnet_info *vi) in virtnet_ack_link_announce() argument
3686 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, in virtnet_ack_link_announce()
3688 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); in virtnet_ack_link_announce()
3691 static bool virtnet_commit_rss_command(struct virtnet_info *vi);
3693 static void virtnet_rss_update_by_qpairs(struct virtnet_info *vi, u16 queue_pairs) in virtnet_rss_update_by_qpairs() argument
3698 for (; i < vi->rss_indir_table_size; ++i) { in virtnet_rss_update_by_qpairs()
3700 vi->rss_hdr->indirection_table[i] = cpu_to_le16(indir_val); in virtnet_rss_update_by_qpairs()
3702 vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs); in virtnet_rss_update_by_qpairs()
3705 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) in virtnet_set_queues() argument
3710 struct net_device *dev = vi->dev; in virtnet_set_queues()
3713 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) in virtnet_set_queues()
3723 if (vi->has_rss && !netif_is_rxfh_configured(dev)) { in virtnet_set_queues()
3724 old_rss_hdr = vi->rss_hdr; in virtnet_set_queues()
3725 old_rss_trailer = vi->rss_trailer; in virtnet_set_queues()
3726 vi->rss_hdr = devm_kzalloc(&vi->vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); in virtnet_set_queues()
3727 if (!vi->rss_hdr) { in virtnet_set_queues()
3728 vi->rss_hdr = old_rss_hdr; in virtnet_set_queues()
3732 *vi->rss_hdr = *old_rss_hdr; in virtnet_set_queues()
3733 virtnet_rss_update_by_qpairs(vi, queue_pairs); in virtnet_set_queues()
3735 if (!virtnet_commit_rss_command(vi)) { in virtnet_set_queues()
3737 devm_kfree(&vi->vdev->dev, vi->rss_hdr); in virtnet_set_queues()
3738 vi->rss_hdr = old_rss_hdr; in virtnet_set_queues()
3739 vi->rss_trailer = old_rss_trailer; in virtnet_set_queues()
3745 devm_kfree(&vi->vdev->dev, old_rss_hdr); in virtnet_set_queues()
3753 mq->virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); in virtnet_set_queues()
3756 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, in virtnet_set_queues()
3766 if (vi->has_rss) in virtnet_set_queues()
3767 vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs); in virtnet_set_queues()
3769 vi->curr_queue_pairs = queue_pairs; in virtnet_set_queues()
3772 for (int i = 0; i < vi->curr_queue_pairs; ++i) in virtnet_set_queues()
3773 virtqueue_napi_schedule(&vi->rq[i].napi, vi->rq[i].vq); in virtnet_set_queues()
3782 struct virtnet_info *vi = netdev_priv(dev); in virtnet_close() local
3788 virtio_config_driver_disable(vi->vdev); in virtnet_close()
3792 cancel_work_sync(&vi->config_work); in virtnet_close()
3794 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_close()
3795 virtnet_disable_queue_pair(vi, i); in virtnet_close()
3796 virtnet_cancel_dim(vi, &vi->rq[i].dim); in virtnet_close()
3806 struct virtnet_info *vi = in virtnet_rx_mode_work() local
3809 struct net_device *dev = vi->dev; in virtnet_rx_mode_work()
3819 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) in virtnet_rx_mode_work()
3833 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, in virtnet_rx_mode_work()
3841 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, in virtnet_rx_mode_work()
3863 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); in virtnet_rx_mode_work()
3874 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); in virtnet_rx_mode_work()
3884 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, in virtnet_rx_mode_work()
3895 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_rx_mode() local
3897 if (vi->rx_mode_work_enabled) in virtnet_set_rx_mode()
3898 schedule_work(&vi->rx_mode_work); in virtnet_set_rx_mode()
3904 struct virtnet_info *vi = netdev_priv(dev); in virtnet_vlan_rx_add_vid() local
3912 *_vid = cpu_to_virtio16(vi->vdev, vid); in virtnet_vlan_rx_add_vid()
3915 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, in virtnet_vlan_rx_add_vid()
3924 struct virtnet_info *vi = netdev_priv(dev); in virtnet_vlan_rx_kill_vid() local
3932 *_vid = cpu_to_virtio16(vi->vdev, vid); in virtnet_vlan_rx_kill_vid()
3935 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, in virtnet_vlan_rx_kill_vid()
3941 static void virtnet_clean_affinity(struct virtnet_info *vi) in virtnet_clean_affinity() argument
3945 if (vi->affinity_hint_set) { in virtnet_clean_affinity()
3946 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_clean_affinity()
3947 virtqueue_set_affinity(vi->rq[i].vq, NULL); in virtnet_clean_affinity()
3948 virtqueue_set_affinity(vi->sq[i].vq, NULL); in virtnet_clean_affinity()
3951 vi->affinity_hint_set = false; in virtnet_clean_affinity()
3955 static void virtnet_set_affinity(struct virtnet_info *vi) in virtnet_set_affinity() argument
3965 virtnet_clean_affinity(vi); in virtnet_set_affinity()
3970 stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); in virtnet_set_affinity()
3971 stragglers = num_cpu >= vi->curr_queue_pairs ? in virtnet_set_affinity()
3972 num_cpu % vi->curr_queue_pairs : in virtnet_set_affinity()
3975 for (i = 0; i < vi->curr_queue_pairs; i++) { in virtnet_set_affinity()
3986 virtqueue_set_affinity(vi->rq[i].vq, mask); in virtnet_set_affinity()
3987 virtqueue_set_affinity(vi->sq[i].vq, mask); in virtnet_set_affinity()
3988 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); in virtnet_set_affinity()
3992 vi->affinity_hint_set = true; in virtnet_set_affinity()
3998 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, in virtnet_cpu_online() local
4000 virtnet_set_affinity(vi); in virtnet_cpu_online()
4006 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, in virtnet_cpu_dead() local
4008 virtnet_set_affinity(vi); in virtnet_cpu_dead()
4014 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, in virtnet_cpu_down_prep() local
4017 virtnet_clean_affinity(vi); in virtnet_cpu_down_prep()
4023 static int virtnet_cpu_notif_add(struct virtnet_info *vi) in virtnet_cpu_notif_add() argument
4027 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); in virtnet_cpu_notif_add()
4031 &vi->node_dead); in virtnet_cpu_notif_add()
4034 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); in virtnet_cpu_notif_add()
4038 static void virtnet_cpu_notif_remove(struct virtnet_info *vi) in virtnet_cpu_notif_remove() argument
4040 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); in virtnet_cpu_notif_remove()
4042 &vi->node_dead); in virtnet_cpu_notif_remove()
4045 static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, in virtnet_send_ctrl_coal_vq_cmd() argument
4060 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, in virtnet_send_ctrl_coal_vq_cmd()
4068 static int virtnet_send_rx_ctrl_coal_vq_cmd(struct virtnet_info *vi, in virtnet_send_rx_ctrl_coal_vq_cmd() argument
4074 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_send_rx_ctrl_coal_vq_cmd()
4077 err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue), in virtnet_send_rx_ctrl_coal_vq_cmd()
4082 vi->rq[queue].intr_coal.max_usecs = max_usecs; in virtnet_send_rx_ctrl_coal_vq_cmd()
4083 vi->rq[queue].intr_coal.max_packets = max_packets; in virtnet_send_rx_ctrl_coal_vq_cmd()
4088 static int virtnet_send_tx_ctrl_coal_vq_cmd(struct virtnet_info *vi, in virtnet_send_tx_ctrl_coal_vq_cmd() argument
4094 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_send_tx_ctrl_coal_vq_cmd()
4097 err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue), in virtnet_send_tx_ctrl_coal_vq_cmd()
4102 vi->sq[queue].intr_coal.max_usecs = max_usecs; in virtnet_send_tx_ctrl_coal_vq_cmd()
4103 vi->sq[queue].intr_coal.max_packets = max_packets; in virtnet_send_tx_ctrl_coal_vq_cmd()
4113 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_ringparam() local
4115 ring->rx_max_pending = vi->rq[0].vq->num_max; in virtnet_get_ringparam()
4116 ring->tx_max_pending = vi->sq[0].vq->num_max; in virtnet_get_ringparam()
4117 ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); in virtnet_get_ringparam()
4118 ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); in virtnet_get_ringparam()
4126 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_ringparam() local
4135 rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); in virtnet_set_ringparam()
4136 tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); in virtnet_set_ringparam()
4142 if (ring->rx_pending > vi->rq[0].vq->num_max) in virtnet_set_ringparam()
4145 if (ring->tx_pending > vi->sq[0].vq->num_max) in virtnet_set_ringparam()
4148 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_set_ringparam()
4149 rq = vi->rq + i; in virtnet_set_ringparam()
4150 sq = vi->sq + i; in virtnet_set_ringparam()
4153 err = virtnet_tx_resize(vi, sq, ring->tx_pending); in virtnet_set_ringparam()
4162 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, i, in virtnet_set_ringparam()
4163 vi->intr_coal_tx.max_usecs, in virtnet_set_ringparam()
4164 vi->intr_coal_tx.max_packets); in virtnet_set_ringparam()
4174 err = virtnet_rx_resize(vi, rq, ring->rx_pending); in virtnet_set_ringparam()
4179 mutex_lock(&vi->rq[i].dim_lock); in virtnet_set_ringparam()
4180 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, i, in virtnet_set_ringparam()
4181 vi->intr_coal_rx.max_usecs, in virtnet_set_ringparam()
4182 vi->intr_coal_rx.max_packets); in virtnet_set_ringparam()
4183 mutex_unlock(&vi->rq[i].dim_lock); in virtnet_set_ringparam()
4192 static bool virtnet_commit_rss_command(struct virtnet_info *vi) in virtnet_commit_rss_command() argument
4194 struct net_device *dev = vi->dev; in virtnet_commit_rss_command()
4199 sg_set_buf(&sgs[0], vi->rss_hdr, virtnet_rss_hdr_size(vi)); in virtnet_commit_rss_command()
4200 sg_set_buf(&sgs[1], &vi->rss_trailer, virtnet_rss_trailer_size(vi)); in virtnet_commit_rss_command()
4202 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, in virtnet_commit_rss_command()
4203 vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG in virtnet_commit_rss_command()
4215 static void virtnet_init_default_rss(struct virtnet_info *vi) in virtnet_init_default_rss() argument
4217 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_supported); in virtnet_init_default_rss()
4218 vi->rss_hash_types_saved = vi->rss_hash_types_supported; in virtnet_init_default_rss()
4219 vi->rss_hdr->indirection_table_mask = vi->rss_indir_table_size in virtnet_init_default_rss()
4220 ? cpu_to_le16(vi->rss_indir_table_size - 1) : 0; in virtnet_init_default_rss()
4221 vi->rss_hdr->unclassified_queue = 0; in virtnet_init_default_rss()
4223 virtnet_rss_update_by_qpairs(vi, vi->curr_queue_pairs); in virtnet_init_default_rss()
4225 vi->rss_trailer.hash_key_length = vi->rss_key_size; in virtnet_init_default_rss()
4227 netdev_rss_key_fill(vi->rss_hash_key_data, vi->rss_key_size); in virtnet_init_default_rss()
4233 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_hashflow() local
4238 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { in virtnet_get_hashflow()
4241 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { in virtnet_get_hashflow()
4246 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { in virtnet_get_hashflow()
4249 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { in virtnet_get_hashflow()
4254 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { in virtnet_get_hashflow()
4257 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { in virtnet_get_hashflow()
4262 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { in virtnet_get_hashflow()
4265 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { in virtnet_get_hashflow()
4270 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) in virtnet_get_hashflow()
4275 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) in virtnet_get_hashflow()
4291 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_hashflow() local
4292 u32 new_hashtypes = vi->rss_hash_types_saved; in virtnet_set_hashflow()
4341 if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) in virtnet_set_hashflow()
4344 if (new_hashtypes != vi->rss_hash_types_saved) { in virtnet_set_hashflow()
4345 vi->rss_hash_types_saved = new_hashtypes; in virtnet_set_hashflow()
4346 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); in virtnet_set_hashflow()
4347 if (vi->dev->features & NETIF_F_RXHASH) in virtnet_set_hashflow()
4348 if (!virtnet_commit_rss_command(vi)) in virtnet_set_hashflow()
4358 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_drvinfo() local
4359 struct virtio_device *vdev = vi->vdev; in virtnet_get_drvinfo()
4371 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_channels() local
4381 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) in virtnet_set_channels()
4388 if (vi->rq[0].xdp_prog) in virtnet_set_channels()
4392 err = virtnet_set_queues(vi, queue_pairs); in virtnet_set_channels()
4397 virtnet_set_affinity(vi); in virtnet_set_channels()
4421 static void virtnet_get_stats_string(struct virtnet_info *vi, int type, int qid, u8 **data) in virtnet_get_stats_string() argument
4431 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { in virtnet_get_stats_string()
4451 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { in virtnet_get_stats_string()
4458 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { in virtnet_get_stats_string()
4465 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { in virtnet_get_stats_string()
4485 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { in virtnet_get_stats_string()
4492 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { in virtnet_get_stats_string()
4499 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { in virtnet_get_stats_string()
4527 static void virtnet_stats_ctx_init(struct virtnet_info *vi, in virtnet_stats_ctx_init() argument
4542 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { in virtnet_stats_ctx_init()
4548 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { in virtnet_stats_ctx_init()
4554 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { in virtnet_stats_ctx_init()
4560 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { in virtnet_stats_ctx_init()
4568 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { in virtnet_stats_ctx_init()
4574 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { in virtnet_stats_ctx_init()
4580 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { in virtnet_stats_ctx_init()
4586 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { in virtnet_stats_ctx_init()
4598 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { in virtnet_stats_ctx_init()
4608 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { in virtnet_stats_ctx_init()
4614 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { in virtnet_stats_ctx_init()
4620 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { in virtnet_stats_ctx_init()
4628 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { in virtnet_stats_ctx_init()
4634 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { in virtnet_stats_ctx_init()
4640 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { in virtnet_stats_ctx_init()
4668 static void virtnet_fill_total_fields(struct virtnet_info *vi, in virtnet_fill_total_fields() argument
4679 first_tx_q = first_rx_q + vi->curr_queue_pairs * num_rx; in virtnet_fill_total_fields()
4683 stats_sum_queue(data, num_rx, first_rx_q, vi->curr_queue_pairs); in virtnet_fill_total_fields()
4687 stats_sum_queue(data, num_tx, first_tx_q, vi->curr_queue_pairs); in virtnet_fill_total_fields()
4690 static void virtnet_fill_stats_qstat(struct virtnet_info *vi, u32 qid, in virtnet_fill_stats_qstat() argument
4701 queue_type = vq_type(vi, qid); in virtnet_fill_stats_qstat()
4790 * @vi: virtio net info
4797 static void virtnet_fill_stats(struct virtnet_info *vi, u32 qid, in virtnet_fill_stats() argument
4809 return virtnet_fill_stats_qstat(vi, qid, ctx, base, drv_stats, reply_type); in virtnet_fill_stats()
4815 queue_type = vq_type(vi, qid); in virtnet_fill_stats()
4822 offset += num_cq + num_rx * vi->curr_queue_pairs + num_tx * (qid / 2); in virtnet_fill_stats()
4924 static int __virtnet_get_hw_stats(struct virtnet_info *vi, in __virtnet_get_hw_stats() argument
4938 ok = virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, in __virtnet_get_hw_stats()
4948 virtnet_fill_stats(vi, qid, ctx, p, false, hdr->type); in __virtnet_get_hw_stats()
4954 static void virtnet_make_stat_req(struct virtnet_info *vi, in virtnet_make_stat_req() argument
4959 int qtype = vq_type(vi, qid); in virtnet_make_stat_req()
4973 static int virtnet_get_hw_stats(struct virtnet_info *vi, in virtnet_get_hw_stats() argument
4982 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) in virtnet_get_hw_stats()
4986 last_vq = vi->curr_queue_pairs * 2 - 1; in virtnet_get_hw_stats()
4998 qtype = vq_type(vi, i); in virtnet_get_hw_stats()
5022 virtnet_make_stat_req(vi, ctx, req, i, &j); in virtnet_get_hw_stats()
5025 virtnet_make_stat_req(vi, ctx, req, vi->max_queue_pairs * 2, &j); in virtnet_get_hw_stats()
5027 ok = __virtnet_get_hw_stats(vi, ctx, req, sizeof(*req) * j, reply, res_size); in virtnet_get_hw_stats()
5037 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_strings() local
5044 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, -1, &p); in virtnet_get_strings()
5045 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, -1, &p); in virtnet_get_strings()
5047 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_CQ, 0, &p); in virtnet_get_strings()
5049 for (i = 0; i < vi->curr_queue_pairs; ++i) in virtnet_get_strings()
5050 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, i, &p); in virtnet_get_strings()
5052 for (i = 0; i < vi->curr_queue_pairs; ++i) in virtnet_get_strings()
5053 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, i, &p); in virtnet_get_strings()
5060 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_sset_count() local
5066 virtnet_stats_ctx_init(vi, &ctx, NULL, false); in virtnet_get_sset_count()
5071 vi->curr_queue_pairs * pair_count; in virtnet_get_sset_count()
5080 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_ethtool_stats() local
5085 virtnet_stats_ctx_init(vi, &ctx, data, false); in virtnet_get_ethtool_stats()
5086 if (virtnet_get_hw_stats(vi, &ctx, -1)) in virtnet_get_ethtool_stats()
5087 dev_warn(&vi->dev->dev, "Failed to get hw stats.\n"); in virtnet_get_ethtool_stats()
5089 for (i = 0; i < vi->curr_queue_pairs; i++) { in virtnet_get_ethtool_stats()
5090 struct receive_queue *rq = &vi->rq[i]; in virtnet_get_ethtool_stats()
5091 struct send_queue *sq = &vi->sq[i]; in virtnet_get_ethtool_stats()
5096 virtnet_fill_stats(vi, i * 2, &ctx, stats_base, true, 0); in virtnet_get_ethtool_stats()
5102 virtnet_fill_stats(vi, i * 2 + 1, &ctx, stats_base, true, 0); in virtnet_get_ethtool_stats()
5106 virtnet_fill_total_fields(vi, &ctx); in virtnet_get_ethtool_stats()
5112 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_channels() local
5114 channels->combined_count = vi->curr_queue_pairs; in virtnet_get_channels()
5115 channels->max_combined = vi->max_queue_pairs; in virtnet_get_channels()
5125 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_link_ksettings() local
5128 &vi->speed, &vi->duplex); in virtnet_set_link_ksettings()
5134 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_link_ksettings() local
5136 cmd->base.speed = vi->speed; in virtnet_get_link_ksettings()
5137 cmd->base.duplex = vi->duplex; in virtnet_get_link_ksettings()
5143 static int virtnet_send_tx_notf_coal_cmds(struct virtnet_info *vi, in virtnet_send_tx_notf_coal_cmds() argument
5158 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, in virtnet_send_tx_notf_coal_cmds()
5163 vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs; in virtnet_send_tx_notf_coal_cmds()
5164 vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames; in virtnet_send_tx_notf_coal_cmds()
5165 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_send_tx_notf_coal_cmds()
5166 vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs; in virtnet_send_tx_notf_coal_cmds()
5167 vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames; in virtnet_send_tx_notf_coal_cmds()
5173 static int virtnet_send_rx_notf_coal_cmds(struct virtnet_info *vi, in virtnet_send_rx_notf_coal_cmds() argument
5181 if (rx_ctrl_dim_on && !virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_send_rx_notf_coal_cmds()
5184 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != vi->intr_coal_rx.max_usecs || in virtnet_send_rx_notf_coal_cmds()
5185 ec->rx_max_coalesced_frames != vi->intr_coal_rx.max_packets)) in virtnet_send_rx_notf_coal_cmds()
5188 if (rx_ctrl_dim_on && !vi->rx_dim_enabled) { in virtnet_send_rx_notf_coal_cmds()
5189 vi->rx_dim_enabled = true; in virtnet_send_rx_notf_coal_cmds()
5190 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_send_rx_notf_coal_cmds()
5191 mutex_lock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5192 vi->rq[i].dim_enabled = true; in virtnet_send_rx_notf_coal_cmds()
5193 mutex_unlock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5202 if (!rx_ctrl_dim_on && vi->rx_dim_enabled) { in virtnet_send_rx_notf_coal_cmds()
5203 vi->rx_dim_enabled = false; in virtnet_send_rx_notf_coal_cmds()
5204 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_send_rx_notf_coal_cmds()
5205 mutex_lock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5206 vi->rq[i].dim_enabled = false; in virtnet_send_rx_notf_coal_cmds()
5207 mutex_unlock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5219 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, in virtnet_send_rx_notf_coal_cmds()
5224 vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs; in virtnet_send_rx_notf_coal_cmds()
5225 vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames; in virtnet_send_rx_notf_coal_cmds()
5226 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_send_rx_notf_coal_cmds()
5227 mutex_lock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5228 vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs; in virtnet_send_rx_notf_coal_cmds()
5229 vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames; in virtnet_send_rx_notf_coal_cmds()
5230 mutex_unlock(&vi->rq[i].dim_lock); in virtnet_send_rx_notf_coal_cmds()
5236 static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, in virtnet_send_notf_coal_cmds() argument
5241 err = virtnet_send_tx_notf_coal_cmds(vi, ec); in virtnet_send_notf_coal_cmds()
5245 err = virtnet_send_rx_notf_coal_cmds(vi, ec); in virtnet_send_notf_coal_cmds()
5252 static int virtnet_send_rx_notf_coal_vq_cmds(struct virtnet_info *vi, in virtnet_send_rx_notf_coal_vq_cmds() argument
5261 mutex_lock(&vi->rq[queue].dim_lock); in virtnet_send_rx_notf_coal_vq_cmds()
5262 cur_rx_dim = vi->rq[queue].dim_enabled; in virtnet_send_rx_notf_coal_vq_cmds()
5263 max_usecs = vi->rq[queue].intr_coal.max_usecs; in virtnet_send_rx_notf_coal_vq_cmds()
5264 max_packets = vi->rq[queue].intr_coal.max_packets; in virtnet_send_rx_notf_coal_vq_cmds()
5268 mutex_unlock(&vi->rq[queue].dim_lock); in virtnet_send_rx_notf_coal_vq_cmds()
5273 vi->rq[queue].dim_enabled = true; in virtnet_send_rx_notf_coal_vq_cmds()
5274 mutex_unlock(&vi->rq[queue].dim_lock); in virtnet_send_rx_notf_coal_vq_cmds()
5279 vi->rq[queue].dim_enabled = false; in virtnet_send_rx_notf_coal_vq_cmds()
5284 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, queue, in virtnet_send_rx_notf_coal_vq_cmds()
5287 mutex_unlock(&vi->rq[queue].dim_lock); in virtnet_send_rx_notf_coal_vq_cmds()
5291 static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, in virtnet_send_notf_coal_vq_cmds() argument
5297 err = virtnet_send_rx_notf_coal_vq_cmds(vi, ec, queue); in virtnet_send_notf_coal_vq_cmds()
5301 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, queue, in virtnet_send_notf_coal_vq_cmds()
5315 struct virtnet_info *vi = rq->vq->vdev->priv; in virtnet_rx_dim_work() local
5316 struct net_device *dev = vi->dev; in virtnet_rx_dim_work()
5320 qnum = rq - vi->rq; in virtnet_rx_dim_work()
5329 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum, in virtnet_rx_dim_work()
5373 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_coalesce() local
5379 for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) { in virtnet_set_coalesce()
5381 vi->sq[queue_number].napi.weight, in virtnet_set_coalesce()
5387 /* All queues that belong to [queue_number, vi->max_queue_pairs] will be in virtnet_set_coalesce()
5394 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) in virtnet_set_coalesce()
5395 ret = virtnet_send_notf_coal_cmds(vi, ec); in virtnet_set_coalesce()
5406 for (i = queue_number; i < vi->max_queue_pairs; i++) { in virtnet_set_coalesce()
5407 if (vi->sq[i].xsk_pool) in virtnet_set_coalesce()
5411 for (; queue_number < vi->max_queue_pairs; queue_number++) in virtnet_set_coalesce()
5412 vi->sq[queue_number].napi.weight = napi_weight; in virtnet_set_coalesce()
5423 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_coalesce() local
5425 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { in virtnet_get_coalesce()
5426 ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs; in virtnet_get_coalesce()
5427 ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs; in virtnet_get_coalesce()
5428 ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets; in virtnet_get_coalesce()
5429 ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets; in virtnet_get_coalesce()
5430 ec->use_adaptive_rx_coalesce = vi->rx_dim_enabled; in virtnet_get_coalesce()
5434 if (vi->sq[0].napi.weight) in virtnet_get_coalesce()
5445 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_per_queue_coalesce() local
5449 if (queue >= vi->max_queue_pairs) in virtnet_set_per_queue_coalesce()
5455 vi->sq[queue].napi.weight, in virtnet_set_per_queue_coalesce()
5460 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_set_per_queue_coalesce()
5461 ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue); in virtnet_set_per_queue_coalesce()
5469 vi->sq[queue].napi.weight = napi_weight; in virtnet_set_per_queue_coalesce()
5478 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_per_queue_coalesce() local
5480 if (queue >= vi->max_queue_pairs) in virtnet_get_per_queue_coalesce()
5483 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { in virtnet_get_per_queue_coalesce()
5484 mutex_lock(&vi->rq[queue].dim_lock); in virtnet_get_per_queue_coalesce()
5485 ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs; in virtnet_get_per_queue_coalesce()
5486 ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs; in virtnet_get_per_queue_coalesce()
5487 ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets; in virtnet_get_per_queue_coalesce()
5488 ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets; in virtnet_get_per_queue_coalesce()
5489 ec->use_adaptive_rx_coalesce = vi->rq[queue].dim_enabled; in virtnet_get_per_queue_coalesce()
5490 mutex_unlock(&vi->rq[queue].dim_lock); in virtnet_get_per_queue_coalesce()
5494 if (vi->sq[queue].napi.weight) in virtnet_get_per_queue_coalesce()
5503 struct virtnet_info *vi = netdev_priv(dev); in virtnet_init_settings() local
5505 vi->speed = SPEED_UNKNOWN; in virtnet_init_settings()
5506 vi->duplex = DUPLEX_UNKNOWN; in virtnet_init_settings()
5522 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_rxfh() local
5526 for (i = 0; i < vi->rss_indir_table_size; ++i) in virtnet_get_rxfh()
5527 rxfh->indir[i] = le16_to_cpu(vi->rss_hdr->indirection_table[i]); in virtnet_get_rxfh()
5531 memcpy(rxfh->key, vi->rss_hash_key_data, vi->rss_key_size); in virtnet_get_rxfh()
5542 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_rxfh() local
5551 if (!vi->has_rss) in virtnet_set_rxfh()
5554 for (i = 0; i < vi->rss_indir_table_size; ++i) in virtnet_set_rxfh()
5555 vi->rss_hdr->indirection_table[i] = cpu_to_le16(rxfh->indir[i]); in virtnet_set_rxfh()
5564 if (!vi->has_rss && !vi->has_rss_hash_report) in virtnet_set_rxfh()
5567 memcpy(vi->rss_hash_key_data, rxfh->key, vi->rss_key_size); in virtnet_set_rxfh()
5572 virtnet_commit_rss_command(vi); in virtnet_set_rxfh()
5579 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_rx_ring_count() local
5581 return vi->curr_queue_pairs; in virtnet_get_rx_ring_count()
5615 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_queue_stats_rx() local
5616 struct receive_queue *rq = &vi->rq[i]; in virtnet_get_queue_stats_rx()
5619 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); in virtnet_get_queue_stats_rx()
5621 virtnet_get_hw_stats(vi, &ctx, i * 2); in virtnet_get_queue_stats_rx()
5622 virtnet_fill_stats(vi, i * 2, &ctx, (void *)&rq->stats, true, 0); in virtnet_get_queue_stats_rx()
5628 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_queue_stats_tx() local
5629 struct send_queue *sq = &vi->sq[i]; in virtnet_get_queue_stats_tx()
5632 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); in virtnet_get_queue_stats_tx()
5634 virtnet_get_hw_stats(vi, &ctx, i * 2 + 1); in virtnet_get_queue_stats_tx()
5635 virtnet_fill_stats(vi, i * 2 + 1, &ctx, (void *)&sq->stats, true, 0); in virtnet_get_queue_stats_tx()
5642 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_base_stats() local
5650 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { in virtnet_get_base_stats()
5655 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { in virtnet_get_base_stats()
5661 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { in virtnet_get_base_stats()
5668 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) in virtnet_get_base_stats()
5676 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { in virtnet_get_base_stats()
5681 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { in virtnet_get_base_stats()
5686 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { in virtnet_get_base_stats()
5693 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) in virtnet_get_base_stats()
5697 dev->real_num_rx_queues, vi->max_queue_pairs, rx, in virtnet_get_base_stats()
5698 dev->real_num_tx_queues, vi->max_queue_pairs, tx); in virtnet_get_base_stats()
5709 struct virtnet_info *vi = vdev->priv; in virtnet_freeze_down() local
5712 flush_work(&vi->config_work); in virtnet_freeze_down()
5713 disable_rx_mode_work(vi); in virtnet_freeze_down()
5714 flush_work(&vi->rx_mode_work); in virtnet_freeze_down()
5716 if (netif_running(vi->dev)) { in virtnet_freeze_down()
5718 virtnet_close(vi->dev); in virtnet_freeze_down()
5722 netif_tx_lock_bh(vi->dev); in virtnet_freeze_down()
5723 netif_device_detach(vi->dev); in virtnet_freeze_down()
5724 netif_tx_unlock_bh(vi->dev); in virtnet_freeze_down()
5727 static int init_vqs(struct virtnet_info *vi);
5731 struct virtnet_info *vi = vdev->priv; in virtnet_restore_up() local
5734 err = init_vqs(vi); in virtnet_restore_up()
5738 err = virtnet_create_page_pools(vi); in virtnet_restore_up()
5744 enable_rx_mode_work(vi); in virtnet_restore_up()
5746 if (netif_running(vi->dev)) { in virtnet_restore_up()
5748 err = virtnet_open(vi->dev); in virtnet_restore_up()
5754 netif_tx_lock_bh(vi->dev); in virtnet_restore_up()
5755 netif_device_attach(vi->dev); in virtnet_restore_up()
5756 netif_tx_unlock_bh(vi->dev); in virtnet_restore_up()
5761 free_unused_bufs(vi); in virtnet_restore_up()
5762 virtnet_destroy_page_pools(vi); in virtnet_restore_up()
5763 virtnet_del_vqs(vi); in virtnet_restore_up()
5768 virtnet_del_vqs(vi); in virtnet_restore_up()
5772 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) in virtnet_set_guest_offloads() argument
5781 *_offloads = cpu_to_virtio64(vi->vdev, offloads); in virtnet_set_guest_offloads()
5785 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, in virtnet_set_guest_offloads()
5787 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); in virtnet_set_guest_offloads()
5794 static int virtnet_clear_guest_offloads(struct virtnet_info *vi) in virtnet_clear_guest_offloads() argument
5798 if (!vi->guest_offloads) in virtnet_clear_guest_offloads()
5801 return virtnet_set_guest_offloads(vi, offloads); in virtnet_clear_guest_offloads()
5804 static int virtnet_restore_guest_offloads(struct virtnet_info *vi) in virtnet_restore_guest_offloads() argument
5806 u64 offloads = vi->guest_offloads; in virtnet_restore_guest_offloads()
5808 if (!vi->guest_offloads) in virtnet_restore_guest_offloads()
5811 return virtnet_set_guest_offloads(vi, offloads); in virtnet_restore_guest_offloads()
5814 static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq, in virtnet_rq_bind_xsk_pool() argument
5819 qindex = rq - vi->rq; in virtnet_rq_bind_xsk_pool()
5822 err = xdp_rxq_info_reg(&rq->xsk_rxq_info, vi->dev, qindex, rq->napi.napi_id); in virtnet_rq_bind_xsk_pool()
5834 virtnet_rx_pause(vi, rq); in virtnet_rq_bind_xsk_pool()
5838 netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err); in virtnet_rq_bind_xsk_pool()
5845 virtnet_rx_resume(vi, rq, true); in virtnet_rq_bind_xsk_pool()
5855 static int virtnet_sq_bind_xsk_pool(struct virtnet_info *vi, in virtnet_sq_bind_xsk_pool() argument
5861 qindex = sq - vi->sq; in virtnet_sq_bind_xsk_pool()
5863 virtnet_tx_pause(vi, sq); in virtnet_sq_bind_xsk_pool()
5868 netdev_err(vi->dev, "reset tx fail: tx queue index: %d err: %d\n", qindex, err); in virtnet_sq_bind_xsk_pool()
5874 virtnet_tx_resume(vi, sq); in virtnet_sq_bind_xsk_pool()
5883 struct virtnet_info *vi = netdev_priv(dev); in virtnet_xsk_pool_enable() local
5890 if (vi->hdr_len > xsk_pool_get_headroom(pool)) in virtnet_xsk_pool_enable()
5896 if (!vi->rq[qid].page_pool) in virtnet_xsk_pool_enable()
5899 if (qid >= vi->curr_queue_pairs) in virtnet_xsk_pool_enable()
5902 sq = &vi->sq[qid]; in virtnet_xsk_pool_enable()
5903 rq = &vi->rq[qid]; in virtnet_xsk_pool_enable()
5925 hdr_dma = virtqueue_map_single_attrs(sq->vq, &xsk_hdr, vi->hdr_len, in virtnet_xsk_pool_enable()
5936 err = virtnet_rq_bind_xsk_pool(vi, rq, pool); in virtnet_xsk_pool_enable()
5940 err = virtnet_sq_bind_xsk_pool(vi, sq, pool); in virtnet_xsk_pool_enable()
5952 virtnet_rq_bind_xsk_pool(vi, rq, NULL); in virtnet_xsk_pool_enable()
5956 virtqueue_unmap_single_attrs(rq->vq, hdr_dma, vi->hdr_len, in virtnet_xsk_pool_enable()
5965 struct virtnet_info *vi = netdev_priv(dev); in virtnet_xsk_pool_disable() local
5971 if (qid >= vi->curr_queue_pairs) in virtnet_xsk_pool_disable()
5974 sq = &vi->sq[qid]; in virtnet_xsk_pool_disable()
5975 rq = &vi->rq[qid]; in virtnet_xsk_pool_disable()
5979 err = virtnet_rq_bind_xsk_pool(vi, rq, NULL); in virtnet_xsk_pool_disable()
5980 err |= virtnet_sq_bind_xsk_pool(vi, sq, NULL); in virtnet_xsk_pool_disable()
5985 vi->hdr_len, DMA_TO_DEVICE, 0); in virtnet_xsk_pool_disable()
6006 struct virtnet_info *vi = netdev_priv(dev); in virtnet_xdp_set() local
6011 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) in virtnet_xdp_set()
6012 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || in virtnet_xdp_set()
6013 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || in virtnet_xdp_set()
6014 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || in virtnet_xdp_set()
6015 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || in virtnet_xdp_set()
6016 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || in virtnet_xdp_set()
6017 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || in virtnet_xdp_set()
6018 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { in virtnet_xdp_set()
6023 if (vi->mergeable_rx_bufs && !vi->any_header_sg) { in virtnet_xdp_set()
6034 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; in virtnet_xdp_set()
6039 if (curr_qp + xdp_qp > vi->max_queue_pairs) { in virtnet_xdp_set()
6041 curr_qp + xdp_qp, vi->max_queue_pairs); in virtnet_xdp_set()
6045 old_prog = rtnl_dereference(vi->rq[0].xdp_prog); in virtnet_xdp_set()
6050 bpf_prog_add(prog, vi->max_queue_pairs - 1); in virtnet_xdp_set()
6052 virtnet_rx_pause_all(vi); in virtnet_xdp_set()
6056 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_xdp_set()
6057 virtnet_napi_tx_disable(&vi->sq[i]); in virtnet_xdp_set()
6061 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_xdp_set()
6062 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); in virtnet_xdp_set()
6064 virtnet_restore_guest_offloads(vi); in virtnet_xdp_set()
6069 err = virtnet_set_queues(vi, curr_qp + xdp_qp); in virtnet_xdp_set()
6073 vi->xdp_queue_pairs = xdp_qp; in virtnet_xdp_set()
6076 vi->xdp_enabled = true; in virtnet_xdp_set()
6077 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_xdp_set()
6078 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); in virtnet_xdp_set()
6080 virtnet_clear_guest_offloads(vi); in virtnet_xdp_set()
6086 vi->xdp_enabled = false; in virtnet_xdp_set()
6089 virtnet_rx_resume_all(vi); in virtnet_xdp_set()
6090 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_xdp_set()
6094 virtnet_napi_tx_enable(&vi->sq[i]); in virtnet_xdp_set()
6101 virtnet_clear_guest_offloads(vi); in virtnet_xdp_set()
6102 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_xdp_set()
6103 rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); in virtnet_xdp_set()
6106 virtnet_rx_resume_all(vi); in virtnet_xdp_set()
6108 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_xdp_set()
6109 virtnet_napi_tx_enable(&vi->sq[i]); in virtnet_xdp_set()
6112 bpf_prog_sub(prog, vi->max_queue_pairs - 1); in virtnet_xdp_set()
6131 struct virtnet_info *vi = netdev_priv(dev); in virtnet_get_phys_port_name() local
6134 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) in virtnet_get_phys_port_name()
6147 struct virtnet_info *vi = netdev_priv(dev); in virtnet_set_features() local
6152 if (vi->xdp_enabled) in virtnet_set_features()
6156 offloads = vi->guest_offloads_capable; in virtnet_set_features()
6158 offloads = vi->guest_offloads_capable & in virtnet_set_features()
6161 err = virtnet_set_guest_offloads(vi, offloads); in virtnet_set_features()
6164 vi->guest_offloads = offloads; in virtnet_set_features()
6169 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); in virtnet_set_features()
6171 vi->rss_hdr->hash_types = cpu_to_le32(VIRTIO_NET_HASH_REPORT_NONE); in virtnet_set_features()
6173 if (!virtnet_commit_rss_command(vi)) in virtnet_set_features()
6195 static int virtnet_init_irq_moder(struct virtnet_info *vi) in virtnet_init_irq_moder() argument
6202 ret = net_dim_init_irq_moder(vi->dev, profile_flags, coal_flags, in virtnet_init_irq_moder()
6209 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_init_irq_moder()
6210 net_dim_setting(vi->dev, &vi->rq[i].dim, false); in virtnet_init_irq_moder()
6215 static void virtnet_free_irq_moder(struct virtnet_info *vi) in virtnet_free_irq_moder() argument
6217 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) in virtnet_free_irq_moder()
6221 net_dim_free_irq_moder(vi->dev); in virtnet_free_irq_moder()
6246 struct virtnet_info *vi = in virtnet_config_changed_work() local
6250 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, in virtnet_config_changed_work()
6255 netdev_notify_peers(vi->dev); in virtnet_config_changed_work()
6256 virtnet_ack_link_announce(vi); in virtnet_config_changed_work()
6262 if (vi->status == v) in virtnet_config_changed_work()
6265 vi->status = v; in virtnet_config_changed_work()
6267 if (vi->status & VIRTIO_NET_S_LINK_UP) { in virtnet_config_changed_work()
6268 virtnet_update_settings(vi); in virtnet_config_changed_work()
6269 netif_carrier_on(vi->dev); in virtnet_config_changed_work()
6270 netif_tx_wake_all_queues(vi->dev); in virtnet_config_changed_work()
6272 netif_carrier_off(vi->dev); in virtnet_config_changed_work()
6273 netif_tx_stop_all_queues(vi->dev); in virtnet_config_changed_work()
6279 struct virtnet_info *vi = vdev->priv; in virtnet_config_changed() local
6281 schedule_work(&vi->config_work); in virtnet_config_changed()
6284 static void virtnet_free_queues(struct virtnet_info *vi) in virtnet_free_queues() argument
6288 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_free_queues()
6289 __netif_napi_del(&vi->rq[i].napi); in virtnet_free_queues()
6290 __netif_napi_del(&vi->sq[i].napi); in virtnet_free_queues()
6294 * we need to respect an RCU grace period before freeing vi->rq in virtnet_free_queues()
6298 kfree(vi->rq); in virtnet_free_queues()
6299 kfree(vi->sq); in virtnet_free_queues()
6300 kfree(vi->ctrl); in virtnet_free_queues()
6303 static void _free_receive_bufs(struct virtnet_info *vi) in _free_receive_bufs() argument
6308 for (i = 0; i < vi->max_queue_pairs; i++) { in _free_receive_bufs()
6309 while (vi->rq[i].pages) in _free_receive_bufs()
6310 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); in _free_receive_bufs()
6312 old_prog = rtnl_dereference(vi->rq[i].xdp_prog); in _free_receive_bufs()
6313 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); in _free_receive_bufs()
6319 static void free_receive_bufs(struct virtnet_info *vi) in free_receive_bufs() argument
6322 _free_receive_bufs(vi); in free_receive_bufs()
6328 struct virtnet_info *vi = vq->vdev->priv; in virtnet_sq_free_unused_buf() local
6332 sq = &vi->sq[i]; in virtnet_sq_free_unused_buf()
6352 struct virtnet_info *vi = vq->vdev->priv; in virtnet_sq_free_unused_buf_done() local
6355 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); in virtnet_sq_free_unused_buf_done()
6358 static void free_unused_bufs(struct virtnet_info *vi) in free_unused_bufs() argument
6363 for (i = 0; i < vi->max_queue_pairs; i++) { in free_unused_bufs()
6364 struct virtqueue *vq = vi->sq[i].vq; in free_unused_bufs()
6370 for (i = 0; i < vi->max_queue_pairs; i++) { in free_unused_bufs()
6371 struct virtqueue *vq = vi->rq[i].vq; in free_unused_bufs()
6379 static void virtnet_del_vqs(struct virtnet_info *vi) in virtnet_del_vqs() argument
6381 struct virtio_device *vdev = vi->vdev; in virtnet_del_vqs()
6383 virtnet_clean_affinity(vi); in virtnet_del_vqs()
6387 virtnet_free_queues(vi); in virtnet_del_vqs()
6394 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) in mergeable_min_buf_len() argument
6396 const unsigned int hdr_len = vi->hdr_len; in mergeable_min_buf_len()
6398 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; in mergeable_min_buf_len()
6406 static int virtnet_find_vqs(struct virtnet_info *vi) in virtnet_find_vqs() argument
6419 total_vqs = vi->max_queue_pairs * 2 + in virtnet_find_vqs()
6420 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); in virtnet_find_vqs()
6429 if (vi->mergeable_rx_bufs || !vi->big_packets) { in virtnet_find_vqs()
6438 if (vi->has_cvq) { in virtnet_find_vqs()
6443 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_find_vqs()
6446 sprintf(vi->rq[i].name, "input.%u", i); in virtnet_find_vqs()
6447 sprintf(vi->sq[i].name, "output.%u", i); in virtnet_find_vqs()
6448 vqs_info[rxq2vq(i)].name = vi->rq[i].name; in virtnet_find_vqs()
6449 vqs_info[txq2vq(i)].name = vi->sq[i].name; in virtnet_find_vqs()
6454 ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); in virtnet_find_vqs()
6458 if (vi->has_cvq) { in virtnet_find_vqs()
6459 vi->cvq = vqs[total_vqs - 1]; in virtnet_find_vqs()
6460 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) in virtnet_find_vqs()
6461 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; in virtnet_find_vqs()
6464 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_find_vqs()
6465 vi->rq[i].vq = vqs[rxq2vq(i)]; in virtnet_find_vqs()
6466 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); in virtnet_find_vqs()
6467 vi->sq[i].vq = vqs[txq2vq(i)]; in virtnet_find_vqs()
6481 static int virtnet_alloc_queues(struct virtnet_info *vi) in virtnet_alloc_queues() argument
6485 if (vi->has_cvq) { in virtnet_alloc_queues()
6486 vi->ctrl = kzalloc_obj(*vi->ctrl); in virtnet_alloc_queues()
6487 if (!vi->ctrl) in virtnet_alloc_queues()
6490 vi->ctrl = NULL; in virtnet_alloc_queues()
6492 vi->sq = kzalloc_objs(*vi->sq, vi->max_queue_pairs); in virtnet_alloc_queues()
6493 if (!vi->sq) in virtnet_alloc_queues()
6495 vi->rq = kzalloc_objs(*vi->rq, vi->max_queue_pairs); in virtnet_alloc_queues()
6496 if (!vi->rq) in virtnet_alloc_queues()
6499 for (i = 0; i < vi->max_queue_pairs; i++) { in virtnet_alloc_queues()
6500 vi->rq[i].pages = NULL; in virtnet_alloc_queues()
6501 netif_napi_add_config(vi->dev, &vi->rq[i].napi, virtnet_poll, in virtnet_alloc_queues()
6503 vi->rq[i].napi.weight = napi_weight; in virtnet_alloc_queues()
6504 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi, in virtnet_alloc_queues()
6508 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); in virtnet_alloc_queues()
6509 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); in virtnet_alloc_queues()
6510 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); in virtnet_alloc_queues()
6512 u64_stats_init(&vi->rq[i].stats.syncp); in virtnet_alloc_queues()
6513 u64_stats_init(&vi->sq[i].stats.syncp); in virtnet_alloc_queues()
6514 mutex_init(&vi->rq[i].dim_lock); in virtnet_alloc_queues()
6520 kfree(vi->sq); in virtnet_alloc_queues()
6522 kfree(vi->ctrl); in virtnet_alloc_queues()
6527 static int init_vqs(struct virtnet_info *vi) in init_vqs() argument
6532 ret = virtnet_alloc_queues(vi); in init_vqs()
6536 ret = virtnet_find_vqs(vi); in init_vqs()
6541 virtnet_set_affinity(vi); in init_vqs()
6547 virtnet_free_queues(vi); in init_vqs()
6556 struct virtnet_info *vi = netdev_priv(queue->dev); in mergeable_rx_buffer_size_show() local
6558 unsigned int headroom = virtnet_get_headroom(vi); in mergeable_rx_buffer_size_show()
6562 BUG_ON(queue_index >= vi->max_queue_pairs); in mergeable_rx_buffer_size_show()
6563 avg = &vi->rq[queue_index].mrg_avg_pkt_len; in mergeable_rx_buffer_size_show()
6565 get_mergeable_buf_len(&vi->rq[queue_index], avg, in mergeable_rx_buffer_size_show()
6656 static bool virtnet_check_guest_gso(const struct virtnet_info *vi) in virtnet_check_guest_gso() argument
6658 return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || in virtnet_check_guest_gso()
6659 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || in virtnet_check_guest_gso()
6660 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || in virtnet_check_guest_gso()
6661 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || in virtnet_check_guest_gso()
6662 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && in virtnet_check_guest_gso()
6663 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); in virtnet_check_guest_gso()
6666 static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) in virtnet_set_big_packets() argument
6668 bool guest_gso = virtnet_check_guest_gso(vi); in virtnet_set_big_packets()
6675 vi->big_packets = true; in virtnet_set_big_packets()
6676 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); in virtnet_set_big_packets()
6700 struct virtnet_info *vi; in virtnet_xdp_rx_hash() local
6706 vi = netdev_priv(xdp->rxq->dev); in virtnet_xdp_rx_hash()
6707 hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len); in virtnet_xdp_rx_hash()
6726 struct virtnet_info *vi; in virtnet_probe() local
6832 vi = netdev_priv(dev); in virtnet_probe()
6833 vi->dev = dev; in virtnet_probe()
6834 vi->vdev = vdev; in virtnet_probe()
6835 vdev->priv = vi; in virtnet_probe()
6837 INIT_WORK(&vi->config_work, virtnet_config_changed_work); in virtnet_probe()
6838 INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work); in virtnet_probe()
6841 vi->mergeable_rx_bufs = true; in virtnet_probe()
6846 vi->has_rss_hash_report = true; in virtnet_probe()
6849 vi->has_rss = true; in virtnet_probe()
6851 vi->rss_indir_table_size = in virtnet_probe()
6855 vi->rss_hdr = devm_kzalloc(&vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); in virtnet_probe()
6856 if (!vi->rss_hdr) { in virtnet_probe()
6861 if (vi->has_rss || vi->has_rss_hash_report) { in virtnet_probe()
6864 vi->rss_key_size = min_t(u16, key_sz, NETDEV_RSS_KEY_LEN); in virtnet_probe()
6865 if (key_sz > vi->rss_key_size) in virtnet_probe()
6868 key_sz, vi->rss_key_size); in virtnet_probe()
6870 vi->rss_hash_types_supported = in virtnet_probe()
6872 vi->rss_hash_types_supported &= in virtnet_probe()
6883 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash_tunnel); in virtnet_probe()
6884 else if (vi->has_rss_hash_report) in virtnet_probe()
6885 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); in virtnet_probe()
6888 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); in virtnet_probe()
6890 vi->hdr_len = sizeof(struct virtio_net_hdr); in virtnet_probe()
6893 vi->rx_tnl_csum = true; in virtnet_probe()
6895 vi->rx_tnl = true; in virtnet_probe()
6897 vi->tx_tnl = true; in virtnet_probe()
6901 vi->any_header_sg = true; in virtnet_probe()
6904 vi->has_cvq = true; in virtnet_probe()
6906 mutex_init(&vi->cvq_lock); in virtnet_probe()
6927 virtnet_set_big_packets(vi, mtu); in virtnet_probe()
6929 if (vi->any_header_sg) in virtnet_probe()
6930 dev->needed_headroom = vi->hdr_len; in virtnet_probe()
6934 vi->curr_queue_pairs = max_queue_pairs; in virtnet_probe()
6936 vi->curr_queue_pairs = num_online_cpus(); in virtnet_probe()
6937 vi->max_queue_pairs = max_queue_pairs; in virtnet_probe()
6940 err = init_vqs(vi); in virtnet_probe()
6944 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { in virtnet_probe()
6945 vi->intr_coal_rx.max_usecs = 0; in virtnet_probe()
6946 vi->intr_coal_tx.max_usecs = 0; in virtnet_probe()
6947 vi->intr_coal_rx.max_packets = 0; in virtnet_probe()
6952 if (vi->sq[0].napi.weight) in virtnet_probe()
6953 vi->intr_coal_tx.max_packets = 1; in virtnet_probe()
6955 vi->intr_coal_tx.max_packets = 0; in virtnet_probe()
6958 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { in virtnet_probe()
6960 for (i = 0; i < vi->max_queue_pairs; i++) in virtnet_probe()
6961 if (vi->sq[i].napi.weight) in virtnet_probe()
6962 vi->sq[i].intr_coal.max_packets = 1; in virtnet_probe()
6964 err = virtnet_init_irq_moder(vi); in virtnet_probe()
6973 err = virtnet_create_page_pools(vi); in virtnet_probe()
6978 if (vi->mergeable_rx_bufs) in virtnet_probe()
6981 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); in virtnet_probe()
6982 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); in virtnet_probe()
6987 vi->failover = net_failover_create(vi->dev); in virtnet_probe()
6988 if (IS_ERR(vi->failover)) { in virtnet_probe()
6989 err = PTR_ERR(vi->failover); in virtnet_probe()
6994 if (vi->has_rss || vi->has_rss_hash_report) in virtnet_probe()
6995 virtnet_init_default_rss(vi); in virtnet_probe()
6997 enable_rx_mode_work(vi); in virtnet_probe()
7003 if (virtio_has_feature(vi->vdev, fbit)) in virtnet_probe()
7004 set_bit(guest_offloads[i], &vi->guest_offloads); in virtnet_probe()
7006 vi->guest_offloads_capable = vi->guest_offloads; in virtnet_probe()
7009 (vi->guest_offloads_capable & GUEST_OFFLOAD_GRO_HW_MASK)) in virtnet_probe()
7023 virtio_config_driver_disable(vi->vdev); in virtnet_probe()
7027 if (vi->has_rss || vi->has_rss_hash_report) { in virtnet_probe()
7028 if (!virtnet_commit_rss_command(vi)) { in virtnet_probe()
7031 vi->has_rss_hash_report = false; in virtnet_probe()
7032 vi->has_rss = false; in virtnet_probe()
7036 virtnet_set_queues(vi, vi->curr_queue_pairs); in virtnet_probe()
7043 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { in virtnet_probe()
7047 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, in virtnet_probe()
7056 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) { in virtnet_probe()
7070 if (!virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, in virtnet_probe()
7080 vi->device_stats_cap = le64_to_cpu(v); in virtnet_probe()
7086 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { in virtnet_probe()
7087 virtio_config_changed(vi->vdev); in virtnet_probe()
7089 vi->status = VIRTIO_NET_S_LINK_UP; in virtnet_probe()
7090 virtnet_update_settings(vi); in virtnet_probe()
7096 err = virtnet_cpu_notif_add(vi); in virtnet_probe()
7110 net_failover_destroy(vi->failover); in virtnet_probe()
7112 virtnet_destroy_page_pools(vi); in virtnet_probe()
7114 virtnet_free_irq_moder(vi); in virtnet_probe()
7116 virtnet_del_vqs(vi); in virtnet_probe()
7122 static void remove_vq_common(struct virtnet_info *vi) in remove_vq_common() argument
7126 virtio_reset_device(vi->vdev); in remove_vq_common()
7129 free_unused_bufs(vi); in remove_vq_common()
7135 for (i = 0; i < vi->max_queue_pairs; i++) in remove_vq_common()
7136 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); in remove_vq_common()
7138 free_receive_bufs(vi); in remove_vq_common()
7140 virtnet_destroy_page_pools(vi); in remove_vq_common()
7142 virtnet_del_vqs(vi); in remove_vq_common()
7147 struct virtnet_info *vi = vdev->priv; in virtnet_remove() local
7149 virtnet_cpu_notif_remove(vi); in virtnet_remove()
7152 flush_work(&vi->config_work); in virtnet_remove()
7153 disable_rx_mode_work(vi); in virtnet_remove()
7154 flush_work(&vi->rx_mode_work); in virtnet_remove()
7156 virtnet_free_irq_moder(vi); in virtnet_remove()
7158 unregister_netdev(vi->dev); in virtnet_remove()
7160 net_failover_destroy(vi->failover); in virtnet_remove()
7162 remove_vq_common(vi); in virtnet_remove()
7164 free_netdev(vi->dev); in virtnet_remove()
7169 struct virtnet_info *vi = vdev->priv; in virtnet_freeze() local
7171 virtnet_cpu_notif_remove(vi); in virtnet_freeze()
7173 remove_vq_common(vi); in virtnet_freeze()
7180 struct virtnet_info *vi = vdev->priv; in virtnet_restore() local
7186 virtnet_set_queues(vi, vi->curr_queue_pairs); in virtnet_restore()
7188 err = virtnet_cpu_notif_add(vi); in virtnet_restore()
7191 remove_vq_common(vi); in virtnet_restore()