netvsc_drv.c (3d4743131b8de970faa4b979ead0fadfe5d2de9d) netvsc_drv.c (351e1581395fcc7fb952bbd7dda01238f69968fd)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */

--- 11 unchanged lines hidden (view full) ---

20#include <linux/etherdevice.h>
21#include <linux/pci.h>
22#include <linux/skbuff.h>
23#include <linux/if_vlan.h>
24#include <linux/in.h>
25#include <linux/slab.h>
26#include <linux/rtnetlink.h>
27#include <linux/netpoll.h>
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */

--- 11 unchanged lines hidden (view full) ---

20#include <linux/etherdevice.h>
21#include <linux/pci.h>
22#include <linux/skbuff.h>
23#include <linux/if_vlan.h>
24#include <linux/in.h>
25#include <linux/slab.h>
26#include <linux/rtnetlink.h>
27#include <linux/netpoll.h>
28#include <linux/bpf.h>
28
29#include <net/arp.h>
30#include <net/route.h>
31#include <net/sock.h>
32#include <net/pkt_sched.h>
33#include <net/checksum.h>
34#include <net/ip6_checksum.h>
35

--- 478 unchanged lines hidden (view full) ---

514 u64_stats_update_end(&pcpu_stats->syncp);
515 } else {
516 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
517 }
518
519 return rc;
520}
521
29
30#include <net/arp.h>
31#include <net/route.h>
32#include <net/sock.h>
33#include <net/pkt_sched.h>
34#include <net/checksum.h>
35#include <net/ip6_checksum.h>
36

--- 478 unchanged lines hidden (view full) ---

515 u64_stats_update_end(&pcpu_stats->syncp);
516 } else {
517 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
518 }
519
520 return rc;
521}
522
522static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
523static int netvsc_xmit(struct sk_buff *skb, struct net_device *net, bool xdp_tx)
523{
524 struct net_device_context *net_device_ctx = netdev_priv(net);
525 struct hv_netvsc_packet *packet = NULL;
526 int ret;
527 unsigned int num_data_pgs;
528 struct rndis_message *rndis_msg;
529 struct net_device *vf_netdev;
530 u32 rndis_msg_size;

--- 150 unchanged lines hidden (view full) ---

681 rndis_msg->msg_len += rndis_msg_size;
682 packet->total_data_buflen = rndis_msg->msg_len;
683 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
684 skb, packet, pb);
685
686 /* timestamp packet in software */
687 skb_tx_timestamp(skb);
688
524{
525 struct net_device_context *net_device_ctx = netdev_priv(net);
526 struct hv_netvsc_packet *packet = NULL;
527 int ret;
528 unsigned int num_data_pgs;
529 struct rndis_message *rndis_msg;
530 struct net_device *vf_netdev;
531 u32 rndis_msg_size;

--- 150 unchanged lines hidden (view full) ---

682 rndis_msg->msg_len += rndis_msg_size;
683 packet->total_data_buflen = rndis_msg->msg_len;
684 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
685 skb, packet, pb);
686
687 /* timestamp packet in software */
688 skb_tx_timestamp(skb);
689
689 ret = netvsc_send(net, packet, rndis_msg, pb, skb);
690 ret = netvsc_send(net, packet, rndis_msg, pb, skb, xdp_tx);
690 if (likely(ret == 0))
691 return NETDEV_TX_OK;
692
693 if (ret == -EAGAIN) {
694 ++net_device_ctx->eth_stats.tx_busy;
695 return NETDEV_TX_BUSY;
696 }
697

--- 6 unchanged lines hidden (view full) ---

704
705 return NETDEV_TX_OK;
706
707no_memory:
708 ++net_device_ctx->eth_stats.tx_no_memory;
709 goto drop;
710}
711
691 if (likely(ret == 0))
692 return NETDEV_TX_OK;
693
694 if (ret == -EAGAIN) {
695 ++net_device_ctx->eth_stats.tx_busy;
696 return NETDEV_TX_BUSY;
697 }
698

--- 6 unchanged lines hidden (view full) ---

705
706 return NETDEV_TX_OK;
707
708no_memory:
709 ++net_device_ctx->eth_stats.tx_no_memory;
710 goto drop;
711}
712
713static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *ndev)
714{
715 return netvsc_xmit(skb, ndev, false);
716}
717
712/*
713 * netvsc_linkstatus_callback - Link up/down notification
714 */
715void netvsc_linkstatus_callback(struct net_device *net,
716 struct rndis_message *resp)
717{
718 struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
719 struct net_device_context *ndev_ctx = netdev_priv(net);

--- 26 unchanged lines hidden (view full) ---

746
747 spin_lock_irqsave(&ndev_ctx->lock, flags);
748 list_add_tail(&event->list, &ndev_ctx->reconfig_events);
749 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
750
751 schedule_delayed_work(&ndev_ctx->dwork, 0);
752}
753
718/*
719 * netvsc_linkstatus_callback - Link up/down notification
720 */
721void netvsc_linkstatus_callback(struct net_device *net,
722 struct rndis_message *resp)
723{
724 struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
725 struct net_device_context *ndev_ctx = netdev_priv(net);

--- 26 unchanged lines hidden (view full) ---

752
753 spin_lock_irqsave(&ndev_ctx->lock, flags);
754 list_add_tail(&event->list, &ndev_ctx->reconfig_events);
755 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
756
757 schedule_delayed_work(&ndev_ctx->dwork, 0);
758}
759
760static void netvsc_xdp_xmit(struct sk_buff *skb, struct net_device *ndev)
761{
762 int rc;
763
764 skb->queue_mapping = skb_get_rx_queue(skb);
765 __skb_push(skb, ETH_HLEN);
766
767 rc = netvsc_xmit(skb, ndev, true);
768
769 if (dev_xmit_complete(rc))
770 return;
771
772 dev_kfree_skb_any(skb);
773 ndev->stats.tx_dropped++;
774}
775
754static void netvsc_comp_ipcsum(struct sk_buff *skb)
755{
756 struct iphdr *iph = (struct iphdr *)skb->data;
757
758 iph->check = 0;
759 iph->check = ip_fast_csum(iph, iph->ihl);
760}
761
762static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
776static void netvsc_comp_ipcsum(struct sk_buff *skb)
777{
778 struct iphdr *iph = (struct iphdr *)skb->data;
779
780 iph->check = 0;
781 iph->check = ip_fast_csum(iph, iph->ihl);
782}
783
784static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
763 struct netvsc_channel *nvchan)
785 struct netvsc_channel *nvchan,
786 struct xdp_buff *xdp)
764{
765 struct napi_struct *napi = &nvchan->napi;
766 const struct ndis_pkt_8021q_info *vlan = nvchan->rsc.vlan;
767 const struct ndis_tcp_ip_checksum_info *csum_info =
768 nvchan->rsc.csum_info;
769 const u32 *hash_info = nvchan->rsc.hash_info;
770 struct sk_buff *skb;
787{
788 struct napi_struct *napi = &nvchan->napi;
789 const struct ndis_pkt_8021q_info *vlan = nvchan->rsc.vlan;
790 const struct ndis_tcp_ip_checksum_info *csum_info =
791 nvchan->rsc.csum_info;
792 const u32 *hash_info = nvchan->rsc.hash_info;
793 struct sk_buff *skb;
794 void *xbuf = xdp->data_hard_start;
771 int i;
772
795 int i;
796
773 skb = napi_alloc_skb(napi, nvchan->rsc.pktlen);
774 if (!skb)
775 return skb;
797 if (xbuf) {
798 unsigned int hdroom = xdp->data - xdp->data_hard_start;
799 unsigned int xlen = xdp->data_end - xdp->data;
800 unsigned int frag_size = netvsc_xdp_fraglen(hdroom + xlen);
776
801
777 /*
778 * Copy to skb. This copy is needed here since the memory pointed by
779 * hv_netvsc_packet cannot be deallocated
780 */
781 for (i = 0; i < nvchan->rsc.cnt; i++)
782 skb_put_data(skb, nvchan->rsc.data[i], nvchan->rsc.len[i]);
802 skb = build_skb(xbuf, frag_size);
783
803
804 if (!skb) {
805 __free_page(virt_to_page(xbuf));
806 return NULL;
807 }
808
809 skb_reserve(skb, hdroom);
810 skb_put(skb, xlen);
811 skb->dev = napi->dev;
812 } else {
813 skb = napi_alloc_skb(napi, nvchan->rsc.pktlen);
814
815 if (!skb)
816 return NULL;
817
818 /* Copy to skb. This copy is needed here since the memory
819 * pointed by hv_netvsc_packet cannot be deallocated.
820 */
821 for (i = 0; i < nvchan->rsc.cnt; i++)
822 skb_put_data(skb, nvchan->rsc.data[i],
823 nvchan->rsc.len[i]);
824 }
825
784 skb->protocol = eth_type_trans(skb, net);
785
786 /* skb is already created with CHECKSUM_NONE */
787 skb_checksum_none_assert(skb);
788
789 /* Incoming packets may have IP header checksum verified by the host.
790 * They may not have IP header checksum computed after coalescing.
791 * We compute it here if the flags are set, because on Linux, the IP

--- 32 unchanged lines hidden (view full) ---

824int netvsc_recv_callback(struct net_device *net,
825 struct netvsc_device *net_device,
826 struct netvsc_channel *nvchan)
827{
828 struct net_device_context *net_device_ctx = netdev_priv(net);
829 struct vmbus_channel *channel = nvchan->channel;
830 u16 q_idx = channel->offermsg.offer.sub_channel_index;
831 struct sk_buff *skb;
826 skb->protocol = eth_type_trans(skb, net);
827
828 /* skb is already created with CHECKSUM_NONE */
829 skb_checksum_none_assert(skb);
830
831 /* Incoming packets may have IP header checksum verified by the host.
832 * They may not have IP header checksum computed after coalescing.
833 * We compute it here if the flags are set, because on Linux, the IP

--- 32 unchanged lines hidden (view full) ---

866int netvsc_recv_callback(struct net_device *net,
867 struct netvsc_device *net_device,
868 struct netvsc_channel *nvchan)
869{
870 struct net_device_context *net_device_ctx = netdev_priv(net);
871 struct vmbus_channel *channel = nvchan->channel;
872 u16 q_idx = channel->offermsg.offer.sub_channel_index;
873 struct sk_buff *skb;
832 struct netvsc_stats *rx_stats;
874 struct netvsc_stats *rx_stats = &nvchan->rx_stats;
875 struct xdp_buff xdp;
876 u32 act;
833
834 if (net->reg_state != NETREG_REGISTERED)
835 return NVSP_STAT_FAIL;
836
877
878 if (net->reg_state != NETREG_REGISTERED)
879 return NVSP_STAT_FAIL;
880
881 act = netvsc_run_xdp(net, nvchan, &xdp);
882
883 if (act != XDP_PASS && act != XDP_TX) {
884 u64_stats_update_begin(&rx_stats->syncp);
885 rx_stats->xdp_drop++;
886 u64_stats_update_end(&rx_stats->syncp);
887
888 return NVSP_STAT_SUCCESS; /* consumed by XDP */
889 }
890
837 /* Allocate a skb - TODO direct I/O to pages? */
891 /* Allocate a skb - TODO direct I/O to pages? */
838 skb = netvsc_alloc_recv_skb(net, nvchan);
892 skb = netvsc_alloc_recv_skb(net, nvchan, &xdp);
839
840 if (unlikely(!skb)) {
841 ++net_device_ctx->eth_stats.rx_no_memory;
842 return NVSP_STAT_FAIL;
843 }
844
845 skb_record_rx_queue(skb, q_idx);
846
847 /*
848 * Even if injecting the packet, record the statistics
849 * on the synthetic device because modifying the VF device
850 * statistics will not work correctly.
851 */
893
894 if (unlikely(!skb)) {
895 ++net_device_ctx->eth_stats.rx_no_memory;
896 return NVSP_STAT_FAIL;
897 }
898
899 skb_record_rx_queue(skb, q_idx);
900
901 /*
902 * Even if injecting the packet, record the statistics
903 * on the synthetic device because modifying the VF device
904 * statistics will not work correctly.
905 */
852 rx_stats = &nvchan->rx_stats;
853 u64_stats_update_begin(&rx_stats->syncp);
854 rx_stats->packets++;
855 rx_stats->bytes += nvchan->rsc.pktlen;
856
857 if (skb->pkt_type == PACKET_BROADCAST)
858 ++rx_stats->broadcast;
859 else if (skb->pkt_type == PACKET_MULTICAST)
860 ++rx_stats->multicast;
861 u64_stats_update_end(&rx_stats->syncp);
862
906 u64_stats_update_begin(&rx_stats->syncp);
907 rx_stats->packets++;
908 rx_stats->bytes += nvchan->rsc.pktlen;
909
910 if (skb->pkt_type == PACKET_BROADCAST)
911 ++rx_stats->broadcast;
912 else if (skb->pkt_type == PACKET_MULTICAST)
913 ++rx_stats->multicast;
914 u64_stats_update_end(&rx_stats->syncp);
915
916 if (act == XDP_TX) {
917 netvsc_xdp_xmit(skb, net);
918 return NVSP_STAT_SUCCESS;
919 }
920
863 napi_gro_receive(&nvchan->napi, skb);
864 return NVSP_STAT_SUCCESS;
865}
866
867static void netvsc_get_drvinfo(struct net_device *net,
868 struct ethtool_drvinfo *info)
869{
870 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));

--- 10 unchanged lines hidden (view full) ---

881 channel->max_combined = nvdev->max_chn;
882 channel->combined_count = nvdev->num_chn;
883 }
884}
885
886/* Alloc struct netvsc_device_info, and initialize it from either existing
887 * struct netvsc_device, or from default values.
888 */
921 napi_gro_receive(&nvchan->napi, skb);
922 return NVSP_STAT_SUCCESS;
923}
924
925static void netvsc_get_drvinfo(struct net_device *net,
926 struct ethtool_drvinfo *info)
927{
928 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));

--- 10 unchanged lines hidden (view full) ---

939 channel->max_combined = nvdev->max_chn;
940 channel->combined_count = nvdev->num_chn;
941 }
942}
943
944/* Alloc struct netvsc_device_info, and initialize it from either existing
945 * struct netvsc_device, or from default values.
946 */
889static struct netvsc_device_info *netvsc_devinfo_get
890 (struct netvsc_device *nvdev)
947static
948struct netvsc_device_info *netvsc_devinfo_get(struct netvsc_device *nvdev)
891{
892 struct netvsc_device_info *dev_info;
949{
950 struct netvsc_device_info *dev_info;
951 struct bpf_prog *prog;
893
894 dev_info = kzalloc(sizeof(*dev_info), GFP_ATOMIC);
895
896 if (!dev_info)
897 return NULL;
898
899 if (nvdev) {
952
953 dev_info = kzalloc(sizeof(*dev_info), GFP_ATOMIC);
954
955 if (!dev_info)
956 return NULL;
957
958 if (nvdev) {
959 ASSERT_RTNL();
960
900 dev_info->num_chn = nvdev->num_chn;
901 dev_info->send_sections = nvdev->send_section_cnt;
902 dev_info->send_section_size = nvdev->send_section_size;
903 dev_info->recv_sections = nvdev->recv_section_cnt;
904 dev_info->recv_section_size = nvdev->recv_section_size;
905
906 memcpy(dev_info->rss_key, nvdev->extension->rss_key,
907 NETVSC_HASH_KEYLEN);
961 dev_info->num_chn = nvdev->num_chn;
962 dev_info->send_sections = nvdev->send_section_cnt;
963 dev_info->send_section_size = nvdev->send_section_size;
964 dev_info->recv_sections = nvdev->recv_section_cnt;
965 dev_info->recv_section_size = nvdev->recv_section_size;
966
967 memcpy(dev_info->rss_key, nvdev->extension->rss_key,
968 NETVSC_HASH_KEYLEN);
969
970 prog = netvsc_xdp_get(nvdev);
971 if (prog) {
972 bpf_prog_inc(prog);
973 dev_info->bprog = prog;
974 }
908 } else {
909 dev_info->num_chn = VRSS_CHANNEL_DEFAULT;
910 dev_info->send_sections = NETVSC_DEFAULT_TX;
911 dev_info->send_section_size = NETVSC_SEND_SECTION_SIZE;
912 dev_info->recv_sections = NETVSC_DEFAULT_RX;
913 dev_info->recv_section_size = NETVSC_RECV_SECTION_SIZE;
914 }
915
916 return dev_info;
917}
918
975 } else {
976 dev_info->num_chn = VRSS_CHANNEL_DEFAULT;
977 dev_info->send_sections = NETVSC_DEFAULT_TX;
978 dev_info->send_section_size = NETVSC_SEND_SECTION_SIZE;
979 dev_info->recv_sections = NETVSC_DEFAULT_RX;
980 dev_info->recv_section_size = NETVSC_RECV_SECTION_SIZE;
981 }
982
983 return dev_info;
984}
985
986/* Free struct netvsc_device_info */
987static void netvsc_devinfo_put(struct netvsc_device_info *dev_info)
988{
989 if (dev_info->bprog) {
990 ASSERT_RTNL();
991 bpf_prog_put(dev_info->bprog);
992 }
993
994 kfree(dev_info);
995}
996
919static int netvsc_detach(struct net_device *ndev,
920 struct netvsc_device *nvdev)
921{
922 struct net_device_context *ndev_ctx = netdev_priv(ndev);
923 struct hv_device *hdev = ndev_ctx->device_ctx;
924 int ret;
925
926 /* Don't try continuing to try and setup sub channels */
927 if (cancel_work_sync(&nvdev->subchan_work))
928 nvdev->num_chn = 1;
929
997static int netvsc_detach(struct net_device *ndev,
998 struct netvsc_device *nvdev)
999{
1000 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1001 struct hv_device *hdev = ndev_ctx->device_ctx;
1002 int ret;
1003
1004 /* Don't try continuing to try and setup sub channels */
1005 if (cancel_work_sync(&nvdev->subchan_work))
1006 nvdev->num_chn = 1;
1007
1008 netvsc_xdp_set(ndev, NULL, NULL, nvdev);
1009
930 /* If device was up (receiving) then shutdown */
931 if (netif_running(ndev)) {
932 netvsc_tx_disable(nvdev, ndev);
933
934 ret = rndis_filter_close(nvdev);
935 if (ret) {
936 netdev_err(ndev,
937 "unable to close device (ret %d).\n", ret);

--- 17 unchanged lines hidden (view full) ---

955
956static int netvsc_attach(struct net_device *ndev,
957 struct netvsc_device_info *dev_info)
958{
959 struct net_device_context *ndev_ctx = netdev_priv(ndev);
960 struct hv_device *hdev = ndev_ctx->device_ctx;
961 struct netvsc_device *nvdev;
962 struct rndis_device *rdev;
1010 /* If device was up (receiving) then shutdown */
1011 if (netif_running(ndev)) {
1012 netvsc_tx_disable(nvdev, ndev);
1013
1014 ret = rndis_filter_close(nvdev);
1015 if (ret) {
1016 netdev_err(ndev,
1017 "unable to close device (ret %d).\n", ret);

--- 17 unchanged lines hidden (view full) ---

1035
1036static int netvsc_attach(struct net_device *ndev,
1037 struct netvsc_device_info *dev_info)
1038{
1039 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1040 struct hv_device *hdev = ndev_ctx->device_ctx;
1041 struct netvsc_device *nvdev;
1042 struct rndis_device *rdev;
963 int ret;
1043 struct bpf_prog *prog;
1044 int ret = 0;
964
965 nvdev = rndis_filter_device_add(hdev, dev_info);
966 if (IS_ERR(nvdev))
967 return PTR_ERR(nvdev);
968
969 if (nvdev->num_chn > 1) {
970 ret = rndis_set_subchannel(ndev, nvdev, dev_info);
971
972 /* if unavailable, just proceed with one queue */
973 if (ret) {
974 nvdev->max_chn = 1;
975 nvdev->num_chn = 1;
976 }
977 }
978
1045
1046 nvdev = rndis_filter_device_add(hdev, dev_info);
1047 if (IS_ERR(nvdev))
1048 return PTR_ERR(nvdev);
1049
1050 if (nvdev->num_chn > 1) {
1051 ret = rndis_set_subchannel(ndev, nvdev, dev_info);
1052
1053 /* if unavailable, just proceed with one queue */
1054 if (ret) {
1055 nvdev->max_chn = 1;
1056 nvdev->num_chn = 1;
1057 }
1058 }
1059
1060 prog = dev_info->bprog;
1061 if (prog) {
1062 ret = netvsc_xdp_set(ndev, prog, NULL, nvdev);
1063 if (ret)
1064 goto err1;
1065 }
1066
979 /* In any case device is now ready */
980 netif_device_attach(ndev);
981
982 /* Note: enable and attach happen when sub-channels setup */
983 netif_carrier_off(ndev);
984
985 if (netif_running(ndev)) {
986 ret = rndis_filter_open(nvdev);
987 if (ret)
1067 /* In any case device is now ready */
1068 netif_device_attach(ndev);
1069
1070 /* Note: enable and attach happen when sub-channels setup */
1071 netif_carrier_off(ndev);
1072
1073 if (netif_running(ndev)) {
1074 ret = rndis_filter_open(nvdev);
1075 if (ret)
988 goto err;
1076 goto err2;
989
990 rdev = nvdev->extension;
991 if (!rdev->link_state)
992 netif_carrier_on(ndev);
993 }
994
995 return 0;
996
1077
1078 rdev = nvdev->extension;
1079 if (!rdev->link_state)
1080 netif_carrier_on(ndev);
1081 }
1082
1083 return 0;
1084
997err:
1085err2:
998 netif_device_detach(ndev);
999
1086 netif_device_detach(ndev);
1087
1088err1:
1000 rndis_filter_device_remove(hdev, nvdev);
1001
1002 return ret;
1003}
1004
1005static int netvsc_set_channels(struct net_device *net,
1006 struct ethtool_channels *channels)
1007{

--- 33 unchanged lines hidden (view full) ---

1041 ret = netvsc_attach(net, device_info);
1042 if (ret) {
1043 device_info->num_chn = orig;
1044 if (netvsc_attach(net, device_info))
1045 netdev_err(net, "restoring channel setting failed\n");
1046 }
1047
1048out:
1089 rndis_filter_device_remove(hdev, nvdev);
1090
1091 return ret;
1092}
1093
1094static int netvsc_set_channels(struct net_device *net,
1095 struct ethtool_channels *channels)
1096{

--- 33 unchanged lines hidden (view full) ---

1130 ret = netvsc_attach(net, device_info);
1131 if (ret) {
1132 device_info->num_chn = orig;
1133 if (netvsc_attach(net, device_info))
1134 netdev_err(net, "restoring channel setting failed\n");
1135 }
1136
1137out:
1049 kfree(device_info);
1138 netvsc_devinfo_put(device_info);
1050 return ret;
1051}
1052
1053static bool
1054netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
1055{
1056 struct ethtool_link_ksettings diff1 = *cmd;
1057 struct ethtool_link_ksettings diff2 = {};

--- 90 unchanged lines hidden (view full) ---

1148
1149 if (netvsc_attach(ndev, device_info))
1150 netdev_err(ndev, "restoring mtu failed\n");
1151rollback_vf:
1152 if (vf_netdev)
1153 dev_set_mtu(vf_netdev, orig_mtu);
1154
1155out:
1139 return ret;
1140}
1141
1142static bool
1143netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
1144{
1145 struct ethtool_link_ksettings diff1 = *cmd;
1146 struct ethtool_link_ksettings diff2 = {};

--- 90 unchanged lines hidden (view full) ---

1237
1238 if (netvsc_attach(ndev, device_info))
1239 netdev_err(ndev, "restoring mtu failed\n");
1240rollback_vf:
1241 if (vf_netdev)
1242 dev_set_mtu(vf_netdev, orig_mtu);
1243
1244out:
1156 kfree(device_info);
1245 netvsc_devinfo_put(device_info);
1157 return ret;
1158}
1159
1160static void netvsc_get_vf_stats(struct net_device *net,
1161 struct netvsc_vf_pcpu_stats *tot)
1162{
1163 struct net_device_context *ndev_ctx = netdev_priv(net);
1164 int i;

--- 208 unchanged lines hidden (view full) ---

1373};
1374
1375#define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1376#define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
1377
1378/* statistics per queue (rx/tx packets/bytes) */
1379#define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1380
1246 return ret;
1247}
1248
1249static void netvsc_get_vf_stats(struct net_device *net,
1250 struct netvsc_vf_pcpu_stats *tot)
1251{
1252 struct net_device_context *ndev_ctx = netdev_priv(net);
1253 int i;

--- 208 unchanged lines hidden (view full) ---

1462};
1463
1464#define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1465#define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
1466
1467/* statistics per queue (rx/tx packets/bytes) */
1468#define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1469
1381/* 4 statistics per queue (rx/tx packets/bytes) */
1382#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1470/* 5 statistics per queue (rx/tx packets/bytes, rx xdp_drop) */
1471#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 5)
1383
1384static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1385{
1386 struct net_device_context *ndc = netdev_priv(dev);
1387 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1388
1389 if (!nvdev)
1390 return -ENODEV;

--- 15 unchanged lines hidden (view full) ---

1406 struct net_device_context *ndc = netdev_priv(dev);
1407 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1408 const void *nds = &ndc->eth_stats;
1409 const struct netvsc_stats *qstats;
1410 struct netvsc_vf_pcpu_stats sum;
1411 struct netvsc_ethtool_pcpu_stats *pcpu_sum;
1412 unsigned int start;
1413 u64 packets, bytes;
1472
1473static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1474{
1475 struct net_device_context *ndc = netdev_priv(dev);
1476 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1477
1478 if (!nvdev)
1479 return -ENODEV;

--- 15 unchanged lines hidden (view full) ---

1495 struct net_device_context *ndc = netdev_priv(dev);
1496 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1497 const void *nds = &ndc->eth_stats;
1498 const struct netvsc_stats *qstats;
1499 struct netvsc_vf_pcpu_stats sum;
1500 struct netvsc_ethtool_pcpu_stats *pcpu_sum;
1501 unsigned int start;
1502 u64 packets, bytes;
1503 u64 xdp_drop;
1414 int i, j, cpu;
1415
1416 if (!nvdev)
1417 return;
1418
1419 for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1420 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1421

--- 12 unchanged lines hidden (view full) ---

1434 data[i++] = packets;
1435 data[i++] = bytes;
1436
1437 qstats = &nvdev->chan_table[j].rx_stats;
1438 do {
1439 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1440 packets = qstats->packets;
1441 bytes = qstats->bytes;
1504 int i, j, cpu;
1505
1506 if (!nvdev)
1507 return;
1508
1509 for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1510 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1511

--- 12 unchanged lines hidden (view full) ---

1524 data[i++] = packets;
1525 data[i++] = bytes;
1526
1527 qstats = &nvdev->chan_table[j].rx_stats;
1528 do {
1529 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1530 packets = qstats->packets;
1531 bytes = qstats->bytes;
1532 xdp_drop = qstats->xdp_drop;
1442 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1443 data[i++] = packets;
1444 data[i++] = bytes;
1533 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1534 data[i++] = packets;
1535 data[i++] = bytes;
1536 data[i++] = xdp_drop;
1445 }
1446
1447 pcpu_sum = kvmalloc_array(num_possible_cpus(),
1448 sizeof(struct netvsc_ethtool_pcpu_stats),
1449 GFP_KERNEL);
1450 netvsc_get_pcpu_stats(dev, pcpu_sum);
1451 for_each_present_cpu(cpu) {
1452 struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];

--- 31 unchanged lines hidden (view full) ---

1484 sprintf(p, "tx_queue_%u_packets", i);
1485 p += ETH_GSTRING_LEN;
1486 sprintf(p, "tx_queue_%u_bytes", i);
1487 p += ETH_GSTRING_LEN;
1488 sprintf(p, "rx_queue_%u_packets", i);
1489 p += ETH_GSTRING_LEN;
1490 sprintf(p, "rx_queue_%u_bytes", i);
1491 p += ETH_GSTRING_LEN;
1537 }
1538
1539 pcpu_sum = kvmalloc_array(num_possible_cpus(),
1540 sizeof(struct netvsc_ethtool_pcpu_stats),
1541 GFP_KERNEL);
1542 netvsc_get_pcpu_stats(dev, pcpu_sum);
1543 for_each_present_cpu(cpu) {
1544 struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];

--- 31 unchanged lines hidden (view full) ---

1576 sprintf(p, "tx_queue_%u_packets", i);
1577 p += ETH_GSTRING_LEN;
1578 sprintf(p, "tx_queue_%u_bytes", i);
1579 p += ETH_GSTRING_LEN;
1580 sprintf(p, "rx_queue_%u_packets", i);
1581 p += ETH_GSTRING_LEN;
1582 sprintf(p, "rx_queue_%u_bytes", i);
1583 p += ETH_GSTRING_LEN;
1584 sprintf(p, "rx_queue_%u_xdp_drop", i);
1585 p += ETH_GSTRING_LEN;
1492 }
1493
1494 for_each_present_cpu(cpu) {
1495 for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) {
1496 sprintf(p, pcpu_stats[i].name, cpu);
1497 p += ETH_GSTRING_LEN;
1498 }
1499 }

--- 280 unchanged lines hidden (view full) ---

1780 device_info->send_sections = orig.tx_pending;
1781 device_info->recv_sections = orig.rx_pending;
1782
1783 if (netvsc_attach(ndev, device_info))
1784 netdev_err(ndev, "restoring ringparam failed");
1785 }
1786
1787out:
1586 }
1587
1588 for_each_present_cpu(cpu) {
1589 for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) {
1590 sprintf(p, pcpu_stats[i].name, cpu);
1591 p += ETH_GSTRING_LEN;
1592 }
1593 }

--- 280 unchanged lines hidden (view full) ---

1874 device_info->send_sections = orig.tx_pending;
1875 device_info->recv_sections = orig.rx_pending;
1876
1877 if (netvsc_attach(ndev, device_info))
1878 netdev_err(ndev, "restoring ringparam failed");
1879 }
1880
1881out:
1788 kfree(device_info);
1882 netvsc_devinfo_put(device_info);
1789 return ret;
1790}
1791
1883 return ret;
1884}
1885
1886static netdev_features_t netvsc_fix_features(struct net_device *ndev,
1887 netdev_features_t features)
1888{
1889 struct net_device_context *ndevctx = netdev_priv(ndev);
1890 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1891
1892 if (!nvdev || nvdev->destroy)
1893 return features;
1894
1895 if ((features & NETIF_F_LRO) && netvsc_xdp_get(nvdev)) {
1896 features ^= NETIF_F_LRO;
1897 netdev_info(ndev, "Skip LRO - unsupported with XDP\n");
1898 }
1899
1900 return features;
1901}
1902
1792static int netvsc_set_features(struct net_device *ndev,
1793 netdev_features_t features)
1794{
1795 netdev_features_t change = features ^ ndev->features;
1796 struct net_device_context *ndevctx = netdev_priv(ndev);
1797 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1798 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1799 struct ndis_offload_params offloads;

--- 70 unchanged lines hidden (view full) ---

1870};
1871
1872static const struct net_device_ops device_ops = {
1873 .ndo_open = netvsc_open,
1874 .ndo_stop = netvsc_close,
1875 .ndo_start_xmit = netvsc_start_xmit,
1876 .ndo_change_rx_flags = netvsc_change_rx_flags,
1877 .ndo_set_rx_mode = netvsc_set_rx_mode,
1903static int netvsc_set_features(struct net_device *ndev,
1904 netdev_features_t features)
1905{
1906 netdev_features_t change = features ^ ndev->features;
1907 struct net_device_context *ndevctx = netdev_priv(ndev);
1908 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1909 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1910 struct ndis_offload_params offloads;

--- 70 unchanged lines hidden (view full) ---

1981};
1982
1983static const struct net_device_ops device_ops = {
1984 .ndo_open = netvsc_open,
1985 .ndo_stop = netvsc_close,
1986 .ndo_start_xmit = netvsc_start_xmit,
1987 .ndo_change_rx_flags = netvsc_change_rx_flags,
1988 .ndo_set_rx_mode = netvsc_set_rx_mode,
1989 .ndo_fix_features = netvsc_fix_features,
1878 .ndo_set_features = netvsc_set_features,
1879 .ndo_change_mtu = netvsc_change_mtu,
1880 .ndo_validate_addr = eth_validate_addr,
1881 .ndo_set_mac_address = netvsc_set_mac_addr,
1882 .ndo_select_queue = netvsc_select_queue,
1883 .ndo_get_stats64 = netvsc_get_stats64,
1990 .ndo_set_features = netvsc_set_features,
1991 .ndo_change_mtu = netvsc_change_mtu,
1992 .ndo_validate_addr = eth_validate_addr,
1993 .ndo_set_mac_address = netvsc_set_mac_addr,
1994 .ndo_select_queue = netvsc_select_queue,
1995 .ndo_get_stats64 = netvsc_get_stats64,
1996 .ndo_bpf = netvsc_bpf,
1884};
1885
1886/*
1887 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1888 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1889 * present send GARP packet to network peers with netif_notify_peers().
1890 */
1891static void netvsc_link_change(struct work_struct *w)

--- 270 unchanged lines hidden (view full) ---

2162 "no netdev found for vf serial:%u\n", serial);
2163 return NULL;
2164}
2165
2166static int netvsc_register_vf(struct net_device *vf_netdev)
2167{
2168 struct net_device_context *net_device_ctx;
2169 struct netvsc_device *netvsc_dev;
1997};
1998
1999/*
2000 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
2001 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
2002 * present send GARP packet to network peers with netif_notify_peers().
2003 */
2004static void netvsc_link_change(struct work_struct *w)

--- 270 unchanged lines hidden (view full) ---

2275 "no netdev found for vf serial:%u\n", serial);
2276 return NULL;
2277}
2278
2279static int netvsc_register_vf(struct net_device *vf_netdev)
2280{
2281 struct net_device_context *net_device_ctx;
2282 struct netvsc_device *netvsc_dev;
2283 struct bpf_prog *prog;
2170 struct net_device *ndev;
2171 int ret;
2172
2173 if (vf_netdev->addr_len != ETH_ALEN)
2174 return NOTIFY_DONE;
2175
2176 ndev = get_netvsc_byslot(vf_netdev);
2177 if (!ndev)

--- 28 unchanged lines hidden (view full) ---

2206 return NOTIFY_DONE;
2207
2208 dev_hold(vf_netdev);
2209 rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
2210
2211 vf_netdev->wanted_features = ndev->features;
2212 netdev_update_features(vf_netdev);
2213
2284 struct net_device *ndev;
2285 int ret;
2286
2287 if (vf_netdev->addr_len != ETH_ALEN)
2288 return NOTIFY_DONE;
2289
2290 ndev = get_netvsc_byslot(vf_netdev);
2291 if (!ndev)

--- 28 unchanged lines hidden (view full) ---

2320 return NOTIFY_DONE;
2321
2322 dev_hold(vf_netdev);
2323 rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
2324
2325 vf_netdev->wanted_features = ndev->features;
2326 netdev_update_features(vf_netdev);
2327
2328 prog = netvsc_xdp_get(netvsc_dev);
2329 netvsc_vf_setxdp(vf_netdev, prog);
2330
2214 return NOTIFY_OK;
2215}
2216
2217/* VF up/down change detected, schedule to change data path */
2218static int netvsc_vf_changed(struct net_device *vf_netdev)
2219{
2220 struct net_device_context *net_device_ctx;
2221 struct netvsc_device *netvsc_dev;

--- 25 unchanged lines hidden (view full) ---

2247 if (!ndev)
2248 return NOTIFY_DONE;
2249
2250 net_device_ctx = netdev_priv(ndev);
2251 cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2252
2253 netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2254
2331 return NOTIFY_OK;
2332}
2333
2334/* VF up/down change detected, schedule to change data path */
2335static int netvsc_vf_changed(struct net_device *vf_netdev)
2336{
2337 struct net_device_context *net_device_ctx;
2338 struct netvsc_device *netvsc_dev;

--- 25 unchanged lines hidden (view full) ---

2364 if (!ndev)
2365 return NOTIFY_DONE;
2366
2367 net_device_ctx = netdev_priv(ndev);
2368 cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2369
2370 netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2371
2372 netvsc_vf_setxdp(vf_netdev, NULL);
2373
2255 netdev_rx_handler_unregister(vf_netdev);
2256 netdev_upper_dev_unlink(vf_netdev, ndev);
2257 RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2258 dev_put(vf_netdev);
2259
2260 return NOTIFY_OK;
2261}
2262

--- 95 unchanged lines hidden (view full) ---

2358 if (ret != 0) {
2359 pr_err("Unable to register netdev.\n");
2360 goto register_failed;
2361 }
2362
2363 list_add(&net_device_ctx->list, &netvsc_dev_list);
2364 rtnl_unlock();
2365
2374 netdev_rx_handler_unregister(vf_netdev);
2375 netdev_upper_dev_unlink(vf_netdev, ndev);
2376 RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2377 dev_put(vf_netdev);
2378
2379 return NOTIFY_OK;
2380}
2381

--- 95 unchanged lines hidden (view full) ---

2477 if (ret != 0) {
2478 pr_err("Unable to register netdev.\n");
2479 goto register_failed;
2480 }
2481
2482 list_add(&net_device_ctx->list, &netvsc_dev_list);
2483 rtnl_unlock();
2484
2366 kfree(device_info);
2485 netvsc_devinfo_put(device_info);
2367 return 0;
2368
2369register_failed:
2370 rtnl_unlock();
2371 rndis_filter_device_remove(dev, nvdev);
2372rndis_failed:
2486 return 0;
2487
2488register_failed:
2489 rtnl_unlock();
2490 rndis_filter_device_remove(dev, nvdev);
2491rndis_failed:
2373 kfree(device_info);
2492 netvsc_devinfo_put(device_info);
2374devinfo_failed:
2375 free_percpu(net_device_ctx->vf_stats);
2376no_stats:
2377 hv_set_drvdata(dev, NULL);
2378 free_netdev(net);
2379no_net:
2380 return ret;
2381}

--- 11 unchanged lines hidden (view full) ---

2393 }
2394
2395 ndev_ctx = netdev_priv(net);
2396
2397 cancel_delayed_work_sync(&ndev_ctx->dwork);
2398
2399 rtnl_lock();
2400 nvdev = rtnl_dereference(ndev_ctx->nvdev);
2493devinfo_failed:
2494 free_percpu(net_device_ctx->vf_stats);
2495no_stats:
2496 hv_set_drvdata(dev, NULL);
2497 free_netdev(net);
2498no_net:
2499 return ret;
2500}

--- 11 unchanged lines hidden (view full) ---

2512 }
2513
2514 ndev_ctx = netdev_priv(net);
2515
2516 cancel_delayed_work_sync(&ndev_ctx->dwork);
2517
2518 rtnl_lock();
2519 nvdev = rtnl_dereference(ndev_ctx->nvdev);
2401 if (nvdev)
2520 if (nvdev) {
2402 cancel_work_sync(&nvdev->subchan_work);
2521 cancel_work_sync(&nvdev->subchan_work);
2522 netvsc_xdp_set(net, NULL, NULL, nvdev);
2523 }
2403
2404 /*
2405 * Call to the vsc driver to let it know that the device is being
2406 * removed. Also blocks mtu and channel changes.
2407 */
2408 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2409 if (vf_netdev)
2410 netvsc_unregister_vf(vf_netdev);

--- 56 unchanged lines hidden (view full) ---

2467
2468 rtnl_lock();
2469
2470 net_device_ctx = netdev_priv(net);
2471 device_info = net_device_ctx->saved_netvsc_dev_info;
2472
2473 ret = netvsc_attach(net, device_info);
2474
2524
2525 /*
2526 * Call to the vsc driver to let it know that the device is being
2527 * removed. Also blocks mtu and channel changes.
2528 */
2529 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2530 if (vf_netdev)
2531 netvsc_unregister_vf(vf_netdev);

--- 56 unchanged lines hidden (view full) ---

2588
2589 rtnl_lock();
2590
2591 net_device_ctx = netdev_priv(net);
2592 device_info = net_device_ctx->saved_netvsc_dev_info;
2593
2594 ret = netvsc_attach(net, device_info);
2595
2475 rtnl_unlock();
2476
2477 kfree(device_info);
2596 netvsc_devinfo_put(device_info);
2478 net_device_ctx->saved_netvsc_dev_info = NULL;
2479
2597 net_device_ctx->saved_netvsc_dev_info = NULL;
2598
2599 rtnl_unlock();
2600
2480 return ret;
2481}
2482static const struct hv_vmbus_device_id id_table[] = {
2483 /* Network guid */
2484 { HV_NIC_GUID, },
2485 { },
2486};
2487

--- 90 unchanged lines hidden --- 		2601 return ret;
2602}
2603static const struct hv_vmbus_device_id id_table[] = {
2604 /* Network guid */
2605 { HV_NIC_GUID, },
2606 { },
2607};
2608

--- 90 unchanged lines hidden ---