1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2019, Microsoft Corporation. 3 * 4 * Author: 5 * Haiyang Zhang <haiyangz@microsoft.com> 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/netdevice.h> 11 #include <linux/etherdevice.h> 12 #include <linux/ethtool.h> 13 #include <linux/bpf.h> 14 #include <linux/bpf_trace.h> 15 #include <linux/kernel.h> 16 #include <net/xdp.h> 17 18 #include <linux/mutex.h> 19 #include <linux/rtnetlink.h> 20 21 #include "hyperv_net.h" 22 23 u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan, 24 struct xdp_buff *xdp) 25 { 26 void *data = nvchan->rsc.data[0]; 27 u32 len = nvchan->rsc.len[0]; 28 struct page *page = NULL; 29 struct bpf_prog *prog; 30 u32 act = XDP_PASS; 31 32 xdp->data_hard_start = NULL; 33 34 rcu_read_lock(); 35 prog = rcu_dereference(nvchan->bpf_prog); 36 37 if (!prog) 38 goto out; 39 40 /* allocate page buffer for data */ 41 page = alloc_page(GFP_ATOMIC); 42 if (!page) { 43 act = XDP_DROP; 44 goto out; 45 } 46 47 xdp->data_hard_start = page_address(page); 48 xdp->data = xdp->data_hard_start + NETVSC_XDP_HDRM; 49 xdp_set_data_meta_invalid(xdp); 50 xdp->data_end = xdp->data + len; 51 xdp->rxq = &nvchan->xdp_rxq; 52 xdp->frame_sz = PAGE_SIZE; 53 xdp->handle = 0; 54 55 memcpy(xdp->data, data, len); 56 57 act = bpf_prog_run_xdp(prog, xdp); 58 59 switch (act) { 60 case XDP_PASS: 61 case XDP_TX: 62 case XDP_DROP: 63 break; 64 65 case XDP_ABORTED: 66 trace_xdp_exception(ndev, prog, act); 67 break; 68 69 default: 70 bpf_warn_invalid_xdp_action(act); 71 } 72 73 out: 74 rcu_read_unlock(); 75 76 if (page && act != XDP_PASS && act != XDP_TX) { 77 __free_page(page); 78 xdp->data_hard_start = NULL; 79 } 80 81 return act; 82 } 83 84 unsigned int netvsc_xdp_fraglen(unsigned int len) 85 { 86 return SKB_DATA_ALIGN(len) + 87 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 88 } 89 90 struct bpf_prog *netvsc_xdp_get(struct netvsc_device *nvdev) 91 { 92 return rtnl_dereference(nvdev->chan_table[0].bpf_prog); 93 } 94 95 int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog, 96 struct netlink_ext_ack *extack, 97 struct netvsc_device *nvdev) 98 { 99 struct bpf_prog *old_prog; 100 int buf_max, i; 101 102 old_prog = netvsc_xdp_get(nvdev); 103 104 if (!old_prog && !prog) 105 return 0; 106 107 buf_max = NETVSC_XDP_HDRM + netvsc_xdp_fraglen(dev->mtu + ETH_HLEN); 108 if (prog && buf_max > PAGE_SIZE) { 109 netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", 110 dev->mtu, buf_max); 111 NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large"); 112 113 return -EOPNOTSUPP; 114 } 115 116 if (prog && (dev->features & NETIF_F_LRO)) { 117 netdev_err(dev, "XDP: not support LRO\n"); 118 NL_SET_ERR_MSG_MOD(extack, "XDP: not support LRO"); 119 120 return -EOPNOTSUPP; 121 } 122 123 if (prog) 124 bpf_prog_add(prog, nvdev->num_chn - 1); 125 126 for (i = 0; i < nvdev->num_chn; i++) 127 rcu_assign_pointer(nvdev->chan_table[i].bpf_prog, prog); 128 129 if (old_prog) 130 for (i = 0; i < nvdev->num_chn; i++) 131 bpf_prog_put(old_prog); 132 133 return 0; 134 } 135 136 int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) 137 { 138 struct netdev_bpf xdp; 139 bpf_op_t ndo_bpf; 140 int ret; 141 142 ASSERT_RTNL(); 143 144 if (!vf_netdev) 145 return 0; 146 147 ndo_bpf = vf_netdev->netdev_ops->ndo_bpf; 148 if (!ndo_bpf) 149 return 0; 150 151 memset(&xdp, 0, sizeof(xdp)); 152 153 if (prog) 154 bpf_prog_inc(prog); 155 156 xdp.command = XDP_SETUP_PROG; 157 xdp.prog = prog; 158 159 ret = ndo_bpf(vf_netdev, &xdp); 160 161 if (ret && prog) 162 bpf_prog_put(prog); 163 164 return ret; 165 } 166 167 static u32 netvsc_xdp_query(struct netvsc_device *nvdev) 168 { 169 struct bpf_prog *prog = netvsc_xdp_get(nvdev); 170 171 if (prog) 172 return prog->aux->id; 173 174 return 0; 175 } 176 177 int netvsc_bpf(struct net_device *dev, struct netdev_bpf *bpf) 178 { 179 struct net_device_context *ndevctx = netdev_priv(dev); 180 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); 181 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); 182 struct netlink_ext_ack *extack = bpf->extack; 183 int ret; 184 185 if (!nvdev || nvdev->destroy) { 186 if (bpf->command == XDP_QUERY_PROG) { 187 bpf->prog_id = 0; 188 return 0; /* Query must always succeed */ 189 } else { 190 return -ENODEV; 191 } 192 } 193 194 switch (bpf->command) { 195 case XDP_SETUP_PROG: 196 ret = netvsc_xdp_set(dev, bpf->prog, extack, nvdev); 197 198 if (ret) 199 return ret; 200 201 ret = netvsc_vf_setxdp(vf_netdev, bpf->prog); 202 203 if (ret) { 204 netdev_err(dev, "vf_setxdp failed:%d\n", ret); 205 NL_SET_ERR_MSG_MOD(extack, "vf_setxdp failed"); 206 207 netvsc_xdp_set(dev, NULL, extack, nvdev); 208 } 209 210 return ret; 211 212 case XDP_QUERY_PROG: 213 bpf->prog_id = netvsc_xdp_query(nvdev); 214 return 0; 215 216 default: 217 return -EINVAL; 218 } 219 } 220