1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * drivers/net/veth.c 4 * 5 * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6 * 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9 * 10 */ 11 12 #include <linux/netdevice.h> 13 #include <linux/slab.h> 14 #include <linux/ethtool.h> 15 #include <linux/etherdevice.h> 16 #include <linux/u64_stats_sync.h> 17 18 #include <net/rtnetlink.h> 19 #include <net/dst.h> 20 #include <net/netdev_lock.h> 21 #include <net/xfrm.h> 22 #include <net/xdp.h> 23 #include <linux/veth.h> 24 #include <linux/module.h> 25 #include <linux/bpf.h> 26 #include <linux/filter.h> 27 #include <linux/ptr_ring.h> 28 #include <linux/bpf_trace.h> 29 #include <linux/net_tstamp.h> 30 #include <linux/skbuff_ref.h> 31 #include <net/page_pool/helpers.h> 32 33 #define DRV_NAME "veth" 34 #define DRV_VERSION "1.0" 35 36 #define VETH_XDP_FLAG BIT(0) 37 #define VETH_RING_SIZE 256 38 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 39 40 #define VETH_XDP_TX_BULK_SIZE 16 41 #define VETH_XDP_BATCH 16 42 43 struct veth_stats { 44 u64 rx_drops; 45 /* xdp */ 46 u64 xdp_packets; 47 u64 xdp_bytes; 48 u64 xdp_redirect; 49 u64 xdp_drops; 50 u64 xdp_tx; 51 u64 xdp_tx_err; 52 u64 peer_tq_xdp_xmit; 53 u64 peer_tq_xdp_xmit_err; 54 }; 55 56 struct veth_rq_stats { 57 struct veth_stats vs; 58 struct u64_stats_sync syncp; 59 }; 60 61 struct veth_rq { 62 struct napi_struct xdp_napi; 63 struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */ 64 struct net_device *dev; 65 struct bpf_prog __rcu *xdp_prog; 66 struct xdp_mem_info xdp_mem; 67 struct veth_rq_stats stats; 68 bool rx_notify_masked; 69 struct ptr_ring xdp_ring; 70 struct xdp_rxq_info xdp_rxq; 71 struct page_pool *page_pool; 72 }; 73 74 struct veth_priv { 75 struct net_device __rcu *peer; 76 atomic64_t dropped; 77 struct bpf_prog *_xdp_prog; 78 struct veth_rq *rq; 79 unsigned int requested_headroom; 80 netdevice_tracker peer_tracker; 81 }; 82 83 struct veth_xdp_tx_bq { 84 struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 85 unsigned int count; 86 }; 87 88 /* 89 * ethtool interface 90 */ 91 92 struct veth_q_stat_desc { 93 char desc[ETH_GSTRING_LEN]; 94 size_t offset; 95 }; 96 97 #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 98 99 static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 100 { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 101 { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 102 { "drops", VETH_RQ_STAT(rx_drops) }, 103 { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 104 { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 105 { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 106 { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 107 }; 108 109 #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 110 111 static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 112 { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 113 { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 114 }; 115 116 #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 117 118 static struct { 119 const char string[ETH_GSTRING_LEN]; 120 } ethtool_stats_keys[] = { 121 { "peer_ifindex" }, 122 }; 123 124 struct veth_xdp_buff { 125 struct xdp_buff xdp; 126 struct sk_buff *skb; 127 }; 128 129 static int veth_get_link_ksettings(struct net_device *dev, 130 struct ethtool_link_ksettings *cmd) 131 { 132 cmd->base.speed = SPEED_10000; 133 cmd->base.duplex = DUPLEX_FULL; 134 cmd->base.port = PORT_TP; 135 cmd->base.autoneg = AUTONEG_DISABLE; 136 return 0; 137 } 138 139 static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 140 { 141 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 142 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 143 } 144 145 static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 146 { 147 u8 *p = buf; 148 int i, j; 149 150 switch(stringset) { 151 case ETH_SS_STATS: 152 memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 153 p += sizeof(ethtool_stats_keys); 154 for (i = 0; i < dev->real_num_rx_queues; i++) 155 for (j = 0; j < VETH_RQ_STATS_LEN; j++) 156 ethtool_sprintf(&p, "rx_queue_%u_%.18s", 157 i, veth_rq_stats_desc[j].desc); 158 159 for (i = 0; i < dev->real_num_tx_queues; i++) 160 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 161 ethtool_sprintf(&p, "tx_queue_%u_%.18s", 162 i, veth_tq_stats_desc[j].desc); 163 164 page_pool_ethtool_stats_get_strings(p); 165 break; 166 } 167 } 168 169 static int veth_get_sset_count(struct net_device *dev, int sset) 170 { 171 switch (sset) { 172 case ETH_SS_STATS: 173 return ARRAY_SIZE(ethtool_stats_keys) + 174 VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 175 VETH_TQ_STATS_LEN * dev->real_num_tx_queues + 176 page_pool_ethtool_stats_get_count(); 177 default: 178 return -EOPNOTSUPP; 179 } 180 } 181 182 static void veth_get_page_pool_stats(struct net_device *dev, u64 *data) 183 { 184 #ifdef CONFIG_PAGE_POOL_STATS 185 struct veth_priv *priv = netdev_priv(dev); 186 struct page_pool_stats pp_stats = {}; 187 int i; 188 189 for (i = 0; i < dev->real_num_rx_queues; i++) { 190 if (!priv->rq[i].page_pool) 191 continue; 192 page_pool_get_stats(priv->rq[i].page_pool, &pp_stats); 193 } 194 page_pool_ethtool_stats_get(data, &pp_stats); 195 #endif /* CONFIG_PAGE_POOL_STATS */ 196 } 197 198 static void veth_get_ethtool_stats(struct net_device *dev, 199 struct ethtool_stats *stats, u64 *data) 200 { 201 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 202 struct net_device *peer = rtnl_dereference(priv->peer); 203 int i, j, idx, pp_idx; 204 205 data[0] = peer ? peer->ifindex : 0; 206 idx = 1; 207 for (i = 0; i < dev->real_num_rx_queues; i++) { 208 const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 209 const void *stats_base = (void *)&rq_stats->vs; 210 unsigned int start; 211 size_t offset; 212 213 do { 214 start = u64_stats_fetch_begin(&rq_stats->syncp); 215 for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 216 offset = veth_rq_stats_desc[j].offset; 217 data[idx + j] = *(u64 *)(stats_base + offset); 218 } 219 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 220 idx += VETH_RQ_STATS_LEN; 221 } 222 pp_idx = idx; 223 224 if (!peer) 225 goto page_pool_stats; 226 227 rcv_priv = netdev_priv(peer); 228 for (i = 0; i < peer->real_num_rx_queues; i++) { 229 const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 230 const void *base = (void *)&rq_stats->vs; 231 unsigned int start, tx_idx = idx; 232 u64 buf[VETH_TQ_STATS_LEN]; 233 size_t offset; 234 235 do { 236 start = u64_stats_fetch_begin(&rq_stats->syncp); 237 for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 238 offset = veth_tq_stats_desc[j].offset; 239 buf[j] = *(u64 *)(base + offset); 240 } 241 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 242 243 tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 244 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 245 data[tx_idx + j] += buf[j]; 246 } 247 pp_idx = idx + dev->real_num_tx_queues * VETH_TQ_STATS_LEN; 248 249 page_pool_stats: 250 veth_get_page_pool_stats(dev, &data[pp_idx]); 251 } 252 253 static void veth_get_channels(struct net_device *dev, 254 struct ethtool_channels *channels) 255 { 256 channels->tx_count = dev->real_num_tx_queues; 257 channels->rx_count = dev->real_num_rx_queues; 258 channels->max_tx = dev->num_tx_queues; 259 channels->max_rx = dev->num_rx_queues; 260 } 261 262 static int veth_set_channels(struct net_device *dev, 263 struct ethtool_channels *ch); 264 265 static const struct ethtool_ops veth_ethtool_ops = { 266 .get_drvinfo = veth_get_drvinfo, 267 .get_link = ethtool_op_get_link, 268 .get_strings = veth_get_strings, 269 .get_sset_count = veth_get_sset_count, 270 .get_ethtool_stats = veth_get_ethtool_stats, 271 .get_link_ksettings = veth_get_link_ksettings, 272 .get_ts_info = ethtool_op_get_ts_info, 273 .get_channels = veth_get_channels, 274 .set_channels = veth_set_channels, 275 }; 276 277 /* general routines */ 278 279 static bool veth_is_xdp_frame(void *ptr) 280 { 281 return (unsigned long)ptr & VETH_XDP_FLAG; 282 } 283 284 static struct xdp_frame *veth_ptr_to_xdp(void *ptr) 285 { 286 return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 287 } 288 289 static void *veth_xdp_to_ptr(struct xdp_frame *xdp) 290 { 291 return (void *)((unsigned long)xdp | VETH_XDP_FLAG); 292 } 293 294 static void veth_ptr_free(void *ptr) 295 { 296 if (veth_is_xdp_frame(ptr)) 297 xdp_return_frame(veth_ptr_to_xdp(ptr)); 298 else 299 kfree_skb(ptr); 300 } 301 302 static void __veth_xdp_flush(struct veth_rq *rq) 303 { 304 /* Write ptr_ring before reading rx_notify_masked */ 305 smp_mb(); 306 if (!READ_ONCE(rq->rx_notify_masked) && 307 napi_schedule_prep(&rq->xdp_napi)) { 308 WRITE_ONCE(rq->rx_notify_masked, true); 309 __napi_schedule(&rq->xdp_napi); 310 } 311 } 312 313 static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 314 { 315 if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) 316 return NETDEV_TX_BUSY; /* signal qdisc layer */ 317 318 return NET_RX_SUCCESS; /* same as NETDEV_TX_OK */ 319 } 320 321 static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 322 struct veth_rq *rq, bool xdp) 323 { 324 return __dev_forward_skb(dev, skb) ?: xdp ? 325 veth_xdp_rx(rq, skb) : 326 __netif_rx(skb); 327 } 328 329 /* return true if the specified skb has chances of GRO aggregation 330 * Don't strive for accuracy, but try to avoid GRO overhead in the most 331 * common scenarios. 332 * When XDP is enabled, all traffic is considered eligible, as the xmit 333 * device has TSO off. 334 * When TSO is enabled on the xmit device, we are likely interested only 335 * in UDP aggregation, explicitly check for that if the skb is suspected 336 * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets - 337 * to belong to locally generated UDP traffic. 338 */ 339 static bool veth_skb_is_eligible_for_gro(const struct net_device *dev, 340 const struct net_device *rcv, 341 const struct sk_buff *skb) 342 { 343 return !(dev->features & NETIF_F_ALL_TSO) || 344 (skb->destructor == sock_wfree && 345 rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD)); 346 } 347 348 static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 349 { 350 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 351 struct veth_rq *rq = NULL; 352 struct netdev_queue *txq; 353 struct net_device *rcv; 354 int length = skb->len; 355 bool use_napi = false; 356 int ret, rxq; 357 358 rcu_read_lock(); 359 rcv = rcu_dereference(priv->peer); 360 if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) { 361 kfree_skb(skb); 362 goto drop; 363 } 364 365 rcv_priv = netdev_priv(rcv); 366 rxq = skb_get_queue_mapping(skb); 367 if (rxq < rcv->real_num_rx_queues) { 368 rq = &rcv_priv->rq[rxq]; 369 370 /* The napi pointer is available when an XDP program is 371 * attached or when GRO is enabled 372 * Don't bother with napi/GRO if the skb can't be aggregated 373 */ 374 use_napi = rcu_access_pointer(rq->napi) && 375 veth_skb_is_eligible_for_gro(dev, rcv, skb); 376 } 377 378 skb_tx_timestamp(skb); 379 380 ret = veth_forward_skb(rcv, skb, rq, use_napi); 381 switch (ret) { 382 case NET_RX_SUCCESS: /* same as NETDEV_TX_OK */ 383 if (!use_napi) 384 dev_sw_netstats_tx_add(dev, 1, length); 385 else 386 __veth_xdp_flush(rq); 387 break; 388 case NETDEV_TX_BUSY: 389 /* If a qdisc is attached to our virtual device, returning 390 * NETDEV_TX_BUSY is allowed. 391 */ 392 txq = netdev_get_tx_queue(dev, rxq); 393 394 if (qdisc_txq_has_no_queue(txq)) { 395 dev_kfree_skb_any(skb); 396 goto drop; 397 } 398 /* Restore Eth hdr pulled by dev_forward_skb/eth_type_trans */ 399 __skb_push(skb, ETH_HLEN); 400 netif_tx_stop_queue(txq); 401 /* Makes sure NAPI peer consumer runs. Consumer is responsible 402 * for starting txq again, until then ndo_start_xmit (this 403 * function) will not be invoked by the netstack again. 404 */ 405 __veth_xdp_flush(rq); 406 break; 407 case NET_RX_DROP: /* same as NET_XMIT_DROP */ 408 drop: 409 atomic64_inc(&priv->dropped); 410 ret = NET_XMIT_DROP; 411 break; 412 default: 413 net_crit_ratelimited("%s(%s): Invalid return code(%d)", 414 __func__, dev->name, ret); 415 } 416 rcu_read_unlock(); 417 418 return ret; 419 } 420 421 static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 422 { 423 struct veth_priv *priv = netdev_priv(dev); 424 int i; 425 426 result->peer_tq_xdp_xmit_err = 0; 427 result->xdp_packets = 0; 428 result->xdp_tx_err = 0; 429 result->xdp_bytes = 0; 430 result->rx_drops = 0; 431 for (i = 0; i < dev->num_rx_queues; i++) { 432 u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 433 struct veth_rq_stats *stats = &priv->rq[i].stats; 434 unsigned int start; 435 436 do { 437 start = u64_stats_fetch_begin(&stats->syncp); 438 peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 439 xdp_tx_err = stats->vs.xdp_tx_err; 440 packets = stats->vs.xdp_packets; 441 bytes = stats->vs.xdp_bytes; 442 drops = stats->vs.rx_drops; 443 } while (u64_stats_fetch_retry(&stats->syncp, start)); 444 result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 445 result->xdp_tx_err += xdp_tx_err; 446 result->xdp_packets += packets; 447 result->xdp_bytes += bytes; 448 result->rx_drops += drops; 449 } 450 } 451 452 static void veth_get_stats64(struct net_device *dev, 453 struct rtnl_link_stats64 *tot) 454 { 455 struct veth_priv *priv = netdev_priv(dev); 456 struct net_device *peer; 457 struct veth_stats rx; 458 459 tot->tx_dropped = atomic64_read(&priv->dropped); 460 dev_fetch_sw_netstats(tot, dev->tstats); 461 462 veth_stats_rx(&rx, dev); 463 tot->tx_dropped += rx.xdp_tx_err; 464 tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 465 tot->rx_bytes += rx.xdp_bytes; 466 tot->rx_packets += rx.xdp_packets; 467 468 rcu_read_lock(); 469 peer = rcu_dereference(priv->peer); 470 if (peer) { 471 struct rtnl_link_stats64 tot_peer = {}; 472 473 dev_fetch_sw_netstats(&tot_peer, peer->tstats); 474 tot->rx_bytes += tot_peer.tx_bytes; 475 tot->rx_packets += tot_peer.tx_packets; 476 477 veth_stats_rx(&rx, peer); 478 tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 479 tot->rx_dropped += rx.xdp_tx_err; 480 tot->tx_bytes += rx.xdp_bytes; 481 tot->tx_packets += rx.xdp_packets; 482 } 483 rcu_read_unlock(); 484 } 485 486 /* fake multicast ability */ 487 static void veth_set_multicast_list(struct net_device *dev) 488 { 489 } 490 491 static int veth_select_rxq(struct net_device *dev) 492 { 493 return smp_processor_id() % dev->real_num_rx_queues; 494 } 495 496 static struct net_device *veth_peer_dev(struct net_device *dev) 497 { 498 struct veth_priv *priv = netdev_priv(dev); 499 500 /* Callers must be under RCU read side. */ 501 return rcu_dereference(priv->peer); 502 } 503 504 static int veth_xdp_xmit(struct net_device *dev, int n, 505 struct xdp_frame **frames, 506 u32 flags, bool ndo_xmit) 507 { 508 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 509 int i, ret = -ENXIO, nxmit = 0; 510 struct net_device *rcv; 511 unsigned int max_len; 512 struct veth_rq *rq; 513 514 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 515 return -EINVAL; 516 517 rcu_read_lock(); 518 rcv = rcu_dereference(priv->peer); 519 if (unlikely(!rcv)) 520 goto out; 521 522 rcv_priv = netdev_priv(rcv); 523 rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 524 /* The napi pointer is set if NAPI is enabled, which ensures that 525 * xdp_ring is initialized on receive side and the peer device is up. 526 */ 527 if (!rcu_access_pointer(rq->napi)) 528 goto out; 529 530 max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 531 532 spin_lock(&rq->xdp_ring.producer_lock); 533 for (i = 0; i < n; i++) { 534 struct xdp_frame *frame = frames[i]; 535 void *ptr = veth_xdp_to_ptr(frame); 536 537 if (unlikely(xdp_get_frame_len(frame) > max_len || 538 __ptr_ring_produce(&rq->xdp_ring, ptr))) 539 break; 540 nxmit++; 541 } 542 spin_unlock(&rq->xdp_ring.producer_lock); 543 544 if (flags & XDP_XMIT_FLUSH) 545 __veth_xdp_flush(rq); 546 547 ret = nxmit; 548 if (ndo_xmit) { 549 u64_stats_update_begin(&rq->stats.syncp); 550 rq->stats.vs.peer_tq_xdp_xmit += nxmit; 551 rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 552 u64_stats_update_end(&rq->stats.syncp); 553 } 554 555 out: 556 rcu_read_unlock(); 557 558 return ret; 559 } 560 561 static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 562 struct xdp_frame **frames, u32 flags) 563 { 564 int err; 565 566 err = veth_xdp_xmit(dev, n, frames, flags, true); 567 if (err < 0) { 568 struct veth_priv *priv = netdev_priv(dev); 569 570 atomic64_add(n, &priv->dropped); 571 } 572 573 return err; 574 } 575 576 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 577 { 578 int sent, i, err = 0, drops; 579 580 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 581 if (sent < 0) { 582 err = sent; 583 sent = 0; 584 } 585 586 for (i = sent; unlikely(i < bq->count); i++) 587 xdp_return_frame(bq->q[i]); 588 589 drops = bq->count - sent; 590 trace_xdp_bulk_tx(rq->dev, sent, drops, err); 591 592 u64_stats_update_begin(&rq->stats.syncp); 593 rq->stats.vs.xdp_tx += sent; 594 rq->stats.vs.xdp_tx_err += drops; 595 u64_stats_update_end(&rq->stats.syncp); 596 597 bq->count = 0; 598 } 599 600 static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 601 { 602 struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 603 struct net_device *rcv; 604 struct veth_rq *rcv_rq; 605 606 rcu_read_lock(); 607 veth_xdp_flush_bq(rq, bq); 608 rcv = rcu_dereference(priv->peer); 609 if (unlikely(!rcv)) 610 goto out; 611 612 rcv_priv = netdev_priv(rcv); 613 rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 614 /* xdp_ring is initialized on receive side? */ 615 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) 616 goto out; 617 618 __veth_xdp_flush(rcv_rq); 619 out: 620 rcu_read_unlock(); 621 } 622 623 static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 624 struct veth_xdp_tx_bq *bq) 625 { 626 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 627 628 if (unlikely(!frame)) 629 return -EOVERFLOW; 630 631 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 632 veth_xdp_flush_bq(rq, bq); 633 634 bq->q[bq->count++] = frame; 635 636 return 0; 637 } 638 639 static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 640 struct xdp_frame *frame, 641 struct veth_xdp_tx_bq *bq, 642 struct veth_stats *stats) 643 { 644 struct xdp_frame orig_frame; 645 struct bpf_prog *xdp_prog; 646 647 rcu_read_lock(); 648 xdp_prog = rcu_dereference(rq->xdp_prog); 649 if (likely(xdp_prog)) { 650 struct veth_xdp_buff vxbuf; 651 struct xdp_buff *xdp = &vxbuf.xdp; 652 u32 act; 653 654 xdp_convert_frame_to_buff(frame, xdp); 655 xdp->rxq = &rq->xdp_rxq; 656 vxbuf.skb = NULL; 657 658 act = bpf_prog_run_xdp(xdp_prog, xdp); 659 660 switch (act) { 661 case XDP_PASS: 662 if (xdp_update_frame_from_buff(xdp, frame)) 663 goto err_xdp; 664 break; 665 case XDP_TX: 666 orig_frame = *frame; 667 xdp->rxq->mem.type = frame->mem_type; 668 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 669 trace_xdp_exception(rq->dev, xdp_prog, act); 670 frame = &orig_frame; 671 stats->rx_drops++; 672 goto err_xdp; 673 } 674 stats->xdp_tx++; 675 rcu_read_unlock(); 676 goto xdp_xmit; 677 case XDP_REDIRECT: 678 orig_frame = *frame; 679 xdp->rxq->mem.type = frame->mem_type; 680 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 681 frame = &orig_frame; 682 stats->rx_drops++; 683 goto err_xdp; 684 } 685 stats->xdp_redirect++; 686 rcu_read_unlock(); 687 goto xdp_xmit; 688 default: 689 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 690 fallthrough; 691 case XDP_ABORTED: 692 trace_xdp_exception(rq->dev, xdp_prog, act); 693 fallthrough; 694 case XDP_DROP: 695 stats->xdp_drops++; 696 goto err_xdp; 697 } 698 } 699 rcu_read_unlock(); 700 701 return frame; 702 err_xdp: 703 rcu_read_unlock(); 704 xdp_return_frame(frame); 705 xdp_xmit: 706 return NULL; 707 } 708 709 /* frames array contains VETH_XDP_BATCH at most */ 710 static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, 711 int n_xdpf, struct veth_xdp_tx_bq *bq, 712 struct veth_stats *stats) 713 { 714 void *skbs[VETH_XDP_BATCH]; 715 int i; 716 717 if (unlikely(!napi_skb_cache_get_bulk(skbs, n_xdpf))) { 718 for (i = 0; i < n_xdpf; i++) 719 xdp_return_frame(frames[i]); 720 stats->rx_drops += n_xdpf; 721 722 return; 723 } 724 725 for (i = 0; i < n_xdpf; i++) { 726 struct sk_buff *skb = skbs[i]; 727 728 skb = __xdp_build_skb_from_frame(frames[i], skb, 729 rq->dev); 730 if (!skb) { 731 xdp_return_frame(frames[i]); 732 stats->rx_drops++; 733 continue; 734 } 735 napi_gro_receive(&rq->xdp_napi, skb); 736 } 737 } 738 739 static void veth_xdp_get(struct xdp_buff *xdp) 740 { 741 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 742 int i; 743 744 get_page(virt_to_page(xdp->data)); 745 if (likely(!xdp_buff_has_frags(xdp))) 746 return; 747 748 for (i = 0; i < sinfo->nr_frags; i++) 749 __skb_frag_ref(&sinfo->frags[i]); 750 } 751 752 static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, 753 struct xdp_buff *xdp, 754 struct sk_buff **pskb) 755 { 756 struct sk_buff *skb = *pskb; 757 u32 frame_sz; 758 759 if (skb_shared(skb) || skb_head_is_locked(skb) || 760 skb_shinfo(skb)->nr_frags || 761 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 762 if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM)) 763 goto drop; 764 765 skb = *pskb; 766 } 767 768 /* SKB "head" area always have tailroom for skb_shared_info */ 769 frame_sz = skb_end_pointer(skb) - skb->head; 770 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 771 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 772 xdp_prepare_buff(xdp, skb->head, skb_headroom(skb), 773 skb_headlen(skb), true); 774 775 if (skb_is_nonlinear(skb)) { 776 skb_shinfo(skb)->xdp_frags_size = skb->data_len; 777 xdp_buff_set_frags_flag(xdp); 778 } else { 779 xdp_buff_clear_frags_flag(xdp); 780 } 781 *pskb = skb; 782 783 return 0; 784 drop: 785 consume_skb(skb); 786 *pskb = NULL; 787 788 return -ENOMEM; 789 } 790 791 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 792 struct sk_buff *skb, 793 struct veth_xdp_tx_bq *bq, 794 struct veth_stats *stats) 795 { 796 void *orig_data, *orig_data_end; 797 struct bpf_prog *xdp_prog; 798 struct veth_xdp_buff vxbuf; 799 struct xdp_buff *xdp = &vxbuf.xdp; 800 u32 act, metalen; 801 int off; 802 803 skb_prepare_for_gro(skb); 804 805 rcu_read_lock(); 806 xdp_prog = rcu_dereference(rq->xdp_prog); 807 if (unlikely(!xdp_prog)) { 808 rcu_read_unlock(); 809 goto out; 810 } 811 812 __skb_push(skb, skb->data - skb_mac_header(skb)); 813 if (veth_convert_skb_to_xdp_buff(rq, xdp, &skb)) 814 goto drop; 815 vxbuf.skb = skb; 816 817 orig_data = xdp->data; 818 orig_data_end = xdp->data_end; 819 820 act = bpf_prog_run_xdp(xdp_prog, xdp); 821 822 switch (act) { 823 case XDP_PASS: 824 break; 825 case XDP_TX: 826 veth_xdp_get(xdp); 827 consume_skb(skb); 828 xdp->rxq->mem = rq->xdp_mem; 829 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 830 trace_xdp_exception(rq->dev, xdp_prog, act); 831 stats->rx_drops++; 832 goto err_xdp; 833 } 834 stats->xdp_tx++; 835 rcu_read_unlock(); 836 goto xdp_xmit; 837 case XDP_REDIRECT: 838 veth_xdp_get(xdp); 839 consume_skb(skb); 840 xdp->rxq->mem = rq->xdp_mem; 841 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 842 stats->rx_drops++; 843 goto err_xdp; 844 } 845 stats->xdp_redirect++; 846 rcu_read_unlock(); 847 goto xdp_xmit; 848 default: 849 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 850 fallthrough; 851 case XDP_ABORTED: 852 trace_xdp_exception(rq->dev, xdp_prog, act); 853 fallthrough; 854 case XDP_DROP: 855 stats->xdp_drops++; 856 goto xdp_drop; 857 } 858 rcu_read_unlock(); 859 860 /* check if bpf_xdp_adjust_head was used */ 861 off = orig_data - xdp->data; 862 if (off > 0) 863 __skb_push(skb, off); 864 else if (off < 0) 865 __skb_pull(skb, -off); 866 867 skb_reset_mac_header(skb); 868 869 /* check if bpf_xdp_adjust_tail was used */ 870 off = xdp->data_end - orig_data_end; 871 if (off != 0) 872 __skb_put(skb, off); /* positive on grow, negative on shrink */ 873 874 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers 875 * (e.g. bpf_xdp_adjust_tail), we need to update data_len here. 876 */ 877 if (xdp_buff_has_frags(xdp)) 878 skb->data_len = skb_shinfo(skb)->xdp_frags_size; 879 else 880 skb->data_len = 0; 881 882 skb->protocol = eth_type_trans(skb, rq->dev); 883 884 metalen = xdp->data - xdp->data_meta; 885 if (metalen) 886 skb_metadata_set(skb, metalen); 887 out: 888 return skb; 889 drop: 890 stats->rx_drops++; 891 xdp_drop: 892 rcu_read_unlock(); 893 kfree_skb(skb); 894 return NULL; 895 err_xdp: 896 rcu_read_unlock(); 897 xdp_return_buff(xdp); 898 xdp_xmit: 899 return NULL; 900 } 901 902 static int veth_xdp_rcv(struct veth_rq *rq, int budget, 903 struct veth_xdp_tx_bq *bq, 904 struct veth_stats *stats) 905 { 906 int i, done = 0, n_xdpf = 0; 907 void *xdpf[VETH_XDP_BATCH]; 908 909 for (i = 0; i < budget; i++) { 910 void *ptr = __ptr_ring_consume(&rq->xdp_ring); 911 912 if (!ptr) 913 break; 914 915 if (veth_is_xdp_frame(ptr)) { 916 /* ndo_xdp_xmit */ 917 struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 918 919 stats->xdp_bytes += xdp_get_frame_len(frame); 920 frame = veth_xdp_rcv_one(rq, frame, bq, stats); 921 if (frame) { 922 /* XDP_PASS */ 923 xdpf[n_xdpf++] = frame; 924 if (n_xdpf == VETH_XDP_BATCH) { 925 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, 926 bq, stats); 927 n_xdpf = 0; 928 } 929 } 930 } else { 931 /* ndo_start_xmit */ 932 struct sk_buff *skb = ptr; 933 934 stats->xdp_bytes += skb->len; 935 skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 936 if (skb) { 937 if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC)) 938 netif_receive_skb(skb); 939 else 940 napi_gro_receive(&rq->xdp_napi, skb); 941 } 942 } 943 done++; 944 } 945 946 if (n_xdpf) 947 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats); 948 949 u64_stats_update_begin(&rq->stats.syncp); 950 rq->stats.vs.xdp_redirect += stats->xdp_redirect; 951 rq->stats.vs.xdp_bytes += stats->xdp_bytes; 952 rq->stats.vs.xdp_drops += stats->xdp_drops; 953 rq->stats.vs.rx_drops += stats->rx_drops; 954 rq->stats.vs.xdp_packets += done; 955 u64_stats_update_end(&rq->stats.syncp); 956 957 return done; 958 } 959 960 static int veth_poll(struct napi_struct *napi, int budget) 961 { 962 struct veth_rq *rq = 963 container_of(napi, struct veth_rq, xdp_napi); 964 struct veth_priv *priv = netdev_priv(rq->dev); 965 int queue_idx = rq->xdp_rxq.queue_index; 966 struct netdev_queue *peer_txq; 967 struct veth_stats stats = {}; 968 struct net_device *peer_dev; 969 struct veth_xdp_tx_bq bq; 970 int done; 971 972 bq.count = 0; 973 974 /* NAPI functions as RCU section */ 975 peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held()); 976 peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ? 977 netdev_get_tx_queue(peer_dev, queue_idx) : NULL; 978 979 xdp_set_return_frame_no_direct(); 980 done = veth_xdp_rcv(rq, budget, &bq, &stats); 981 982 if (stats.xdp_redirect > 0) 983 xdp_do_flush(); 984 if (stats.xdp_tx > 0) 985 veth_xdp_flush(rq, &bq); 986 xdp_clear_return_frame_no_direct(); 987 988 if (done < budget && napi_complete_done(napi, done)) { 989 /* Write rx_notify_masked before reading ptr_ring */ 990 smp_store_mb(rq->rx_notify_masked, false); 991 if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 992 if (napi_schedule_prep(&rq->xdp_napi)) { 993 WRITE_ONCE(rq->rx_notify_masked, true); 994 __napi_schedule(&rq->xdp_napi); 995 } 996 } 997 } 998 999 /* Release backpressure per NAPI poll */ 1000 smp_rmb(); /* Paired with netif_tx_stop_queue set_bit */ 1001 if (peer_txq && netif_tx_queue_stopped(peer_txq)) { 1002 txq_trans_cond_update(peer_txq); 1003 netif_tx_wake_queue(peer_txq); 1004 } 1005 1006 return done; 1007 } 1008 1009 static int veth_create_page_pool(struct veth_rq *rq) 1010 { 1011 struct page_pool_params pp_params = { 1012 .order = 0, 1013 .pool_size = VETH_RING_SIZE, 1014 .nid = NUMA_NO_NODE, 1015 .dev = &rq->dev->dev, 1016 }; 1017 1018 rq->page_pool = page_pool_create(&pp_params); 1019 if (IS_ERR(rq->page_pool)) { 1020 int err = PTR_ERR(rq->page_pool); 1021 1022 rq->page_pool = NULL; 1023 return err; 1024 } 1025 1026 return 0; 1027 } 1028 1029 static int __veth_napi_enable_range(struct net_device *dev, int start, int end) 1030 { 1031 struct veth_priv *priv = netdev_priv(dev); 1032 int err, i; 1033 1034 for (i = start; i < end; i++) { 1035 err = veth_create_page_pool(&priv->rq[i]); 1036 if (err) 1037 goto err_page_pool; 1038 } 1039 1040 for (i = start; i < end; i++) { 1041 struct veth_rq *rq = &priv->rq[i]; 1042 1043 err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 1044 if (err) 1045 goto err_xdp_ring; 1046 } 1047 1048 for (i = start; i < end; i++) { 1049 struct veth_rq *rq = &priv->rq[i]; 1050 1051 napi_enable(&rq->xdp_napi); 1052 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1053 } 1054 1055 return 0; 1056 1057 err_xdp_ring: 1058 for (i--; i >= start; i--) 1059 ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 1060 i = end; 1061 err_page_pool: 1062 for (i--; i >= start; i--) { 1063 page_pool_destroy(priv->rq[i].page_pool); 1064 priv->rq[i].page_pool = NULL; 1065 } 1066 1067 return err; 1068 } 1069 1070 static int __veth_napi_enable(struct net_device *dev) 1071 { 1072 return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1073 } 1074 1075 static void veth_napi_del_range(struct net_device *dev, int start, int end) 1076 { 1077 struct veth_priv *priv = netdev_priv(dev); 1078 int i; 1079 1080 for (i = start; i < end; i++) { 1081 struct veth_rq *rq = &priv->rq[i]; 1082 1083 rcu_assign_pointer(priv->rq[i].napi, NULL); 1084 napi_disable(&rq->xdp_napi); 1085 __netif_napi_del(&rq->xdp_napi); 1086 } 1087 synchronize_net(); 1088 1089 for (i = start; i < end; i++) { 1090 struct veth_rq *rq = &priv->rq[i]; 1091 1092 rq->rx_notify_masked = false; 1093 ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 1094 } 1095 1096 for (i = start; i < end; i++) { 1097 page_pool_destroy(priv->rq[i].page_pool); 1098 priv->rq[i].page_pool = NULL; 1099 } 1100 } 1101 1102 static void veth_napi_del(struct net_device *dev) 1103 { 1104 veth_napi_del_range(dev, 0, dev->real_num_rx_queues); 1105 } 1106 1107 static bool veth_gro_requested(const struct net_device *dev) 1108 { 1109 return !!(dev->wanted_features & NETIF_F_GRO); 1110 } 1111 1112 static int veth_enable_xdp_range(struct net_device *dev, int start, int end, 1113 bool napi_already_on) 1114 { 1115 struct veth_priv *priv = netdev_priv(dev); 1116 int err, i; 1117 1118 for (i = start; i < end; i++) { 1119 struct veth_rq *rq = &priv->rq[i]; 1120 1121 if (!napi_already_on) 1122 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1123 err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); 1124 if (err < 0) 1125 goto err_rxq_reg; 1126 1127 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1128 MEM_TYPE_PAGE_SHARED, 1129 NULL); 1130 if (err < 0) 1131 goto err_reg_mem; 1132 1133 /* Save original mem info as it can be overwritten */ 1134 rq->xdp_mem = rq->xdp_rxq.mem; 1135 } 1136 return 0; 1137 1138 err_reg_mem: 1139 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 1140 err_rxq_reg: 1141 if (!napi_already_on) 1142 netif_napi_del(&priv->rq[i].xdp_napi); 1143 for (i--; i >= start; i--) { 1144 struct veth_rq *rq = &priv->rq[i]; 1145 1146 xdp_rxq_info_unreg(&rq->xdp_rxq); 1147 if (!napi_already_on) 1148 netif_napi_del(&rq->xdp_napi); 1149 } 1150 1151 return err; 1152 } 1153 1154 static void veth_disable_xdp_range(struct net_device *dev, int start, int end, 1155 bool delete_napi) 1156 { 1157 struct veth_priv *priv = netdev_priv(dev); 1158 int i; 1159 1160 for (i = start; i < end; i++) { 1161 struct veth_rq *rq = &priv->rq[i]; 1162 1163 rq->xdp_rxq.mem = rq->xdp_mem; 1164 xdp_rxq_info_unreg(&rq->xdp_rxq); 1165 1166 if (delete_napi) 1167 netif_napi_del(&rq->xdp_napi); 1168 } 1169 } 1170 1171 static int veth_enable_xdp(struct net_device *dev) 1172 { 1173 bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP); 1174 struct veth_priv *priv = netdev_priv(dev); 1175 int err, i; 1176 1177 if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 1178 err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on); 1179 if (err) 1180 return err; 1181 1182 if (!napi_already_on) { 1183 err = __veth_napi_enable(dev); 1184 if (err) { 1185 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true); 1186 return err; 1187 } 1188 } 1189 } 1190 1191 for (i = 0; i < dev->real_num_rx_queues; i++) { 1192 rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 1193 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1194 } 1195 1196 return 0; 1197 } 1198 1199 static void veth_disable_xdp(struct net_device *dev) 1200 { 1201 struct veth_priv *priv = netdev_priv(dev); 1202 int i; 1203 1204 for (i = 0; i < dev->real_num_rx_queues; i++) 1205 rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 1206 1207 if (!netif_running(dev) || !veth_gro_requested(dev)) 1208 veth_napi_del(dev); 1209 1210 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false); 1211 } 1212 1213 static int veth_napi_enable_range(struct net_device *dev, int start, int end) 1214 { 1215 struct veth_priv *priv = netdev_priv(dev); 1216 int err, i; 1217 1218 for (i = start; i < end; i++) { 1219 struct veth_rq *rq = &priv->rq[i]; 1220 1221 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1222 } 1223 1224 err = __veth_napi_enable_range(dev, start, end); 1225 if (err) { 1226 for (i = start; i < end; i++) { 1227 struct veth_rq *rq = &priv->rq[i]; 1228 1229 netif_napi_del(&rq->xdp_napi); 1230 } 1231 return err; 1232 } 1233 return err; 1234 } 1235 1236 static int veth_napi_enable(struct net_device *dev) 1237 { 1238 return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1239 } 1240 1241 static void veth_disable_range_safe(struct net_device *dev, int start, int end) 1242 { 1243 struct veth_priv *priv = netdev_priv(dev); 1244 1245 if (start >= end) 1246 return; 1247 1248 if (priv->_xdp_prog) { 1249 veth_napi_del_range(dev, start, end); 1250 veth_disable_xdp_range(dev, start, end, false); 1251 } else if (veth_gro_requested(dev)) { 1252 veth_napi_del_range(dev, start, end); 1253 } 1254 } 1255 1256 static int veth_enable_range_safe(struct net_device *dev, int start, int end) 1257 { 1258 struct veth_priv *priv = netdev_priv(dev); 1259 int err; 1260 1261 if (start >= end) 1262 return 0; 1263 1264 if (priv->_xdp_prog) { 1265 /* these channels are freshly initialized, napi is not on there even 1266 * when GRO is requeste 1267 */ 1268 err = veth_enable_xdp_range(dev, start, end, false); 1269 if (err) 1270 return err; 1271 1272 err = __veth_napi_enable_range(dev, start, end); 1273 if (err) { 1274 /* on error always delete the newly added napis */ 1275 veth_disable_xdp_range(dev, start, end, true); 1276 return err; 1277 } 1278 } else if (veth_gro_requested(dev)) { 1279 return veth_napi_enable_range(dev, start, end); 1280 } 1281 return 0; 1282 } 1283 1284 static void veth_set_xdp_features(struct net_device *dev) 1285 { 1286 struct veth_priv *priv = netdev_priv(dev); 1287 struct net_device *peer; 1288 1289 peer = rtnl_dereference(priv->peer); 1290 if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) { 1291 struct veth_priv *priv_peer = netdev_priv(peer); 1292 xdp_features_t val = NETDEV_XDP_ACT_BASIC | 1293 NETDEV_XDP_ACT_REDIRECT | 1294 NETDEV_XDP_ACT_RX_SG; 1295 1296 if (priv_peer->_xdp_prog || veth_gro_requested(peer)) 1297 val |= NETDEV_XDP_ACT_NDO_XMIT | 1298 NETDEV_XDP_ACT_NDO_XMIT_SG; 1299 xdp_set_features_flag(dev, val); 1300 } else { 1301 xdp_clear_features_flag(dev); 1302 } 1303 } 1304 1305 static int veth_set_channels(struct net_device *dev, 1306 struct ethtool_channels *ch) 1307 { 1308 struct veth_priv *priv = netdev_priv(dev); 1309 unsigned int old_rx_count, new_rx_count; 1310 struct veth_priv *peer_priv; 1311 struct net_device *peer; 1312 int err; 1313 1314 /* sanity check. Upper bounds are already enforced by the caller */ 1315 if (!ch->rx_count || !ch->tx_count) 1316 return -EINVAL; 1317 1318 /* avoid braking XDP, if that is enabled */ 1319 peer = rtnl_dereference(priv->peer); 1320 peer_priv = peer ? netdev_priv(peer) : NULL; 1321 if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues) 1322 return -EINVAL; 1323 1324 if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues) 1325 return -EINVAL; 1326 1327 old_rx_count = dev->real_num_rx_queues; 1328 new_rx_count = ch->rx_count; 1329 if (netif_running(dev)) { 1330 /* turn device off */ 1331 netif_carrier_off(dev); 1332 if (peer) 1333 netif_carrier_off(peer); 1334 1335 /* try to allocate new resources, as needed*/ 1336 err = veth_enable_range_safe(dev, old_rx_count, new_rx_count); 1337 if (err) 1338 goto out; 1339 } 1340 1341 err = netif_set_real_num_rx_queues(dev, ch->rx_count); 1342 if (err) 1343 goto revert; 1344 1345 err = netif_set_real_num_tx_queues(dev, ch->tx_count); 1346 if (err) { 1347 int err2 = netif_set_real_num_rx_queues(dev, old_rx_count); 1348 1349 /* this error condition could happen only if rx and tx change 1350 * in opposite directions (e.g. tx nr raises, rx nr decreases) 1351 * and we can't do anything to fully restore the original 1352 * status 1353 */ 1354 if (err2) 1355 pr_warn("Can't restore rx queues config %d -> %d %d", 1356 new_rx_count, old_rx_count, err2); 1357 else 1358 goto revert; 1359 } 1360 1361 out: 1362 if (netif_running(dev)) { 1363 /* note that we need to swap the arguments WRT the enable part 1364 * to identify the range we have to disable 1365 */ 1366 veth_disable_range_safe(dev, new_rx_count, old_rx_count); 1367 netif_carrier_on(dev); 1368 if (peer) 1369 netif_carrier_on(peer); 1370 } 1371 1372 /* update XDP supported features */ 1373 veth_set_xdp_features(dev); 1374 if (peer) 1375 veth_set_xdp_features(peer); 1376 1377 return err; 1378 1379 revert: 1380 new_rx_count = old_rx_count; 1381 old_rx_count = ch->rx_count; 1382 goto out; 1383 } 1384 1385 static int veth_open(struct net_device *dev) 1386 { 1387 struct veth_priv *priv = netdev_priv(dev); 1388 struct net_device *peer = rtnl_dereference(priv->peer); 1389 int err; 1390 1391 if (!peer) 1392 return -ENOTCONN; 1393 1394 if (priv->_xdp_prog) { 1395 err = veth_enable_xdp(dev); 1396 if (err) 1397 return err; 1398 } else if (veth_gro_requested(dev)) { 1399 err = veth_napi_enable(dev); 1400 if (err) 1401 return err; 1402 } 1403 1404 if (peer->flags & IFF_UP) { 1405 netif_carrier_on(dev); 1406 netif_carrier_on(peer); 1407 } 1408 1409 veth_set_xdp_features(dev); 1410 1411 return 0; 1412 } 1413 1414 static int veth_close(struct net_device *dev) 1415 { 1416 struct veth_priv *priv = netdev_priv(dev); 1417 struct net_device *peer = rtnl_dereference(priv->peer); 1418 1419 netif_carrier_off(dev); 1420 if (peer) 1421 netif_carrier_off(peer); 1422 1423 if (priv->_xdp_prog) 1424 veth_disable_xdp(dev); 1425 else if (veth_gro_requested(dev)) 1426 veth_napi_del(dev); 1427 1428 return 0; 1429 } 1430 1431 static int is_valid_veth_mtu(int mtu) 1432 { 1433 return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 1434 } 1435 1436 static int veth_alloc_queues(struct net_device *dev) 1437 { 1438 struct veth_priv *priv = netdev_priv(dev); 1439 int i; 1440 1441 priv->rq = kvzalloc_objs(*priv->rq, dev->num_rx_queues, 1442 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 1443 if (!priv->rq) 1444 return -ENOMEM; 1445 1446 for (i = 0; i < dev->num_rx_queues; i++) { 1447 priv->rq[i].dev = dev; 1448 u64_stats_init(&priv->rq[i].stats.syncp); 1449 } 1450 1451 return 0; 1452 } 1453 1454 static void veth_free_queues(struct net_device *dev) 1455 { 1456 struct veth_priv *priv = netdev_priv(dev); 1457 1458 kvfree(priv->rq); 1459 } 1460 1461 static int veth_dev_init(struct net_device *dev) 1462 { 1463 netdev_lockdep_set_classes(dev); 1464 return veth_alloc_queues(dev); 1465 } 1466 1467 static void veth_dev_free(struct net_device *dev) 1468 { 1469 veth_free_queues(dev); 1470 } 1471 1472 #ifdef CONFIG_NET_POLL_CONTROLLER 1473 static void veth_poll_controller(struct net_device *dev) 1474 { 1475 /* veth only receives frames when its peer sends one 1476 * Since it has nothing to do with disabling irqs, we are guaranteed 1477 * never to have pending data when we poll for it so 1478 * there is nothing to do here. 1479 * 1480 * We need this though so netpoll recognizes us as an interface that 1481 * supports polling, which enables bridge devices in virt setups to 1482 * still use netconsole 1483 */ 1484 } 1485 #endif /* CONFIG_NET_POLL_CONTROLLER */ 1486 1487 static int veth_get_iflink(const struct net_device *dev) 1488 { 1489 struct veth_priv *priv = netdev_priv(dev); 1490 struct net_device *peer; 1491 int iflink; 1492 1493 rcu_read_lock(); 1494 peer = rcu_dereference(priv->peer); 1495 iflink = peer ? READ_ONCE(peer->ifindex) : 0; 1496 rcu_read_unlock(); 1497 1498 return iflink; 1499 } 1500 1501 static netdev_features_t veth_fix_features(struct net_device *dev, 1502 netdev_features_t features) 1503 { 1504 struct veth_priv *priv = netdev_priv(dev); 1505 struct net_device *peer; 1506 1507 peer = rtnl_dereference(priv->peer); 1508 if (peer) { 1509 struct veth_priv *peer_priv = netdev_priv(peer); 1510 1511 if (peer_priv->_xdp_prog) 1512 features &= ~NETIF_F_GSO_SOFTWARE; 1513 } 1514 1515 return features; 1516 } 1517 1518 static int veth_set_features(struct net_device *dev, 1519 netdev_features_t features) 1520 { 1521 netdev_features_t changed = features ^ dev->features; 1522 struct veth_priv *priv = netdev_priv(dev); 1523 struct net_device *peer; 1524 int err; 1525 1526 if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog) 1527 return 0; 1528 1529 peer = rtnl_dereference(priv->peer); 1530 if (features & NETIF_F_GRO) { 1531 err = veth_napi_enable(dev); 1532 if (err) 1533 return err; 1534 1535 if (peer) 1536 xdp_features_set_redirect_target(peer, true); 1537 } else { 1538 if (peer) 1539 xdp_features_clear_redirect_target(peer); 1540 veth_napi_del(dev); 1541 } 1542 return 0; 1543 } 1544 1545 static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1546 { 1547 struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1548 struct net_device *peer; 1549 1550 if (new_hr < 0) 1551 new_hr = 0; 1552 1553 rcu_read_lock(); 1554 peer = rcu_dereference(priv->peer); 1555 if (unlikely(!peer)) 1556 goto out; 1557 1558 peer_priv = netdev_priv(peer); 1559 priv->requested_headroom = new_hr; 1560 new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1561 dev->needed_headroom = new_hr; 1562 peer->needed_headroom = new_hr; 1563 1564 out: 1565 rcu_read_unlock(); 1566 } 1567 1568 static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1569 struct netlink_ext_ack *extack) 1570 { 1571 struct veth_priv *priv = netdev_priv(dev); 1572 struct bpf_prog *old_prog; 1573 struct net_device *peer; 1574 unsigned int max_mtu; 1575 int err; 1576 1577 old_prog = priv->_xdp_prog; 1578 priv->_xdp_prog = prog; 1579 peer = rtnl_dereference(priv->peer); 1580 1581 if (prog) { 1582 if (!peer) { 1583 NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1584 err = -ENOTCONN; 1585 goto err; 1586 } 1587 1588 max_mtu = SKB_WITH_OVERHEAD(PAGE_SIZE - VETH_XDP_HEADROOM) - 1589 peer->hard_header_len; 1590 /* Allow increasing the max_mtu if the program supports 1591 * XDP fragments. 1592 */ 1593 if (prog->aux->xdp_has_frags) 1594 max_mtu += PAGE_SIZE * MAX_SKB_FRAGS; 1595 1596 if (peer->mtu > max_mtu) { 1597 NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1598 err = -ERANGE; 1599 goto err; 1600 } 1601 1602 if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1603 NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1604 err = -ENOSPC; 1605 goto err; 1606 } 1607 1608 if (dev->flags & IFF_UP) { 1609 err = veth_enable_xdp(dev); 1610 if (err) { 1611 NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1612 goto err; 1613 } 1614 } 1615 1616 if (!old_prog) { 1617 peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1618 peer->max_mtu = max_mtu; 1619 } 1620 1621 xdp_features_set_redirect_target(peer, true); 1622 } 1623 1624 if (old_prog) { 1625 if (!prog) { 1626 if (peer && !veth_gro_requested(dev)) 1627 xdp_features_clear_redirect_target(peer); 1628 1629 if (dev->flags & IFF_UP) 1630 veth_disable_xdp(dev); 1631 1632 if (peer) { 1633 peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1634 peer->max_mtu = ETH_MAX_MTU; 1635 } 1636 } 1637 bpf_prog_put(old_prog); 1638 } 1639 1640 if ((!!old_prog ^ !!prog) && peer) 1641 netdev_update_features(peer); 1642 1643 return 0; 1644 err: 1645 priv->_xdp_prog = old_prog; 1646 1647 return err; 1648 } 1649 1650 static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1651 { 1652 switch (xdp->command) { 1653 case XDP_SETUP_PROG: 1654 return veth_xdp_set(dev, xdp->prog, xdp->extack); 1655 default: 1656 return -EINVAL; 1657 } 1658 } 1659 1660 static int veth_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp) 1661 { 1662 struct veth_xdp_buff *_ctx = (void *)ctx; 1663 1664 if (!_ctx->skb) 1665 return -ENODATA; 1666 1667 *timestamp = skb_hwtstamps(_ctx->skb)->hwtstamp; 1668 return 0; 1669 } 1670 1671 static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, 1672 enum xdp_rss_hash_type *rss_type) 1673 { 1674 struct veth_xdp_buff *_ctx = (void *)ctx; 1675 struct sk_buff *skb = _ctx->skb; 1676 1677 if (!skb) 1678 return -ENODATA; 1679 1680 *hash = skb_get_hash(skb); 1681 *rss_type = skb->l4_hash ? XDP_RSS_TYPE_L4_ANY : XDP_RSS_TYPE_NONE; 1682 1683 return 0; 1684 } 1685 1686 static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, 1687 u16 *vlan_tci) 1688 { 1689 const struct veth_xdp_buff *_ctx = (void *)ctx; 1690 const struct sk_buff *skb = _ctx->skb; 1691 int err; 1692 1693 if (!skb) 1694 return -ENODATA; 1695 1696 err = __vlan_hwaccel_get_tag(skb, vlan_tci); 1697 if (err) 1698 return err; 1699 1700 *vlan_proto = skb->vlan_proto; 1701 return err; 1702 } 1703 1704 static const struct net_device_ops veth_netdev_ops = { 1705 .ndo_init = veth_dev_init, 1706 .ndo_open = veth_open, 1707 .ndo_stop = veth_close, 1708 .ndo_start_xmit = veth_xmit, 1709 .ndo_get_stats64 = veth_get_stats64, 1710 .ndo_set_rx_mode = veth_set_multicast_list, 1711 .ndo_set_mac_address = eth_mac_addr, 1712 #ifdef CONFIG_NET_POLL_CONTROLLER 1713 .ndo_poll_controller = veth_poll_controller, 1714 #endif 1715 .ndo_get_iflink = veth_get_iflink, 1716 .ndo_fix_features = veth_fix_features, 1717 .ndo_set_features = veth_set_features, 1718 .ndo_features_check = passthru_features_check, 1719 .ndo_set_rx_headroom = veth_set_rx_headroom, 1720 .ndo_bpf = veth_xdp, 1721 .ndo_xdp_xmit = veth_ndo_xdp_xmit, 1722 .ndo_get_peer_dev = veth_peer_dev, 1723 }; 1724 1725 static const struct xdp_metadata_ops veth_xdp_metadata_ops = { 1726 .xmo_rx_timestamp = veth_xdp_rx_timestamp, 1727 .xmo_rx_hash = veth_xdp_rx_hash, 1728 .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag, 1729 }; 1730 1731 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1732 NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1733 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 1734 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 1735 NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 1736 1737 static void veth_setup(struct net_device *dev) 1738 { 1739 ether_setup(dev); 1740 1741 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1742 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1743 dev->priv_flags |= IFF_NO_QUEUE; 1744 dev->priv_flags |= IFF_PHONY_HEADROOM; 1745 dev->priv_flags |= IFF_DISABLE_NETPOLL; 1746 dev->lltx = true; 1747 1748 dev->netdev_ops = &veth_netdev_ops; 1749 dev->xdp_metadata_ops = &veth_xdp_metadata_ops; 1750 dev->ethtool_ops = &veth_ethtool_ops; 1751 dev->features |= VETH_FEATURES; 1752 dev->vlan_features = dev->features & 1753 ~(NETIF_F_HW_VLAN_CTAG_TX | 1754 NETIF_F_HW_VLAN_STAG_TX | 1755 NETIF_F_HW_VLAN_CTAG_RX | 1756 NETIF_F_HW_VLAN_STAG_RX); 1757 dev->needs_free_netdev = true; 1758 dev->priv_destructor = veth_dev_free; 1759 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 1760 dev->max_mtu = ETH_MAX_MTU; 1761 1762 dev->hw_features = VETH_FEATURES; 1763 dev->hw_enc_features = VETH_FEATURES; 1764 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1765 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 1766 } 1767 1768 /* 1769 * netlink interface 1770 */ 1771 1772 static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1773 struct netlink_ext_ack *extack) 1774 { 1775 if (tb[IFLA_ADDRESS]) { 1776 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1777 return -EINVAL; 1778 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1779 return -EADDRNOTAVAIL; 1780 } 1781 if (tb[IFLA_MTU]) { 1782 if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 1783 return -EINVAL; 1784 } 1785 return 0; 1786 } 1787 1788 static struct rtnl_link_ops veth_link_ops; 1789 1790 static void veth_disable_gro(struct net_device *dev) 1791 { 1792 dev->features &= ~NETIF_F_GRO; 1793 dev->wanted_features &= ~NETIF_F_GRO; 1794 netdev_update_features(dev); 1795 } 1796 1797 static int veth_init_queues(struct net_device *dev, struct nlattr *tb[]) 1798 { 1799 int err; 1800 1801 if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) { 1802 err = netif_set_real_num_tx_queues(dev, 1); 1803 if (err) 1804 return err; 1805 } 1806 if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) { 1807 err = netif_set_real_num_rx_queues(dev, 1); 1808 if (err) 1809 return err; 1810 } 1811 return 0; 1812 } 1813 1814 static int veth_newlink(struct net_device *dev, 1815 struct rtnl_newlink_params *params, 1816 struct netlink_ext_ack *extack) 1817 { 1818 struct net *peer_net = rtnl_newlink_peer_net(params); 1819 struct nlattr **data = params->data; 1820 struct nlattr **tb = params->tb; 1821 int err; 1822 struct net_device *peer; 1823 struct veth_priv *priv; 1824 char ifname[IFNAMSIZ]; 1825 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 1826 unsigned char name_assign_type; 1827 struct ifinfomsg *ifmp; 1828 1829 /* 1830 * create and register peer first 1831 */ 1832 if (data && data[VETH_INFO_PEER]) { 1833 struct nlattr *nla_peer = data[VETH_INFO_PEER]; 1834 1835 ifmp = nla_data(nla_peer); 1836 rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack); 1837 tbp = peer_tb; 1838 } else { 1839 ifmp = NULL; 1840 tbp = tb; 1841 } 1842 1843 if (ifmp && tbp[IFLA_IFNAME]) { 1844 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 1845 name_assign_type = NET_NAME_USER; 1846 } else { 1847 snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 1848 name_assign_type = NET_NAME_ENUM; 1849 } 1850 1851 peer = rtnl_create_link(peer_net, ifname, name_assign_type, 1852 &veth_link_ops, tbp, extack); 1853 if (IS_ERR(peer)) 1854 return PTR_ERR(peer); 1855 1856 if (!ifmp || !tbp[IFLA_ADDRESS]) 1857 eth_hw_addr_random(peer); 1858 1859 if (ifmp && (dev->ifindex != 0)) 1860 peer->ifindex = ifmp->ifi_index; 1861 1862 netif_inherit_tso_max(peer, dev); 1863 1864 err = register_netdevice(peer); 1865 if (err < 0) 1866 goto err_register_peer; 1867 1868 /* keep GRO disabled by default to be consistent with the established 1869 * veth behavior 1870 */ 1871 veth_disable_gro(peer); 1872 netif_carrier_off(peer); 1873 1874 err = rtnl_configure_link(peer, ifmp, 0, NULL); 1875 if (err < 0) 1876 goto err_configure_peer; 1877 1878 /* 1879 * register dev last 1880 * 1881 * note, that since we've registered new device the dev's name 1882 * should be re-allocated 1883 */ 1884 1885 if (tb[IFLA_ADDRESS] == NULL) 1886 eth_hw_addr_random(dev); 1887 1888 if (tb[IFLA_IFNAME]) 1889 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 1890 else 1891 snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 1892 1893 err = register_netdevice(dev); 1894 if (err < 0) 1895 goto err_register_dev; 1896 1897 netif_carrier_off(dev); 1898 1899 /* 1900 * tie the deviced together 1901 */ 1902 1903 priv = netdev_priv(dev); 1904 rcu_assign_pointer(priv->peer, peer); 1905 netdev_hold(peer, &priv->peer_tracker, GFP_KERNEL); 1906 err = veth_init_queues(dev, tb); 1907 if (err) 1908 goto err_queues; 1909 1910 priv = netdev_priv(peer); 1911 rcu_assign_pointer(priv->peer, dev); 1912 netdev_hold(dev, &priv->peer_tracker, GFP_KERNEL); 1913 err = veth_init_queues(peer, tb); 1914 if (err) 1915 goto err_peer_queues; 1916 1917 veth_disable_gro(dev); 1918 /* update XDP supported features */ 1919 veth_set_xdp_features(dev); 1920 veth_set_xdp_features(peer); 1921 1922 return 0; 1923 1924 err_peer_queues: 1925 netdev_put(dev, &priv->peer_tracker); 1926 priv = netdev_priv(dev); 1927 err_queues: 1928 netdev_put(peer, &priv->peer_tracker); 1929 unregister_netdevice(dev); 1930 err_register_dev: 1931 /* nothing to do */ 1932 err_configure_peer: 1933 unregister_netdevice(peer); 1934 return err; 1935 1936 err_register_peer: 1937 free_netdev(peer); 1938 return err; 1939 } 1940 1941 static void veth_dellink(struct net_device *dev, struct list_head *head) 1942 { 1943 netdevice_tracker *peer_tracker; 1944 struct net_device *peer; 1945 struct veth_priv *priv; 1946 1947 priv = netdev_priv(dev); 1948 peer_tracker = &priv->peer_tracker; 1949 peer = unrcu_pointer(xchg(&priv->peer, NULL)); 1950 if (!peer) 1951 return; 1952 1953 unregister_netdevice_queue(dev, head); 1954 1955 priv = netdev_priv(peer); 1956 dev = unrcu_pointer(xchg(&priv->peer, NULL)); 1957 if (dev) 1958 unregister_netdevice_queue_net(dev_net(dev), peer, head); 1959 1960 netdev_put(peer, peer_tracker); 1961 netdev_put(dev, &priv->peer_tracker); 1962 } 1963 1964 static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 1965 [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 1966 }; 1967 1968 static struct net *veth_get_link_net(const struct net_device *dev) 1969 { 1970 struct veth_priv *priv = netdev_priv(dev); 1971 struct net_device *peer = rtnl_dereference(priv->peer); 1972 1973 return peer ? dev_net(peer) : dev_net(dev); 1974 } 1975 1976 static unsigned int veth_get_num_queues(void) 1977 { 1978 /* enforce the same queue limit as rtnl_create_link */ 1979 int queues = num_possible_cpus(); 1980 1981 if (queues > 4096) 1982 queues = 4096; 1983 return queues; 1984 } 1985 1986 static struct rtnl_link_ops veth_link_ops = { 1987 .kind = DRV_NAME, 1988 .priv_size = sizeof(struct veth_priv), 1989 .setup = veth_setup, 1990 .validate = veth_validate, 1991 .newlink = veth_newlink, 1992 .dellink = veth_dellink, 1993 .policy = veth_policy, 1994 .peer_type = VETH_INFO_PEER, 1995 .maxtype = VETH_INFO_MAX, 1996 .get_link_net = veth_get_link_net, 1997 .get_num_tx_queues = veth_get_num_queues, 1998 .get_num_rx_queues = veth_get_num_queues, 1999 }; 2000 2001 /* 2002 * init/fini 2003 */ 2004 2005 static __init int veth_init(void) 2006 { 2007 return rtnl_link_register(&veth_link_ops); 2008 } 2009 2010 static __exit void veth_exit(void) 2011 { 2012 rtnl_link_unregister(&veth_link_ops); 2013 } 2014 2015 module_init(veth_init); 2016 module_exit(veth_exit); 2017 2018 MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 2019 MODULE_LICENSE("GPL v2"); 2020 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 2021