1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * drivers/net/veth.c 4 * 5 * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6 * 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9 * 10 */ 11 12 #include <linux/netdevice.h> 13 #include <linux/slab.h> 14 #include <linux/ethtool.h> 15 #include <linux/etherdevice.h> 16 #include <linux/u64_stats_sync.h> 17 18 #include <net/rtnetlink.h> 19 #include <net/dst.h> 20 #include <net/netdev_lock.h> 21 #include <net/xfrm.h> 22 #include <net/xdp.h> 23 #include <linux/veth.h> 24 #include <linux/module.h> 25 #include <linux/bpf.h> 26 #include <linux/filter.h> 27 #include <linux/ptr_ring.h> 28 #include <linux/bpf_trace.h> 29 #include <linux/net_tstamp.h> 30 #include <linux/skbuff_ref.h> 31 #include <net/page_pool/helpers.h> 32 33 #define DRV_NAME "veth" 34 #define DRV_VERSION "1.0" 35 36 #define VETH_XDP_FLAG BIT(0) 37 #define VETH_RING_SIZE 256 38 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 39 40 #define VETH_XDP_TX_BULK_SIZE 16 41 #define VETH_XDP_BATCH 16 42 43 struct veth_stats { 44 u64 rx_drops; 45 /* xdp */ 46 u64 xdp_packets; 47 u64 xdp_bytes; 48 u64 xdp_redirect; 49 u64 xdp_drops; 50 u64 xdp_tx; 51 u64 xdp_tx_err; 52 u64 peer_tq_xdp_xmit; 53 u64 peer_tq_xdp_xmit_err; 54 }; 55 56 struct veth_rq_stats { 57 struct veth_stats vs; 58 struct u64_stats_sync syncp; 59 }; 60 61 struct veth_rq { 62 struct napi_struct xdp_napi; 63 struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */ 64 struct net_device *dev; 65 struct bpf_prog __rcu *xdp_prog; 66 struct xdp_mem_info xdp_mem; 67 struct veth_rq_stats stats; 68 bool rx_notify_masked; 69 struct ptr_ring xdp_ring; 70 struct xdp_rxq_info xdp_rxq; 71 struct page_pool *page_pool; 72 }; 73 74 struct veth_priv { 75 struct net_device __rcu *peer; 76 atomic64_t dropped; 77 struct bpf_prog *_xdp_prog; 78 struct veth_rq *rq; 79 unsigned int requested_headroom; 80 netdevice_tracker peer_tracker; 81 }; 82 83 struct veth_xdp_tx_bq { 84 struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 85 unsigned int count; 86 }; 87 88 /* 89 * ethtool interface 90 */ 91 92 struct veth_q_stat_desc { 93 char desc[ETH_GSTRING_LEN]; 94 size_t offset; 95 }; 96 97 #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 98 99 static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 100 { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 101 { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 102 { "drops", VETH_RQ_STAT(rx_drops) }, 103 { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 104 { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 105 { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 106 { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 107 }; 108 109 #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 110 111 static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 112 { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 113 { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 114 }; 115 116 #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 117 118 static struct { 119 const char string[ETH_GSTRING_LEN]; 120 } ethtool_stats_keys[] = { 121 { "peer_ifindex" }, 122 }; 123 124 struct veth_xdp_buff { 125 struct xdp_buff xdp; 126 struct sk_buff *skb; 127 }; 128 129 static int veth_get_link_ksettings(struct net_device *dev, 130 struct ethtool_link_ksettings *cmd) 131 { 132 cmd->base.speed = SPEED_10000; 133 cmd->base.duplex = DUPLEX_FULL; 134 cmd->base.port = PORT_TP; 135 cmd->base.autoneg = AUTONEG_DISABLE; 136 return 0; 137 } 138 139 static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 140 { 141 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 142 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 143 } 144 145 static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 146 { 147 u8 *p = buf; 148 int i, j; 149 150 switch(stringset) { 151 case ETH_SS_STATS: 152 memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 153 p += sizeof(ethtool_stats_keys); 154 for (i = 0; i < dev->real_num_rx_queues; i++) 155 for (j = 0; j < VETH_RQ_STATS_LEN; j++) 156 ethtool_sprintf(&p, "rx_queue_%u_%.18s", 157 i, veth_rq_stats_desc[j].desc); 158 159 for (i = 0; i < dev->real_num_tx_queues; i++) 160 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 161 ethtool_sprintf(&p, "tx_queue_%u_%.18s", 162 i, veth_tq_stats_desc[j].desc); 163 164 page_pool_ethtool_stats_get_strings(p); 165 break; 166 } 167 } 168 169 static int veth_get_sset_count(struct net_device *dev, int sset) 170 { 171 switch (sset) { 172 case ETH_SS_STATS: 173 return ARRAY_SIZE(ethtool_stats_keys) + 174 VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 175 VETH_TQ_STATS_LEN * dev->real_num_tx_queues + 176 page_pool_ethtool_stats_get_count(); 177 default: 178 return -EOPNOTSUPP; 179 } 180 } 181 182 static void veth_get_page_pool_stats(struct net_device *dev, u64 *data) 183 { 184 #ifdef CONFIG_PAGE_POOL_STATS 185 struct veth_priv *priv = netdev_priv(dev); 186 struct page_pool_stats pp_stats = {}; 187 int i; 188 189 for (i = 0; i < dev->real_num_rx_queues; i++) { 190 if (!priv->rq[i].page_pool) 191 continue; 192 page_pool_get_stats(priv->rq[i].page_pool, &pp_stats); 193 } 194 page_pool_ethtool_stats_get(data, &pp_stats); 195 #endif /* CONFIG_PAGE_POOL_STATS */ 196 } 197 198 static void veth_get_ethtool_stats(struct net_device *dev, 199 struct ethtool_stats *stats, u64 *data) 200 { 201 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 202 struct net_device *peer = rtnl_dereference(priv->peer); 203 int i, j, idx, pp_idx; 204 205 data[0] = peer ? peer->ifindex : 0; 206 idx = 1; 207 for (i = 0; i < dev->real_num_rx_queues; i++) { 208 const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 209 const void *stats_base = (void *)&rq_stats->vs; 210 unsigned int start; 211 size_t offset; 212 213 do { 214 start = u64_stats_fetch_begin(&rq_stats->syncp); 215 for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 216 offset = veth_rq_stats_desc[j].offset; 217 data[idx + j] = *(u64 *)(stats_base + offset); 218 } 219 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 220 idx += VETH_RQ_STATS_LEN; 221 } 222 pp_idx = idx; 223 224 if (!peer) 225 goto page_pool_stats; 226 227 rcv_priv = netdev_priv(peer); 228 for (i = 0; i < peer->real_num_rx_queues; i++) { 229 const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 230 const void *base = (void *)&rq_stats->vs; 231 unsigned int start, tx_idx = idx; 232 u64 buf[VETH_TQ_STATS_LEN]; 233 size_t offset; 234 235 do { 236 start = u64_stats_fetch_begin(&rq_stats->syncp); 237 for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 238 offset = veth_tq_stats_desc[j].offset; 239 buf[j] = *(u64 *)(base + offset); 240 } 241 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 242 243 tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 244 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 245 data[tx_idx + j] += buf[j]; 246 } 247 pp_idx = idx + dev->real_num_tx_queues * VETH_TQ_STATS_LEN; 248 249 page_pool_stats: 250 veth_get_page_pool_stats(dev, &data[pp_idx]); 251 } 252 253 static void veth_get_channels(struct net_device *dev, 254 struct ethtool_channels *channels) 255 { 256 channels->tx_count = dev->real_num_tx_queues; 257 channels->rx_count = dev->real_num_rx_queues; 258 channels->max_tx = dev->num_tx_queues; 259 channels->max_rx = dev->num_rx_queues; 260 } 261 262 static int veth_set_channels(struct net_device *dev, 263 struct ethtool_channels *ch); 264 265 static const struct ethtool_ops veth_ethtool_ops = { 266 .get_drvinfo = veth_get_drvinfo, 267 .get_link = ethtool_op_get_link, 268 .get_strings = veth_get_strings, 269 .get_sset_count = veth_get_sset_count, 270 .get_ethtool_stats = veth_get_ethtool_stats, 271 .get_link_ksettings = veth_get_link_ksettings, 272 .get_ts_info = ethtool_op_get_ts_info, 273 .get_channels = veth_get_channels, 274 .set_channels = veth_set_channels, 275 }; 276 277 /* general routines */ 278 279 static bool veth_is_xdp_frame(void *ptr) 280 { 281 return (unsigned long)ptr & VETH_XDP_FLAG; 282 } 283 284 static struct xdp_frame *veth_ptr_to_xdp(void *ptr) 285 { 286 return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 287 } 288 289 static void *veth_xdp_to_ptr(struct xdp_frame *xdp) 290 { 291 return (void *)((unsigned long)xdp | VETH_XDP_FLAG); 292 } 293 294 static void veth_ptr_free(void *ptr) 295 { 296 if (veth_is_xdp_frame(ptr)) 297 xdp_return_frame(veth_ptr_to_xdp(ptr)); 298 else 299 kfree_skb(ptr); 300 } 301 302 static void __veth_xdp_flush(struct veth_rq *rq) 303 { 304 /* Write ptr_ring before reading rx_notify_masked */ 305 smp_mb(); 306 if (!READ_ONCE(rq->rx_notify_masked) && 307 napi_schedule_prep(&rq->xdp_napi)) { 308 WRITE_ONCE(rq->rx_notify_masked, true); 309 __napi_schedule(&rq->xdp_napi); 310 } 311 } 312 313 static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 314 { 315 if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) 316 return NETDEV_TX_BUSY; /* signal qdisc layer */ 317 318 return NET_RX_SUCCESS; /* same as NETDEV_TX_OK */ 319 } 320 321 static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 322 struct veth_rq *rq, bool xdp) 323 { 324 return __dev_forward_skb(dev, skb) ?: xdp ? 325 veth_xdp_rx(rq, skb) : 326 __netif_rx(skb); 327 } 328 329 /* return true if the specified skb has chances of GRO aggregation 330 * Don't strive for accuracy, but try to avoid GRO overhead in the most 331 * common scenarios. 332 * When XDP is enabled, all traffic is considered eligible, as the xmit 333 * device has TSO off. 334 * When TSO is enabled on the xmit device, we are likely interested only 335 * in UDP aggregation, explicitly check for that if the skb is suspected 336 * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets - 337 * to belong to locally generated UDP traffic. 338 */ 339 static bool veth_skb_is_eligible_for_gro(const struct net_device *dev, 340 const struct net_device *rcv, 341 const struct sk_buff *skb) 342 { 343 return !(dev->features & NETIF_F_ALL_TSO) || 344 (skb->destructor == sock_wfree && 345 rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD)); 346 } 347 348 static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 349 { 350 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 351 struct veth_rq *rq = NULL; 352 struct netdev_queue *txq; 353 struct net_device *rcv; 354 int length = skb->len; 355 bool use_napi = false; 356 int ret, rxq; 357 358 rcu_read_lock(); 359 rcv = rcu_dereference(priv->peer); 360 if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) { 361 kfree_skb(skb); 362 goto drop; 363 } 364 365 rcv_priv = netdev_priv(rcv); 366 rxq = skb_get_queue_mapping(skb); 367 if (rxq < rcv->real_num_rx_queues) { 368 rq = &rcv_priv->rq[rxq]; 369 370 /* The napi pointer is available when an XDP program is 371 * attached or when GRO is enabled 372 * Don't bother with napi/GRO if the skb can't be aggregated 373 */ 374 use_napi = rcu_access_pointer(rq->napi) && 375 veth_skb_is_eligible_for_gro(dev, rcv, skb); 376 } 377 378 skb_tx_timestamp(skb); 379 380 /* tx queue index of this device, meaningless as rx queue of the peer */ 381 skb_set_queue_mapping(skb, 0); 382 383 ret = veth_forward_skb(rcv, skb, rq, use_napi); 384 switch (ret) { 385 case NET_RX_SUCCESS: /* same as NETDEV_TX_OK */ 386 if (!use_napi) 387 dev_sw_netstats_tx_add(dev, 1, length); 388 else 389 __veth_xdp_flush(rq); 390 break; 391 case NETDEV_TX_BUSY: 392 /* If a qdisc is attached to our virtual device, returning 393 * NETDEV_TX_BUSY is allowed. 394 */ 395 txq = netdev_get_tx_queue(dev, rxq); 396 397 if (qdisc_txq_has_no_queue(txq)) { 398 dev_kfree_skb_any(skb); 399 goto drop; 400 } 401 /* Restore Eth hdr pulled by dev_forward_skb/eth_type_trans */ 402 __skb_push(skb, ETH_HLEN); 403 /* qdisc requeues the skb on the txq it points to */ 404 skb_set_queue_mapping(skb, rxq); 405 netif_tx_stop_queue(txq); 406 /* Makes sure NAPI peer consumer runs. Consumer is responsible 407 * for starting txq again, until then ndo_start_xmit (this 408 * function) will not be invoked by the netstack again. 409 */ 410 __veth_xdp_flush(rq); 411 break; 412 case NET_RX_DROP: /* same as NET_XMIT_DROP */ 413 drop: 414 atomic64_inc(&priv->dropped); 415 ret = NET_XMIT_DROP; 416 break; 417 default: 418 net_crit_ratelimited("%s(%s): Invalid return code(%d)", 419 __func__, dev->name, ret); 420 } 421 rcu_read_unlock(); 422 423 return ret; 424 } 425 426 static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 427 { 428 struct veth_priv *priv = netdev_priv(dev); 429 int i; 430 431 result->peer_tq_xdp_xmit_err = 0; 432 result->xdp_packets = 0; 433 result->xdp_tx_err = 0; 434 result->xdp_bytes = 0; 435 result->rx_drops = 0; 436 for (i = 0; i < dev->num_rx_queues; i++) { 437 u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 438 struct veth_rq_stats *stats = &priv->rq[i].stats; 439 unsigned int start; 440 441 do { 442 start = u64_stats_fetch_begin(&stats->syncp); 443 peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 444 xdp_tx_err = stats->vs.xdp_tx_err; 445 packets = stats->vs.xdp_packets; 446 bytes = stats->vs.xdp_bytes; 447 drops = stats->vs.rx_drops; 448 } while (u64_stats_fetch_retry(&stats->syncp, start)); 449 result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 450 result->xdp_tx_err += xdp_tx_err; 451 result->xdp_packets += packets; 452 result->xdp_bytes += bytes; 453 result->rx_drops += drops; 454 } 455 } 456 457 static void veth_get_stats64(struct net_device *dev, 458 struct rtnl_link_stats64 *tot) 459 { 460 struct veth_priv *priv = netdev_priv(dev); 461 struct net_device *peer; 462 struct veth_stats rx; 463 464 tot->tx_dropped = atomic64_read(&priv->dropped); 465 dev_fetch_sw_netstats(tot, dev->tstats); 466 467 veth_stats_rx(&rx, dev); 468 tot->tx_dropped += rx.xdp_tx_err; 469 tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 470 tot->rx_bytes += rx.xdp_bytes; 471 tot->rx_packets += rx.xdp_packets; 472 473 rcu_read_lock(); 474 peer = rcu_dereference(priv->peer); 475 if (peer) { 476 struct rtnl_link_stats64 tot_peer = {}; 477 478 dev_fetch_sw_netstats(&tot_peer, peer->tstats); 479 tot->rx_bytes += tot_peer.tx_bytes; 480 tot->rx_packets += tot_peer.tx_packets; 481 482 veth_stats_rx(&rx, peer); 483 tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 484 tot->rx_dropped += rx.xdp_tx_err; 485 tot->tx_bytes += rx.xdp_bytes; 486 tot->tx_packets += rx.xdp_packets; 487 } 488 rcu_read_unlock(); 489 } 490 491 /* fake multicast ability */ 492 static void veth_set_multicast_list(struct net_device *dev) 493 { 494 } 495 496 static int veth_select_rxq(struct net_device *dev) 497 { 498 return smp_processor_id() % dev->real_num_rx_queues; 499 } 500 501 static struct net_device *veth_peer_dev(struct net_device *dev) 502 { 503 struct veth_priv *priv = netdev_priv(dev); 504 505 /* Callers must be under RCU read side. */ 506 return rcu_dereference(priv->peer); 507 } 508 509 static int veth_xdp_xmit(struct net_device *dev, int n, 510 struct xdp_frame **frames, 511 u32 flags, bool ndo_xmit) 512 { 513 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 514 int i, ret = -ENXIO, nxmit = 0; 515 struct net_device *rcv; 516 unsigned int max_len; 517 struct veth_rq *rq; 518 519 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 520 return -EINVAL; 521 522 rcu_read_lock(); 523 rcv = rcu_dereference(priv->peer); 524 if (unlikely(!rcv)) 525 goto out; 526 527 rcv_priv = netdev_priv(rcv); 528 rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 529 /* The napi pointer is set if NAPI is enabled, which ensures that 530 * xdp_ring is initialized on receive side and the peer device is up. 531 */ 532 if (!rcu_access_pointer(rq->napi)) 533 goto out; 534 535 max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 536 537 spin_lock(&rq->xdp_ring.producer_lock); 538 for (i = 0; i < n; i++) { 539 struct xdp_frame *frame = frames[i]; 540 void *ptr = veth_xdp_to_ptr(frame); 541 542 if (unlikely(xdp_get_frame_len(frame) > max_len || 543 __ptr_ring_produce(&rq->xdp_ring, ptr))) 544 break; 545 nxmit++; 546 } 547 spin_unlock(&rq->xdp_ring.producer_lock); 548 549 if (flags & XDP_XMIT_FLUSH) 550 __veth_xdp_flush(rq); 551 552 ret = nxmit; 553 if (ndo_xmit) { 554 u64_stats_update_begin(&rq->stats.syncp); 555 rq->stats.vs.peer_tq_xdp_xmit += nxmit; 556 rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 557 u64_stats_update_end(&rq->stats.syncp); 558 } 559 560 out: 561 rcu_read_unlock(); 562 563 return ret; 564 } 565 566 static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 567 struct xdp_frame **frames, u32 flags) 568 { 569 int err; 570 571 err = veth_xdp_xmit(dev, n, frames, flags, true); 572 if (err < 0) { 573 struct veth_priv *priv = netdev_priv(dev); 574 575 atomic64_add(n, &priv->dropped); 576 } 577 578 return err; 579 } 580 581 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 582 { 583 int sent, i, err = 0, drops; 584 585 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 586 if (sent < 0) { 587 err = sent; 588 sent = 0; 589 } 590 591 for (i = sent; unlikely(i < bq->count); i++) 592 xdp_return_frame(bq->q[i]); 593 594 drops = bq->count - sent; 595 trace_xdp_bulk_tx(rq->dev, sent, drops, err); 596 597 u64_stats_update_begin(&rq->stats.syncp); 598 rq->stats.vs.xdp_tx += sent; 599 rq->stats.vs.xdp_tx_err += drops; 600 u64_stats_update_end(&rq->stats.syncp); 601 602 bq->count = 0; 603 } 604 605 static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 606 { 607 struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 608 struct net_device *rcv; 609 struct veth_rq *rcv_rq; 610 611 rcu_read_lock(); 612 veth_xdp_flush_bq(rq, bq); 613 rcv = rcu_dereference(priv->peer); 614 if (unlikely(!rcv)) 615 goto out; 616 617 rcv_priv = netdev_priv(rcv); 618 rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 619 /* xdp_ring is initialized on receive side? */ 620 if (unlikely(!rcu_access_pointer(rcv_rq->napi))) 621 goto out; 622 623 __veth_xdp_flush(rcv_rq); 624 out: 625 rcu_read_unlock(); 626 } 627 628 static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 629 struct veth_xdp_tx_bq *bq) 630 { 631 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 632 633 if (unlikely(!frame)) 634 return -EOVERFLOW; 635 636 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 637 veth_xdp_flush_bq(rq, bq); 638 639 bq->q[bq->count++] = frame; 640 641 return 0; 642 } 643 644 static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 645 struct xdp_frame *frame, 646 struct veth_xdp_tx_bq *bq, 647 struct veth_stats *stats) 648 { 649 struct xdp_frame orig_frame; 650 struct bpf_prog *xdp_prog; 651 652 rcu_read_lock(); 653 xdp_prog = rcu_dereference(rq->xdp_prog); 654 if (likely(xdp_prog)) { 655 struct veth_xdp_buff vxbuf; 656 struct xdp_buff *xdp = &vxbuf.xdp; 657 u32 act; 658 659 xdp_convert_frame_to_buff(frame, xdp); 660 xdp->rxq = &rq->xdp_rxq; 661 vxbuf.skb = NULL; 662 663 act = bpf_prog_run_xdp(xdp_prog, xdp); 664 665 switch (act) { 666 case XDP_PASS: 667 if (xdp_update_frame_from_buff(xdp, frame)) 668 goto err_xdp; 669 break; 670 case XDP_TX: 671 orig_frame = *frame; 672 xdp->rxq->mem.type = frame->mem_type; 673 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 674 trace_xdp_exception(rq->dev, xdp_prog, act); 675 frame = &orig_frame; 676 stats->rx_drops++; 677 goto err_xdp; 678 } 679 stats->xdp_tx++; 680 rcu_read_unlock(); 681 goto xdp_xmit; 682 case XDP_REDIRECT: 683 orig_frame = *frame; 684 xdp->rxq->mem.type = frame->mem_type; 685 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 686 frame = &orig_frame; 687 stats->rx_drops++; 688 goto err_xdp; 689 } 690 stats->xdp_redirect++; 691 rcu_read_unlock(); 692 goto xdp_xmit; 693 default: 694 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 695 fallthrough; 696 case XDP_ABORTED: 697 trace_xdp_exception(rq->dev, xdp_prog, act); 698 fallthrough; 699 case XDP_DROP: 700 stats->xdp_drops++; 701 goto err_xdp; 702 } 703 } 704 rcu_read_unlock(); 705 706 return frame; 707 err_xdp: 708 rcu_read_unlock(); 709 xdp_return_frame(frame); 710 xdp_xmit: 711 return NULL; 712 } 713 714 /* frames array contains VETH_XDP_BATCH at most */ 715 static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, 716 int n_xdpf, struct veth_xdp_tx_bq *bq, 717 struct veth_stats *stats) 718 { 719 void *skbs[VETH_XDP_BATCH]; 720 int i; 721 722 if (unlikely(!napi_skb_cache_get_bulk(skbs, n_xdpf))) { 723 for (i = 0; i < n_xdpf; i++) 724 xdp_return_frame(frames[i]); 725 stats->rx_drops += n_xdpf; 726 727 return; 728 } 729 730 for (i = 0; i < n_xdpf; i++) { 731 struct sk_buff *skb = skbs[i]; 732 733 skb = __xdp_build_skb_from_frame(frames[i], skb, 734 rq->dev); 735 if (!skb) { 736 xdp_return_frame(frames[i]); 737 stats->rx_drops++; 738 continue; 739 } 740 napi_gro_receive(&rq->xdp_napi, skb); 741 } 742 } 743 744 static void veth_xdp_get(struct xdp_buff *xdp) 745 { 746 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 747 int i; 748 749 get_page(virt_to_page(xdp->data)); 750 if (likely(!xdp_buff_has_frags(xdp))) 751 return; 752 753 for (i = 0; i < sinfo->nr_frags; i++) 754 netmem_inc_ref(skb_frag_netmem(&sinfo->frags[i])); 755 } 756 757 static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, 758 struct xdp_buff *xdp, 759 struct sk_buff **pskb) 760 { 761 struct sk_buff *skb = *pskb; 762 u32 frame_sz; 763 764 if (skb_shared(skb) || skb_head_is_locked(skb) || 765 skb_is_nonlinear(skb) || 766 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 767 if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM)) 768 goto drop; 769 770 skb = *pskb; 771 } 772 773 /* SKB "head" area always have tailroom for skb_shared_info */ 774 frame_sz = skb_end_pointer(skb) - skb->head; 775 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 776 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 777 xdp_prepare_buff(xdp, skb->head, skb_headroom(skb), 778 skb_headlen(skb), true); 779 780 if (skb_shinfo(skb)->nr_frags) { 781 skb_shinfo(skb)->xdp_frags_size = skb->data_len; 782 xdp_buff_set_frags_flag(xdp); 783 } else { 784 xdp_buff_clear_frags_flag(xdp); 785 } 786 *pskb = skb; 787 788 return 0; 789 drop: 790 consume_skb(skb); 791 *pskb = NULL; 792 793 return -ENOMEM; 794 } 795 796 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 797 struct sk_buff *skb, 798 struct veth_xdp_tx_bq *bq, 799 struct veth_stats *stats) 800 { 801 void *orig_data, *orig_data_end; 802 struct bpf_prog *xdp_prog; 803 struct veth_xdp_buff vxbuf; 804 struct xdp_buff *xdp = &vxbuf.xdp; 805 u32 act, metalen; 806 int off; 807 808 skb_prepare_for_gro(skb); 809 810 rcu_read_lock(); 811 xdp_prog = rcu_dereference(rq->xdp_prog); 812 if (unlikely(!xdp_prog)) { 813 rcu_read_unlock(); 814 goto out; 815 } 816 817 __skb_push(skb, skb->data - skb_mac_header(skb)); 818 if (veth_convert_skb_to_xdp_buff(rq, xdp, &skb)) 819 goto drop; 820 vxbuf.skb = skb; 821 822 orig_data = xdp->data; 823 orig_data_end = xdp->data_end; 824 825 act = bpf_prog_run_xdp(xdp_prog, xdp); 826 827 switch (act) { 828 case XDP_PASS: 829 break; 830 case XDP_TX: 831 veth_xdp_get(xdp); 832 consume_skb(skb); 833 xdp->rxq->mem = rq->xdp_mem; 834 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 835 trace_xdp_exception(rq->dev, xdp_prog, act); 836 stats->rx_drops++; 837 goto err_xdp; 838 } 839 stats->xdp_tx++; 840 rcu_read_unlock(); 841 goto xdp_xmit; 842 case XDP_REDIRECT: 843 veth_xdp_get(xdp); 844 consume_skb(skb); 845 xdp->rxq->mem = rq->xdp_mem; 846 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 847 stats->rx_drops++; 848 goto err_xdp; 849 } 850 stats->xdp_redirect++; 851 rcu_read_unlock(); 852 goto xdp_xmit; 853 default: 854 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 855 fallthrough; 856 case XDP_ABORTED: 857 trace_xdp_exception(rq->dev, xdp_prog, act); 858 fallthrough; 859 case XDP_DROP: 860 stats->xdp_drops++; 861 goto xdp_drop; 862 } 863 rcu_read_unlock(); 864 865 /* check if bpf_xdp_adjust_head was used */ 866 off = orig_data - xdp->data; 867 if (off > 0) 868 __skb_push(skb, off); 869 else if (off < 0) 870 __skb_pull(skb, -off); 871 872 skb_reset_mac_header(skb); 873 874 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers 875 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution 876 * from skb->len before updating data_len, then add the new one back. 877 */ 878 skb->len -= skb->data_len; 879 if (xdp_buff_has_frags(xdp)) { 880 skb->data_len = skb_shinfo(skb)->xdp_frags_size; 881 skb->len += skb->data_len; 882 } else { 883 skb->data_len = 0; 884 } 885 886 /* Synchronize the skb tail with XDP's updated linear area. */ 887 off = xdp->data_end - orig_data_end; 888 if (off != 0) { 889 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 890 skb->len += off; /* positive on grow, negative on shrink */ 891 } 892 893 skb->protocol = eth_type_trans(skb, rq->dev); 894 895 metalen = xdp->data - xdp->data_meta; 896 if (metalen) 897 skb_metadata_set(skb, metalen); 898 out: 899 return skb; 900 drop: 901 stats->rx_drops++; 902 xdp_drop: 903 rcu_read_unlock(); 904 kfree_skb(skb); 905 return NULL; 906 err_xdp: 907 rcu_read_unlock(); 908 xdp_return_buff(xdp); 909 xdp_xmit: 910 return NULL; 911 } 912 913 static int veth_xdp_rcv(struct veth_rq *rq, int budget, 914 struct veth_xdp_tx_bq *bq, 915 struct veth_stats *stats) 916 { 917 int i, done = 0, n_xdpf = 0; 918 void *xdpf[VETH_XDP_BATCH]; 919 920 for (i = 0; i < budget; i++) { 921 void *ptr = __ptr_ring_consume(&rq->xdp_ring); 922 923 if (!ptr) 924 break; 925 926 if (veth_is_xdp_frame(ptr)) { 927 /* ndo_xdp_xmit */ 928 struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 929 930 stats->xdp_bytes += xdp_get_frame_len(frame); 931 frame = veth_xdp_rcv_one(rq, frame, bq, stats); 932 if (frame) { 933 /* XDP_PASS */ 934 xdpf[n_xdpf++] = frame; 935 if (n_xdpf == VETH_XDP_BATCH) { 936 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, 937 bq, stats); 938 n_xdpf = 0; 939 } 940 } 941 } else { 942 /* ndo_start_xmit */ 943 struct sk_buff *skb = ptr; 944 945 stats->xdp_bytes += skb->len; 946 skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 947 if (skb) { 948 if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC)) 949 netif_receive_skb(skb); 950 else 951 napi_gro_receive(&rq->xdp_napi, skb); 952 } 953 } 954 done++; 955 } 956 957 if (n_xdpf) 958 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats); 959 960 u64_stats_update_begin(&rq->stats.syncp); 961 rq->stats.vs.xdp_redirect += stats->xdp_redirect; 962 rq->stats.vs.xdp_bytes += stats->xdp_bytes; 963 rq->stats.vs.xdp_drops += stats->xdp_drops; 964 rq->stats.vs.rx_drops += stats->rx_drops; 965 rq->stats.vs.xdp_packets += done; 966 u64_stats_update_end(&rq->stats.syncp); 967 968 return done; 969 } 970 971 static int veth_poll(struct napi_struct *napi, int budget) 972 { 973 struct veth_rq *rq = 974 container_of(napi, struct veth_rq, xdp_napi); 975 struct veth_priv *priv = netdev_priv(rq->dev); 976 int queue_idx = rq - priv->rq; 977 struct netdev_queue *peer_txq; 978 struct veth_stats stats = {}; 979 struct net_device *peer_dev; 980 struct veth_xdp_tx_bq bq; 981 int done; 982 983 bq.count = 0; 984 985 /* NAPI functions as RCU section */ 986 peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held()); 987 peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ? 988 netdev_get_tx_queue(peer_dev, queue_idx) : NULL; 989 990 xdp_set_return_frame_no_direct(); 991 done = veth_xdp_rcv(rq, budget, &bq, &stats); 992 993 if (stats.xdp_redirect > 0) 994 xdp_do_flush(); 995 if (stats.xdp_tx > 0) 996 veth_xdp_flush(rq, &bq); 997 xdp_clear_return_frame_no_direct(); 998 999 if (done < budget && napi_complete_done(napi, done)) { 1000 /* Write rx_notify_masked before reading ptr_ring */ 1001 smp_store_mb(rq->rx_notify_masked, false); 1002 if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 1003 if (napi_schedule_prep(&rq->xdp_napi)) { 1004 WRITE_ONCE(rq->rx_notify_masked, true); 1005 __napi_schedule(&rq->xdp_napi); 1006 } 1007 } 1008 } 1009 1010 /* Release backpressure per NAPI poll */ 1011 smp_rmb(); /* Paired with netif_tx_stop_queue set_bit */ 1012 if (peer_txq && netif_tx_queue_stopped(peer_txq)) { 1013 txq_trans_cond_update(peer_txq); 1014 netif_tx_wake_queue(peer_txq); 1015 } 1016 1017 return done; 1018 } 1019 1020 static int veth_create_page_pool(struct veth_rq *rq) 1021 { 1022 struct page_pool_params pp_params = { 1023 .order = 0, 1024 .pool_size = VETH_RING_SIZE, 1025 .nid = NUMA_NO_NODE, 1026 .dev = &rq->dev->dev, 1027 }; 1028 1029 rq->page_pool = page_pool_create(&pp_params); 1030 if (IS_ERR(rq->page_pool)) { 1031 int err = PTR_ERR(rq->page_pool); 1032 1033 rq->page_pool = NULL; 1034 return err; 1035 } 1036 1037 return 0; 1038 } 1039 1040 static int __veth_napi_enable_range(struct net_device *dev, int start, int end) 1041 { 1042 struct veth_priv *priv = netdev_priv(dev); 1043 int err, i; 1044 1045 for (i = start; i < end; i++) { 1046 err = veth_create_page_pool(&priv->rq[i]); 1047 if (err) 1048 goto err_page_pool; 1049 } 1050 1051 for (i = start; i < end; i++) { 1052 struct veth_rq *rq = &priv->rq[i]; 1053 1054 err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 1055 if (err) 1056 goto err_xdp_ring; 1057 } 1058 1059 for (i = start; i < end; i++) { 1060 struct veth_rq *rq = &priv->rq[i]; 1061 1062 rcu_assign_pointer(rq->xdp_prog, priv->_xdp_prog); 1063 napi_enable(&rq->xdp_napi); 1064 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1065 } 1066 1067 return 0; 1068 1069 err_xdp_ring: 1070 for (i--; i >= start; i--) 1071 ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 1072 i = end; 1073 err_page_pool: 1074 for (i--; i >= start; i--) { 1075 page_pool_destroy(priv->rq[i].page_pool); 1076 priv->rq[i].page_pool = NULL; 1077 } 1078 1079 return err; 1080 } 1081 1082 static int __veth_napi_enable(struct net_device *dev) 1083 { 1084 return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1085 } 1086 1087 static void veth_napi_del_range(struct net_device *dev, int start, int end) 1088 { 1089 struct veth_priv *priv = netdev_priv(dev); 1090 int i; 1091 1092 for (i = start; i < end; i++) { 1093 struct veth_rq *rq = &priv->rq[i]; 1094 1095 rcu_assign_pointer(priv->rq[i].napi, NULL); 1096 napi_disable(&rq->xdp_napi); 1097 rcu_assign_pointer(rq->xdp_prog, NULL); 1098 __netif_napi_del(&rq->xdp_napi); 1099 } 1100 synchronize_net(); 1101 1102 for (i = start; i < end; i++) { 1103 struct veth_rq *rq = &priv->rq[i]; 1104 1105 rq->rx_notify_masked = false; 1106 ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 1107 } 1108 1109 for (i = start; i < end; i++) { 1110 page_pool_destroy(priv->rq[i].page_pool); 1111 priv->rq[i].page_pool = NULL; 1112 } 1113 } 1114 1115 static void veth_napi_del(struct net_device *dev) 1116 { 1117 veth_napi_del_range(dev, 0, dev->real_num_rx_queues); 1118 } 1119 1120 static bool veth_gro_requested(const struct net_device *dev) 1121 { 1122 return !!(dev->wanted_features & NETIF_F_GRO); 1123 } 1124 1125 static int veth_enable_xdp_range(struct net_device *dev, int start, int end, 1126 bool napi_already_on) 1127 { 1128 struct veth_priv *priv = netdev_priv(dev); 1129 int err, i; 1130 1131 for (i = start; i < end; i++) { 1132 struct veth_rq *rq = &priv->rq[i]; 1133 1134 if (!napi_already_on) 1135 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1136 err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); 1137 if (err < 0) 1138 goto err_rxq_reg; 1139 1140 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1141 MEM_TYPE_PAGE_SHARED, 1142 NULL); 1143 if (err < 0) 1144 goto err_reg_mem; 1145 1146 /* Save original mem info as it can be overwritten */ 1147 rq->xdp_mem = rq->xdp_rxq.mem; 1148 } 1149 return 0; 1150 1151 err_reg_mem: 1152 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 1153 err_rxq_reg: 1154 if (!napi_already_on) 1155 netif_napi_del(&priv->rq[i].xdp_napi); 1156 for (i--; i >= start; i--) { 1157 struct veth_rq *rq = &priv->rq[i]; 1158 1159 xdp_rxq_info_unreg(&rq->xdp_rxq); 1160 if (!napi_already_on) 1161 netif_napi_del(&rq->xdp_napi); 1162 } 1163 1164 return err; 1165 } 1166 1167 static void veth_disable_xdp_range(struct net_device *dev, int start, int end, 1168 bool delete_napi) 1169 { 1170 struct veth_priv *priv = netdev_priv(dev); 1171 int i; 1172 1173 for (i = start; i < end; i++) { 1174 struct veth_rq *rq = &priv->rq[i]; 1175 1176 rq->xdp_rxq.mem = rq->xdp_mem; 1177 xdp_rxq_info_unreg(&rq->xdp_rxq); 1178 1179 if (delete_napi) 1180 netif_napi_del(&rq->xdp_napi); 1181 } 1182 } 1183 1184 static int veth_enable_xdp(struct net_device *dev) 1185 { 1186 bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP); 1187 struct veth_priv *priv = netdev_priv(dev); 1188 int err, i; 1189 1190 if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 1191 err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on); 1192 if (err) 1193 return err; 1194 1195 if (!napi_already_on) { 1196 err = __veth_napi_enable(dev); 1197 if (err) { 1198 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true); 1199 return err; 1200 } 1201 } 1202 } 1203 1204 for (i = 0; i < dev->real_num_rx_queues; i++) { 1205 rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 1206 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1207 } 1208 1209 return 0; 1210 } 1211 1212 static void veth_disable_xdp(struct net_device *dev) 1213 { 1214 struct veth_priv *priv = netdev_priv(dev); 1215 int i; 1216 1217 for (i = 0; i < dev->real_num_rx_queues; i++) 1218 rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 1219 1220 if (!netif_running(dev) || !veth_gro_requested(dev)) 1221 veth_napi_del(dev); 1222 1223 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false); 1224 } 1225 1226 static int veth_napi_enable_range(struct net_device *dev, int start, int end) 1227 { 1228 struct veth_priv *priv = netdev_priv(dev); 1229 int err, i; 1230 1231 for (i = start; i < end; i++) { 1232 struct veth_rq *rq = &priv->rq[i]; 1233 1234 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1235 } 1236 1237 err = __veth_napi_enable_range(dev, start, end); 1238 if (err) { 1239 for (i = start; i < end; i++) { 1240 struct veth_rq *rq = &priv->rq[i]; 1241 1242 netif_napi_del(&rq->xdp_napi); 1243 } 1244 return err; 1245 } 1246 return err; 1247 } 1248 1249 static int veth_napi_enable(struct net_device *dev) 1250 { 1251 return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1252 } 1253 1254 static void veth_disable_range_safe(struct net_device *dev, int start, int end) 1255 { 1256 struct veth_priv *priv = netdev_priv(dev); 1257 1258 if (start >= end) 1259 return; 1260 1261 if (priv->_xdp_prog) { 1262 veth_napi_del_range(dev, start, end); 1263 veth_disable_xdp_range(dev, start, end, false); 1264 } else if (veth_gro_requested(dev)) { 1265 veth_napi_del_range(dev, start, end); 1266 } 1267 } 1268 1269 static int veth_enable_range_safe(struct net_device *dev, int start, int end) 1270 { 1271 struct veth_priv *priv = netdev_priv(dev); 1272 int err; 1273 1274 if (start >= end) 1275 return 0; 1276 1277 if (priv->_xdp_prog) { 1278 /* these channels are freshly initialized, napi is not on there even 1279 * when GRO is requeste 1280 */ 1281 err = veth_enable_xdp_range(dev, start, end, false); 1282 if (err) 1283 return err; 1284 1285 err = __veth_napi_enable_range(dev, start, end); 1286 if (err) { 1287 /* on error always delete the newly added napis */ 1288 veth_disable_xdp_range(dev, start, end, true); 1289 return err; 1290 } 1291 } else if (veth_gro_requested(dev)) { 1292 return veth_napi_enable_range(dev, start, end); 1293 } 1294 return 0; 1295 } 1296 1297 static void veth_set_xdp_features(struct net_device *dev) 1298 { 1299 struct veth_priv *priv = netdev_priv(dev); 1300 struct net_device *peer; 1301 1302 peer = rtnl_dereference(priv->peer); 1303 if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) { 1304 struct veth_priv *priv_peer = netdev_priv(peer); 1305 xdp_features_t val = NETDEV_XDP_ACT_BASIC | 1306 NETDEV_XDP_ACT_REDIRECT | 1307 NETDEV_XDP_ACT_RX_SG; 1308 1309 if (priv_peer->_xdp_prog || veth_gro_requested(peer)) 1310 val |= NETDEV_XDP_ACT_NDO_XMIT | 1311 NETDEV_XDP_ACT_NDO_XMIT_SG; 1312 xdp_set_features_flag(dev, val); 1313 } else { 1314 xdp_clear_features_flag(dev); 1315 } 1316 } 1317 1318 static int veth_set_channels(struct net_device *dev, 1319 struct ethtool_channels *ch) 1320 { 1321 struct veth_priv *priv = netdev_priv(dev); 1322 unsigned int old_rx_count, new_rx_count; 1323 struct veth_priv *peer_priv; 1324 struct net_device *peer; 1325 int err; 1326 1327 /* sanity check. Upper bounds are already enforced by the caller */ 1328 if (!ch->rx_count || !ch->tx_count) 1329 return -EINVAL; 1330 1331 /* avoid braking XDP, if that is enabled */ 1332 peer = rtnl_dereference(priv->peer); 1333 peer_priv = peer ? netdev_priv(peer) : NULL; 1334 if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues) 1335 return -EINVAL; 1336 1337 if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues) 1338 return -EINVAL; 1339 1340 old_rx_count = dev->real_num_rx_queues; 1341 new_rx_count = ch->rx_count; 1342 if (netif_running(dev)) { 1343 /* turn device off */ 1344 netif_carrier_off(dev); 1345 if (peer) 1346 netif_carrier_off(peer); 1347 1348 /* try to allocate new resources, as needed*/ 1349 err = veth_enable_range_safe(dev, old_rx_count, new_rx_count); 1350 if (err) 1351 goto out; 1352 } 1353 1354 err = netif_set_real_num_rx_queues(dev, ch->rx_count); 1355 if (err) 1356 goto revert; 1357 1358 err = netif_set_real_num_tx_queues(dev, ch->tx_count); 1359 if (err) { 1360 int err2 = netif_set_real_num_rx_queues(dev, old_rx_count); 1361 1362 /* this error condition could happen only if rx and tx change 1363 * in opposite directions (e.g. tx nr raises, rx nr decreases) 1364 * and we can't do anything to fully restore the original 1365 * status 1366 */ 1367 if (err2) 1368 pr_warn("Can't restore rx queues config %d -> %d %d", 1369 new_rx_count, old_rx_count, err2); 1370 else 1371 goto revert; 1372 } 1373 1374 out: 1375 if (netif_running(dev)) { 1376 /* note that we need to swap the arguments WRT the enable part 1377 * to identify the range we have to disable 1378 */ 1379 veth_disable_range_safe(dev, new_rx_count, old_rx_count); 1380 netif_carrier_on(dev); 1381 if (peer) 1382 netif_carrier_on(peer); 1383 } 1384 1385 /* update XDP supported features */ 1386 veth_set_xdp_features(dev); 1387 if (peer) 1388 veth_set_xdp_features(peer); 1389 1390 return err; 1391 1392 revert: 1393 new_rx_count = old_rx_count; 1394 old_rx_count = ch->rx_count; 1395 goto out; 1396 } 1397 1398 static int veth_open(struct net_device *dev) 1399 { 1400 struct veth_priv *priv = netdev_priv(dev); 1401 struct net_device *peer = rtnl_dereference(priv->peer); 1402 int err; 1403 1404 if (!peer) 1405 return -ENOTCONN; 1406 1407 if (priv->_xdp_prog) { 1408 err = veth_enable_xdp(dev); 1409 if (err) 1410 return err; 1411 } else if (veth_gro_requested(dev)) { 1412 err = veth_napi_enable(dev); 1413 if (err) 1414 return err; 1415 } 1416 1417 if (peer->flags & IFF_UP) { 1418 netif_carrier_on(dev); 1419 netif_carrier_on(peer); 1420 } 1421 1422 veth_set_xdp_features(dev); 1423 veth_set_xdp_features(peer); 1424 1425 return 0; 1426 } 1427 1428 static int veth_close(struct net_device *dev) 1429 { 1430 struct veth_priv *priv = netdev_priv(dev); 1431 struct net_device *peer = rtnl_dereference(priv->peer); 1432 1433 netif_carrier_off(dev); 1434 if (peer) 1435 netif_carrier_off(peer); 1436 1437 if (priv->_xdp_prog) 1438 veth_disable_xdp(dev); 1439 else if (veth_gro_requested(dev)) 1440 veth_napi_del(dev); 1441 1442 return 0; 1443 } 1444 1445 static int is_valid_veth_mtu(int mtu) 1446 { 1447 return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 1448 } 1449 1450 static int veth_alloc_queues(struct net_device *dev) 1451 { 1452 struct veth_priv *priv = netdev_priv(dev); 1453 int i; 1454 1455 priv->rq = kvzalloc_objs(*priv->rq, dev->num_rx_queues, 1456 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 1457 if (!priv->rq) 1458 return -ENOMEM; 1459 1460 for (i = 0; i < dev->num_rx_queues; i++) { 1461 priv->rq[i].dev = dev; 1462 u64_stats_init(&priv->rq[i].stats.syncp); 1463 } 1464 1465 return 0; 1466 } 1467 1468 static void veth_free_queues(struct net_device *dev) 1469 { 1470 struct veth_priv *priv = netdev_priv(dev); 1471 1472 kvfree(priv->rq); 1473 } 1474 1475 static int veth_dev_init(struct net_device *dev) 1476 { 1477 netdev_lockdep_set_classes(dev); 1478 return veth_alloc_queues(dev); 1479 } 1480 1481 static void veth_dev_free(struct net_device *dev) 1482 { 1483 veth_free_queues(dev); 1484 } 1485 1486 #ifdef CONFIG_NET_POLL_CONTROLLER 1487 static void veth_poll_controller(struct net_device *dev) 1488 { 1489 /* veth only receives frames when its peer sends one 1490 * Since it has nothing to do with disabling irqs, we are guaranteed 1491 * never to have pending data when we poll for it so 1492 * there is nothing to do here. 1493 * 1494 * We need this though so netpoll recognizes us as an interface that 1495 * supports polling, which enables bridge devices in virt setups to 1496 * still use netconsole 1497 */ 1498 } 1499 #endif /* CONFIG_NET_POLL_CONTROLLER */ 1500 1501 static int veth_get_iflink(const struct net_device *dev) 1502 { 1503 struct veth_priv *priv = netdev_priv(dev); 1504 struct net_device *peer; 1505 int iflink; 1506 1507 rcu_read_lock(); 1508 peer = rcu_dereference(priv->peer); 1509 iflink = peer ? READ_ONCE(peer->ifindex) : 0; 1510 rcu_read_unlock(); 1511 1512 return iflink; 1513 } 1514 1515 static netdev_features_t veth_fix_features(struct net_device *dev, 1516 netdev_features_t features) 1517 { 1518 struct veth_priv *priv = netdev_priv(dev); 1519 struct net_device *peer; 1520 1521 peer = rtnl_dereference(priv->peer); 1522 if (peer) { 1523 struct veth_priv *peer_priv = netdev_priv(peer); 1524 1525 if (peer_priv->_xdp_prog) 1526 features &= ~NETIF_F_GSO_SOFTWARE; 1527 } 1528 1529 return features; 1530 } 1531 1532 static int veth_set_features(struct net_device *dev, 1533 netdev_features_t features) 1534 { 1535 netdev_features_t changed = features ^ dev->features; 1536 struct veth_priv *priv = netdev_priv(dev); 1537 struct net_device *peer; 1538 int err; 1539 1540 if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog) 1541 return 0; 1542 1543 peer = rtnl_dereference(priv->peer); 1544 if (features & NETIF_F_GRO) { 1545 err = veth_napi_enable(dev); 1546 if (err) 1547 return err; 1548 1549 if (peer) 1550 xdp_features_set_redirect_target(peer, true); 1551 } else { 1552 if (peer) 1553 xdp_features_clear_redirect_target(peer); 1554 veth_napi_del(dev); 1555 } 1556 return 0; 1557 } 1558 1559 static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1560 { 1561 struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1562 struct net_device *peer; 1563 1564 if (new_hr < 0) 1565 new_hr = 0; 1566 1567 rcu_read_lock(); 1568 peer = rcu_dereference(priv->peer); 1569 if (unlikely(!peer)) 1570 goto out; 1571 1572 peer_priv = netdev_priv(peer); 1573 priv->requested_headroom = new_hr; 1574 new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1575 dev->needed_headroom = new_hr; 1576 peer->needed_headroom = new_hr; 1577 1578 out: 1579 rcu_read_unlock(); 1580 } 1581 1582 static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1583 struct netlink_ext_ack *extack) 1584 { 1585 struct veth_priv *priv = netdev_priv(dev); 1586 struct bpf_prog *old_prog; 1587 struct net_device *peer; 1588 unsigned int max_mtu; 1589 int err; 1590 1591 old_prog = priv->_xdp_prog; 1592 priv->_xdp_prog = prog; 1593 peer = rtnl_dereference(priv->peer); 1594 1595 if (prog) { 1596 if (!peer) { 1597 NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1598 err = -ENOTCONN; 1599 goto err; 1600 } 1601 1602 max_mtu = SKB_WITH_OVERHEAD(PAGE_SIZE - VETH_XDP_HEADROOM) - 1603 peer->hard_header_len; 1604 /* Allow increasing the max_mtu if the program supports 1605 * XDP fragments. 1606 */ 1607 if (prog->aux->xdp_has_frags) 1608 max_mtu += PAGE_SIZE * MAX_SKB_FRAGS; 1609 1610 if (peer->mtu > max_mtu) { 1611 NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1612 err = -ERANGE; 1613 goto err; 1614 } 1615 1616 if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1617 NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1618 err = -ENOSPC; 1619 goto err; 1620 } 1621 1622 if (dev->flags & IFF_UP) { 1623 err = veth_enable_xdp(dev); 1624 if (err) { 1625 NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1626 goto err; 1627 } 1628 } 1629 1630 if (!old_prog) 1631 peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1632 peer->max_mtu = max_mtu; 1633 1634 xdp_features_set_redirect_target(peer, true); 1635 } 1636 1637 if (old_prog) { 1638 if (!prog) { 1639 if (peer && !veth_gro_requested(dev)) 1640 xdp_features_clear_redirect_target(peer); 1641 1642 if (dev->flags & IFF_UP) 1643 veth_disable_xdp(dev); 1644 1645 if (peer) { 1646 peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1647 peer->max_mtu = ETH_MAX_MTU; 1648 } 1649 } 1650 bpf_prog_put(old_prog); 1651 } 1652 1653 if ((!!old_prog ^ !!prog) && peer) 1654 netdev_update_features(peer); 1655 1656 return 0; 1657 err: 1658 priv->_xdp_prog = old_prog; 1659 1660 return err; 1661 } 1662 1663 static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1664 { 1665 switch (xdp->command) { 1666 case XDP_SETUP_PROG: 1667 return veth_xdp_set(dev, xdp->prog, xdp->extack); 1668 default: 1669 return -EINVAL; 1670 } 1671 } 1672 1673 static int veth_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp) 1674 { 1675 struct veth_xdp_buff *_ctx = (void *)ctx; 1676 1677 if (!_ctx->skb) 1678 return -ENODATA; 1679 1680 *timestamp = skb_hwtstamps(_ctx->skb)->hwtstamp; 1681 return 0; 1682 } 1683 1684 static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, 1685 enum xdp_rss_hash_type *rss_type) 1686 { 1687 struct veth_xdp_buff *_ctx = (void *)ctx; 1688 struct sk_buff *skb = _ctx->skb; 1689 1690 if (!skb) 1691 return -ENODATA; 1692 1693 *hash = skb_get_hash(skb); 1694 *rss_type = skb->l4_hash ? XDP_RSS_TYPE_L4_ANY : XDP_RSS_TYPE_NONE; 1695 1696 return 0; 1697 } 1698 1699 static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, 1700 u16 *vlan_tci) 1701 { 1702 const struct veth_xdp_buff *_ctx = (void *)ctx; 1703 const struct sk_buff *skb = _ctx->skb; 1704 int err; 1705 1706 if (!skb) 1707 return -ENODATA; 1708 1709 err = __vlan_hwaccel_get_tag(skb, vlan_tci); 1710 if (err) 1711 return err; 1712 1713 *vlan_proto = skb->vlan_proto; 1714 return err; 1715 } 1716 1717 static const struct net_device_ops veth_netdev_ops = { 1718 .ndo_init = veth_dev_init, 1719 .ndo_open = veth_open, 1720 .ndo_stop = veth_close, 1721 .ndo_start_xmit = veth_xmit, 1722 .ndo_get_stats64 = veth_get_stats64, 1723 .ndo_set_rx_mode = veth_set_multicast_list, 1724 .ndo_set_mac_address = eth_mac_addr, 1725 #ifdef CONFIG_NET_POLL_CONTROLLER 1726 .ndo_poll_controller = veth_poll_controller, 1727 #endif 1728 .ndo_get_iflink = veth_get_iflink, 1729 .ndo_fix_features = veth_fix_features, 1730 .ndo_set_features = veth_set_features, 1731 .ndo_features_check = passthru_features_check, 1732 .ndo_set_rx_headroom = veth_set_rx_headroom, 1733 .ndo_bpf = veth_xdp, 1734 .ndo_xdp_xmit = veth_ndo_xdp_xmit, 1735 .ndo_get_peer_dev = veth_peer_dev, 1736 }; 1737 1738 static const struct xdp_metadata_ops veth_xdp_metadata_ops = { 1739 .xmo_rx_timestamp = veth_xdp_rx_timestamp, 1740 .xmo_rx_hash = veth_xdp_rx_hash, 1741 .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag, 1742 }; 1743 1744 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1745 NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1746 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 1747 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 1748 NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 1749 1750 static void veth_setup(struct net_device *dev) 1751 { 1752 ether_setup(dev); 1753 1754 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1755 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1756 dev->priv_flags |= IFF_NO_QUEUE; 1757 dev->priv_flags |= IFF_PHONY_HEADROOM; 1758 dev->priv_flags |= IFF_DISABLE_NETPOLL; 1759 dev->lltx = true; 1760 1761 dev->netdev_ops = &veth_netdev_ops; 1762 dev->xdp_metadata_ops = &veth_xdp_metadata_ops; 1763 dev->ethtool_ops = &veth_ethtool_ops; 1764 dev->features |= VETH_FEATURES; 1765 dev->vlan_features = dev->features & 1766 ~(NETIF_F_HW_VLAN_CTAG_TX | 1767 NETIF_F_HW_VLAN_STAG_TX | 1768 NETIF_F_HW_VLAN_CTAG_RX | 1769 NETIF_F_HW_VLAN_STAG_RX); 1770 dev->needs_free_netdev = true; 1771 dev->priv_destructor = veth_dev_free; 1772 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 1773 dev->max_mtu = ETH_MAX_MTU; 1774 1775 dev->hw_features = VETH_FEATURES; 1776 dev->hw_enc_features = VETH_FEATURES; 1777 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1778 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 1779 } 1780 1781 /* 1782 * netlink interface 1783 */ 1784 1785 static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1786 struct netlink_ext_ack *extack) 1787 { 1788 if (tb[IFLA_ADDRESS]) { 1789 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1790 return -EINVAL; 1791 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1792 return -EADDRNOTAVAIL; 1793 } 1794 if (tb[IFLA_MTU]) { 1795 if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 1796 return -EINVAL; 1797 } 1798 return 0; 1799 } 1800 1801 static struct rtnl_link_ops veth_link_ops; 1802 1803 static void veth_disable_gro(struct net_device *dev) 1804 { 1805 dev->features &= ~NETIF_F_GRO; 1806 dev->wanted_features &= ~NETIF_F_GRO; 1807 netdev_update_features(dev); 1808 } 1809 1810 static int veth_init_queues(struct net_device *dev, struct nlattr *tb[]) 1811 { 1812 int err; 1813 1814 if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) { 1815 err = netif_set_real_num_tx_queues(dev, 1); 1816 if (err) 1817 return err; 1818 } 1819 if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) { 1820 err = netif_set_real_num_rx_queues(dev, 1); 1821 if (err) 1822 return err; 1823 } 1824 return 0; 1825 } 1826 1827 static int veth_newlink(struct net_device *dev, 1828 struct rtnl_newlink_params *params, 1829 struct netlink_ext_ack *extack) 1830 { 1831 struct net *peer_net = rtnl_newlink_peer_net(params); 1832 struct nlattr **data = params->data; 1833 struct nlattr **tb = params->tb; 1834 int err; 1835 struct net_device *peer; 1836 struct veth_priv *priv; 1837 char ifname[IFNAMSIZ]; 1838 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 1839 unsigned char name_assign_type; 1840 struct ifinfomsg *ifmp; 1841 1842 /* 1843 * create and register peer first 1844 */ 1845 if (data && data[VETH_INFO_PEER]) { 1846 struct nlattr *nla_peer = data[VETH_INFO_PEER]; 1847 1848 ifmp = nla_data(nla_peer); 1849 rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack); 1850 tbp = peer_tb; 1851 } else { 1852 ifmp = NULL; 1853 tbp = tb; 1854 } 1855 1856 if (ifmp && tbp[IFLA_IFNAME]) { 1857 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 1858 name_assign_type = NET_NAME_USER; 1859 } else { 1860 snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 1861 name_assign_type = NET_NAME_ENUM; 1862 } 1863 1864 peer = rtnl_create_link(peer_net, ifname, name_assign_type, 1865 &veth_link_ops, tbp, extack); 1866 if (IS_ERR(peer)) 1867 return PTR_ERR(peer); 1868 1869 if (!ifmp || !tbp[IFLA_ADDRESS]) 1870 eth_hw_addr_random(peer); 1871 1872 if (ifmp && (dev->ifindex != 0)) 1873 peer->ifindex = ifmp->ifi_index; 1874 1875 netif_inherit_tso_max(peer, dev); 1876 1877 err = register_netdevice(peer); 1878 if (err < 0) 1879 goto err_register_peer; 1880 1881 /* keep GRO disabled by default to be consistent with the established 1882 * veth behavior 1883 */ 1884 veth_disable_gro(peer); 1885 netif_carrier_off(peer); 1886 1887 err = rtnl_configure_link(peer, ifmp, 0, NULL); 1888 if (err < 0) 1889 goto err_configure_peer; 1890 1891 /* 1892 * register dev last 1893 * 1894 * note, that since we've registered new device the dev's name 1895 * should be re-allocated 1896 */ 1897 1898 if (tb[IFLA_ADDRESS] == NULL) 1899 eth_hw_addr_random(dev); 1900 1901 if (tb[IFLA_IFNAME]) 1902 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 1903 else 1904 snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 1905 1906 err = register_netdevice(dev); 1907 if (err < 0) 1908 goto err_register_dev; 1909 1910 netif_carrier_off(dev); 1911 1912 /* 1913 * tie the deviced together 1914 */ 1915 1916 priv = netdev_priv(dev); 1917 rcu_assign_pointer(priv->peer, peer); 1918 netdev_hold(peer, &priv->peer_tracker, GFP_KERNEL); 1919 err = veth_init_queues(dev, tb); 1920 if (err) 1921 goto err_queues; 1922 1923 priv = netdev_priv(peer); 1924 rcu_assign_pointer(priv->peer, dev); 1925 netdev_hold(dev, &priv->peer_tracker, GFP_KERNEL); 1926 err = veth_init_queues(peer, tbp); 1927 if (err) 1928 goto err_peer_queues; 1929 1930 veth_disable_gro(dev); 1931 /* update XDP supported features */ 1932 veth_set_xdp_features(dev); 1933 veth_set_xdp_features(peer); 1934 1935 return 0; 1936 1937 err_peer_queues: 1938 netdev_put(dev, &priv->peer_tracker); 1939 priv = netdev_priv(dev); 1940 err_queues: 1941 netdev_put(peer, &priv->peer_tracker); 1942 unregister_netdevice(dev); 1943 err_register_dev: 1944 /* nothing to do */ 1945 err_configure_peer: 1946 unregister_netdevice(peer); 1947 return err; 1948 1949 err_register_peer: 1950 free_netdev(peer); 1951 return err; 1952 } 1953 1954 static void veth_dellink(struct net_device *dev, struct list_head *head) 1955 { 1956 netdevice_tracker *peer_tracker; 1957 struct net_device *peer; 1958 struct veth_priv *priv; 1959 1960 priv = netdev_priv(dev); 1961 peer_tracker = &priv->peer_tracker; 1962 peer = unrcu_pointer(xchg(&priv->peer, NULL)); 1963 if (!peer) 1964 return; 1965 1966 unregister_netdevice_queue(dev, head); 1967 1968 priv = netdev_priv(peer); 1969 dev = unrcu_pointer(xchg(&priv->peer, NULL)); 1970 if (dev) 1971 unregister_netdevice_queue_net(dev_net(dev), peer, head); 1972 1973 netdev_put(peer, peer_tracker); 1974 netdev_put(dev, &priv->peer_tracker); 1975 } 1976 1977 static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 1978 [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 1979 }; 1980 1981 static struct net *veth_get_link_net(const struct net_device *dev) 1982 { 1983 struct veth_priv *priv = netdev_priv(dev); 1984 struct net_device *peer = rtnl_dereference(priv->peer); 1985 1986 return peer ? dev_net(peer) : dev_net(dev); 1987 } 1988 1989 static unsigned int veth_get_num_queues(void) 1990 { 1991 /* enforce the same queue limit as rtnl_create_link */ 1992 int queues = num_possible_cpus(); 1993 1994 if (queues > 4096) 1995 queues = 4096; 1996 return queues; 1997 } 1998 1999 static struct rtnl_link_ops veth_link_ops = { 2000 .kind = DRV_NAME, 2001 .priv_size = sizeof(struct veth_priv), 2002 .setup = veth_setup, 2003 .validate = veth_validate, 2004 .newlink = veth_newlink, 2005 .dellink = veth_dellink, 2006 .policy = veth_policy, 2007 .peer_type = VETH_INFO_PEER, 2008 .maxtype = VETH_INFO_MAX, 2009 .get_link_net = veth_get_link_net, 2010 .get_num_tx_queues = veth_get_num_queues, 2011 .get_num_rx_queues = veth_get_num_queues, 2012 }; 2013 2014 /* 2015 * init/fini 2016 */ 2017 2018 static __init int veth_init(void) 2019 { 2020 return rtnl_link_register(&veth_link_ops); 2021 } 2022 2023 static __exit void veth_exit(void) 2024 { 2025 rtnl_link_unregister(&veth_link_ops); 2026 } 2027 2028 module_init(veth_init); 2029 module_exit(veth_exit); 2030 2031 MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 2032 MODULE_LICENSE("GPL v2"); 2033 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 2034