1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * drivers/net/veth.c 4 * 5 * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6 * 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9 * 10 */ 11 12 #include <linux/netdevice.h> 13 #include <linux/slab.h> 14 #include <linux/ethtool.h> 15 #include <linux/etherdevice.h> 16 #include <linux/u64_stats_sync.h> 17 18 #include <net/rtnetlink.h> 19 #include <net/dst.h> 20 #include <net/netdev_lock.h> 21 #include <net/xfrm.h> 22 #include <net/xdp.h> 23 #include <linux/veth.h> 24 #include <linux/module.h> 25 #include <linux/bpf.h> 26 #include <linux/filter.h> 27 #include <linux/ptr_ring.h> 28 #include <linux/bpf_trace.h> 29 #include <linux/net_tstamp.h> 30 #include <linux/skbuff_ref.h> 31 #include <net/page_pool/helpers.h> 32 33 #define DRV_NAME "veth" 34 #define DRV_VERSION "1.0" 35 36 #define VETH_XDP_FLAG BIT(0) 37 #define VETH_RING_SIZE 256 38 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 39 40 #define VETH_XDP_TX_BULK_SIZE 16 41 #define VETH_XDP_BATCH 16 42 43 struct veth_stats { 44 u64 rx_drops; 45 /* xdp */ 46 u64 xdp_packets; 47 u64 xdp_bytes; 48 u64 xdp_redirect; 49 u64 xdp_drops; 50 u64 xdp_tx; 51 u64 xdp_tx_err; 52 u64 peer_tq_xdp_xmit; 53 u64 peer_tq_xdp_xmit_err; 54 }; 55 56 struct veth_rq_stats { 57 struct veth_stats vs; 58 struct u64_stats_sync syncp; 59 }; 60 61 struct veth_rq { 62 struct napi_struct xdp_napi; 63 struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */ 64 struct net_device *dev; 65 struct bpf_prog __rcu *xdp_prog; 66 struct xdp_mem_info xdp_mem; 67 struct veth_rq_stats stats; 68 bool rx_notify_masked; 69 struct ptr_ring xdp_ring; 70 struct xdp_rxq_info xdp_rxq; 71 struct page_pool *page_pool; 72 }; 73 74 struct veth_priv { 75 struct net_device __rcu *peer; 76 atomic64_t dropped; 77 struct bpf_prog *_xdp_prog; 78 struct veth_rq *rq; 79 unsigned int requested_headroom; 80 }; 81 82 struct veth_xdp_tx_bq { 83 struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 84 unsigned int count; 85 }; 86 87 /* 88 * ethtool interface 89 */ 90 91 struct veth_q_stat_desc { 92 char desc[ETH_GSTRING_LEN]; 93 size_t offset; 94 }; 95 96 #define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) 97 98 static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 99 { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 100 { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 101 { "drops", VETH_RQ_STAT(rx_drops) }, 102 { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, 103 { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 104 { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, 105 { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, 106 }; 107 108 #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 109 110 static const struct veth_q_stat_desc veth_tq_stats_desc[] = { 111 { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, 112 { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, 113 }; 114 115 #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) 116 117 static struct { 118 const char string[ETH_GSTRING_LEN]; 119 } ethtool_stats_keys[] = { 120 { "peer_ifindex" }, 121 }; 122 123 struct veth_xdp_buff { 124 struct xdp_buff xdp; 125 struct sk_buff *skb; 126 }; 127 128 static int veth_get_link_ksettings(struct net_device *dev, 129 struct ethtool_link_ksettings *cmd) 130 { 131 cmd->base.speed = SPEED_10000; 132 cmd->base.duplex = DUPLEX_FULL; 133 cmd->base.port = PORT_TP; 134 cmd->base.autoneg = AUTONEG_DISABLE; 135 return 0; 136 } 137 138 static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 139 { 140 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 141 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 142 } 143 144 static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 145 { 146 u8 *p = buf; 147 int i, j; 148 149 switch(stringset) { 150 case ETH_SS_STATS: 151 memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 152 p += sizeof(ethtool_stats_keys); 153 for (i = 0; i < dev->real_num_rx_queues; i++) 154 for (j = 0; j < VETH_RQ_STATS_LEN; j++) 155 ethtool_sprintf(&p, "rx_queue_%u_%.18s", 156 i, veth_rq_stats_desc[j].desc); 157 158 for (i = 0; i < dev->real_num_tx_queues; i++) 159 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 160 ethtool_sprintf(&p, "tx_queue_%u_%.18s", 161 i, veth_tq_stats_desc[j].desc); 162 163 page_pool_ethtool_stats_get_strings(p); 164 break; 165 } 166 } 167 168 static int veth_get_sset_count(struct net_device *dev, int sset) 169 { 170 switch (sset) { 171 case ETH_SS_STATS: 172 return ARRAY_SIZE(ethtool_stats_keys) + 173 VETH_RQ_STATS_LEN * dev->real_num_rx_queues + 174 VETH_TQ_STATS_LEN * dev->real_num_tx_queues + 175 page_pool_ethtool_stats_get_count(); 176 default: 177 return -EOPNOTSUPP; 178 } 179 } 180 181 static void veth_get_page_pool_stats(struct net_device *dev, u64 *data) 182 { 183 #ifdef CONFIG_PAGE_POOL_STATS 184 struct veth_priv *priv = netdev_priv(dev); 185 struct page_pool_stats pp_stats = {}; 186 int i; 187 188 for (i = 0; i < dev->real_num_rx_queues; i++) { 189 if (!priv->rq[i].page_pool) 190 continue; 191 page_pool_get_stats(priv->rq[i].page_pool, &pp_stats); 192 } 193 page_pool_ethtool_stats_get(data, &pp_stats); 194 #endif /* CONFIG_PAGE_POOL_STATS */ 195 } 196 197 static void veth_get_ethtool_stats(struct net_device *dev, 198 struct ethtool_stats *stats, u64 *data) 199 { 200 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 201 struct net_device *peer = rtnl_dereference(priv->peer); 202 int i, j, idx, pp_idx; 203 204 data[0] = peer ? peer->ifindex : 0; 205 idx = 1; 206 for (i = 0; i < dev->real_num_rx_queues; i++) { 207 const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 208 const void *stats_base = (void *)&rq_stats->vs; 209 unsigned int start; 210 size_t offset; 211 212 do { 213 start = u64_stats_fetch_begin(&rq_stats->syncp); 214 for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 215 offset = veth_rq_stats_desc[j].offset; 216 data[idx + j] = *(u64 *)(stats_base + offset); 217 } 218 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 219 idx += VETH_RQ_STATS_LEN; 220 } 221 pp_idx = idx; 222 223 if (!peer) 224 goto page_pool_stats; 225 226 rcv_priv = netdev_priv(peer); 227 for (i = 0; i < peer->real_num_rx_queues; i++) { 228 const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; 229 const void *base = (void *)&rq_stats->vs; 230 unsigned int start, tx_idx = idx; 231 u64 buf[VETH_TQ_STATS_LEN]; 232 size_t offset; 233 234 do { 235 start = u64_stats_fetch_begin(&rq_stats->syncp); 236 for (j = 0; j < VETH_TQ_STATS_LEN; j++) { 237 offset = veth_tq_stats_desc[j].offset; 238 buf[j] = *(u64 *)(base + offset); 239 } 240 } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); 241 242 tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; 243 for (j = 0; j < VETH_TQ_STATS_LEN; j++) 244 data[tx_idx + j] += buf[j]; 245 } 246 pp_idx = idx + dev->real_num_tx_queues * VETH_TQ_STATS_LEN; 247 248 page_pool_stats: 249 veth_get_page_pool_stats(dev, &data[pp_idx]); 250 } 251 252 static void veth_get_channels(struct net_device *dev, 253 struct ethtool_channels *channels) 254 { 255 channels->tx_count = dev->real_num_tx_queues; 256 channels->rx_count = dev->real_num_rx_queues; 257 channels->max_tx = dev->num_tx_queues; 258 channels->max_rx = dev->num_rx_queues; 259 } 260 261 static int veth_set_channels(struct net_device *dev, 262 struct ethtool_channels *ch); 263 264 static const struct ethtool_ops veth_ethtool_ops = { 265 .get_drvinfo = veth_get_drvinfo, 266 .get_link = ethtool_op_get_link, 267 .get_strings = veth_get_strings, 268 .get_sset_count = veth_get_sset_count, 269 .get_ethtool_stats = veth_get_ethtool_stats, 270 .get_link_ksettings = veth_get_link_ksettings, 271 .get_ts_info = ethtool_op_get_ts_info, 272 .get_channels = veth_get_channels, 273 .set_channels = veth_set_channels, 274 }; 275 276 /* general routines */ 277 278 static bool veth_is_xdp_frame(void *ptr) 279 { 280 return (unsigned long)ptr & VETH_XDP_FLAG; 281 } 282 283 static struct xdp_frame *veth_ptr_to_xdp(void *ptr) 284 { 285 return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 286 } 287 288 static void *veth_xdp_to_ptr(struct xdp_frame *xdp) 289 { 290 return (void *)((unsigned long)xdp | VETH_XDP_FLAG); 291 } 292 293 static void veth_ptr_free(void *ptr) 294 { 295 if (veth_is_xdp_frame(ptr)) 296 xdp_return_frame(veth_ptr_to_xdp(ptr)); 297 else 298 kfree_skb(ptr); 299 } 300 301 static void __veth_xdp_flush(struct veth_rq *rq) 302 { 303 /* Write ptr_ring before reading rx_notify_masked */ 304 smp_mb(); 305 if (!READ_ONCE(rq->rx_notify_masked) && 306 napi_schedule_prep(&rq->xdp_napi)) { 307 WRITE_ONCE(rq->rx_notify_masked, true); 308 __napi_schedule(&rq->xdp_napi); 309 } 310 } 311 312 static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 313 { 314 if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) 315 return NETDEV_TX_BUSY; /* signal qdisc layer */ 316 317 return NET_RX_SUCCESS; /* same as NETDEV_TX_OK */ 318 } 319 320 static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 321 struct veth_rq *rq, bool xdp) 322 { 323 return __dev_forward_skb(dev, skb) ?: xdp ? 324 veth_xdp_rx(rq, skb) : 325 __netif_rx(skb); 326 } 327 328 /* return true if the specified skb has chances of GRO aggregation 329 * Don't strive for accuracy, but try to avoid GRO overhead in the most 330 * common scenarios. 331 * When XDP is enabled, all traffic is considered eligible, as the xmit 332 * device has TSO off. 333 * When TSO is enabled on the xmit device, we are likely interested only 334 * in UDP aggregation, explicitly check for that if the skb is suspected 335 * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets - 336 * to belong to locally generated UDP traffic. 337 */ 338 static bool veth_skb_is_eligible_for_gro(const struct net_device *dev, 339 const struct net_device *rcv, 340 const struct sk_buff *skb) 341 { 342 return !(dev->features & NETIF_F_ALL_TSO) || 343 (skb->destructor == sock_wfree && 344 rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD)); 345 } 346 347 static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 348 { 349 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 350 struct veth_rq *rq = NULL; 351 struct netdev_queue *txq; 352 struct net_device *rcv; 353 int length = skb->len; 354 bool use_napi = false; 355 int ret, rxq; 356 357 rcu_read_lock(); 358 rcv = rcu_dereference(priv->peer); 359 if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) { 360 kfree_skb(skb); 361 goto drop; 362 } 363 364 rcv_priv = netdev_priv(rcv); 365 rxq = skb_get_queue_mapping(skb); 366 if (rxq < rcv->real_num_rx_queues) { 367 rq = &rcv_priv->rq[rxq]; 368 369 /* The napi pointer is available when an XDP program is 370 * attached or when GRO is enabled 371 * Don't bother with napi/GRO if the skb can't be aggregated 372 */ 373 use_napi = rcu_access_pointer(rq->napi) && 374 veth_skb_is_eligible_for_gro(dev, rcv, skb); 375 } 376 377 skb_tx_timestamp(skb); 378 379 ret = veth_forward_skb(rcv, skb, rq, use_napi); 380 switch (ret) { 381 case NET_RX_SUCCESS: /* same as NETDEV_TX_OK */ 382 if (!use_napi) 383 dev_sw_netstats_tx_add(dev, 1, length); 384 else 385 __veth_xdp_flush(rq); 386 break; 387 case NETDEV_TX_BUSY: 388 /* If a qdisc is attached to our virtual device, returning 389 * NETDEV_TX_BUSY is allowed. 390 */ 391 txq = netdev_get_tx_queue(dev, rxq); 392 393 if (qdisc_txq_has_no_queue(txq)) { 394 dev_kfree_skb_any(skb); 395 goto drop; 396 } 397 /* Restore Eth hdr pulled by dev_forward_skb/eth_type_trans */ 398 __skb_push(skb, ETH_HLEN); 399 netif_tx_stop_queue(txq); 400 /* Makes sure NAPI peer consumer runs. Consumer is responsible 401 * for starting txq again, until then ndo_start_xmit (this 402 * function) will not be invoked by the netstack again. 403 */ 404 __veth_xdp_flush(rq); 405 break; 406 case NET_RX_DROP: /* same as NET_XMIT_DROP */ 407 drop: 408 atomic64_inc(&priv->dropped); 409 ret = NET_XMIT_DROP; 410 break; 411 default: 412 net_crit_ratelimited("%s(%s): Invalid return code(%d)", 413 __func__, dev->name, ret); 414 } 415 rcu_read_unlock(); 416 417 return ret; 418 } 419 420 static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) 421 { 422 struct veth_priv *priv = netdev_priv(dev); 423 int i; 424 425 result->peer_tq_xdp_xmit_err = 0; 426 result->xdp_packets = 0; 427 result->xdp_tx_err = 0; 428 result->xdp_bytes = 0; 429 result->rx_drops = 0; 430 for (i = 0; i < dev->num_rx_queues; i++) { 431 u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; 432 struct veth_rq_stats *stats = &priv->rq[i].stats; 433 unsigned int start; 434 435 do { 436 start = u64_stats_fetch_begin(&stats->syncp); 437 peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; 438 xdp_tx_err = stats->vs.xdp_tx_err; 439 packets = stats->vs.xdp_packets; 440 bytes = stats->vs.xdp_bytes; 441 drops = stats->vs.rx_drops; 442 } while (u64_stats_fetch_retry(&stats->syncp, start)); 443 result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; 444 result->xdp_tx_err += xdp_tx_err; 445 result->xdp_packets += packets; 446 result->xdp_bytes += bytes; 447 result->rx_drops += drops; 448 } 449 } 450 451 static void veth_get_stats64(struct net_device *dev, 452 struct rtnl_link_stats64 *tot) 453 { 454 struct veth_priv *priv = netdev_priv(dev); 455 struct net_device *peer; 456 struct veth_stats rx; 457 458 tot->tx_dropped = atomic64_read(&priv->dropped); 459 dev_fetch_sw_netstats(tot, dev->tstats); 460 461 veth_stats_rx(&rx, dev); 462 tot->tx_dropped += rx.xdp_tx_err; 463 tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; 464 tot->rx_bytes += rx.xdp_bytes; 465 tot->rx_packets += rx.xdp_packets; 466 467 rcu_read_lock(); 468 peer = rcu_dereference(priv->peer); 469 if (peer) { 470 struct rtnl_link_stats64 tot_peer = {}; 471 472 dev_fetch_sw_netstats(&tot_peer, peer->tstats); 473 tot->rx_bytes += tot_peer.tx_bytes; 474 tot->rx_packets += tot_peer.tx_packets; 475 476 veth_stats_rx(&rx, peer); 477 tot->tx_dropped += rx.peer_tq_xdp_xmit_err; 478 tot->rx_dropped += rx.xdp_tx_err; 479 tot->tx_bytes += rx.xdp_bytes; 480 tot->tx_packets += rx.xdp_packets; 481 } 482 rcu_read_unlock(); 483 } 484 485 /* fake multicast ability */ 486 static void veth_set_multicast_list(struct net_device *dev) 487 { 488 } 489 490 static int veth_select_rxq(struct net_device *dev) 491 { 492 return smp_processor_id() % dev->real_num_rx_queues; 493 } 494 495 static struct net_device *veth_peer_dev(struct net_device *dev) 496 { 497 struct veth_priv *priv = netdev_priv(dev); 498 499 /* Callers must be under RCU read side. */ 500 return rcu_dereference(priv->peer); 501 } 502 503 static int veth_xdp_xmit(struct net_device *dev, int n, 504 struct xdp_frame **frames, 505 u32 flags, bool ndo_xmit) 506 { 507 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 508 int i, ret = -ENXIO, nxmit = 0; 509 struct net_device *rcv; 510 unsigned int max_len; 511 struct veth_rq *rq; 512 513 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 514 return -EINVAL; 515 516 rcu_read_lock(); 517 rcv = rcu_dereference(priv->peer); 518 if (unlikely(!rcv)) 519 goto out; 520 521 rcv_priv = netdev_priv(rcv); 522 rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 523 /* The napi pointer is set if NAPI is enabled, which ensures that 524 * xdp_ring is initialized on receive side and the peer device is up. 525 */ 526 if (!rcu_access_pointer(rq->napi)) 527 goto out; 528 529 max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 530 531 spin_lock(&rq->xdp_ring.producer_lock); 532 for (i = 0; i < n; i++) { 533 struct xdp_frame *frame = frames[i]; 534 void *ptr = veth_xdp_to_ptr(frame); 535 536 if (unlikely(xdp_get_frame_len(frame) > max_len || 537 __ptr_ring_produce(&rq->xdp_ring, ptr))) 538 break; 539 nxmit++; 540 } 541 spin_unlock(&rq->xdp_ring.producer_lock); 542 543 if (flags & XDP_XMIT_FLUSH) 544 __veth_xdp_flush(rq); 545 546 ret = nxmit; 547 if (ndo_xmit) { 548 u64_stats_update_begin(&rq->stats.syncp); 549 rq->stats.vs.peer_tq_xdp_xmit += nxmit; 550 rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 551 u64_stats_update_end(&rq->stats.syncp); 552 } 553 554 out: 555 rcu_read_unlock(); 556 557 return ret; 558 } 559 560 static int veth_ndo_xdp_xmit(struct net_device *dev, int n, 561 struct xdp_frame **frames, u32 flags) 562 { 563 int err; 564 565 err = veth_xdp_xmit(dev, n, frames, flags, true); 566 if (err < 0) { 567 struct veth_priv *priv = netdev_priv(dev); 568 569 atomic64_add(n, &priv->dropped); 570 } 571 572 return err; 573 } 574 575 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 576 { 577 int sent, i, err = 0, drops; 578 579 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 580 if (sent < 0) { 581 err = sent; 582 sent = 0; 583 } 584 585 for (i = sent; unlikely(i < bq->count); i++) 586 xdp_return_frame(bq->q[i]); 587 588 drops = bq->count - sent; 589 trace_xdp_bulk_tx(rq->dev, sent, drops, err); 590 591 u64_stats_update_begin(&rq->stats.syncp); 592 rq->stats.vs.xdp_tx += sent; 593 rq->stats.vs.xdp_tx_err += drops; 594 u64_stats_update_end(&rq->stats.syncp); 595 596 bq->count = 0; 597 } 598 599 static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 600 { 601 struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); 602 struct net_device *rcv; 603 struct veth_rq *rcv_rq; 604 605 rcu_read_lock(); 606 veth_xdp_flush_bq(rq, bq); 607 rcv = rcu_dereference(priv->peer); 608 if (unlikely(!rcv)) 609 goto out; 610 611 rcv_priv = netdev_priv(rcv); 612 rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 613 /* xdp_ring is initialized on receive side? */ 614 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) 615 goto out; 616 617 __veth_xdp_flush(rcv_rq); 618 out: 619 rcu_read_unlock(); 620 } 621 622 static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, 623 struct veth_xdp_tx_bq *bq) 624 { 625 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 626 627 if (unlikely(!frame)) 628 return -EOVERFLOW; 629 630 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 631 veth_xdp_flush_bq(rq, bq); 632 633 bq->q[bq->count++] = frame; 634 635 return 0; 636 } 637 638 static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, 639 struct xdp_frame *frame, 640 struct veth_xdp_tx_bq *bq, 641 struct veth_stats *stats) 642 { 643 struct xdp_frame orig_frame; 644 struct bpf_prog *xdp_prog; 645 646 rcu_read_lock(); 647 xdp_prog = rcu_dereference(rq->xdp_prog); 648 if (likely(xdp_prog)) { 649 struct veth_xdp_buff vxbuf; 650 struct xdp_buff *xdp = &vxbuf.xdp; 651 u32 act; 652 653 xdp_convert_frame_to_buff(frame, xdp); 654 xdp->rxq = &rq->xdp_rxq; 655 vxbuf.skb = NULL; 656 657 act = bpf_prog_run_xdp(xdp_prog, xdp); 658 659 switch (act) { 660 case XDP_PASS: 661 if (xdp_update_frame_from_buff(xdp, frame)) 662 goto err_xdp; 663 break; 664 case XDP_TX: 665 orig_frame = *frame; 666 xdp->rxq->mem.type = frame->mem_type; 667 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 668 trace_xdp_exception(rq->dev, xdp_prog, act); 669 frame = &orig_frame; 670 stats->rx_drops++; 671 goto err_xdp; 672 } 673 stats->xdp_tx++; 674 rcu_read_unlock(); 675 goto xdp_xmit; 676 case XDP_REDIRECT: 677 orig_frame = *frame; 678 xdp->rxq->mem.type = frame->mem_type; 679 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 680 frame = &orig_frame; 681 stats->rx_drops++; 682 goto err_xdp; 683 } 684 stats->xdp_redirect++; 685 rcu_read_unlock(); 686 goto xdp_xmit; 687 default: 688 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 689 fallthrough; 690 case XDP_ABORTED: 691 trace_xdp_exception(rq->dev, xdp_prog, act); 692 fallthrough; 693 case XDP_DROP: 694 stats->xdp_drops++; 695 goto err_xdp; 696 } 697 } 698 rcu_read_unlock(); 699 700 return frame; 701 err_xdp: 702 rcu_read_unlock(); 703 xdp_return_frame(frame); 704 xdp_xmit: 705 return NULL; 706 } 707 708 /* frames array contains VETH_XDP_BATCH at most */ 709 static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, 710 int n_xdpf, struct veth_xdp_tx_bq *bq, 711 struct veth_stats *stats) 712 { 713 void *skbs[VETH_XDP_BATCH]; 714 int i; 715 716 if (unlikely(!napi_skb_cache_get_bulk(skbs, n_xdpf))) { 717 for (i = 0; i < n_xdpf; i++) 718 xdp_return_frame(frames[i]); 719 stats->rx_drops += n_xdpf; 720 721 return; 722 } 723 724 for (i = 0; i < n_xdpf; i++) { 725 struct sk_buff *skb = skbs[i]; 726 727 skb = __xdp_build_skb_from_frame(frames[i], skb, 728 rq->dev); 729 if (!skb) { 730 xdp_return_frame(frames[i]); 731 stats->rx_drops++; 732 continue; 733 } 734 napi_gro_receive(&rq->xdp_napi, skb); 735 } 736 } 737 738 static void veth_xdp_get(struct xdp_buff *xdp) 739 { 740 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 741 int i; 742 743 get_page(virt_to_page(xdp->data)); 744 if (likely(!xdp_buff_has_frags(xdp))) 745 return; 746 747 for (i = 0; i < sinfo->nr_frags; i++) 748 __skb_frag_ref(&sinfo->frags[i]); 749 } 750 751 static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, 752 struct xdp_buff *xdp, 753 struct sk_buff **pskb) 754 { 755 struct sk_buff *skb = *pskb; 756 u32 frame_sz; 757 758 if (skb_shared(skb) || skb_head_is_locked(skb) || 759 skb_is_nonlinear(skb) || 760 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 761 if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM)) 762 goto drop; 763 764 skb = *pskb; 765 } 766 767 /* SKB "head" area always have tailroom for skb_shared_info */ 768 frame_sz = skb_end_pointer(skb) - skb->head; 769 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 770 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 771 xdp_prepare_buff(xdp, skb->head, skb_headroom(skb), 772 skb_headlen(skb), true); 773 774 if (skb_shinfo(skb)->nr_frags) { 775 skb_shinfo(skb)->xdp_frags_size = skb->data_len; 776 xdp_buff_set_frags_flag(xdp); 777 } else { 778 xdp_buff_clear_frags_flag(xdp); 779 } 780 *pskb = skb; 781 782 return 0; 783 drop: 784 consume_skb(skb); 785 *pskb = NULL; 786 787 return -ENOMEM; 788 } 789 790 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, 791 struct sk_buff *skb, 792 struct veth_xdp_tx_bq *bq, 793 struct veth_stats *stats) 794 { 795 void *orig_data, *orig_data_end; 796 struct bpf_prog *xdp_prog; 797 struct veth_xdp_buff vxbuf; 798 struct xdp_buff *xdp = &vxbuf.xdp; 799 u32 act, metalen; 800 int off; 801 802 skb_prepare_for_gro(skb); 803 804 rcu_read_lock(); 805 xdp_prog = rcu_dereference(rq->xdp_prog); 806 if (unlikely(!xdp_prog)) { 807 rcu_read_unlock(); 808 goto out; 809 } 810 811 __skb_push(skb, skb->data - skb_mac_header(skb)); 812 if (veth_convert_skb_to_xdp_buff(rq, xdp, &skb)) 813 goto drop; 814 vxbuf.skb = skb; 815 816 orig_data = xdp->data; 817 orig_data_end = xdp->data_end; 818 819 act = bpf_prog_run_xdp(xdp_prog, xdp); 820 821 switch (act) { 822 case XDP_PASS: 823 break; 824 case XDP_TX: 825 veth_xdp_get(xdp); 826 consume_skb(skb); 827 xdp->rxq->mem = rq->xdp_mem; 828 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { 829 trace_xdp_exception(rq->dev, xdp_prog, act); 830 stats->rx_drops++; 831 goto err_xdp; 832 } 833 stats->xdp_tx++; 834 rcu_read_unlock(); 835 goto xdp_xmit; 836 case XDP_REDIRECT: 837 veth_xdp_get(xdp); 838 consume_skb(skb); 839 xdp->rxq->mem = rq->xdp_mem; 840 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { 841 stats->rx_drops++; 842 goto err_xdp; 843 } 844 stats->xdp_redirect++; 845 rcu_read_unlock(); 846 goto xdp_xmit; 847 default: 848 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); 849 fallthrough; 850 case XDP_ABORTED: 851 trace_xdp_exception(rq->dev, xdp_prog, act); 852 fallthrough; 853 case XDP_DROP: 854 stats->xdp_drops++; 855 goto xdp_drop; 856 } 857 rcu_read_unlock(); 858 859 /* check if bpf_xdp_adjust_head was used */ 860 off = orig_data - xdp->data; 861 if (off > 0) 862 __skb_push(skb, off); 863 else if (off < 0) 864 __skb_pull(skb, -off); 865 866 skb_reset_mac_header(skb); 867 868 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers 869 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution 870 * from skb->len before updating data_len, then add the new one back. 871 */ 872 skb->len -= skb->data_len; 873 if (xdp_buff_has_frags(xdp)) { 874 skb->data_len = skb_shinfo(skb)->xdp_frags_size; 875 skb->len += skb->data_len; 876 } else { 877 skb->data_len = 0; 878 } 879 880 /* Synchronize the skb tail with XDP's updated linear area. */ 881 off = xdp->data_end - orig_data_end; 882 if (off != 0) { 883 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 884 skb->len += off; /* positive on grow, negative on shrink */ 885 } 886 887 skb->protocol = eth_type_trans(skb, rq->dev); 888 889 metalen = xdp->data - xdp->data_meta; 890 if (metalen) 891 skb_metadata_set(skb, metalen); 892 out: 893 return skb; 894 drop: 895 stats->rx_drops++; 896 xdp_drop: 897 rcu_read_unlock(); 898 kfree_skb(skb); 899 return NULL; 900 err_xdp: 901 rcu_read_unlock(); 902 xdp_return_buff(xdp); 903 xdp_xmit: 904 return NULL; 905 } 906 907 static int veth_xdp_rcv(struct veth_rq *rq, int budget, 908 struct veth_xdp_tx_bq *bq, 909 struct veth_stats *stats) 910 { 911 int i, done = 0, n_xdpf = 0; 912 void *xdpf[VETH_XDP_BATCH]; 913 914 for (i = 0; i < budget; i++) { 915 void *ptr = __ptr_ring_consume(&rq->xdp_ring); 916 917 if (!ptr) 918 break; 919 920 if (veth_is_xdp_frame(ptr)) { 921 /* ndo_xdp_xmit */ 922 struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 923 924 stats->xdp_bytes += xdp_get_frame_len(frame); 925 frame = veth_xdp_rcv_one(rq, frame, bq, stats); 926 if (frame) { 927 /* XDP_PASS */ 928 xdpf[n_xdpf++] = frame; 929 if (n_xdpf == VETH_XDP_BATCH) { 930 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, 931 bq, stats); 932 n_xdpf = 0; 933 } 934 } 935 } else { 936 /* ndo_start_xmit */ 937 struct sk_buff *skb = ptr; 938 939 stats->xdp_bytes += skb->len; 940 skb = veth_xdp_rcv_skb(rq, skb, bq, stats); 941 if (skb) { 942 if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC)) 943 netif_receive_skb(skb); 944 else 945 napi_gro_receive(&rq->xdp_napi, skb); 946 } 947 } 948 done++; 949 } 950 951 if (n_xdpf) 952 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats); 953 954 u64_stats_update_begin(&rq->stats.syncp); 955 rq->stats.vs.xdp_redirect += stats->xdp_redirect; 956 rq->stats.vs.xdp_bytes += stats->xdp_bytes; 957 rq->stats.vs.xdp_drops += stats->xdp_drops; 958 rq->stats.vs.rx_drops += stats->rx_drops; 959 rq->stats.vs.xdp_packets += done; 960 u64_stats_update_end(&rq->stats.syncp); 961 962 return done; 963 } 964 965 static int veth_poll(struct napi_struct *napi, int budget) 966 { 967 struct veth_rq *rq = 968 container_of(napi, struct veth_rq, xdp_napi); 969 struct veth_priv *priv = netdev_priv(rq->dev); 970 int queue_idx = rq - priv->rq; 971 struct netdev_queue *peer_txq; 972 struct veth_stats stats = {}; 973 struct net_device *peer_dev; 974 struct veth_xdp_tx_bq bq; 975 int done; 976 977 bq.count = 0; 978 979 /* NAPI functions as RCU section */ 980 peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held()); 981 peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ? 982 netdev_get_tx_queue(peer_dev, queue_idx) : NULL; 983 984 xdp_set_return_frame_no_direct(); 985 done = veth_xdp_rcv(rq, budget, &bq, &stats); 986 987 if (stats.xdp_redirect > 0) 988 xdp_do_flush(); 989 if (stats.xdp_tx > 0) 990 veth_xdp_flush(rq, &bq); 991 xdp_clear_return_frame_no_direct(); 992 993 if (done < budget && napi_complete_done(napi, done)) { 994 /* Write rx_notify_masked before reading ptr_ring */ 995 smp_store_mb(rq->rx_notify_masked, false); 996 if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 997 if (napi_schedule_prep(&rq->xdp_napi)) { 998 WRITE_ONCE(rq->rx_notify_masked, true); 999 __napi_schedule(&rq->xdp_napi); 1000 } 1001 } 1002 } 1003 1004 /* Release backpressure per NAPI poll */ 1005 smp_rmb(); /* Paired with netif_tx_stop_queue set_bit */ 1006 if (peer_txq && netif_tx_queue_stopped(peer_txq)) { 1007 txq_trans_cond_update(peer_txq); 1008 netif_tx_wake_queue(peer_txq); 1009 } 1010 1011 return done; 1012 } 1013 1014 static int veth_create_page_pool(struct veth_rq *rq) 1015 { 1016 struct page_pool_params pp_params = { 1017 .order = 0, 1018 .pool_size = VETH_RING_SIZE, 1019 .nid = NUMA_NO_NODE, 1020 .dev = &rq->dev->dev, 1021 }; 1022 1023 rq->page_pool = page_pool_create(&pp_params); 1024 if (IS_ERR(rq->page_pool)) { 1025 int err = PTR_ERR(rq->page_pool); 1026 1027 rq->page_pool = NULL; 1028 return err; 1029 } 1030 1031 return 0; 1032 } 1033 1034 static int __veth_napi_enable_range(struct net_device *dev, int start, int end) 1035 { 1036 struct veth_priv *priv = netdev_priv(dev); 1037 int err, i; 1038 1039 for (i = start; i < end; i++) { 1040 err = veth_create_page_pool(&priv->rq[i]); 1041 if (err) 1042 goto err_page_pool; 1043 } 1044 1045 for (i = start; i < end; i++) { 1046 struct veth_rq *rq = &priv->rq[i]; 1047 1048 err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 1049 if (err) 1050 goto err_xdp_ring; 1051 } 1052 1053 for (i = start; i < end; i++) { 1054 struct veth_rq *rq = &priv->rq[i]; 1055 1056 napi_enable(&rq->xdp_napi); 1057 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1058 } 1059 1060 return 0; 1061 1062 err_xdp_ring: 1063 for (i--; i >= start; i--) 1064 ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 1065 i = end; 1066 err_page_pool: 1067 for (i--; i >= start; i--) { 1068 page_pool_destroy(priv->rq[i].page_pool); 1069 priv->rq[i].page_pool = NULL; 1070 } 1071 1072 return err; 1073 } 1074 1075 static int __veth_napi_enable(struct net_device *dev) 1076 { 1077 return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1078 } 1079 1080 static void veth_napi_del_range(struct net_device *dev, int start, int end) 1081 { 1082 struct veth_priv *priv = netdev_priv(dev); 1083 int i; 1084 1085 for (i = start; i < end; i++) { 1086 struct veth_rq *rq = &priv->rq[i]; 1087 1088 rcu_assign_pointer(priv->rq[i].napi, NULL); 1089 napi_disable(&rq->xdp_napi); 1090 __netif_napi_del(&rq->xdp_napi); 1091 } 1092 synchronize_net(); 1093 1094 for (i = start; i < end; i++) { 1095 struct veth_rq *rq = &priv->rq[i]; 1096 1097 rq->rx_notify_masked = false; 1098 ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 1099 } 1100 1101 for (i = start; i < end; i++) { 1102 page_pool_destroy(priv->rq[i].page_pool); 1103 priv->rq[i].page_pool = NULL; 1104 } 1105 } 1106 1107 static void veth_napi_del(struct net_device *dev) 1108 { 1109 veth_napi_del_range(dev, 0, dev->real_num_rx_queues); 1110 } 1111 1112 static bool veth_gro_requested(const struct net_device *dev) 1113 { 1114 return !!(dev->wanted_features & NETIF_F_GRO); 1115 } 1116 1117 static int veth_enable_xdp_range(struct net_device *dev, int start, int end, 1118 bool napi_already_on) 1119 { 1120 struct veth_priv *priv = netdev_priv(dev); 1121 int err, i; 1122 1123 for (i = start; i < end; i++) { 1124 struct veth_rq *rq = &priv->rq[i]; 1125 1126 if (!napi_already_on) 1127 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1128 err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); 1129 if (err < 0) 1130 goto err_rxq_reg; 1131 1132 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1133 MEM_TYPE_PAGE_SHARED, 1134 NULL); 1135 if (err < 0) 1136 goto err_reg_mem; 1137 1138 /* Save original mem info as it can be overwritten */ 1139 rq->xdp_mem = rq->xdp_rxq.mem; 1140 } 1141 return 0; 1142 1143 err_reg_mem: 1144 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 1145 err_rxq_reg: 1146 if (!napi_already_on) 1147 netif_napi_del(&priv->rq[i].xdp_napi); 1148 for (i--; i >= start; i--) { 1149 struct veth_rq *rq = &priv->rq[i]; 1150 1151 xdp_rxq_info_unreg(&rq->xdp_rxq); 1152 if (!napi_already_on) 1153 netif_napi_del(&rq->xdp_napi); 1154 } 1155 1156 return err; 1157 } 1158 1159 static void veth_disable_xdp_range(struct net_device *dev, int start, int end, 1160 bool delete_napi) 1161 { 1162 struct veth_priv *priv = netdev_priv(dev); 1163 int i; 1164 1165 for (i = start; i < end; i++) { 1166 struct veth_rq *rq = &priv->rq[i]; 1167 1168 rq->xdp_rxq.mem = rq->xdp_mem; 1169 xdp_rxq_info_unreg(&rq->xdp_rxq); 1170 1171 if (delete_napi) 1172 netif_napi_del(&rq->xdp_napi); 1173 } 1174 } 1175 1176 static int veth_enable_xdp(struct net_device *dev) 1177 { 1178 bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP); 1179 struct veth_priv *priv = netdev_priv(dev); 1180 int err, i; 1181 1182 if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 1183 err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on); 1184 if (err) 1185 return err; 1186 1187 if (!napi_already_on) { 1188 err = __veth_napi_enable(dev); 1189 if (err) { 1190 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true); 1191 return err; 1192 } 1193 } 1194 } 1195 1196 for (i = 0; i < dev->real_num_rx_queues; i++) { 1197 rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 1198 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); 1199 } 1200 1201 return 0; 1202 } 1203 1204 static void veth_disable_xdp(struct net_device *dev) 1205 { 1206 struct veth_priv *priv = netdev_priv(dev); 1207 int i; 1208 1209 for (i = 0; i < dev->real_num_rx_queues; i++) 1210 rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 1211 1212 if (!netif_running(dev) || !veth_gro_requested(dev)) 1213 veth_napi_del(dev); 1214 1215 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false); 1216 } 1217 1218 static int veth_napi_enable_range(struct net_device *dev, int start, int end) 1219 { 1220 struct veth_priv *priv = netdev_priv(dev); 1221 int err, i; 1222 1223 for (i = start; i < end; i++) { 1224 struct veth_rq *rq = &priv->rq[i]; 1225 1226 netif_napi_add(dev, &rq->xdp_napi, veth_poll); 1227 } 1228 1229 err = __veth_napi_enable_range(dev, start, end); 1230 if (err) { 1231 for (i = start; i < end; i++) { 1232 struct veth_rq *rq = &priv->rq[i]; 1233 1234 netif_napi_del(&rq->xdp_napi); 1235 } 1236 return err; 1237 } 1238 return err; 1239 } 1240 1241 static int veth_napi_enable(struct net_device *dev) 1242 { 1243 return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues); 1244 } 1245 1246 static void veth_disable_range_safe(struct net_device *dev, int start, int end) 1247 { 1248 struct veth_priv *priv = netdev_priv(dev); 1249 1250 if (start >= end) 1251 return; 1252 1253 if (priv->_xdp_prog) { 1254 veth_napi_del_range(dev, start, end); 1255 veth_disable_xdp_range(dev, start, end, false); 1256 } else if (veth_gro_requested(dev)) { 1257 veth_napi_del_range(dev, start, end); 1258 } 1259 } 1260 1261 static int veth_enable_range_safe(struct net_device *dev, int start, int end) 1262 { 1263 struct veth_priv *priv = netdev_priv(dev); 1264 int err; 1265 1266 if (start >= end) 1267 return 0; 1268 1269 if (priv->_xdp_prog) { 1270 /* these channels are freshly initialized, napi is not on there even 1271 * when GRO is requeste 1272 */ 1273 err = veth_enable_xdp_range(dev, start, end, false); 1274 if (err) 1275 return err; 1276 1277 err = __veth_napi_enable_range(dev, start, end); 1278 if (err) { 1279 /* on error always delete the newly added napis */ 1280 veth_disable_xdp_range(dev, start, end, true); 1281 return err; 1282 } 1283 } else if (veth_gro_requested(dev)) { 1284 return veth_napi_enable_range(dev, start, end); 1285 } 1286 return 0; 1287 } 1288 1289 static void veth_set_xdp_features(struct net_device *dev) 1290 { 1291 struct veth_priv *priv = netdev_priv(dev); 1292 struct net_device *peer; 1293 1294 peer = rtnl_dereference(priv->peer); 1295 if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) { 1296 struct veth_priv *priv_peer = netdev_priv(peer); 1297 xdp_features_t val = NETDEV_XDP_ACT_BASIC | 1298 NETDEV_XDP_ACT_REDIRECT | 1299 NETDEV_XDP_ACT_RX_SG; 1300 1301 if (priv_peer->_xdp_prog || veth_gro_requested(peer)) 1302 val |= NETDEV_XDP_ACT_NDO_XMIT | 1303 NETDEV_XDP_ACT_NDO_XMIT_SG; 1304 xdp_set_features_flag(dev, val); 1305 } else { 1306 xdp_clear_features_flag(dev); 1307 } 1308 } 1309 1310 static int veth_set_channels(struct net_device *dev, 1311 struct ethtool_channels *ch) 1312 { 1313 struct veth_priv *priv = netdev_priv(dev); 1314 unsigned int old_rx_count, new_rx_count; 1315 struct veth_priv *peer_priv; 1316 struct net_device *peer; 1317 int err; 1318 1319 /* sanity check. Upper bounds are already enforced by the caller */ 1320 if (!ch->rx_count || !ch->tx_count) 1321 return -EINVAL; 1322 1323 /* avoid braking XDP, if that is enabled */ 1324 peer = rtnl_dereference(priv->peer); 1325 peer_priv = peer ? netdev_priv(peer) : NULL; 1326 if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues) 1327 return -EINVAL; 1328 1329 if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues) 1330 return -EINVAL; 1331 1332 old_rx_count = dev->real_num_rx_queues; 1333 new_rx_count = ch->rx_count; 1334 if (netif_running(dev)) { 1335 /* turn device off */ 1336 netif_carrier_off(dev); 1337 if (peer) 1338 netif_carrier_off(peer); 1339 1340 /* try to allocate new resources, as needed*/ 1341 err = veth_enable_range_safe(dev, old_rx_count, new_rx_count); 1342 if (err) 1343 goto out; 1344 } 1345 1346 err = netif_set_real_num_rx_queues(dev, ch->rx_count); 1347 if (err) 1348 goto revert; 1349 1350 err = netif_set_real_num_tx_queues(dev, ch->tx_count); 1351 if (err) { 1352 int err2 = netif_set_real_num_rx_queues(dev, old_rx_count); 1353 1354 /* this error condition could happen only if rx and tx change 1355 * in opposite directions (e.g. tx nr raises, rx nr decreases) 1356 * and we can't do anything to fully restore the original 1357 * status 1358 */ 1359 if (err2) 1360 pr_warn("Can't restore rx queues config %d -> %d %d", 1361 new_rx_count, old_rx_count, err2); 1362 else 1363 goto revert; 1364 } 1365 1366 out: 1367 if (netif_running(dev)) { 1368 /* note that we need to swap the arguments WRT the enable part 1369 * to identify the range we have to disable 1370 */ 1371 veth_disable_range_safe(dev, new_rx_count, old_rx_count); 1372 netif_carrier_on(dev); 1373 if (peer) 1374 netif_carrier_on(peer); 1375 } 1376 1377 /* update XDP supported features */ 1378 veth_set_xdp_features(dev); 1379 if (peer) 1380 veth_set_xdp_features(peer); 1381 1382 return err; 1383 1384 revert: 1385 new_rx_count = old_rx_count; 1386 old_rx_count = ch->rx_count; 1387 goto out; 1388 } 1389 1390 static int veth_open(struct net_device *dev) 1391 { 1392 struct veth_priv *priv = netdev_priv(dev); 1393 struct net_device *peer = rtnl_dereference(priv->peer); 1394 int err; 1395 1396 if (!peer) 1397 return -ENOTCONN; 1398 1399 if (priv->_xdp_prog) { 1400 err = veth_enable_xdp(dev); 1401 if (err) 1402 return err; 1403 } else if (veth_gro_requested(dev)) { 1404 err = veth_napi_enable(dev); 1405 if (err) 1406 return err; 1407 } 1408 1409 if (peer->flags & IFF_UP) { 1410 netif_carrier_on(dev); 1411 netif_carrier_on(peer); 1412 } 1413 1414 veth_set_xdp_features(dev); 1415 1416 return 0; 1417 } 1418 1419 static int veth_close(struct net_device *dev) 1420 { 1421 struct veth_priv *priv = netdev_priv(dev); 1422 struct net_device *peer = rtnl_dereference(priv->peer); 1423 1424 netif_carrier_off(dev); 1425 if (peer) 1426 netif_carrier_off(peer); 1427 1428 if (priv->_xdp_prog) 1429 veth_disable_xdp(dev); 1430 else if (veth_gro_requested(dev)) 1431 veth_napi_del(dev); 1432 1433 return 0; 1434 } 1435 1436 static int is_valid_veth_mtu(int mtu) 1437 { 1438 return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 1439 } 1440 1441 static int veth_alloc_queues(struct net_device *dev) 1442 { 1443 struct veth_priv *priv = netdev_priv(dev); 1444 int i; 1445 1446 priv->rq = kvzalloc_objs(*priv->rq, dev->num_rx_queues, 1447 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 1448 if (!priv->rq) 1449 return -ENOMEM; 1450 1451 for (i = 0; i < dev->num_rx_queues; i++) { 1452 priv->rq[i].dev = dev; 1453 u64_stats_init(&priv->rq[i].stats.syncp); 1454 } 1455 1456 return 0; 1457 } 1458 1459 static void veth_free_queues(struct net_device *dev) 1460 { 1461 struct veth_priv *priv = netdev_priv(dev); 1462 1463 kvfree(priv->rq); 1464 } 1465 1466 static int veth_dev_init(struct net_device *dev) 1467 { 1468 netdev_lockdep_set_classes(dev); 1469 return veth_alloc_queues(dev); 1470 } 1471 1472 static void veth_dev_free(struct net_device *dev) 1473 { 1474 veth_free_queues(dev); 1475 } 1476 1477 #ifdef CONFIG_NET_POLL_CONTROLLER 1478 static void veth_poll_controller(struct net_device *dev) 1479 { 1480 /* veth only receives frames when its peer sends one 1481 * Since it has nothing to do with disabling irqs, we are guaranteed 1482 * never to have pending data when we poll for it so 1483 * there is nothing to do here. 1484 * 1485 * We need this though so netpoll recognizes us as an interface that 1486 * supports polling, which enables bridge devices in virt setups to 1487 * still use netconsole 1488 */ 1489 } 1490 #endif /* CONFIG_NET_POLL_CONTROLLER */ 1491 1492 static int veth_get_iflink(const struct net_device *dev) 1493 { 1494 struct veth_priv *priv = netdev_priv(dev); 1495 struct net_device *peer; 1496 int iflink; 1497 1498 rcu_read_lock(); 1499 peer = rcu_dereference(priv->peer); 1500 iflink = peer ? READ_ONCE(peer->ifindex) : 0; 1501 rcu_read_unlock(); 1502 1503 return iflink; 1504 } 1505 1506 static netdev_features_t veth_fix_features(struct net_device *dev, 1507 netdev_features_t features) 1508 { 1509 struct veth_priv *priv = netdev_priv(dev); 1510 struct net_device *peer; 1511 1512 peer = rtnl_dereference(priv->peer); 1513 if (peer) { 1514 struct veth_priv *peer_priv = netdev_priv(peer); 1515 1516 if (peer_priv->_xdp_prog) 1517 features &= ~NETIF_F_GSO_SOFTWARE; 1518 } 1519 1520 return features; 1521 } 1522 1523 static int veth_set_features(struct net_device *dev, 1524 netdev_features_t features) 1525 { 1526 netdev_features_t changed = features ^ dev->features; 1527 struct veth_priv *priv = netdev_priv(dev); 1528 struct net_device *peer; 1529 int err; 1530 1531 if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog) 1532 return 0; 1533 1534 peer = rtnl_dereference(priv->peer); 1535 if (features & NETIF_F_GRO) { 1536 err = veth_napi_enable(dev); 1537 if (err) 1538 return err; 1539 1540 if (peer) 1541 xdp_features_set_redirect_target(peer, true); 1542 } else { 1543 if (peer) 1544 xdp_features_clear_redirect_target(peer); 1545 veth_napi_del(dev); 1546 } 1547 return 0; 1548 } 1549 1550 static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1551 { 1552 struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1553 struct net_device *peer; 1554 1555 if (new_hr < 0) 1556 new_hr = 0; 1557 1558 rcu_read_lock(); 1559 peer = rcu_dereference(priv->peer); 1560 if (unlikely(!peer)) 1561 goto out; 1562 1563 peer_priv = netdev_priv(peer); 1564 priv->requested_headroom = new_hr; 1565 new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1566 dev->needed_headroom = new_hr; 1567 peer->needed_headroom = new_hr; 1568 1569 out: 1570 rcu_read_unlock(); 1571 } 1572 1573 static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1574 struct netlink_ext_ack *extack) 1575 { 1576 struct veth_priv *priv = netdev_priv(dev); 1577 struct bpf_prog *old_prog; 1578 struct net_device *peer; 1579 unsigned int max_mtu; 1580 int err; 1581 1582 old_prog = priv->_xdp_prog; 1583 priv->_xdp_prog = prog; 1584 peer = rtnl_dereference(priv->peer); 1585 1586 if (prog) { 1587 if (!peer) { 1588 NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1589 err = -ENOTCONN; 1590 goto err; 1591 } 1592 1593 max_mtu = SKB_WITH_OVERHEAD(PAGE_SIZE - VETH_XDP_HEADROOM) - 1594 peer->hard_header_len; 1595 /* Allow increasing the max_mtu if the program supports 1596 * XDP fragments. 1597 */ 1598 if (prog->aux->xdp_has_frags) 1599 max_mtu += PAGE_SIZE * MAX_SKB_FRAGS; 1600 1601 if (peer->mtu > max_mtu) { 1602 NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1603 err = -ERANGE; 1604 goto err; 1605 } 1606 1607 if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1608 NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1609 err = -ENOSPC; 1610 goto err; 1611 } 1612 1613 if (dev->flags & IFF_UP) { 1614 err = veth_enable_xdp(dev); 1615 if (err) { 1616 NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1617 goto err; 1618 } 1619 } 1620 1621 if (!old_prog) { 1622 peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1623 peer->max_mtu = max_mtu; 1624 } 1625 1626 xdp_features_set_redirect_target(peer, true); 1627 } 1628 1629 if (old_prog) { 1630 if (!prog) { 1631 if (peer && !veth_gro_requested(dev)) 1632 xdp_features_clear_redirect_target(peer); 1633 1634 if (dev->flags & IFF_UP) 1635 veth_disable_xdp(dev); 1636 1637 if (peer) { 1638 peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1639 peer->max_mtu = ETH_MAX_MTU; 1640 } 1641 } 1642 bpf_prog_put(old_prog); 1643 } 1644 1645 if ((!!old_prog ^ !!prog) && peer) 1646 netdev_update_features(peer); 1647 1648 return 0; 1649 err: 1650 priv->_xdp_prog = old_prog; 1651 1652 return err; 1653 } 1654 1655 static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1656 { 1657 switch (xdp->command) { 1658 case XDP_SETUP_PROG: 1659 return veth_xdp_set(dev, xdp->prog, xdp->extack); 1660 default: 1661 return -EINVAL; 1662 } 1663 } 1664 1665 static int veth_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp) 1666 { 1667 struct veth_xdp_buff *_ctx = (void *)ctx; 1668 1669 if (!_ctx->skb) 1670 return -ENODATA; 1671 1672 *timestamp = skb_hwtstamps(_ctx->skb)->hwtstamp; 1673 return 0; 1674 } 1675 1676 static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, 1677 enum xdp_rss_hash_type *rss_type) 1678 { 1679 struct veth_xdp_buff *_ctx = (void *)ctx; 1680 struct sk_buff *skb = _ctx->skb; 1681 1682 if (!skb) 1683 return -ENODATA; 1684 1685 *hash = skb_get_hash(skb); 1686 *rss_type = skb->l4_hash ? XDP_RSS_TYPE_L4_ANY : XDP_RSS_TYPE_NONE; 1687 1688 return 0; 1689 } 1690 1691 static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, 1692 u16 *vlan_tci) 1693 { 1694 const struct veth_xdp_buff *_ctx = (void *)ctx; 1695 const struct sk_buff *skb = _ctx->skb; 1696 int err; 1697 1698 if (!skb) 1699 return -ENODATA; 1700 1701 err = __vlan_hwaccel_get_tag(skb, vlan_tci); 1702 if (err) 1703 return err; 1704 1705 *vlan_proto = skb->vlan_proto; 1706 return err; 1707 } 1708 1709 static const struct net_device_ops veth_netdev_ops = { 1710 .ndo_init = veth_dev_init, 1711 .ndo_open = veth_open, 1712 .ndo_stop = veth_close, 1713 .ndo_start_xmit = veth_xmit, 1714 .ndo_get_stats64 = veth_get_stats64, 1715 .ndo_set_rx_mode = veth_set_multicast_list, 1716 .ndo_set_mac_address = eth_mac_addr, 1717 #ifdef CONFIG_NET_POLL_CONTROLLER 1718 .ndo_poll_controller = veth_poll_controller, 1719 #endif 1720 .ndo_get_iflink = veth_get_iflink, 1721 .ndo_fix_features = veth_fix_features, 1722 .ndo_set_features = veth_set_features, 1723 .ndo_features_check = passthru_features_check, 1724 .ndo_set_rx_headroom = veth_set_rx_headroom, 1725 .ndo_bpf = veth_xdp, 1726 .ndo_xdp_xmit = veth_ndo_xdp_xmit, 1727 .ndo_get_peer_dev = veth_peer_dev, 1728 }; 1729 1730 static const struct xdp_metadata_ops veth_xdp_metadata_ops = { 1731 .xmo_rx_timestamp = veth_xdp_rx_timestamp, 1732 .xmo_rx_hash = veth_xdp_rx_hash, 1733 .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag, 1734 }; 1735 1736 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1737 NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1738 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 1739 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 1740 NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 1741 1742 static void veth_setup(struct net_device *dev) 1743 { 1744 ether_setup(dev); 1745 1746 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1747 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1748 dev->priv_flags |= IFF_NO_QUEUE; 1749 dev->priv_flags |= IFF_PHONY_HEADROOM; 1750 dev->priv_flags |= IFF_DISABLE_NETPOLL; 1751 dev->lltx = true; 1752 1753 dev->netdev_ops = &veth_netdev_ops; 1754 dev->xdp_metadata_ops = &veth_xdp_metadata_ops; 1755 dev->ethtool_ops = &veth_ethtool_ops; 1756 dev->features |= VETH_FEATURES; 1757 dev->vlan_features = dev->features & 1758 ~(NETIF_F_HW_VLAN_CTAG_TX | 1759 NETIF_F_HW_VLAN_STAG_TX | 1760 NETIF_F_HW_VLAN_CTAG_RX | 1761 NETIF_F_HW_VLAN_STAG_RX); 1762 dev->needs_free_netdev = true; 1763 dev->priv_destructor = veth_dev_free; 1764 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 1765 dev->max_mtu = ETH_MAX_MTU; 1766 1767 dev->hw_features = VETH_FEATURES; 1768 dev->hw_enc_features = VETH_FEATURES; 1769 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1770 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 1771 } 1772 1773 /* 1774 * netlink interface 1775 */ 1776 1777 static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1778 struct netlink_ext_ack *extack) 1779 { 1780 if (tb[IFLA_ADDRESS]) { 1781 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1782 return -EINVAL; 1783 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1784 return -EADDRNOTAVAIL; 1785 } 1786 if (tb[IFLA_MTU]) { 1787 if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 1788 return -EINVAL; 1789 } 1790 return 0; 1791 } 1792 1793 static struct rtnl_link_ops veth_link_ops; 1794 1795 static void veth_disable_gro(struct net_device *dev) 1796 { 1797 dev->features &= ~NETIF_F_GRO; 1798 dev->wanted_features &= ~NETIF_F_GRO; 1799 netdev_update_features(dev); 1800 } 1801 1802 static int veth_init_queues(struct net_device *dev, struct nlattr *tb[]) 1803 { 1804 int err; 1805 1806 if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) { 1807 err = netif_set_real_num_tx_queues(dev, 1); 1808 if (err) 1809 return err; 1810 } 1811 if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) { 1812 err = netif_set_real_num_rx_queues(dev, 1); 1813 if (err) 1814 return err; 1815 } 1816 return 0; 1817 } 1818 1819 static int veth_newlink(struct net_device *dev, 1820 struct rtnl_newlink_params *params, 1821 struct netlink_ext_ack *extack) 1822 { 1823 struct net *peer_net = rtnl_newlink_peer_net(params); 1824 struct nlattr **data = params->data; 1825 struct nlattr **tb = params->tb; 1826 int err; 1827 struct net_device *peer; 1828 struct veth_priv *priv; 1829 char ifname[IFNAMSIZ]; 1830 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 1831 unsigned char name_assign_type; 1832 struct ifinfomsg *ifmp; 1833 1834 /* 1835 * create and register peer first 1836 */ 1837 if (data && data[VETH_INFO_PEER]) { 1838 struct nlattr *nla_peer = data[VETH_INFO_PEER]; 1839 1840 ifmp = nla_data(nla_peer); 1841 rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack); 1842 tbp = peer_tb; 1843 } else { 1844 ifmp = NULL; 1845 tbp = tb; 1846 } 1847 1848 if (ifmp && tbp[IFLA_IFNAME]) { 1849 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 1850 name_assign_type = NET_NAME_USER; 1851 } else { 1852 snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 1853 name_assign_type = NET_NAME_ENUM; 1854 } 1855 1856 peer = rtnl_create_link(peer_net, ifname, name_assign_type, 1857 &veth_link_ops, tbp, extack); 1858 if (IS_ERR(peer)) 1859 return PTR_ERR(peer); 1860 1861 if (!ifmp || !tbp[IFLA_ADDRESS]) 1862 eth_hw_addr_random(peer); 1863 1864 if (ifmp && (dev->ifindex != 0)) 1865 peer->ifindex = ifmp->ifi_index; 1866 1867 netif_inherit_tso_max(peer, dev); 1868 1869 err = register_netdevice(peer); 1870 if (err < 0) 1871 goto err_register_peer; 1872 1873 /* keep GRO disabled by default to be consistent with the established 1874 * veth behavior 1875 */ 1876 veth_disable_gro(peer); 1877 netif_carrier_off(peer); 1878 1879 err = rtnl_configure_link(peer, ifmp, 0, NULL); 1880 if (err < 0) 1881 goto err_configure_peer; 1882 1883 /* 1884 * register dev last 1885 * 1886 * note, that since we've registered new device the dev's name 1887 * should be re-allocated 1888 */ 1889 1890 if (tb[IFLA_ADDRESS] == NULL) 1891 eth_hw_addr_random(dev); 1892 1893 if (tb[IFLA_IFNAME]) 1894 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 1895 else 1896 snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 1897 1898 err = register_netdevice(dev); 1899 if (err < 0) 1900 goto err_register_dev; 1901 1902 netif_carrier_off(dev); 1903 1904 /* 1905 * tie the deviced together 1906 */ 1907 1908 priv = netdev_priv(dev); 1909 rcu_assign_pointer(priv->peer, peer); 1910 err = veth_init_queues(dev, tb); 1911 if (err) 1912 goto err_queues; 1913 1914 priv = netdev_priv(peer); 1915 rcu_assign_pointer(priv->peer, dev); 1916 err = veth_init_queues(peer, tb); 1917 if (err) 1918 goto err_queues; 1919 1920 veth_disable_gro(dev); 1921 /* update XDP supported features */ 1922 veth_set_xdp_features(dev); 1923 veth_set_xdp_features(peer); 1924 1925 return 0; 1926 1927 err_queues: 1928 unregister_netdevice(dev); 1929 err_register_dev: 1930 /* nothing to do */ 1931 err_configure_peer: 1932 unregister_netdevice(peer); 1933 return err; 1934 1935 err_register_peer: 1936 free_netdev(peer); 1937 return err; 1938 } 1939 1940 static void veth_dellink(struct net_device *dev, struct list_head *head) 1941 { 1942 struct veth_priv *priv; 1943 struct net_device *peer; 1944 1945 priv = netdev_priv(dev); 1946 peer = rtnl_dereference(priv->peer); 1947 1948 /* Note : dellink() is called from default_device_exit_batch(), 1949 * before a rcu_synchronize() point. The devices are guaranteed 1950 * not being freed before one RCU grace period. 1951 */ 1952 RCU_INIT_POINTER(priv->peer, NULL); 1953 unregister_netdevice_queue(dev, head); 1954 1955 if (peer) { 1956 priv = netdev_priv(peer); 1957 RCU_INIT_POINTER(priv->peer, NULL); 1958 unregister_netdevice_queue(peer, head); 1959 } 1960 } 1961 1962 static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 1963 [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 1964 }; 1965 1966 static struct net *veth_get_link_net(const struct net_device *dev) 1967 { 1968 struct veth_priv *priv = netdev_priv(dev); 1969 struct net_device *peer = rtnl_dereference(priv->peer); 1970 1971 return peer ? dev_net(peer) : dev_net(dev); 1972 } 1973 1974 static unsigned int veth_get_num_queues(void) 1975 { 1976 /* enforce the same queue limit as rtnl_create_link */ 1977 int queues = num_possible_cpus(); 1978 1979 if (queues > 4096) 1980 queues = 4096; 1981 return queues; 1982 } 1983 1984 static struct rtnl_link_ops veth_link_ops = { 1985 .kind = DRV_NAME, 1986 .priv_size = sizeof(struct veth_priv), 1987 .setup = veth_setup, 1988 .validate = veth_validate, 1989 .newlink = veth_newlink, 1990 .dellink = veth_dellink, 1991 .policy = veth_policy, 1992 .peer_type = VETH_INFO_PEER, 1993 .maxtype = VETH_INFO_MAX, 1994 .get_link_net = veth_get_link_net, 1995 .get_num_tx_queues = veth_get_num_queues, 1996 .get_num_rx_queues = veth_get_num_queues, 1997 }; 1998 1999 /* 2000 * init/fini 2001 */ 2002 2003 static __init int veth_init(void) 2004 { 2005 return rtnl_link_register(&veth_link_ops); 2006 } 2007 2008 static __exit void veth_exit(void) 2009 { 2010 rtnl_link_unregister(&veth_link_ops); 2011 } 2012 2013 module_init(veth_init); 2014 module_exit(veth_exit); 2015 2016 MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 2017 MODULE_LICENSE("GPL v2"); 2018 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 2019