1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * drivers/net/veth.c 4 * 5 * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc 6 * 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> 9 * 10 */ 11 12 #include <linux/netdevice.h> 13 #include <linux/slab.h> 14 #include <linux/ethtool.h> 15 #include <linux/etherdevice.h> 16 #include <linux/u64_stats_sync.h> 17 18 #include <net/rtnetlink.h> 19 #include <net/dst.h> 20 #include <net/xfrm.h> 21 #include <net/xdp.h> 22 #include <linux/veth.h> 23 #include <linux/module.h> 24 #include <linux/bpf.h> 25 #include <linux/filter.h> 26 #include <linux/ptr_ring.h> 27 #include <linux/bpf_trace.h> 28 #include <linux/net_tstamp.h> 29 30 #define DRV_NAME "veth" 31 #define DRV_VERSION "1.0" 32 33 #define VETH_XDP_FLAG BIT(0) 34 #define VETH_RING_SIZE 256 35 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) 36 37 /* Separating two types of XDP xmit */ 38 #define VETH_XDP_TX BIT(0) 39 #define VETH_XDP_REDIR BIT(1) 40 41 #define VETH_XDP_TX_BULK_SIZE 16 42 43 struct veth_rq_stats { 44 u64 xdp_packets; 45 u64 xdp_bytes; 46 u64 xdp_drops; 47 struct u64_stats_sync syncp; 48 }; 49 50 struct veth_rq { 51 struct napi_struct xdp_napi; 52 struct net_device *dev; 53 struct bpf_prog __rcu *xdp_prog; 54 struct xdp_mem_info xdp_mem; 55 struct veth_rq_stats stats; 56 bool rx_notify_masked; 57 struct ptr_ring xdp_ring; 58 struct xdp_rxq_info xdp_rxq; 59 }; 60 61 struct veth_priv { 62 struct net_device __rcu *peer; 63 atomic64_t dropped; 64 struct bpf_prog *_xdp_prog; 65 struct veth_rq *rq; 66 unsigned int requested_headroom; 67 }; 68 69 struct veth_xdp_tx_bq { 70 struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; 71 unsigned int count; 72 }; 73 74 /* 75 * ethtool interface 76 */ 77 78 struct veth_q_stat_desc { 79 char desc[ETH_GSTRING_LEN]; 80 size_t offset; 81 }; 82 83 #define VETH_RQ_STAT(m) offsetof(struct veth_rq_stats, m) 84 85 static const struct veth_q_stat_desc veth_rq_stats_desc[] = { 86 { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, 87 { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, 88 { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, 89 }; 90 91 #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) 92 93 static struct { 94 const char string[ETH_GSTRING_LEN]; 95 } ethtool_stats_keys[] = { 96 { "peer_ifindex" }, 97 }; 98 99 static int veth_get_link_ksettings(struct net_device *dev, 100 struct ethtool_link_ksettings *cmd) 101 { 102 cmd->base.speed = SPEED_10000; 103 cmd->base.duplex = DUPLEX_FULL; 104 cmd->base.port = PORT_TP; 105 cmd->base.autoneg = AUTONEG_DISABLE; 106 return 0; 107 } 108 109 static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 110 { 111 strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); 112 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 113 } 114 115 static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) 116 { 117 char *p = (char *)buf; 118 int i, j; 119 120 switch(stringset) { 121 case ETH_SS_STATS: 122 memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); 123 p += sizeof(ethtool_stats_keys); 124 for (i = 0; i < dev->real_num_rx_queues; i++) { 125 for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 126 snprintf(p, ETH_GSTRING_LEN, 127 "rx_queue_%u_%.11s", 128 i, veth_rq_stats_desc[j].desc); 129 p += ETH_GSTRING_LEN; 130 } 131 } 132 break; 133 } 134 } 135 136 static int veth_get_sset_count(struct net_device *dev, int sset) 137 { 138 switch (sset) { 139 case ETH_SS_STATS: 140 return ARRAY_SIZE(ethtool_stats_keys) + 141 VETH_RQ_STATS_LEN * dev->real_num_rx_queues; 142 default: 143 return -EOPNOTSUPP; 144 } 145 } 146 147 static void veth_get_ethtool_stats(struct net_device *dev, 148 struct ethtool_stats *stats, u64 *data) 149 { 150 struct veth_priv *priv = netdev_priv(dev); 151 struct net_device *peer = rtnl_dereference(priv->peer); 152 int i, j, idx; 153 154 data[0] = peer ? peer->ifindex : 0; 155 idx = 1; 156 for (i = 0; i < dev->real_num_rx_queues; i++) { 157 const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; 158 const void *stats_base = (void *)rq_stats; 159 unsigned int start; 160 size_t offset; 161 162 do { 163 start = u64_stats_fetch_begin_irq(&rq_stats->syncp); 164 for (j = 0; j < VETH_RQ_STATS_LEN; j++) { 165 offset = veth_rq_stats_desc[j].offset; 166 data[idx + j] = *(u64 *)(stats_base + offset); 167 } 168 } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); 169 idx += VETH_RQ_STATS_LEN; 170 } 171 } 172 173 static const struct ethtool_ops veth_ethtool_ops = { 174 .get_drvinfo = veth_get_drvinfo, 175 .get_link = ethtool_op_get_link, 176 .get_strings = veth_get_strings, 177 .get_sset_count = veth_get_sset_count, 178 .get_ethtool_stats = veth_get_ethtool_stats, 179 .get_link_ksettings = veth_get_link_ksettings, 180 .get_ts_info = ethtool_op_get_ts_info, 181 }; 182 183 /* general routines */ 184 185 static bool veth_is_xdp_frame(void *ptr) 186 { 187 return (unsigned long)ptr & VETH_XDP_FLAG; 188 } 189 190 static void *veth_ptr_to_xdp(void *ptr) 191 { 192 return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); 193 } 194 195 static void *veth_xdp_to_ptr(void *ptr) 196 { 197 return (void *)((unsigned long)ptr | VETH_XDP_FLAG); 198 } 199 200 static void veth_ptr_free(void *ptr) 201 { 202 if (veth_is_xdp_frame(ptr)) 203 xdp_return_frame(veth_ptr_to_xdp(ptr)); 204 else 205 kfree_skb(ptr); 206 } 207 208 static void __veth_xdp_flush(struct veth_rq *rq) 209 { 210 /* Write ptr_ring before reading rx_notify_masked */ 211 smp_mb(); 212 if (!rq->rx_notify_masked) { 213 rq->rx_notify_masked = true; 214 napi_schedule(&rq->xdp_napi); 215 } 216 } 217 218 static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) 219 { 220 if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { 221 dev_kfree_skb_any(skb); 222 return NET_RX_DROP; 223 } 224 225 return NET_RX_SUCCESS; 226 } 227 228 static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, 229 struct veth_rq *rq, bool xdp) 230 { 231 return __dev_forward_skb(dev, skb) ?: xdp ? 232 veth_xdp_rx(rq, skb) : 233 netif_rx(skb); 234 } 235 236 static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) 237 { 238 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 239 struct veth_rq *rq = NULL; 240 struct net_device *rcv; 241 int length = skb->len; 242 bool rcv_xdp = false; 243 int rxq; 244 245 rcu_read_lock(); 246 rcv = rcu_dereference(priv->peer); 247 if (unlikely(!rcv)) { 248 kfree_skb(skb); 249 goto drop; 250 } 251 252 rcv_priv = netdev_priv(rcv); 253 rxq = skb_get_queue_mapping(skb); 254 if (rxq < rcv->real_num_rx_queues) { 255 rq = &rcv_priv->rq[rxq]; 256 rcv_xdp = rcu_access_pointer(rq->xdp_prog); 257 if (rcv_xdp) 258 skb_record_rx_queue(skb, rxq); 259 } 260 261 skb_tx_timestamp(skb); 262 if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) { 263 if (!rcv_xdp) 264 dev_lstats_add(dev, length); 265 } else { 266 drop: 267 atomic64_inc(&priv->dropped); 268 } 269 270 if (rcv_xdp) 271 __veth_xdp_flush(rq); 272 273 rcu_read_unlock(); 274 275 return NETDEV_TX_OK; 276 } 277 278 static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes) 279 { 280 struct veth_priv *priv = netdev_priv(dev); 281 282 dev_lstats_read(dev, packets, bytes); 283 return atomic64_read(&priv->dropped); 284 } 285 286 static void veth_stats_rx(struct veth_rq_stats *result, struct net_device *dev) 287 { 288 struct veth_priv *priv = netdev_priv(dev); 289 int i; 290 291 result->xdp_packets = 0; 292 result->xdp_bytes = 0; 293 result->xdp_drops = 0; 294 for (i = 0; i < dev->num_rx_queues; i++) { 295 struct veth_rq_stats *stats = &priv->rq[i].stats; 296 u64 packets, bytes, drops; 297 unsigned int start; 298 299 do { 300 start = u64_stats_fetch_begin_irq(&stats->syncp); 301 packets = stats->xdp_packets; 302 bytes = stats->xdp_bytes; 303 drops = stats->xdp_drops; 304 } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); 305 result->xdp_packets += packets; 306 result->xdp_bytes += bytes; 307 result->xdp_drops += drops; 308 } 309 } 310 311 static void veth_get_stats64(struct net_device *dev, 312 struct rtnl_link_stats64 *tot) 313 { 314 struct veth_priv *priv = netdev_priv(dev); 315 struct net_device *peer; 316 struct veth_rq_stats rx; 317 u64 packets, bytes; 318 319 tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes); 320 tot->tx_bytes = bytes; 321 tot->tx_packets = packets; 322 323 veth_stats_rx(&rx, dev); 324 tot->rx_dropped = rx.xdp_drops; 325 tot->rx_bytes = rx.xdp_bytes; 326 tot->rx_packets = rx.xdp_packets; 327 328 rcu_read_lock(); 329 peer = rcu_dereference(priv->peer); 330 if (peer) { 331 tot->rx_dropped += veth_stats_tx(peer, &packets, &bytes); 332 tot->rx_bytes += bytes; 333 tot->rx_packets += packets; 334 335 veth_stats_rx(&rx, peer); 336 tot->tx_bytes += rx.xdp_bytes; 337 tot->tx_packets += rx.xdp_packets; 338 } 339 rcu_read_unlock(); 340 } 341 342 /* fake multicast ability */ 343 static void veth_set_multicast_list(struct net_device *dev) 344 { 345 } 346 347 static struct sk_buff *veth_build_skb(void *head, int headroom, int len, 348 int buflen) 349 { 350 struct sk_buff *skb; 351 352 if (!buflen) { 353 buflen = SKB_DATA_ALIGN(headroom + len) + 354 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 355 } 356 skb = build_skb(head, buflen); 357 if (!skb) 358 return NULL; 359 360 skb_reserve(skb, headroom); 361 skb_put(skb, len); 362 363 return skb; 364 } 365 366 static int veth_select_rxq(struct net_device *dev) 367 { 368 return smp_processor_id() % dev->real_num_rx_queues; 369 } 370 371 static int veth_xdp_xmit(struct net_device *dev, int n, 372 struct xdp_frame **frames, u32 flags) 373 { 374 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 375 struct net_device *rcv; 376 int i, ret, drops = n; 377 unsigned int max_len; 378 struct veth_rq *rq; 379 380 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 381 ret = -EINVAL; 382 goto drop; 383 } 384 385 rcv = rcu_dereference(priv->peer); 386 if (unlikely(!rcv)) { 387 ret = -ENXIO; 388 goto drop; 389 } 390 391 rcv_priv = netdev_priv(rcv); 392 rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 393 /* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive 394 * side. This means an XDP program is loaded on the peer and the peer 395 * device is up. 396 */ 397 if (!rcu_access_pointer(rq->xdp_prog)) { 398 ret = -ENXIO; 399 goto drop; 400 } 401 402 drops = 0; 403 max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; 404 405 spin_lock(&rq->xdp_ring.producer_lock); 406 for (i = 0; i < n; i++) { 407 struct xdp_frame *frame = frames[i]; 408 void *ptr = veth_xdp_to_ptr(frame); 409 410 if (unlikely(frame->len > max_len || 411 __ptr_ring_produce(&rq->xdp_ring, ptr))) { 412 xdp_return_frame_rx_napi(frame); 413 drops++; 414 } 415 } 416 spin_unlock(&rq->xdp_ring.producer_lock); 417 418 if (flags & XDP_XMIT_FLUSH) 419 __veth_xdp_flush(rq); 420 421 if (likely(!drops)) 422 return n; 423 424 ret = n - drops; 425 drop: 426 atomic64_add(drops, &priv->dropped); 427 428 return ret; 429 } 430 431 static void veth_xdp_flush_bq(struct net_device *dev, struct veth_xdp_tx_bq *bq) 432 { 433 int sent, i, err = 0; 434 435 sent = veth_xdp_xmit(dev, bq->count, bq->q, 0); 436 if (sent < 0) { 437 err = sent; 438 sent = 0; 439 for (i = 0; i < bq->count; i++) 440 xdp_return_frame(bq->q[i]); 441 } 442 trace_xdp_bulk_tx(dev, sent, bq->count - sent, err); 443 444 bq->count = 0; 445 } 446 447 static void veth_xdp_flush(struct net_device *dev, struct veth_xdp_tx_bq *bq) 448 { 449 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 450 struct net_device *rcv; 451 struct veth_rq *rq; 452 453 rcu_read_lock(); 454 veth_xdp_flush_bq(dev, bq); 455 rcv = rcu_dereference(priv->peer); 456 if (unlikely(!rcv)) 457 goto out; 458 459 rcv_priv = netdev_priv(rcv); 460 rq = &rcv_priv->rq[veth_select_rxq(rcv)]; 461 /* xdp_ring is initialized on receive side? */ 462 if (unlikely(!rcu_access_pointer(rq->xdp_prog))) 463 goto out; 464 465 __veth_xdp_flush(rq); 466 out: 467 rcu_read_unlock(); 468 } 469 470 static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp, 471 struct veth_xdp_tx_bq *bq) 472 { 473 struct xdp_frame *frame = convert_to_xdp_frame(xdp); 474 475 if (unlikely(!frame)) 476 return -EOVERFLOW; 477 478 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) 479 veth_xdp_flush_bq(dev, bq); 480 481 bq->q[bq->count++] = frame; 482 483 return 0; 484 } 485 486 static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, 487 struct xdp_frame *frame, 488 unsigned int *xdp_xmit, 489 struct veth_xdp_tx_bq *bq) 490 { 491 void *hard_start = frame->data - frame->headroom; 492 void *head = hard_start - sizeof(struct xdp_frame); 493 int len = frame->len, delta = 0; 494 struct xdp_frame orig_frame; 495 struct bpf_prog *xdp_prog; 496 unsigned int headroom; 497 struct sk_buff *skb; 498 499 rcu_read_lock(); 500 xdp_prog = rcu_dereference(rq->xdp_prog); 501 if (likely(xdp_prog)) { 502 struct xdp_buff xdp; 503 u32 act; 504 505 xdp.data_hard_start = hard_start; 506 xdp.data = frame->data; 507 xdp.data_end = frame->data + frame->len; 508 xdp.data_meta = frame->data - frame->metasize; 509 xdp.rxq = &rq->xdp_rxq; 510 511 act = bpf_prog_run_xdp(xdp_prog, &xdp); 512 513 switch (act) { 514 case XDP_PASS: 515 delta = frame->data - xdp.data; 516 len = xdp.data_end - xdp.data; 517 break; 518 case XDP_TX: 519 orig_frame = *frame; 520 xdp.data_hard_start = head; 521 xdp.rxq->mem = frame->mem; 522 if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) { 523 trace_xdp_exception(rq->dev, xdp_prog, act); 524 frame = &orig_frame; 525 goto err_xdp; 526 } 527 *xdp_xmit |= VETH_XDP_TX; 528 rcu_read_unlock(); 529 goto xdp_xmit; 530 case XDP_REDIRECT: 531 orig_frame = *frame; 532 xdp.data_hard_start = head; 533 xdp.rxq->mem = frame->mem; 534 if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { 535 frame = &orig_frame; 536 goto err_xdp; 537 } 538 *xdp_xmit |= VETH_XDP_REDIR; 539 rcu_read_unlock(); 540 goto xdp_xmit; 541 default: 542 bpf_warn_invalid_xdp_action(act); 543 /* fall through */ 544 case XDP_ABORTED: 545 trace_xdp_exception(rq->dev, xdp_prog, act); 546 /* fall through */ 547 case XDP_DROP: 548 goto err_xdp; 549 } 550 } 551 rcu_read_unlock(); 552 553 headroom = sizeof(struct xdp_frame) + frame->headroom - delta; 554 skb = veth_build_skb(head, headroom, len, 0); 555 if (!skb) { 556 xdp_return_frame(frame); 557 goto err; 558 } 559 560 xdp_release_frame(frame); 561 xdp_scrub_frame(frame); 562 skb->protocol = eth_type_trans(skb, rq->dev); 563 err: 564 return skb; 565 err_xdp: 566 rcu_read_unlock(); 567 xdp_return_frame(frame); 568 xdp_xmit: 569 return NULL; 570 } 571 572 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb, 573 unsigned int *xdp_xmit, 574 struct veth_xdp_tx_bq *bq) 575 { 576 u32 pktlen, headroom, act, metalen; 577 void *orig_data, *orig_data_end; 578 struct bpf_prog *xdp_prog; 579 int mac_len, delta, off; 580 struct xdp_buff xdp; 581 582 skb_orphan(skb); 583 584 rcu_read_lock(); 585 xdp_prog = rcu_dereference(rq->xdp_prog); 586 if (unlikely(!xdp_prog)) { 587 rcu_read_unlock(); 588 goto out; 589 } 590 591 mac_len = skb->data - skb_mac_header(skb); 592 pktlen = skb->len + mac_len; 593 headroom = skb_headroom(skb) - mac_len; 594 595 if (skb_shared(skb) || skb_head_is_locked(skb) || 596 skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { 597 struct sk_buff *nskb; 598 int size, head_off; 599 void *head, *start; 600 struct page *page; 601 602 size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + 603 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 604 if (size > PAGE_SIZE) 605 goto drop; 606 607 page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 608 if (!page) 609 goto drop; 610 611 head = page_address(page); 612 start = head + VETH_XDP_HEADROOM; 613 if (skb_copy_bits(skb, -mac_len, start, pktlen)) { 614 page_frag_free(head); 615 goto drop; 616 } 617 618 nskb = veth_build_skb(head, 619 VETH_XDP_HEADROOM + mac_len, skb->len, 620 PAGE_SIZE); 621 if (!nskb) { 622 page_frag_free(head); 623 goto drop; 624 } 625 626 skb_copy_header(nskb, skb); 627 head_off = skb_headroom(nskb) - skb_headroom(skb); 628 skb_headers_offset_update(nskb, head_off); 629 consume_skb(skb); 630 skb = nskb; 631 } 632 633 xdp.data_hard_start = skb->head; 634 xdp.data = skb_mac_header(skb); 635 xdp.data_end = xdp.data + pktlen; 636 xdp.data_meta = xdp.data; 637 xdp.rxq = &rq->xdp_rxq; 638 orig_data = xdp.data; 639 orig_data_end = xdp.data_end; 640 641 act = bpf_prog_run_xdp(xdp_prog, &xdp); 642 643 switch (act) { 644 case XDP_PASS: 645 break; 646 case XDP_TX: 647 get_page(virt_to_page(xdp.data)); 648 consume_skb(skb); 649 xdp.rxq->mem = rq->xdp_mem; 650 if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) { 651 trace_xdp_exception(rq->dev, xdp_prog, act); 652 goto err_xdp; 653 } 654 *xdp_xmit |= VETH_XDP_TX; 655 rcu_read_unlock(); 656 goto xdp_xmit; 657 case XDP_REDIRECT: 658 get_page(virt_to_page(xdp.data)); 659 consume_skb(skb); 660 xdp.rxq->mem = rq->xdp_mem; 661 if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) 662 goto err_xdp; 663 *xdp_xmit |= VETH_XDP_REDIR; 664 rcu_read_unlock(); 665 goto xdp_xmit; 666 default: 667 bpf_warn_invalid_xdp_action(act); 668 /* fall through */ 669 case XDP_ABORTED: 670 trace_xdp_exception(rq->dev, xdp_prog, act); 671 /* fall through */ 672 case XDP_DROP: 673 goto drop; 674 } 675 rcu_read_unlock(); 676 677 delta = orig_data - xdp.data; 678 off = mac_len + delta; 679 if (off > 0) 680 __skb_push(skb, off); 681 else if (off < 0) 682 __skb_pull(skb, -off); 683 skb->mac_header -= delta; 684 off = xdp.data_end - orig_data_end; 685 if (off != 0) 686 __skb_put(skb, off); 687 skb->protocol = eth_type_trans(skb, rq->dev); 688 689 metalen = xdp.data - xdp.data_meta; 690 if (metalen) 691 skb_metadata_set(skb, metalen); 692 out: 693 return skb; 694 drop: 695 rcu_read_unlock(); 696 kfree_skb(skb); 697 return NULL; 698 err_xdp: 699 rcu_read_unlock(); 700 page_frag_free(xdp.data); 701 xdp_xmit: 702 return NULL; 703 } 704 705 static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit, 706 struct veth_xdp_tx_bq *bq) 707 { 708 int i, done = 0, drops = 0, bytes = 0; 709 710 for (i = 0; i < budget; i++) { 711 void *ptr = __ptr_ring_consume(&rq->xdp_ring); 712 unsigned int xdp_xmit_one = 0; 713 struct sk_buff *skb; 714 715 if (!ptr) 716 break; 717 718 if (veth_is_xdp_frame(ptr)) { 719 struct xdp_frame *frame = veth_ptr_to_xdp(ptr); 720 721 bytes += frame->len; 722 skb = veth_xdp_rcv_one(rq, frame, &xdp_xmit_one, bq); 723 } else { 724 skb = ptr; 725 bytes += skb->len; 726 skb = veth_xdp_rcv_skb(rq, skb, &xdp_xmit_one, bq); 727 } 728 *xdp_xmit |= xdp_xmit_one; 729 730 if (skb) 731 napi_gro_receive(&rq->xdp_napi, skb); 732 else if (!xdp_xmit_one) 733 drops++; 734 735 done++; 736 } 737 738 u64_stats_update_begin(&rq->stats.syncp); 739 rq->stats.xdp_packets += done; 740 rq->stats.xdp_bytes += bytes; 741 rq->stats.xdp_drops += drops; 742 u64_stats_update_end(&rq->stats.syncp); 743 744 return done; 745 } 746 747 static int veth_poll(struct napi_struct *napi, int budget) 748 { 749 struct veth_rq *rq = 750 container_of(napi, struct veth_rq, xdp_napi); 751 unsigned int xdp_xmit = 0; 752 struct veth_xdp_tx_bq bq; 753 int done; 754 755 bq.count = 0; 756 757 xdp_set_return_frame_no_direct(); 758 done = veth_xdp_rcv(rq, budget, &xdp_xmit, &bq); 759 760 if (done < budget && napi_complete_done(napi, done)) { 761 /* Write rx_notify_masked before reading ptr_ring */ 762 smp_store_mb(rq->rx_notify_masked, false); 763 if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { 764 rq->rx_notify_masked = true; 765 napi_schedule(&rq->xdp_napi); 766 } 767 } 768 769 if (xdp_xmit & VETH_XDP_TX) 770 veth_xdp_flush(rq->dev, &bq); 771 if (xdp_xmit & VETH_XDP_REDIR) 772 xdp_do_flush_map(); 773 xdp_clear_return_frame_no_direct(); 774 775 return done; 776 } 777 778 static int veth_napi_add(struct net_device *dev) 779 { 780 struct veth_priv *priv = netdev_priv(dev); 781 int err, i; 782 783 for (i = 0; i < dev->real_num_rx_queues; i++) { 784 struct veth_rq *rq = &priv->rq[i]; 785 786 err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); 787 if (err) 788 goto err_xdp_ring; 789 } 790 791 for (i = 0; i < dev->real_num_rx_queues; i++) { 792 struct veth_rq *rq = &priv->rq[i]; 793 794 netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); 795 napi_enable(&rq->xdp_napi); 796 } 797 798 return 0; 799 err_xdp_ring: 800 for (i--; i >= 0; i--) 801 ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); 802 803 return err; 804 } 805 806 static void veth_napi_del(struct net_device *dev) 807 { 808 struct veth_priv *priv = netdev_priv(dev); 809 int i; 810 811 for (i = 0; i < dev->real_num_rx_queues; i++) { 812 struct veth_rq *rq = &priv->rq[i]; 813 814 napi_disable(&rq->xdp_napi); 815 napi_hash_del(&rq->xdp_napi); 816 } 817 synchronize_net(); 818 819 for (i = 0; i < dev->real_num_rx_queues; i++) { 820 struct veth_rq *rq = &priv->rq[i]; 821 822 netif_napi_del(&rq->xdp_napi); 823 rq->rx_notify_masked = false; 824 ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); 825 } 826 } 827 828 static int veth_enable_xdp(struct net_device *dev) 829 { 830 struct veth_priv *priv = netdev_priv(dev); 831 int err, i; 832 833 if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { 834 for (i = 0; i < dev->real_num_rx_queues; i++) { 835 struct veth_rq *rq = &priv->rq[i]; 836 837 err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); 838 if (err < 0) 839 goto err_rxq_reg; 840 841 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 842 MEM_TYPE_PAGE_SHARED, 843 NULL); 844 if (err < 0) 845 goto err_reg_mem; 846 847 /* Save original mem info as it can be overwritten */ 848 rq->xdp_mem = rq->xdp_rxq.mem; 849 } 850 851 err = veth_napi_add(dev); 852 if (err) 853 goto err_rxq_reg; 854 } 855 856 for (i = 0; i < dev->real_num_rx_queues; i++) 857 rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); 858 859 return 0; 860 err_reg_mem: 861 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 862 err_rxq_reg: 863 for (i--; i >= 0; i--) 864 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); 865 866 return err; 867 } 868 869 static void veth_disable_xdp(struct net_device *dev) 870 { 871 struct veth_priv *priv = netdev_priv(dev); 872 int i; 873 874 for (i = 0; i < dev->real_num_rx_queues; i++) 875 rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); 876 veth_napi_del(dev); 877 for (i = 0; i < dev->real_num_rx_queues; i++) { 878 struct veth_rq *rq = &priv->rq[i]; 879 880 rq->xdp_rxq.mem = rq->xdp_mem; 881 xdp_rxq_info_unreg(&rq->xdp_rxq); 882 } 883 } 884 885 static int veth_open(struct net_device *dev) 886 { 887 struct veth_priv *priv = netdev_priv(dev); 888 struct net_device *peer = rtnl_dereference(priv->peer); 889 int err; 890 891 if (!peer) 892 return -ENOTCONN; 893 894 if (priv->_xdp_prog) { 895 err = veth_enable_xdp(dev); 896 if (err) 897 return err; 898 } 899 900 if (peer->flags & IFF_UP) { 901 netif_carrier_on(dev); 902 netif_carrier_on(peer); 903 } 904 905 return 0; 906 } 907 908 static int veth_close(struct net_device *dev) 909 { 910 struct veth_priv *priv = netdev_priv(dev); 911 struct net_device *peer = rtnl_dereference(priv->peer); 912 913 netif_carrier_off(dev); 914 if (peer) 915 netif_carrier_off(peer); 916 917 if (priv->_xdp_prog) 918 veth_disable_xdp(dev); 919 920 return 0; 921 } 922 923 static int is_valid_veth_mtu(int mtu) 924 { 925 return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; 926 } 927 928 static int veth_alloc_queues(struct net_device *dev) 929 { 930 struct veth_priv *priv = netdev_priv(dev); 931 int i; 932 933 priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); 934 if (!priv->rq) 935 return -ENOMEM; 936 937 for (i = 0; i < dev->num_rx_queues; i++) { 938 priv->rq[i].dev = dev; 939 u64_stats_init(&priv->rq[i].stats.syncp); 940 } 941 942 return 0; 943 } 944 945 static void veth_free_queues(struct net_device *dev) 946 { 947 struct veth_priv *priv = netdev_priv(dev); 948 949 kfree(priv->rq); 950 } 951 952 static int veth_dev_init(struct net_device *dev) 953 { 954 int err; 955 956 dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 957 if (!dev->lstats) 958 return -ENOMEM; 959 960 err = veth_alloc_queues(dev); 961 if (err) { 962 free_percpu(dev->lstats); 963 return err; 964 } 965 966 return 0; 967 } 968 969 static void veth_dev_free(struct net_device *dev) 970 { 971 veth_free_queues(dev); 972 free_percpu(dev->lstats); 973 } 974 975 #ifdef CONFIG_NET_POLL_CONTROLLER 976 static void veth_poll_controller(struct net_device *dev) 977 { 978 /* veth only receives frames when its peer sends one 979 * Since it has nothing to do with disabling irqs, we are guaranteed 980 * never to have pending data when we poll for it so 981 * there is nothing to do here. 982 * 983 * We need this though so netpoll recognizes us as an interface that 984 * supports polling, which enables bridge devices in virt setups to 985 * still use netconsole 986 */ 987 } 988 #endif /* CONFIG_NET_POLL_CONTROLLER */ 989 990 static int veth_get_iflink(const struct net_device *dev) 991 { 992 struct veth_priv *priv = netdev_priv(dev); 993 struct net_device *peer; 994 int iflink; 995 996 rcu_read_lock(); 997 peer = rcu_dereference(priv->peer); 998 iflink = peer ? peer->ifindex : 0; 999 rcu_read_unlock(); 1000 1001 return iflink; 1002 } 1003 1004 static netdev_features_t veth_fix_features(struct net_device *dev, 1005 netdev_features_t features) 1006 { 1007 struct veth_priv *priv = netdev_priv(dev); 1008 struct net_device *peer; 1009 1010 peer = rtnl_dereference(priv->peer); 1011 if (peer) { 1012 struct veth_priv *peer_priv = netdev_priv(peer); 1013 1014 if (peer_priv->_xdp_prog) 1015 features &= ~NETIF_F_GSO_SOFTWARE; 1016 } 1017 1018 return features; 1019 } 1020 1021 static void veth_set_rx_headroom(struct net_device *dev, int new_hr) 1022 { 1023 struct veth_priv *peer_priv, *priv = netdev_priv(dev); 1024 struct net_device *peer; 1025 1026 if (new_hr < 0) 1027 new_hr = 0; 1028 1029 rcu_read_lock(); 1030 peer = rcu_dereference(priv->peer); 1031 if (unlikely(!peer)) 1032 goto out; 1033 1034 peer_priv = netdev_priv(peer); 1035 priv->requested_headroom = new_hr; 1036 new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); 1037 dev->needed_headroom = new_hr; 1038 peer->needed_headroom = new_hr; 1039 1040 out: 1041 rcu_read_unlock(); 1042 } 1043 1044 static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1045 struct netlink_ext_ack *extack) 1046 { 1047 struct veth_priv *priv = netdev_priv(dev); 1048 struct bpf_prog *old_prog; 1049 struct net_device *peer; 1050 unsigned int max_mtu; 1051 int err; 1052 1053 old_prog = priv->_xdp_prog; 1054 priv->_xdp_prog = prog; 1055 peer = rtnl_dereference(priv->peer); 1056 1057 if (prog) { 1058 if (!peer) { 1059 NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); 1060 err = -ENOTCONN; 1061 goto err; 1062 } 1063 1064 max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - 1065 peer->hard_header_len - 1066 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1067 if (peer->mtu > max_mtu) { 1068 NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); 1069 err = -ERANGE; 1070 goto err; 1071 } 1072 1073 if (dev->real_num_rx_queues < peer->real_num_tx_queues) { 1074 NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); 1075 err = -ENOSPC; 1076 goto err; 1077 } 1078 1079 if (dev->flags & IFF_UP) { 1080 err = veth_enable_xdp(dev); 1081 if (err) { 1082 NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); 1083 goto err; 1084 } 1085 } 1086 1087 if (!old_prog) { 1088 peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; 1089 peer->max_mtu = max_mtu; 1090 } 1091 } 1092 1093 if (old_prog) { 1094 if (!prog) { 1095 if (dev->flags & IFF_UP) 1096 veth_disable_xdp(dev); 1097 1098 if (peer) { 1099 peer->hw_features |= NETIF_F_GSO_SOFTWARE; 1100 peer->max_mtu = ETH_MAX_MTU; 1101 } 1102 } 1103 bpf_prog_put(old_prog); 1104 } 1105 1106 if ((!!old_prog ^ !!prog) && peer) 1107 netdev_update_features(peer); 1108 1109 return 0; 1110 err: 1111 priv->_xdp_prog = old_prog; 1112 1113 return err; 1114 } 1115 1116 static u32 veth_xdp_query(struct net_device *dev) 1117 { 1118 struct veth_priv *priv = netdev_priv(dev); 1119 const struct bpf_prog *xdp_prog; 1120 1121 xdp_prog = priv->_xdp_prog; 1122 if (xdp_prog) 1123 return xdp_prog->aux->id; 1124 1125 return 0; 1126 } 1127 1128 static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1129 { 1130 switch (xdp->command) { 1131 case XDP_SETUP_PROG: 1132 return veth_xdp_set(dev, xdp->prog, xdp->extack); 1133 case XDP_QUERY_PROG: 1134 xdp->prog_id = veth_xdp_query(dev); 1135 return 0; 1136 default: 1137 return -EINVAL; 1138 } 1139 } 1140 1141 static const struct net_device_ops veth_netdev_ops = { 1142 .ndo_init = veth_dev_init, 1143 .ndo_open = veth_open, 1144 .ndo_stop = veth_close, 1145 .ndo_start_xmit = veth_xmit, 1146 .ndo_get_stats64 = veth_get_stats64, 1147 .ndo_set_rx_mode = veth_set_multicast_list, 1148 .ndo_set_mac_address = eth_mac_addr, 1149 #ifdef CONFIG_NET_POLL_CONTROLLER 1150 .ndo_poll_controller = veth_poll_controller, 1151 #endif 1152 .ndo_get_iflink = veth_get_iflink, 1153 .ndo_fix_features = veth_fix_features, 1154 .ndo_features_check = passthru_features_check, 1155 .ndo_set_rx_headroom = veth_set_rx_headroom, 1156 .ndo_bpf = veth_xdp, 1157 .ndo_xdp_xmit = veth_xdp_xmit, 1158 }; 1159 1160 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ 1161 NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ 1162 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ 1163 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ 1164 NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) 1165 1166 static void veth_setup(struct net_device *dev) 1167 { 1168 ether_setup(dev); 1169 1170 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1171 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1172 dev->priv_flags |= IFF_NO_QUEUE; 1173 dev->priv_flags |= IFF_PHONY_HEADROOM; 1174 1175 dev->netdev_ops = &veth_netdev_ops; 1176 dev->ethtool_ops = &veth_ethtool_ops; 1177 dev->features |= NETIF_F_LLTX; 1178 dev->features |= VETH_FEATURES; 1179 dev->vlan_features = dev->features & 1180 ~(NETIF_F_HW_VLAN_CTAG_TX | 1181 NETIF_F_HW_VLAN_STAG_TX | 1182 NETIF_F_HW_VLAN_CTAG_RX | 1183 NETIF_F_HW_VLAN_STAG_RX); 1184 dev->needs_free_netdev = true; 1185 dev->priv_destructor = veth_dev_free; 1186 dev->max_mtu = ETH_MAX_MTU; 1187 1188 dev->hw_features = VETH_FEATURES; 1189 dev->hw_enc_features = VETH_FEATURES; 1190 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 1191 } 1192 1193 /* 1194 * netlink interface 1195 */ 1196 1197 static int veth_validate(struct nlattr *tb[], struct nlattr *data[], 1198 struct netlink_ext_ack *extack) 1199 { 1200 if (tb[IFLA_ADDRESS]) { 1201 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) 1202 return -EINVAL; 1203 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) 1204 return -EADDRNOTAVAIL; 1205 } 1206 if (tb[IFLA_MTU]) { 1207 if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) 1208 return -EINVAL; 1209 } 1210 return 0; 1211 } 1212 1213 static struct rtnl_link_ops veth_link_ops; 1214 1215 static int veth_newlink(struct net *src_net, struct net_device *dev, 1216 struct nlattr *tb[], struct nlattr *data[], 1217 struct netlink_ext_ack *extack) 1218 { 1219 int err; 1220 struct net_device *peer; 1221 struct veth_priv *priv; 1222 char ifname[IFNAMSIZ]; 1223 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; 1224 unsigned char name_assign_type; 1225 struct ifinfomsg *ifmp; 1226 struct net *net; 1227 1228 /* 1229 * create and register peer first 1230 */ 1231 if (data != NULL && data[VETH_INFO_PEER] != NULL) { 1232 struct nlattr *nla_peer; 1233 1234 nla_peer = data[VETH_INFO_PEER]; 1235 ifmp = nla_data(nla_peer); 1236 err = rtnl_nla_parse_ifla(peer_tb, 1237 nla_data(nla_peer) + sizeof(struct ifinfomsg), 1238 nla_len(nla_peer) - sizeof(struct ifinfomsg), 1239 NULL); 1240 if (err < 0) 1241 return err; 1242 1243 err = veth_validate(peer_tb, NULL, extack); 1244 if (err < 0) 1245 return err; 1246 1247 tbp = peer_tb; 1248 } else { 1249 ifmp = NULL; 1250 tbp = tb; 1251 } 1252 1253 if (ifmp && tbp[IFLA_IFNAME]) { 1254 nla_strlcpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 1255 name_assign_type = NET_NAME_USER; 1256 } else { 1257 snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); 1258 name_assign_type = NET_NAME_ENUM; 1259 } 1260 1261 net = rtnl_link_get_net(src_net, tbp); 1262 if (IS_ERR(net)) 1263 return PTR_ERR(net); 1264 1265 peer = rtnl_create_link(net, ifname, name_assign_type, 1266 &veth_link_ops, tbp, extack); 1267 if (IS_ERR(peer)) { 1268 put_net(net); 1269 return PTR_ERR(peer); 1270 } 1271 1272 if (!ifmp || !tbp[IFLA_ADDRESS]) 1273 eth_hw_addr_random(peer); 1274 1275 if (ifmp && (dev->ifindex != 0)) 1276 peer->ifindex = ifmp->ifi_index; 1277 1278 peer->gso_max_size = dev->gso_max_size; 1279 peer->gso_max_segs = dev->gso_max_segs; 1280 1281 err = register_netdevice(peer); 1282 put_net(net); 1283 net = NULL; 1284 if (err < 0) 1285 goto err_register_peer; 1286 1287 netif_carrier_off(peer); 1288 1289 err = rtnl_configure_link(peer, ifmp); 1290 if (err < 0) 1291 goto err_configure_peer; 1292 1293 /* 1294 * register dev last 1295 * 1296 * note, that since we've registered new device the dev's name 1297 * should be re-allocated 1298 */ 1299 1300 if (tb[IFLA_ADDRESS] == NULL) 1301 eth_hw_addr_random(dev); 1302 1303 if (tb[IFLA_IFNAME]) 1304 nla_strlcpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 1305 else 1306 snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); 1307 1308 err = register_netdevice(dev); 1309 if (err < 0) 1310 goto err_register_dev; 1311 1312 netif_carrier_off(dev); 1313 1314 /* 1315 * tie the deviced together 1316 */ 1317 1318 priv = netdev_priv(dev); 1319 rcu_assign_pointer(priv->peer, peer); 1320 1321 priv = netdev_priv(peer); 1322 rcu_assign_pointer(priv->peer, dev); 1323 1324 return 0; 1325 1326 err_register_dev: 1327 /* nothing to do */ 1328 err_configure_peer: 1329 unregister_netdevice(peer); 1330 return err; 1331 1332 err_register_peer: 1333 free_netdev(peer); 1334 return err; 1335 } 1336 1337 static void veth_dellink(struct net_device *dev, struct list_head *head) 1338 { 1339 struct veth_priv *priv; 1340 struct net_device *peer; 1341 1342 priv = netdev_priv(dev); 1343 peer = rtnl_dereference(priv->peer); 1344 1345 /* Note : dellink() is called from default_device_exit_batch(), 1346 * before a rcu_synchronize() point. The devices are guaranteed 1347 * not being freed before one RCU grace period. 1348 */ 1349 RCU_INIT_POINTER(priv->peer, NULL); 1350 unregister_netdevice_queue(dev, head); 1351 1352 if (peer) { 1353 priv = netdev_priv(peer); 1354 RCU_INIT_POINTER(priv->peer, NULL); 1355 unregister_netdevice_queue(peer, head); 1356 } 1357 } 1358 1359 static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { 1360 [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, 1361 }; 1362 1363 static struct net *veth_get_link_net(const struct net_device *dev) 1364 { 1365 struct veth_priv *priv = netdev_priv(dev); 1366 struct net_device *peer = rtnl_dereference(priv->peer); 1367 1368 return peer ? dev_net(peer) : dev_net(dev); 1369 } 1370 1371 static struct rtnl_link_ops veth_link_ops = { 1372 .kind = DRV_NAME, 1373 .priv_size = sizeof(struct veth_priv), 1374 .setup = veth_setup, 1375 .validate = veth_validate, 1376 .newlink = veth_newlink, 1377 .dellink = veth_dellink, 1378 .policy = veth_policy, 1379 .maxtype = VETH_INFO_MAX, 1380 .get_link_net = veth_get_link_net, 1381 }; 1382 1383 /* 1384 * init/fini 1385 */ 1386 1387 static __init int veth_init(void) 1388 { 1389 return rtnl_link_register(&veth_link_ops); 1390 } 1391 1392 static __exit void veth_exit(void) 1393 { 1394 rtnl_link_unregister(&veth_link_ops); 1395 } 1396 1397 module_init(veth_init); 1398 module_exit(veth_exit); 1399 1400 MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); 1401 MODULE_LICENSE("GPL v2"); 1402 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1403