1 /* 2 * Virtual network driver for conversing with remote driver backends. 3 * 4 * Copyright (c) 2002-2005, K A Fraser 5 * Copyright (c) 2005, XenSource Ltd 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License version 2 9 * as published by the Free Software Foundation; or, when distributed 10 * separately from the Linux kernel or incorporated into other 11 * software packages, subject to the following license: 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a copy 14 * of this source file (the "Software"), to deal in the Software without 15 * restriction, including without limitation the rights to use, copy, modify, 16 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 17 * and to permit persons to whom the Software is furnished to do so, subject to 18 * the following conditions: 19 * 20 * The above copyright notice and this permission notice shall be included in 21 * all copies or substantial portions of the Software. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 29 * IN THE SOFTWARE. 30 */ 31 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34 #include <linux/module.h> 35 #include <linux/kernel.h> 36 #include <linux/netdevice.h> 37 #include <linux/etherdevice.h> 38 #include <linux/skbuff.h> 39 #include <linux/ethtool.h> 40 #include <linux/if_ether.h> 41 #include <net/tcp.h> 42 #include <linux/udp.h> 43 #include <linux/moduleparam.h> 44 #include <linux/mm.h> 45 #include <linux/slab.h> 46 #include <net/ip.h> 47 #include <linux/bpf.h> 48 #include <net/page_pool.h> 49 #include <linux/bpf_trace.h> 50 51 #include <xen/xen.h> 52 #include <xen/xenbus.h> 53 #include <xen/events.h> 54 #include <xen/page.h> 55 #include <xen/platform_pci.h> 56 #include <xen/grant_table.h> 57 58 #include <xen/interface/io/netif.h> 59 #include <xen/interface/memory.h> 60 #include <xen/interface/grant_table.h> 61 62 /* Module parameters */ 63 #define MAX_QUEUES_DEFAULT 8 64 static unsigned int xennet_max_queues; 65 module_param_named(max_queues, xennet_max_queues, uint, 0644); 66 MODULE_PARM_DESC(max_queues, 67 "Maximum number of queues per virtual interface"); 68 69 #define XENNET_TIMEOUT (5 * HZ) 70 71 static const struct ethtool_ops xennet_ethtool_ops; 72 73 struct netfront_cb { 74 int pull_to; 75 }; 76 77 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb)) 78 79 #define RX_COPY_THRESHOLD 256 80 81 #define GRANT_INVALID_REF 0 82 83 #define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE) 84 #define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE) 85 86 /* Minimum number of Rx slots (includes slot for GSO metadata). */ 87 #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1) 88 89 /* Queue name is interface name with "-qNNN" appended */ 90 #define QUEUE_NAME_SIZE (IFNAMSIZ + 6) 91 92 /* IRQ name is queue name with "-tx" or "-rx" appended */ 93 #define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3) 94 95 static DECLARE_WAIT_QUEUE_HEAD(module_wq); 96 97 struct netfront_stats { 98 u64 packets; 99 u64 bytes; 100 struct u64_stats_sync syncp; 101 }; 102 103 struct netfront_info; 104 105 struct netfront_queue { 106 unsigned int id; /* Queue ID, 0-based */ 107 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ 108 struct netfront_info *info; 109 110 struct bpf_prog __rcu *xdp_prog; 111 112 struct napi_struct napi; 113 114 /* Split event channels support, tx_* == rx_* when using 115 * single event channel. 116 */ 117 unsigned int tx_evtchn, rx_evtchn; 118 unsigned int tx_irq, rx_irq; 119 /* Only used when split event channels support is enabled */ 120 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */ 121 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */ 122 123 spinlock_t tx_lock; 124 struct xen_netif_tx_front_ring tx; 125 int tx_ring_ref; 126 127 /* 128 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries 129 * are linked from tx_skb_freelist through tx_link. 130 */ 131 struct sk_buff *tx_skbs[NET_TX_RING_SIZE]; 132 unsigned short tx_link[NET_TX_RING_SIZE]; 133 #define TX_LINK_NONE 0xffff 134 #define TX_PENDING 0xfffe 135 grant_ref_t gref_tx_head; 136 grant_ref_t grant_tx_ref[NET_TX_RING_SIZE]; 137 struct page *grant_tx_page[NET_TX_RING_SIZE]; 138 unsigned tx_skb_freelist; 139 unsigned int tx_pend_queue; 140 141 spinlock_t rx_lock ____cacheline_aligned_in_smp; 142 struct xen_netif_rx_front_ring rx; 143 int rx_ring_ref; 144 145 struct timer_list rx_refill_timer; 146 147 struct sk_buff *rx_skbs[NET_RX_RING_SIZE]; 148 grant_ref_t gref_rx_head; 149 grant_ref_t grant_rx_ref[NET_RX_RING_SIZE]; 150 151 unsigned int rx_rsp_unconsumed; 152 spinlock_t rx_cons_lock; 153 154 struct page_pool *page_pool; 155 struct xdp_rxq_info xdp_rxq; 156 }; 157 158 struct netfront_info { 159 struct list_head list; 160 struct net_device *netdev; 161 162 struct xenbus_device *xbdev; 163 164 /* Multi-queue support */ 165 struct netfront_queue *queues; 166 167 /* Statistics */ 168 struct netfront_stats __percpu *rx_stats; 169 struct netfront_stats __percpu *tx_stats; 170 171 /* XDP state */ 172 bool netback_has_xdp_headroom; 173 bool netfront_xdp_enabled; 174 175 /* Is device behaving sane? */ 176 bool broken; 177 178 atomic_t rx_gso_checksum_fixup; 179 }; 180 181 struct netfront_rx_info { 182 struct xen_netif_rx_response rx; 183 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; 184 }; 185 186 /* 187 * Access macros for acquiring freeing slots in tx_skbs[]. 188 */ 189 190 static void add_id_to_list(unsigned *head, unsigned short *list, 191 unsigned short id) 192 { 193 list[id] = *head; 194 *head = id; 195 } 196 197 static unsigned short get_id_from_list(unsigned *head, unsigned short *list) 198 { 199 unsigned int id = *head; 200 201 if (id != TX_LINK_NONE) { 202 *head = list[id]; 203 list[id] = TX_LINK_NONE; 204 } 205 return id; 206 } 207 208 static int xennet_rxidx(RING_IDX idx) 209 { 210 return idx & (NET_RX_RING_SIZE - 1); 211 } 212 213 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, 214 RING_IDX ri) 215 { 216 int i = xennet_rxidx(ri); 217 struct sk_buff *skb = queue->rx_skbs[i]; 218 queue->rx_skbs[i] = NULL; 219 return skb; 220 } 221 222 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, 223 RING_IDX ri) 224 { 225 int i = xennet_rxidx(ri); 226 grant_ref_t ref = queue->grant_rx_ref[i]; 227 queue->grant_rx_ref[i] = GRANT_INVALID_REF; 228 return ref; 229 } 230 231 #ifdef CONFIG_SYSFS 232 static const struct attribute_group xennet_dev_group; 233 #endif 234 235 static bool xennet_can_sg(struct net_device *dev) 236 { 237 return dev->features & NETIF_F_SG; 238 } 239 240 241 static void rx_refill_timeout(struct timer_list *t) 242 { 243 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); 244 napi_schedule(&queue->napi); 245 } 246 247 static int netfront_tx_slot_available(struct netfront_queue *queue) 248 { 249 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < 250 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); 251 } 252 253 static void xennet_maybe_wake_tx(struct netfront_queue *queue) 254 { 255 struct net_device *dev = queue->info->netdev; 256 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); 257 258 if (unlikely(netif_tx_queue_stopped(dev_queue)) && 259 netfront_tx_slot_available(queue) && 260 likely(netif_running(dev))) 261 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); 262 } 263 264 265 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) 266 { 267 struct sk_buff *skb; 268 struct page *page; 269 270 skb = __netdev_alloc_skb(queue->info->netdev, 271 RX_COPY_THRESHOLD + NET_IP_ALIGN, 272 GFP_ATOMIC | __GFP_NOWARN); 273 if (unlikely(!skb)) 274 return NULL; 275 276 page = page_pool_dev_alloc_pages(queue->page_pool); 277 if (unlikely(!page)) { 278 kfree_skb(skb); 279 return NULL; 280 } 281 skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE); 282 283 /* Align ip header to a 16 bytes boundary */ 284 skb_reserve(skb, NET_IP_ALIGN); 285 skb->dev = queue->info->netdev; 286 287 return skb; 288 } 289 290 291 static void xennet_alloc_rx_buffers(struct netfront_queue *queue) 292 { 293 RING_IDX req_prod = queue->rx.req_prod_pvt; 294 int notify; 295 int err = 0; 296 297 if (unlikely(!netif_carrier_ok(queue->info->netdev))) 298 return; 299 300 for (req_prod = queue->rx.req_prod_pvt; 301 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; 302 req_prod++) { 303 struct sk_buff *skb; 304 unsigned short id; 305 grant_ref_t ref; 306 struct page *page; 307 struct xen_netif_rx_request *req; 308 309 skb = xennet_alloc_one_rx_buffer(queue); 310 if (!skb) { 311 err = -ENOMEM; 312 break; 313 } 314 315 id = xennet_rxidx(req_prod); 316 317 BUG_ON(queue->rx_skbs[id]); 318 queue->rx_skbs[id] = skb; 319 320 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); 321 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); 322 queue->grant_rx_ref[id] = ref; 323 324 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); 325 326 req = RING_GET_REQUEST(&queue->rx, req_prod); 327 gnttab_page_grant_foreign_access_ref_one(ref, 328 queue->info->xbdev->otherend_id, 329 page, 330 0); 331 req->id = id; 332 req->gref = ref; 333 } 334 335 queue->rx.req_prod_pvt = req_prod; 336 337 /* Try again later if there are not enough requests or skb allocation 338 * failed. 339 * Enough requests is quantified as the sum of newly created slots and 340 * the unconsumed slots at the backend. 341 */ 342 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || 343 unlikely(err)) { 344 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); 345 return; 346 } 347 348 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); 349 if (notify) 350 notify_remote_via_irq(queue->rx_irq); 351 } 352 353 static int xennet_open(struct net_device *dev) 354 { 355 struct netfront_info *np = netdev_priv(dev); 356 unsigned int num_queues = dev->real_num_tx_queues; 357 unsigned int i = 0; 358 struct netfront_queue *queue = NULL; 359 360 if (!np->queues || np->broken) 361 return -ENODEV; 362 363 for (i = 0; i < num_queues; ++i) { 364 queue = &np->queues[i]; 365 napi_enable(&queue->napi); 366 367 spin_lock_bh(&queue->rx_lock); 368 if (netif_carrier_ok(dev)) { 369 xennet_alloc_rx_buffers(queue); 370 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; 371 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) 372 napi_schedule(&queue->napi); 373 } 374 spin_unlock_bh(&queue->rx_lock); 375 } 376 377 netif_tx_start_all_queues(dev); 378 379 return 0; 380 } 381 382 static bool xennet_tx_buf_gc(struct netfront_queue *queue) 383 { 384 RING_IDX cons, prod; 385 unsigned short id; 386 struct sk_buff *skb; 387 bool more_to_do; 388 bool work_done = false; 389 const struct device *dev = &queue->info->netdev->dev; 390 391 BUG_ON(!netif_carrier_ok(queue->info->netdev)); 392 393 do { 394 prod = queue->tx.sring->rsp_prod; 395 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { 396 dev_alert(dev, "Illegal number of responses %u\n", 397 prod - queue->tx.rsp_cons); 398 goto err; 399 } 400 rmb(); /* Ensure we see responses up to 'rp'. */ 401 402 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { 403 struct xen_netif_tx_response txrsp; 404 405 work_done = true; 406 407 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); 408 if (txrsp.status == XEN_NETIF_RSP_NULL) 409 continue; 410 411 id = txrsp.id; 412 if (id >= RING_SIZE(&queue->tx)) { 413 dev_alert(dev, 414 "Response has incorrect id (%u)\n", 415 id); 416 goto err; 417 } 418 if (queue->tx_link[id] != TX_PENDING) { 419 dev_alert(dev, 420 "Response for inactive request\n"); 421 goto err; 422 } 423 424 queue->tx_link[id] = TX_LINK_NONE; 425 skb = queue->tx_skbs[id]; 426 queue->tx_skbs[id] = NULL; 427 if (unlikely(gnttab_query_foreign_access( 428 queue->grant_tx_ref[id]) != 0)) { 429 dev_alert(dev, 430 "Grant still in use by backend domain\n"); 431 goto err; 432 } 433 gnttab_end_foreign_access_ref( 434 queue->grant_tx_ref[id], GNTMAP_readonly); 435 gnttab_release_grant_reference( 436 &queue->gref_tx_head, queue->grant_tx_ref[id]); 437 queue->grant_tx_ref[id] = GRANT_INVALID_REF; 438 queue->grant_tx_page[id] = NULL; 439 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); 440 dev_kfree_skb_irq(skb); 441 } 442 443 queue->tx.rsp_cons = prod; 444 445 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); 446 } while (more_to_do); 447 448 xennet_maybe_wake_tx(queue); 449 450 return work_done; 451 452 err: 453 queue->info->broken = true; 454 dev_alert(dev, "Disabled for further use\n"); 455 456 return work_done; 457 } 458 459 struct xennet_gnttab_make_txreq { 460 struct netfront_queue *queue; 461 struct sk_buff *skb; 462 struct page *page; 463 struct xen_netif_tx_request *tx; /* Last request on ring page */ 464 struct xen_netif_tx_request tx_local; /* Last request local copy*/ 465 unsigned int size; 466 }; 467 468 static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset, 469 unsigned int len, void *data) 470 { 471 struct xennet_gnttab_make_txreq *info = data; 472 unsigned int id; 473 struct xen_netif_tx_request *tx; 474 grant_ref_t ref; 475 /* convenient aliases */ 476 struct page *page = info->page; 477 struct netfront_queue *queue = info->queue; 478 struct sk_buff *skb = info->skb; 479 480 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); 481 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); 482 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); 483 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); 484 485 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, 486 gfn, GNTMAP_readonly); 487 488 queue->tx_skbs[id] = skb; 489 queue->grant_tx_page[id] = page; 490 queue->grant_tx_ref[id] = ref; 491 492 info->tx_local.id = id; 493 info->tx_local.gref = ref; 494 info->tx_local.offset = offset; 495 info->tx_local.size = len; 496 info->tx_local.flags = 0; 497 498 *tx = info->tx_local; 499 500 /* 501 * Put the request in the pending queue, it will be set to be pending 502 * when the producer index is about to be raised. 503 */ 504 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); 505 506 info->tx = tx; 507 info->size += info->tx_local.size; 508 } 509 510 static struct xen_netif_tx_request *xennet_make_first_txreq( 511 struct xennet_gnttab_make_txreq *info, 512 unsigned int offset, unsigned int len) 513 { 514 info->size = 0; 515 516 gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info); 517 518 return info->tx; 519 } 520 521 static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset, 522 unsigned int len, void *data) 523 { 524 struct xennet_gnttab_make_txreq *info = data; 525 526 info->tx->flags |= XEN_NETTXF_more_data; 527 skb_get(info->skb); 528 xennet_tx_setup_grant(gfn, offset, len, data); 529 } 530 531 static void xennet_make_txreqs( 532 struct xennet_gnttab_make_txreq *info, 533 struct page *page, 534 unsigned int offset, unsigned int len) 535 { 536 /* Skip unused frames from start of page */ 537 page += offset >> PAGE_SHIFT; 538 offset &= ~PAGE_MASK; 539 540 while (len) { 541 info->page = page; 542 info->size = 0; 543 544 gnttab_foreach_grant_in_range(page, offset, len, 545 xennet_make_one_txreq, 546 info); 547 548 page++; 549 offset = 0; 550 len -= info->size; 551 } 552 } 553 554 /* 555 * Count how many ring slots are required to send this skb. Each frag 556 * might be a compound page. 557 */ 558 static int xennet_count_skb_slots(struct sk_buff *skb) 559 { 560 int i, frags = skb_shinfo(skb)->nr_frags; 561 int slots; 562 563 slots = gnttab_count_grant(offset_in_page(skb->data), 564 skb_headlen(skb)); 565 566 for (i = 0; i < frags; i++) { 567 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 568 unsigned long size = skb_frag_size(frag); 569 unsigned long offset = skb_frag_off(frag); 570 571 /* Skip unused frames from start of page */ 572 offset &= ~PAGE_MASK; 573 574 slots += gnttab_count_grant(offset, size); 575 } 576 577 return slots; 578 } 579 580 static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb, 581 struct net_device *sb_dev) 582 { 583 unsigned int num_queues = dev->real_num_tx_queues; 584 u32 hash; 585 u16 queue_idx; 586 587 /* First, check if there is only one queue */ 588 if (num_queues == 1) { 589 queue_idx = 0; 590 } else { 591 hash = skb_get_hash(skb); 592 queue_idx = hash % num_queues; 593 } 594 595 return queue_idx; 596 } 597 598 static void xennet_mark_tx_pending(struct netfront_queue *queue) 599 { 600 unsigned int i; 601 602 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != 603 TX_LINK_NONE) 604 queue->tx_link[i] = TX_PENDING; 605 } 606 607 static int xennet_xdp_xmit_one(struct net_device *dev, 608 struct netfront_queue *queue, 609 struct xdp_frame *xdpf) 610 { 611 struct netfront_info *np = netdev_priv(dev); 612 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); 613 struct xennet_gnttab_make_txreq info = { 614 .queue = queue, 615 .skb = NULL, 616 .page = virt_to_page(xdpf->data), 617 }; 618 int notify; 619 620 xennet_make_first_txreq(&info, 621 offset_in_page(xdpf->data), 622 xdpf->len); 623 624 xennet_mark_tx_pending(queue); 625 626 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); 627 if (notify) 628 notify_remote_via_irq(queue->tx_irq); 629 630 u64_stats_update_begin(&tx_stats->syncp); 631 tx_stats->bytes += xdpf->len; 632 tx_stats->packets++; 633 u64_stats_update_end(&tx_stats->syncp); 634 635 xennet_tx_buf_gc(queue); 636 637 return 0; 638 } 639 640 static int xennet_xdp_xmit(struct net_device *dev, int n, 641 struct xdp_frame **frames, u32 flags) 642 { 643 unsigned int num_queues = dev->real_num_tx_queues; 644 struct netfront_info *np = netdev_priv(dev); 645 struct netfront_queue *queue = NULL; 646 unsigned long irq_flags; 647 int nxmit = 0; 648 int i; 649 650 if (unlikely(np->broken)) 651 return -ENODEV; 652 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 653 return -EINVAL; 654 655 queue = &np->queues[smp_processor_id() % num_queues]; 656 657 spin_lock_irqsave(&queue->tx_lock, irq_flags); 658 for (i = 0; i < n; i++) { 659 struct xdp_frame *xdpf = frames[i]; 660 661 if (!xdpf) 662 continue; 663 if (xennet_xdp_xmit_one(dev, queue, xdpf)) 664 break; 665 nxmit++; 666 } 667 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); 668 669 return nxmit; 670 } 671 672 673 #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) 674 675 static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) 676 { 677 struct netfront_info *np = netdev_priv(dev); 678 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); 679 struct xen_netif_tx_request *first_tx; 680 unsigned int i; 681 int notify; 682 int slots; 683 struct page *page; 684 unsigned int offset; 685 unsigned int len; 686 unsigned long flags; 687 struct netfront_queue *queue = NULL; 688 struct xennet_gnttab_make_txreq info = { }; 689 unsigned int num_queues = dev->real_num_tx_queues; 690 u16 queue_index; 691 struct sk_buff *nskb; 692 693 /* Drop the packet if no queues are set up */ 694 if (num_queues < 1) 695 goto drop; 696 if (unlikely(np->broken)) 697 goto drop; 698 /* Determine which queue to transmit this SKB on */ 699 queue_index = skb_get_queue_mapping(skb); 700 queue = &np->queues[queue_index]; 701 702 /* If skb->len is too big for wire format, drop skb and alert 703 * user about misconfiguration. 704 */ 705 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { 706 net_alert_ratelimited( 707 "xennet: skb->len = %u, too big for wire format\n", 708 skb->len); 709 goto drop; 710 } 711 712 slots = xennet_count_skb_slots(skb); 713 if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) { 714 net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n", 715 slots, skb->len); 716 if (skb_linearize(skb)) 717 goto drop; 718 } 719 720 page = virt_to_page(skb->data); 721 offset = offset_in_page(skb->data); 722 723 /* The first req should be at least ETH_HLEN size or the packet will be 724 * dropped by netback. 725 */ 726 if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { 727 nskb = skb_copy(skb, GFP_ATOMIC); 728 if (!nskb) 729 goto drop; 730 dev_consume_skb_any(skb); 731 skb = nskb; 732 page = virt_to_page(skb->data); 733 offset = offset_in_page(skb->data); 734 } 735 736 len = skb_headlen(skb); 737 738 spin_lock_irqsave(&queue->tx_lock, flags); 739 740 if (unlikely(!netif_carrier_ok(dev) || 741 (slots > 1 && !xennet_can_sg(dev)) || 742 netif_needs_gso(skb, netif_skb_features(skb)))) { 743 spin_unlock_irqrestore(&queue->tx_lock, flags); 744 goto drop; 745 } 746 747 /* First request for the linear area. */ 748 info.queue = queue; 749 info.skb = skb; 750 info.page = page; 751 first_tx = xennet_make_first_txreq(&info, offset, len); 752 offset += info.tx_local.size; 753 if (offset == PAGE_SIZE) { 754 page++; 755 offset = 0; 756 } 757 len -= info.tx_local.size; 758 759 if (skb->ip_summed == CHECKSUM_PARTIAL) 760 /* local packet? */ 761 first_tx->flags |= XEN_NETTXF_csum_blank | 762 XEN_NETTXF_data_validated; 763 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) 764 /* remote but checksummed. */ 765 first_tx->flags |= XEN_NETTXF_data_validated; 766 767 /* Optional extra info after the first request. */ 768 if (skb_shinfo(skb)->gso_size) { 769 struct xen_netif_extra_info *gso; 770 771 gso = (struct xen_netif_extra_info *) 772 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); 773 774 first_tx->flags |= XEN_NETTXF_extra_info; 775 776 gso->u.gso.size = skb_shinfo(skb)->gso_size; 777 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? 778 XEN_NETIF_GSO_TYPE_TCPV6 : 779 XEN_NETIF_GSO_TYPE_TCPV4; 780 gso->u.gso.pad = 0; 781 gso->u.gso.features = 0; 782 783 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; 784 gso->flags = 0; 785 } 786 787 /* Requests for the rest of the linear area. */ 788 xennet_make_txreqs(&info, page, offset, len); 789 790 /* Requests for all the frags. */ 791 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 792 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 793 xennet_make_txreqs(&info, skb_frag_page(frag), 794 skb_frag_off(frag), 795 skb_frag_size(frag)); 796 } 797 798 /* First request has the packet length. */ 799 first_tx->size = skb->len; 800 801 /* timestamp packet in software */ 802 skb_tx_timestamp(skb); 803 804 xennet_mark_tx_pending(queue); 805 806 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); 807 if (notify) 808 notify_remote_via_irq(queue->tx_irq); 809 810 u64_stats_update_begin(&tx_stats->syncp); 811 tx_stats->bytes += skb->len; 812 tx_stats->packets++; 813 u64_stats_update_end(&tx_stats->syncp); 814 815 /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ 816 xennet_tx_buf_gc(queue); 817 818 if (!netfront_tx_slot_available(queue)) 819 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); 820 821 spin_unlock_irqrestore(&queue->tx_lock, flags); 822 823 return NETDEV_TX_OK; 824 825 drop: 826 dev->stats.tx_dropped++; 827 dev_kfree_skb_any(skb); 828 return NETDEV_TX_OK; 829 } 830 831 static int xennet_close(struct net_device *dev) 832 { 833 struct netfront_info *np = netdev_priv(dev); 834 unsigned int num_queues = dev->real_num_tx_queues; 835 unsigned int i; 836 struct netfront_queue *queue; 837 netif_tx_stop_all_queues(np->netdev); 838 for (i = 0; i < num_queues; ++i) { 839 queue = &np->queues[i]; 840 napi_disable(&queue->napi); 841 } 842 return 0; 843 } 844 845 static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val) 846 { 847 unsigned long flags; 848 849 spin_lock_irqsave(&queue->rx_cons_lock, flags); 850 queue->rx.rsp_cons = val; 851 queue->rx_rsp_unconsumed = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); 852 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 853 } 854 855 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, 856 grant_ref_t ref) 857 { 858 int new = xennet_rxidx(queue->rx.req_prod_pvt); 859 860 BUG_ON(queue->rx_skbs[new]); 861 queue->rx_skbs[new] = skb; 862 queue->grant_rx_ref[new] = ref; 863 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; 864 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; 865 queue->rx.req_prod_pvt++; 866 } 867 868 static int xennet_get_extras(struct netfront_queue *queue, 869 struct xen_netif_extra_info *extras, 870 RING_IDX rp) 871 872 { 873 struct xen_netif_extra_info extra; 874 struct device *dev = &queue->info->netdev->dev; 875 RING_IDX cons = queue->rx.rsp_cons; 876 int err = 0; 877 878 do { 879 struct sk_buff *skb; 880 grant_ref_t ref; 881 882 if (unlikely(cons + 1 == rp)) { 883 if (net_ratelimit()) 884 dev_warn(dev, "Missing extra info\n"); 885 err = -EBADR; 886 break; 887 } 888 889 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); 890 891 if (unlikely(!extra.type || 892 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 893 if (net_ratelimit()) 894 dev_warn(dev, "Invalid extra type: %d\n", 895 extra.type); 896 err = -EINVAL; 897 } else { 898 extras[extra.type - 1] = extra; 899 } 900 901 skb = xennet_get_rx_skb(queue, cons); 902 ref = xennet_get_rx_ref(queue, cons); 903 xennet_move_rx_slot(queue, skb, ref); 904 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 905 906 xennet_set_rx_rsp_cons(queue, cons); 907 return err; 908 } 909 910 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, 911 struct xen_netif_rx_response *rx, struct bpf_prog *prog, 912 struct xdp_buff *xdp, bool *need_xdp_flush) 913 { 914 struct xdp_frame *xdpf; 915 u32 len = rx->status; 916 u32 act; 917 int err; 918 919 xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, 920 &queue->xdp_rxq); 921 xdp_prepare_buff(xdp, page_address(pdata), XDP_PACKET_HEADROOM, 922 len, false); 923 924 act = bpf_prog_run_xdp(prog, xdp); 925 switch (act) { 926 case XDP_TX: 927 get_page(pdata); 928 xdpf = xdp_convert_buff_to_frame(xdp); 929 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); 930 if (unlikely(!err)) 931 xdp_return_frame_rx_napi(xdpf); 932 else if (unlikely(err < 0)) 933 trace_xdp_exception(queue->info->netdev, prog, act); 934 break; 935 case XDP_REDIRECT: 936 get_page(pdata); 937 err = xdp_do_redirect(queue->info->netdev, xdp, prog); 938 *need_xdp_flush = true; 939 if (unlikely(err)) 940 trace_xdp_exception(queue->info->netdev, prog, act); 941 break; 942 case XDP_PASS: 943 case XDP_DROP: 944 break; 945 946 case XDP_ABORTED: 947 trace_xdp_exception(queue->info->netdev, prog, act); 948 break; 949 950 default: 951 bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act); 952 } 953 954 return act; 955 } 956 957 static int xennet_get_responses(struct netfront_queue *queue, 958 struct netfront_rx_info *rinfo, RING_IDX rp, 959 struct sk_buff_head *list, 960 bool *need_xdp_flush) 961 { 962 struct xen_netif_rx_response *rx = &rinfo->rx, rx_local; 963 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); 964 RING_IDX cons = queue->rx.rsp_cons; 965 struct sk_buff *skb = xennet_get_rx_skb(queue, cons); 966 struct xen_netif_extra_info *extras = rinfo->extras; 967 grant_ref_t ref = xennet_get_rx_ref(queue, cons); 968 struct device *dev = &queue->info->netdev->dev; 969 struct bpf_prog *xdp_prog; 970 struct xdp_buff xdp; 971 unsigned long ret; 972 int slots = 1; 973 int err = 0; 974 u32 verdict; 975 976 if (rx->flags & XEN_NETRXF_extra_info) { 977 err = xennet_get_extras(queue, extras, rp); 978 if (!err) { 979 if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) { 980 struct xen_netif_extra_info *xdp; 981 982 xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1]; 983 rx->offset = xdp->u.xdp.headroom; 984 } 985 } 986 cons = queue->rx.rsp_cons; 987 } 988 989 for (;;) { 990 if (unlikely(rx->status < 0 || 991 rx->offset + rx->status > XEN_PAGE_SIZE)) { 992 if (net_ratelimit()) 993 dev_warn(dev, "rx->offset: %u, size: %d\n", 994 rx->offset, rx->status); 995 xennet_move_rx_slot(queue, skb, ref); 996 err = -EINVAL; 997 goto next; 998 } 999 1000 /* 1001 * This definitely indicates a bug, either in this driver or in 1002 * the backend driver. In future this should flag the bad 1003 * situation to the system controller to reboot the backend. 1004 */ 1005 if (ref == GRANT_INVALID_REF) { 1006 if (net_ratelimit()) 1007 dev_warn(dev, "Bad rx response id %d.\n", 1008 rx->id); 1009 err = -EINVAL; 1010 goto next; 1011 } 1012 1013 ret = gnttab_end_foreign_access_ref(ref, 0); 1014 BUG_ON(!ret); 1015 1016 gnttab_release_grant_reference(&queue->gref_rx_head, ref); 1017 1018 rcu_read_lock(); 1019 xdp_prog = rcu_dereference(queue->xdp_prog); 1020 if (xdp_prog) { 1021 if (!(rx->flags & XEN_NETRXF_more_data)) { 1022 /* currently only a single page contains data */ 1023 verdict = xennet_run_xdp(queue, 1024 skb_frag_page(&skb_shinfo(skb)->frags[0]), 1025 rx, xdp_prog, &xdp, need_xdp_flush); 1026 if (verdict != XDP_PASS) 1027 err = -EINVAL; 1028 } else { 1029 /* drop the frame */ 1030 err = -EINVAL; 1031 } 1032 } 1033 rcu_read_unlock(); 1034 next: 1035 __skb_queue_tail(list, skb); 1036 if (!(rx->flags & XEN_NETRXF_more_data)) 1037 break; 1038 1039 if (cons + slots == rp) { 1040 if (net_ratelimit()) 1041 dev_warn(dev, "Need more slots\n"); 1042 err = -ENOENT; 1043 break; 1044 } 1045 1046 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); 1047 rx = &rx_local; 1048 skb = xennet_get_rx_skb(queue, cons + slots); 1049 ref = xennet_get_rx_ref(queue, cons + slots); 1050 slots++; 1051 } 1052 1053 if (unlikely(slots > max)) { 1054 if (net_ratelimit()) 1055 dev_warn(dev, "Too many slots\n"); 1056 err = -E2BIG; 1057 } 1058 1059 if (unlikely(err)) 1060 xennet_set_rx_rsp_cons(queue, cons + slots); 1061 1062 return err; 1063 } 1064 1065 static int xennet_set_skb_gso(struct sk_buff *skb, 1066 struct xen_netif_extra_info *gso) 1067 { 1068 if (!gso->u.gso.size) { 1069 if (net_ratelimit()) 1070 pr_warn("GSO size must not be zero\n"); 1071 return -EINVAL; 1072 } 1073 1074 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && 1075 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { 1076 if (net_ratelimit()) 1077 pr_warn("Bad GSO type %d\n", gso->u.gso.type); 1078 return -EINVAL; 1079 } 1080 1081 skb_shinfo(skb)->gso_size = gso->u.gso.size; 1082 skb_shinfo(skb)->gso_type = 1083 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? 1084 SKB_GSO_TCPV4 : 1085 SKB_GSO_TCPV6; 1086 1087 /* Header must be checked, and gso_segs computed. */ 1088 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 1089 skb_shinfo(skb)->gso_segs = 0; 1090 1091 return 0; 1092 } 1093 1094 static int xennet_fill_frags(struct netfront_queue *queue, 1095 struct sk_buff *skb, 1096 struct sk_buff_head *list) 1097 { 1098 RING_IDX cons = queue->rx.rsp_cons; 1099 struct sk_buff *nskb; 1100 1101 while ((nskb = __skb_dequeue(list))) { 1102 struct xen_netif_rx_response rx; 1103 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; 1104 1105 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); 1106 1107 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { 1108 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; 1109 1110 BUG_ON(pull_to < skb_headlen(skb)); 1111 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); 1112 } 1113 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { 1114 xennet_set_rx_rsp_cons(queue, 1115 ++cons + skb_queue_len(list)); 1116 kfree_skb(nskb); 1117 return -ENOENT; 1118 } 1119 1120 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 1121 skb_frag_page(nfrag), 1122 rx.offset, rx.status, PAGE_SIZE); 1123 1124 skb_shinfo(nskb)->nr_frags = 0; 1125 kfree_skb(nskb); 1126 } 1127 1128 xennet_set_rx_rsp_cons(queue, cons); 1129 1130 return 0; 1131 } 1132 1133 static int checksum_setup(struct net_device *dev, struct sk_buff *skb) 1134 { 1135 bool recalculate_partial_csum = false; 1136 1137 /* 1138 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 1139 * peers can fail to set NETRXF_csum_blank when sending a GSO 1140 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 1141 * recalculate the partial checksum. 1142 */ 1143 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 1144 struct netfront_info *np = netdev_priv(dev); 1145 atomic_inc(&np->rx_gso_checksum_fixup); 1146 skb->ip_summed = CHECKSUM_PARTIAL; 1147 recalculate_partial_csum = true; 1148 } 1149 1150 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 1151 if (skb->ip_summed != CHECKSUM_PARTIAL) 1152 return 0; 1153 1154 return skb_checksum_setup(skb, recalculate_partial_csum); 1155 } 1156 1157 static int handle_incoming_queue(struct netfront_queue *queue, 1158 struct sk_buff_head *rxq) 1159 { 1160 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); 1161 int packets_dropped = 0; 1162 struct sk_buff *skb; 1163 1164 while ((skb = __skb_dequeue(rxq)) != NULL) { 1165 int pull_to = NETFRONT_SKB_CB(skb)->pull_to; 1166 1167 if (pull_to > skb_headlen(skb)) 1168 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); 1169 1170 /* Ethernet work: Delayed to here as it peeks the header. */ 1171 skb->protocol = eth_type_trans(skb, queue->info->netdev); 1172 skb_reset_network_header(skb); 1173 1174 if (checksum_setup(queue->info->netdev, skb)) { 1175 kfree_skb(skb); 1176 packets_dropped++; 1177 queue->info->netdev->stats.rx_errors++; 1178 continue; 1179 } 1180 1181 u64_stats_update_begin(&rx_stats->syncp); 1182 rx_stats->packets++; 1183 rx_stats->bytes += skb->len; 1184 u64_stats_update_end(&rx_stats->syncp); 1185 1186 /* Pass it up. */ 1187 napi_gro_receive(&queue->napi, skb); 1188 } 1189 1190 return packets_dropped; 1191 } 1192 1193 static int xennet_poll(struct napi_struct *napi, int budget) 1194 { 1195 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); 1196 struct net_device *dev = queue->info->netdev; 1197 struct sk_buff *skb; 1198 struct netfront_rx_info rinfo; 1199 struct xen_netif_rx_response *rx = &rinfo.rx; 1200 struct xen_netif_extra_info *extras = rinfo.extras; 1201 RING_IDX i, rp; 1202 int work_done; 1203 struct sk_buff_head rxq; 1204 struct sk_buff_head errq; 1205 struct sk_buff_head tmpq; 1206 int err; 1207 bool need_xdp_flush = false; 1208 1209 spin_lock(&queue->rx_lock); 1210 1211 skb_queue_head_init(&rxq); 1212 skb_queue_head_init(&errq); 1213 skb_queue_head_init(&tmpq); 1214 1215 rp = queue->rx.sring->rsp_prod; 1216 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { 1217 dev_alert(&dev->dev, "Illegal number of responses %u\n", 1218 rp - queue->rx.rsp_cons); 1219 queue->info->broken = true; 1220 spin_unlock(&queue->rx_lock); 1221 return 0; 1222 } 1223 rmb(); /* Ensure we see queued responses up to 'rp'. */ 1224 1225 i = queue->rx.rsp_cons; 1226 work_done = 0; 1227 while ((i != rp) && (work_done < budget)) { 1228 RING_COPY_RESPONSE(&queue->rx, i, rx); 1229 memset(extras, 0, sizeof(rinfo.extras)); 1230 1231 err = xennet_get_responses(queue, &rinfo, rp, &tmpq, 1232 &need_xdp_flush); 1233 1234 if (unlikely(err)) { 1235 err: 1236 while ((skb = __skb_dequeue(&tmpq))) 1237 __skb_queue_tail(&errq, skb); 1238 dev->stats.rx_errors++; 1239 i = queue->rx.rsp_cons; 1240 continue; 1241 } 1242 1243 skb = __skb_dequeue(&tmpq); 1244 1245 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 1246 struct xen_netif_extra_info *gso; 1247 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1248 1249 if (unlikely(xennet_set_skb_gso(skb, gso))) { 1250 __skb_queue_head(&tmpq, skb); 1251 xennet_set_rx_rsp_cons(queue, 1252 queue->rx.rsp_cons + 1253 skb_queue_len(&tmpq)); 1254 goto err; 1255 } 1256 } 1257 1258 NETFRONT_SKB_CB(skb)->pull_to = rx->status; 1259 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) 1260 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; 1261 1262 skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset); 1263 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); 1264 skb->data_len = rx->status; 1265 skb->len += rx->status; 1266 1267 if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) 1268 goto err; 1269 1270 if (rx->flags & XEN_NETRXF_csum_blank) 1271 skb->ip_summed = CHECKSUM_PARTIAL; 1272 else if (rx->flags & XEN_NETRXF_data_validated) 1273 skb->ip_summed = CHECKSUM_UNNECESSARY; 1274 1275 __skb_queue_tail(&rxq, skb); 1276 1277 i = queue->rx.rsp_cons + 1; 1278 xennet_set_rx_rsp_cons(queue, i); 1279 work_done++; 1280 } 1281 if (need_xdp_flush) 1282 xdp_do_flush(); 1283 1284 __skb_queue_purge(&errq); 1285 1286 work_done -= handle_incoming_queue(queue, &rxq); 1287 1288 xennet_alloc_rx_buffers(queue); 1289 1290 if (work_done < budget) { 1291 int more_to_do = 0; 1292 1293 napi_complete_done(napi, work_done); 1294 1295 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); 1296 if (more_to_do) 1297 napi_schedule(napi); 1298 } 1299 1300 spin_unlock(&queue->rx_lock); 1301 1302 return work_done; 1303 } 1304 1305 static int xennet_change_mtu(struct net_device *dev, int mtu) 1306 { 1307 int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN; 1308 1309 if (mtu > max) 1310 return -EINVAL; 1311 dev->mtu = mtu; 1312 return 0; 1313 } 1314 1315 static void xennet_get_stats64(struct net_device *dev, 1316 struct rtnl_link_stats64 *tot) 1317 { 1318 struct netfront_info *np = netdev_priv(dev); 1319 int cpu; 1320 1321 for_each_possible_cpu(cpu) { 1322 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); 1323 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); 1324 u64 rx_packets, rx_bytes, tx_packets, tx_bytes; 1325 unsigned int start; 1326 1327 do { 1328 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 1329 tx_packets = tx_stats->packets; 1330 tx_bytes = tx_stats->bytes; 1331 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 1332 1333 do { 1334 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 1335 rx_packets = rx_stats->packets; 1336 rx_bytes = rx_stats->bytes; 1337 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 1338 1339 tot->rx_packets += rx_packets; 1340 tot->tx_packets += tx_packets; 1341 tot->rx_bytes += rx_bytes; 1342 tot->tx_bytes += tx_bytes; 1343 } 1344 1345 tot->rx_errors = dev->stats.rx_errors; 1346 tot->tx_dropped = dev->stats.tx_dropped; 1347 } 1348 1349 static void xennet_release_tx_bufs(struct netfront_queue *queue) 1350 { 1351 struct sk_buff *skb; 1352 int i; 1353 1354 for (i = 0; i < NET_TX_RING_SIZE; i++) { 1355 /* Skip over entries which are actually freelist references */ 1356 if (!queue->tx_skbs[i]) 1357 continue; 1358 1359 skb = queue->tx_skbs[i]; 1360 queue->tx_skbs[i] = NULL; 1361 get_page(queue->grant_tx_page[i]); 1362 gnttab_end_foreign_access(queue->grant_tx_ref[i], 1363 GNTMAP_readonly, 1364 (unsigned long)page_address(queue->grant_tx_page[i])); 1365 queue->grant_tx_page[i] = NULL; 1366 queue->grant_tx_ref[i] = GRANT_INVALID_REF; 1367 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); 1368 dev_kfree_skb_irq(skb); 1369 } 1370 } 1371 1372 static void xennet_release_rx_bufs(struct netfront_queue *queue) 1373 { 1374 int id, ref; 1375 1376 spin_lock_bh(&queue->rx_lock); 1377 1378 for (id = 0; id < NET_RX_RING_SIZE; id++) { 1379 struct sk_buff *skb; 1380 struct page *page; 1381 1382 skb = queue->rx_skbs[id]; 1383 if (!skb) 1384 continue; 1385 1386 ref = queue->grant_rx_ref[id]; 1387 if (ref == GRANT_INVALID_REF) 1388 continue; 1389 1390 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); 1391 1392 /* gnttab_end_foreign_access() needs a page ref until 1393 * foreign access is ended (which may be deferred). 1394 */ 1395 get_page(page); 1396 gnttab_end_foreign_access(ref, 0, 1397 (unsigned long)page_address(page)); 1398 queue->grant_rx_ref[id] = GRANT_INVALID_REF; 1399 1400 kfree_skb(skb); 1401 } 1402 1403 spin_unlock_bh(&queue->rx_lock); 1404 } 1405 1406 static netdev_features_t xennet_fix_features(struct net_device *dev, 1407 netdev_features_t features) 1408 { 1409 struct netfront_info *np = netdev_priv(dev); 1410 1411 if (features & NETIF_F_SG && 1412 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) 1413 features &= ~NETIF_F_SG; 1414 1415 if (features & NETIF_F_IPV6_CSUM && 1416 !xenbus_read_unsigned(np->xbdev->otherend, 1417 "feature-ipv6-csum-offload", 0)) 1418 features &= ~NETIF_F_IPV6_CSUM; 1419 1420 if (features & NETIF_F_TSO && 1421 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) 1422 features &= ~NETIF_F_TSO; 1423 1424 if (features & NETIF_F_TSO6 && 1425 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) 1426 features &= ~NETIF_F_TSO6; 1427 1428 return features; 1429 } 1430 1431 static int xennet_set_features(struct net_device *dev, 1432 netdev_features_t features) 1433 { 1434 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { 1435 netdev_info(dev, "Reducing MTU because no SG offload"); 1436 dev->mtu = ETH_DATA_LEN; 1437 } 1438 1439 return 0; 1440 } 1441 1442 static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi) 1443 { 1444 unsigned long flags; 1445 1446 if (unlikely(queue->info->broken)) 1447 return false; 1448 1449 spin_lock_irqsave(&queue->tx_lock, flags); 1450 if (xennet_tx_buf_gc(queue)) 1451 *eoi = 0; 1452 spin_unlock_irqrestore(&queue->tx_lock, flags); 1453 1454 return true; 1455 } 1456 1457 static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id) 1458 { 1459 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1460 1461 if (likely(xennet_handle_tx(dev_id, &eoiflag))) 1462 xen_irq_lateeoi(irq, eoiflag); 1463 1464 return IRQ_HANDLED; 1465 } 1466 1467 static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi) 1468 { 1469 unsigned int work_queued; 1470 unsigned long flags; 1471 1472 if (unlikely(queue->info->broken)) 1473 return false; 1474 1475 spin_lock_irqsave(&queue->rx_cons_lock, flags); 1476 work_queued = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); 1477 if (work_queued > queue->rx_rsp_unconsumed) { 1478 queue->rx_rsp_unconsumed = work_queued; 1479 *eoi = 0; 1480 } else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) { 1481 const struct device *dev = &queue->info->netdev->dev; 1482 1483 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 1484 dev_alert(dev, "RX producer index going backwards\n"); 1485 dev_alert(dev, "Disabled for further use\n"); 1486 queue->info->broken = true; 1487 return false; 1488 } 1489 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); 1490 1491 if (likely(netif_carrier_ok(queue->info->netdev) && work_queued)) 1492 napi_schedule(&queue->napi); 1493 1494 return true; 1495 } 1496 1497 static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id) 1498 { 1499 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1500 1501 if (likely(xennet_handle_rx(dev_id, &eoiflag))) 1502 xen_irq_lateeoi(irq, eoiflag); 1503 1504 return IRQ_HANDLED; 1505 } 1506 1507 static irqreturn_t xennet_interrupt(int irq, void *dev_id) 1508 { 1509 unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; 1510 1511 if (xennet_handle_tx(dev_id, &eoiflag) && 1512 xennet_handle_rx(dev_id, &eoiflag)) 1513 xen_irq_lateeoi(irq, eoiflag); 1514 1515 return IRQ_HANDLED; 1516 } 1517 1518 #ifdef CONFIG_NET_POLL_CONTROLLER 1519 static void xennet_poll_controller(struct net_device *dev) 1520 { 1521 /* Poll each queue */ 1522 struct netfront_info *info = netdev_priv(dev); 1523 unsigned int num_queues = dev->real_num_tx_queues; 1524 unsigned int i; 1525 1526 if (info->broken) 1527 return; 1528 1529 for (i = 0; i < num_queues; ++i) 1530 xennet_interrupt(0, &info->queues[i]); 1531 } 1532 #endif 1533 1534 #define NETBACK_XDP_HEADROOM_DISABLE 0 1535 #define NETBACK_XDP_HEADROOM_ENABLE 1 1536 1537 static int talk_to_netback_xdp(struct netfront_info *np, int xdp) 1538 { 1539 int err; 1540 unsigned short headroom; 1541 1542 headroom = xdp ? XDP_PACKET_HEADROOM : 0; 1543 err = xenbus_printf(XBT_NIL, np->xbdev->nodename, 1544 "xdp-headroom", "%hu", 1545 headroom); 1546 if (err) 1547 pr_warn("Error writing xdp-headroom\n"); 1548 1549 return err; 1550 } 1551 1552 static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1553 struct netlink_ext_ack *extack) 1554 { 1555 unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; 1556 struct netfront_info *np = netdev_priv(dev); 1557 struct bpf_prog *old_prog; 1558 unsigned int i, err; 1559 1560 if (dev->mtu > max_mtu) { 1561 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu); 1562 return -EINVAL; 1563 } 1564 1565 if (!np->netback_has_xdp_headroom) 1566 return 0; 1567 1568 xenbus_switch_state(np->xbdev, XenbusStateReconfiguring); 1569 1570 err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE : 1571 NETBACK_XDP_HEADROOM_DISABLE); 1572 if (err) 1573 return err; 1574 1575 /* avoid the race with XDP headroom adjustment */ 1576 wait_event(module_wq, 1577 xenbus_read_driver_state(np->xbdev->otherend) == 1578 XenbusStateReconfigured); 1579 np->netfront_xdp_enabled = true; 1580 1581 old_prog = rtnl_dereference(np->queues[0].xdp_prog); 1582 1583 if (prog) 1584 bpf_prog_add(prog, dev->real_num_tx_queues); 1585 1586 for (i = 0; i < dev->real_num_tx_queues; ++i) 1587 rcu_assign_pointer(np->queues[i].xdp_prog, prog); 1588 1589 if (old_prog) 1590 for (i = 0; i < dev->real_num_tx_queues; ++i) 1591 bpf_prog_put(old_prog); 1592 1593 xenbus_switch_state(np->xbdev, XenbusStateConnected); 1594 1595 return 0; 1596 } 1597 1598 static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1599 { 1600 struct netfront_info *np = netdev_priv(dev); 1601 1602 if (np->broken) 1603 return -ENODEV; 1604 1605 switch (xdp->command) { 1606 case XDP_SETUP_PROG: 1607 return xennet_xdp_set(dev, xdp->prog, xdp->extack); 1608 default: 1609 return -EINVAL; 1610 } 1611 } 1612 1613 static const struct net_device_ops xennet_netdev_ops = { 1614 .ndo_open = xennet_open, 1615 .ndo_stop = xennet_close, 1616 .ndo_start_xmit = xennet_start_xmit, 1617 .ndo_change_mtu = xennet_change_mtu, 1618 .ndo_get_stats64 = xennet_get_stats64, 1619 .ndo_set_mac_address = eth_mac_addr, 1620 .ndo_validate_addr = eth_validate_addr, 1621 .ndo_fix_features = xennet_fix_features, 1622 .ndo_set_features = xennet_set_features, 1623 .ndo_select_queue = xennet_select_queue, 1624 .ndo_bpf = xennet_xdp, 1625 .ndo_xdp_xmit = xennet_xdp_xmit, 1626 #ifdef CONFIG_NET_POLL_CONTROLLER 1627 .ndo_poll_controller = xennet_poll_controller, 1628 #endif 1629 }; 1630 1631 static void xennet_free_netdev(struct net_device *netdev) 1632 { 1633 struct netfront_info *np = netdev_priv(netdev); 1634 1635 free_percpu(np->rx_stats); 1636 free_percpu(np->tx_stats); 1637 free_netdev(netdev); 1638 } 1639 1640 static struct net_device *xennet_create_dev(struct xenbus_device *dev) 1641 { 1642 int err; 1643 struct net_device *netdev; 1644 struct netfront_info *np; 1645 1646 netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues); 1647 if (!netdev) 1648 return ERR_PTR(-ENOMEM); 1649 1650 np = netdev_priv(netdev); 1651 np->xbdev = dev; 1652 1653 np->queues = NULL; 1654 1655 err = -ENOMEM; 1656 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); 1657 if (np->rx_stats == NULL) 1658 goto exit; 1659 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); 1660 if (np->tx_stats == NULL) 1661 goto exit; 1662 1663 netdev->netdev_ops = &xennet_netdev_ops; 1664 1665 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | 1666 NETIF_F_GSO_ROBUST; 1667 netdev->hw_features = NETIF_F_SG | 1668 NETIF_F_IPV6_CSUM | 1669 NETIF_F_TSO | NETIF_F_TSO6; 1670 1671 /* 1672 * Assume that all hw features are available for now. This set 1673 * will be adjusted by the call to netdev_update_features() in 1674 * xennet_connect() which is the earliest point where we can 1675 * negotiate with the backend regarding supported features. 1676 */ 1677 netdev->features |= netdev->hw_features; 1678 1679 netdev->ethtool_ops = &xennet_ethtool_ops; 1680 netdev->min_mtu = ETH_MIN_MTU; 1681 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; 1682 SET_NETDEV_DEV(netdev, &dev->dev); 1683 1684 np->netdev = netdev; 1685 np->netfront_xdp_enabled = false; 1686 1687 netif_carrier_off(netdev); 1688 1689 do { 1690 xenbus_switch_state(dev, XenbusStateInitialising); 1691 err = wait_event_timeout(module_wq, 1692 xenbus_read_driver_state(dev->otherend) != 1693 XenbusStateClosed && 1694 xenbus_read_driver_state(dev->otherend) != 1695 XenbusStateUnknown, XENNET_TIMEOUT); 1696 } while (!err); 1697 1698 return netdev; 1699 1700 exit: 1701 xennet_free_netdev(netdev); 1702 return ERR_PTR(err); 1703 } 1704 1705 /* 1706 * Entry point to this code when a new device is created. Allocate the basic 1707 * structures and the ring buffers for communication with the backend, and 1708 * inform the backend of the appropriate details for those. 1709 */ 1710 static int netfront_probe(struct xenbus_device *dev, 1711 const struct xenbus_device_id *id) 1712 { 1713 int err; 1714 struct net_device *netdev; 1715 struct netfront_info *info; 1716 1717 netdev = xennet_create_dev(dev); 1718 if (IS_ERR(netdev)) { 1719 err = PTR_ERR(netdev); 1720 xenbus_dev_fatal(dev, err, "creating netdev"); 1721 return err; 1722 } 1723 1724 info = netdev_priv(netdev); 1725 dev_set_drvdata(&dev->dev, info); 1726 #ifdef CONFIG_SYSFS 1727 info->netdev->sysfs_groups[0] = &xennet_dev_group; 1728 #endif 1729 1730 return 0; 1731 } 1732 1733 static void xennet_end_access(int ref, void *page) 1734 { 1735 /* This frees the page as a side-effect */ 1736 if (ref != GRANT_INVALID_REF) 1737 gnttab_end_foreign_access(ref, 0, (unsigned long)page); 1738 } 1739 1740 static void xennet_disconnect_backend(struct netfront_info *info) 1741 { 1742 unsigned int i = 0; 1743 unsigned int num_queues = info->netdev->real_num_tx_queues; 1744 1745 netif_carrier_off(info->netdev); 1746 1747 for (i = 0; i < num_queues && info->queues; ++i) { 1748 struct netfront_queue *queue = &info->queues[i]; 1749 1750 del_timer_sync(&queue->rx_refill_timer); 1751 1752 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) 1753 unbind_from_irqhandler(queue->tx_irq, queue); 1754 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { 1755 unbind_from_irqhandler(queue->tx_irq, queue); 1756 unbind_from_irqhandler(queue->rx_irq, queue); 1757 } 1758 queue->tx_evtchn = queue->rx_evtchn = 0; 1759 queue->tx_irq = queue->rx_irq = 0; 1760 1761 if (netif_running(info->netdev)) 1762 napi_synchronize(&queue->napi); 1763 1764 xennet_release_tx_bufs(queue); 1765 xennet_release_rx_bufs(queue); 1766 gnttab_free_grant_references(queue->gref_tx_head); 1767 gnttab_free_grant_references(queue->gref_rx_head); 1768 1769 /* End access and free the pages */ 1770 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); 1771 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); 1772 1773 queue->tx_ring_ref = GRANT_INVALID_REF; 1774 queue->rx_ring_ref = GRANT_INVALID_REF; 1775 queue->tx.sring = NULL; 1776 queue->rx.sring = NULL; 1777 1778 page_pool_destroy(queue->page_pool); 1779 } 1780 } 1781 1782 /* 1783 * We are reconnecting to the backend, due to a suspend/resume, or a backend 1784 * driver restart. We tear down our netif structure and recreate it, but 1785 * leave the device-layer structures intact so that this is transparent to the 1786 * rest of the kernel. 1787 */ 1788 static int netfront_resume(struct xenbus_device *dev) 1789 { 1790 struct netfront_info *info = dev_get_drvdata(&dev->dev); 1791 1792 dev_dbg(&dev->dev, "%s\n", dev->nodename); 1793 1794 netif_tx_lock_bh(info->netdev); 1795 netif_device_detach(info->netdev); 1796 netif_tx_unlock_bh(info->netdev); 1797 1798 xennet_disconnect_backend(info); 1799 return 0; 1800 } 1801 1802 static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[]) 1803 { 1804 char *s, *e, *macstr; 1805 int i; 1806 1807 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); 1808 if (IS_ERR(macstr)) 1809 return PTR_ERR(macstr); 1810 1811 for (i = 0; i < ETH_ALEN; i++) { 1812 mac[i] = simple_strtoul(s, &e, 16); 1813 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { 1814 kfree(macstr); 1815 return -ENOENT; 1816 } 1817 s = e+1; 1818 } 1819 1820 kfree(macstr); 1821 return 0; 1822 } 1823 1824 static int setup_netfront_single(struct netfront_queue *queue) 1825 { 1826 int err; 1827 1828 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); 1829 if (err < 0) 1830 goto fail; 1831 1832 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, 1833 xennet_interrupt, 0, 1834 queue->info->netdev->name, 1835 queue); 1836 if (err < 0) 1837 goto bind_fail; 1838 queue->rx_evtchn = queue->tx_evtchn; 1839 queue->rx_irq = queue->tx_irq = err; 1840 1841 return 0; 1842 1843 bind_fail: 1844 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); 1845 queue->tx_evtchn = 0; 1846 fail: 1847 return err; 1848 } 1849 1850 static int setup_netfront_split(struct netfront_queue *queue) 1851 { 1852 int err; 1853 1854 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); 1855 if (err < 0) 1856 goto fail; 1857 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); 1858 if (err < 0) 1859 goto alloc_rx_evtchn_fail; 1860 1861 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), 1862 "%s-tx", queue->name); 1863 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, 1864 xennet_tx_interrupt, 0, 1865 queue->tx_irq_name, queue); 1866 if (err < 0) 1867 goto bind_tx_fail; 1868 queue->tx_irq = err; 1869 1870 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), 1871 "%s-rx", queue->name); 1872 err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn, 1873 xennet_rx_interrupt, 0, 1874 queue->rx_irq_name, queue); 1875 if (err < 0) 1876 goto bind_rx_fail; 1877 queue->rx_irq = err; 1878 1879 return 0; 1880 1881 bind_rx_fail: 1882 unbind_from_irqhandler(queue->tx_irq, queue); 1883 queue->tx_irq = 0; 1884 bind_tx_fail: 1885 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); 1886 queue->rx_evtchn = 0; 1887 alloc_rx_evtchn_fail: 1888 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); 1889 queue->tx_evtchn = 0; 1890 fail: 1891 return err; 1892 } 1893 1894 static int setup_netfront(struct xenbus_device *dev, 1895 struct netfront_queue *queue, unsigned int feature_split_evtchn) 1896 { 1897 struct xen_netif_tx_sring *txs; 1898 struct xen_netif_rx_sring *rxs; 1899 grant_ref_t gref; 1900 int err; 1901 1902 queue->tx_ring_ref = GRANT_INVALID_REF; 1903 queue->rx_ring_ref = GRANT_INVALID_REF; 1904 queue->rx.sring = NULL; 1905 queue->tx.sring = NULL; 1906 1907 txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); 1908 if (!txs) { 1909 err = -ENOMEM; 1910 xenbus_dev_fatal(dev, err, "allocating tx ring page"); 1911 goto fail; 1912 } 1913 SHARED_RING_INIT(txs); 1914 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); 1915 1916 err = xenbus_grant_ring(dev, txs, 1, &gref); 1917 if (err < 0) 1918 goto grant_tx_ring_fail; 1919 queue->tx_ring_ref = gref; 1920 1921 rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); 1922 if (!rxs) { 1923 err = -ENOMEM; 1924 xenbus_dev_fatal(dev, err, "allocating rx ring page"); 1925 goto alloc_rx_ring_fail; 1926 } 1927 SHARED_RING_INIT(rxs); 1928 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); 1929 1930 err = xenbus_grant_ring(dev, rxs, 1, &gref); 1931 if (err < 0) 1932 goto grant_rx_ring_fail; 1933 queue->rx_ring_ref = gref; 1934 1935 if (feature_split_evtchn) 1936 err = setup_netfront_split(queue); 1937 /* setup single event channel if 1938 * a) feature-split-event-channels == 0 1939 * b) feature-split-event-channels == 1 but failed to setup 1940 */ 1941 if (!feature_split_evtchn || err) 1942 err = setup_netfront_single(queue); 1943 1944 if (err) 1945 goto alloc_evtchn_fail; 1946 1947 return 0; 1948 1949 /* If we fail to setup netfront, it is safe to just revoke access to 1950 * granted pages because backend is not accessing it at this point. 1951 */ 1952 alloc_evtchn_fail: 1953 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); 1954 grant_rx_ring_fail: 1955 free_page((unsigned long)rxs); 1956 alloc_rx_ring_fail: 1957 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); 1958 grant_tx_ring_fail: 1959 free_page((unsigned long)txs); 1960 fail: 1961 return err; 1962 } 1963 1964 /* Queue-specific initialisation 1965 * This used to be done in xennet_create_dev() but must now 1966 * be run per-queue. 1967 */ 1968 static int xennet_init_queue(struct netfront_queue *queue) 1969 { 1970 unsigned short i; 1971 int err = 0; 1972 char *devid; 1973 1974 spin_lock_init(&queue->tx_lock); 1975 spin_lock_init(&queue->rx_lock); 1976 spin_lock_init(&queue->rx_cons_lock); 1977 1978 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); 1979 1980 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; 1981 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", 1982 devid, queue->id); 1983 1984 /* Initialise tx_skb_freelist as a free chain containing every entry. */ 1985 queue->tx_skb_freelist = 0; 1986 queue->tx_pend_queue = TX_LINK_NONE; 1987 for (i = 0; i < NET_TX_RING_SIZE; i++) { 1988 queue->tx_link[i] = i + 1; 1989 queue->grant_tx_ref[i] = GRANT_INVALID_REF; 1990 queue->grant_tx_page[i] = NULL; 1991 } 1992 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; 1993 1994 /* Clear out rx_skbs */ 1995 for (i = 0; i < NET_RX_RING_SIZE; i++) { 1996 queue->rx_skbs[i] = NULL; 1997 queue->grant_rx_ref[i] = GRANT_INVALID_REF; 1998 } 1999 2000 /* A grant for every tx ring slot */ 2001 if (gnttab_alloc_grant_references(NET_TX_RING_SIZE, 2002 &queue->gref_tx_head) < 0) { 2003 pr_alert("can't alloc tx grant refs\n"); 2004 err = -ENOMEM; 2005 goto exit; 2006 } 2007 2008 /* A grant for every rx ring slot */ 2009 if (gnttab_alloc_grant_references(NET_RX_RING_SIZE, 2010 &queue->gref_rx_head) < 0) { 2011 pr_alert("can't alloc rx grant refs\n"); 2012 err = -ENOMEM; 2013 goto exit_free_tx; 2014 } 2015 2016 return 0; 2017 2018 exit_free_tx: 2019 gnttab_free_grant_references(queue->gref_tx_head); 2020 exit: 2021 return err; 2022 } 2023 2024 static int write_queue_xenstore_keys(struct netfront_queue *queue, 2025 struct xenbus_transaction *xbt, int write_hierarchical) 2026 { 2027 /* Write the queue-specific keys into XenStore in the traditional 2028 * way for a single queue, or in a queue subkeys for multiple 2029 * queues. 2030 */ 2031 struct xenbus_device *dev = queue->info->xbdev; 2032 int err; 2033 const char *message; 2034 char *path; 2035 size_t pathsize; 2036 2037 /* Choose the correct place to write the keys */ 2038 if (write_hierarchical) { 2039 pathsize = strlen(dev->nodename) + 10; 2040 path = kzalloc(pathsize, GFP_KERNEL); 2041 if (!path) { 2042 err = -ENOMEM; 2043 message = "out of memory while writing ring references"; 2044 goto error; 2045 } 2046 snprintf(path, pathsize, "%s/queue-%u", 2047 dev->nodename, queue->id); 2048 } else { 2049 path = (char *)dev->nodename; 2050 } 2051 2052 /* Write ring references */ 2053 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", 2054 queue->tx_ring_ref); 2055 if (err) { 2056 message = "writing tx-ring-ref"; 2057 goto error; 2058 } 2059 2060 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", 2061 queue->rx_ring_ref); 2062 if (err) { 2063 message = "writing rx-ring-ref"; 2064 goto error; 2065 } 2066 2067 /* Write event channels; taking into account both shared 2068 * and split event channel scenarios. 2069 */ 2070 if (queue->tx_evtchn == queue->rx_evtchn) { 2071 /* Shared event channel */ 2072 err = xenbus_printf(*xbt, path, 2073 "event-channel", "%u", queue->tx_evtchn); 2074 if (err) { 2075 message = "writing event-channel"; 2076 goto error; 2077 } 2078 } else { 2079 /* Split event channels */ 2080 err = xenbus_printf(*xbt, path, 2081 "event-channel-tx", "%u", queue->tx_evtchn); 2082 if (err) { 2083 message = "writing event-channel-tx"; 2084 goto error; 2085 } 2086 2087 err = xenbus_printf(*xbt, path, 2088 "event-channel-rx", "%u", queue->rx_evtchn); 2089 if (err) { 2090 message = "writing event-channel-rx"; 2091 goto error; 2092 } 2093 } 2094 2095 if (write_hierarchical) 2096 kfree(path); 2097 return 0; 2098 2099 error: 2100 if (write_hierarchical) 2101 kfree(path); 2102 xenbus_dev_fatal(dev, err, "%s", message); 2103 return err; 2104 } 2105 2106 static void xennet_destroy_queues(struct netfront_info *info) 2107 { 2108 unsigned int i; 2109 2110 for (i = 0; i < info->netdev->real_num_tx_queues; i++) { 2111 struct netfront_queue *queue = &info->queues[i]; 2112 2113 if (netif_running(info->netdev)) 2114 napi_disable(&queue->napi); 2115 netif_napi_del(&queue->napi); 2116 } 2117 2118 kfree(info->queues); 2119 info->queues = NULL; 2120 } 2121 2122 2123 2124 static int xennet_create_page_pool(struct netfront_queue *queue) 2125 { 2126 int err; 2127 struct page_pool_params pp_params = { 2128 .order = 0, 2129 .flags = 0, 2130 .pool_size = NET_RX_RING_SIZE, 2131 .nid = NUMA_NO_NODE, 2132 .dev = &queue->info->netdev->dev, 2133 .offset = XDP_PACKET_HEADROOM, 2134 .max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, 2135 }; 2136 2137 queue->page_pool = page_pool_create(&pp_params); 2138 if (IS_ERR(queue->page_pool)) { 2139 err = PTR_ERR(queue->page_pool); 2140 queue->page_pool = NULL; 2141 return err; 2142 } 2143 2144 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, 2145 queue->id, 0); 2146 if (err) { 2147 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); 2148 goto err_free_pp; 2149 } 2150 2151 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, 2152 MEM_TYPE_PAGE_POOL, queue->page_pool); 2153 if (err) { 2154 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); 2155 goto err_unregister_rxq; 2156 } 2157 return 0; 2158 2159 err_unregister_rxq: 2160 xdp_rxq_info_unreg(&queue->xdp_rxq); 2161 err_free_pp: 2162 page_pool_destroy(queue->page_pool); 2163 queue->page_pool = NULL; 2164 return err; 2165 } 2166 2167 static int xennet_create_queues(struct netfront_info *info, 2168 unsigned int *num_queues) 2169 { 2170 unsigned int i; 2171 int ret; 2172 2173 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), 2174 GFP_KERNEL); 2175 if (!info->queues) 2176 return -ENOMEM; 2177 2178 for (i = 0; i < *num_queues; i++) { 2179 struct netfront_queue *queue = &info->queues[i]; 2180 2181 queue->id = i; 2182 queue->info = info; 2183 2184 ret = xennet_init_queue(queue); 2185 if (ret < 0) { 2186 dev_warn(&info->xbdev->dev, 2187 "only created %d queues\n", i); 2188 *num_queues = i; 2189 break; 2190 } 2191 2192 /* use page pool recycling instead of buddy allocator */ 2193 ret = xennet_create_page_pool(queue); 2194 if (ret < 0) { 2195 dev_err(&info->xbdev->dev, "can't allocate page pool\n"); 2196 *num_queues = i; 2197 return ret; 2198 } 2199 2200 netif_napi_add(queue->info->netdev, &queue->napi, 2201 xennet_poll, 64); 2202 if (netif_running(info->netdev)) 2203 napi_enable(&queue->napi); 2204 } 2205 2206 netif_set_real_num_tx_queues(info->netdev, *num_queues); 2207 2208 if (*num_queues == 0) { 2209 dev_err(&info->xbdev->dev, "no queues\n"); 2210 return -EINVAL; 2211 } 2212 return 0; 2213 } 2214 2215 /* Common code used when first setting up, and when resuming. */ 2216 static int talk_to_netback(struct xenbus_device *dev, 2217 struct netfront_info *info) 2218 { 2219 const char *message; 2220 struct xenbus_transaction xbt; 2221 int err; 2222 unsigned int feature_split_evtchn; 2223 unsigned int i = 0; 2224 unsigned int max_queues = 0; 2225 struct netfront_queue *queue = NULL; 2226 unsigned int num_queues = 1; 2227 u8 addr[ETH_ALEN]; 2228 2229 info->netdev->irq = 0; 2230 2231 /* Check if backend supports multiple queues */ 2232 max_queues = xenbus_read_unsigned(info->xbdev->otherend, 2233 "multi-queue-max-queues", 1); 2234 num_queues = min(max_queues, xennet_max_queues); 2235 2236 /* Check feature-split-event-channels */ 2237 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, 2238 "feature-split-event-channels", 0); 2239 2240 /* Read mac addr. */ 2241 err = xen_net_read_mac(dev, addr); 2242 if (err) { 2243 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); 2244 goto out_unlocked; 2245 } 2246 eth_hw_addr_set(info->netdev, addr); 2247 2248 info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend, 2249 "feature-xdp-headroom", 0); 2250 if (info->netback_has_xdp_headroom) { 2251 /* set the current xen-netfront xdp state */ 2252 err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ? 2253 NETBACK_XDP_HEADROOM_ENABLE : 2254 NETBACK_XDP_HEADROOM_DISABLE); 2255 if (err) 2256 goto out_unlocked; 2257 } 2258 2259 rtnl_lock(); 2260 if (info->queues) 2261 xennet_destroy_queues(info); 2262 2263 /* For the case of a reconnect reset the "broken" indicator. */ 2264 info->broken = false; 2265 2266 err = xennet_create_queues(info, &num_queues); 2267 if (err < 0) { 2268 xenbus_dev_fatal(dev, err, "creating queues"); 2269 kfree(info->queues); 2270 info->queues = NULL; 2271 goto out; 2272 } 2273 rtnl_unlock(); 2274 2275 /* Create shared ring, alloc event channel -- for each queue */ 2276 for (i = 0; i < num_queues; ++i) { 2277 queue = &info->queues[i]; 2278 err = setup_netfront(dev, queue, feature_split_evtchn); 2279 if (err) 2280 goto destroy_ring; 2281 } 2282 2283 again: 2284 err = xenbus_transaction_start(&xbt); 2285 if (err) { 2286 xenbus_dev_fatal(dev, err, "starting transaction"); 2287 goto destroy_ring; 2288 } 2289 2290 if (xenbus_exists(XBT_NIL, 2291 info->xbdev->otherend, "multi-queue-max-queues")) { 2292 /* Write the number of queues */ 2293 err = xenbus_printf(xbt, dev->nodename, 2294 "multi-queue-num-queues", "%u", num_queues); 2295 if (err) { 2296 message = "writing multi-queue-num-queues"; 2297 goto abort_transaction_no_dev_fatal; 2298 } 2299 } 2300 2301 if (num_queues == 1) { 2302 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ 2303 if (err) 2304 goto abort_transaction_no_dev_fatal; 2305 } else { 2306 /* Write the keys for each queue */ 2307 for (i = 0; i < num_queues; ++i) { 2308 queue = &info->queues[i]; 2309 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ 2310 if (err) 2311 goto abort_transaction_no_dev_fatal; 2312 } 2313 } 2314 2315 /* The remaining keys are not queue-specific */ 2316 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", 2317 1); 2318 if (err) { 2319 message = "writing request-rx-copy"; 2320 goto abort_transaction; 2321 } 2322 2323 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); 2324 if (err) { 2325 message = "writing feature-rx-notify"; 2326 goto abort_transaction; 2327 } 2328 2329 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); 2330 if (err) { 2331 message = "writing feature-sg"; 2332 goto abort_transaction; 2333 } 2334 2335 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); 2336 if (err) { 2337 message = "writing feature-gso-tcpv4"; 2338 goto abort_transaction; 2339 } 2340 2341 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); 2342 if (err) { 2343 message = "writing feature-gso-tcpv6"; 2344 goto abort_transaction; 2345 } 2346 2347 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", 2348 "1"); 2349 if (err) { 2350 message = "writing feature-ipv6-csum-offload"; 2351 goto abort_transaction; 2352 } 2353 2354 err = xenbus_transaction_end(xbt, 0); 2355 if (err) { 2356 if (err == -EAGAIN) 2357 goto again; 2358 xenbus_dev_fatal(dev, err, "completing transaction"); 2359 goto destroy_ring; 2360 } 2361 2362 return 0; 2363 2364 abort_transaction: 2365 xenbus_dev_fatal(dev, err, "%s", message); 2366 abort_transaction_no_dev_fatal: 2367 xenbus_transaction_end(xbt, 1); 2368 destroy_ring: 2369 xennet_disconnect_backend(info); 2370 rtnl_lock(); 2371 xennet_destroy_queues(info); 2372 out: 2373 rtnl_unlock(); 2374 out_unlocked: 2375 device_unregister(&dev->dev); 2376 return err; 2377 } 2378 2379 static int xennet_connect(struct net_device *dev) 2380 { 2381 struct netfront_info *np = netdev_priv(dev); 2382 unsigned int num_queues = 0; 2383 int err; 2384 unsigned int j = 0; 2385 struct netfront_queue *queue = NULL; 2386 2387 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { 2388 dev_info(&dev->dev, 2389 "backend does not support copying receive path\n"); 2390 return -ENODEV; 2391 } 2392 2393 err = talk_to_netback(np->xbdev, np); 2394 if (err) 2395 return err; 2396 if (np->netback_has_xdp_headroom) 2397 pr_info("backend supports XDP headroom\n"); 2398 2399 /* talk_to_netback() sets the correct number of queues */ 2400 num_queues = dev->real_num_tx_queues; 2401 2402 if (dev->reg_state == NETREG_UNINITIALIZED) { 2403 err = register_netdev(dev); 2404 if (err) { 2405 pr_warn("%s: register_netdev err=%d\n", __func__, err); 2406 device_unregister(&np->xbdev->dev); 2407 return err; 2408 } 2409 } 2410 2411 rtnl_lock(); 2412 netdev_update_features(dev); 2413 rtnl_unlock(); 2414 2415 /* 2416 * All public and private state should now be sane. Get 2417 * ready to start sending and receiving packets and give the driver 2418 * domain a kick because we've probably just requeued some 2419 * packets. 2420 */ 2421 netif_tx_lock_bh(np->netdev); 2422 netif_device_attach(np->netdev); 2423 netif_tx_unlock_bh(np->netdev); 2424 2425 netif_carrier_on(np->netdev); 2426 for (j = 0; j < num_queues; ++j) { 2427 queue = &np->queues[j]; 2428 2429 notify_remote_via_irq(queue->tx_irq); 2430 if (queue->tx_irq != queue->rx_irq) 2431 notify_remote_via_irq(queue->rx_irq); 2432 2433 spin_lock_irq(&queue->tx_lock); 2434 xennet_tx_buf_gc(queue); 2435 spin_unlock_irq(&queue->tx_lock); 2436 2437 spin_lock_bh(&queue->rx_lock); 2438 xennet_alloc_rx_buffers(queue); 2439 spin_unlock_bh(&queue->rx_lock); 2440 } 2441 2442 return 0; 2443 } 2444 2445 /* 2446 * Callback received when the backend's state changes. 2447 */ 2448 static void netback_changed(struct xenbus_device *dev, 2449 enum xenbus_state backend_state) 2450 { 2451 struct netfront_info *np = dev_get_drvdata(&dev->dev); 2452 struct net_device *netdev = np->netdev; 2453 2454 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); 2455 2456 wake_up_all(&module_wq); 2457 2458 switch (backend_state) { 2459 case XenbusStateInitialising: 2460 case XenbusStateInitialised: 2461 case XenbusStateReconfiguring: 2462 case XenbusStateReconfigured: 2463 case XenbusStateUnknown: 2464 break; 2465 2466 case XenbusStateInitWait: 2467 if (dev->state != XenbusStateInitialising) 2468 break; 2469 if (xennet_connect(netdev) != 0) 2470 break; 2471 xenbus_switch_state(dev, XenbusStateConnected); 2472 break; 2473 2474 case XenbusStateConnected: 2475 netdev_notify_peers(netdev); 2476 break; 2477 2478 case XenbusStateClosed: 2479 if (dev->state == XenbusStateClosed) 2480 break; 2481 fallthrough; /* Missed the backend's CLOSING state */ 2482 case XenbusStateClosing: 2483 xenbus_frontend_closed(dev); 2484 break; 2485 } 2486 } 2487 2488 static const struct xennet_stat { 2489 char name[ETH_GSTRING_LEN]; 2490 u16 offset; 2491 } xennet_stats[] = { 2492 { 2493 "rx_gso_checksum_fixup", 2494 offsetof(struct netfront_info, rx_gso_checksum_fixup) 2495 }, 2496 }; 2497 2498 static int xennet_get_sset_count(struct net_device *dev, int string_set) 2499 { 2500 switch (string_set) { 2501 case ETH_SS_STATS: 2502 return ARRAY_SIZE(xennet_stats); 2503 default: 2504 return -EINVAL; 2505 } 2506 } 2507 2508 static void xennet_get_ethtool_stats(struct net_device *dev, 2509 struct ethtool_stats *stats, u64 * data) 2510 { 2511 void *np = netdev_priv(dev); 2512 int i; 2513 2514 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) 2515 data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset)); 2516 } 2517 2518 static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data) 2519 { 2520 int i; 2521 2522 switch (stringset) { 2523 case ETH_SS_STATS: 2524 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) 2525 memcpy(data + i * ETH_GSTRING_LEN, 2526 xennet_stats[i].name, ETH_GSTRING_LEN); 2527 break; 2528 } 2529 } 2530 2531 static const struct ethtool_ops xennet_ethtool_ops = 2532 { 2533 .get_link = ethtool_op_get_link, 2534 2535 .get_sset_count = xennet_get_sset_count, 2536 .get_ethtool_stats = xennet_get_ethtool_stats, 2537 .get_strings = xennet_get_strings, 2538 .get_ts_info = ethtool_op_get_ts_info, 2539 }; 2540 2541 #ifdef CONFIG_SYSFS 2542 static ssize_t show_rxbuf(struct device *dev, 2543 struct device_attribute *attr, char *buf) 2544 { 2545 return sprintf(buf, "%lu\n", NET_RX_RING_SIZE); 2546 } 2547 2548 static ssize_t store_rxbuf(struct device *dev, 2549 struct device_attribute *attr, 2550 const char *buf, size_t len) 2551 { 2552 char *endp; 2553 2554 if (!capable(CAP_NET_ADMIN)) 2555 return -EPERM; 2556 2557 simple_strtoul(buf, &endp, 0); 2558 if (endp == buf) 2559 return -EBADMSG; 2560 2561 /* rxbuf_min and rxbuf_max are no longer configurable. */ 2562 2563 return len; 2564 } 2565 2566 static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf); 2567 static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf); 2568 static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL); 2569 2570 static struct attribute *xennet_dev_attrs[] = { 2571 &dev_attr_rxbuf_min.attr, 2572 &dev_attr_rxbuf_max.attr, 2573 &dev_attr_rxbuf_cur.attr, 2574 NULL 2575 }; 2576 2577 static const struct attribute_group xennet_dev_group = { 2578 .attrs = xennet_dev_attrs 2579 }; 2580 #endif /* CONFIG_SYSFS */ 2581 2582 static void xennet_bus_close(struct xenbus_device *dev) 2583 { 2584 int ret; 2585 2586 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) 2587 return; 2588 do { 2589 xenbus_switch_state(dev, XenbusStateClosing); 2590 ret = wait_event_timeout(module_wq, 2591 xenbus_read_driver_state(dev->otherend) == 2592 XenbusStateClosing || 2593 xenbus_read_driver_state(dev->otherend) == 2594 XenbusStateClosed || 2595 xenbus_read_driver_state(dev->otherend) == 2596 XenbusStateUnknown, 2597 XENNET_TIMEOUT); 2598 } while (!ret); 2599 2600 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) 2601 return; 2602 2603 do { 2604 xenbus_switch_state(dev, XenbusStateClosed); 2605 ret = wait_event_timeout(module_wq, 2606 xenbus_read_driver_state(dev->otherend) == 2607 XenbusStateClosed || 2608 xenbus_read_driver_state(dev->otherend) == 2609 XenbusStateUnknown, 2610 XENNET_TIMEOUT); 2611 } while (!ret); 2612 } 2613 2614 static int xennet_remove(struct xenbus_device *dev) 2615 { 2616 struct netfront_info *info = dev_get_drvdata(&dev->dev); 2617 2618 xennet_bus_close(dev); 2619 xennet_disconnect_backend(info); 2620 2621 if (info->netdev->reg_state == NETREG_REGISTERED) 2622 unregister_netdev(info->netdev); 2623 2624 if (info->queues) { 2625 rtnl_lock(); 2626 xennet_destroy_queues(info); 2627 rtnl_unlock(); 2628 } 2629 xennet_free_netdev(info->netdev); 2630 2631 return 0; 2632 } 2633 2634 static const struct xenbus_device_id netfront_ids[] = { 2635 { "vif" }, 2636 { "" } 2637 }; 2638 2639 static struct xenbus_driver netfront_driver = { 2640 .ids = netfront_ids, 2641 .probe = netfront_probe, 2642 .remove = xennet_remove, 2643 .resume = netfront_resume, 2644 .otherend_changed = netback_changed, 2645 }; 2646 2647 static int __init netif_init(void) 2648 { 2649 if (!xen_domain()) 2650 return -ENODEV; 2651 2652 if (!xen_has_pv_nic_devices()) 2653 return -ENODEV; 2654 2655 pr_info("Initialising Xen virtual ethernet driver\n"); 2656 2657 /* Allow as many queues as there are CPUs inut max. 8 if user has not 2658 * specified a value. 2659 */ 2660 if (xennet_max_queues == 0) 2661 xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, 2662 num_online_cpus()); 2663 2664 return xenbus_register_frontend(&netfront_driver); 2665 } 2666 module_init(netif_init); 2667 2668 2669 static void __exit netif_exit(void) 2670 { 2671 xenbus_unregister_driver(&netfront_driver); 2672 } 2673 module_exit(netif_exit); 2674 2675 MODULE_DESCRIPTION("Xen virtual network device frontend"); 2676 MODULE_LICENSE("GPL"); 2677 MODULE_ALIAS("xen:vif"); 2678 MODULE_ALIAS("xennet"); 2679