1 /* 2 * Back-end of the driver for virtual network devices. This portion of the 3 * driver exports a 'unified' network-device interface that can be accessed 4 * by any operating system that implements a compatible front end. A 5 * reference front-end implementation can be found in: 6 * drivers/net/xen-netfront.c 7 * 8 * Copyright (c) 2002-2005, K A Fraser 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License version 2 12 * as published by the Free Software Foundation; or, when distributed 13 * separately from the Linux kernel or incorporated into other 14 * software packages, subject to the following license: 15 * 16 * Permission is hereby granted, free of charge, to any person obtaining a copy 17 * of this source file (the "Software"), to deal in the Software without 18 * restriction, including without limitation the rights to use, copy, modify, 19 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 20 * and to permit persons to whom the Software is furnished to do so, subject to 21 * the following conditions: 22 * 23 * The above copyright notice and this permission notice shall be included in 24 * all copies or substantial portions of the Software. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 32 * IN THE SOFTWARE. 33 */ 34 35 #include "common.h" 36 37 #include <linux/kthread.h> 38 #include <linux/if_vlan.h> 39 #include <linux/udp.h> 40 #include <linux/highmem.h> 41 #include <linux/skbuff_ref.h> 42 43 #include <net/tcp.h> 44 45 #include <xen/xen.h> 46 #include <xen/events.h> 47 #include <xen/interface/memory.h> 48 #include <xen/page.h> 49 50 #include <asm/xen/hypercall.h> 51 52 /* Provide an option to disable split event channels at load time as 53 * event channels are limited resource. Split event channels are 54 * enabled by default. 55 */ 56 bool separate_tx_rx_irq = true; 57 module_param(separate_tx_rx_irq, bool, 0644); 58 59 /* The time that packets can stay on the guest Rx internal queue 60 * before they are dropped. 61 */ 62 unsigned int rx_drain_timeout_msecs = 10000; 63 module_param(rx_drain_timeout_msecs, uint, 0444); 64 65 /* The length of time before the frontend is considered unresponsive 66 * because it isn't providing Rx slots. 67 */ 68 unsigned int rx_stall_timeout_msecs = 60000; 69 module_param(rx_stall_timeout_msecs, uint, 0444); 70 71 #define MAX_QUEUES_DEFAULT 8 72 unsigned int xenvif_max_queues; 73 module_param_named(max_queues, xenvif_max_queues, uint, 0644); 74 MODULE_PARM_DESC(max_queues, 75 "Maximum number of queues per virtual interface"); 76 77 /* 78 * This is the maximum slots a skb can have. If a guest sends a skb 79 * which exceeds this limit it is considered malicious. 80 */ 81 #define FATAL_SKB_SLOTS_DEFAULT 20 82 static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT; 83 module_param(fatal_skb_slots, uint, 0444); 84 85 /* The amount to copy out of the first guest Tx slot into the skb's 86 * linear area. If the first slot has more data, it will be mapped 87 * and put into the first frag. 88 * 89 * This is sized to avoid pulling headers from the frags for most 90 * TCP/IP packets. 91 */ 92 #define XEN_NETBACK_TX_COPY_LEN 128 93 94 /* This is the maximum number of flows in the hash cache. */ 95 #define XENVIF_HASH_CACHE_SIZE_DEFAULT 64 96 unsigned int xenvif_hash_cache_size = XENVIF_HASH_CACHE_SIZE_DEFAULT; 97 module_param_named(hash_cache_size, xenvif_hash_cache_size, uint, 0644); 98 MODULE_PARM_DESC(hash_cache_size, "Number of flows in the hash cache"); 99 100 /* The module parameter tells that we have to put data 101 * for xen-netfront with the XDP_PACKET_HEADROOM offset 102 * needed for XDP processing 103 */ 104 bool provides_xdp_headroom = true; 105 module_param(provides_xdp_headroom, bool, 0644); 106 107 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 108 s8 status); 109 110 static void make_tx_response(struct xenvif_queue *queue, 111 const struct xen_netif_tx_request *txp, 112 unsigned int extra_count, 113 s8 status); 114 115 static void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx); 116 117 static inline int tx_work_todo(struct xenvif_queue *queue); 118 119 static inline unsigned long idx_to_pfn(struct xenvif_queue *queue, 120 u16 idx) 121 { 122 return page_to_pfn(queue->mmap_pages[idx]); 123 } 124 125 static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, 126 u16 idx) 127 { 128 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx)); 129 } 130 131 #define callback_param(vif, pending_idx) \ 132 (vif->pending_tx_info[pending_idx].callback_struct) 133 134 /* Find the containing VIF's structure from a pointer in pending_tx_info array 135 */ 136 static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info_msgzc *ubuf) 137 { 138 u16 pending_idx = ubuf->desc; 139 struct pending_tx_info *temp = 140 container_of(ubuf, struct pending_tx_info, callback_struct); 141 return container_of(temp - pending_idx, 142 struct xenvif_queue, 143 pending_tx_info[0]); 144 } 145 146 static u16 frag_get_pending_idx(skb_frag_t *frag) 147 { 148 return (u16)skb_frag_off(frag); 149 } 150 151 static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx) 152 { 153 skb_frag_off_set(frag, pending_idx); 154 } 155 156 static inline pending_ring_idx_t pending_index(unsigned i) 157 { 158 return i & (MAX_PENDING_REQS-1); 159 } 160 161 void xenvif_kick_thread(struct xenvif_queue *queue) 162 { 163 wake_up(&queue->wq); 164 } 165 166 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue) 167 { 168 int more_to_do; 169 170 RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do); 171 172 if (more_to_do) 173 napi_schedule(&queue->napi); 174 else if (atomic_fetch_andnot(NETBK_TX_EOI | NETBK_COMMON_EOI, 175 &queue->eoi_pending) & 176 (NETBK_TX_EOI | NETBK_COMMON_EOI)) 177 xen_irq_lateeoi(queue->tx_irq, 0); 178 } 179 180 static void tx_add_credit(struct xenvif_queue *queue) 181 { 182 unsigned long max_burst, max_credit; 183 184 /* 185 * Allow a burst big enough to transmit a jumbo packet of up to 128kB. 186 * Otherwise the interface can seize up due to insufficient credit. 187 */ 188 max_burst = max(131072UL, queue->credit_bytes); 189 190 /* Take care that adding a new chunk of credit doesn't wrap to zero. */ 191 max_credit = queue->remaining_credit + queue->credit_bytes; 192 if (max_credit < queue->remaining_credit) 193 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ 194 195 queue->remaining_credit = min(max_credit, max_burst); 196 queue->rate_limited = false; 197 } 198 199 void xenvif_tx_credit_callback(struct timer_list *t) 200 { 201 struct xenvif_queue *queue = timer_container_of(queue, t, 202 credit_timeout); 203 tx_add_credit(queue); 204 xenvif_napi_schedule_or_enable_events(queue); 205 } 206 207 static void xenvif_tx_err(struct xenvif_queue *queue, 208 struct xen_netif_tx_request *txp, 209 unsigned int extra_count, RING_IDX end) 210 { 211 RING_IDX cons = queue->tx.req_cons; 212 213 do { 214 make_tx_response(queue, txp, extra_count, XEN_NETIF_RSP_ERROR); 215 if (cons == end) 216 break; 217 RING_COPY_REQUEST(&queue->tx, cons++, txp); 218 extra_count = 0; /* only the first frag can have extras */ 219 } while (1); 220 queue->tx.req_cons = cons; 221 } 222 223 static void xenvif_fatal_tx_err(struct xenvif *vif) 224 { 225 netdev_err(vif->dev, "fatal error; disabling device\n"); 226 vif->disabled = true; 227 /* Disable the vif from queue 0's kthread */ 228 if (vif->num_queues) 229 xenvif_kick_thread(&vif->queues[0]); 230 } 231 232 static int xenvif_count_requests(struct xenvif_queue *queue, 233 struct xen_netif_tx_request *first, 234 unsigned int extra_count, 235 struct xen_netif_tx_request *txp, 236 int work_to_do) 237 { 238 RING_IDX cons = queue->tx.req_cons; 239 int slots = 0; 240 int drop_err = 0; 241 int more_data; 242 243 if (!(first->flags & XEN_NETTXF_more_data)) 244 return 0; 245 246 do { 247 struct xen_netif_tx_request dropped_tx = { 0 }; 248 249 if (slots >= work_to_do) { 250 netdev_err(queue->vif->dev, 251 "Asked for %d slots but exceeds this limit\n", 252 work_to_do); 253 xenvif_fatal_tx_err(queue->vif); 254 return -ENODATA; 255 } 256 257 /* This guest is really using too many slots and 258 * considered malicious. 259 */ 260 if (unlikely(slots >= fatal_skb_slots)) { 261 netdev_err(queue->vif->dev, 262 "Malicious frontend using %d slots, threshold %u\n", 263 slots, fatal_skb_slots); 264 xenvif_fatal_tx_err(queue->vif); 265 return -E2BIG; 266 } 267 268 /* Xen network protocol had implicit dependency on 269 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to 270 * the historical MAX_SKB_FRAGS value 18 to honor the 271 * same behavior as before. Any packet using more than 272 * 18 slots but less than fatal_skb_slots slots is 273 * dropped 274 */ 275 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) { 276 if (net_ratelimit()) 277 netdev_dbg(queue->vif->dev, 278 "Too many slots (%d) exceeding limit (%d), dropping packet\n", 279 slots, XEN_NETBK_LEGACY_SLOTS_MAX); 280 drop_err = -E2BIG; 281 } 282 283 if (drop_err) 284 txp = &dropped_tx; 285 286 RING_COPY_REQUEST(&queue->tx, cons + slots, txp); 287 288 /* If the guest submitted a frame >= 64 KiB then 289 * first->size overflowed and following slots will 290 * appear to be larger than the frame. 291 * 292 * This cannot be fatal error as there are buggy 293 * frontends that do this. 294 * 295 * Consume all slots and drop the packet. 296 */ 297 if (!drop_err && txp->size > first->size) { 298 if (net_ratelimit()) 299 netdev_dbg(queue->vif->dev, 300 "Invalid tx request, slot size %u > remaining size %u\n", 301 txp->size, first->size); 302 drop_err = -EIO; 303 } 304 305 first->size -= txp->size; 306 slots++; 307 308 if (unlikely((txp->offset + txp->size) > XEN_PAGE_SIZE)) { 309 netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %u, size: %u\n", 310 txp->offset, txp->size); 311 xenvif_fatal_tx_err(queue->vif); 312 return -EINVAL; 313 } 314 315 more_data = txp->flags & XEN_NETTXF_more_data; 316 317 if (!drop_err) 318 txp++; 319 320 } while (more_data); 321 322 if (drop_err) { 323 xenvif_tx_err(queue, first, extra_count, cons + slots); 324 return drop_err; 325 } 326 327 return slots; 328 } 329 330 331 struct xenvif_tx_cb { 332 u16 copy_pending_idx[XEN_NETBK_LEGACY_SLOTS_MAX + 1]; 333 u8 copy_count; 334 u32 split_mask; 335 }; 336 337 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb) 338 #define copy_pending_idx(skb, i) (XENVIF_TX_CB(skb)->copy_pending_idx[i]) 339 #define copy_count(skb) (XENVIF_TX_CB(skb)->copy_count) 340 341 static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue, 342 u16 pending_idx, 343 struct xen_netif_tx_request *txp, 344 unsigned int extra_count, 345 struct gnttab_map_grant_ref *mop) 346 { 347 queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx]; 348 gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx), 349 GNTMAP_host_map | GNTMAP_readonly, 350 txp->gref, queue->vif->domid); 351 352 memcpy(&queue->pending_tx_info[pending_idx].req, txp, 353 sizeof(*txp)); 354 queue->pending_tx_info[pending_idx].extra_count = extra_count; 355 } 356 357 static inline struct sk_buff *xenvif_alloc_skb(unsigned int size) 358 { 359 struct sk_buff *skb = 360 alloc_skb(size + NET_SKB_PAD + NET_IP_ALIGN, 361 GFP_ATOMIC | __GFP_NOWARN); 362 363 BUILD_BUG_ON(sizeof(*XENVIF_TX_CB(skb)) > sizeof(skb->cb)); 364 if (unlikely(skb == NULL)) 365 return NULL; 366 367 /* Packets passed to netif_rx() must have some headroom. */ 368 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 369 370 /* Initialize it here to avoid later surprises */ 371 skb_shinfo(skb)->destructor_arg = NULL; 372 373 return skb; 374 } 375 376 static void xenvif_get_requests(struct xenvif_queue *queue, 377 struct sk_buff *skb, 378 struct xen_netif_tx_request *first, 379 struct xen_netif_tx_request *txfrags, 380 unsigned *copy_ops, 381 unsigned *map_ops, 382 unsigned int frag_overflow, 383 struct sk_buff *nskb, 384 unsigned int extra_count, 385 unsigned int data_len) 386 { 387 struct skb_shared_info *shinfo = skb_shinfo(skb); 388 skb_frag_t *frags = shinfo->frags; 389 u16 pending_idx; 390 pending_ring_idx_t index; 391 unsigned int nr_slots; 392 struct gnttab_copy *cop = queue->tx_copy_ops + *copy_ops; 393 struct gnttab_map_grant_ref *gop = queue->tx_map_ops + *map_ops; 394 struct xen_netif_tx_request *txp = first; 395 396 nr_slots = shinfo->nr_frags + frag_overflow + 1; 397 398 copy_count(skb) = 0; 399 XENVIF_TX_CB(skb)->split_mask = 0; 400 401 /* Create copy ops for exactly data_len bytes into the skb head. */ 402 __skb_put(skb, data_len); 403 while (data_len > 0) { 404 int amount = data_len > txp->size ? txp->size : data_len; 405 bool split = false; 406 407 cop->source.u.ref = txp->gref; 408 cop->source.domid = queue->vif->domid; 409 cop->source.offset = txp->offset; 410 411 cop->dest.domid = DOMID_SELF; 412 cop->dest.offset = (offset_in_page(skb->data + 413 skb_headlen(skb) - 414 data_len)) & ~XEN_PAGE_MASK; 415 cop->dest.u.gmfn = virt_to_gfn(skb->data + skb_headlen(skb) 416 - data_len); 417 418 /* Don't cross local page boundary! */ 419 if (cop->dest.offset + amount > XEN_PAGE_SIZE) { 420 amount = XEN_PAGE_SIZE - cop->dest.offset; 421 XENVIF_TX_CB(skb)->split_mask |= 1U << copy_count(skb); 422 split = true; 423 } 424 425 cop->len = amount; 426 cop->flags = GNTCOPY_source_gref; 427 428 index = pending_index(queue->pending_cons); 429 pending_idx = queue->pending_ring[index]; 430 callback_param(queue, pending_idx).ctx = NULL; 431 copy_pending_idx(skb, copy_count(skb)) = pending_idx; 432 if (!split) 433 copy_count(skb)++; 434 435 cop++; 436 data_len -= amount; 437 438 if (amount == txp->size) { 439 /* The copy op covered the full tx_request */ 440 441 memcpy(&queue->pending_tx_info[pending_idx].req, 442 txp, sizeof(*txp)); 443 queue->pending_tx_info[pending_idx].extra_count = 444 (txp == first) ? extra_count : 0; 445 446 if (txp == first) 447 txp = txfrags; 448 else 449 txp++; 450 queue->pending_cons++; 451 nr_slots--; 452 } else { 453 /* The copy op partially covered the tx_request. 454 * The remainder will be mapped or copied in the next 455 * iteration. 456 */ 457 txp->offset += amount; 458 txp->size -= amount; 459 } 460 } 461 462 for (shinfo->nr_frags = 0; nr_slots > 0 && shinfo->nr_frags < MAX_SKB_FRAGS; 463 nr_slots--) { 464 if (unlikely(!txp->size)) { 465 make_tx_response(queue, txp, 0, XEN_NETIF_RSP_OKAY); 466 ++txp; 467 continue; 468 } 469 470 index = pending_index(queue->pending_cons++); 471 pending_idx = queue->pending_ring[index]; 472 xenvif_tx_create_map_op(queue, pending_idx, txp, 473 txp == first ? extra_count : 0, gop); 474 frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); 475 ++shinfo->nr_frags; 476 ++gop; 477 478 if (txp == first) 479 txp = txfrags; 480 else 481 txp++; 482 } 483 484 if (nr_slots > 0) { 485 486 shinfo = skb_shinfo(nskb); 487 frags = shinfo->frags; 488 489 for (shinfo->nr_frags = 0; shinfo->nr_frags < nr_slots; ++txp) { 490 if (unlikely(!txp->size)) { 491 make_tx_response(queue, txp, 0, 492 XEN_NETIF_RSP_OKAY); 493 continue; 494 } 495 496 index = pending_index(queue->pending_cons++); 497 pending_idx = queue->pending_ring[index]; 498 xenvif_tx_create_map_op(queue, pending_idx, txp, 0, 499 gop); 500 frag_set_pending_idx(&frags[shinfo->nr_frags], 501 pending_idx); 502 ++shinfo->nr_frags; 503 ++gop; 504 } 505 506 if (shinfo->nr_frags) { 507 skb_shinfo(skb)->frag_list = nskb; 508 nskb = NULL; 509 } 510 } 511 512 if (nskb) { 513 /* A frag_list skb was allocated but it is no longer needed 514 * because enough slots were converted to copy ops above or some 515 * were empty. 516 */ 517 kfree_skb(nskb); 518 } 519 520 (*copy_ops) = cop - queue->tx_copy_ops; 521 (*map_ops) = gop - queue->tx_map_ops; 522 } 523 524 static inline void xenvif_grant_handle_set(struct xenvif_queue *queue, 525 u16 pending_idx, 526 grant_handle_t handle) 527 { 528 if (unlikely(queue->grant_tx_handle[pending_idx] != 529 NETBACK_INVALID_HANDLE)) { 530 netdev_err(queue->vif->dev, 531 "Trying to overwrite active handle! pending_idx: 0x%x\n", 532 pending_idx); 533 BUG(); 534 } 535 queue->grant_tx_handle[pending_idx] = handle; 536 } 537 538 static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue, 539 u16 pending_idx) 540 { 541 if (unlikely(queue->grant_tx_handle[pending_idx] == 542 NETBACK_INVALID_HANDLE)) { 543 netdev_err(queue->vif->dev, 544 "Trying to unmap invalid handle! pending_idx: 0x%x\n", 545 pending_idx); 546 BUG(); 547 } 548 queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE; 549 } 550 551 static int xenvif_tx_check_gop(struct xenvif_queue *queue, 552 struct sk_buff *skb, 553 struct gnttab_map_grant_ref **gopp_map, 554 struct gnttab_copy **gopp_copy) 555 { 556 struct gnttab_map_grant_ref *gop_map = *gopp_map; 557 u16 pending_idx; 558 /* This always points to the shinfo of the skb being checked, which 559 * could be either the first or the one on the frag_list 560 */ 561 struct skb_shared_info *shinfo = skb_shinfo(skb); 562 /* If this is non-NULL, we are currently checking the frag_list skb, and 563 * this points to the shinfo of the first one 564 */ 565 struct skb_shared_info *first_shinfo = NULL; 566 int nr_frags = shinfo->nr_frags; 567 const bool sharedslot = nr_frags && 568 frag_get_pending_idx(&shinfo->frags[0]) == 569 copy_pending_idx(skb, copy_count(skb) - 1); 570 int i, err = 0; 571 572 for (i = 0; i < copy_count(skb); i++) { 573 int newerr; 574 575 /* Check status of header. */ 576 pending_idx = copy_pending_idx(skb, i); 577 578 newerr = (*gopp_copy)->status; 579 580 /* Split copies need to be handled together. */ 581 if (XENVIF_TX_CB(skb)->split_mask & (1U << i)) { 582 (*gopp_copy)++; 583 if (!newerr) 584 newerr = (*gopp_copy)->status; 585 } 586 if (likely(!newerr)) { 587 /* The first frag might still have this slot mapped */ 588 if (i < copy_count(skb) - 1 || !sharedslot) 589 xenvif_idx_release(queue, pending_idx, 590 XEN_NETIF_RSP_OKAY); 591 } else { 592 err = newerr; 593 if (net_ratelimit()) 594 netdev_dbg(queue->vif->dev, 595 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", 596 (*gopp_copy)->status, 597 pending_idx, 598 (*gopp_copy)->source.u.ref); 599 /* The first frag might still have this slot mapped */ 600 if (i < copy_count(skb) - 1 || !sharedslot) 601 xenvif_idx_release(queue, pending_idx, 602 XEN_NETIF_RSP_ERROR); 603 } 604 (*gopp_copy)++; 605 } 606 607 check_frags: 608 for (i = 0; i < nr_frags; i++, gop_map++) { 609 int j, newerr; 610 611 pending_idx = frag_get_pending_idx(&shinfo->frags[i]); 612 613 /* Check error status: if okay then remember grant handle. */ 614 newerr = gop_map->status; 615 616 if (likely(!newerr)) { 617 xenvif_grant_handle_set(queue, 618 pending_idx, 619 gop_map->handle); 620 /* Had a previous error? Invalidate this fragment. */ 621 if (unlikely(err)) { 622 xenvif_idx_unmap(queue, pending_idx); 623 /* If the mapping of the first frag was OK, but 624 * the header's copy failed, and they are 625 * sharing a slot, send an error 626 */ 627 if (i == 0 && !first_shinfo && sharedslot) 628 xenvif_idx_release(queue, pending_idx, 629 XEN_NETIF_RSP_ERROR); 630 else 631 xenvif_idx_release(queue, pending_idx, 632 XEN_NETIF_RSP_OKAY); 633 } 634 continue; 635 } 636 637 /* Error on this fragment: respond to client with an error. */ 638 if (net_ratelimit()) 639 netdev_dbg(queue->vif->dev, 640 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n", 641 i, 642 gop_map->status, 643 pending_idx, 644 gop_map->ref); 645 646 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR); 647 648 /* Not the first error? Preceding frags already invalidated. */ 649 if (err) 650 continue; 651 652 /* Invalidate preceding fragments of this skb. */ 653 for (j = 0; j < i; j++) { 654 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 655 xenvif_idx_unmap(queue, pending_idx); 656 xenvif_idx_release(queue, pending_idx, 657 XEN_NETIF_RSP_OKAY); 658 } 659 660 /* And if we found the error while checking the frag_list, unmap 661 * the first skb's frags 662 */ 663 if (first_shinfo) { 664 for (j = 0; j < first_shinfo->nr_frags; j++) { 665 pending_idx = frag_get_pending_idx(&first_shinfo->frags[j]); 666 xenvif_idx_unmap(queue, pending_idx); 667 xenvif_idx_release(queue, pending_idx, 668 XEN_NETIF_RSP_OKAY); 669 } 670 } 671 672 /* Remember the error: invalidate all subsequent fragments. */ 673 err = newerr; 674 } 675 676 if (skb_has_frag_list(skb) && !first_shinfo) { 677 first_shinfo = shinfo; 678 shinfo = skb_shinfo(shinfo->frag_list); 679 nr_frags = shinfo->nr_frags; 680 681 goto check_frags; 682 } 683 684 *gopp_map = gop_map; 685 return err; 686 } 687 688 static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) 689 { 690 struct skb_shared_info *shinfo = skb_shinfo(skb); 691 int nr_frags = shinfo->nr_frags; 692 int i; 693 u16 prev_pending_idx = INVALID_PENDING_IDX; 694 695 for (i = 0; i < nr_frags; i++) { 696 skb_frag_t *frag = shinfo->frags + i; 697 struct xen_netif_tx_request *txp; 698 struct page *page; 699 u16 pending_idx; 700 701 pending_idx = frag_get_pending_idx(frag); 702 703 /* If this is not the first frag, chain it to the previous*/ 704 if (prev_pending_idx == INVALID_PENDING_IDX) 705 skb_shinfo(skb)->destructor_arg = 706 &callback_param(queue, pending_idx); 707 else 708 callback_param(queue, prev_pending_idx).ctx = 709 &callback_param(queue, pending_idx); 710 711 callback_param(queue, pending_idx).ctx = NULL; 712 prev_pending_idx = pending_idx; 713 714 txp = &queue->pending_tx_info[pending_idx].req; 715 page = virt_to_page((void *)idx_to_kaddr(queue, pending_idx)); 716 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 717 skb->len += txp->size; 718 skb->data_len += txp->size; 719 skb->truesize += txp->size; 720 721 /* Take an extra reference to offset network stack's put_page */ 722 get_page(queue->mmap_pages[pending_idx]); 723 } 724 } 725 726 static int xenvif_get_extras(struct xenvif_queue *queue, 727 struct xen_netif_extra_info *extras, 728 unsigned int *extra_count, 729 int work_to_do) 730 { 731 struct xen_netif_extra_info extra; 732 RING_IDX cons = queue->tx.req_cons; 733 734 do { 735 if (unlikely(work_to_do-- <= 0)) { 736 netdev_err(queue->vif->dev, "Missing extra info\n"); 737 xenvif_fatal_tx_err(queue->vif); 738 return -EBADR; 739 } 740 741 RING_COPY_REQUEST(&queue->tx, cons, &extra); 742 743 queue->tx.req_cons = ++cons; 744 (*extra_count)++; 745 746 if (unlikely(!extra.type || 747 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 748 netdev_err(queue->vif->dev, 749 "Invalid extra type: %d\n", extra.type); 750 xenvif_fatal_tx_err(queue->vif); 751 return -EINVAL; 752 } 753 754 memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); 755 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 756 757 return work_to_do; 758 } 759 760 static int xenvif_set_skb_gso(struct xenvif *vif, 761 struct sk_buff *skb, 762 struct xen_netif_extra_info *gso) 763 { 764 if (!gso->u.gso.size) { 765 netdev_err(vif->dev, "GSO size must not be zero.\n"); 766 xenvif_fatal_tx_err(vif); 767 return -EINVAL; 768 } 769 770 switch (gso->u.gso.type) { 771 case XEN_NETIF_GSO_TYPE_TCPV4: 772 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 773 break; 774 case XEN_NETIF_GSO_TYPE_TCPV6: 775 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 776 break; 777 default: 778 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type); 779 xenvif_fatal_tx_err(vif); 780 return -EINVAL; 781 } 782 783 skb_shinfo(skb)->gso_size = gso->u.gso.size; 784 /* gso_segs will be calculated later */ 785 786 return 0; 787 } 788 789 static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb) 790 { 791 bool recalculate_partial_csum = false; 792 793 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy 794 * peers can fail to set NETRXF_csum_blank when sending a GSO 795 * frame. In this case force the SKB to CHECKSUM_PARTIAL and 796 * recalculate the partial checksum. 797 */ 798 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 799 queue->stats.rx_gso_checksum_fixup++; 800 skb->ip_summed = CHECKSUM_PARTIAL; 801 recalculate_partial_csum = true; 802 } 803 804 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ 805 if (skb->ip_summed != CHECKSUM_PARTIAL) 806 return 0; 807 808 return skb_checksum_setup(skb, recalculate_partial_csum); 809 } 810 811 static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size) 812 { 813 u64 now = get_jiffies_64(); 814 u64 next_credit = queue->credit_window_start + 815 msecs_to_jiffies(queue->credit_usec / 1000); 816 817 /* Timer could already be pending in rare cases. */ 818 if (timer_pending(&queue->credit_timeout)) { 819 queue->rate_limited = true; 820 return true; 821 } 822 823 /* Passed the point where we can replenish credit? */ 824 if (time_after_eq64(now, next_credit)) { 825 queue->credit_window_start = now; 826 tx_add_credit(queue); 827 } 828 829 /* Still too big to send right now? Set a callback. */ 830 if (size > queue->remaining_credit) { 831 mod_timer(&queue->credit_timeout, 832 next_credit); 833 queue->credit_window_start = next_credit; 834 queue->rate_limited = true; 835 836 return true; 837 } 838 839 return false; 840 } 841 842 /* No locking is required in xenvif_mcast_add/del() as they are 843 * only ever invoked from NAPI poll. An RCU list is used because 844 * xenvif_mcast_match() is called asynchronously, during start_xmit. 845 */ 846 847 static int xenvif_mcast_add(struct xenvif *vif, const u8 *addr) 848 { 849 struct xenvif_mcast_addr *mcast; 850 851 if (vif->fe_mcast_count == XEN_NETBK_MCAST_MAX) { 852 if (net_ratelimit()) 853 netdev_err(vif->dev, 854 "Too many multicast addresses\n"); 855 return -ENOSPC; 856 } 857 858 mcast = kzalloc(sizeof(*mcast), GFP_ATOMIC); 859 if (!mcast) 860 return -ENOMEM; 861 862 ether_addr_copy(mcast->addr, addr); 863 list_add_tail_rcu(&mcast->entry, &vif->fe_mcast_addr); 864 vif->fe_mcast_count++; 865 866 return 0; 867 } 868 869 static void xenvif_mcast_del(struct xenvif *vif, const u8 *addr) 870 { 871 struct xenvif_mcast_addr *mcast; 872 873 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 874 if (ether_addr_equal(addr, mcast->addr)) { 875 --vif->fe_mcast_count; 876 list_del_rcu(&mcast->entry); 877 kfree_rcu(mcast, rcu); 878 break; 879 } 880 } 881 } 882 883 bool xenvif_mcast_match(struct xenvif *vif, const u8 *addr) 884 { 885 struct xenvif_mcast_addr *mcast; 886 887 rcu_read_lock(); 888 list_for_each_entry_rcu(mcast, &vif->fe_mcast_addr, entry) { 889 if (ether_addr_equal(addr, mcast->addr)) { 890 rcu_read_unlock(); 891 return true; 892 } 893 } 894 rcu_read_unlock(); 895 896 return false; 897 } 898 899 void xenvif_mcast_addr_list_free(struct xenvif *vif) 900 { 901 /* No need for locking or RCU here. NAPI poll and TX queue 902 * are stopped. 903 */ 904 while (!list_empty(&vif->fe_mcast_addr)) { 905 struct xenvif_mcast_addr *mcast; 906 907 mcast = list_first_entry(&vif->fe_mcast_addr, 908 struct xenvif_mcast_addr, 909 entry); 910 --vif->fe_mcast_count; 911 list_del(&mcast->entry); 912 kfree(mcast); 913 } 914 } 915 916 static void xenvif_tx_build_gops(struct xenvif_queue *queue, 917 int budget, 918 unsigned *copy_ops, 919 unsigned *map_ops) 920 { 921 struct sk_buff *skb, *nskb; 922 int ret; 923 unsigned int frag_overflow; 924 925 while (skb_queue_len(&queue->tx_queue) < budget) { 926 struct xen_netif_tx_request txreq; 927 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX]; 928 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1]; 929 unsigned int extra_count; 930 RING_IDX idx; 931 int work_to_do; 932 unsigned int data_len; 933 934 if (queue->tx.sring->req_prod - queue->tx.req_cons > 935 XEN_NETIF_TX_RING_SIZE) { 936 netdev_err(queue->vif->dev, 937 "Impossible number of requests. " 938 "req_prod %d, req_cons %d, size %ld\n", 939 queue->tx.sring->req_prod, queue->tx.req_cons, 940 XEN_NETIF_TX_RING_SIZE); 941 xenvif_fatal_tx_err(queue->vif); 942 break; 943 } 944 945 work_to_do = XEN_RING_NR_UNCONSUMED_REQUESTS(&queue->tx); 946 if (!work_to_do) 947 break; 948 949 idx = queue->tx.req_cons; 950 rmb(); /* Ensure that we see the request before we copy it. */ 951 RING_COPY_REQUEST(&queue->tx, idx, &txreq); 952 953 /* Credit-based scheduling. */ 954 if (txreq.size > queue->remaining_credit && 955 tx_credit_exceeded(queue, txreq.size)) 956 break; 957 958 queue->remaining_credit -= txreq.size; 959 960 work_to_do--; 961 queue->tx.req_cons = ++idx; 962 963 memset(extras, 0, sizeof(extras)); 964 extra_count = 0; 965 if (txreq.flags & XEN_NETTXF_extra_info) { 966 work_to_do = xenvif_get_extras(queue, extras, 967 &extra_count, 968 work_to_do); 969 idx = queue->tx.req_cons; 970 if (unlikely(work_to_do < 0)) 971 break; 972 } 973 974 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1].type) { 975 struct xen_netif_extra_info *extra; 976 977 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_ADD - 1]; 978 ret = xenvif_mcast_add(queue->vif, extra->u.mcast.addr); 979 980 make_tx_response(queue, &txreq, extra_count, 981 (ret == 0) ? 982 XEN_NETIF_RSP_OKAY : 983 XEN_NETIF_RSP_ERROR); 984 continue; 985 } 986 987 if (extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1].type) { 988 struct xen_netif_extra_info *extra; 989 990 extra = &extras[XEN_NETIF_EXTRA_TYPE_MCAST_DEL - 1]; 991 xenvif_mcast_del(queue->vif, extra->u.mcast.addr); 992 993 make_tx_response(queue, &txreq, extra_count, 994 XEN_NETIF_RSP_OKAY); 995 continue; 996 } 997 998 data_len = (txreq.size > XEN_NETBACK_TX_COPY_LEN) ? 999 XEN_NETBACK_TX_COPY_LEN : txreq.size; 1000 1001 ret = xenvif_count_requests(queue, &txreq, extra_count, 1002 txfrags, work_to_do); 1003 1004 if (unlikely(ret < 0)) 1005 break; 1006 1007 idx += ret; 1008 1009 if (unlikely(txreq.size < ETH_HLEN)) { 1010 netdev_dbg(queue->vif->dev, 1011 "Bad packet size: %d\n", txreq.size); 1012 xenvif_tx_err(queue, &txreq, extra_count, idx); 1013 break; 1014 } 1015 1016 /* No crossing a page as the payload mustn't fragment. */ 1017 if (unlikely((txreq.offset + txreq.size) > XEN_PAGE_SIZE)) { 1018 netdev_err(queue->vif->dev, "Cross page boundary, txreq.offset: %u, size: %u\n", 1019 txreq.offset, txreq.size); 1020 xenvif_fatal_tx_err(queue->vif); 1021 break; 1022 } 1023 1024 if (ret >= XEN_NETBK_LEGACY_SLOTS_MAX - 1 && data_len < txreq.size) 1025 data_len = txreq.size; 1026 1027 skb = xenvif_alloc_skb(data_len); 1028 if (unlikely(skb == NULL)) { 1029 netdev_dbg(queue->vif->dev, 1030 "Can't allocate a skb in start_xmit.\n"); 1031 xenvif_tx_err(queue, &txreq, extra_count, idx); 1032 break; 1033 } 1034 1035 skb_shinfo(skb)->nr_frags = ret; 1036 /* At this point shinfo->nr_frags is in fact the number of 1037 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX. 1038 */ 1039 frag_overflow = 0; 1040 nskb = NULL; 1041 if (skb_shinfo(skb)->nr_frags > MAX_SKB_FRAGS) { 1042 frag_overflow = skb_shinfo(skb)->nr_frags - MAX_SKB_FRAGS; 1043 BUG_ON(frag_overflow > MAX_SKB_FRAGS); 1044 skb_shinfo(skb)->nr_frags = MAX_SKB_FRAGS; 1045 nskb = xenvif_alloc_skb(0); 1046 if (unlikely(nskb == NULL)) { 1047 skb_shinfo(skb)->nr_frags = 0; 1048 kfree_skb(skb); 1049 xenvif_tx_err(queue, &txreq, extra_count, idx); 1050 if (net_ratelimit()) 1051 netdev_err(queue->vif->dev, 1052 "Can't allocate the frag_list skb.\n"); 1053 break; 1054 } 1055 } 1056 1057 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { 1058 struct xen_netif_extra_info *gso; 1059 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1060 1061 if (xenvif_set_skb_gso(queue->vif, skb, gso)) { 1062 /* Failure in xenvif_set_skb_gso is fatal. */ 1063 skb_shinfo(skb)->nr_frags = 0; 1064 kfree_skb(skb); 1065 kfree_skb(nskb); 1066 break; 1067 } 1068 } 1069 1070 if (extras[XEN_NETIF_EXTRA_TYPE_HASH - 1].type) { 1071 struct xen_netif_extra_info *extra; 1072 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 1073 1074 extra = &extras[XEN_NETIF_EXTRA_TYPE_HASH - 1]; 1075 1076 switch (extra->u.hash.type) { 1077 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4: 1078 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6: 1079 type = PKT_HASH_TYPE_L3; 1080 break; 1081 1082 case _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP: 1083 case _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP: 1084 type = PKT_HASH_TYPE_L4; 1085 break; 1086 1087 default: 1088 break; 1089 } 1090 1091 if (type != PKT_HASH_TYPE_NONE) 1092 skb_set_hash(skb, 1093 *(u32 *)extra->u.hash.value, 1094 type); 1095 } 1096 1097 xenvif_get_requests(queue, skb, &txreq, txfrags, copy_ops, 1098 map_ops, frag_overflow, nskb, extra_count, 1099 data_len); 1100 1101 __skb_queue_tail(&queue->tx_queue, skb); 1102 1103 queue->tx.req_cons = idx; 1104 } 1105 1106 return; 1107 } 1108 1109 /* Consolidate skb with a frag_list into a brand new one with local pages on 1110 * frags. Returns 0 or -ENOMEM if can't allocate new pages. 1111 */ 1112 static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb) 1113 { 1114 unsigned int offset = skb_headlen(skb); 1115 skb_frag_t frags[MAX_SKB_FRAGS]; 1116 int i, f; 1117 struct ubuf_info *uarg; 1118 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1119 1120 queue->stats.tx_zerocopy_sent += 2; 1121 queue->stats.tx_frag_overflow++; 1122 1123 xenvif_fill_frags(queue, nskb); 1124 /* Subtract frags size, we will correct it later */ 1125 skb->truesize -= skb->data_len; 1126 skb->len += nskb->len; 1127 skb->data_len += nskb->len; 1128 1129 /* create a brand new frags array and coalesce there */ 1130 for (i = 0; offset < skb->len; i++) { 1131 struct page *page; 1132 unsigned int len; 1133 1134 BUG_ON(i >= MAX_SKB_FRAGS); 1135 page = alloc_page(GFP_ATOMIC); 1136 if (!page) { 1137 int j; 1138 skb->truesize += skb->data_len; 1139 for (j = 0; j < i; j++) 1140 put_page(skb_frag_page(&frags[j])); 1141 return -ENOMEM; 1142 } 1143 1144 if (offset + PAGE_SIZE < skb->len) 1145 len = PAGE_SIZE; 1146 else 1147 len = skb->len - offset; 1148 if (skb_copy_bits(skb, offset, page_address(page), len)) 1149 BUG(); 1150 1151 offset += len; 1152 skb_frag_fill_page_desc(&frags[i], page, 0, len); 1153 } 1154 1155 /* Release all the original (foreign) frags. */ 1156 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1157 skb_frag_unref(skb, f); 1158 uarg = skb_shinfo(skb)->destructor_arg; 1159 /* increase inflight counter to offset decrement in callback */ 1160 atomic_inc(&queue->inflight_packets); 1161 uarg->ops->complete(NULL, uarg, true); 1162 skb_shinfo(skb)->destructor_arg = NULL; 1163 1164 /* Fill the skb with the new (local) frags. */ 1165 memcpy(skb_shinfo(skb)->frags, frags, i * sizeof(skb_frag_t)); 1166 skb_shinfo(skb)->nr_frags = i; 1167 skb->truesize += i * PAGE_SIZE; 1168 1169 return 0; 1170 } 1171 1172 static int xenvif_tx_submit(struct xenvif_queue *queue) 1173 { 1174 struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops; 1175 struct gnttab_copy *gop_copy = queue->tx_copy_ops; 1176 struct sk_buff *skb; 1177 int work_done = 0; 1178 1179 while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) { 1180 struct xen_netif_tx_request *txp; 1181 u16 pending_idx; 1182 1183 pending_idx = copy_pending_idx(skb, 0); 1184 txp = &queue->pending_tx_info[pending_idx].req; 1185 1186 /* Check the remap error code. */ 1187 if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) { 1188 /* If there was an error, xenvif_tx_check_gop is 1189 * expected to release all the frags which were mapped, 1190 * so kfree_skb shouldn't do it again 1191 */ 1192 skb_shinfo(skb)->nr_frags = 0; 1193 if (skb_has_frag_list(skb)) { 1194 struct sk_buff *nskb = 1195 skb_shinfo(skb)->frag_list; 1196 skb_shinfo(nskb)->nr_frags = 0; 1197 } 1198 kfree_skb(skb); 1199 continue; 1200 } 1201 1202 if (txp->flags & XEN_NETTXF_csum_blank) 1203 skb->ip_summed = CHECKSUM_PARTIAL; 1204 else if (txp->flags & XEN_NETTXF_data_validated) 1205 skb->ip_summed = CHECKSUM_UNNECESSARY; 1206 1207 xenvif_fill_frags(queue, skb); 1208 1209 if (unlikely(skb_has_frag_list(skb))) { 1210 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1211 xenvif_skb_zerocopy_prepare(queue, nskb); 1212 if (xenvif_handle_frag_list(queue, skb)) { 1213 if (net_ratelimit()) 1214 netdev_err(queue->vif->dev, 1215 "Not enough memory to consolidate frag_list!\n"); 1216 xenvif_skb_zerocopy_prepare(queue, skb); 1217 kfree_skb(skb); 1218 continue; 1219 } 1220 /* Copied all the bits from the frag list -- free it. */ 1221 skb_frag_list_init(skb); 1222 kfree_skb(nskb); 1223 } 1224 1225 skb->dev = queue->vif->dev; 1226 skb->protocol = eth_type_trans(skb, skb->dev); 1227 skb_reset_network_header(skb); 1228 1229 if (checksum_setup(queue, skb)) { 1230 netdev_dbg(queue->vif->dev, 1231 "Can't setup checksum in net_tx_action\n"); 1232 /* We have to set this flag to trigger the callback */ 1233 if (skb_shinfo(skb)->destructor_arg) 1234 xenvif_skb_zerocopy_prepare(queue, skb); 1235 kfree_skb(skb); 1236 continue; 1237 } 1238 1239 skb_probe_transport_header(skb); 1240 1241 /* If the packet is GSO then we will have just set up the 1242 * transport header offset in checksum_setup so it's now 1243 * straightforward to calculate gso_segs. 1244 */ 1245 if (skb_is_gso(skb)) { 1246 int mss, hdrlen; 1247 1248 /* GSO implies having the L4 header. */ 1249 WARN_ON_ONCE(!skb_transport_header_was_set(skb)); 1250 if (unlikely(!skb_transport_header_was_set(skb))) { 1251 kfree_skb(skb); 1252 continue; 1253 } 1254 1255 mss = skb_shinfo(skb)->gso_size; 1256 hdrlen = skb_tcp_all_headers(skb); 1257 1258 skb_shinfo(skb)->gso_segs = 1259 DIV_ROUND_UP(skb->len - hdrlen, mss); 1260 } 1261 1262 queue->stats.rx_bytes += skb->len; 1263 queue->stats.rx_packets++; 1264 1265 work_done++; 1266 1267 /* Set this flag right before netif_receive_skb, otherwise 1268 * someone might think this packet already left netback, and 1269 * do a skb_copy_ubufs while we are still in control of the 1270 * skb. E.g. the __pskb_pull_tail earlier can do such thing. 1271 */ 1272 if (skb_shinfo(skb)->destructor_arg) { 1273 xenvif_skb_zerocopy_prepare(queue, skb); 1274 queue->stats.tx_zerocopy_sent++; 1275 } 1276 1277 netif_receive_skb(skb); 1278 } 1279 1280 return work_done; 1281 } 1282 1283 static void xenvif_zerocopy_callback(struct sk_buff *skb, 1284 struct ubuf_info *ubuf_base, 1285 bool zerocopy_success) 1286 { 1287 unsigned long flags; 1288 pending_ring_idx_t index; 1289 struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); 1290 struct xenvif_queue *queue = ubuf_to_queue(ubuf); 1291 1292 /* This is the only place where we grab this lock, to protect callbacks 1293 * from each other. 1294 */ 1295 spin_lock_irqsave(&queue->callback_lock, flags); 1296 do { 1297 u16 pending_idx = ubuf->desc; 1298 ubuf = (struct ubuf_info_msgzc *) ubuf->ctx; 1299 BUG_ON(queue->dealloc_prod - queue->dealloc_cons >= 1300 MAX_PENDING_REQS); 1301 index = pending_index(queue->dealloc_prod); 1302 queue->dealloc_ring[index] = pending_idx; 1303 /* Sync with xenvif_tx_dealloc_action: 1304 * insert idx then incr producer. 1305 */ 1306 smp_wmb(); 1307 queue->dealloc_prod++; 1308 } while (ubuf); 1309 spin_unlock_irqrestore(&queue->callback_lock, flags); 1310 1311 if (likely(zerocopy_success)) 1312 queue->stats.tx_zerocopy_success++; 1313 else 1314 queue->stats.tx_zerocopy_fail++; 1315 xenvif_skb_zerocopy_complete(queue); 1316 } 1317 1318 const struct ubuf_info_ops xenvif_ubuf_ops = { 1319 .complete = xenvif_zerocopy_callback, 1320 }; 1321 1322 static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) 1323 { 1324 struct gnttab_unmap_grant_ref *gop; 1325 pending_ring_idx_t dc, dp; 1326 u16 pending_idx, pending_idx_release[MAX_PENDING_REQS]; 1327 unsigned int i = 0; 1328 1329 dc = queue->dealloc_cons; 1330 gop = queue->tx_unmap_ops; 1331 1332 /* Free up any grants we have finished using */ 1333 do { 1334 dp = queue->dealloc_prod; 1335 1336 /* Ensure we see all indices enqueued by all 1337 * xenvif_zerocopy_callback(). 1338 */ 1339 smp_rmb(); 1340 1341 while (dc != dp) { 1342 BUG_ON(gop - queue->tx_unmap_ops >= MAX_PENDING_REQS); 1343 pending_idx = 1344 queue->dealloc_ring[pending_index(dc++)]; 1345 1346 pending_idx_release[gop - queue->tx_unmap_ops] = 1347 pending_idx; 1348 queue->pages_to_unmap[gop - queue->tx_unmap_ops] = 1349 queue->mmap_pages[pending_idx]; 1350 gnttab_set_unmap_op(gop, 1351 idx_to_kaddr(queue, pending_idx), 1352 GNTMAP_host_map, 1353 queue->grant_tx_handle[pending_idx]); 1354 xenvif_grant_handle_reset(queue, pending_idx); 1355 ++gop; 1356 } 1357 1358 } while (dp != queue->dealloc_prod); 1359 1360 queue->dealloc_cons = dc; 1361 1362 if (gop - queue->tx_unmap_ops > 0) { 1363 int ret; 1364 ret = gnttab_unmap_refs(queue->tx_unmap_ops, 1365 NULL, 1366 queue->pages_to_unmap, 1367 gop - queue->tx_unmap_ops); 1368 if (ret) { 1369 netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tu ret %d\n", 1370 gop - queue->tx_unmap_ops, ret); 1371 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) { 1372 if (gop[i].status != GNTST_okay) 1373 netdev_err(queue->vif->dev, 1374 " host_addr: 0x%llx handle: 0x%x status: %d\n", 1375 gop[i].host_addr, 1376 gop[i].handle, 1377 gop[i].status); 1378 } 1379 BUG(); 1380 } 1381 } 1382 1383 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) 1384 xenvif_idx_release(queue, pending_idx_release[i], 1385 XEN_NETIF_RSP_OKAY); 1386 } 1387 1388 1389 /* Called after netfront has transmitted */ 1390 int xenvif_tx_action(struct xenvif_queue *queue, int budget) 1391 { 1392 unsigned nr_mops = 0, nr_cops = 0; 1393 int work_done, ret; 1394 1395 if (unlikely(!tx_work_todo(queue))) 1396 return 0; 1397 1398 xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops); 1399 1400 if (nr_cops == 0) 1401 return 0; 1402 1403 gnttab_batch_copy(queue->tx_copy_ops, nr_cops); 1404 if (nr_mops != 0) { 1405 ret = gnttab_map_refs(queue->tx_map_ops, 1406 NULL, 1407 queue->pages_to_map, 1408 nr_mops); 1409 if (ret) { 1410 unsigned int i; 1411 1412 netdev_err(queue->vif->dev, "Map fail: nr %u ret %d\n", 1413 nr_mops, ret); 1414 for (i = 0; i < nr_mops; ++i) 1415 WARN_ON_ONCE(queue->tx_map_ops[i].status == 1416 GNTST_okay); 1417 } 1418 } 1419 1420 work_done = xenvif_tx_submit(queue); 1421 1422 return work_done; 1423 } 1424 1425 static void _make_tx_response(struct xenvif_queue *queue, 1426 const struct xen_netif_tx_request *txp, 1427 unsigned int extra_count, 1428 s8 status) 1429 { 1430 RING_IDX i = queue->tx.rsp_prod_pvt; 1431 struct xen_netif_tx_response *resp; 1432 1433 resp = RING_GET_RESPONSE(&queue->tx, i); 1434 resp->id = txp->id; 1435 resp->status = status; 1436 1437 while (extra_count-- != 0) 1438 RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL; 1439 1440 queue->tx.rsp_prod_pvt = ++i; 1441 } 1442 1443 static void push_tx_responses(struct xenvif_queue *queue) 1444 { 1445 int notify; 1446 1447 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify); 1448 if (notify) 1449 notify_remote_via_irq(queue->tx_irq); 1450 } 1451 1452 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx, 1453 s8 status) 1454 { 1455 struct pending_tx_info *pending_tx_info; 1456 pending_ring_idx_t index; 1457 unsigned long flags; 1458 1459 pending_tx_info = &queue->pending_tx_info[pending_idx]; 1460 1461 spin_lock_irqsave(&queue->response_lock, flags); 1462 1463 _make_tx_response(queue, &pending_tx_info->req, 1464 pending_tx_info->extra_count, status); 1465 1466 /* Release the pending index before pusing the Tx response so 1467 * its available before a new Tx request is pushed by the 1468 * frontend. 1469 */ 1470 index = pending_index(queue->pending_prod++); 1471 queue->pending_ring[index] = pending_idx; 1472 1473 push_tx_responses(queue); 1474 1475 spin_unlock_irqrestore(&queue->response_lock, flags); 1476 } 1477 1478 static void make_tx_response(struct xenvif_queue *queue, 1479 const struct xen_netif_tx_request *txp, 1480 unsigned int extra_count, 1481 s8 status) 1482 { 1483 unsigned long flags; 1484 1485 spin_lock_irqsave(&queue->response_lock, flags); 1486 1487 _make_tx_response(queue, txp, extra_count, status); 1488 push_tx_responses(queue); 1489 1490 spin_unlock_irqrestore(&queue->response_lock, flags); 1491 } 1492 1493 static void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx) 1494 { 1495 int ret; 1496 struct gnttab_unmap_grant_ref tx_unmap_op; 1497 1498 gnttab_set_unmap_op(&tx_unmap_op, 1499 idx_to_kaddr(queue, pending_idx), 1500 GNTMAP_host_map, 1501 queue->grant_tx_handle[pending_idx]); 1502 xenvif_grant_handle_reset(queue, pending_idx); 1503 1504 ret = gnttab_unmap_refs(&tx_unmap_op, NULL, 1505 &queue->mmap_pages[pending_idx], 1); 1506 if (ret) { 1507 netdev_err(queue->vif->dev, 1508 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: 0x%x status: %d\n", 1509 ret, 1510 pending_idx, 1511 tx_unmap_op.host_addr, 1512 tx_unmap_op.handle, 1513 tx_unmap_op.status); 1514 BUG(); 1515 } 1516 } 1517 1518 static inline int tx_work_todo(struct xenvif_queue *queue) 1519 { 1520 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))) 1521 return 1; 1522 1523 return 0; 1524 } 1525 1526 static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue) 1527 { 1528 return queue->dealloc_cons != queue->dealloc_prod; 1529 } 1530 1531 void xenvif_unmap_frontend_data_rings(struct xenvif_queue *queue) 1532 { 1533 if (queue->tx.sring) 1534 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1535 queue->tx.sring); 1536 if (queue->rx.sring) 1537 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif), 1538 queue->rx.sring); 1539 } 1540 1541 int xenvif_map_frontend_data_rings(struct xenvif_queue *queue, 1542 grant_ref_t tx_ring_ref, 1543 grant_ref_t rx_ring_ref) 1544 { 1545 void *addr; 1546 struct xen_netif_tx_sring *txs; 1547 struct xen_netif_rx_sring *rxs; 1548 RING_IDX rsp_prod, req_prod; 1549 int err; 1550 1551 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1552 &tx_ring_ref, 1, &addr); 1553 if (err) 1554 goto err; 1555 1556 txs = (struct xen_netif_tx_sring *)addr; 1557 rsp_prod = READ_ONCE(txs->rsp_prod); 1558 req_prod = READ_ONCE(txs->req_prod); 1559 1560 BACK_RING_ATTACH(&queue->tx, txs, rsp_prod, XEN_PAGE_SIZE); 1561 1562 err = -EIO; 1563 if (req_prod - rsp_prod > RING_SIZE(&queue->tx)) 1564 goto err; 1565 1566 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif), 1567 &rx_ring_ref, 1, &addr); 1568 if (err) 1569 goto err; 1570 1571 rxs = (struct xen_netif_rx_sring *)addr; 1572 rsp_prod = READ_ONCE(rxs->rsp_prod); 1573 req_prod = READ_ONCE(rxs->req_prod); 1574 1575 BACK_RING_ATTACH(&queue->rx, rxs, rsp_prod, XEN_PAGE_SIZE); 1576 1577 err = -EIO; 1578 if (req_prod - rsp_prod > RING_SIZE(&queue->rx)) 1579 goto err; 1580 1581 return 0; 1582 1583 err: 1584 xenvif_unmap_frontend_data_rings(queue); 1585 return err; 1586 } 1587 1588 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue) 1589 { 1590 /* Dealloc thread must remain running until all inflight 1591 * packets complete. 1592 */ 1593 return kthread_should_stop() && 1594 !atomic_read(&queue->inflight_packets); 1595 } 1596 1597 int xenvif_dealloc_kthread(void *data) 1598 { 1599 struct xenvif_queue *queue = data; 1600 1601 for (;;) { 1602 wait_event_interruptible(queue->dealloc_wq, 1603 tx_dealloc_work_todo(queue) || 1604 xenvif_dealloc_kthread_should_stop(queue)); 1605 if (xenvif_dealloc_kthread_should_stop(queue)) 1606 break; 1607 1608 xenvif_tx_dealloc_action(queue); 1609 cond_resched(); 1610 } 1611 1612 /* Unmap anything remaining*/ 1613 if (tx_dealloc_work_todo(queue)) 1614 xenvif_tx_dealloc_action(queue); 1615 1616 return 0; 1617 } 1618 1619 static void make_ctrl_response(struct xenvif *vif, 1620 const struct xen_netif_ctrl_request *req, 1621 u32 status, u32 data) 1622 { 1623 RING_IDX idx = vif->ctrl.rsp_prod_pvt; 1624 struct xen_netif_ctrl_response rsp = { 1625 .id = req->id, 1626 .type = req->type, 1627 .status = status, 1628 .data = data, 1629 }; 1630 1631 *RING_GET_RESPONSE(&vif->ctrl, idx) = rsp; 1632 vif->ctrl.rsp_prod_pvt = ++idx; 1633 } 1634 1635 static void push_ctrl_response(struct xenvif *vif) 1636 { 1637 int notify; 1638 1639 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->ctrl, notify); 1640 if (notify) 1641 notify_remote_via_irq(vif->ctrl_irq); 1642 } 1643 1644 static void process_ctrl_request(struct xenvif *vif, 1645 const struct xen_netif_ctrl_request *req) 1646 { 1647 u32 status = XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 1648 u32 data = 0; 1649 1650 switch (req->type) { 1651 case XEN_NETIF_CTRL_TYPE_SET_HASH_ALGORITHM: 1652 status = xenvif_set_hash_alg(vif, req->data[0]); 1653 break; 1654 1655 case XEN_NETIF_CTRL_TYPE_GET_HASH_FLAGS: 1656 status = xenvif_get_hash_flags(vif, &data); 1657 break; 1658 1659 case XEN_NETIF_CTRL_TYPE_SET_HASH_FLAGS: 1660 status = xenvif_set_hash_flags(vif, req->data[0]); 1661 break; 1662 1663 case XEN_NETIF_CTRL_TYPE_SET_HASH_KEY: 1664 status = xenvif_set_hash_key(vif, req->data[0], 1665 req->data[1]); 1666 break; 1667 1668 case XEN_NETIF_CTRL_TYPE_GET_HASH_MAPPING_SIZE: 1669 status = XEN_NETIF_CTRL_STATUS_SUCCESS; 1670 data = XEN_NETBK_MAX_HASH_MAPPING_SIZE; 1671 break; 1672 1673 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING_SIZE: 1674 status = xenvif_set_hash_mapping_size(vif, 1675 req->data[0]); 1676 break; 1677 1678 case XEN_NETIF_CTRL_TYPE_SET_HASH_MAPPING: 1679 status = xenvif_set_hash_mapping(vif, req->data[0], 1680 req->data[1], 1681 req->data[2]); 1682 break; 1683 1684 default: 1685 break; 1686 } 1687 1688 make_ctrl_response(vif, req, status, data); 1689 push_ctrl_response(vif); 1690 } 1691 1692 static void xenvif_ctrl_action(struct xenvif *vif) 1693 { 1694 for (;;) { 1695 RING_IDX req_prod, req_cons; 1696 1697 req_prod = vif->ctrl.sring->req_prod; 1698 req_cons = vif->ctrl.req_cons; 1699 1700 /* Make sure we can see requests before we process them. */ 1701 rmb(); 1702 1703 if (req_cons == req_prod) 1704 break; 1705 1706 while (req_cons != req_prod) { 1707 struct xen_netif_ctrl_request req; 1708 1709 RING_COPY_REQUEST(&vif->ctrl, req_cons, &req); 1710 req_cons++; 1711 1712 process_ctrl_request(vif, &req); 1713 } 1714 1715 vif->ctrl.req_cons = req_cons; 1716 vif->ctrl.sring->req_event = req_cons + 1; 1717 } 1718 } 1719 1720 static bool xenvif_ctrl_work_todo(struct xenvif *vif) 1721 { 1722 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->ctrl))) 1723 return true; 1724 1725 return false; 1726 } 1727 1728 irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data) 1729 { 1730 struct xenvif *vif = data; 1731 unsigned int eoi_flag = XEN_EOI_FLAG_SPURIOUS; 1732 1733 while (xenvif_ctrl_work_todo(vif)) { 1734 xenvif_ctrl_action(vif); 1735 eoi_flag = 0; 1736 } 1737 1738 xen_irq_lateeoi(irq, eoi_flag); 1739 1740 return IRQ_HANDLED; 1741 } 1742 1743 static int __init netback_init(void) 1744 { 1745 int rc = 0; 1746 1747 if (!xen_domain()) 1748 return -ENODEV; 1749 1750 /* Allow as many queues as there are CPUs but max. 8 if user has not 1751 * specified a value. 1752 */ 1753 if (xenvif_max_queues == 0) 1754 xenvif_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, 1755 num_online_cpus()); 1756 1757 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) { 1758 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n", 1759 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX); 1760 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX; 1761 } 1762 1763 rc = xenvif_xenbus_init(); 1764 if (rc) 1765 goto failed_init; 1766 1767 #ifdef CONFIG_DEBUG_FS 1768 xen_netback_dbg_root = debugfs_create_dir("xen-netback", NULL); 1769 #endif /* CONFIG_DEBUG_FS */ 1770 1771 return 0; 1772 1773 failed_init: 1774 return rc; 1775 } 1776 1777 module_init(netback_init); 1778 1779 static void __exit netback_fini(void) 1780 { 1781 #ifdef CONFIG_DEBUG_FS 1782 debugfs_remove_recursive(xen_netback_dbg_root); 1783 #endif /* CONFIG_DEBUG_FS */ 1784 xenvif_xenbus_fini(); 1785 } 1786 module_exit(netback_fini); 1787 1788 MODULE_DESCRIPTION("Xen backend network device module"); 1789 MODULE_LICENSE("Dual BSD/GPL"); 1790 MODULE_ALIAS("xen-backend:vif"); 1791