1 /* 2 * Copyright (C) 2013-2016 Vincenzo Maffione 3 * Copyright (C) 2013-2016 Luigi Rizzo 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * This module implements netmap support on top of standard, 30 * unmodified device drivers. 31 * 32 * A NIOCREGIF request is handled here if the device does not 33 * have native support. TX and RX rings are emulated as follows: 34 * 35 * NIOCREGIF 36 * We preallocate a block of TX mbufs (roughly as many as 37 * tx descriptors; the number is not critical) to speed up 38 * operation during transmissions. The refcount on most of 39 * these buffers is artificially bumped up so we can recycle 40 * them more easily. Also, the destructor is intercepted 41 * so we use it as an interrupt notification to wake up 42 * processes blocked on a poll(). 43 * 44 * For each receive ring we allocate one "struct mbq" 45 * (an mbuf tailq plus a spinlock). We intercept packets 46 * (through if_input) 47 * on the receive path and put them in the mbq from which 48 * netmap receive routines can grab them. 49 * 50 * TX: 51 * in the generic_txsync() routine, netmap buffers are copied 52 * (or linked, in a future) to the preallocated mbufs 53 * and pushed to the transmit queue. Some of these mbufs 54 * (those with NS_REPORT, or otherwise every half ring) 55 * have the refcount=1, others have refcount=2. 56 * When the destructor is invoked, we take that as 57 * a notification that all mbufs up to that one in 58 * the specific ring have been completed, and generate 59 * the equivalent of a transmit interrupt. 60 * 61 * RX: 62 * 63 */ 64 65 #ifdef __FreeBSD__ 66 67 #include <sys/cdefs.h> /* prerequisite */ 68 __FBSDID("$FreeBSD$"); 69 70 #include <sys/types.h> 71 #include <sys/errno.h> 72 #include <sys/malloc.h> 73 #include <sys/lock.h> /* PROT_EXEC */ 74 #include <sys/rwlock.h> 75 #include <sys/socket.h> /* sockaddrs */ 76 #include <sys/selinfo.h> 77 #include <net/if.h> 78 #include <net/if_var.h> 79 #include <machine/bus.h> /* bus_dmamap_* in netmap_kern.h */ 80 81 // XXX temporary - D() defined here 82 #include <net/netmap.h> 83 #include <dev/netmap/netmap_kern.h> 84 #include <dev/netmap/netmap_mem2.h> 85 86 #define rtnl_lock() ND("rtnl_lock called") 87 #define rtnl_unlock() ND("rtnl_unlock called") 88 #define MBUF_RXQ(m) ((m)->m_pkthdr.flowid) 89 #define smp_mb() 90 91 /* 92 * FreeBSD mbuf allocator/deallocator in emulation mode: 93 */ 94 #if __FreeBSD_version < 1100000 95 96 /* 97 * For older versions of FreeBSD: 98 * 99 * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE 100 * so that the destructor, if invoked, will not free the packet. 101 * In principle we should set the destructor only on demand, 102 * but since there might be a race we better do it on allocation. 103 * As a consequence, we also need to set the destructor or we 104 * would leak buffers. 105 */ 106 107 /* mbuf destructor, also need to change the type to EXT_EXTREF, 108 * add an M_NOFREE flag, and then clear the flag and 109 * chain into uma_zfree(zone_pack, mf) 110 * (or reinstall the buffer ?) 111 */ 112 #define SET_MBUF_DESTRUCTOR(m, fn) do { \ 113 (m)->m_ext.ext_free = (void *)fn; \ 114 (m)->m_ext.ext_type = EXT_EXTREF; \ 115 } while (0) 116 117 static int 118 void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) 119 { 120 /* restore original mbuf */ 121 m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1; 122 m->m_ext.ext_arg1 = NULL; 123 m->m_ext.ext_type = EXT_PACKET; 124 m->m_ext.ext_free = NULL; 125 if (MBUF_REFCNT(m) == 0) 126 SET_MBUF_REFCNT(m, 1); 127 uma_zfree(zone_pack, m); 128 129 return 0; 130 } 131 132 static inline struct mbuf * 133 nm_os_get_mbuf(struct ifnet *ifp, int len) 134 { 135 struct mbuf *m; 136 137 (void)ifp; 138 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 139 if (m) { 140 /* m_getcl() (mb_ctor_mbuf) has an assert that checks that 141 * M_NOFREE flag is not specified as third argument, 142 * so we have to set M_NOFREE after m_getcl(). */ 143 m->m_flags |= M_NOFREE; 144 m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save 145 m->m_ext.ext_free = (void *)void_mbuf_dtor; 146 m->m_ext.ext_type = EXT_EXTREF; 147 ND(5, "create m %p refcnt %d", m, MBUF_REFCNT(m)); 148 } 149 return m; 150 } 151 152 #else /* __FreeBSD_version >= 1100000 */ 153 154 /* 155 * Newer versions of FreeBSD, using a straightforward scheme. 156 * 157 * We allocate mbufs with m_gethdr(), since the mbuf header is needed 158 * by the driver. We also attach a customly-provided external storage, 159 * which in this case is a netmap buffer. When calling m_extadd(), however 160 * we pass a NULL address, since the real address (and length) will be 161 * filled in by nm_os_generic_xmit_frame() right before calling 162 * if_transmit(). 163 * 164 * The dtor function does nothing, however we need it since mb_free_ext() 165 * has a KASSERT(), checking that the mbuf dtor function is not NULL. 166 */ 167 168 static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { } 169 170 #define SET_MBUF_DESTRUCTOR(m, fn) do { \ 171 (m)->m_ext.ext_free = fn ? (void *)fn : (void *)void_mbuf_dtor; \ 172 } while (0) 173 174 static inline struct mbuf * 175 nm_os_get_mbuf(struct ifnet *ifp, int len) 176 { 177 struct mbuf *m; 178 179 (void)ifp; 180 (void)len; 181 182 m = m_gethdr(M_NOWAIT, MT_DATA); 183 if (m == NULL) { 184 return m; 185 } 186 187 m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor, 188 NULL, NULL, 0, EXT_NET_DRV); 189 190 return m; 191 } 192 193 #endif /* __FreeBSD_version >= 1100000 */ 194 195 #elif defined _WIN32 196 197 #include "win_glue.h" 198 199 #define rtnl_lock() ND("rtnl_lock called") 200 #define rtnl_unlock() ND("rtnl_unlock called") 201 #define MBUF_TXQ(m) 0//((m)->m_pkthdr.flowid) 202 #define MBUF_RXQ(m) 0//((m)->m_pkthdr.flowid) 203 #define smp_mb() //XXX: to be correctly defined 204 205 #else /* linux */ 206 207 #include "bsd_glue.h" 208 209 #include <linux/rtnetlink.h> /* rtnl_[un]lock() */ 210 #include <linux/ethtool.h> /* struct ethtool_ops, get_ringparam */ 211 #include <linux/hrtimer.h> 212 213 static inline struct mbuf * 214 nm_os_get_mbuf(struct ifnet *ifp, int len) 215 { 216 return alloc_skb(ifp->needed_headroom + len + 217 ifp->needed_tailroom, GFP_ATOMIC); 218 } 219 220 #endif /* linux */ 221 222 223 /* Common headers. */ 224 #include <net/netmap.h> 225 #include <dev/netmap/netmap_kern.h> 226 #include <dev/netmap/netmap_mem2.h> 227 228 229 #define for_each_kring_n(_i, _k, _karr, _n) \ 230 for (_k=_karr, _i = 0; _i < _n; (_k)++, (_i)++) 231 232 #define for_each_tx_kring(_i, _k, _na) \ 233 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings) 234 #define for_each_tx_kring_h(_i, _k, _na) \ 235 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings + 1) 236 237 #define for_each_rx_kring(_i, _k, _na) \ 238 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings) 239 #define for_each_rx_kring_h(_i, _k, _na) \ 240 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings + 1) 241 242 243 /* ======================== PERFORMANCE STATISTICS =========================== */ 244 245 #ifdef RATE_GENERIC 246 #define IFRATE(x) x 247 struct rate_stats { 248 unsigned long txpkt; 249 unsigned long txsync; 250 unsigned long txirq; 251 unsigned long txrepl; 252 unsigned long txdrop; 253 unsigned long rxpkt; 254 unsigned long rxirq; 255 unsigned long rxsync; 256 }; 257 258 struct rate_context { 259 unsigned refcount; 260 struct timer_list timer; 261 struct rate_stats new; 262 struct rate_stats old; 263 }; 264 265 #define RATE_PRINTK(_NAME_) \ 266 printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); 267 #define RATE_PERIOD 2 268 static void rate_callback(unsigned long arg) 269 { 270 struct rate_context * ctx = (struct rate_context *)arg; 271 struct rate_stats cur = ctx->new; 272 int r; 273 274 RATE_PRINTK(txpkt); 275 RATE_PRINTK(txsync); 276 RATE_PRINTK(txirq); 277 RATE_PRINTK(txrepl); 278 RATE_PRINTK(txdrop); 279 RATE_PRINTK(rxpkt); 280 RATE_PRINTK(rxsync); 281 RATE_PRINTK(rxirq); 282 printk("\n"); 283 284 ctx->old = cur; 285 r = mod_timer(&ctx->timer, jiffies + 286 msecs_to_jiffies(RATE_PERIOD * 1000)); 287 if (unlikely(r)) 288 D("[v1000] Error: mod_timer()"); 289 } 290 291 static struct rate_context rate_ctx; 292 293 void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi) 294 { 295 if (txp) rate_ctx.new.txpkt++; 296 if (txs) rate_ctx.new.txsync++; 297 if (txi) rate_ctx.new.txirq++; 298 if (rxp) rate_ctx.new.rxpkt++; 299 if (rxs) rate_ctx.new.rxsync++; 300 if (rxi) rate_ctx.new.rxirq++; 301 } 302 303 #else /* !RATE */ 304 #define IFRATE(x) 305 #endif /* !RATE */ 306 307 308 /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */ 309 310 /* 311 * Wrapper used by the generic adapter layer to notify 312 * the poller threads. Differently from netmap_rx_irq(), we check 313 * only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq. 314 */ 315 void 316 netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done) 317 { 318 if (unlikely(!nm_netmap_on(na))) 319 return; 320 321 netmap_common_irq(na, q, work_done); 322 #ifdef RATE_GENERIC 323 if (work_done) 324 rate_ctx.new.rxirq++; 325 else 326 rate_ctx.new.txirq++; 327 #endif /* RATE_GENERIC */ 328 } 329 330 static int 331 generic_netmap_unregister(struct netmap_adapter *na) 332 { 333 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 334 struct netmap_kring *kring = NULL; 335 int i, r; 336 337 if (na->active_fds == 0) { 338 D("Generic adapter %p goes off", na); 339 rtnl_lock(); 340 341 na->na_flags &= ~NAF_NETMAP_ON; 342 343 /* Release packet steering control. */ 344 nm_os_catch_tx(gna, 0); 345 346 /* Stop intercepting packets on the RX path. */ 347 nm_os_catch_rx(gna, 0); 348 349 rtnl_unlock(); 350 } 351 352 for_each_rx_kring_h(r, kring, na) { 353 if (nm_kring_pending_off(kring)) { 354 D("RX ring %d of generic adapter %p goes off", r, na); 355 kring->nr_mode = NKR_NETMAP_OFF; 356 } 357 } 358 for_each_tx_kring_h(r, kring, na) { 359 if (nm_kring_pending_off(kring)) { 360 kring->nr_mode = NKR_NETMAP_OFF; 361 D("TX ring %d of generic adapter %p goes off", r, na); 362 } 363 } 364 365 for_each_rx_kring(r, kring, na) { 366 /* Free the mbufs still pending in the RX queues, 367 * that did not end up into the corresponding netmap 368 * RX rings. */ 369 mbq_safe_purge(&kring->rx_queue); 370 nm_os_mitigation_cleanup(&gna->mit[r]); 371 } 372 373 /* Decrement reference counter for the mbufs in the 374 * TX pools. These mbufs can be still pending in drivers, 375 * (e.g. this happens with virtio-net driver, which 376 * does lazy reclaiming of transmitted mbufs). */ 377 for_each_tx_kring(r, kring, na) { 378 /* We must remove the destructor on the TX event, 379 * because the destructor invokes netmap code, and 380 * the netmap module may disappear before the 381 * TX event is consumed. */ 382 mtx_lock_spin(&kring->tx_event_lock); 383 if (kring->tx_event) { 384 SET_MBUF_DESTRUCTOR(kring->tx_event, NULL); 385 } 386 kring->tx_event = NULL; 387 mtx_unlock_spin(&kring->tx_event_lock); 388 } 389 390 if (na->active_fds == 0) { 391 free(gna->mit, M_DEVBUF); 392 393 for_each_rx_kring(r, kring, na) { 394 mbq_safe_fini(&kring->rx_queue); 395 } 396 397 for_each_tx_kring(r, kring, na) { 398 mtx_destroy(&kring->tx_event_lock); 399 if (kring->tx_pool == NULL) { 400 continue; 401 } 402 403 for (i=0; i<na->num_tx_desc; i++) { 404 if (kring->tx_pool[i]) { 405 m_freem(kring->tx_pool[i]); 406 } 407 } 408 free(kring->tx_pool, M_DEVBUF); 409 kring->tx_pool = NULL; 410 } 411 412 #ifdef RATE_GENERIC 413 if (--rate_ctx.refcount == 0) { 414 D("del_timer()"); 415 del_timer(&rate_ctx.timer); 416 } 417 #endif 418 } 419 420 return 0; 421 } 422 423 /* Enable/disable netmap mode for a generic network interface. */ 424 static int 425 generic_netmap_register(struct netmap_adapter *na, int enable) 426 { 427 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 428 struct netmap_kring *kring = NULL; 429 int error; 430 int i, r; 431 432 if (!na) { 433 return EINVAL; 434 } 435 436 if (!enable) { 437 /* This is actually an unregif. */ 438 return generic_netmap_unregister(na); 439 } 440 441 if (na->active_fds == 0) { 442 D("Generic adapter %p goes on", na); 443 /* Do all memory allocations when (na->active_fds == 0), to 444 * simplify error management. */ 445 446 /* Allocate memory for mitigation support on all the rx queues. */ 447 gna->mit = malloc(na->num_rx_rings * sizeof(struct nm_generic_mit), 448 M_DEVBUF, M_NOWAIT | M_ZERO); 449 if (!gna->mit) { 450 D("mitigation allocation failed"); 451 error = ENOMEM; 452 goto out; 453 } 454 455 for_each_rx_kring(r, kring, na) { 456 /* Init mitigation support. */ 457 nm_os_mitigation_init(&gna->mit[r], r, na); 458 459 /* Initialize the rx queue, as generic_rx_handler() can 460 * be called as soon as nm_os_catch_rx() returns. 461 */ 462 mbq_safe_init(&kring->rx_queue); 463 } 464 465 /* 466 * Prepare mbuf pools (parallel to the tx rings), for packet 467 * transmission. Don't preallocate the mbufs here, it's simpler 468 * to leave this task to txsync. 469 */ 470 for_each_tx_kring(r, kring, na) { 471 kring->tx_pool = NULL; 472 } 473 for_each_tx_kring(r, kring, na) { 474 kring->tx_pool = 475 malloc(na->num_tx_desc * sizeof(struct mbuf *), 476 M_DEVBUF, M_NOWAIT | M_ZERO); 477 if (!kring->tx_pool) { 478 D("tx_pool allocation failed"); 479 error = ENOMEM; 480 goto free_tx_pools; 481 } 482 mtx_init(&kring->tx_event_lock, "tx_event_lock", 483 NULL, MTX_SPIN); 484 } 485 } 486 487 for_each_rx_kring_h(r, kring, na) { 488 if (nm_kring_pending_on(kring)) { 489 D("RX ring %d of generic adapter %p goes on", r, na); 490 kring->nr_mode = NKR_NETMAP_ON; 491 } 492 493 } 494 for_each_tx_kring_h(r, kring, na) { 495 if (nm_kring_pending_on(kring)) { 496 D("TX ring %d of generic adapter %p goes on", r, na); 497 kring->nr_mode = NKR_NETMAP_ON; 498 } 499 } 500 501 for_each_tx_kring(r, kring, na) { 502 /* Initialize tx_pool and tx_event. */ 503 for (i=0; i<na->num_tx_desc; i++) { 504 kring->tx_pool[i] = NULL; 505 } 506 507 kring->tx_event = NULL; 508 } 509 510 if (na->active_fds == 0) { 511 rtnl_lock(); 512 513 /* Prepare to intercept incoming traffic. */ 514 error = nm_os_catch_rx(gna, 1); 515 if (error) { 516 D("nm_os_catch_rx(1) failed (%d)", error); 517 goto register_handler; 518 } 519 520 /* Make netmap control the packet steering. */ 521 error = nm_os_catch_tx(gna, 1); 522 if (error) { 523 D("nm_os_catch_tx(1) failed (%d)", error); 524 goto catch_rx; 525 } 526 527 rtnl_unlock(); 528 529 na->na_flags |= NAF_NETMAP_ON; 530 531 #ifdef RATE_GENERIC 532 if (rate_ctx.refcount == 0) { 533 D("setup_timer()"); 534 memset(&rate_ctx, 0, sizeof(rate_ctx)); 535 setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); 536 if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { 537 D("Error: mod_timer()"); 538 } 539 } 540 rate_ctx.refcount++; 541 #endif /* RATE */ 542 } 543 544 return 0; 545 546 /* Here (na->active_fds == 0) holds. */ 547 catch_rx: 548 nm_os_catch_rx(gna, 0); 549 register_handler: 550 rtnl_unlock(); 551 free_tx_pools: 552 for_each_tx_kring(r, kring, na) { 553 mtx_destroy(&kring->tx_event_lock); 554 if (kring->tx_pool == NULL) { 555 continue; 556 } 557 free(kring->tx_pool, M_DEVBUF); 558 kring->tx_pool = NULL; 559 } 560 for_each_rx_kring(r, kring, na) { 561 mbq_safe_fini(&kring->rx_queue); 562 } 563 free(gna->mit, M_DEVBUF); 564 out: 565 566 return error; 567 } 568 569 /* 570 * Callback invoked when the device driver frees an mbuf used 571 * by netmap to transmit a packet. This usually happens when 572 * the NIC notifies the driver that transmission is completed. 573 */ 574 static void 575 generic_mbuf_destructor(struct mbuf *m) 576 { 577 struct netmap_adapter *na = NA(GEN_TX_MBUF_IFP(m)); 578 struct netmap_kring *kring; 579 unsigned int r = MBUF_TXQ(m); 580 unsigned int r_orig = r; 581 582 if (unlikely(!nm_netmap_on(na) || r >= na->num_tx_rings)) { 583 D("Error: no netmap adapter on device %p", 584 GEN_TX_MBUF_IFP(m)); 585 return; 586 } 587 588 /* 589 * First, clear the event mbuf. 590 * In principle, the event 'm' should match the one stored 591 * on ring 'r'. However we check it explicitely to stay 592 * safe against lower layers (qdisc, driver, etc.) changing 593 * MBUF_TXQ(m) under our feet. If the match is not found 594 * on 'r', we try to see if it belongs to some other ring. 595 */ 596 for (;;) { 597 bool match = false; 598 599 kring = &na->tx_rings[r]; 600 mtx_lock_spin(&kring->tx_event_lock); 601 if (kring->tx_event == m) { 602 kring->tx_event = NULL; 603 match = true; 604 } 605 mtx_unlock_spin(&kring->tx_event_lock); 606 607 if (match) { 608 if (r != r_orig) { 609 RD(1, "event %p migrated: ring %u --> %u", 610 m, r_orig, r); 611 } 612 break; 613 } 614 615 if (++r == na->num_tx_rings) r = 0; 616 617 if (r == r_orig) { 618 RD(1, "Cannot match event %p", m); 619 return; 620 } 621 } 622 623 /* Second, wake up clients. They will reclaim the event through 624 * txsync. */ 625 netmap_generic_irq(na, r, NULL); 626 #ifdef __FreeBSD__ 627 void_mbuf_dtor(m, NULL, NULL); 628 #endif 629 } 630 631 /* Record completed transmissions and update hwtail. 632 * 633 * The oldest tx buffer not yet completed is at nr_hwtail + 1, 634 * nr_hwcur is the first unsent buffer. 635 */ 636 static u_int 637 generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc) 638 { 639 u_int const lim = kring->nkr_num_slots - 1; 640 u_int nm_i = nm_next(kring->nr_hwtail, lim); 641 u_int hwcur = kring->nr_hwcur; 642 u_int n = 0; 643 struct mbuf **tx_pool = kring->tx_pool; 644 645 ND("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail); 646 647 while (nm_i != hwcur) { /* buffers not completed */ 648 struct mbuf *m = tx_pool[nm_i]; 649 650 if (txqdisc) { 651 if (m == NULL) { 652 /* Nothing to do, this is going 653 * to be replenished. */ 654 RD(3, "Is this happening?"); 655 656 } else if (MBUF_QUEUED(m)) { 657 break; /* Not dequeued yet. */ 658 659 } else if (MBUF_REFCNT(m) != 1) { 660 /* This mbuf has been dequeued but is still busy 661 * (refcount is 2). 662 * Leave it to the driver and replenish. */ 663 m_freem(m); 664 tx_pool[nm_i] = NULL; 665 } 666 667 } else { 668 if (unlikely(m == NULL)) { 669 int event_consumed; 670 671 /* This slot was used to place an event. */ 672 mtx_lock_spin(&kring->tx_event_lock); 673 event_consumed = (kring->tx_event == NULL); 674 mtx_unlock_spin(&kring->tx_event_lock); 675 if (!event_consumed) { 676 /* The event has not been consumed yet, 677 * still busy in the driver. */ 678 break; 679 } 680 /* The event has been consumed, we can go 681 * ahead. */ 682 683 } else if (MBUF_REFCNT(m) != 1) { 684 /* This mbuf is still busy: its refcnt is 2. */ 685 break; 686 } 687 } 688 689 n++; 690 nm_i = nm_next(nm_i, lim); 691 } 692 kring->nr_hwtail = nm_prev(nm_i, lim); 693 ND("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail); 694 695 return n; 696 } 697 698 /* Compute a slot index in the middle between inf and sup. */ 699 static inline u_int 700 ring_middle(u_int inf, u_int sup, u_int lim) 701 { 702 u_int n = lim + 1; 703 u_int e; 704 705 if (sup >= inf) { 706 e = (sup + inf) / 2; 707 } else { /* wrap around */ 708 e = (sup + n + inf) / 2; 709 if (e >= n) { 710 e -= n; 711 } 712 } 713 714 if (unlikely(e >= n)) { 715 D("This cannot happen"); 716 e = 0; 717 } 718 719 return e; 720 } 721 722 static void 723 generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) 724 { 725 u_int lim = kring->nkr_num_slots - 1; 726 struct mbuf *m; 727 u_int e; 728 u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */ 729 730 if (ntc == hwcur) { 731 return; /* all buffers are free */ 732 } 733 734 /* 735 * We have pending packets in the driver between hwtail+1 736 * and hwcur, and we have to chose one of these slot to 737 * generate a notification. 738 * There is a race but this is only called within txsync which 739 * does a double check. 740 */ 741 #if 0 742 /* Choose a slot in the middle, so that we don't risk ending 743 * up in a situation where the client continuously wake up, 744 * fills one or a few TX slots and go to sleep again. */ 745 e = ring_middle(ntc, hwcur, lim); 746 #else 747 /* Choose the first pending slot, to be safe against driver 748 * reordering mbuf transmissions. */ 749 e = ntc; 750 #endif 751 752 m = kring->tx_pool[e]; 753 if (m == NULL) { 754 /* An event is already in place. */ 755 return; 756 } 757 758 mtx_lock_spin(&kring->tx_event_lock); 759 if (kring->tx_event) { 760 /* An event is already in place. */ 761 mtx_unlock_spin(&kring->tx_event_lock); 762 return; 763 } 764 765 SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor); 766 kring->tx_event = m; 767 mtx_unlock_spin(&kring->tx_event_lock); 768 769 kring->tx_pool[e] = NULL; 770 771 ND(5, "Request Event at %d mbuf %p refcnt %d", e, m, m ? MBUF_REFCNT(m) : -2 ); 772 773 /* Decrement the refcount. This will free it if we lose the race 774 * with the driver. */ 775 m_freem(m); 776 smp_mb(); 777 } 778 779 780 /* 781 * generic_netmap_txsync() transforms netmap buffers into mbufs 782 * and passes them to the standard device driver 783 * (ndo_start_xmit() or ifp->if_transmit() ). 784 * On linux this is not done directly, but using dev_queue_xmit(), 785 * since it implements the TX flow control (and takes some locks). 786 */ 787 static int 788 generic_netmap_txsync(struct netmap_kring *kring, int flags) 789 { 790 struct netmap_adapter *na = kring->na; 791 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 792 struct ifnet *ifp = na->ifp; 793 struct netmap_ring *ring = kring->ring; 794 u_int nm_i; /* index into the netmap ring */ // j 795 u_int const lim = kring->nkr_num_slots - 1; 796 u_int const head = kring->rhead; 797 u_int ring_nr = kring->ring_id; 798 799 IFRATE(rate_ctx.new.txsync++); 800 801 rmb(); 802 803 /* 804 * First part: process new packets to send. 805 */ 806 nm_i = kring->nr_hwcur; 807 if (nm_i != head) { /* we have new packets to send */ 808 struct nm_os_gen_arg a; 809 u_int event = -1; 810 811 if (gna->txqdisc && nm_kr_txempty(kring)) { 812 /* In txqdisc mode, we ask for a delayed notification, 813 * but only when cur == hwtail, which means that the 814 * client is going to block. */ 815 event = ring_middle(nm_i, head, lim); 816 ND(3, "Place txqdisc event (hwcur=%u,event=%u," 817 "head=%u,hwtail=%u)", nm_i, event, head, 818 kring->nr_hwtail); 819 } 820 821 a.ifp = ifp; 822 a.ring_nr = ring_nr; 823 a.head = a.tail = NULL; 824 825 while (nm_i != head) { 826 struct netmap_slot *slot = &ring->slot[nm_i]; 827 u_int len = slot->len; 828 void *addr = NMB(na, slot); 829 /* device-specific */ 830 struct mbuf *m; 831 int tx_ret; 832 833 NM_CHECK_ADDR_LEN(na, addr, len); 834 835 /* Tale a mbuf from the tx pool (replenishing the pool 836 * entry if necessary) and copy in the user packet. */ 837 m = kring->tx_pool[nm_i]; 838 if (unlikely(m == NULL)) { 839 kring->tx_pool[nm_i] = m = 840 nm_os_get_mbuf(ifp, NETMAP_BUF_SIZE(na)); 841 if (m == NULL) { 842 RD(2, "Failed to replenish mbuf"); 843 /* Here we could schedule a timer which 844 * retries to replenish after a while, 845 * and notifies the client when it 846 * manages to replenish some slots. In 847 * any case we break early to avoid 848 * crashes. */ 849 break; 850 } 851 IFRATE(rate_ctx.new.txrepl++); 852 } 853 854 a.m = m; 855 a.addr = addr; 856 a.len = len; 857 a.qevent = (nm_i == event); 858 /* When not in txqdisc mode, we should ask 859 * notifications when NS_REPORT is set, or roughly 860 * every half ring. To optimize this, we set a 861 * notification event when the client runs out of 862 * TX ring space, or when transmission fails. In 863 * the latter case we also break early. 864 */ 865 tx_ret = nm_os_generic_xmit_frame(&a); 866 if (unlikely(tx_ret)) { 867 if (!gna->txqdisc) { 868 /* 869 * No room for this mbuf in the device driver. 870 * Request a notification FOR A PREVIOUS MBUF, 871 * then call generic_netmap_tx_clean(kring) to do the 872 * double check and see if we can free more buffers. 873 * If there is space continue, else break; 874 * NOTE: the double check is necessary if the problem 875 * occurs in the txsync call after selrecord(). 876 * Also, we need some way to tell the caller that not 877 * all buffers were queued onto the device (this was 878 * not a problem with native netmap driver where space 879 * is preallocated). The bridge has a similar problem 880 * and we solve it there by dropping the excess packets. 881 */ 882 generic_set_tx_event(kring, nm_i); 883 if (generic_netmap_tx_clean(kring, gna->txqdisc)) { 884 /* space now available */ 885 continue; 886 } else { 887 break; 888 } 889 } 890 891 /* In txqdisc mode, the netmap-aware qdisc 892 * queue has the same length as the number of 893 * netmap slots (N). Since tail is advanced 894 * only when packets are dequeued, qdisc 895 * queue overrun cannot happen, so 896 * nm_os_generic_xmit_frame() did not fail 897 * because of that. 898 * However, packets can be dropped because 899 * carrier is off, or because our qdisc is 900 * being deactivated, or possibly for other 901 * reasons. In these cases, we just let the 902 * packet to be dropped. */ 903 IFRATE(rate_ctx.new.txdrop++); 904 } 905 906 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 907 nm_i = nm_next(nm_i, lim); 908 IFRATE(rate_ctx.new.txpkt++); 909 } 910 if (a.head != NULL) { 911 a.addr = NULL; 912 nm_os_generic_xmit_frame(&a); 913 } 914 /* Update hwcur to the next slot to transmit. Here nm_i 915 * is not necessarily head, we could break early. */ 916 kring->nr_hwcur = nm_i; 917 } 918 919 /* 920 * Second, reclaim completed buffers 921 */ 922 if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) { 923 /* No more available slots? Set a notification event 924 * on a netmap slot that will be cleaned in the future. 925 * No doublecheck is performed, since txsync() will be 926 * called twice by netmap_poll(). 927 */ 928 generic_set_tx_event(kring, nm_i); 929 } 930 931 generic_netmap_tx_clean(kring, gna->txqdisc); 932 933 return 0; 934 } 935 936 937 /* 938 * This handler is registered (through nm_os_catch_rx()) 939 * within the attached network interface 940 * in the RX subsystem, so that every mbuf passed up by 941 * the driver can be stolen to the network stack. 942 * Stolen packets are put in a queue where the 943 * generic_netmap_rxsync() callback can extract them. 944 * Returns 1 if the packet was stolen, 0 otherwise. 945 */ 946 int 947 generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 948 { 949 struct netmap_adapter *na = NA(ifp); 950 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 951 struct netmap_kring *kring; 952 u_int work_done; 953 u_int r = MBUF_RXQ(m); /* receive ring number */ 954 955 if (r >= na->num_rx_rings) { 956 r = r % na->num_rx_rings; 957 } 958 959 kring = &na->rx_rings[r]; 960 961 if (kring->nr_mode == NKR_NETMAP_OFF) { 962 /* We must not intercept this mbuf. */ 963 return 0; 964 } 965 966 /* limit the size of the queue */ 967 if (unlikely(!gna->rxsg && MBUF_LEN(m) > NETMAP_BUF_SIZE(na))) { 968 /* This may happen when GRO/LRO features are enabled for 969 * the NIC driver when the generic adapter does not 970 * support RX scatter-gather. */ 971 RD(2, "Warning: driver pushed up big packet " 972 "(size=%d)", (int)MBUF_LEN(m)); 973 m_freem(m); 974 } else if (unlikely(mbq_len(&kring->rx_queue) > 1024)) { 975 m_freem(m); 976 } else { 977 mbq_safe_enqueue(&kring->rx_queue, m); 978 } 979 980 if (netmap_generic_mit < 32768) { 981 /* no rx mitigation, pass notification up */ 982 netmap_generic_irq(na, r, &work_done); 983 } else { 984 /* same as send combining, filter notification if there is a 985 * pending timer, otherwise pass it up and start a timer. 986 */ 987 if (likely(nm_os_mitigation_active(&gna->mit[r]))) { 988 /* Record that there is some pending work. */ 989 gna->mit[r].mit_pending = 1; 990 } else { 991 netmap_generic_irq(na, r, &work_done); 992 nm_os_mitigation_start(&gna->mit[r]); 993 } 994 } 995 996 /* We have intercepted the mbuf. */ 997 return 1; 998 } 999 1000 /* 1001 * generic_netmap_rxsync() extracts mbufs from the queue filled by 1002 * generic_netmap_rx_handler() and puts their content in the netmap 1003 * receive ring. 1004 * Access must be protected because the rx handler is asynchronous, 1005 */ 1006 static int 1007 generic_netmap_rxsync(struct netmap_kring *kring, int flags) 1008 { 1009 struct netmap_ring *ring = kring->ring; 1010 struct netmap_adapter *na = kring->na; 1011 u_int nm_i; /* index into the netmap ring */ //j, 1012 u_int n; 1013 u_int const lim = kring->nkr_num_slots - 1; 1014 u_int const head = kring->rhead; 1015 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 1016 1017 /* Adapter-specific variables. */ 1018 uint16_t slot_flags = kring->nkr_slot_flags; 1019 u_int nm_buf_len = NETMAP_BUF_SIZE(na); 1020 struct mbq tmpq; 1021 struct mbuf *m; 1022 int avail; /* in bytes */ 1023 int mlen; 1024 int copy; 1025 1026 if (head > lim) 1027 return netmap_ring_reinit(kring); 1028 1029 IFRATE(rate_ctx.new.rxsync++); 1030 1031 /* 1032 * First part: skip past packets that userspace has released. 1033 * This can possibly make room for the second part. 1034 */ 1035 nm_i = kring->nr_hwcur; 1036 if (nm_i != head) { 1037 /* Userspace has released some packets. */ 1038 for (n = 0; nm_i != head; n++) { 1039 struct netmap_slot *slot = &ring->slot[nm_i]; 1040 1041 slot->flags &= ~NS_BUF_CHANGED; 1042 nm_i = nm_next(nm_i, lim); 1043 } 1044 kring->nr_hwcur = head; 1045 } 1046 1047 /* 1048 * Second part: import newly received packets. 1049 */ 1050 if (!netmap_no_pendintr && !force_update) { 1051 return 0; 1052 } 1053 1054 nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */ 1055 1056 /* Compute the available space (in bytes) in this netmap ring. 1057 * The first slot that is not considered in is the one before 1058 * nr_hwcur. */ 1059 1060 avail = nm_prev(kring->nr_hwcur, lim) - nm_i; 1061 if (avail < 0) 1062 avail += lim + 1; 1063 avail *= nm_buf_len; 1064 1065 /* First pass: While holding the lock on the RX mbuf queue, 1066 * extract as many mbufs as they fit the available space, 1067 * and put them in a temporary queue. 1068 * To avoid performing a per-mbuf division (mlen / nm_buf_len) to 1069 * to update avail, we do the update in a while loop that we 1070 * also use to set the RX slots, but without performing the copy. */ 1071 mbq_init(&tmpq); 1072 mbq_lock(&kring->rx_queue); 1073 for (n = 0;; n++) { 1074 m = mbq_peek(&kring->rx_queue); 1075 if (!m) { 1076 /* No more packets from the driver. */ 1077 break; 1078 } 1079 1080 mlen = MBUF_LEN(m); 1081 if (mlen > avail) { 1082 /* No more space in the ring. */ 1083 break; 1084 } 1085 1086 mbq_dequeue(&kring->rx_queue); 1087 1088 while (mlen) { 1089 copy = nm_buf_len; 1090 if (mlen < copy) { 1091 copy = mlen; 1092 } 1093 mlen -= copy; 1094 avail -= nm_buf_len; 1095 1096 ring->slot[nm_i].len = copy; 1097 ring->slot[nm_i].flags = slot_flags | (mlen ? NS_MOREFRAG : 0); 1098 nm_i = nm_next(nm_i, lim); 1099 } 1100 1101 mbq_enqueue(&tmpq, m); 1102 } 1103 mbq_unlock(&kring->rx_queue); 1104 1105 /* Second pass: Drain the temporary queue, going over the used RX slots, 1106 * and perform the copy out of the RX queue lock. */ 1107 nm_i = kring->nr_hwtail; 1108 1109 for (;;) { 1110 void *nmaddr; 1111 int ofs = 0; 1112 int morefrag; 1113 1114 m = mbq_dequeue(&tmpq); 1115 if (!m) { 1116 break; 1117 } 1118 1119 do { 1120 nmaddr = NMB(na, &ring->slot[nm_i]); 1121 /* We only check the address here on generic rx rings. */ 1122 if (nmaddr == NETMAP_BUF_BASE(na)) { /* Bad buffer */ 1123 m_freem(m); 1124 mbq_purge(&tmpq); 1125 mbq_fini(&tmpq); 1126 return netmap_ring_reinit(kring); 1127 } 1128 1129 copy = ring->slot[nm_i].len; 1130 m_copydata(m, ofs, copy, nmaddr); 1131 ofs += copy; 1132 morefrag = ring->slot[nm_i].flags & NS_MOREFRAG; 1133 nm_i = nm_next(nm_i, lim); 1134 } while (morefrag); 1135 1136 m_freem(m); 1137 } 1138 1139 mbq_fini(&tmpq); 1140 1141 if (n) { 1142 kring->nr_hwtail = nm_i; 1143 IFRATE(rate_ctx.new.rxpkt += n); 1144 } 1145 kring->nr_kflags &= ~NKR_PENDINTR; 1146 1147 return 0; 1148 } 1149 1150 static void 1151 generic_netmap_dtor(struct netmap_adapter *na) 1152 { 1153 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; 1154 struct ifnet *ifp = netmap_generic_getifp(gna); 1155 struct netmap_adapter *prev_na = gna->prev; 1156 1157 if (prev_na != NULL) { 1158 D("Released generic NA %p", gna); 1159 netmap_adapter_put(prev_na); 1160 if (nm_iszombie(na)) { 1161 /* 1162 * The driver has been removed without releasing 1163 * the reference so we need to do it here. 1164 */ 1165 netmap_adapter_put(prev_na); 1166 } 1167 } 1168 NM_ATTACH_NA(ifp, prev_na); 1169 /* 1170 * netmap_detach_common(), that it's called after this function, 1171 * overrides WNA(ifp) if na->ifp is not NULL. 1172 */ 1173 na->ifp = NULL; 1174 D("Restored native NA %p", prev_na); 1175 } 1176 1177 /* 1178 * generic_netmap_attach() makes it possible to use netmap on 1179 * a device without native netmap support. 1180 * This is less performant than native support but potentially 1181 * faster than raw sockets or similar schemes. 1182 * 1183 * In this "emulated" mode, netmap rings do not necessarily 1184 * have the same size as those in the NIC. We use a default 1185 * value and possibly override it if the OS has ways to fetch the 1186 * actual configuration. 1187 */ 1188 int 1189 generic_netmap_attach(struct ifnet *ifp) 1190 { 1191 struct netmap_adapter *na; 1192 struct netmap_generic_adapter *gna; 1193 int retval; 1194 u_int num_tx_desc, num_rx_desc; 1195 1196 num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ 1197 1198 nm_os_generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */ 1199 ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc); 1200 if (num_tx_desc == 0 || num_rx_desc == 0) { 1201 D("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc); 1202 return EINVAL; 1203 } 1204 1205 gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO); 1206 if (gna == NULL) { 1207 D("no memory on attach, give up"); 1208 return ENOMEM; 1209 } 1210 na = (struct netmap_adapter *)gna; 1211 strncpy(na->name, ifp->if_xname, sizeof(na->name)); 1212 na->ifp = ifp; 1213 na->num_tx_desc = num_tx_desc; 1214 na->num_rx_desc = num_rx_desc; 1215 na->nm_register = &generic_netmap_register; 1216 na->nm_txsync = &generic_netmap_txsync; 1217 na->nm_rxsync = &generic_netmap_rxsync; 1218 na->nm_dtor = &generic_netmap_dtor; 1219 /* when using generic, NAF_NETMAP_ON is set so we force 1220 * NAF_SKIP_INTR to use the regular interrupt handler 1221 */ 1222 na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS; 1223 1224 ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", 1225 ifp->num_tx_queues, ifp->real_num_tx_queues, 1226 ifp->tx_queue_len); 1227 ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", 1228 ifp->num_rx_queues, ifp->real_num_rx_queues); 1229 1230 nm_os_generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); 1231 1232 retval = netmap_attach_common(na); 1233 if (retval) { 1234 free(gna, M_DEVBUF); 1235 return retval; 1236 } 1237 1238 gna->prev = NA(ifp); /* save old na */ 1239 if (gna->prev != NULL) { 1240 netmap_adapter_get(gna->prev); 1241 } 1242 NM_ATTACH_NA(ifp, na); 1243 1244 nm_os_generic_set_features(gna); 1245 1246 D("Created generic NA %p (prev %p)", gna, gna->prev); 1247 1248 return retval; 1249 } 1250