1 /* 2 * Copyright (C) 2013 Universita` di Pisa. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * This module implements netmap support on top of standard, 28 * unmodified device drivers. 29 * 30 * A NIOCREGIF request is handled here if the device does not 31 * have native support. TX and RX rings are emulated as follows: 32 * 33 * NIOCREGIF 34 * We preallocate a block of TX mbufs (roughly as many as 35 * tx descriptors; the number is not critical) to speed up 36 * operation during transmissions. The refcount on most of 37 * these buffers is artificially bumped up so we can recycle 38 * them more easily. Also, the destructor is intercepted 39 * so we use it as an interrupt notification to wake up 40 * processes blocked on a poll(). 41 * 42 * For each receive ring we allocate one "struct mbq" 43 * (an mbuf tailq plus a spinlock). We intercept packets 44 * (through if_input) 45 * on the receive path and put them in the mbq from which 46 * netmap receive routines can grab them. 47 * 48 * TX: 49 * in the generic_txsync() routine, netmap buffers are copied 50 * (or linked, in a future) to the preallocated mbufs 51 * and pushed to the transmit queue. Some of these mbufs 52 * (those with NS_REPORT, or otherwise every half ring) 53 * have the refcount=1, others have refcount=2. 54 * When the destructor is invoked, we take that as 55 * a notification that all mbufs up to that one in 56 * the specific ring have been completed, and generate 57 * the equivalent of a transmit interrupt. 58 * 59 * RX: 60 * 61 */ 62 63 #ifdef __FreeBSD__ 64 65 #include <sys/cdefs.h> /* prerequisite */ 66 __FBSDID("$FreeBSD$"); 67 68 #include <sys/types.h> 69 #include <sys/errno.h> 70 #include <sys/malloc.h> 71 #include <sys/lock.h> /* PROT_EXEC */ 72 #include <sys/rwlock.h> 73 #include <sys/socket.h> /* sockaddrs */ 74 #include <sys/selinfo.h> 75 #include <net/if.h> 76 #include <net/if_var.h> 77 #include <machine/bus.h> /* bus_dmamap_* in netmap_kern.h */ 78 79 // XXX temporary - D() defined here 80 #include <net/netmap.h> 81 #include <dev/netmap/netmap_kern.h> 82 #include <dev/netmap/netmap_mem2.h> 83 84 #define rtnl_lock() D("rtnl_lock called"); 85 #define rtnl_unlock() D("rtnl_lock called"); 86 #define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) 87 #define smp_mb() 88 89 /* 90 * mbuf wrappers 91 */ 92 93 /* 94 * we allocate an EXT_PACKET 95 */ 96 #define netmap_get_mbuf(len) m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR|M_NOFREE) 97 98 /* mbuf destructor, also need to change the type to EXT_EXTREF, 99 * add an M_NOFREE flag, and then clear the flag and 100 * chain into uma_zfree(zone_pack, mf) 101 * (or reinstall the buffer ?) 102 */ 103 #define SET_MBUF_DESTRUCTOR(m, fn) do { \ 104 (m)->m_ext.ext_free = (void *)fn; \ 105 (m)->m_ext.ext_type = EXT_EXTREF; \ 106 } while (0) 107 108 109 #define GET_MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *(m)->m_ext.ref_cnt : -1) 110 111 112 113 #else /* linux */ 114 115 #include "bsd_glue.h" 116 117 #include <linux/rtnetlink.h> /* rtnl_[un]lock() */ 118 #include <linux/ethtool.h> /* struct ethtool_ops, get_ringparam */ 119 #include <linux/hrtimer.h> 120 121 //#define RATE /* Enables communication statistics. */ 122 123 //#define REG_RESET 124 125 #endif /* linux */ 126 127 128 /* Common headers. */ 129 #include <net/netmap.h> 130 #include <dev/netmap/netmap_kern.h> 131 #include <dev/netmap/netmap_mem2.h> 132 133 134 135 /* ======================== usage stats =========================== */ 136 137 #ifdef RATE 138 #define IFRATE(x) x 139 struct rate_stats { 140 unsigned long txpkt; 141 unsigned long txsync; 142 unsigned long txirq; 143 unsigned long rxpkt; 144 unsigned long rxirq; 145 unsigned long rxsync; 146 }; 147 148 struct rate_context { 149 unsigned refcount; 150 struct timer_list timer; 151 struct rate_stats new; 152 struct rate_stats old; 153 }; 154 155 #define RATE_PRINTK(_NAME_) \ 156 printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); 157 #define RATE_PERIOD 2 158 static void rate_callback(unsigned long arg) 159 { 160 struct rate_context * ctx = (struct rate_context *)arg; 161 struct rate_stats cur = ctx->new; 162 int r; 163 164 RATE_PRINTK(txpkt); 165 RATE_PRINTK(txsync); 166 RATE_PRINTK(txirq); 167 RATE_PRINTK(rxpkt); 168 RATE_PRINTK(rxsync); 169 RATE_PRINTK(rxirq); 170 printk("\n"); 171 172 ctx->old = cur; 173 r = mod_timer(&ctx->timer, jiffies + 174 msecs_to_jiffies(RATE_PERIOD * 1000)); 175 if (unlikely(r)) 176 D("[v1000] Error: mod_timer()"); 177 } 178 179 static struct rate_context rate_ctx; 180 181 #else /* !RATE */ 182 #define IFRATE(x) 183 #endif /* !RATE */ 184 185 186 /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */ 187 #define GENERIC_BUF_SIZE netmap_buf_size /* Size of the mbufs in the Tx pool. */ 188 189 /* 190 * Wrapper used by the generic adapter layer to notify 191 * the poller threads. Differently from netmap_rx_irq(), we check 192 * only IFCAP_NETMAP instead of NAF_NATIVE_ON to enable the irq. 193 */ 194 static void 195 netmap_generic_irq(struct ifnet *ifp, u_int q, u_int *work_done) 196 { 197 if (unlikely(!(ifp->if_capenable & IFCAP_NETMAP))) 198 return; 199 200 netmap_common_irq(ifp, q, work_done); 201 } 202 203 204 /* Enable/disable netmap mode for a generic network interface. */ 205 int generic_netmap_register(struct netmap_adapter *na, int enable) 206 { 207 struct ifnet *ifp = na->ifp; 208 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 209 struct mbuf *m; 210 int error; 211 int i, r; 212 213 if (!na) 214 return EINVAL; 215 216 #ifdef REG_RESET 217 error = ifp->netdev_ops->ndo_stop(ifp); 218 if (error) { 219 return error; 220 } 221 #endif /* REG_RESET */ 222 223 if (enable) { /* Enable netmap mode. */ 224 /* Initialize the rx queue, as generic_rx_handler() can 225 * be called as soon as netmap_catch_rx() returns. 226 */ 227 for (r=0; r<na->num_rx_rings; r++) { 228 mbq_safe_init(&na->rx_rings[r].rx_queue); 229 na->rx_rings[r].nr_ntc = 0; 230 } 231 232 /* Init the mitigation timer. */ 233 netmap_mitigation_init(gna); 234 235 /* 236 * Preallocate packet buffers for the tx rings. 237 */ 238 for (r=0; r<na->num_tx_rings; r++) { 239 na->tx_rings[r].nr_ntc = 0; 240 na->tx_rings[r].tx_pool = malloc(na->num_tx_desc * sizeof(struct mbuf *), 241 M_DEVBUF, M_NOWAIT | M_ZERO); 242 if (!na->tx_rings[r].tx_pool) { 243 D("tx_pool allocation failed"); 244 error = ENOMEM; 245 goto free_tx_pool; 246 } 247 for (i=0; i<na->num_tx_desc; i++) { 248 m = netmap_get_mbuf(GENERIC_BUF_SIZE); 249 if (!m) { 250 D("tx_pool[%d] allocation failed", i); 251 error = ENOMEM; 252 goto free_mbufs; 253 } 254 na->tx_rings[r].tx_pool[i] = m; 255 } 256 } 257 rtnl_lock(); 258 /* Prepare to intercept incoming traffic. */ 259 error = netmap_catch_rx(na, 1); 260 if (error) { 261 D("netdev_rx_handler_register() failed"); 262 goto register_handler; 263 } 264 ifp->if_capenable |= IFCAP_NETMAP; 265 266 /* Make netmap control the packet steering. */ 267 netmap_catch_packet_steering(gna, 1); 268 269 rtnl_unlock(); 270 271 #ifdef RATE 272 if (rate_ctx.refcount == 0) { 273 D("setup_timer()"); 274 memset(&rate_ctx, 0, sizeof(rate_ctx)); 275 setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); 276 if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { 277 D("Error: mod_timer()"); 278 } 279 } 280 rate_ctx.refcount++; 281 #endif /* RATE */ 282 283 } else { /* Disable netmap mode. */ 284 rtnl_lock(); 285 286 ifp->if_capenable &= ~IFCAP_NETMAP; 287 288 /* Release packet steering control. */ 289 netmap_catch_packet_steering(gna, 0); 290 291 /* Do not intercept packets on the rx path. */ 292 netmap_catch_rx(na, 0); 293 294 rtnl_unlock(); 295 296 /* Free the mbufs going to the netmap rings */ 297 for (r=0; r<na->num_rx_rings; r++) { 298 mbq_safe_purge(&na->rx_rings[r].rx_queue); 299 mbq_safe_destroy(&na->rx_rings[r].rx_queue); 300 } 301 302 netmap_mitigation_cleanup(gna); 303 304 for (r=0; r<na->num_tx_rings; r++) { 305 for (i=0; i<na->num_tx_desc; i++) { 306 m_freem(na->tx_rings[r].tx_pool[i]); 307 } 308 free(na->tx_rings[r].tx_pool, M_DEVBUF); 309 } 310 311 #ifdef RATE 312 if (--rate_ctx.refcount == 0) { 313 D("del_timer()"); 314 del_timer(&rate_ctx.timer); 315 } 316 #endif 317 } 318 319 #ifdef REG_RESET 320 error = ifp->netdev_ops->ndo_open(ifp); 321 if (error) { 322 goto alloc_tx_pool; 323 } 324 #endif 325 326 return 0; 327 328 register_handler: 329 rtnl_unlock(); 330 free_tx_pool: 331 r--; 332 i = na->num_tx_desc; /* Useless, but just to stay safe. */ 333 free_mbufs: 334 i--; 335 for (; r>=0; r--) { 336 for (; i>=0; i--) { 337 m_freem(na->tx_rings[r].tx_pool[i]); 338 } 339 free(na->tx_rings[r].tx_pool, M_DEVBUF); 340 i = na->num_tx_desc - 1; 341 } 342 343 return error; 344 } 345 346 /* 347 * Callback invoked when the device driver frees an mbuf used 348 * by netmap to transmit a packet. This usually happens when 349 * the NIC notifies the driver that transmission is completed. 350 */ 351 static void 352 generic_mbuf_destructor(struct mbuf *m) 353 { 354 if (netmap_verbose) 355 D("Tx irq (%p) queue %d", m, MBUF_TXQ(m)); 356 netmap_generic_irq(MBUF_IFP(m), MBUF_TXQ(m), NULL); 357 #ifdef __FreeBSD__ 358 m->m_ext.ext_type = EXT_PACKET; 359 m->m_ext.ext_free = NULL; 360 if (*(m->m_ext.ref_cnt) == 0) 361 *(m->m_ext.ref_cnt) = 1; 362 uma_zfree(zone_pack, m); 363 #endif /* __FreeBSD__ */ 364 IFRATE(rate_ctx.new.txirq++); 365 } 366 367 /* Record completed transmissions and update hwavail. 368 * 369 * nr_ntc is the oldest tx buffer not yet completed 370 * (same as nr_hwavail + nr_hwcur + 1), 371 * nr_hwcur is the first unsent buffer. 372 * When cleaning, we try to recover buffers between nr_ntc and nr_hwcur. 373 */ 374 static int 375 generic_netmap_tx_clean(struct netmap_kring *kring) 376 { 377 u_int num_slots = kring->nkr_num_slots; 378 u_int ntc = kring->nr_ntc; 379 u_int hwcur = kring->nr_hwcur; 380 u_int n = 0; 381 struct mbuf **tx_pool = kring->tx_pool; 382 383 while (ntc != hwcur) { /* buffers not completed */ 384 struct mbuf *m = tx_pool[ntc]; 385 386 if (unlikely(m == NULL)) { 387 /* try to replenish the entry */ 388 tx_pool[ntc] = m = netmap_get_mbuf(GENERIC_BUF_SIZE); 389 if (unlikely(m == NULL)) { 390 D("mbuf allocation failed, XXX error"); 391 // XXX how do we proceed ? break ? 392 return -ENOMEM; 393 } 394 } else if (GET_MBUF_REFCNT(m) != 1) { 395 break; /* This mbuf is still busy: its refcnt is 2. */ 396 } 397 if (unlikely(++ntc == num_slots)) { 398 ntc = 0; 399 } 400 n++; 401 } 402 kring->nr_ntc = ntc; 403 kring->nr_hwavail += n; 404 ND("tx completed [%d] -> hwavail %d", n, kring->nr_hwavail); 405 406 return n; 407 } 408 409 410 /* 411 * We have pending packets in the driver between nr_ntc and j. 412 * Compute a position in the middle, to be used to generate 413 * a notification. 414 */ 415 static inline u_int 416 generic_tx_event_middle(struct netmap_kring *kring, u_int hwcur) 417 { 418 u_int n = kring->nkr_num_slots; 419 u_int ntc = kring->nr_ntc; 420 u_int e; 421 422 if (hwcur >= ntc) { 423 e = (hwcur + ntc) / 2; 424 } else { /* wrap around */ 425 e = (hwcur + n + ntc) / 2; 426 if (e >= n) { 427 e -= n; 428 } 429 } 430 431 if (unlikely(e >= n)) { 432 D("This cannot happen"); 433 e = 0; 434 } 435 436 return e; 437 } 438 439 /* 440 * We have pending packets in the driver between nr_ntc and hwcur. 441 * Schedule a notification approximately in the middle of the two. 442 * There is a race but this is only called within txsync which does 443 * a double check. 444 */ 445 static void 446 generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) 447 { 448 struct mbuf *m; 449 u_int e; 450 451 if (kring->nr_ntc == hwcur) { 452 return; 453 } 454 e = generic_tx_event_middle(kring, hwcur); 455 456 m = kring->tx_pool[e]; 457 if (m == NULL) { 458 /* This can happen if there is already an event on the netmap 459 slot 'e': There is nothing to do. */ 460 return; 461 } 462 ND("Event at %d mbuf %p refcnt %d", e, m, GET_MBUF_REFCNT(m)); 463 kring->tx_pool[e] = NULL; 464 SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor); 465 466 // XXX wmb() ? 467 /* Decrement the refcount an free it if we have the last one. */ 468 m_freem(m); 469 smp_mb(); 470 } 471 472 473 /* 474 * generic_netmap_txsync() transforms netmap buffers into mbufs 475 * and passes them to the standard device driver 476 * (ndo_start_xmit() or ifp->if_transmit() ). 477 * On linux this is not done directly, but using dev_queue_xmit(), 478 * since it implements the TX flow control (and takes some locks). 479 */ 480 static int 481 generic_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) 482 { 483 struct ifnet *ifp = na->ifp; 484 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 485 struct netmap_ring *ring = kring->ring; 486 u_int j, k, num_slots = kring->nkr_num_slots; 487 int new_slots, ntx; 488 489 IFRATE(rate_ctx.new.txsync++); 490 491 // TODO: handle the case of mbuf allocation failure 492 /* first, reclaim completed buffers */ 493 generic_netmap_tx_clean(kring); 494 495 /* Take a copy of ring->cur now, and never read it again. */ 496 k = ring->cur; 497 if (unlikely(k >= num_slots)) { 498 return netmap_ring_reinit(kring); 499 } 500 501 rmb(); 502 j = kring->nr_hwcur; 503 /* 504 * 'new_slots' counts how many new slots have been added: 505 * everything from hwcur to cur, excluding reserved ones, if any. 506 * nr_hwreserved start from hwcur and counts how many slots were 507 * not sent to the NIC from the previous round. 508 */ 509 new_slots = k - j - kring->nr_hwreserved; 510 if (new_slots < 0) { 511 new_slots += num_slots; 512 } 513 ntx = 0; 514 if (j != k) { 515 /* Process new packets to send: 516 * j is the current index in the netmap ring. 517 */ 518 while (j != k) { 519 struct netmap_slot *slot = &ring->slot[j]; /* Current slot in the netmap ring */ 520 void *addr = NMB(slot); 521 u_int len = slot->len; 522 struct mbuf *m; 523 int tx_ret; 524 525 if (unlikely(addr == netmap_buffer_base || len > NETMAP_BUF_SIZE)) { 526 return netmap_ring_reinit(kring); 527 } 528 /* Tale a mbuf from the tx pool and copy in the user packet. */ 529 m = kring->tx_pool[j]; 530 if (unlikely(!m)) { 531 RD(5, "This should never happen"); 532 kring->tx_pool[j] = m = netmap_get_mbuf(GENERIC_BUF_SIZE); 533 if (unlikely(m == NULL)) { 534 D("mbuf allocation failed"); 535 break; 536 } 537 } 538 /* XXX we should ask notifications when NS_REPORT is set, 539 * or roughly every half frame. We can optimize this 540 * by lazily requesting notifications only when a 541 * transmission fails. Probably the best way is to 542 * break on failures and set notifications when 543 * ring->avail == 0 || j != k 544 */ 545 tx_ret = generic_xmit_frame(ifp, m, addr, len, ring_nr); 546 if (unlikely(tx_ret)) { 547 RD(5, "start_xmit failed: err %d [%u,%u,%u,%u]", 548 tx_ret, kring->nr_ntc, j, k, kring->nr_hwavail); 549 /* 550 * No room for this mbuf in the device driver. 551 * Request a notification FOR A PREVIOUS MBUF, 552 * then call generic_netmap_tx_clean(kring) to do the 553 * double check and see if we can free more buffers. 554 * If there is space continue, else break; 555 * NOTE: the double check is necessary if the problem 556 * occurs in the txsync call after selrecord(). 557 * Also, we need some way to tell the caller that not 558 * all buffers were queued onto the device (this was 559 * not a problem with native netmap driver where space 560 * is preallocated). The bridge has a similar problem 561 * and we solve it there by dropping the excess packets. 562 */ 563 generic_set_tx_event(kring, j); 564 if (generic_netmap_tx_clean(kring)) { /* space now available */ 565 continue; 566 } else { 567 break; 568 } 569 } 570 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 571 if (unlikely(++j == num_slots)) 572 j = 0; 573 ntx++; 574 } 575 576 /* Update hwcur to the next slot to transmit. */ 577 kring->nr_hwcur = j; 578 579 /* 580 * Report all new slots as unavailable, even those not sent. 581 * We account for them with with hwreserved, so that 582 * nr_hwreserved =:= cur - nr_hwcur 583 */ 584 kring->nr_hwavail -= new_slots; 585 kring->nr_hwreserved = k - j; 586 if (kring->nr_hwreserved < 0) { 587 kring->nr_hwreserved += num_slots; 588 } 589 590 IFRATE(rate_ctx.new.txpkt += ntx); 591 592 if (!kring->nr_hwavail) { 593 /* No more available slots? Set a notification event 594 * on a netmap slot that will be cleaned in the future. 595 * No doublecheck is performed, since txsync() will be 596 * called twice by netmap_poll(). 597 */ 598 generic_set_tx_event(kring, j); 599 } 600 ND("tx #%d, hwavail = %d", n, kring->nr_hwavail); 601 } 602 603 /* Synchronize the user's view to the kernel view. */ 604 ring->avail = kring->nr_hwavail; 605 ring->reserved = kring->nr_hwreserved; 606 607 return 0; 608 } 609 610 /* 611 * This handler is registered (through netmap_catch_rx()) 612 * within the attached network interface 613 * in the RX subsystem, so that every mbuf passed up by 614 * the driver can be stolen to the network stack. 615 * Stolen packets are put in a queue where the 616 * generic_netmap_rxsync() callback can extract them. 617 */ 618 void generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 619 { 620 struct netmap_adapter *na = NA(ifp); 621 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 622 u_int work_done; 623 u_int rr = 0; // receive ring number 624 625 ND("called"); 626 /* limit the size of the queue */ 627 if (unlikely(mbq_len(&na->rx_rings[rr].rx_queue) > 1024)) { 628 m_freem(m); 629 } else { 630 mbq_safe_enqueue(&na->rx_rings[rr].rx_queue, m); 631 } 632 633 if (netmap_generic_mit < 32768) { 634 /* no rx mitigation, pass notification up */ 635 netmap_generic_irq(na->ifp, rr, &work_done); 636 IFRATE(rate_ctx.new.rxirq++); 637 } else { 638 /* same as send combining, filter notification if there is a 639 * pending timer, otherwise pass it up and start a timer. 640 */ 641 if (likely(netmap_mitigation_active(gna))) { 642 /* Record that there is some pending work. */ 643 gna->mit_pending = 1; 644 } else { 645 netmap_generic_irq(na->ifp, rr, &work_done); 646 IFRATE(rate_ctx.new.rxirq++); 647 netmap_mitigation_start(gna); 648 } 649 } 650 } 651 652 /* 653 * generic_netmap_rxsync() extracts mbufs from the queue filled by 654 * generic_netmap_rx_handler() and puts their content in the netmap 655 * receive ring. 656 * Access must be protected because the rx handler is asynchronous, 657 */ 658 static int 659 generic_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) 660 { 661 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 662 struct netmap_ring *ring = kring->ring; 663 u_int j, n, lim = kring->nkr_num_slots - 1; 664 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 665 u_int k, resvd = ring->reserved; 666 667 if (ring->cur > lim) 668 return netmap_ring_reinit(kring); 669 670 /* Import newly received packets into the netmap ring. */ 671 if (netmap_no_pendintr || force_update) { 672 uint16_t slot_flags = kring->nkr_slot_flags; 673 struct mbuf *m; 674 675 n = 0; 676 j = kring->nr_ntc; /* first empty slot in the receive ring */ 677 /* extract buffers from the rx queue, stop at most one 678 * slot before nr_hwcur (index k) 679 */ 680 k = (kring->nr_hwcur) ? kring->nr_hwcur-1 : lim; 681 while (j != k) { 682 int len; 683 void *addr = NMB(&ring->slot[j]); 684 685 if (addr == netmap_buffer_base) { /* Bad buffer */ 686 return netmap_ring_reinit(kring); 687 } 688 /* 689 * Call the locked version of the function. 690 * XXX Ideally we could grab a batch of mbufs at once, 691 * by changing rx_queue into a ring. 692 */ 693 m = mbq_safe_dequeue(&kring->rx_queue); 694 if (!m) 695 break; 696 len = MBUF_LEN(m); 697 m_copydata(m, 0, len, addr); 698 ring->slot[j].len = len; 699 ring->slot[j].flags = slot_flags; 700 m_freem(m); 701 if (unlikely(j++ == lim)) 702 j = 0; 703 n++; 704 } 705 if (n) { 706 kring->nr_ntc = j; 707 kring->nr_hwavail += n; 708 IFRATE(rate_ctx.new.rxpkt += n); 709 } 710 kring->nr_kflags &= ~NKR_PENDINTR; 711 } 712 713 // XXX should we invert the order ? 714 /* Skip past packets that userspace has released */ 715 j = kring->nr_hwcur; 716 k = ring->cur; 717 if (resvd > 0) { 718 if (resvd + ring->avail >= lim + 1) { 719 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 720 ring->reserved = resvd = 0; // XXX panic... 721 } 722 k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; 723 } 724 if (j != k) { 725 /* Userspace has released some packets. */ 726 for (n = 0; j != k; n++) { 727 struct netmap_slot *slot = &ring->slot[j]; 728 729 slot->flags &= ~NS_BUF_CHANGED; 730 if (unlikely(j++ == lim)) 731 j = 0; 732 } 733 kring->nr_hwavail -= n; 734 kring->nr_hwcur = k; 735 } 736 /* Tell userspace that there are new packets. */ 737 ring->avail = kring->nr_hwavail - resvd; 738 IFRATE(rate_ctx.new.rxsync++); 739 740 return 0; 741 } 742 743 static void 744 generic_netmap_dtor(struct netmap_adapter *na) 745 { 746 struct ifnet *ifp = na->ifp; 747 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; 748 struct netmap_adapter *prev_na = gna->prev; 749 750 if (prev_na != NULL) { 751 D("Released generic NA %p", gna); 752 if_rele(na->ifp); 753 netmap_adapter_put(prev_na); 754 } 755 if (ifp != NULL) { 756 WNA(ifp) = prev_na; 757 D("Restored native NA %p", prev_na); 758 na->ifp = NULL; 759 } 760 } 761 762 /* 763 * generic_netmap_attach() makes it possible to use netmap on 764 * a device without native netmap support. 765 * This is less performant than native support but potentially 766 * faster than raw sockets or similar schemes. 767 * 768 * In this "emulated" mode, netmap rings do not necessarily 769 * have the same size as those in the NIC. We use a default 770 * value and possibly override it if the OS has ways to fetch the 771 * actual configuration. 772 */ 773 int 774 generic_netmap_attach(struct ifnet *ifp) 775 { 776 struct netmap_adapter *na; 777 struct netmap_generic_adapter *gna; 778 int retval; 779 u_int num_tx_desc, num_rx_desc; 780 781 num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ 782 783 generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); 784 ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc); 785 786 gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO); 787 if (gna == NULL) { 788 D("no memory on attach, give up"); 789 return ENOMEM; 790 } 791 na = (struct netmap_adapter *)gna; 792 na->ifp = ifp; 793 na->num_tx_desc = num_tx_desc; 794 na->num_rx_desc = num_rx_desc; 795 na->nm_register = &generic_netmap_register; 796 na->nm_txsync = &generic_netmap_txsync; 797 na->nm_rxsync = &generic_netmap_rxsync; 798 na->nm_dtor = &generic_netmap_dtor; 799 /* when using generic, IFCAP_NETMAP is set so we force 800 * NAF_SKIP_INTR to use the regular interrupt handler 801 */ 802 na->na_flags = NAF_SKIP_INTR; 803 804 ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", 805 ifp->num_tx_queues, ifp->real_num_tx_queues, 806 ifp->tx_queue_len); 807 ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", 808 ifp->num_rx_queues, ifp->real_num_rx_queues); 809 810 generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); 811 812 retval = netmap_attach_common(na); 813 if (retval) { 814 free(gna, M_DEVBUF); 815 } 816 817 return retval; 818 } 819