1 /* 2 * Copyright (C) 2011-2012 Matteo Landi, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #define NM_BRIDGE 27 28 /* 29 * This module supports memory mapped access to network devices, 30 * see netmap(4). 31 * 32 * The module uses a large, memory pool allocated by the kernel 33 * and accessible as mmapped memory by multiple userspace threads/processes. 34 * The memory pool contains packet buffers and "netmap rings", 35 * i.e. user-accessible copies of the interface's queues. 36 * 37 * Access to the network card works like this: 38 * 1. a process/thread issues one or more open() on /dev/netmap, to create 39 * select()able file descriptor on which events are reported. 40 * 2. on each descriptor, the process issues an ioctl() to identify 41 * the interface that should report events to the file descriptor. 42 * 3. on each descriptor, the process issues an mmap() request to 43 * map the shared memory region within the process' address space. 44 * The list of interesting queues is indicated by a location in 45 * the shared memory region. 46 * 4. using the functions in the netmap(4) userspace API, a process 47 * can look up the occupation state of a queue, access memory buffers, 48 * and retrieve received packets or enqueue packets to transmit. 49 * 5. using some ioctl()s the process can synchronize the userspace view 50 * of the queue with the actual status in the kernel. This includes both 51 * receiving the notification of new packets, and transmitting new 52 * packets on the output interface. 53 * 6. select() or poll() can be used to wait for events on individual 54 * transmit or receive queues (or all queues for a given interface). 55 */ 56 57 #ifdef linux 58 #include "bsd_glue.h" 59 static netdev_tx_t linux_netmap_start(struct sk_buff *skb, struct net_device *dev); 60 #endif /* linux */ 61 62 #ifdef __APPLE__ 63 #include "osx_glue.h" 64 #endif /* __APPLE__ */ 65 66 #ifdef __FreeBSD__ 67 #include <sys/cdefs.h> /* prerequisite */ 68 __FBSDID("$FreeBSD$"); 69 70 #include <sys/types.h> 71 #include <sys/module.h> 72 #include <sys/errno.h> 73 #include <sys/param.h> /* defines used in kernel.h */ 74 #include <sys/jail.h> 75 #include <sys/kernel.h> /* types used in module initialization */ 76 #include <sys/conf.h> /* cdevsw struct */ 77 #include <sys/uio.h> /* uio struct */ 78 #include <sys/sockio.h> 79 #include <sys/socketvar.h> /* struct socket */ 80 #include <sys/malloc.h> 81 #include <sys/mman.h> /* PROT_EXEC */ 82 #include <sys/poll.h> 83 #include <sys/proc.h> 84 #include <sys/rwlock.h> 85 #include <vm/vm.h> /* vtophys */ 86 #include <vm/pmap.h> /* vtophys */ 87 #include <sys/socket.h> /* sockaddrs */ 88 #include <machine/bus.h> 89 #include <sys/selinfo.h> 90 #include <sys/sysctl.h> 91 #include <net/if.h> 92 #include <net/bpf.h> /* BIOCIMMEDIATE */ 93 #include <net/vnet.h> 94 #include <machine/bus.h> /* bus_dmamap_* */ 95 96 MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map"); 97 #endif /* __FreeBSD__ */ 98 99 #include <net/netmap.h> 100 #include <dev/netmap/netmap_kern.h> 101 102 u_int netmap_total_buffers; 103 u_int netmap_buf_size; 104 char *netmap_buffer_base; /* address of an invalid buffer */ 105 106 /* user-controlled variables */ 107 int netmap_verbose; 108 109 static int netmap_no_timestamp; /* don't timestamp on rxsync */ 110 111 SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args"); 112 SYSCTL_INT(_dev_netmap, OID_AUTO, verbose, 113 CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode"); 114 SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp, 115 CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp"); 116 int netmap_mitigate = 1; 117 SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, ""); 118 int netmap_no_pendintr = 1; 119 SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr, 120 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets."); 121 122 int netmap_drop = 0; /* debugging */ 123 int netmap_flags = 0; /* debug flags */ 124 int netmap_fwd = 0; /* force transparent mode */ 125 int netmap_copy = 0; /* debugging, copy content */ 126 127 SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , ""); 128 SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , ""); 129 SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , ""); 130 SYSCTL_INT(_dev_netmap, OID_AUTO, copy, CTLFLAG_RW, &netmap_copy, 0 , ""); 131 132 #ifdef NM_BRIDGE /* support for netmap bridge */ 133 134 /* 135 * system parameters. 136 * 137 * All switched ports have prefix NM_NAME. 138 * The switch has a max of NM_BDG_MAXPORTS ports (often stored in a bitmap, 139 * so a practical upper bound is 64). 140 * Each tx ring is read-write, whereas rx rings are readonly (XXX not done yet). 141 * The virtual interfaces use per-queue lock instead of core lock. 142 * In the tx loop, we aggregate traffic in batches to make all operations 143 * faster. The batch size is NM_BDG_BATCH 144 */ 145 #define NM_NAME "vale" /* prefix for the interface */ 146 #define NM_BDG_MAXPORTS 16 /* up to 64 ? */ 147 #define NM_BRIDGE_RINGSIZE 1024 /* in the device */ 148 #define NM_BDG_HASH 1024 /* forwarding table entries */ 149 #define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */ 150 #define NM_BRIDGES 4 /* number of bridges */ 151 int netmap_bridge = NM_BDG_BATCH; /* bridge batch size */ 152 SYSCTL_INT(_dev_netmap, OID_AUTO, bridge, CTLFLAG_RW, &netmap_bridge, 0 , ""); 153 154 #ifdef linux 155 #define ADD_BDG_REF(ifp) (NA(ifp)->if_refcount++) 156 #define DROP_BDG_REF(ifp) (NA(ifp)->if_refcount-- <= 1) 157 #else /* !linux */ 158 #define ADD_BDG_REF(ifp) (ifp)->if_refcount++ 159 #define DROP_BDG_REF(ifp) refcount_release(&(ifp)->if_refcount) 160 #ifdef __FreeBSD__ 161 #include <sys/endian.h> 162 #include <sys/refcount.h> 163 #endif /* __FreeBSD__ */ 164 #define prefetch(x) __builtin_prefetch(x) 165 #endif /* !linux */ 166 167 static void bdg_netmap_attach(struct ifnet *ifp); 168 static int bdg_netmap_reg(struct ifnet *ifp, int onoff); 169 /* per-tx-queue entry */ 170 struct nm_bdg_fwd { /* forwarding entry for a bridge */ 171 void *buf; 172 uint64_t dst; /* dst mask */ 173 uint32_t src; /* src index ? */ 174 uint16_t len; /* src len */ 175 }; 176 177 struct nm_hash_ent { 178 uint64_t mac; /* the top 2 bytes are the epoch */ 179 uint64_t ports; 180 }; 181 182 /* 183 * Interfaces for a bridge are all in ports[]. 184 * The array has fixed size, an empty entry does not terminate 185 * the search. 186 */ 187 struct nm_bridge { 188 struct ifnet *bdg_ports[NM_BDG_MAXPORTS]; 189 int n_ports; 190 uint64_t act_ports; 191 int freelist; /* first buffer index */ 192 NM_SELINFO_T si; /* poll/select wait queue */ 193 NM_LOCK_T bdg_lock; /* protect the selinfo ? */ 194 195 /* the forwarding table, MAC+ports */ 196 struct nm_hash_ent ht[NM_BDG_HASH]; 197 198 int namelen; /* 0 means free */ 199 char basename[IFNAMSIZ]; 200 }; 201 202 struct nm_bridge nm_bridges[NM_BRIDGES]; 203 204 #define BDG_LOCK(b) mtx_lock(&(b)->bdg_lock) 205 #define BDG_UNLOCK(b) mtx_unlock(&(b)->bdg_lock) 206 207 /* 208 * NA(ifp)->bdg_port port index 209 */ 210 211 // XXX only for multiples of 64 bytes, non overlapped. 212 static inline void 213 pkt_copy(void *_src, void *_dst, int l) 214 { 215 uint64_t *src = _src; 216 uint64_t *dst = _dst; 217 if (unlikely(l >= 1024)) { 218 bcopy(src, dst, l); 219 return; 220 } 221 for (; likely(l > 0); l-=64) { 222 *dst++ = *src++; 223 *dst++ = *src++; 224 *dst++ = *src++; 225 *dst++ = *src++; 226 *dst++ = *src++; 227 *dst++ = *src++; 228 *dst++ = *src++; 229 *dst++ = *src++; 230 } 231 } 232 233 /* 234 * locate a bridge among the existing ones. 235 * a ':' in the name terminates the bridge name. Otherwise, just NM_NAME. 236 * We assume that this is called with a name of at least NM_NAME chars. 237 */ 238 static struct nm_bridge * 239 nm_find_bridge(const char *name) 240 { 241 int i, l, namelen, e; 242 struct nm_bridge *b = NULL; 243 244 namelen = strlen(NM_NAME); /* base length */ 245 l = strlen(name); /* actual length */ 246 for (i = namelen + 1; i < l; i++) { 247 if (name[i] == ':') { 248 namelen = i; 249 break; 250 } 251 } 252 if (namelen >= IFNAMSIZ) 253 namelen = IFNAMSIZ; 254 ND("--- prefix is '%.*s' ---", namelen, name); 255 256 /* use the first entry for locking */ 257 BDG_LOCK(nm_bridges); // XXX do better 258 for (e = -1, i = 1; i < NM_BRIDGES; i++) { 259 b = nm_bridges + i; 260 if (b->namelen == 0) 261 e = i; /* record empty slot */ 262 else if (strncmp(name, b->basename, namelen) == 0) { 263 ND("found '%.*s' at %d", namelen, name, i); 264 break; 265 } 266 } 267 if (i == NM_BRIDGES) { /* all full */ 268 if (e == -1) { /* no empty slot */ 269 b = NULL; 270 } else { 271 b = nm_bridges + e; 272 strncpy(b->basename, name, namelen); 273 b->namelen = namelen; 274 } 275 } 276 BDG_UNLOCK(nm_bridges); 277 return b; 278 } 279 #endif /* NM_BRIDGE */ 280 281 282 /* 283 * Fetch configuration from the device, to cope with dynamic 284 * reconfigurations after loading the module. 285 */ 286 static int 287 netmap_update_config(struct netmap_adapter *na) 288 { 289 struct ifnet *ifp = na->ifp; 290 u_int txr, txd, rxr, rxd; 291 292 txr = txd = rxr = rxd = 0; 293 if (na->nm_config) { 294 na->nm_config(ifp, &txr, &txd, &rxr, &rxd); 295 } else { 296 /* take whatever we had at init time */ 297 txr = na->num_tx_rings; 298 txd = na->num_tx_desc; 299 rxr = na->num_rx_rings; 300 rxd = na->num_rx_desc; 301 } 302 303 if (na->num_tx_rings == txr && na->num_tx_desc == txd && 304 na->num_rx_rings == rxr && na->num_rx_desc == rxd) 305 return 0; /* nothing changed */ 306 if (netmap_verbose || na->refcount > 0) { 307 D("stored config %s: txring %d x %d, rxring %d x %d", 308 ifp->if_xname, 309 na->num_tx_rings, na->num_tx_desc, 310 na->num_rx_rings, na->num_rx_desc); 311 D("new config %s: txring %d x %d, rxring %d x %d", 312 ifp->if_xname, txr, txd, rxr, rxd); 313 } 314 if (na->refcount == 0) { 315 D("configuration changed (but fine)"); 316 na->num_tx_rings = txr; 317 na->num_tx_desc = txd; 318 na->num_rx_rings = rxr; 319 na->num_rx_desc = rxd; 320 return 0; 321 } 322 D("configuration changed while active, this is bad..."); 323 return 1; 324 } 325 326 /*------------- memory allocator -----------------*/ 327 #include "netmap_mem2.c" 328 /*------------ end of memory allocator ----------*/ 329 330 331 /* Structure associated to each thread which registered an interface. 332 * 333 * The first 4 fields of this structure are written by NIOCREGIF and 334 * read by poll() and NIOC?XSYNC. 335 * There is low contention among writers (actually, a correct user program 336 * should have no contention among writers) and among writers and readers, 337 * so we use a single global lock to protect the structure initialization. 338 * Since initialization involves the allocation of memory, we reuse the memory 339 * allocator lock. 340 * Read access to the structure is lock free. Readers must check that 341 * np_nifp is not NULL before using the other fields. 342 * If np_nifp is NULL initialization has not been performed, so they should 343 * return an error to userlevel. 344 * 345 * The ref_done field is used to regulate access to the refcount in the 346 * memory allocator. The refcount must be incremented at most once for 347 * each open("/dev/netmap"). The increment is performed by the first 348 * function that calls netmap_get_memory() (currently called by 349 * mmap(), NIOCGINFO and NIOCREGIF). 350 * If the refcount is incremented, it is then decremented when the 351 * private structure is destroyed. 352 */ 353 struct netmap_priv_d { 354 struct netmap_if * volatile np_nifp; /* netmap interface descriptor. */ 355 356 struct ifnet *np_ifp; /* device for which we hold a reference */ 357 int np_ringid; /* from the ioctl */ 358 u_int np_qfirst, np_qlast; /* range of rings to scan */ 359 uint16_t np_txpoll; 360 361 unsigned long ref_done; /* use with NMA_LOCK held */ 362 }; 363 364 365 static int 366 netmap_get_memory(struct netmap_priv_d* p) 367 { 368 int error = 0; 369 NMA_LOCK(); 370 if (!p->ref_done) { 371 error = netmap_memory_finalize(); 372 if (!error) 373 p->ref_done = 1; 374 } 375 NMA_UNLOCK(); 376 return error; 377 } 378 379 /* 380 * File descriptor's private data destructor. 381 * 382 * Call nm_register(ifp,0) to stop netmap mode on the interface and 383 * revert to normal operation. We expect that np_ifp has not gone. 384 */ 385 /* call with NMA_LOCK held */ 386 static void 387 netmap_dtor_locked(void *data) 388 { 389 struct netmap_priv_d *priv = data; 390 struct ifnet *ifp = priv->np_ifp; 391 struct netmap_adapter *na = NA(ifp); 392 struct netmap_if *nifp = priv->np_nifp; 393 394 na->refcount--; 395 if (na->refcount <= 0) { /* last instance */ 396 u_int i, j, lim; 397 398 if (netmap_verbose) 399 D("deleting last instance for %s", ifp->if_xname); 400 /* 401 * there is a race here with *_netmap_task() and 402 * netmap_poll(), which don't run under NETMAP_REG_LOCK. 403 * na->refcount == 0 && na->ifp->if_capenable & IFCAP_NETMAP 404 * (aka NETMAP_DELETING(na)) are a unique marker that the 405 * device is dying. 406 * Before destroying stuff we sleep a bit, and then complete 407 * the job. NIOCREG should realize the condition and 408 * loop until they can continue; the other routines 409 * should check the condition at entry and quit if 410 * they cannot run. 411 */ 412 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 413 tsleep(na, 0, "NIOCUNREG", 4); 414 na->nm_lock(ifp, NETMAP_REG_LOCK, 0); 415 na->nm_register(ifp, 0); /* off, clear IFCAP_NETMAP */ 416 /* Wake up any sleeping threads. netmap_poll will 417 * then return POLLERR 418 */ 419 for (i = 0; i < na->num_tx_rings + 1; i++) 420 selwakeuppri(&na->tx_rings[i].si, PI_NET); 421 for (i = 0; i < na->num_rx_rings + 1; i++) 422 selwakeuppri(&na->rx_rings[i].si, PI_NET); 423 selwakeuppri(&na->tx_si, PI_NET); 424 selwakeuppri(&na->rx_si, PI_NET); 425 /* release all buffers */ 426 for (i = 0; i < na->num_tx_rings + 1; i++) { 427 struct netmap_ring *ring = na->tx_rings[i].ring; 428 lim = na->tx_rings[i].nkr_num_slots; 429 for (j = 0; j < lim; j++) 430 netmap_free_buf(nifp, ring->slot[j].buf_idx); 431 /* knlist_destroy(&na->tx_rings[i].si.si_note); */ 432 mtx_destroy(&na->tx_rings[i].q_lock); 433 } 434 for (i = 0; i < na->num_rx_rings + 1; i++) { 435 struct netmap_ring *ring = na->rx_rings[i].ring; 436 lim = na->rx_rings[i].nkr_num_slots; 437 for (j = 0; j < lim; j++) 438 netmap_free_buf(nifp, ring->slot[j].buf_idx); 439 /* knlist_destroy(&na->rx_rings[i].si.si_note); */ 440 mtx_destroy(&na->rx_rings[i].q_lock); 441 } 442 /* XXX kqueue(9) needed; these will mirror knlist_init. */ 443 /* knlist_destroy(&na->tx_si.si_note); */ 444 /* knlist_destroy(&na->rx_si.si_note); */ 445 netmap_free_rings(na); 446 wakeup(na); 447 } 448 netmap_if_free(nifp); 449 } 450 451 static void 452 nm_if_rele(struct ifnet *ifp) 453 { 454 #ifndef NM_BRIDGE 455 if_rele(ifp); 456 #else /* NM_BRIDGE */ 457 int i, full; 458 struct nm_bridge *b; 459 460 if (strncmp(ifp->if_xname, NM_NAME, sizeof(NM_NAME) - 1)) { 461 if_rele(ifp); 462 return; 463 } 464 if (!DROP_BDG_REF(ifp)) 465 return; 466 b = ifp->if_bridge; 467 BDG_LOCK(nm_bridges); 468 BDG_LOCK(b); 469 ND("want to disconnect %s from the bridge", ifp->if_xname); 470 full = 0; 471 for (i = 0; i < NM_BDG_MAXPORTS; i++) { 472 if (b->bdg_ports[i] == ifp) { 473 b->bdg_ports[i] = NULL; 474 bzero(ifp, sizeof(*ifp)); 475 free(ifp, M_DEVBUF); 476 break; 477 } 478 else if (b->bdg_ports[i] != NULL) 479 full = 1; 480 } 481 BDG_UNLOCK(b); 482 if (full == 0) { 483 ND("freeing bridge %d", b - nm_bridges); 484 b->namelen = 0; 485 } 486 BDG_UNLOCK(nm_bridges); 487 if (i == NM_BDG_MAXPORTS) 488 D("ouch, cannot find ifp to remove"); 489 #endif /* NM_BRIDGE */ 490 } 491 492 static void 493 netmap_dtor(void *data) 494 { 495 struct netmap_priv_d *priv = data; 496 struct ifnet *ifp = priv->np_ifp; 497 498 NMA_LOCK(); 499 if (ifp) { 500 struct netmap_adapter *na = NA(ifp); 501 502 na->nm_lock(ifp, NETMAP_REG_LOCK, 0); 503 netmap_dtor_locked(data); 504 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 505 506 nm_if_rele(ifp); /* might also destroy *na */ 507 } 508 if (priv->ref_done) { 509 netmap_memory_deref(); 510 } 511 NMA_UNLOCK(); 512 bzero(priv, sizeof(*priv)); /* XXX for safety */ 513 free(priv, M_DEVBUF); 514 } 515 516 #ifdef __FreeBSD__ 517 #include <vm/vm.h> 518 #include <vm/vm_param.h> 519 #include <vm/vm_object.h> 520 #include <vm/vm_page.h> 521 #include <vm/vm_pager.h> 522 #include <vm/uma.h> 523 524 static struct cdev_pager_ops saved_cdev_pager_ops; 525 526 static int 527 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 528 vm_ooffset_t foff, struct ucred *cred, u_short *color) 529 { 530 if (netmap_verbose) 531 D("first mmap for %p", handle); 532 return saved_cdev_pager_ops.cdev_pg_ctor(handle, 533 size, prot, foff, cred, color); 534 } 535 536 static void 537 netmap_dev_pager_dtor(void *handle) 538 { 539 saved_cdev_pager_ops.cdev_pg_dtor(handle); 540 ND("ready to release memory for %p", handle); 541 } 542 543 544 static struct cdev_pager_ops netmap_cdev_pager_ops = { 545 .cdev_pg_ctor = netmap_dev_pager_ctor, 546 .cdev_pg_dtor = netmap_dev_pager_dtor, 547 .cdev_pg_fault = NULL, 548 }; 549 550 static int 551 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 552 vm_size_t objsize, vm_object_t *objp, int prot) 553 { 554 vm_object_t obj; 555 556 ND("cdev %p foff %jd size %jd objp %p prot %d", cdev, 557 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 558 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 559 curthread->td_ucred); 560 ND("returns obj %p", obj); 561 if (obj == NULL) 562 return EINVAL; 563 if (saved_cdev_pager_ops.cdev_pg_fault == NULL) { 564 ND("initialize cdev_pager_ops"); 565 saved_cdev_pager_ops = *(obj->un_pager.devp.ops); 566 netmap_cdev_pager_ops.cdev_pg_fault = 567 saved_cdev_pager_ops.cdev_pg_fault; 568 }; 569 obj->un_pager.devp.ops = &netmap_cdev_pager_ops; 570 *objp = obj; 571 return 0; 572 } 573 #endif /* __FreeBSD__ */ 574 575 576 /* 577 * mmap(2) support for the "netmap" device. 578 * 579 * Expose all the memory previously allocated by our custom memory 580 * allocator: this way the user has only to issue a single mmap(2), and 581 * can work on all the data structures flawlessly. 582 * 583 * Return 0 on success, -1 otherwise. 584 */ 585 586 #ifdef __FreeBSD__ 587 static int 588 netmap_mmap(__unused struct cdev *dev, 589 #if __FreeBSD_version < 900000 590 vm_offset_t offset, vm_paddr_t *paddr, int nprot 591 #else 592 vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, 593 __unused vm_memattr_t *memattr 594 #endif 595 ) 596 { 597 int error = 0; 598 struct netmap_priv_d *priv; 599 600 if (nprot & PROT_EXEC) 601 return (-1); // XXX -1 or EINVAL ? 602 603 error = devfs_get_cdevpriv((void **)&priv); 604 if (error == EBADF) { /* called on fault, memory is initialized */ 605 ND(5, "handling fault at ofs 0x%x", offset); 606 error = 0; 607 } else if (error == 0) /* make sure memory is set */ 608 error = netmap_get_memory(priv); 609 if (error) 610 return (error); 611 612 ND("request for offset 0x%x", (uint32_t)offset); 613 *paddr = netmap_ofstophys(offset); 614 615 return (*paddr ? 0 : ENOMEM); 616 } 617 618 static int 619 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 620 { 621 if (netmap_verbose) 622 D("dev %p fflag 0x%x devtype %d td %p", 623 dev, fflag, devtype, td); 624 return 0; 625 } 626 627 static int 628 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 629 { 630 struct netmap_priv_d *priv; 631 int error; 632 633 priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 634 M_NOWAIT | M_ZERO); 635 if (priv == NULL) 636 return ENOMEM; 637 638 error = devfs_set_cdevpriv(priv, netmap_dtor); 639 if (error) 640 return error; 641 642 return 0; 643 } 644 #endif /* __FreeBSD__ */ 645 646 647 /* 648 * Handlers for synchronization of the queues from/to the host. 649 * Netmap has two operating modes: 650 * - in the default mode, the rings connected to the host stack are 651 * just another ring pair managed by userspace; 652 * - in transparent mode (XXX to be defined) incoming packets 653 * (from the host or the NIC) are marked as NS_FORWARD upon 654 * arrival, and the user application has a chance to reset the 655 * flag for packets that should be dropped. 656 * On the RXSYNC or poll(), packets in RX rings between 657 * kring->nr_kcur and ring->cur with NS_FORWARD still set are moved 658 * to the other side. 659 * The transfer NIC --> host is relatively easy, just encapsulate 660 * into mbufs and we are done. The host --> NIC side is slightly 661 * harder because there might not be room in the tx ring so it 662 * might take a while before releasing the buffer. 663 */ 664 665 /* 666 * pass a chain of buffers to the host stack as coming from 'dst' 667 */ 668 static void 669 netmap_send_up(struct ifnet *dst, struct mbuf *head) 670 { 671 struct mbuf *m; 672 673 /* send packets up, outside the lock */ 674 while ((m = head) != NULL) { 675 head = head->m_nextpkt; 676 m->m_nextpkt = NULL; 677 if (netmap_verbose & NM_VERB_HOST) 678 D("sending up pkt %p size %d", m, MBUF_LEN(m)); 679 NM_SEND_UP(dst, m); 680 } 681 } 682 683 struct mbq { 684 struct mbuf *head; 685 struct mbuf *tail; 686 int count; 687 }; 688 689 /* 690 * put a copy of the buffers marked NS_FORWARD into an mbuf chain. 691 * Run from hwcur to cur - reserved 692 */ 693 static void 694 netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force) 695 { 696 /* Take packets from hwcur to cur-reserved and pass them up. 697 * In case of no buffers we give up. At the end of the loop, 698 * the queue is drained in all cases. 699 * XXX handle reserved 700 */ 701 int k = kring->ring->cur - kring->ring->reserved; 702 u_int n, lim = kring->nkr_num_slots - 1; 703 struct mbuf *m, *tail = q->tail; 704 705 if (k < 0) 706 k = k + kring->nkr_num_slots; 707 for (n = kring->nr_hwcur; n != k;) { 708 struct netmap_slot *slot = &kring->ring->slot[n]; 709 710 n = (n == lim) ? 0 : n + 1; 711 if ((slot->flags & NS_FORWARD) == 0 && !force) 712 continue; 713 if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE) { 714 D("bad pkt at %d len %d", n, slot->len); 715 continue; 716 } 717 slot->flags &= ~NS_FORWARD; // XXX needed ? 718 m = m_devget(NMB(slot), slot->len, 0, kring->na->ifp, NULL); 719 720 if (m == NULL) 721 break; 722 if (tail) 723 tail->m_nextpkt = m; 724 else 725 q->head = m; 726 tail = m; 727 q->count++; 728 m->m_nextpkt = NULL; 729 } 730 q->tail = tail; 731 } 732 733 /* 734 * called under main lock to send packets from the host to the NIC 735 * The host ring has packets from nr_hwcur to (cur - reserved) 736 * to be sent down. We scan the tx rings, which have just been 737 * flushed so nr_hwcur == cur. Pushing packets down means 738 * increment cur and decrement avail. 739 * XXX to be verified 740 */ 741 static void 742 netmap_sw_to_nic(struct netmap_adapter *na) 743 { 744 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 745 struct netmap_kring *k1 = &na->tx_rings[0]; 746 int i, howmany, src_lim, dst_lim; 747 748 howmany = kring->nr_hwavail; /* XXX otherwise cur - reserved - nr_hwcur */ 749 750 src_lim = kring->nkr_num_slots; 751 for (i = 0; howmany > 0 && i < na->num_tx_rings; i++, k1++) { 752 ND("%d packets left to ring %d (space %d)", howmany, i, k1->nr_hwavail); 753 dst_lim = k1->nkr_num_slots; 754 while (howmany > 0 && k1->ring->avail > 0) { 755 struct netmap_slot *src, *dst, tmp; 756 src = &kring->ring->slot[kring->nr_hwcur]; 757 dst = &k1->ring->slot[k1->ring->cur]; 758 tmp = *src; 759 src->buf_idx = dst->buf_idx; 760 src->flags = NS_BUF_CHANGED; 761 762 dst->buf_idx = tmp.buf_idx; 763 dst->len = tmp.len; 764 dst->flags = NS_BUF_CHANGED; 765 ND("out len %d buf %d from %d to %d", 766 dst->len, dst->buf_idx, 767 kring->nr_hwcur, k1->ring->cur); 768 769 if (++kring->nr_hwcur >= src_lim) 770 kring->nr_hwcur = 0; 771 howmany--; 772 kring->nr_hwavail--; 773 if (++k1->ring->cur >= dst_lim) 774 k1->ring->cur = 0; 775 k1->ring->avail--; 776 } 777 kring->ring->cur = kring->nr_hwcur; // XXX 778 k1++; 779 } 780 } 781 782 /* 783 * netmap_sync_to_host() passes packets up. We are called from a 784 * system call in user process context, and the only contention 785 * can be among multiple user threads erroneously calling 786 * this routine concurrently. 787 */ 788 static void 789 netmap_sync_to_host(struct netmap_adapter *na) 790 { 791 struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings]; 792 struct netmap_ring *ring = kring->ring; 793 u_int k, lim = kring->nkr_num_slots - 1; 794 struct mbq q = { NULL, NULL }; 795 796 k = ring->cur; 797 if (k > lim) { 798 netmap_ring_reinit(kring); 799 return; 800 } 801 // na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0); 802 803 /* Take packets from hwcur to cur and pass them up. 804 * In case of no buffers we give up. At the end of the loop, 805 * the queue is drained in all cases. 806 */ 807 netmap_grab_packets(kring, &q, 1); 808 kring->nr_hwcur = k; 809 kring->nr_hwavail = ring->avail = lim; 810 // na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0); 811 812 netmap_send_up(na->ifp, q.head); 813 } 814 815 /* 816 * rxsync backend for packets coming from the host stack. 817 * They have been put in the queue by netmap_start() so we 818 * need to protect access to the kring using a lock. 819 * 820 * This routine also does the selrecord if called from the poll handler 821 * (we know because td != NULL). 822 * 823 * NOTE: on linux, selrecord() is defined as a macro and uses pwait 824 * as an additional hidden argument. 825 */ 826 static void 827 netmap_sync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait) 828 { 829 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 830 struct netmap_ring *ring = kring->ring; 831 u_int j, n, lim = kring->nkr_num_slots; 832 u_int k = ring->cur, resvd = ring->reserved; 833 834 (void)pwait; /* disable unused warnings */ 835 na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0); 836 if (k >= lim) { 837 netmap_ring_reinit(kring); 838 return; 839 } 840 /* new packets are already set in nr_hwavail */ 841 /* skip past packets that userspace has released */ 842 j = kring->nr_hwcur; 843 if (resvd > 0) { 844 if (resvd + ring->avail >= lim + 1) { 845 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 846 ring->reserved = resvd = 0; // XXX panic... 847 } 848 k = (k >= resvd) ? k - resvd : k + lim - resvd; 849 } 850 if (j != k) { 851 n = k >= j ? k - j : k + lim - j; 852 kring->nr_hwavail -= n; 853 kring->nr_hwcur = k; 854 } 855 k = ring->avail = kring->nr_hwavail - resvd; 856 if (k == 0 && td) 857 selrecord(td, &kring->si); 858 if (k && (netmap_verbose & NM_VERB_HOST)) 859 D("%d pkts from stack", k); 860 na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0); 861 } 862 863 864 /* 865 * get a refcounted reference to an interface. 866 * Return ENXIO if the interface does not exist, EINVAL if netmap 867 * is not supported by the interface. 868 * If successful, hold a reference. 869 */ 870 static int 871 get_ifp(const char *name, struct ifnet **ifp) 872 { 873 #ifdef NM_BRIDGE 874 struct ifnet *iter = NULL; 875 876 do { 877 struct nm_bridge *b; 878 int i, l, cand = -1; 879 880 if (strncmp(name, NM_NAME, sizeof(NM_NAME) - 1)) 881 break; 882 b = nm_find_bridge(name); 883 if (b == NULL) { 884 D("no bridges available for '%s'", name); 885 return (ENXIO); 886 } 887 /* XXX locking */ 888 BDG_LOCK(b); 889 /* lookup in the local list of ports */ 890 for (i = 0; i < NM_BDG_MAXPORTS; i++) { 891 iter = b->bdg_ports[i]; 892 if (iter == NULL) { 893 if (cand == -1) 894 cand = i; /* potential insert point */ 895 continue; 896 } 897 if (!strcmp(iter->if_xname, name)) { 898 ADD_BDG_REF(iter); 899 ND("found existing interface"); 900 BDG_UNLOCK(b); 901 break; 902 } 903 } 904 if (i < NM_BDG_MAXPORTS) /* already unlocked */ 905 break; 906 if (cand == -1) { 907 D("bridge full, cannot create new port"); 908 no_port: 909 BDG_UNLOCK(b); 910 *ifp = NULL; 911 return EINVAL; 912 } 913 ND("create new bridge port %s", name); 914 /* space for forwarding list after the ifnet */ 915 l = sizeof(*iter) + 916 sizeof(struct nm_bdg_fwd)*NM_BDG_BATCH ; 917 iter = malloc(l, M_DEVBUF, M_NOWAIT | M_ZERO); 918 if (!iter) 919 goto no_port; 920 strcpy(iter->if_xname, name); 921 bdg_netmap_attach(iter); 922 b->bdg_ports[cand] = iter; 923 iter->if_bridge = b; 924 ADD_BDG_REF(iter); 925 BDG_UNLOCK(b); 926 ND("attaching virtual bridge %p", b); 927 } while (0); 928 *ifp = iter; 929 if (! *ifp) 930 #endif /* NM_BRIDGE */ 931 *ifp = ifunit_ref(name); 932 if (*ifp == NULL) 933 return (ENXIO); 934 /* can do this if the capability exists and if_pspare[0] 935 * points to the netmap descriptor. 936 */ 937 if (NETMAP_CAPABLE(*ifp)) 938 return 0; /* valid pointer, we hold the refcount */ 939 nm_if_rele(*ifp); 940 return EINVAL; // not NETMAP capable 941 } 942 943 944 /* 945 * Error routine called when txsync/rxsync detects an error. 946 * Can't do much more than resetting cur = hwcur, avail = hwavail. 947 * Return 1 on reinit. 948 * 949 * This routine is only called by the upper half of the kernel. 950 * It only reads hwcur (which is changed only by the upper half, too) 951 * and hwavail (which may be changed by the lower half, but only on 952 * a tx ring and only to increase it, so any error will be recovered 953 * on the next call). For the above, we don't strictly need to call 954 * it under lock. 955 */ 956 int 957 netmap_ring_reinit(struct netmap_kring *kring) 958 { 959 struct netmap_ring *ring = kring->ring; 960 u_int i, lim = kring->nkr_num_slots - 1; 961 int errors = 0; 962 963 RD(10, "called for %s", kring->na->ifp->if_xname); 964 if (ring->cur > lim) 965 errors++; 966 for (i = 0; i <= lim; i++) { 967 u_int idx = ring->slot[i].buf_idx; 968 u_int len = ring->slot[i].len; 969 if (idx < 2 || idx >= netmap_total_buffers) { 970 if (!errors++) 971 D("bad buffer at slot %d idx %d len %d ", i, idx, len); 972 ring->slot[i].buf_idx = 0; 973 ring->slot[i].len = 0; 974 } else if (len > NETMAP_BUF_SIZE) { 975 ring->slot[i].len = 0; 976 if (!errors++) 977 D("bad len %d at slot %d idx %d", 978 len, i, idx); 979 } 980 } 981 if (errors) { 982 int pos = kring - kring->na->tx_rings; 983 int n = kring->na->num_tx_rings + 1; 984 985 RD(10, "total %d errors", errors); 986 errors++; 987 RD(10, "%s %s[%d] reinit, cur %d -> %d avail %d -> %d", 988 kring->na->ifp->if_xname, 989 pos < n ? "TX" : "RX", pos < n ? pos : pos - n, 990 ring->cur, kring->nr_hwcur, 991 ring->avail, kring->nr_hwavail); 992 ring->cur = kring->nr_hwcur; 993 ring->avail = kring->nr_hwavail; 994 } 995 return (errors ? 1 : 0); 996 } 997 998 999 /* 1000 * Set the ring ID. For devices with a single queue, a request 1001 * for all rings is the same as a single ring. 1002 */ 1003 static int 1004 netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid) 1005 { 1006 struct ifnet *ifp = priv->np_ifp; 1007 struct netmap_adapter *na = NA(ifp); 1008 u_int i = ringid & NETMAP_RING_MASK; 1009 /* initially (np_qfirst == np_qlast) we don't want to lock */ 1010 int need_lock = (priv->np_qfirst != priv->np_qlast); 1011 int lim = na->num_rx_rings; 1012 1013 if (na->num_tx_rings > lim) 1014 lim = na->num_tx_rings; 1015 if ( (ringid & NETMAP_HW_RING) && i >= lim) { 1016 D("invalid ring id %d", i); 1017 return (EINVAL); 1018 } 1019 if (need_lock) 1020 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1021 priv->np_ringid = ringid; 1022 if (ringid & NETMAP_SW_RING) { 1023 priv->np_qfirst = NETMAP_SW_RING; 1024 priv->np_qlast = 0; 1025 } else if (ringid & NETMAP_HW_RING) { 1026 priv->np_qfirst = i; 1027 priv->np_qlast = i + 1; 1028 } else { 1029 priv->np_qfirst = 0; 1030 priv->np_qlast = NETMAP_HW_RING ; 1031 } 1032 priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1; 1033 if (need_lock) 1034 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 1035 if (netmap_verbose) { 1036 if (ringid & NETMAP_SW_RING) 1037 D("ringid %s set to SW RING", ifp->if_xname); 1038 else if (ringid & NETMAP_HW_RING) 1039 D("ringid %s set to HW RING %d", ifp->if_xname, 1040 priv->np_qfirst); 1041 else 1042 D("ringid %s set to all %d HW RINGS", ifp->if_xname, lim); 1043 } 1044 return 0; 1045 } 1046 1047 /* 1048 * ioctl(2) support for the "netmap" device. 1049 * 1050 * Following a list of accepted commands: 1051 * - NIOCGINFO 1052 * - SIOCGIFADDR just for convenience 1053 * - NIOCREGIF 1054 * - NIOCUNREGIF 1055 * - NIOCTXSYNC 1056 * - NIOCRXSYNC 1057 * 1058 * Return 0 on success, errno otherwise. 1059 */ 1060 static int 1061 netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, 1062 int fflag, struct thread *td) 1063 { 1064 struct netmap_priv_d *priv = NULL; 1065 struct ifnet *ifp; 1066 struct nmreq *nmr = (struct nmreq *) data; 1067 struct netmap_adapter *na; 1068 int error; 1069 u_int i, lim; 1070 struct netmap_if *nifp; 1071 1072 (void)dev; /* UNUSED */ 1073 (void)fflag; /* UNUSED */ 1074 #ifdef linux 1075 #define devfs_get_cdevpriv(pp) \ 1076 ({ *(struct netmap_priv_d **)pp = ((struct file *)td)->private_data; \ 1077 (*pp ? 0 : ENOENT); }) 1078 1079 /* devfs_set_cdevpriv cannot fail on linux */ 1080 #define devfs_set_cdevpriv(p, fn) \ 1081 ({ ((struct file *)td)->private_data = p; (p ? 0 : EINVAL); }) 1082 1083 1084 #define devfs_clear_cdevpriv() do { \ 1085 netmap_dtor(priv); ((struct file *)td)->private_data = 0; \ 1086 } while (0) 1087 #endif /* linux */ 1088 1089 CURVNET_SET(TD_TO_VNET(td)); 1090 1091 error = devfs_get_cdevpriv((void **)&priv); 1092 if (error) { 1093 CURVNET_RESTORE(); 1094 /* XXX ENOENT should be impossible, since the priv 1095 * is now created in the open */ 1096 return (error == ENOENT ? ENXIO : error); 1097 } 1098 1099 nmr->nr_name[sizeof(nmr->nr_name) - 1] = '\0'; /* truncate name */ 1100 switch (cmd) { 1101 case NIOCGINFO: /* return capabilities etc */ 1102 if (nmr->nr_version != NETMAP_API) { 1103 D("API mismatch got %d have %d", 1104 nmr->nr_version, NETMAP_API); 1105 nmr->nr_version = NETMAP_API; 1106 error = EINVAL; 1107 break; 1108 } 1109 /* update configuration */ 1110 error = netmap_get_memory(priv); 1111 ND("get_memory returned %d", error); 1112 if (error) 1113 break; 1114 /* memsize is always valid */ 1115 nmr->nr_memsize = nm_mem.nm_totalsize; 1116 nmr->nr_offset = 0; 1117 nmr->nr_rx_rings = nmr->nr_tx_rings = 0; 1118 nmr->nr_rx_slots = nmr->nr_tx_slots = 0; 1119 if (nmr->nr_name[0] == '\0') /* just get memory info */ 1120 break; 1121 error = get_ifp(nmr->nr_name, &ifp); /* get a refcount */ 1122 if (error) 1123 break; 1124 na = NA(ifp); /* retrieve netmap_adapter */ 1125 netmap_update_config(na); 1126 nmr->nr_rx_rings = na->num_rx_rings; 1127 nmr->nr_tx_rings = na->num_tx_rings; 1128 nmr->nr_rx_slots = na->num_rx_desc; 1129 nmr->nr_tx_slots = na->num_tx_desc; 1130 nm_if_rele(ifp); /* return the refcount */ 1131 break; 1132 1133 case NIOCREGIF: 1134 if (nmr->nr_version != NETMAP_API) { 1135 nmr->nr_version = NETMAP_API; 1136 error = EINVAL; 1137 break; 1138 } 1139 /* ensure allocators are ready */ 1140 error = netmap_get_memory(priv); 1141 ND("get_memory returned %d", error); 1142 if (error) 1143 break; 1144 1145 /* protect access to priv from concurrent NIOCREGIF */ 1146 NMA_LOCK(); 1147 if (priv->np_ifp != NULL) { /* thread already registered */ 1148 error = netmap_set_ringid(priv, nmr->nr_ringid); 1149 NMA_UNLOCK(); 1150 break; 1151 } 1152 /* find the interface and a reference */ 1153 error = get_ifp(nmr->nr_name, &ifp); /* keep reference */ 1154 if (error) { 1155 NMA_UNLOCK(); 1156 break; 1157 } 1158 na = NA(ifp); /* retrieve netmap adapter */ 1159 1160 for (i = 10; i > 0; i--) { 1161 na->nm_lock(ifp, NETMAP_REG_LOCK, 0); 1162 if (!NETMAP_DELETING(na)) 1163 break; 1164 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 1165 tsleep(na, 0, "NIOCREGIF", hz/10); 1166 } 1167 if (i == 0) { 1168 D("too many NIOCREGIF attempts, give up"); 1169 error = EINVAL; 1170 nm_if_rele(ifp); /* return the refcount */ 1171 NMA_UNLOCK(); 1172 break; 1173 } 1174 1175 /* ring configuration may have changed, fetch from the card */ 1176 netmap_update_config(na); 1177 priv->np_ifp = ifp; /* store the reference */ 1178 error = netmap_set_ringid(priv, nmr->nr_ringid); 1179 if (error) 1180 goto error; 1181 nifp = netmap_if_new(nmr->nr_name, na); 1182 if (nifp == NULL) { /* allocation failed */ 1183 error = ENOMEM; 1184 } else if (ifp->if_capenable & IFCAP_NETMAP) { 1185 /* was already set */ 1186 } else { 1187 /* Otherwise set the card in netmap mode 1188 * and make it use the shared buffers. 1189 */ 1190 for (i = 0 ; i < na->num_tx_rings + 1; i++) 1191 mtx_init(&na->tx_rings[i].q_lock, "nm_txq_lock", MTX_NETWORK_LOCK, MTX_DEF); 1192 for (i = 0 ; i < na->num_rx_rings + 1; i++) { 1193 mtx_init(&na->rx_rings[i].q_lock, "nm_rxq_lock", MTX_NETWORK_LOCK, MTX_DEF); 1194 } 1195 error = na->nm_register(ifp, 1); /* mode on */ 1196 if (error) { 1197 netmap_dtor_locked(priv); 1198 netmap_if_free(nifp); 1199 } 1200 } 1201 1202 if (error) { /* reg. failed, release priv and ref */ 1203 error: 1204 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 1205 nm_if_rele(ifp); /* return the refcount */ 1206 priv->np_ifp = NULL; 1207 priv->np_nifp = NULL; 1208 NMA_UNLOCK(); 1209 break; 1210 } 1211 1212 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 1213 1214 /* the following assignment is a commitment. 1215 * Readers (i.e., poll and *SYNC) check for 1216 * np_nifp != NULL without locking 1217 */ 1218 wmb(); /* make sure previous writes are visible to all CPUs */ 1219 priv->np_nifp = nifp; 1220 NMA_UNLOCK(); 1221 1222 /* return the offset of the netmap_if object */ 1223 nmr->nr_rx_rings = na->num_rx_rings; 1224 nmr->nr_tx_rings = na->num_tx_rings; 1225 nmr->nr_rx_slots = na->num_rx_desc; 1226 nmr->nr_tx_slots = na->num_tx_desc; 1227 nmr->nr_memsize = nm_mem.nm_totalsize; 1228 nmr->nr_offset = netmap_if_offset(nifp); 1229 break; 1230 1231 case NIOCUNREGIF: 1232 // XXX we have no data here ? 1233 D("deprecated, data is %p", nmr); 1234 error = EINVAL; 1235 break; 1236 1237 case NIOCTXSYNC: 1238 case NIOCRXSYNC: 1239 nifp = priv->np_nifp; 1240 1241 if (nifp == NULL) { 1242 error = ENXIO; 1243 break; 1244 } 1245 rmb(); /* make sure following reads are not from cache */ 1246 1247 1248 ifp = priv->np_ifp; /* we have a reference */ 1249 1250 if (ifp == NULL) { 1251 D("Internal error: nifp != NULL && ifp == NULL"); 1252 error = ENXIO; 1253 break; 1254 } 1255 1256 na = NA(ifp); /* retrieve netmap adapter */ 1257 if (priv->np_qfirst == NETMAP_SW_RING) { /* host rings */ 1258 if (cmd == NIOCTXSYNC) 1259 netmap_sync_to_host(na); 1260 else 1261 netmap_sync_from_host(na, NULL, NULL); 1262 break; 1263 } 1264 /* find the last ring to scan */ 1265 lim = priv->np_qlast; 1266 if (lim == NETMAP_HW_RING) 1267 lim = (cmd == NIOCTXSYNC) ? 1268 na->num_tx_rings : na->num_rx_rings; 1269 1270 for (i = priv->np_qfirst; i < lim; i++) { 1271 if (cmd == NIOCTXSYNC) { 1272 struct netmap_kring *kring = &na->tx_rings[i]; 1273 if (netmap_verbose & NM_VERB_TXSYNC) 1274 D("pre txsync ring %d cur %d hwcur %d", 1275 i, kring->ring->cur, 1276 kring->nr_hwcur); 1277 na->nm_txsync(ifp, i, 1 /* do lock */); 1278 if (netmap_verbose & NM_VERB_TXSYNC) 1279 D("post txsync ring %d cur %d hwcur %d", 1280 i, kring->ring->cur, 1281 kring->nr_hwcur); 1282 } else { 1283 na->nm_rxsync(ifp, i, 1 /* do lock */); 1284 microtime(&na->rx_rings[i].ring->ts); 1285 } 1286 } 1287 1288 break; 1289 1290 #ifdef __FreeBSD__ 1291 case BIOCIMMEDIATE: 1292 case BIOCGHDRCMPLT: 1293 case BIOCSHDRCMPLT: 1294 case BIOCSSEESENT: 1295 D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT"); 1296 break; 1297 1298 default: /* allow device-specific ioctls */ 1299 { 1300 struct socket so; 1301 bzero(&so, sizeof(so)); 1302 error = get_ifp(nmr->nr_name, &ifp); /* keep reference */ 1303 if (error) 1304 break; 1305 so.so_vnet = ifp->if_vnet; 1306 // so->so_proto not null. 1307 error = ifioctl(&so, cmd, data, td); 1308 nm_if_rele(ifp); 1309 break; 1310 } 1311 1312 #else /* linux */ 1313 default: 1314 error = EOPNOTSUPP; 1315 #endif /* linux */ 1316 } 1317 1318 CURVNET_RESTORE(); 1319 return (error); 1320 } 1321 1322 1323 /* 1324 * select(2) and poll(2) handlers for the "netmap" device. 1325 * 1326 * Can be called for one or more queues. 1327 * Return true the event mask corresponding to ready events. 1328 * If there are no ready events, do a selrecord on either individual 1329 * selfd or on the global one. 1330 * Device-dependent parts (locking and sync of tx/rx rings) 1331 * are done through callbacks. 1332 * 1333 * On linux, arguments are really pwait, the poll table, and 'td' is struct file * 1334 * The first one is remapped to pwait as selrecord() uses the name as an 1335 * hidden argument. 1336 */ 1337 static int 1338 netmap_poll(struct cdev *dev, int events, struct thread *td) 1339 { 1340 struct netmap_priv_d *priv = NULL; 1341 struct netmap_adapter *na; 1342 struct ifnet *ifp; 1343 struct netmap_kring *kring; 1344 u_int core_lock, i, check_all, want_tx, want_rx, revents = 0; 1345 u_int lim_tx, lim_rx, host_forwarded = 0; 1346 struct mbq q = { NULL, NULL, 0 }; 1347 enum {NO_CL, NEED_CL, LOCKED_CL }; /* see below */ 1348 void *pwait = dev; /* linux compatibility */ 1349 1350 (void)pwait; 1351 1352 if (devfs_get_cdevpriv((void **)&priv) != 0 || priv == NULL) 1353 return POLLERR; 1354 1355 if (priv->np_nifp == NULL) { 1356 D("No if registered"); 1357 return POLLERR; 1358 } 1359 rmb(); /* make sure following reads are not from cache */ 1360 1361 ifp = priv->np_ifp; 1362 // XXX check for deleting() ? 1363 if ( (ifp->if_capenable & IFCAP_NETMAP) == 0) 1364 return POLLERR; 1365 1366 if (netmap_verbose & 0x8000) 1367 D("device %s events 0x%x", ifp->if_xname, events); 1368 want_tx = events & (POLLOUT | POLLWRNORM); 1369 want_rx = events & (POLLIN | POLLRDNORM); 1370 1371 na = NA(ifp); /* retrieve netmap adapter */ 1372 1373 lim_tx = na->num_tx_rings; 1374 lim_rx = na->num_rx_rings; 1375 /* how many queues we are scanning */ 1376 if (priv->np_qfirst == NETMAP_SW_RING) { 1377 if (priv->np_txpoll || want_tx) { 1378 /* push any packets up, then we are always ready */ 1379 kring = &na->tx_rings[lim_tx]; 1380 netmap_sync_to_host(na); 1381 revents |= want_tx; 1382 } 1383 if (want_rx) { 1384 kring = &na->rx_rings[lim_rx]; 1385 if (kring->ring->avail == 0) 1386 netmap_sync_from_host(na, td, dev); 1387 if (kring->ring->avail > 0) { 1388 revents |= want_rx; 1389 } 1390 } 1391 return (revents); 1392 } 1393 1394 /* if we are in transparent mode, check also the host rx ring */ 1395 kring = &na->rx_rings[lim_rx]; 1396 if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all 1397 && want_rx 1398 && (netmap_fwd || kring->ring->flags & NR_FORWARD) ) { 1399 if (kring->ring->avail == 0) 1400 netmap_sync_from_host(na, td, dev); 1401 if (kring->ring->avail > 0) 1402 revents |= want_rx; 1403 } 1404 1405 /* 1406 * check_all is set if the card has more than one queue and 1407 * the client is polling all of them. If true, we sleep on 1408 * the "global" selfd, otherwise we sleep on individual selfd 1409 * (we can only sleep on one of them per direction). 1410 * The interrupt routine in the driver should always wake on 1411 * the individual selfd, and also on the global one if the card 1412 * has more than one ring. 1413 * 1414 * If the card has only one lock, we just use that. 1415 * If the card has separate ring locks, we just use those 1416 * unless we are doing check_all, in which case the whole 1417 * loop is wrapped by the global lock. 1418 * We acquire locks only when necessary: if poll is called 1419 * when buffers are available, we can just return without locks. 1420 * 1421 * rxsync() is only called if we run out of buffers on a POLLIN. 1422 * txsync() is called if we run out of buffers on POLLOUT, or 1423 * there are pending packets to send. The latter can be disabled 1424 * passing NETMAP_NO_TX_POLL in the NIOCREG call. 1425 */ 1426 check_all = (priv->np_qlast == NETMAP_HW_RING) && (lim_tx > 1 || lim_rx > 1); 1427 1428 /* 1429 * core_lock indicates what to do with the core lock. 1430 * The core lock is used when either the card has no individual 1431 * locks, or it has individual locks but we are cheking all 1432 * rings so we need the core lock to avoid missing wakeup events. 1433 * 1434 * It has three possible states: 1435 * NO_CL we don't need to use the core lock, e.g. 1436 * because we are protected by individual locks. 1437 * NEED_CL we need the core lock. In this case, when we 1438 * call the lock routine, move to LOCKED_CL 1439 * to remember to release the lock once done. 1440 * LOCKED_CL core lock is set, so we need to release it. 1441 */ 1442 core_lock = (check_all || !na->separate_locks) ? NEED_CL : NO_CL; 1443 #ifdef NM_BRIDGE 1444 /* the bridge uses separate locks */ 1445 if (na->nm_register == bdg_netmap_reg) { 1446 ND("not using core lock for %s", ifp->if_xname); 1447 core_lock = NO_CL; 1448 } 1449 #endif /* NM_BRIDGE */ 1450 if (priv->np_qlast != NETMAP_HW_RING) { 1451 lim_tx = lim_rx = priv->np_qlast; 1452 } 1453 1454 /* 1455 * We start with a lock free round which is good if we have 1456 * data available. If this fails, then lock and call the sync 1457 * routines. 1458 */ 1459 for (i = priv->np_qfirst; want_rx && i < lim_rx; i++) { 1460 kring = &na->rx_rings[i]; 1461 if (kring->ring->avail > 0) { 1462 revents |= want_rx; 1463 want_rx = 0; /* also breaks the loop */ 1464 } 1465 } 1466 for (i = priv->np_qfirst; want_tx && i < lim_tx; i++) { 1467 kring = &na->tx_rings[i]; 1468 if (kring->ring->avail > 0) { 1469 revents |= want_tx; 1470 want_tx = 0; /* also breaks the loop */ 1471 } 1472 } 1473 1474 /* 1475 * If we to push packets out (priv->np_txpoll) or want_tx is 1476 * still set, we do need to run the txsync calls (on all rings, 1477 * to avoid that the tx rings stall). 1478 */ 1479 if (priv->np_txpoll || want_tx) { 1480 flush_tx: 1481 for (i = priv->np_qfirst; i < lim_tx; i++) { 1482 kring = &na->tx_rings[i]; 1483 /* 1484 * Skip the current ring if want_tx == 0 1485 * (we have already done a successful sync on 1486 * a previous ring) AND kring->cur == kring->hwcur 1487 * (there are no pending transmissions for this ring). 1488 */ 1489 if (!want_tx && kring->ring->cur == kring->nr_hwcur) 1490 continue; 1491 if (core_lock == NEED_CL) { 1492 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1493 core_lock = LOCKED_CL; 1494 } 1495 if (na->separate_locks) 1496 na->nm_lock(ifp, NETMAP_TX_LOCK, i); 1497 if (netmap_verbose & NM_VERB_TXSYNC) 1498 D("send %d on %s %d", 1499 kring->ring->cur, 1500 ifp->if_xname, i); 1501 if (na->nm_txsync(ifp, i, 0 /* no lock */)) 1502 revents |= POLLERR; 1503 1504 /* Check avail/call selrecord only if called with POLLOUT */ 1505 if (want_tx) { 1506 if (kring->ring->avail > 0) { 1507 /* stop at the first ring. We don't risk 1508 * starvation. 1509 */ 1510 revents |= want_tx; 1511 want_tx = 0; 1512 } else if (!check_all) 1513 selrecord(td, &kring->si); 1514 } 1515 if (na->separate_locks) 1516 na->nm_lock(ifp, NETMAP_TX_UNLOCK, i); 1517 } 1518 } 1519 1520 /* 1521 * now if want_rx is still set we need to lock and rxsync. 1522 * Do it on all rings because otherwise we starve. 1523 */ 1524 if (want_rx) { 1525 for (i = priv->np_qfirst; i < lim_rx; i++) { 1526 kring = &na->rx_rings[i]; 1527 if (core_lock == NEED_CL) { 1528 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1529 core_lock = LOCKED_CL; 1530 } 1531 if (na->separate_locks) 1532 na->nm_lock(ifp, NETMAP_RX_LOCK, i); 1533 if (netmap_fwd ||kring->ring->flags & NR_FORWARD) { 1534 ND(10, "forwarding some buffers up %d to %d", 1535 kring->nr_hwcur, kring->ring->cur); 1536 netmap_grab_packets(kring, &q, netmap_fwd); 1537 } 1538 1539 if (na->nm_rxsync(ifp, i, 0 /* no lock */)) 1540 revents |= POLLERR; 1541 if (netmap_no_timestamp == 0 || 1542 kring->ring->flags & NR_TIMESTAMP) { 1543 microtime(&kring->ring->ts); 1544 } 1545 1546 if (kring->ring->avail > 0) 1547 revents |= want_rx; 1548 else if (!check_all) 1549 selrecord(td, &kring->si); 1550 if (na->separate_locks) 1551 na->nm_lock(ifp, NETMAP_RX_UNLOCK, i); 1552 } 1553 } 1554 if (check_all && revents == 0) { /* signal on the global queue */ 1555 if (want_tx) 1556 selrecord(td, &na->tx_si); 1557 if (want_rx) 1558 selrecord(td, &na->rx_si); 1559 } 1560 1561 /* forward host to the netmap ring */ 1562 kring = &na->rx_rings[lim_rx]; 1563 if (kring->nr_hwavail > 0) 1564 ND("host rx %d has %d packets", lim_rx, kring->nr_hwavail); 1565 if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all 1566 && (netmap_fwd || kring->ring->flags & NR_FORWARD) 1567 && kring->nr_hwavail > 0 && !host_forwarded) { 1568 if (core_lock == NEED_CL) { 1569 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1570 core_lock = LOCKED_CL; 1571 } 1572 netmap_sw_to_nic(na); 1573 host_forwarded = 1; /* prevent another pass */ 1574 want_rx = 0; 1575 goto flush_tx; 1576 } 1577 1578 if (core_lock == LOCKED_CL) 1579 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 1580 if (q.head) 1581 netmap_send_up(na->ifp, q.head); 1582 1583 return (revents); 1584 } 1585 1586 /*------- driver support routines ------*/ 1587 1588 /* 1589 * default lock wrapper. 1590 */ 1591 static void 1592 netmap_lock_wrapper(struct ifnet *dev, int what, u_int queueid) 1593 { 1594 struct netmap_adapter *na = NA(dev); 1595 1596 switch (what) { 1597 #ifdef linux /* some system do not need lock on register */ 1598 case NETMAP_REG_LOCK: 1599 case NETMAP_REG_UNLOCK: 1600 break; 1601 #endif /* linux */ 1602 1603 case NETMAP_CORE_LOCK: 1604 mtx_lock(&na->core_lock); 1605 break; 1606 1607 case NETMAP_CORE_UNLOCK: 1608 mtx_unlock(&na->core_lock); 1609 break; 1610 1611 case NETMAP_TX_LOCK: 1612 mtx_lock(&na->tx_rings[queueid].q_lock); 1613 break; 1614 1615 case NETMAP_TX_UNLOCK: 1616 mtx_unlock(&na->tx_rings[queueid].q_lock); 1617 break; 1618 1619 case NETMAP_RX_LOCK: 1620 mtx_lock(&na->rx_rings[queueid].q_lock); 1621 break; 1622 1623 case NETMAP_RX_UNLOCK: 1624 mtx_unlock(&na->rx_rings[queueid].q_lock); 1625 break; 1626 } 1627 } 1628 1629 1630 /* 1631 * Initialize a ``netmap_adapter`` object created by driver on attach. 1632 * We allocate a block of memory with room for a struct netmap_adapter 1633 * plus two sets of N+2 struct netmap_kring (where N is the number 1634 * of hardware rings): 1635 * krings 0..N-1 are for the hardware queues. 1636 * kring N is for the host stack queue 1637 * kring N+1 is only used for the selinfo for all queues. 1638 * Return 0 on success, ENOMEM otherwise. 1639 * 1640 * By default the receive and transmit adapter ring counts are both initialized 1641 * to num_queues. na->num_tx_rings can be set for cards with different tx/rx 1642 * setups. 1643 */ 1644 int 1645 netmap_attach(struct netmap_adapter *arg, int num_queues) 1646 { 1647 struct netmap_adapter *na = NULL; 1648 struct ifnet *ifp = arg ? arg->ifp : NULL; 1649 1650 if (arg == NULL || ifp == NULL) 1651 goto fail; 1652 na = malloc(sizeof(*na), M_DEVBUF, M_NOWAIT | M_ZERO); 1653 if (na == NULL) 1654 goto fail; 1655 WNA(ifp) = na; 1656 *na = *arg; /* copy everything, trust the driver to not pass junk */ 1657 NETMAP_SET_CAPABLE(ifp); 1658 if (na->num_tx_rings == 0) 1659 na->num_tx_rings = num_queues; 1660 na->num_rx_rings = num_queues; 1661 na->refcount = na->na_single = na->na_multi = 0; 1662 /* Core lock initialized here, others after netmap_if_new. */ 1663 mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF); 1664 if (na->nm_lock == NULL) { 1665 ND("using default locks for %s", ifp->if_xname); 1666 na->nm_lock = netmap_lock_wrapper; 1667 } 1668 #ifdef linux 1669 if (ifp->netdev_ops) { 1670 ND("netdev_ops %p", ifp->netdev_ops); 1671 /* prepare a clone of the netdev ops */ 1672 na->nm_ndo = *ifp->netdev_ops; 1673 } 1674 na->nm_ndo.ndo_start_xmit = linux_netmap_start; 1675 #endif 1676 D("success for %s", ifp->if_xname); 1677 return 0; 1678 1679 fail: 1680 D("fail, arg %p ifp %p na %p", arg, ifp, na); 1681 return (na ? EINVAL : ENOMEM); 1682 } 1683 1684 1685 /* 1686 * Free the allocated memory linked to the given ``netmap_adapter`` 1687 * object. 1688 */ 1689 void 1690 netmap_detach(struct ifnet *ifp) 1691 { 1692 struct netmap_adapter *na = NA(ifp); 1693 1694 if (!na) 1695 return; 1696 1697 mtx_destroy(&na->core_lock); 1698 1699 if (na->tx_rings) { /* XXX should not happen */ 1700 D("freeing leftover tx_rings"); 1701 free(na->tx_rings, M_DEVBUF); 1702 } 1703 bzero(na, sizeof(*na)); 1704 WNA(ifp) = NULL; 1705 free(na, M_DEVBUF); 1706 } 1707 1708 1709 /* 1710 * Intercept packets from the network stack and pass them 1711 * to netmap as incoming packets on the 'software' ring. 1712 * We are not locked when called. 1713 */ 1714 int 1715 netmap_start(struct ifnet *ifp, struct mbuf *m) 1716 { 1717 struct netmap_adapter *na = NA(ifp); 1718 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 1719 u_int i, len = MBUF_LEN(m); 1720 u_int error = EBUSY, lim = kring->nkr_num_slots - 1; 1721 struct netmap_slot *slot; 1722 1723 if (netmap_verbose & NM_VERB_HOST) 1724 D("%s packet %d len %d from the stack", ifp->if_xname, 1725 kring->nr_hwcur + kring->nr_hwavail, len); 1726 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1727 if (kring->nr_hwavail >= lim) { 1728 if (netmap_verbose) 1729 D("stack ring %s full\n", ifp->if_xname); 1730 goto done; /* no space */ 1731 } 1732 if (len > NETMAP_BUF_SIZE) { 1733 D("%s from_host, drop packet size %d > %d", ifp->if_xname, 1734 len, NETMAP_BUF_SIZE); 1735 goto done; /* too long for us */ 1736 } 1737 1738 /* compute the insert position */ 1739 i = kring->nr_hwcur + kring->nr_hwavail; 1740 if (i > lim) 1741 i -= lim + 1; 1742 slot = &kring->ring->slot[i]; 1743 m_copydata(m, 0, len, NMB(slot)); 1744 slot->len = len; 1745 slot->flags = kring->nkr_slot_flags; 1746 kring->nr_hwavail++; 1747 if (netmap_verbose & NM_VERB_HOST) 1748 D("wake up host ring %s %d", na->ifp->if_xname, na->num_rx_rings); 1749 selwakeuppri(&kring->si, PI_NET); 1750 error = 0; 1751 done: 1752 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 1753 1754 /* release the mbuf in either cases of success or failure. As an 1755 * alternative, put the mbuf in a free list and free the list 1756 * only when really necessary. 1757 */ 1758 m_freem(m); 1759 1760 return (error); 1761 } 1762 1763 1764 /* 1765 * netmap_reset() is called by the driver routines when reinitializing 1766 * a ring. The driver is in charge of locking to protect the kring. 1767 * If netmap mode is not set just return NULL. 1768 */ 1769 struct netmap_slot * 1770 netmap_reset(struct netmap_adapter *na, enum txrx tx, int n, 1771 u_int new_cur) 1772 { 1773 struct netmap_kring *kring; 1774 int new_hwofs, lim; 1775 1776 if (na == NULL) 1777 return NULL; /* no netmap support here */ 1778 if (!(na->ifp->if_capenable & IFCAP_NETMAP)) 1779 return NULL; /* nothing to reinitialize */ 1780 1781 if (tx == NR_TX) { 1782 if (n >= na->num_tx_rings) 1783 return NULL; 1784 kring = na->tx_rings + n; 1785 new_hwofs = kring->nr_hwcur - new_cur; 1786 } else { 1787 if (n >= na->num_rx_rings) 1788 return NULL; 1789 kring = na->rx_rings + n; 1790 new_hwofs = kring->nr_hwcur + kring->nr_hwavail - new_cur; 1791 } 1792 lim = kring->nkr_num_slots - 1; 1793 if (new_hwofs > lim) 1794 new_hwofs -= lim + 1; 1795 1796 /* Alwayws set the new offset value and realign the ring. */ 1797 kring->nkr_hwofs = new_hwofs; 1798 if (tx == NR_TX) 1799 kring->nr_hwavail = kring->nkr_num_slots - 1; 1800 ND(10, "new hwofs %d on %s %s[%d]", 1801 kring->nkr_hwofs, na->ifp->if_xname, 1802 tx == NR_TX ? "TX" : "RX", n); 1803 1804 #if 0 // def linux 1805 /* XXX check that the mappings are correct */ 1806 /* need ring_nr, adapter->pdev, direction */ 1807 buffer_info->dma = dma_map_single(&pdev->dev, addr, adapter->rx_buffer_len, DMA_FROM_DEVICE); 1808 if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 1809 D("error mapping rx netmap buffer %d", i); 1810 // XXX fix error handling 1811 } 1812 1813 #endif /* linux */ 1814 /* 1815 * Wakeup on the individual and global lock 1816 * We do the wakeup here, but the ring is not yet reconfigured. 1817 * However, we are under lock so there are no races. 1818 */ 1819 selwakeuppri(&kring->si, PI_NET); 1820 selwakeuppri(tx == NR_TX ? &na->tx_si : &na->rx_si, PI_NET); 1821 return kring->ring->slot; 1822 } 1823 1824 1825 /* 1826 * Default functions to handle rx/tx interrupts 1827 * we have 4 cases: 1828 * 1 ring, single lock: 1829 * lock(core); wake(i=0); unlock(core) 1830 * N rings, single lock: 1831 * lock(core); wake(i); wake(N+1) unlock(core) 1832 * 1 ring, separate locks: (i=0) 1833 * lock(i); wake(i); unlock(i) 1834 * N rings, separate locks: 1835 * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core) 1836 * work_done is non-null on the RX path. 1837 */ 1838 int 1839 netmap_rx_irq(struct ifnet *ifp, int q, int *work_done) 1840 { 1841 struct netmap_adapter *na; 1842 struct netmap_kring *r; 1843 NM_SELINFO_T *main_wq; 1844 1845 if (!(ifp->if_capenable & IFCAP_NETMAP)) 1846 return 0; 1847 ND(5, "received %s queue %d", work_done ? "RX" : "TX" , q); 1848 na = NA(ifp); 1849 if (na->na_flags & NAF_SKIP_INTR) { 1850 ND("use regular interrupt"); 1851 return 0; 1852 } 1853 1854 if (work_done) { /* RX path */ 1855 if (q >= na->num_rx_rings) 1856 return 0; // regular queue 1857 r = na->rx_rings + q; 1858 r->nr_kflags |= NKR_PENDINTR; 1859 main_wq = (na->num_rx_rings > 1) ? &na->rx_si : NULL; 1860 } else { /* tx path */ 1861 if (q >= na->num_tx_rings) 1862 return 0; // regular queue 1863 r = na->tx_rings + q; 1864 main_wq = (na->num_tx_rings > 1) ? &na->tx_si : NULL; 1865 work_done = &q; /* dummy */ 1866 } 1867 if (na->separate_locks) { 1868 mtx_lock(&r->q_lock); 1869 selwakeuppri(&r->si, PI_NET); 1870 mtx_unlock(&r->q_lock); 1871 if (main_wq) { 1872 mtx_lock(&na->core_lock); 1873 selwakeuppri(main_wq, PI_NET); 1874 mtx_unlock(&na->core_lock); 1875 } 1876 } else { 1877 mtx_lock(&na->core_lock); 1878 selwakeuppri(&r->si, PI_NET); 1879 if (main_wq) 1880 selwakeuppri(main_wq, PI_NET); 1881 mtx_unlock(&na->core_lock); 1882 } 1883 *work_done = 1; /* do not fire napi again */ 1884 return 1; 1885 } 1886 1887 1888 #ifdef linux /* linux-specific routines */ 1889 1890 /* 1891 * Remap linux arguments into the FreeBSD call. 1892 * - pwait is the poll table, passed as 'dev'; 1893 * If pwait == NULL someone else already woke up before. We can report 1894 * events but they are filtered upstream. 1895 * If pwait != NULL, then pwait->key contains the list of events. 1896 * - events is computed from pwait as above. 1897 * - file is passed as 'td'; 1898 */ 1899 static u_int 1900 linux_netmap_poll(struct file * file, struct poll_table_struct *pwait) 1901 { 1902 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0) 1903 int events = pwait ? pwait->key : POLLIN | POLLOUT; 1904 #else /* in 3.4.0 field 'key' was renamed to '_key' */ 1905 int events = pwait ? pwait->_key : POLLIN | POLLOUT; 1906 #endif 1907 return netmap_poll((void *)pwait, events, (void *)file); 1908 } 1909 1910 static int 1911 linux_netmap_mmap(struct file *f, struct vm_area_struct *vma) 1912 { 1913 int lut_skip, i, j; 1914 int user_skip = 0; 1915 struct lut_entry *l_entry; 1916 int error = 0; 1917 unsigned long off, tomap; 1918 /* 1919 * vma->vm_start: start of mapping user address space 1920 * vma->vm_end: end of the mapping user address space 1921 * vma->vm_pfoff: offset of first page in the device 1922 */ 1923 1924 // XXX security checks 1925 1926 error = netmap_get_memory(f->private_data); 1927 ND("get_memory returned %d", error); 1928 if (error) 1929 return -error; 1930 1931 off = vma->vm_pgoff << PAGE_SHIFT; /* offset in bytes */ 1932 tomap = vma->vm_end - vma->vm_start; 1933 for (i = 0; i < NETMAP_POOLS_NR; i++) { /* loop through obj_pools */ 1934 const struct netmap_obj_pool *p = &nm_mem.pools[i]; 1935 /* 1936 * In each pool memory is allocated in clusters 1937 * of size _clustsize, each containing clustentries 1938 * entries. For each object k we already store the 1939 * vtophys mapping in lut[k] so we use that, scanning 1940 * the lut[] array in steps of clustentries, 1941 * and we map each cluster (not individual pages, 1942 * it would be overkill). 1943 */ 1944 1945 /* 1946 * We interpret vm_pgoff as an offset into the whole 1947 * netmap memory, as if all clusters where contiguous. 1948 */ 1949 for (lut_skip = 0, j = 0; j < p->_numclusters; j++, lut_skip += p->clustentries) { 1950 unsigned long paddr, mapsize; 1951 if (p->_clustsize <= off) { 1952 off -= p->_clustsize; 1953 continue; 1954 } 1955 l_entry = &p->lut[lut_skip]; /* first obj in the cluster */ 1956 paddr = l_entry->paddr + off; 1957 mapsize = p->_clustsize - off; 1958 off = 0; 1959 if (mapsize > tomap) 1960 mapsize = tomap; 1961 ND("remap_pfn_range(%lx, %lx, %lx)", 1962 vma->vm_start + user_skip, 1963 paddr >> PAGE_SHIFT, mapsize); 1964 if (remap_pfn_range(vma, vma->vm_start + user_skip, 1965 paddr >> PAGE_SHIFT, mapsize, 1966 vma->vm_page_prot)) 1967 return -EAGAIN; // XXX check return value 1968 user_skip += mapsize; 1969 tomap -= mapsize; 1970 if (tomap == 0) 1971 goto done; 1972 } 1973 } 1974 done: 1975 1976 return 0; 1977 } 1978 1979 static netdev_tx_t 1980 linux_netmap_start(struct sk_buff *skb, struct net_device *dev) 1981 { 1982 netmap_start(dev, skb); 1983 return (NETDEV_TX_OK); 1984 } 1985 1986 1987 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) // XXX was 38 1988 #define LIN_IOCTL_NAME .ioctl 1989 int 1990 linux_netmap_ioctl(struct inode *inode, struct file *file, u_int cmd, u_long data /* arg */) 1991 #else 1992 #define LIN_IOCTL_NAME .unlocked_ioctl 1993 long 1994 linux_netmap_ioctl(struct file *file, u_int cmd, u_long data /* arg */) 1995 #endif 1996 { 1997 int ret; 1998 struct nmreq nmr; 1999 bzero(&nmr, sizeof(nmr)); 2000 2001 if (data && copy_from_user(&nmr, (void *)data, sizeof(nmr) ) != 0) 2002 return -EFAULT; 2003 ret = netmap_ioctl(NULL, cmd, (caddr_t)&nmr, 0, (void *)file); 2004 if (data && copy_to_user((void*)data, &nmr, sizeof(nmr) ) != 0) 2005 return -EFAULT; 2006 return -ret; 2007 } 2008 2009 2010 static int 2011 netmap_release(struct inode *inode, struct file *file) 2012 { 2013 (void)inode; /* UNUSED */ 2014 if (file->private_data) 2015 netmap_dtor(file->private_data); 2016 return (0); 2017 } 2018 2019 static int 2020 linux_netmap_open(struct inode *inode, struct file *file) 2021 { 2022 struct netmap_priv_d *priv; 2023 (void)inode; /* UNUSED */ 2024 2025 priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 2026 M_NOWAIT | M_ZERO); 2027 if (priv == NULL) 2028 return -ENOMEM; 2029 2030 file->private_data = priv; 2031 2032 return (0); 2033 } 2034 2035 static struct file_operations netmap_fops = { 2036 .open = linux_netmap_open, 2037 .mmap = linux_netmap_mmap, 2038 LIN_IOCTL_NAME = linux_netmap_ioctl, 2039 .poll = linux_netmap_poll, 2040 .release = netmap_release, 2041 }; 2042 2043 static struct miscdevice netmap_cdevsw = { /* same name as FreeBSD */ 2044 MISC_DYNAMIC_MINOR, 2045 "netmap", 2046 &netmap_fops, 2047 }; 2048 2049 static int netmap_init(void); 2050 static void netmap_fini(void); 2051 2052 /* Errors have negative values on linux */ 2053 static int linux_netmap_init(void) 2054 { 2055 return -netmap_init(); 2056 } 2057 2058 module_init(linux_netmap_init); 2059 module_exit(netmap_fini); 2060 /* export certain symbols to other modules */ 2061 EXPORT_SYMBOL(netmap_attach); // driver attach routines 2062 EXPORT_SYMBOL(netmap_detach); // driver detach routines 2063 EXPORT_SYMBOL(netmap_ring_reinit); // ring init on error 2064 EXPORT_SYMBOL(netmap_buffer_lut); 2065 EXPORT_SYMBOL(netmap_total_buffers); // index check 2066 EXPORT_SYMBOL(netmap_buffer_base); 2067 EXPORT_SYMBOL(netmap_reset); // ring init routines 2068 EXPORT_SYMBOL(netmap_buf_size); 2069 EXPORT_SYMBOL(netmap_rx_irq); // default irq handler 2070 EXPORT_SYMBOL(netmap_no_pendintr); // XXX mitigation - should go away 2071 2072 2073 MODULE_AUTHOR("http://info.iet.unipi.it/~luigi/netmap/"); 2074 MODULE_DESCRIPTION("The netmap packet I/O framework"); 2075 MODULE_LICENSE("Dual BSD/GPL"); /* the code here is all BSD. */ 2076 2077 #else /* __FreeBSD__ */ 2078 2079 static struct cdevsw netmap_cdevsw = { 2080 .d_version = D_VERSION, 2081 .d_name = "netmap", 2082 .d_open = netmap_open, 2083 .d_mmap = netmap_mmap, 2084 .d_mmap_single = netmap_mmap_single, 2085 .d_ioctl = netmap_ioctl, 2086 .d_poll = netmap_poll, 2087 .d_close = netmap_close, 2088 }; 2089 #endif /* __FreeBSD__ */ 2090 2091 #ifdef NM_BRIDGE 2092 /* 2093 *---- support for virtual bridge ----- 2094 */ 2095 2096 /* ----- FreeBSD if_bridge hash function ------- */ 2097 2098 /* 2099 * The following hash function is adapted from "Hash Functions" by Bob Jenkins 2100 * ("Algorithm Alley", Dr. Dobbs Journal, September 1997). 2101 * 2102 * http://www.burtleburtle.net/bob/hash/spooky.html 2103 */ 2104 #define mix(a, b, c) \ 2105 do { \ 2106 a -= b; a -= c; a ^= (c >> 13); \ 2107 b -= c; b -= a; b ^= (a << 8); \ 2108 c -= a; c -= b; c ^= (b >> 13); \ 2109 a -= b; a -= c; a ^= (c >> 12); \ 2110 b -= c; b -= a; b ^= (a << 16); \ 2111 c -= a; c -= b; c ^= (b >> 5); \ 2112 a -= b; a -= c; a ^= (c >> 3); \ 2113 b -= c; b -= a; b ^= (a << 10); \ 2114 c -= a; c -= b; c ^= (b >> 15); \ 2115 } while (/*CONSTCOND*/0) 2116 2117 static __inline uint32_t 2118 nm_bridge_rthash(const uint8_t *addr) 2119 { 2120 uint32_t a = 0x9e3779b9, b = 0x9e3779b9, c = 0; // hask key 2121 2122 b += addr[5] << 8; 2123 b += addr[4]; 2124 a += addr[3] << 24; 2125 a += addr[2] << 16; 2126 a += addr[1] << 8; 2127 a += addr[0]; 2128 2129 mix(a, b, c); 2130 #define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1) 2131 return (c & BRIDGE_RTHASH_MASK); 2132 } 2133 2134 #undef mix 2135 2136 2137 static int 2138 bdg_netmap_reg(struct ifnet *ifp, int onoff) 2139 { 2140 int i, err = 0; 2141 struct nm_bridge *b = ifp->if_bridge; 2142 2143 BDG_LOCK(b); 2144 if (onoff) { 2145 /* the interface must be already in the list. 2146 * only need to mark the port as active 2147 */ 2148 ND("should attach %s to the bridge", ifp->if_xname); 2149 for (i=0; i < NM_BDG_MAXPORTS; i++) 2150 if (b->bdg_ports[i] == ifp) 2151 break; 2152 if (i == NM_BDG_MAXPORTS) { 2153 D("no more ports available"); 2154 err = EINVAL; 2155 goto done; 2156 } 2157 ND("setting %s in netmap mode", ifp->if_xname); 2158 ifp->if_capenable |= IFCAP_NETMAP; 2159 NA(ifp)->bdg_port = i; 2160 b->act_ports |= (1<<i); 2161 b->bdg_ports[i] = ifp; 2162 } else { 2163 /* should be in the list, too -- remove from the mask */ 2164 ND("removing %s from netmap mode", ifp->if_xname); 2165 ifp->if_capenable &= ~IFCAP_NETMAP; 2166 i = NA(ifp)->bdg_port; 2167 b->act_ports &= ~(1<<i); 2168 } 2169 done: 2170 BDG_UNLOCK(b); 2171 return err; 2172 } 2173 2174 2175 static int 2176 nm_bdg_flush(struct nm_bdg_fwd *ft, int n, struct ifnet *ifp) 2177 { 2178 int i, ifn; 2179 uint64_t all_dst, dst; 2180 uint32_t sh, dh; 2181 uint64_t mysrc = 1 << NA(ifp)->bdg_port; 2182 uint64_t smac, dmac; 2183 struct netmap_slot *slot; 2184 struct nm_bridge *b = ifp->if_bridge; 2185 2186 ND("prepare to send %d packets, act_ports 0x%x", n, b->act_ports); 2187 /* only consider valid destinations */ 2188 all_dst = (b->act_ports & ~mysrc); 2189 /* first pass: hash and find destinations */ 2190 for (i = 0; likely(i < n); i++) { 2191 uint8_t *buf = ft[i].buf; 2192 dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff; 2193 smac = le64toh(*(uint64_t *)(buf + 4)); 2194 smac >>= 16; 2195 if (unlikely(netmap_verbose)) { 2196 uint8_t *s = buf+6, *d = buf; 2197 D("%d len %4d %02x:%02x:%02x:%02x:%02x:%02x -> %02x:%02x:%02x:%02x:%02x:%02x", 2198 i, 2199 ft[i].len, 2200 s[0], s[1], s[2], s[3], s[4], s[5], 2201 d[0], d[1], d[2], d[3], d[4], d[5]); 2202 } 2203 /* 2204 * The hash is somewhat expensive, there might be some 2205 * worthwhile optimizations here. 2206 */ 2207 if ((buf[6] & 1) == 0) { /* valid src */ 2208 uint8_t *s = buf+6; 2209 sh = nm_bridge_rthash(buf+6); // XXX hash of source 2210 /* update source port forwarding entry */ 2211 b->ht[sh].mac = smac; /* XXX expire ? */ 2212 b->ht[sh].ports = mysrc; 2213 if (netmap_verbose) 2214 D("src %02x:%02x:%02x:%02x:%02x:%02x on port %d", 2215 s[0], s[1], s[2], s[3], s[4], s[5], NA(ifp)->bdg_port); 2216 } 2217 dst = 0; 2218 if ( (buf[0] & 1) == 0) { /* unicast */ 2219 uint8_t *d = buf; 2220 dh = nm_bridge_rthash(buf); // XXX hash of dst 2221 if (b->ht[dh].mac == dmac) { /* found dst */ 2222 dst = b->ht[dh].ports; 2223 if (netmap_verbose) 2224 D("dst %02x:%02x:%02x:%02x:%02x:%02x to port %x", 2225 d[0], d[1], d[2], d[3], d[4], d[5], (uint32_t)(dst >> 16)); 2226 } 2227 } 2228 if (dst == 0) 2229 dst = all_dst; 2230 dst &= all_dst; /* only consider valid ports */ 2231 if (unlikely(netmap_verbose)) 2232 D("pkt goes to ports 0x%x", (uint32_t)dst); 2233 ft[i].dst = dst; 2234 } 2235 2236 /* second pass, scan interfaces and forward */ 2237 all_dst = (b->act_ports & ~mysrc); 2238 for (ifn = 0; all_dst; ifn++) { 2239 struct ifnet *dst_ifp = b->bdg_ports[ifn]; 2240 struct netmap_adapter *na; 2241 struct netmap_kring *kring; 2242 struct netmap_ring *ring; 2243 int j, lim, sent, locked; 2244 2245 if (!dst_ifp) 2246 continue; 2247 ND("scan port %d %s", ifn, dst_ifp->if_xname); 2248 dst = 1 << ifn; 2249 if ((dst & all_dst) == 0) /* skip if not set */ 2250 continue; 2251 all_dst &= ~dst; /* clear current node */ 2252 na = NA(dst_ifp); 2253 2254 ring = NULL; 2255 kring = NULL; 2256 lim = sent = locked = 0; 2257 /* inside, scan slots */ 2258 for (i = 0; likely(i < n); i++) { 2259 if ((ft[i].dst & dst) == 0) 2260 continue; /* not here */ 2261 if (!locked) { 2262 kring = &na->rx_rings[0]; 2263 ring = kring->ring; 2264 lim = kring->nkr_num_slots - 1; 2265 na->nm_lock(dst_ifp, NETMAP_RX_LOCK, 0); 2266 locked = 1; 2267 } 2268 if (unlikely(kring->nr_hwavail >= lim)) { 2269 if (netmap_verbose) 2270 D("rx ring full on %s", ifp->if_xname); 2271 break; 2272 } 2273 j = kring->nr_hwcur + kring->nr_hwavail; 2274 if (j > lim) 2275 j -= kring->nkr_num_slots; 2276 slot = &ring->slot[j]; 2277 ND("send %d %d bytes at %s:%d", i, ft[i].len, dst_ifp->if_xname, j); 2278 pkt_copy(ft[i].buf, NMB(slot), ft[i].len); 2279 slot->len = ft[i].len; 2280 kring->nr_hwavail++; 2281 sent++; 2282 } 2283 if (locked) { 2284 ND("sent %d on %s", sent, dst_ifp->if_xname); 2285 if (sent) 2286 selwakeuppri(&kring->si, PI_NET); 2287 na->nm_lock(dst_ifp, NETMAP_RX_UNLOCK, 0); 2288 } 2289 } 2290 return 0; 2291 } 2292 2293 /* 2294 * main dispatch routine 2295 */ 2296 static int 2297 bdg_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 2298 { 2299 struct netmap_adapter *na = NA(ifp); 2300 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 2301 struct netmap_ring *ring = kring->ring; 2302 int i, j, k, lim = kring->nkr_num_slots - 1; 2303 struct nm_bdg_fwd *ft = (struct nm_bdg_fwd *)(ifp + 1); 2304 int ft_i; /* position in the forwarding table */ 2305 2306 k = ring->cur; 2307 if (k > lim) 2308 return netmap_ring_reinit(kring); 2309 if (do_lock) 2310 na->nm_lock(ifp, NETMAP_TX_LOCK, ring_nr); 2311 2312 if (netmap_bridge <= 0) { /* testing only */ 2313 j = k; // used all 2314 goto done; 2315 } 2316 if (netmap_bridge > NM_BDG_BATCH) 2317 netmap_bridge = NM_BDG_BATCH; 2318 2319 ft_i = 0; /* start from 0 */ 2320 for (j = kring->nr_hwcur; likely(j != k); j = unlikely(j == lim) ? 0 : j+1) { 2321 struct netmap_slot *slot = &ring->slot[j]; 2322 int len = ft[ft_i].len = slot->len; 2323 char *buf = ft[ft_i].buf = NMB(slot); 2324 2325 prefetch(buf); 2326 if (unlikely(len < 14)) 2327 continue; 2328 if (unlikely(++ft_i == netmap_bridge)) 2329 ft_i = nm_bdg_flush(ft, ft_i, ifp); 2330 } 2331 if (ft_i) 2332 ft_i = nm_bdg_flush(ft, ft_i, ifp); 2333 /* count how many packets we sent */ 2334 i = k - j; 2335 if (i < 0) 2336 i += kring->nkr_num_slots; 2337 kring->nr_hwavail = kring->nkr_num_slots - 1 - i; 2338 if (j != k) 2339 D("early break at %d/ %d, avail %d", j, k, kring->nr_hwavail); 2340 2341 done: 2342 kring->nr_hwcur = j; 2343 ring->avail = kring->nr_hwavail; 2344 if (do_lock) 2345 na->nm_lock(ifp, NETMAP_TX_UNLOCK, ring_nr); 2346 2347 if (netmap_verbose) 2348 D("%s ring %d lock %d", ifp->if_xname, ring_nr, do_lock); 2349 return 0; 2350 } 2351 2352 static int 2353 bdg_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 2354 { 2355 struct netmap_adapter *na = NA(ifp); 2356 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 2357 struct netmap_ring *ring = kring->ring; 2358 u_int j, n, lim = kring->nkr_num_slots - 1; 2359 u_int k = ring->cur, resvd = ring->reserved; 2360 2361 ND("%s ring %d lock %d avail %d", 2362 ifp->if_xname, ring_nr, do_lock, kring->nr_hwavail); 2363 2364 if (k > lim) 2365 return netmap_ring_reinit(kring); 2366 if (do_lock) 2367 na->nm_lock(ifp, NETMAP_RX_LOCK, ring_nr); 2368 2369 /* skip past packets that userspace has released */ 2370 j = kring->nr_hwcur; /* netmap ring index */ 2371 if (resvd > 0) { 2372 if (resvd + ring->avail >= lim + 1) { 2373 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 2374 ring->reserved = resvd = 0; // XXX panic... 2375 } 2376 k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; 2377 } 2378 2379 if (j != k) { /* userspace has released some packets. */ 2380 n = k - j; 2381 if (n < 0) 2382 n += kring->nkr_num_slots; 2383 ND("userspace releases %d packets", n); 2384 for (n = 0; likely(j != k); n++) { 2385 struct netmap_slot *slot = &ring->slot[j]; 2386 void *addr = NMB(slot); 2387 2388 if (addr == netmap_buffer_base) { /* bad buf */ 2389 if (do_lock) 2390 na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr); 2391 return netmap_ring_reinit(kring); 2392 } 2393 /* decrease refcount for buffer */ 2394 2395 slot->flags &= ~NS_BUF_CHANGED; 2396 j = unlikely(j == lim) ? 0 : j + 1; 2397 } 2398 kring->nr_hwavail -= n; 2399 kring->nr_hwcur = k; 2400 } 2401 /* tell userspace that there are new packets */ 2402 ring->avail = kring->nr_hwavail - resvd; 2403 2404 if (do_lock) 2405 na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr); 2406 return 0; 2407 } 2408 2409 static void 2410 bdg_netmap_attach(struct ifnet *ifp) 2411 { 2412 struct netmap_adapter na; 2413 2414 ND("attaching virtual bridge"); 2415 bzero(&na, sizeof(na)); 2416 2417 na.ifp = ifp; 2418 na.separate_locks = 1; 2419 na.num_tx_desc = NM_BRIDGE_RINGSIZE; 2420 na.num_rx_desc = NM_BRIDGE_RINGSIZE; 2421 na.nm_txsync = bdg_netmap_txsync; 2422 na.nm_rxsync = bdg_netmap_rxsync; 2423 na.nm_register = bdg_netmap_reg; 2424 netmap_attach(&na, 1); 2425 } 2426 2427 #endif /* NM_BRIDGE */ 2428 2429 static struct cdev *netmap_dev; /* /dev/netmap character device. */ 2430 2431 2432 /* 2433 * Module loader. 2434 * 2435 * Create the /dev/netmap device and initialize all global 2436 * variables. 2437 * 2438 * Return 0 on success, errno on failure. 2439 */ 2440 static int 2441 netmap_init(void) 2442 { 2443 int error; 2444 2445 error = netmap_memory_init(); 2446 if (error != 0) { 2447 printf("netmap: unable to initialize the memory allocator.\n"); 2448 return (error); 2449 } 2450 printf("netmap: loaded module\n"); 2451 netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660, 2452 "netmap"); 2453 2454 #ifdef NM_BRIDGE 2455 { 2456 int i; 2457 for (i = 0; i < NM_BRIDGES; i++) 2458 mtx_init(&nm_bridges[i].bdg_lock, "bdg lock", "bdg_lock", MTX_DEF); 2459 } 2460 #endif 2461 return (error); 2462 } 2463 2464 2465 /* 2466 * Module unloader. 2467 * 2468 * Free all the memory, and destroy the ``/dev/netmap`` device. 2469 */ 2470 static void 2471 netmap_fini(void) 2472 { 2473 destroy_dev(netmap_dev); 2474 netmap_memory_fini(); 2475 printf("netmap: unloaded module.\n"); 2476 } 2477 2478 2479 #ifdef __FreeBSD__ 2480 /* 2481 * Kernel entry point. 2482 * 2483 * Initialize/finalize the module and return. 2484 * 2485 * Return 0 on success, errno on failure. 2486 */ 2487 static int 2488 netmap_loader(__unused struct module *module, int event, __unused void *arg) 2489 { 2490 int error = 0; 2491 2492 switch (event) { 2493 case MOD_LOAD: 2494 error = netmap_init(); 2495 break; 2496 2497 case MOD_UNLOAD: 2498 netmap_fini(); 2499 break; 2500 2501 default: 2502 error = EOPNOTSUPP; 2503 break; 2504 } 2505 2506 return (error); 2507 } 2508 2509 2510 DEV_MODULE(netmap, netmap_loader, NULL); 2511 #endif /* __FreeBSD__ */ 2512