1 /* 2 * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #define NM_BRIDGE 27 28 /* 29 * This module supports memory mapped access to network devices, 30 * see netmap(4). 31 * 32 * The module uses a large, memory pool allocated by the kernel 33 * and accessible as mmapped memory by multiple userspace threads/processes. 34 * The memory pool contains packet buffers and "netmap rings", 35 * i.e. user-accessible copies of the interface's queues. 36 * 37 * Access to the network card works like this: 38 * 1. a process/thread issues one or more open() on /dev/netmap, to create 39 * select()able file descriptor on which events are reported. 40 * 2. on each descriptor, the process issues an ioctl() to identify 41 * the interface that should report events to the file descriptor. 42 * 3. on each descriptor, the process issues an mmap() request to 43 * map the shared memory region within the process' address space. 44 * The list of interesting queues is indicated by a location in 45 * the shared memory region. 46 * 4. using the functions in the netmap(4) userspace API, a process 47 * can look up the occupation state of a queue, access memory buffers, 48 * and retrieve received packets or enqueue packets to transmit. 49 * 5. using some ioctl()s the process can synchronize the userspace view 50 * of the queue with the actual status in the kernel. This includes both 51 * receiving the notification of new packets, and transmitting new 52 * packets on the output interface. 53 * 6. select() or poll() can be used to wait for events on individual 54 * transmit or receive queues (or all queues for a given interface). 55 */ 56 57 #ifdef linux 58 #include "bsd_glue.h" 59 static netdev_tx_t linux_netmap_start(struct sk_buff *skb, struct net_device *dev); 60 #endif /* linux */ 61 62 #ifdef __APPLE__ 63 #include "osx_glue.h" 64 #endif /* __APPLE__ */ 65 66 #ifdef __FreeBSD__ 67 #include <sys/cdefs.h> /* prerequisite */ 68 __FBSDID("$FreeBSD$"); 69 70 #include <sys/types.h> 71 #include <sys/module.h> 72 #include <sys/errno.h> 73 #include <sys/param.h> /* defines used in kernel.h */ 74 #include <sys/jail.h> 75 #include <sys/kernel.h> /* types used in module initialization */ 76 #include <sys/conf.h> /* cdevsw struct */ 77 #include <sys/uio.h> /* uio struct */ 78 #include <sys/sockio.h> 79 #include <sys/socketvar.h> /* struct socket */ 80 #include <sys/malloc.h> 81 #include <sys/mman.h> /* PROT_EXEC */ 82 #include <sys/poll.h> 83 #include <sys/proc.h> 84 #include <sys/rwlock.h> 85 #include <vm/vm.h> /* vtophys */ 86 #include <vm/pmap.h> /* vtophys */ 87 #include <sys/socket.h> /* sockaddrs */ 88 #include <machine/bus.h> 89 #include <sys/selinfo.h> 90 #include <sys/sysctl.h> 91 #include <net/if.h> 92 #include <net/if_var.h> 93 #include <net/bpf.h> /* BIOCIMMEDIATE */ 94 #include <net/vnet.h> 95 #include <machine/bus.h> /* bus_dmamap_* */ 96 97 MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map"); 98 #endif /* __FreeBSD__ */ 99 100 #include <net/netmap.h> 101 #include <dev/netmap/netmap_kern.h> 102 103 /* XXX the following variables must be deprecated and included in nm_mem */ 104 u_int netmap_total_buffers; 105 u_int netmap_buf_size; 106 char *netmap_buffer_base; /* address of an invalid buffer */ 107 108 /* user-controlled variables */ 109 int netmap_verbose; 110 111 static int netmap_no_timestamp; /* don't timestamp on rxsync */ 112 113 SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args"); 114 SYSCTL_INT(_dev_netmap, OID_AUTO, verbose, 115 CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode"); 116 SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp, 117 CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp"); 118 int netmap_mitigate = 1; 119 SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, ""); 120 int netmap_no_pendintr = 1; 121 SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr, 122 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets."); 123 int netmap_txsync_retry = 2; 124 SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW, 125 &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush."); 126 127 int netmap_drop = 0; /* debugging */ 128 int netmap_flags = 0; /* debug flags */ 129 int netmap_fwd = 0; /* force transparent mode */ 130 131 SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , ""); 132 SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , ""); 133 SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , ""); 134 135 #ifdef NM_BRIDGE /* support for netmap virtual switch, called VALE */ 136 137 /* 138 * system parameters (most of them in netmap_kern.h) 139 * NM_NAME prefix for switch port names, default "vale" 140 * NM_MAXPORTS number of ports 141 * NM_BRIDGES max number of switches in the system. 142 * XXX should become a sysctl or tunable 143 * 144 * Switch ports are named valeX:Y where X is the switch name and Y 145 * is the port. If Y matches a physical interface name, the port is 146 * connected to a physical device. 147 * 148 * Unlike physical interfaces, switch ports use their own memory region 149 * for rings and buffers. 150 * The virtual interfaces use per-queue lock instead of core lock. 151 * In the tx loop, we aggregate traffic in batches to make all operations 152 * faster. The batch size is NM_BDG_BATCH 153 */ 154 #define NM_BDG_MAXRINGS 16 /* XXX unclear how many. */ 155 #define NM_BRIDGE_RINGSIZE 1024 /* in the device */ 156 #define NM_BDG_HASH 1024 /* forwarding table entries */ 157 #define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */ 158 #define NM_BRIDGES 8 /* number of bridges */ 159 160 161 int netmap_bridge = NM_BDG_BATCH; /* bridge batch size */ 162 SYSCTL_INT(_dev_netmap, OID_AUTO, bridge, CTLFLAG_RW, &netmap_bridge, 0 , ""); 163 164 #ifdef linux 165 166 #define refcount_acquire(_a) atomic_add(1, (atomic_t *)_a) 167 #define refcount_release(_a) atomic_dec_and_test((atomic_t *)_a) 168 169 #else /* !linux */ 170 171 #ifdef __FreeBSD__ 172 #include <sys/endian.h> 173 #include <sys/refcount.h> 174 #endif /* __FreeBSD__ */ 175 176 #define prefetch(x) __builtin_prefetch(x) 177 178 #endif /* !linux */ 179 180 /* 181 * These are used to handle reference counters for bridge ports. 182 */ 183 #define ADD_BDG_REF(ifp) refcount_acquire(&NA(ifp)->na_bdg_refcount) 184 #define DROP_BDG_REF(ifp) refcount_release(&NA(ifp)->na_bdg_refcount) 185 186 static void bdg_netmap_attach(struct netmap_adapter *); 187 static int bdg_netmap_reg(struct ifnet *ifp, int onoff); 188 static int kern_netmap_regif(struct nmreq *nmr); 189 190 /* per-tx-queue entry */ 191 struct nm_bdg_fwd { /* forwarding entry for a bridge */ 192 void *ft_buf; 193 uint16_t _ft_dst; /* dst port, unused */ 194 uint16_t ft_flags; /* flags, e.g. indirect */ 195 uint16_t ft_len; /* src len */ 196 uint16_t ft_next; /* next packet to same destination */ 197 }; 198 199 /* We need to build a list of buffers going to each destination. 200 * Each buffer is in one entry of struct nm_bdg_fwd, we use ft_next 201 * to build the list, and struct nm_bdg_q below for the queue. 202 * The structure should compact because potentially we have a lot 203 * of destinations. 204 */ 205 struct nm_bdg_q { 206 uint16_t bq_head; 207 uint16_t bq_tail; 208 }; 209 210 struct nm_hash_ent { 211 uint64_t mac; /* the top 2 bytes are the epoch */ 212 uint64_t ports; 213 }; 214 215 /* 216 * Interfaces for a bridge are all in bdg_ports[]. 217 * The array has fixed size, an empty entry does not terminate 218 * the search. But lookups only occur on attach/detach so we 219 * don't mind if they are slow. 220 * 221 * The bridge is non blocking on the transmit ports. 222 * 223 * bdg_lock protects accesses to the bdg_ports array. 224 * This is a rw lock (or equivalent). 225 */ 226 struct nm_bridge { 227 int namelen; /* 0 means free */ 228 229 /* XXX what is the proper alignment/layout ? */ 230 NM_RWLOCK_T bdg_lock; /* protects bdg_ports */ 231 struct netmap_adapter *bdg_ports[NM_BDG_MAXPORTS]; 232 233 char basename[IFNAMSIZ]; 234 /* 235 * The function to decide the destination port. 236 * It returns either of an index of the destination port, 237 * NM_BDG_BROADCAST to broadcast this packet, or NM_BDG_NOPORT not to 238 * forward this packet. ring_nr is the source ring index, and the 239 * function may overwrite this value to forward this packet to a 240 * different ring index. 241 * This function must be set by netmap_bdgctl(). 242 */ 243 bdg_lookup_fn_t nm_bdg_lookup; 244 245 /* the forwarding table, MAC+ports */ 246 struct nm_hash_ent ht[NM_BDG_HASH]; 247 }; 248 249 struct nm_bridge nm_bridges[NM_BRIDGES]; 250 NM_LOCK_T netmap_bridge_mutex; 251 252 /* other OS will have these macros defined in their own glue code. */ 253 254 #ifdef __FreeBSD__ 255 #define BDG_LOCK() mtx_lock(&netmap_bridge_mutex) 256 #define BDG_UNLOCK() mtx_unlock(&netmap_bridge_mutex) 257 #define BDG_WLOCK(b) rw_wlock(&(b)->bdg_lock) 258 #define BDG_WUNLOCK(b) rw_wunlock(&(b)->bdg_lock) 259 #define BDG_RLOCK(b) rw_rlock(&(b)->bdg_lock) 260 #define BDG_RUNLOCK(b) rw_runlock(&(b)->bdg_lock) 261 262 /* set/get variables. OS-specific macros may wrap these 263 * assignments into read/write lock or similar 264 */ 265 #define BDG_SET_VAR(lval, p) (lval = p) 266 #define BDG_GET_VAR(lval) (lval) 267 #define BDG_FREE(p) free(p, M_DEVBUF) 268 #endif /* __FreeBSD__ */ 269 270 static __inline int 271 nma_is_vp(struct netmap_adapter *na) 272 { 273 return na->nm_register == bdg_netmap_reg; 274 } 275 static __inline int 276 nma_is_host(struct netmap_adapter *na) 277 { 278 return na->nm_register == NULL; 279 } 280 static __inline int 281 nma_is_hw(struct netmap_adapter *na) 282 { 283 /* In case of sw adapter, nm_register is NULL */ 284 return !nma_is_vp(na) && !nma_is_host(na); 285 } 286 287 /* 288 * Regarding holding a NIC, if the NIC is owned by the kernel 289 * (i.e., bridge), neither another bridge nor user can use it; 290 * if the NIC is owned by a user, only users can share it. 291 * Evaluation must be done under NMA_LOCK(). 292 */ 293 #define NETMAP_OWNED_BY_KERN(ifp) (!nma_is_vp(NA(ifp)) && NA(ifp)->na_bdg) 294 #define NETMAP_OWNED_BY_ANY(ifp) \ 295 (NETMAP_OWNED_BY_KERN(ifp) || (NA(ifp)->refcount > 0)) 296 297 /* 298 * NA(ifp)->bdg_port port index 299 */ 300 301 // XXX only for multiples of 64 bytes, non overlapped. 302 static inline void 303 pkt_copy(void *_src, void *_dst, int l) 304 { 305 uint64_t *src = _src; 306 uint64_t *dst = _dst; 307 if (unlikely(l >= 1024)) { 308 bcopy(src, dst, l); 309 return; 310 } 311 for (; likely(l > 0); l-=64) { 312 *dst++ = *src++; 313 *dst++ = *src++; 314 *dst++ = *src++; 315 *dst++ = *src++; 316 *dst++ = *src++; 317 *dst++ = *src++; 318 *dst++ = *src++; 319 *dst++ = *src++; 320 } 321 } 322 323 324 /* 325 * locate a bridge among the existing ones. 326 * a ':' in the name terminates the bridge name. Otherwise, just NM_NAME. 327 * We assume that this is called with a name of at least NM_NAME chars. 328 */ 329 static struct nm_bridge * 330 nm_find_bridge(const char *name, int create) 331 { 332 int i, l, namelen; 333 struct nm_bridge *b = NULL; 334 335 namelen = strlen(NM_NAME); /* base length */ 336 l = strlen(name); /* actual length */ 337 for (i = namelen + 1; i < l; i++) { 338 if (name[i] == ':') { 339 namelen = i; 340 break; 341 } 342 } 343 if (namelen >= IFNAMSIZ) 344 namelen = IFNAMSIZ; 345 ND("--- prefix is '%.*s' ---", namelen, name); 346 347 BDG_LOCK(); 348 /* lookup the name, remember empty slot if there is one */ 349 for (i = 0; i < NM_BRIDGES; i++) { 350 struct nm_bridge *x = nm_bridges + i; 351 352 if (x->namelen == 0) { 353 if (create && b == NULL) 354 b = x; /* record empty slot */ 355 } else if (x->namelen != namelen) { 356 continue; 357 } else if (strncmp(name, x->basename, namelen) == 0) { 358 ND("found '%.*s' at %d", namelen, name, i); 359 b = x; 360 break; 361 } 362 } 363 if (i == NM_BRIDGES && b) { /* name not found, can create entry */ 364 strncpy(b->basename, name, namelen); 365 b->namelen = namelen; 366 /* set the default function */ 367 b->nm_bdg_lookup = netmap_bdg_learning; 368 /* reset the MAC address table */ 369 bzero(b->ht, sizeof(struct nm_hash_ent) * NM_BDG_HASH); 370 } 371 BDG_UNLOCK(); 372 return b; 373 } 374 375 376 /* 377 * Free the forwarding tables for rings attached to switch ports. 378 */ 379 static void 380 nm_free_bdgfwd(struct netmap_adapter *na) 381 { 382 int nrings, i; 383 struct netmap_kring *kring; 384 385 nrings = nma_is_vp(na) ? na->num_tx_rings : na->num_rx_rings; 386 kring = nma_is_vp(na) ? na->tx_rings : na->rx_rings; 387 for (i = 0; i < nrings; i++) { 388 if (kring[i].nkr_ft) { 389 free(kring[i].nkr_ft, M_DEVBUF); 390 kring[i].nkr_ft = NULL; /* protect from freeing twice */ 391 } 392 } 393 if (nma_is_hw(na)) 394 nm_free_bdgfwd(SWNA(na->ifp)); 395 } 396 397 398 /* 399 * Allocate the forwarding tables for the rings attached to the bridge ports. 400 */ 401 static int 402 nm_alloc_bdgfwd(struct netmap_adapter *na) 403 { 404 int nrings, l, i, num_dstq; 405 struct netmap_kring *kring; 406 407 /* all port:rings + broadcast */ 408 num_dstq = NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1; 409 l = sizeof(struct nm_bdg_fwd) * NM_BDG_BATCH; 410 l += sizeof(struct nm_bdg_q) * num_dstq; 411 l += sizeof(uint16_t) * NM_BDG_BATCH; 412 413 nrings = nma_is_vp(na) ? na->num_tx_rings : na->num_rx_rings; 414 kring = nma_is_vp(na) ? na->tx_rings : na->rx_rings; 415 for (i = 0; i < nrings; i++) { 416 struct nm_bdg_fwd *ft; 417 struct nm_bdg_q *dstq; 418 int j; 419 420 ft = malloc(l, M_DEVBUF, M_NOWAIT | M_ZERO); 421 if (!ft) { 422 nm_free_bdgfwd(na); 423 return ENOMEM; 424 } 425 dstq = (struct nm_bdg_q *)(ft + NM_BDG_BATCH); 426 for (j = 0; j < num_dstq; j++) 427 dstq[j].bq_head = dstq[j].bq_tail = NM_BDG_BATCH; 428 kring[i].nkr_ft = ft; 429 } 430 if (nma_is_hw(na)) 431 nm_alloc_bdgfwd(SWNA(na->ifp)); 432 return 0; 433 } 434 435 #endif /* NM_BRIDGE */ 436 437 438 /* 439 * Fetch configuration from the device, to cope with dynamic 440 * reconfigurations after loading the module. 441 */ 442 static int 443 netmap_update_config(struct netmap_adapter *na) 444 { 445 struct ifnet *ifp = na->ifp; 446 u_int txr, txd, rxr, rxd; 447 448 txr = txd = rxr = rxd = 0; 449 if (na->nm_config) { 450 na->nm_config(ifp, &txr, &txd, &rxr, &rxd); 451 } else { 452 /* take whatever we had at init time */ 453 txr = na->num_tx_rings; 454 txd = na->num_tx_desc; 455 rxr = na->num_rx_rings; 456 rxd = na->num_rx_desc; 457 } 458 459 if (na->num_tx_rings == txr && na->num_tx_desc == txd && 460 na->num_rx_rings == rxr && na->num_rx_desc == rxd) 461 return 0; /* nothing changed */ 462 if (netmap_verbose || na->refcount > 0) { 463 D("stored config %s: txring %d x %d, rxring %d x %d", 464 ifp->if_xname, 465 na->num_tx_rings, na->num_tx_desc, 466 na->num_rx_rings, na->num_rx_desc); 467 D("new config %s: txring %d x %d, rxring %d x %d", 468 ifp->if_xname, txr, txd, rxr, rxd); 469 } 470 if (na->refcount == 0) { 471 D("configuration changed (but fine)"); 472 na->num_tx_rings = txr; 473 na->num_tx_desc = txd; 474 na->num_rx_rings = rxr; 475 na->num_rx_desc = rxd; 476 return 0; 477 } 478 D("configuration changed while active, this is bad..."); 479 return 1; 480 } 481 482 /*------------- memory allocator -----------------*/ 483 #include "netmap_mem2.c" 484 /*------------ end of memory allocator ----------*/ 485 486 487 /* Structure associated to each thread which registered an interface. 488 * 489 * The first 4 fields of this structure are written by NIOCREGIF and 490 * read by poll() and NIOC?XSYNC. 491 * There is low contention among writers (actually, a correct user program 492 * should have no contention among writers) and among writers and readers, 493 * so we use a single global lock to protect the structure initialization. 494 * Since initialization involves the allocation of memory, we reuse the memory 495 * allocator lock. 496 * Read access to the structure is lock free. Readers must check that 497 * np_nifp is not NULL before using the other fields. 498 * If np_nifp is NULL initialization has not been performed, so they should 499 * return an error to userlevel. 500 * 501 * The ref_done field is used to regulate access to the refcount in the 502 * memory allocator. The refcount must be incremented at most once for 503 * each open("/dev/netmap"). The increment is performed by the first 504 * function that calls netmap_get_memory() (currently called by 505 * mmap(), NIOCGINFO and NIOCREGIF). 506 * If the refcount is incremented, it is then decremented when the 507 * private structure is destroyed. 508 */ 509 struct netmap_priv_d { 510 struct netmap_if * volatile np_nifp; /* netmap interface descriptor. */ 511 512 struct ifnet *np_ifp; /* device for which we hold a reference */ 513 int np_ringid; /* from the ioctl */ 514 u_int np_qfirst, np_qlast; /* range of rings to scan */ 515 uint16_t np_txpoll; 516 517 unsigned long ref_done; /* use with NMA_LOCK held */ 518 }; 519 520 521 static int 522 netmap_get_memory(struct netmap_priv_d* p) 523 { 524 int error = 0; 525 NMA_LOCK(); 526 if (!p->ref_done) { 527 error = netmap_memory_finalize(); 528 if (!error) 529 p->ref_done = 1; 530 } 531 NMA_UNLOCK(); 532 return error; 533 } 534 535 /* 536 * File descriptor's private data destructor. 537 * 538 * Call nm_register(ifp,0) to stop netmap mode on the interface and 539 * revert to normal operation. We expect that np_ifp has not gone. 540 */ 541 /* call with NMA_LOCK held */ 542 static void 543 netmap_dtor_locked(void *data) 544 { 545 struct netmap_priv_d *priv = data; 546 struct ifnet *ifp = priv->np_ifp; 547 struct netmap_adapter *na = NA(ifp); 548 struct netmap_if *nifp = priv->np_nifp; 549 550 na->refcount--; 551 if (na->refcount <= 0) { /* last instance */ 552 u_int i, j, lim; 553 554 if (netmap_verbose) 555 D("deleting last instance for %s", ifp->if_xname); 556 /* 557 * (TO CHECK) This function is only called 558 * when the last reference to this file descriptor goes 559 * away. This means we cannot have any pending poll() 560 * or interrupt routine operating on the structure. 561 */ 562 na->nm_register(ifp, 0); /* off, clear IFCAP_NETMAP */ 563 /* Wake up any sleeping threads. netmap_poll will 564 * then return POLLERR 565 */ 566 for (i = 0; i < na->num_tx_rings + 1; i++) 567 selwakeuppri(&na->tx_rings[i].si, PI_NET); 568 for (i = 0; i < na->num_rx_rings + 1; i++) 569 selwakeuppri(&na->rx_rings[i].si, PI_NET); 570 selwakeuppri(&na->tx_si, PI_NET); 571 selwakeuppri(&na->rx_si, PI_NET); 572 #ifdef NM_BRIDGE 573 nm_free_bdgfwd(na); 574 #endif /* NM_BRIDGE */ 575 /* release all buffers */ 576 for (i = 0; i < na->num_tx_rings + 1; i++) { 577 struct netmap_ring *ring = na->tx_rings[i].ring; 578 lim = na->tx_rings[i].nkr_num_slots; 579 for (j = 0; j < lim; j++) 580 netmap_free_buf(nifp, ring->slot[j].buf_idx); 581 /* knlist_destroy(&na->tx_rings[i].si.si_note); */ 582 mtx_destroy(&na->tx_rings[i].q_lock); 583 } 584 for (i = 0; i < na->num_rx_rings + 1; i++) { 585 struct netmap_ring *ring = na->rx_rings[i].ring; 586 lim = na->rx_rings[i].nkr_num_slots; 587 for (j = 0; j < lim; j++) 588 netmap_free_buf(nifp, ring->slot[j].buf_idx); 589 /* knlist_destroy(&na->rx_rings[i].si.si_note); */ 590 mtx_destroy(&na->rx_rings[i].q_lock); 591 } 592 /* XXX kqueue(9) needed; these will mirror knlist_init. */ 593 /* knlist_destroy(&na->tx_si.si_note); */ 594 /* knlist_destroy(&na->rx_si.si_note); */ 595 netmap_free_rings(na); 596 if (nma_is_hw(na)) 597 SWNA(ifp)->tx_rings = SWNA(ifp)->rx_rings = NULL; 598 } 599 netmap_if_free(nifp); 600 } 601 602 603 /* we assume netmap adapter exists */ 604 static void 605 nm_if_rele(struct ifnet *ifp) 606 { 607 #ifndef NM_BRIDGE 608 if_rele(ifp); 609 #else /* NM_BRIDGE */ 610 int i, full = 0, is_hw; 611 struct nm_bridge *b; 612 struct netmap_adapter *na; 613 614 /* I can be called not only for get_ifp()-ed references where netmap's 615 * capability is guaranteed, but also for non-netmap-capable NICs. 616 */ 617 if (!NETMAP_CAPABLE(ifp) || !NA(ifp)->na_bdg) { 618 if_rele(ifp); 619 return; 620 } 621 if (!DROP_BDG_REF(ifp)) 622 return; 623 624 na = NA(ifp); 625 b = na->na_bdg; 626 is_hw = nma_is_hw(na); 627 628 BDG_WLOCK(b); 629 ND("want to disconnect %s from the bridge", ifp->if_xname); 630 full = 0; 631 /* remove the entry from the bridge, also check 632 * if there are any leftover interfaces 633 * XXX we should optimize this code, e.g. going directly 634 * to na->bdg_port, and having a counter of ports that 635 * are connected. But it is not in a critical path. 636 * In NIC's case, index of sw na is always higher than hw na 637 */ 638 for (i = 0; i < NM_BDG_MAXPORTS; i++) { 639 struct netmap_adapter *tmp = BDG_GET_VAR(b->bdg_ports[i]); 640 641 if (tmp == na) { 642 /* disconnect from bridge */ 643 BDG_SET_VAR(b->bdg_ports[i], NULL); 644 na->na_bdg = NULL; 645 if (is_hw && SWNA(ifp)->na_bdg) { 646 /* disconnect sw adapter too */ 647 int j = SWNA(ifp)->bdg_port; 648 BDG_SET_VAR(b->bdg_ports[j], NULL); 649 SWNA(ifp)->na_bdg = NULL; 650 } 651 } else if (tmp != NULL) { 652 full = 1; 653 } 654 } 655 BDG_WUNLOCK(b); 656 if (full == 0) { 657 ND("marking bridge %d as free", b - nm_bridges); 658 b->namelen = 0; 659 b->nm_bdg_lookup = NULL; 660 } 661 if (na->na_bdg) { /* still attached to the bridge */ 662 D("ouch, cannot find ifp to remove"); 663 } else if (is_hw) { 664 if_rele(ifp); 665 } else { 666 bzero(na, sizeof(*na)); 667 free(na, M_DEVBUF); 668 bzero(ifp, sizeof(*ifp)); 669 free(ifp, M_DEVBUF); 670 } 671 #endif /* NM_BRIDGE */ 672 } 673 674 static void 675 netmap_dtor(void *data) 676 { 677 struct netmap_priv_d *priv = data; 678 struct ifnet *ifp = priv->np_ifp; 679 680 NMA_LOCK(); 681 if (ifp) { 682 struct netmap_adapter *na = NA(ifp); 683 684 if (na->na_bdg) 685 BDG_WLOCK(na->na_bdg); 686 na->nm_lock(ifp, NETMAP_REG_LOCK, 0); 687 netmap_dtor_locked(data); 688 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 689 if (na->na_bdg) 690 BDG_WUNLOCK(na->na_bdg); 691 692 nm_if_rele(ifp); /* might also destroy *na */ 693 } 694 if (priv->ref_done) { 695 netmap_memory_deref(); 696 } 697 NMA_UNLOCK(); 698 bzero(priv, sizeof(*priv)); /* XXX for safety */ 699 free(priv, M_DEVBUF); 700 } 701 702 703 #ifdef __FreeBSD__ 704 #include <vm/vm.h> 705 #include <vm/vm_param.h> 706 #include <vm/vm_object.h> 707 #include <vm/vm_page.h> 708 #include <vm/vm_pager.h> 709 #include <vm/uma.h> 710 711 /* 712 * In order to track whether pages are still mapped, we hook into 713 * the standard cdev_pager and intercept the constructor and 714 * destructor. 715 * XXX but then ? Do we really use the information ? 716 * Need to investigate. 717 */ 718 static struct cdev_pager_ops saved_cdev_pager_ops; 719 720 721 static int 722 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 723 vm_ooffset_t foff, struct ucred *cred, u_short *color) 724 { 725 if (netmap_verbose) 726 D("first mmap for %p", handle); 727 return saved_cdev_pager_ops.cdev_pg_ctor(handle, 728 size, prot, foff, cred, color); 729 } 730 731 732 static void 733 netmap_dev_pager_dtor(void *handle) 734 { 735 saved_cdev_pager_ops.cdev_pg_dtor(handle); 736 ND("ready to release memory for %p", handle); 737 } 738 739 740 static struct cdev_pager_ops netmap_cdev_pager_ops = { 741 .cdev_pg_ctor = netmap_dev_pager_ctor, 742 .cdev_pg_dtor = netmap_dev_pager_dtor, 743 .cdev_pg_fault = NULL, 744 }; 745 746 747 // XXX check whether we need netmap_mmap_single _and_ netmap_mmap 748 static int 749 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 750 vm_size_t objsize, vm_object_t *objp, int prot) 751 { 752 vm_object_t obj; 753 754 ND("cdev %p foff %jd size %jd objp %p prot %d", cdev, 755 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 756 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 757 curthread->td_ucred); 758 ND("returns obj %p", obj); 759 if (obj == NULL) 760 return EINVAL; 761 if (saved_cdev_pager_ops.cdev_pg_fault == NULL) { 762 ND("initialize cdev_pager_ops"); 763 saved_cdev_pager_ops = *(obj->un_pager.devp.ops); 764 netmap_cdev_pager_ops.cdev_pg_fault = 765 saved_cdev_pager_ops.cdev_pg_fault; 766 }; 767 obj->un_pager.devp.ops = &netmap_cdev_pager_ops; 768 *objp = obj; 769 return 0; 770 } 771 #endif /* __FreeBSD__ */ 772 773 774 /* 775 * mmap(2) support for the "netmap" device. 776 * 777 * Expose all the memory previously allocated by our custom memory 778 * allocator: this way the user has only to issue a single mmap(2), and 779 * can work on all the data structures flawlessly. 780 * 781 * Return 0 on success, -1 otherwise. 782 */ 783 784 #ifdef __FreeBSD__ 785 static int 786 netmap_mmap(__unused struct cdev *dev, 787 #if __FreeBSD_version < 900000 788 vm_offset_t offset, vm_paddr_t *paddr, int nprot 789 #else 790 vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, 791 __unused vm_memattr_t *memattr 792 #endif 793 ) 794 { 795 int error = 0; 796 struct netmap_priv_d *priv; 797 798 if (nprot & PROT_EXEC) 799 return (-1); // XXX -1 or EINVAL ? 800 801 error = devfs_get_cdevpriv((void **)&priv); 802 if (error == EBADF) { /* called on fault, memory is initialized */ 803 ND(5, "handling fault at ofs 0x%x", offset); 804 error = 0; 805 } else if (error == 0) /* make sure memory is set */ 806 error = netmap_get_memory(priv); 807 if (error) 808 return (error); 809 810 ND("request for offset 0x%x", (uint32_t)offset); 811 *paddr = netmap_ofstophys(offset); 812 813 return (*paddr ? 0 : ENOMEM); 814 } 815 816 817 static int 818 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 819 { 820 if (netmap_verbose) 821 D("dev %p fflag 0x%x devtype %d td %p", 822 dev, fflag, devtype, td); 823 return 0; 824 } 825 826 827 static int 828 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 829 { 830 struct netmap_priv_d *priv; 831 int error; 832 833 priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 834 M_NOWAIT | M_ZERO); 835 if (priv == NULL) 836 return ENOMEM; 837 838 error = devfs_set_cdevpriv(priv, netmap_dtor); 839 if (error) 840 return error; 841 842 return 0; 843 } 844 #endif /* __FreeBSD__ */ 845 846 847 /* 848 * Handlers for synchronization of the queues from/to the host. 849 * Netmap has two operating modes: 850 * - in the default mode, the rings connected to the host stack are 851 * just another ring pair managed by userspace; 852 * - in transparent mode (XXX to be defined) incoming packets 853 * (from the host or the NIC) are marked as NS_FORWARD upon 854 * arrival, and the user application has a chance to reset the 855 * flag for packets that should be dropped. 856 * On the RXSYNC or poll(), packets in RX rings between 857 * kring->nr_kcur and ring->cur with NS_FORWARD still set are moved 858 * to the other side. 859 * The transfer NIC --> host is relatively easy, just encapsulate 860 * into mbufs and we are done. The host --> NIC side is slightly 861 * harder because there might not be room in the tx ring so it 862 * might take a while before releasing the buffer. 863 */ 864 865 866 /* 867 * pass a chain of buffers to the host stack as coming from 'dst' 868 */ 869 static void 870 netmap_send_up(struct ifnet *dst, struct mbuf *head) 871 { 872 struct mbuf *m; 873 874 /* send packets up, outside the lock */ 875 while ((m = head) != NULL) { 876 head = head->m_nextpkt; 877 m->m_nextpkt = NULL; 878 if (netmap_verbose & NM_VERB_HOST) 879 D("sending up pkt %p size %d", m, MBUF_LEN(m)); 880 NM_SEND_UP(dst, m); 881 } 882 } 883 884 struct mbq { 885 struct mbuf *head; 886 struct mbuf *tail; 887 int count; 888 }; 889 890 891 /* 892 * put a copy of the buffers marked NS_FORWARD into an mbuf chain. 893 * Run from hwcur to cur - reserved 894 */ 895 static void 896 netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force) 897 { 898 /* Take packets from hwcur to cur-reserved and pass them up. 899 * In case of no buffers we give up. At the end of the loop, 900 * the queue is drained in all cases. 901 * XXX handle reserved 902 */ 903 int k = kring->ring->cur - kring->ring->reserved; 904 u_int n, lim = kring->nkr_num_slots - 1; 905 struct mbuf *m, *tail = q->tail; 906 907 if (k < 0) 908 k = k + kring->nkr_num_slots; 909 for (n = kring->nr_hwcur; n != k;) { 910 struct netmap_slot *slot = &kring->ring->slot[n]; 911 912 n = (n == lim) ? 0 : n + 1; 913 if ((slot->flags & NS_FORWARD) == 0 && !force) 914 continue; 915 if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE) { 916 D("bad pkt at %d len %d", n, slot->len); 917 continue; 918 } 919 slot->flags &= ~NS_FORWARD; // XXX needed ? 920 m = m_devget(NMB(slot), slot->len, 0, kring->na->ifp, NULL); 921 922 if (m == NULL) 923 break; 924 if (tail) 925 tail->m_nextpkt = m; 926 else 927 q->head = m; 928 tail = m; 929 q->count++; 930 m->m_nextpkt = NULL; 931 } 932 q->tail = tail; 933 } 934 935 936 /* 937 * called under main lock to send packets from the host to the NIC 938 * The host ring has packets from nr_hwcur to (cur - reserved) 939 * to be sent down. We scan the tx rings, which have just been 940 * flushed so nr_hwcur == cur. Pushing packets down means 941 * increment cur and decrement avail. 942 * XXX to be verified 943 */ 944 static void 945 netmap_sw_to_nic(struct netmap_adapter *na) 946 { 947 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 948 struct netmap_kring *k1 = &na->tx_rings[0]; 949 int i, howmany, src_lim, dst_lim; 950 951 howmany = kring->nr_hwavail; /* XXX otherwise cur - reserved - nr_hwcur */ 952 953 src_lim = kring->nkr_num_slots; 954 for (i = 0; howmany > 0 && i < na->num_tx_rings; i++, k1++) { 955 ND("%d packets left to ring %d (space %d)", howmany, i, k1->nr_hwavail); 956 dst_lim = k1->nkr_num_slots; 957 while (howmany > 0 && k1->ring->avail > 0) { 958 struct netmap_slot *src, *dst, tmp; 959 src = &kring->ring->slot[kring->nr_hwcur]; 960 dst = &k1->ring->slot[k1->ring->cur]; 961 tmp = *src; 962 src->buf_idx = dst->buf_idx; 963 src->flags = NS_BUF_CHANGED; 964 965 dst->buf_idx = tmp.buf_idx; 966 dst->len = tmp.len; 967 dst->flags = NS_BUF_CHANGED; 968 ND("out len %d buf %d from %d to %d", 969 dst->len, dst->buf_idx, 970 kring->nr_hwcur, k1->ring->cur); 971 972 if (++kring->nr_hwcur >= src_lim) 973 kring->nr_hwcur = 0; 974 howmany--; 975 kring->nr_hwavail--; 976 if (++k1->ring->cur >= dst_lim) 977 k1->ring->cur = 0; 978 k1->ring->avail--; 979 } 980 kring->ring->cur = kring->nr_hwcur; // XXX 981 k1++; 982 } 983 } 984 985 986 /* 987 * netmap_sync_to_host() passes packets up. We are called from a 988 * system call in user process context, and the only contention 989 * can be among multiple user threads erroneously calling 990 * this routine concurrently. 991 */ 992 static void 993 netmap_sync_to_host(struct netmap_adapter *na) 994 { 995 struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings]; 996 struct netmap_ring *ring = kring->ring; 997 u_int k, lim = kring->nkr_num_slots - 1; 998 struct mbq q = { NULL, NULL }; 999 1000 k = ring->cur; 1001 if (k > lim) { 1002 netmap_ring_reinit(kring); 1003 return; 1004 } 1005 // na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0); 1006 1007 /* Take packets from hwcur to cur and pass them up. 1008 * In case of no buffers we give up. At the end of the loop, 1009 * the queue is drained in all cases. 1010 */ 1011 netmap_grab_packets(kring, &q, 1); 1012 kring->nr_hwcur = k; 1013 kring->nr_hwavail = ring->avail = lim; 1014 // na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0); 1015 1016 netmap_send_up(na->ifp, q.head); 1017 } 1018 1019 1020 /* SWNA(ifp)->txrings[0] is always NA(ifp)->txrings[NA(ifp)->num_txrings] */ 1021 static int 1022 netmap_bdg_to_host(struct ifnet *ifp, u_int ring_nr, int do_lock) 1023 { 1024 (void)ring_nr; 1025 (void)do_lock; 1026 netmap_sync_to_host(NA(ifp)); 1027 return 0; 1028 } 1029 1030 1031 /* 1032 * rxsync backend for packets coming from the host stack. 1033 * They have been put in the queue by netmap_start() so we 1034 * need to protect access to the kring using a lock. 1035 * 1036 * This routine also does the selrecord if called from the poll handler 1037 * (we know because td != NULL). 1038 * 1039 * NOTE: on linux, selrecord() is defined as a macro and uses pwait 1040 * as an additional hidden argument. 1041 */ 1042 static void 1043 netmap_sync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait) 1044 { 1045 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 1046 struct netmap_ring *ring = kring->ring; 1047 u_int j, n, lim = kring->nkr_num_slots; 1048 u_int k = ring->cur, resvd = ring->reserved; 1049 1050 (void)pwait; /* disable unused warnings */ 1051 na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0); 1052 if (k >= lim) { 1053 netmap_ring_reinit(kring); 1054 return; 1055 } 1056 /* new packets are already set in nr_hwavail */ 1057 /* skip past packets that userspace has released */ 1058 j = kring->nr_hwcur; 1059 if (resvd > 0) { 1060 if (resvd + ring->avail >= lim + 1) { 1061 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 1062 ring->reserved = resvd = 0; // XXX panic... 1063 } 1064 k = (k >= resvd) ? k - resvd : k + lim - resvd; 1065 } 1066 if (j != k) { 1067 n = k >= j ? k - j : k + lim - j; 1068 kring->nr_hwavail -= n; 1069 kring->nr_hwcur = k; 1070 } 1071 k = ring->avail = kring->nr_hwavail - resvd; 1072 if (k == 0 && td) 1073 selrecord(td, &kring->si); 1074 if (k && (netmap_verbose & NM_VERB_HOST)) 1075 D("%d pkts from stack", k); 1076 na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0); 1077 } 1078 1079 1080 /* 1081 * get a refcounted reference to an interface. 1082 * Return ENXIO if the interface does not exist, EINVAL if netmap 1083 * is not supported by the interface. 1084 * If successful, hold a reference. 1085 * 1086 * During the NIC is attached to a bridge, reference is managed 1087 * at na->na_bdg_refcount using ADD/DROP_BDG_REF() as well as 1088 * virtual ports. Hence, on the final DROP_BDG_REF(), the NIC 1089 * is detached from the bridge, then ifp's refcount is dropped (this 1090 * is equivalent to that ifp is destroyed in case of virtual ports. 1091 * 1092 * This function uses if_rele() when we want to prevent the NIC from 1093 * being detached from the bridge in error handling. But once refcount 1094 * is acquired by this function, it must be released using nm_if_rele(). 1095 */ 1096 static int 1097 get_ifp(struct nmreq *nmr, struct ifnet **ifp) 1098 { 1099 const char *name = nmr->nr_name; 1100 int namelen = strlen(name); 1101 #ifdef NM_BRIDGE 1102 struct ifnet *iter = NULL; 1103 int no_prefix = 0; 1104 1105 do { 1106 struct nm_bridge *b; 1107 struct netmap_adapter *na; 1108 int i, cand = -1, cand2 = -1; 1109 1110 if (strncmp(name, NM_NAME, sizeof(NM_NAME) - 1)) { 1111 no_prefix = 1; 1112 break; 1113 } 1114 b = nm_find_bridge(name, 1 /* create a new one if no exist */ ); 1115 if (b == NULL) { 1116 D("no bridges available for '%s'", name); 1117 return (ENXIO); 1118 } 1119 /* Now we are sure that name starts with the bridge's name */ 1120 BDG_WLOCK(b); 1121 /* lookup in the local list of ports */ 1122 for (i = 0; i < NM_BDG_MAXPORTS; i++) { 1123 na = BDG_GET_VAR(b->bdg_ports[i]); 1124 if (na == NULL) { 1125 if (cand == -1) 1126 cand = i; /* potential insert point */ 1127 else if (cand2 == -1) 1128 cand2 = i; /* for host stack */ 1129 continue; 1130 } 1131 iter = na->ifp; 1132 /* XXX make sure the name only contains one : */ 1133 if (!strcmp(iter->if_xname, name) /* virtual port */ || 1134 (namelen > b->namelen && !strcmp(iter->if_xname, 1135 name + b->namelen + 1)) /* NIC */) { 1136 ADD_BDG_REF(iter); 1137 ND("found existing interface"); 1138 BDG_WUNLOCK(b); 1139 break; 1140 } 1141 } 1142 if (i < NM_BDG_MAXPORTS) /* already unlocked */ 1143 break; 1144 if (cand == -1) { 1145 D("bridge full, cannot create new port"); 1146 no_port: 1147 BDG_WUNLOCK(b); 1148 *ifp = NULL; 1149 return EINVAL; 1150 } 1151 ND("create new bridge port %s", name); 1152 /* 1153 * create a struct ifnet for the new port. 1154 * The forwarding table is attached to the kring(s). 1155 */ 1156 /* 1157 * try see if there is a matching NIC with this name 1158 * (after the bridge's name) 1159 */ 1160 iter = ifunit_ref(name + b->namelen + 1); 1161 if (!iter) { /* this is a virtual port */ 1162 /* Create a temporary NA with arguments, then 1163 * bdg_netmap_attach() will allocate the real one 1164 * and attach it to the ifp 1165 */ 1166 struct netmap_adapter tmp_na; 1167 1168 if (nmr->nr_cmd) /* nr_cmd must be for a NIC */ 1169 goto no_port; 1170 bzero(&tmp_na, sizeof(tmp_na)); 1171 /* bound checking */ 1172 if (nmr->nr_tx_rings < 1) 1173 nmr->nr_tx_rings = 1; 1174 if (nmr->nr_tx_rings > NM_BDG_MAXRINGS) 1175 nmr->nr_tx_rings = NM_BDG_MAXRINGS; 1176 tmp_na.num_tx_rings = nmr->nr_tx_rings; 1177 if (nmr->nr_rx_rings < 1) 1178 nmr->nr_rx_rings = 1; 1179 if (nmr->nr_rx_rings > NM_BDG_MAXRINGS) 1180 nmr->nr_rx_rings = NM_BDG_MAXRINGS; 1181 tmp_na.num_rx_rings = nmr->nr_rx_rings; 1182 1183 iter = malloc(sizeof(*iter), M_DEVBUF, M_NOWAIT | M_ZERO); 1184 if (!iter) 1185 goto no_port; 1186 strcpy(iter->if_xname, name); 1187 tmp_na.ifp = iter; 1188 /* bdg_netmap_attach creates a struct netmap_adapter */ 1189 bdg_netmap_attach(&tmp_na); 1190 } else if (NETMAP_CAPABLE(iter)) { /* this is a NIC */ 1191 /* cannot attach the NIC that any user or another 1192 * bridge already holds. 1193 */ 1194 if (NETMAP_OWNED_BY_ANY(iter) || cand2 == -1) { 1195 ifunit_rele: 1196 if_rele(iter); /* don't detach from bridge */ 1197 goto no_port; 1198 } 1199 /* bind the host stack to the bridge */ 1200 if (nmr->nr_arg1 == NETMAP_BDG_HOST) { 1201 BDG_SET_VAR(b->bdg_ports[cand2], SWNA(iter)); 1202 SWNA(iter)->bdg_port = cand2; 1203 SWNA(iter)->na_bdg = b; 1204 } 1205 } else /* not a netmap-capable NIC */ 1206 goto ifunit_rele; 1207 na = NA(iter); 1208 na->bdg_port = cand; 1209 /* bind the port to the bridge (virtual ports are not active) */ 1210 BDG_SET_VAR(b->bdg_ports[cand], na); 1211 na->na_bdg = b; 1212 ADD_BDG_REF(iter); 1213 BDG_WUNLOCK(b); 1214 ND("attaching virtual bridge %p", b); 1215 } while (0); 1216 *ifp = iter; 1217 if (! *ifp) 1218 #endif /* NM_BRIDGE */ 1219 *ifp = ifunit_ref(name); 1220 if (*ifp == NULL) 1221 return (ENXIO); 1222 /* can do this if the capability exists and if_pspare[0] 1223 * points to the netmap descriptor. 1224 */ 1225 if (NETMAP_CAPABLE(*ifp)) { 1226 #ifdef NM_BRIDGE 1227 /* Users cannot use the NIC attached to a bridge directly */ 1228 if (no_prefix && NETMAP_OWNED_BY_KERN(*ifp)) { 1229 if_rele(*ifp); /* don't detach from bridge */ 1230 return EINVAL; 1231 } else 1232 #endif /* NM_BRIDGE */ 1233 return 0; /* valid pointer, we hold the refcount */ 1234 } 1235 nm_if_rele(*ifp); 1236 return EINVAL; // not NETMAP capable 1237 } 1238 1239 1240 /* 1241 * Error routine called when txsync/rxsync detects an error. 1242 * Can't do much more than resetting cur = hwcur, avail = hwavail. 1243 * Return 1 on reinit. 1244 * 1245 * This routine is only called by the upper half of the kernel. 1246 * It only reads hwcur (which is changed only by the upper half, too) 1247 * and hwavail (which may be changed by the lower half, but only on 1248 * a tx ring and only to increase it, so any error will be recovered 1249 * on the next call). For the above, we don't strictly need to call 1250 * it under lock. 1251 */ 1252 int 1253 netmap_ring_reinit(struct netmap_kring *kring) 1254 { 1255 struct netmap_ring *ring = kring->ring; 1256 u_int i, lim = kring->nkr_num_slots - 1; 1257 int errors = 0; 1258 1259 RD(10, "called for %s", kring->na->ifp->if_xname); 1260 if (ring->cur > lim) 1261 errors++; 1262 for (i = 0; i <= lim; i++) { 1263 u_int idx = ring->slot[i].buf_idx; 1264 u_int len = ring->slot[i].len; 1265 if (idx < 2 || idx >= netmap_total_buffers) { 1266 if (!errors++) 1267 D("bad buffer at slot %d idx %d len %d ", i, idx, len); 1268 ring->slot[i].buf_idx = 0; 1269 ring->slot[i].len = 0; 1270 } else if (len > NETMAP_BUF_SIZE) { 1271 ring->slot[i].len = 0; 1272 if (!errors++) 1273 D("bad len %d at slot %d idx %d", 1274 len, i, idx); 1275 } 1276 } 1277 if (errors) { 1278 int pos = kring - kring->na->tx_rings; 1279 int n = kring->na->num_tx_rings + 1; 1280 1281 RD(10, "total %d errors", errors); 1282 errors++; 1283 RD(10, "%s %s[%d] reinit, cur %d -> %d avail %d -> %d", 1284 kring->na->ifp->if_xname, 1285 pos < n ? "TX" : "RX", pos < n ? pos : pos - n, 1286 ring->cur, kring->nr_hwcur, 1287 ring->avail, kring->nr_hwavail); 1288 ring->cur = kring->nr_hwcur; 1289 ring->avail = kring->nr_hwavail; 1290 } 1291 return (errors ? 1 : 0); 1292 } 1293 1294 1295 /* 1296 * Set the ring ID. For devices with a single queue, a request 1297 * for all rings is the same as a single ring. 1298 */ 1299 static int 1300 netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid) 1301 { 1302 struct ifnet *ifp = priv->np_ifp; 1303 struct netmap_adapter *na = NA(ifp); 1304 u_int i = ringid & NETMAP_RING_MASK; 1305 /* initially (np_qfirst == np_qlast) we don't want to lock */ 1306 int need_lock = (priv->np_qfirst != priv->np_qlast); 1307 int lim = na->num_rx_rings; 1308 1309 if (na->num_tx_rings > lim) 1310 lim = na->num_tx_rings; 1311 if ( (ringid & NETMAP_HW_RING) && i >= lim) { 1312 D("invalid ring id %d", i); 1313 return (EINVAL); 1314 } 1315 if (need_lock) 1316 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 1317 priv->np_ringid = ringid; 1318 if (ringid & NETMAP_SW_RING) { 1319 priv->np_qfirst = NETMAP_SW_RING; 1320 priv->np_qlast = 0; 1321 } else if (ringid & NETMAP_HW_RING) { 1322 priv->np_qfirst = i; 1323 priv->np_qlast = i + 1; 1324 } else { 1325 priv->np_qfirst = 0; 1326 priv->np_qlast = NETMAP_HW_RING ; 1327 } 1328 priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1; 1329 if (need_lock) 1330 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 1331 if (netmap_verbose) { 1332 if (ringid & NETMAP_SW_RING) 1333 D("ringid %s set to SW RING", ifp->if_xname); 1334 else if (ringid & NETMAP_HW_RING) 1335 D("ringid %s set to HW RING %d", ifp->if_xname, 1336 priv->np_qfirst); 1337 else 1338 D("ringid %s set to all %d HW RINGS", ifp->if_xname, lim); 1339 } 1340 return 0; 1341 } 1342 1343 1344 /* 1345 * possibly move the interface to netmap-mode. 1346 * If success it returns a pointer to netmap_if, otherwise NULL. 1347 * This must be called with NMA_LOCK held. 1348 */ 1349 static struct netmap_if * 1350 netmap_do_regif(struct netmap_priv_d *priv, struct ifnet *ifp, 1351 uint16_t ringid, int *err) 1352 { 1353 struct netmap_adapter *na = NA(ifp); 1354 struct netmap_if *nifp = NULL; 1355 int i, error; 1356 1357 if (na->na_bdg) 1358 BDG_WLOCK(na->na_bdg); 1359 na->nm_lock(ifp, NETMAP_REG_LOCK, 0); 1360 1361 /* ring configuration may have changed, fetch from the card */ 1362 netmap_update_config(na); 1363 priv->np_ifp = ifp; /* store the reference */ 1364 error = netmap_set_ringid(priv, ringid); 1365 if (error) 1366 goto out; 1367 nifp = netmap_if_new(ifp->if_xname, na); 1368 if (nifp == NULL) { /* allocation failed */ 1369 error = ENOMEM; 1370 } else if (ifp->if_capenable & IFCAP_NETMAP) { 1371 /* was already set */ 1372 } else { 1373 /* Otherwise set the card in netmap mode 1374 * and make it use the shared buffers. 1375 */ 1376 for (i = 0 ; i < na->num_tx_rings + 1; i++) 1377 mtx_init(&na->tx_rings[i].q_lock, "nm_txq_lock", 1378 MTX_NETWORK_LOCK, MTX_DEF); 1379 for (i = 0 ; i < na->num_rx_rings + 1; i++) { 1380 mtx_init(&na->rx_rings[i].q_lock, "nm_rxq_lock", 1381 MTX_NETWORK_LOCK, MTX_DEF); 1382 } 1383 if (nma_is_hw(na)) { 1384 SWNA(ifp)->tx_rings = &na->tx_rings[na->num_tx_rings]; 1385 SWNA(ifp)->rx_rings = &na->rx_rings[na->num_rx_rings]; 1386 } 1387 error = na->nm_register(ifp, 1); /* mode on */ 1388 #ifdef NM_BRIDGE 1389 if (!error) 1390 error = nm_alloc_bdgfwd(na); 1391 #endif /* NM_BRIDGE */ 1392 if (error) { 1393 netmap_dtor_locked(priv); 1394 /* nifp is not yet in priv, so free it separately */ 1395 netmap_if_free(nifp); 1396 nifp = NULL; 1397 } 1398 1399 } 1400 out: 1401 *err = error; 1402 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0); 1403 if (na->na_bdg) 1404 BDG_WUNLOCK(na->na_bdg); 1405 return nifp; 1406 } 1407 1408 1409 /* Process NETMAP_BDG_ATTACH and NETMAP_BDG_DETACH */ 1410 static int 1411 kern_netmap_regif(struct nmreq *nmr) 1412 { 1413 struct ifnet *ifp; 1414 struct netmap_if *nifp; 1415 struct netmap_priv_d *npriv; 1416 int error; 1417 1418 npriv = malloc(sizeof(*npriv), M_DEVBUF, M_NOWAIT|M_ZERO); 1419 if (npriv == NULL) 1420 return ENOMEM; 1421 error = netmap_get_memory(npriv); 1422 if (error) { 1423 free_exit: 1424 bzero(npriv, sizeof(*npriv)); 1425 free(npriv, M_DEVBUF); 1426 return error; 1427 } 1428 1429 NMA_LOCK(); 1430 error = get_ifp(nmr, &ifp); 1431 if (error) { /* no device, or another bridge or user owns the device */ 1432 NMA_UNLOCK(); 1433 goto free_exit; 1434 } else if (!NETMAP_OWNED_BY_KERN(ifp)) { 1435 /* got reference to a virtual port or direct access to a NIC. 1436 * perhaps specified no bridge's prefix or wrong NIC's name 1437 */ 1438 error = EINVAL; 1439 unref_exit: 1440 nm_if_rele(ifp); 1441 NMA_UNLOCK(); 1442 goto free_exit; 1443 } 1444 1445 if (nmr->nr_cmd == NETMAP_BDG_DETACH) { 1446 if (NA(ifp)->refcount == 0) { /* not registered */ 1447 error = EINVAL; 1448 goto unref_exit; 1449 } 1450 NMA_UNLOCK(); 1451 1452 netmap_dtor(NA(ifp)->na_kpriv); /* unregister */ 1453 NA(ifp)->na_kpriv = NULL; 1454 nm_if_rele(ifp); /* detach from the bridge */ 1455 goto free_exit; 1456 } else if (NA(ifp)->refcount > 0) { /* already registered */ 1457 error = EINVAL; 1458 goto unref_exit; 1459 } 1460 1461 nifp = netmap_do_regif(npriv, ifp, nmr->nr_ringid, &error); 1462 if (!nifp) 1463 goto unref_exit; 1464 wmb(); // XXX do we need it ? 1465 npriv->np_nifp = nifp; 1466 NA(ifp)->na_kpriv = npriv; 1467 NMA_UNLOCK(); 1468 D("registered %s to netmap-mode", ifp->if_xname); 1469 return 0; 1470 } 1471 1472 1473 /* CORE_LOCK is not necessary */ 1474 static void 1475 netmap_swlock_wrapper(struct ifnet *dev, int what, u_int queueid) 1476 { 1477 struct netmap_adapter *na = SWNA(dev); 1478 1479 switch (what) { 1480 case NETMAP_TX_LOCK: 1481 mtx_lock(&na->tx_rings[queueid].q_lock); 1482 break; 1483 1484 case NETMAP_TX_UNLOCK: 1485 mtx_unlock(&na->tx_rings[queueid].q_lock); 1486 break; 1487 1488 case NETMAP_RX_LOCK: 1489 mtx_lock(&na->rx_rings[queueid].q_lock); 1490 break; 1491 1492 case NETMAP_RX_UNLOCK: 1493 mtx_unlock(&na->rx_rings[queueid].q_lock); 1494 break; 1495 } 1496 } 1497 1498 1499 /* Initialize necessary fields of sw adapter located in right after hw's 1500 * one. sw adapter attaches a pair of sw rings of the netmap-mode NIC. 1501 * It is always activated and deactivated at the same tie with the hw's one. 1502 * Thus we don't need refcounting on the sw adapter. 1503 * Regardless of NIC's feature we use separate lock so that anybody can lock 1504 * me independently from the hw adapter. 1505 * Make sure nm_register is NULL to be handled as FALSE in nma_is_hw 1506 */ 1507 static void 1508 netmap_attach_sw(struct ifnet *ifp) 1509 { 1510 struct netmap_adapter *hw_na = NA(ifp); 1511 struct netmap_adapter *na = SWNA(ifp); 1512 1513 na->ifp = ifp; 1514 na->separate_locks = 1; 1515 na->nm_lock = netmap_swlock_wrapper; 1516 na->num_rx_rings = na->num_tx_rings = 1; 1517 na->num_tx_desc = hw_na->num_tx_desc; 1518 na->num_rx_desc = hw_na->num_rx_desc; 1519 na->nm_txsync = netmap_bdg_to_host; 1520 } 1521 1522 1523 /* exported to kernel callers */ 1524 int 1525 netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func) 1526 { 1527 struct nm_bridge *b; 1528 struct netmap_adapter *na; 1529 struct ifnet *iter; 1530 char *name = nmr->nr_name; 1531 int cmd = nmr->nr_cmd, namelen = strlen(name); 1532 int error = 0, i, j; 1533 1534 switch (cmd) { 1535 case NETMAP_BDG_ATTACH: 1536 case NETMAP_BDG_DETACH: 1537 error = kern_netmap_regif(nmr); 1538 break; 1539 1540 case NETMAP_BDG_LIST: 1541 /* this is used to enumerate bridges and ports */ 1542 if (namelen) { /* look up indexes of bridge and port */ 1543 if (strncmp(name, NM_NAME, strlen(NM_NAME))) { 1544 error = EINVAL; 1545 break; 1546 } 1547 b = nm_find_bridge(name, 0 /* don't create */); 1548 if (!b) { 1549 error = ENOENT; 1550 break; 1551 } 1552 1553 BDG_RLOCK(b); 1554 error = ENOENT; 1555 for (i = 0; i < NM_BDG_MAXPORTS; i++) { 1556 na = BDG_GET_VAR(b->bdg_ports[i]); 1557 if (na == NULL) 1558 continue; 1559 iter = na->ifp; 1560 /* the former and the latter identify a 1561 * virtual port and a NIC, respectively 1562 */ 1563 if (!strcmp(iter->if_xname, name) || 1564 (namelen > b->namelen && 1565 !strcmp(iter->if_xname, 1566 name + b->namelen + 1))) { 1567 /* bridge index */ 1568 nmr->nr_arg1 = b - nm_bridges; 1569 nmr->nr_arg2 = i; /* port index */ 1570 error = 0; 1571 break; 1572 } 1573 } 1574 BDG_RUNLOCK(b); 1575 } else { 1576 /* return the first non-empty entry starting from 1577 * bridge nr_arg1 and port nr_arg2. 1578 * 1579 * Users can detect the end of the same bridge by 1580 * seeing the new and old value of nr_arg1, and can 1581 * detect the end of all the bridge by error != 0 1582 */ 1583 i = nmr->nr_arg1; 1584 j = nmr->nr_arg2; 1585 1586 for (error = ENOENT; error && i < NM_BRIDGES; i++) { 1587 b = nm_bridges + i; 1588 BDG_RLOCK(b); 1589 for (; j < NM_BDG_MAXPORTS; j++) { 1590 na = BDG_GET_VAR(b->bdg_ports[j]); 1591 if (na == NULL) 1592 continue; 1593 iter = na->ifp; 1594 nmr->nr_arg1 = i; 1595 nmr->nr_arg2 = j; 1596 strncpy(name, iter->if_xname, IFNAMSIZ); 1597 error = 0; 1598 break; 1599 } 1600 BDG_RUNLOCK(b); 1601 j = 0; /* following bridges scan from 0 */ 1602 } 1603 } 1604 break; 1605 1606 case NETMAP_BDG_LOOKUP_REG: 1607 /* register a lookup function to the given bridge. 1608 * nmr->nr_name may be just bridge's name (including ':' 1609 * if it is not just NM_NAME). 1610 */ 1611 if (!func) { 1612 error = EINVAL; 1613 break; 1614 } 1615 b = nm_find_bridge(name, 0 /* don't create */); 1616 if (!b) { 1617 error = EINVAL; 1618 break; 1619 } 1620 BDG_WLOCK(b); 1621 b->nm_bdg_lookup = func; 1622 BDG_WUNLOCK(b); 1623 break; 1624 default: 1625 D("invalid cmd (nmr->nr_cmd) (0x%x)", cmd); 1626 error = EINVAL; 1627 break; 1628 } 1629 return error; 1630 } 1631 1632 1633 /* 1634 * ioctl(2) support for the "netmap" device. 1635 * 1636 * Following a list of accepted commands: 1637 * - NIOCGINFO 1638 * - SIOCGIFADDR just for convenience 1639 * - NIOCREGIF 1640 * - NIOCUNREGIF 1641 * - NIOCTXSYNC 1642 * - NIOCRXSYNC 1643 * 1644 * Return 0 on success, errno otherwise. 1645 */ 1646 static int 1647 netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, 1648 int fflag, struct thread *td) 1649 { 1650 struct netmap_priv_d *priv = NULL; 1651 struct ifnet *ifp; 1652 struct nmreq *nmr = (struct nmreq *) data; 1653 struct netmap_adapter *na; 1654 int error; 1655 u_int i, lim; 1656 struct netmap_if *nifp; 1657 1658 (void)dev; /* UNUSED */ 1659 (void)fflag; /* UNUSED */ 1660 #ifdef linux 1661 #define devfs_get_cdevpriv(pp) \ 1662 ({ *(struct netmap_priv_d **)pp = ((struct file *)td)->private_data; \ 1663 (*pp ? 0 : ENOENT); }) 1664 1665 /* devfs_set_cdevpriv cannot fail on linux */ 1666 #define devfs_set_cdevpriv(p, fn) \ 1667 ({ ((struct file *)td)->private_data = p; (p ? 0 : EINVAL); }) 1668 1669 1670 #define devfs_clear_cdevpriv() do { \ 1671 netmap_dtor(priv); ((struct file *)td)->private_data = 0; \ 1672 } while (0) 1673 #endif /* linux */ 1674 1675 CURVNET_SET(TD_TO_VNET(td)); 1676 1677 error = devfs_get_cdevpriv((void **)&priv); 1678 if (error) { 1679 CURVNET_RESTORE(); 1680 /* XXX ENOENT should be impossible, since the priv 1681 * is now created in the open */ 1682 return (error == ENOENT ? ENXIO : error); 1683 } 1684 1685 nmr->nr_name[sizeof(nmr->nr_name) - 1] = '\0'; /* truncate name */ 1686 switch (cmd) { 1687 case NIOCGINFO: /* return capabilities etc */ 1688 if (nmr->nr_version != NETMAP_API) { 1689 D("API mismatch got %d have %d", 1690 nmr->nr_version, NETMAP_API); 1691 nmr->nr_version = NETMAP_API; 1692 error = EINVAL; 1693 break; 1694 } 1695 if (nmr->nr_cmd == NETMAP_BDG_LIST) { 1696 error = netmap_bdg_ctl(nmr, NULL); 1697 break; 1698 } 1699 /* update configuration */ 1700 error = netmap_get_memory(priv); 1701 ND("get_memory returned %d", error); 1702 if (error) 1703 break; 1704 /* memsize is always valid */ 1705 nmr->nr_memsize = nm_mem.nm_totalsize; 1706 nmr->nr_offset = 0; 1707 nmr->nr_rx_slots = nmr->nr_tx_slots = 0; 1708 if (nmr->nr_name[0] == '\0') /* just get memory info */ 1709 break; 1710 /* lock because get_ifp and update_config see na->refcount */ 1711 NMA_LOCK(); 1712 error = get_ifp(nmr, &ifp); /* get a refcount */ 1713 if (error) { 1714 NMA_UNLOCK(); 1715 break; 1716 } 1717 na = NA(ifp); /* retrieve netmap_adapter */ 1718 netmap_update_config(na); 1719 NMA_UNLOCK(); 1720 nmr->nr_rx_rings = na->num_rx_rings; 1721 nmr->nr_tx_rings = na->num_tx_rings; 1722 nmr->nr_rx_slots = na->num_rx_desc; 1723 nmr->nr_tx_slots = na->num_tx_desc; 1724 nm_if_rele(ifp); /* return the refcount */ 1725 break; 1726 1727 case NIOCREGIF: 1728 if (nmr->nr_version != NETMAP_API) { 1729 nmr->nr_version = NETMAP_API; 1730 error = EINVAL; 1731 break; 1732 } 1733 /* possibly attach/detach NIC and VALE switch */ 1734 i = nmr->nr_cmd; 1735 if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH) { 1736 error = netmap_bdg_ctl(nmr, NULL); 1737 break; 1738 } else if (i != 0) { 1739 D("nr_cmd must be 0 not %d", i); 1740 error = EINVAL; 1741 break; 1742 } 1743 1744 /* ensure allocators are ready */ 1745 error = netmap_get_memory(priv); 1746 ND("get_memory returned %d", error); 1747 if (error) 1748 break; 1749 1750 /* protect access to priv from concurrent NIOCREGIF */ 1751 NMA_LOCK(); 1752 if (priv->np_ifp != NULL) { /* thread already registered */ 1753 error = netmap_set_ringid(priv, nmr->nr_ringid); 1754 unlock_out: 1755 NMA_UNLOCK(); 1756 break; 1757 } 1758 /* find the interface and a reference */ 1759 error = get_ifp(nmr, &ifp); /* keep reference */ 1760 if (error) 1761 goto unlock_out; 1762 else if (NETMAP_OWNED_BY_KERN(ifp)) { 1763 nm_if_rele(ifp); 1764 goto unlock_out; 1765 } 1766 nifp = netmap_do_regif(priv, ifp, nmr->nr_ringid, &error); 1767 if (!nifp) { /* reg. failed, release priv and ref */ 1768 nm_if_rele(ifp); /* return the refcount */ 1769 priv->np_ifp = NULL; 1770 priv->np_nifp = NULL; 1771 goto unlock_out; 1772 } 1773 1774 /* the following assignment is a commitment. 1775 * Readers (i.e., poll and *SYNC) check for 1776 * np_nifp != NULL without locking 1777 */ 1778 wmb(); /* make sure previous writes are visible to all CPUs */ 1779 priv->np_nifp = nifp; 1780 NMA_UNLOCK(); 1781 1782 /* return the offset of the netmap_if object */ 1783 na = NA(ifp); /* retrieve netmap adapter */ 1784 nmr->nr_rx_rings = na->num_rx_rings; 1785 nmr->nr_tx_rings = na->num_tx_rings; 1786 nmr->nr_rx_slots = na->num_rx_desc; 1787 nmr->nr_tx_slots = na->num_tx_desc; 1788 nmr->nr_memsize = nm_mem.nm_totalsize; 1789 nmr->nr_offset = netmap_if_offset(nifp); 1790 break; 1791 1792 case NIOCUNREGIF: 1793 // XXX we have no data here ? 1794 D("deprecated, data is %p", nmr); 1795 error = EINVAL; 1796 break; 1797 1798 case NIOCTXSYNC: 1799 case NIOCRXSYNC: 1800 nifp = priv->np_nifp; 1801 1802 if (nifp == NULL) { 1803 error = ENXIO; 1804 break; 1805 } 1806 rmb(); /* make sure following reads are not from cache */ 1807 1808 1809 ifp = priv->np_ifp; /* we have a reference */ 1810 1811 if (ifp == NULL) { 1812 D("Internal error: nifp != NULL && ifp == NULL"); 1813 error = ENXIO; 1814 break; 1815 } 1816 1817 na = NA(ifp); /* retrieve netmap adapter */ 1818 if (priv->np_qfirst == NETMAP_SW_RING) { /* host rings */ 1819 if (cmd == NIOCTXSYNC) 1820 netmap_sync_to_host(na); 1821 else 1822 netmap_sync_from_host(na, NULL, NULL); 1823 break; 1824 } 1825 /* find the last ring to scan */ 1826 lim = priv->np_qlast; 1827 if (lim == NETMAP_HW_RING) 1828 lim = (cmd == NIOCTXSYNC) ? 1829 na->num_tx_rings : na->num_rx_rings; 1830 1831 for (i = priv->np_qfirst; i < lim; i++) { 1832 if (cmd == NIOCTXSYNC) { 1833 struct netmap_kring *kring = &na->tx_rings[i]; 1834 if (netmap_verbose & NM_VERB_TXSYNC) 1835 D("pre txsync ring %d cur %d hwcur %d", 1836 i, kring->ring->cur, 1837 kring->nr_hwcur); 1838 na->nm_txsync(ifp, i, 1 /* do lock */); 1839 if (netmap_verbose & NM_VERB_TXSYNC) 1840 D("post txsync ring %d cur %d hwcur %d", 1841 i, kring->ring->cur, 1842 kring->nr_hwcur); 1843 } else { 1844 na->nm_rxsync(ifp, i, 1 /* do lock */); 1845 microtime(&na->rx_rings[i].ring->ts); 1846 } 1847 } 1848 1849 break; 1850 1851 #ifdef __FreeBSD__ 1852 case BIOCIMMEDIATE: 1853 case BIOCGHDRCMPLT: 1854 case BIOCSHDRCMPLT: 1855 case BIOCSSEESENT: 1856 D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT"); 1857 break; 1858 1859 default: /* allow device-specific ioctls */ 1860 { 1861 struct socket so; 1862 bzero(&so, sizeof(so)); 1863 error = get_ifp(nmr, &ifp); /* keep reference */ 1864 if (error) 1865 break; 1866 so.so_vnet = ifp->if_vnet; 1867 // so->so_proto not null. 1868 error = ifioctl(&so, cmd, data, td); 1869 nm_if_rele(ifp); 1870 break; 1871 } 1872 1873 #else /* linux */ 1874 default: 1875 error = EOPNOTSUPP; 1876 #endif /* linux */ 1877 } 1878 1879 CURVNET_RESTORE(); 1880 return (error); 1881 } 1882 1883 1884 /* 1885 * select(2) and poll(2) handlers for the "netmap" device. 1886 * 1887 * Can be called for one or more queues. 1888 * Return true the event mask corresponding to ready events. 1889 * If there are no ready events, do a selrecord on either individual 1890 * selfd or on the global one. 1891 * Device-dependent parts (locking and sync of tx/rx rings) 1892 * are done through callbacks. 1893 * 1894 * On linux, arguments are really pwait, the poll table, and 'td' is struct file * 1895 * The first one is remapped to pwait as selrecord() uses the name as an 1896 * hidden argument. 1897 */ 1898 static int 1899 netmap_poll(struct cdev *dev, int events, struct thread *td) 1900 { 1901 struct netmap_priv_d *priv = NULL; 1902 struct netmap_adapter *na; 1903 struct ifnet *ifp; 1904 struct netmap_kring *kring; 1905 u_int core_lock, i, check_all, want_tx, want_rx, revents = 0; 1906 u_int lim_tx, lim_rx, host_forwarded = 0; 1907 struct mbq q = { NULL, NULL, 0 }; 1908 enum {NO_CL, NEED_CL, LOCKED_CL }; /* see below */ 1909 void *pwait = dev; /* linux compatibility */ 1910 1911 (void)pwait; 1912 1913 if (devfs_get_cdevpriv((void **)&priv) != 0 || priv == NULL) 1914 return POLLERR; 1915 1916 if (priv->np_nifp == NULL) { 1917 D("No if registered"); 1918 return POLLERR; 1919 } 1920 rmb(); /* make sure following reads are not from cache */ 1921 1922 ifp = priv->np_ifp; 1923 // XXX check for deleting() ? 1924 if ( (ifp->if_capenable & IFCAP_NETMAP) == 0) 1925 return POLLERR; 1926 1927 if (netmap_verbose & 0x8000) 1928 D("device %s events 0x%x", ifp->if_xname, events); 1929 want_tx = events & (POLLOUT | POLLWRNORM); 1930 want_rx = events & (POLLIN | POLLRDNORM); 1931 1932 na = NA(ifp); /* retrieve netmap adapter */ 1933 1934 lim_tx = na->num_tx_rings; 1935 lim_rx = na->num_rx_rings; 1936 /* how many queues we are scanning */ 1937 if (priv->np_qfirst == NETMAP_SW_RING) { 1938 if (priv->np_txpoll || want_tx) { 1939 /* push any packets up, then we are always ready */ 1940 netmap_sync_to_host(na); 1941 revents |= want_tx; 1942 } 1943 if (want_rx) { 1944 kring = &na->rx_rings[lim_rx]; 1945 if (kring->ring->avail == 0) 1946 netmap_sync_from_host(na, td, dev); 1947 if (kring->ring->avail > 0) { 1948 revents |= want_rx; 1949 } 1950 } 1951 return (revents); 1952 } 1953 1954 /* if we are in transparent mode, check also the host rx ring */ 1955 kring = &na->rx_rings[lim_rx]; 1956 if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all 1957 && want_rx 1958 && (netmap_fwd || kring->ring->flags & NR_FORWARD) ) { 1959 if (kring->ring->avail == 0) 1960 netmap_sync_from_host(na, td, dev); 1961 if (kring->ring->avail > 0) 1962 revents |= want_rx; 1963 } 1964 1965 /* 1966 * check_all is set if the card has more than one queue and 1967 * the client is polling all of them. If true, we sleep on 1968 * the "global" selfd, otherwise we sleep on individual selfd 1969 * (we can only sleep on one of them per direction). 1970 * The interrupt routine in the driver should always wake on 1971 * the individual selfd, and also on the global one if the card 1972 * has more than one ring. 1973 * 1974 * If the card has only one lock, we just use that. 1975 * If the card has separate ring locks, we just use those 1976 * unless we are doing check_all, in which case the whole 1977 * loop is wrapped by the global lock. 1978 * We acquire locks only when necessary: if poll is called 1979 * when buffers are available, we can just return without locks. 1980 * 1981 * rxsync() is only called if we run out of buffers on a POLLIN. 1982 * txsync() is called if we run out of buffers on POLLOUT, or 1983 * there are pending packets to send. The latter can be disabled 1984 * passing NETMAP_NO_TX_POLL in the NIOCREG call. 1985 */ 1986 check_all = (priv->np_qlast == NETMAP_HW_RING) && (lim_tx > 1 || lim_rx > 1); 1987 1988 /* 1989 * core_lock indicates what to do with the core lock. 1990 * The core lock is used when either the card has no individual 1991 * locks, or it has individual locks but we are cheking all 1992 * rings so we need the core lock to avoid missing wakeup events. 1993 * 1994 * It has three possible states: 1995 * NO_CL we don't need to use the core lock, e.g. 1996 * because we are protected by individual locks. 1997 * NEED_CL we need the core lock. In this case, when we 1998 * call the lock routine, move to LOCKED_CL 1999 * to remember to release the lock once done. 2000 * LOCKED_CL core lock is set, so we need to release it. 2001 */ 2002 core_lock = (check_all || !na->separate_locks) ? NEED_CL : NO_CL; 2003 #ifdef NM_BRIDGE 2004 /* the bridge uses separate locks */ 2005 if (na->nm_register == bdg_netmap_reg) { 2006 ND("not using core lock for %s", ifp->if_xname); 2007 core_lock = NO_CL; 2008 } 2009 #endif /* NM_BRIDGE */ 2010 if (priv->np_qlast != NETMAP_HW_RING) { 2011 lim_tx = lim_rx = priv->np_qlast; 2012 } 2013 2014 /* 2015 * We start with a lock free round which is good if we have 2016 * data available. If this fails, then lock and call the sync 2017 * routines. 2018 */ 2019 for (i = priv->np_qfirst; want_rx && i < lim_rx; i++) { 2020 kring = &na->rx_rings[i]; 2021 if (kring->ring->avail > 0) { 2022 revents |= want_rx; 2023 want_rx = 0; /* also breaks the loop */ 2024 } 2025 } 2026 for (i = priv->np_qfirst; want_tx && i < lim_tx; i++) { 2027 kring = &na->tx_rings[i]; 2028 if (kring->ring->avail > 0) { 2029 revents |= want_tx; 2030 want_tx = 0; /* also breaks the loop */ 2031 } 2032 } 2033 2034 /* 2035 * If we to push packets out (priv->np_txpoll) or want_tx is 2036 * still set, we do need to run the txsync calls (on all rings, 2037 * to avoid that the tx rings stall). 2038 */ 2039 if (priv->np_txpoll || want_tx) { 2040 flush_tx: 2041 for (i = priv->np_qfirst; i < lim_tx; i++) { 2042 kring = &na->tx_rings[i]; 2043 /* 2044 * Skip the current ring if want_tx == 0 2045 * (we have already done a successful sync on 2046 * a previous ring) AND kring->cur == kring->hwcur 2047 * (there are no pending transmissions for this ring). 2048 */ 2049 if (!want_tx && kring->ring->cur == kring->nr_hwcur) 2050 continue; 2051 if (core_lock == NEED_CL) { 2052 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2053 core_lock = LOCKED_CL; 2054 } 2055 if (na->separate_locks) 2056 na->nm_lock(ifp, NETMAP_TX_LOCK, i); 2057 if (netmap_verbose & NM_VERB_TXSYNC) 2058 D("send %d on %s %d", 2059 kring->ring->cur, 2060 ifp->if_xname, i); 2061 if (na->nm_txsync(ifp, i, 0 /* no lock */)) 2062 revents |= POLLERR; 2063 2064 /* Check avail/call selrecord only if called with POLLOUT */ 2065 if (want_tx) { 2066 if (kring->ring->avail > 0) { 2067 /* stop at the first ring. We don't risk 2068 * starvation. 2069 */ 2070 revents |= want_tx; 2071 want_tx = 0; 2072 } else if (!check_all) 2073 selrecord(td, &kring->si); 2074 } 2075 if (na->separate_locks) 2076 na->nm_lock(ifp, NETMAP_TX_UNLOCK, i); 2077 } 2078 } 2079 2080 /* 2081 * now if want_rx is still set we need to lock and rxsync. 2082 * Do it on all rings because otherwise we starve. 2083 */ 2084 if (want_rx) { 2085 for (i = priv->np_qfirst; i < lim_rx; i++) { 2086 kring = &na->rx_rings[i]; 2087 if (core_lock == NEED_CL) { 2088 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2089 core_lock = LOCKED_CL; 2090 } 2091 if (na->separate_locks) 2092 na->nm_lock(ifp, NETMAP_RX_LOCK, i); 2093 if (netmap_fwd ||kring->ring->flags & NR_FORWARD) { 2094 ND(10, "forwarding some buffers up %d to %d", 2095 kring->nr_hwcur, kring->ring->cur); 2096 netmap_grab_packets(kring, &q, netmap_fwd); 2097 } 2098 2099 if (na->nm_rxsync(ifp, i, 0 /* no lock */)) 2100 revents |= POLLERR; 2101 if (netmap_no_timestamp == 0 || 2102 kring->ring->flags & NR_TIMESTAMP) { 2103 microtime(&kring->ring->ts); 2104 } 2105 2106 if (kring->ring->avail > 0) 2107 revents |= want_rx; 2108 else if (!check_all) 2109 selrecord(td, &kring->si); 2110 if (na->separate_locks) 2111 na->nm_lock(ifp, NETMAP_RX_UNLOCK, i); 2112 } 2113 } 2114 if (check_all && revents == 0) { /* signal on the global queue */ 2115 if (want_tx) 2116 selrecord(td, &na->tx_si); 2117 if (want_rx) 2118 selrecord(td, &na->rx_si); 2119 } 2120 2121 /* forward host to the netmap ring */ 2122 kring = &na->rx_rings[lim_rx]; 2123 if (kring->nr_hwavail > 0) 2124 ND("host rx %d has %d packets", lim_rx, kring->nr_hwavail); 2125 if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all 2126 && (netmap_fwd || kring->ring->flags & NR_FORWARD) 2127 && kring->nr_hwavail > 0 && !host_forwarded) { 2128 if (core_lock == NEED_CL) { 2129 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2130 core_lock = LOCKED_CL; 2131 } 2132 netmap_sw_to_nic(na); 2133 host_forwarded = 1; /* prevent another pass */ 2134 want_rx = 0; 2135 goto flush_tx; 2136 } 2137 2138 if (core_lock == LOCKED_CL) 2139 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 2140 if (q.head) 2141 netmap_send_up(na->ifp, q.head); 2142 2143 return (revents); 2144 } 2145 2146 /*------- driver support routines ------*/ 2147 2148 2149 /* 2150 * default lock wrapper. 2151 */ 2152 static void 2153 netmap_lock_wrapper(struct ifnet *dev, int what, u_int queueid) 2154 { 2155 struct netmap_adapter *na = NA(dev); 2156 2157 switch (what) { 2158 #ifdef linux /* some system do not need lock on register */ 2159 case NETMAP_REG_LOCK: 2160 case NETMAP_REG_UNLOCK: 2161 break; 2162 #endif /* linux */ 2163 2164 case NETMAP_CORE_LOCK: 2165 mtx_lock(&na->core_lock); 2166 break; 2167 2168 case NETMAP_CORE_UNLOCK: 2169 mtx_unlock(&na->core_lock); 2170 break; 2171 2172 case NETMAP_TX_LOCK: 2173 mtx_lock(&na->tx_rings[queueid].q_lock); 2174 break; 2175 2176 case NETMAP_TX_UNLOCK: 2177 mtx_unlock(&na->tx_rings[queueid].q_lock); 2178 break; 2179 2180 case NETMAP_RX_LOCK: 2181 mtx_lock(&na->rx_rings[queueid].q_lock); 2182 break; 2183 2184 case NETMAP_RX_UNLOCK: 2185 mtx_unlock(&na->rx_rings[queueid].q_lock); 2186 break; 2187 } 2188 } 2189 2190 2191 /* 2192 * Initialize a ``netmap_adapter`` object created by driver on attach. 2193 * We allocate a block of memory with room for a struct netmap_adapter 2194 * plus two sets of N+2 struct netmap_kring (where N is the number 2195 * of hardware rings): 2196 * krings 0..N-1 are for the hardware queues. 2197 * kring N is for the host stack queue 2198 * kring N+1 is only used for the selinfo for all queues. 2199 * Return 0 on success, ENOMEM otherwise. 2200 * 2201 * By default the receive and transmit adapter ring counts are both initialized 2202 * to num_queues. na->num_tx_rings can be set for cards with different tx/rx 2203 * setups. 2204 */ 2205 int 2206 netmap_attach(struct netmap_adapter *arg, int num_queues) 2207 { 2208 struct netmap_adapter *na = NULL; 2209 struct ifnet *ifp = arg ? arg->ifp : NULL; 2210 int len; 2211 2212 if (arg == NULL || ifp == NULL) 2213 goto fail; 2214 len = nma_is_vp(arg) ? sizeof(*na) : sizeof(*na) * 2; 2215 na = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); 2216 if (na == NULL) 2217 goto fail; 2218 WNA(ifp) = na; 2219 *na = *arg; /* copy everything, trust the driver to not pass junk */ 2220 NETMAP_SET_CAPABLE(ifp); 2221 if (na->num_tx_rings == 0) 2222 na->num_tx_rings = num_queues; 2223 na->num_rx_rings = num_queues; 2224 na->refcount = na->na_single = na->na_multi = 0; 2225 /* Core lock initialized here, others after netmap_if_new. */ 2226 mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF); 2227 if (na->nm_lock == NULL) { 2228 ND("using default locks for %s", ifp->if_xname); 2229 na->nm_lock = netmap_lock_wrapper; 2230 } 2231 #ifdef linux 2232 if (ifp->netdev_ops) { 2233 ND("netdev_ops %p", ifp->netdev_ops); 2234 /* prepare a clone of the netdev ops */ 2235 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28) 2236 na->nm_ndo.ndo_start_xmit = ifp->netdev_ops; 2237 #else 2238 na->nm_ndo = *ifp->netdev_ops; 2239 #endif 2240 } 2241 na->nm_ndo.ndo_start_xmit = linux_netmap_start; 2242 #endif 2243 if (!nma_is_vp(arg)) 2244 netmap_attach_sw(ifp); 2245 D("success for %s", ifp->if_xname); 2246 return 0; 2247 2248 fail: 2249 D("fail, arg %p ifp %p na %p", arg, ifp, na); 2250 netmap_detach(ifp); 2251 return (na ? EINVAL : ENOMEM); 2252 } 2253 2254 2255 /* 2256 * Free the allocated memory linked to the given ``netmap_adapter`` 2257 * object. 2258 */ 2259 void 2260 netmap_detach(struct ifnet *ifp) 2261 { 2262 struct netmap_adapter *na = NA(ifp); 2263 2264 if (!na) 2265 return; 2266 2267 mtx_destroy(&na->core_lock); 2268 2269 if (na->tx_rings) { /* XXX should not happen */ 2270 D("freeing leftover tx_rings"); 2271 free(na->tx_rings, M_DEVBUF); 2272 } 2273 bzero(na, sizeof(*na)); 2274 WNA(ifp) = NULL; 2275 free(na, M_DEVBUF); 2276 } 2277 2278 2279 int 2280 nm_bdg_flush(struct nm_bdg_fwd *ft, int n, struct netmap_adapter *na, u_int ring_nr); 2281 2282 /* we don't need to lock myself */ 2283 static int 2284 bdg_netmap_start(struct ifnet *ifp, struct mbuf *m) 2285 { 2286 struct netmap_adapter *na = SWNA(ifp); 2287 struct nm_bdg_fwd *ft = na->rx_rings[0].nkr_ft; 2288 char *buf = NMB(&na->rx_rings[0].ring->slot[0]); 2289 u_int len = MBUF_LEN(m); 2290 2291 if (!na->na_bdg) /* SWNA is not configured to be attached */ 2292 return EBUSY; 2293 m_copydata(m, 0, len, buf); 2294 ft->ft_flags = 0; // XXX could be indirect ? 2295 ft->ft_len = len; 2296 ft->ft_buf = buf; 2297 ft->ft_next = NM_BDG_BATCH; // XXX is it needed ? 2298 nm_bdg_flush(ft, 1, na, 0); 2299 2300 /* release the mbuf in either cases of success or failure. As an 2301 * alternative, put the mbuf in a free list and free the list 2302 * only when really necessary. 2303 */ 2304 m_freem(m); 2305 2306 return (0); 2307 } 2308 2309 2310 /* 2311 * Intercept packets from the network stack and pass them 2312 * to netmap as incoming packets on the 'software' ring. 2313 * We are not locked when called. 2314 */ 2315 int 2316 netmap_start(struct ifnet *ifp, struct mbuf *m) 2317 { 2318 struct netmap_adapter *na = NA(ifp); 2319 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; 2320 u_int i, len = MBUF_LEN(m); 2321 u_int error = EBUSY, lim = kring->nkr_num_slots - 1; 2322 struct netmap_slot *slot; 2323 2324 if (netmap_verbose & NM_VERB_HOST) 2325 D("%s packet %d len %d from the stack", ifp->if_xname, 2326 kring->nr_hwcur + kring->nr_hwavail, len); 2327 if (len > NETMAP_BUF_SIZE) { /* too long for us */ 2328 D("%s from_host, drop packet size %d > %d", ifp->if_xname, 2329 len, NETMAP_BUF_SIZE); 2330 m_freem(m); 2331 return EINVAL; 2332 } 2333 if (na->na_bdg) 2334 return bdg_netmap_start(ifp, m); 2335 2336 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2337 if (kring->nr_hwavail >= lim) { 2338 if (netmap_verbose) 2339 D("stack ring %s full\n", ifp->if_xname); 2340 goto done; /* no space */ 2341 } 2342 2343 /* compute the insert position */ 2344 i = kring->nr_hwcur + kring->nr_hwavail; 2345 if (i > lim) 2346 i -= lim + 1; 2347 slot = &kring->ring->slot[i]; 2348 m_copydata(m, 0, len, NMB(slot)); 2349 slot->len = len; 2350 slot->flags = kring->nkr_slot_flags; 2351 kring->nr_hwavail++; 2352 if (netmap_verbose & NM_VERB_HOST) 2353 D("wake up host ring %s %d", na->ifp->if_xname, na->num_rx_rings); 2354 selwakeuppri(&kring->si, PI_NET); 2355 error = 0; 2356 done: 2357 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 2358 2359 /* release the mbuf in either cases of success or failure. As an 2360 * alternative, put the mbuf in a free list and free the list 2361 * only when really necessary. 2362 */ 2363 m_freem(m); 2364 2365 return (error); 2366 } 2367 2368 2369 /* 2370 * netmap_reset() is called by the driver routines when reinitializing 2371 * a ring. The driver is in charge of locking to protect the kring. 2372 * If netmap mode is not set just return NULL. 2373 */ 2374 struct netmap_slot * 2375 netmap_reset(struct netmap_adapter *na, enum txrx tx, int n, 2376 u_int new_cur) 2377 { 2378 struct netmap_kring *kring; 2379 int new_hwofs, lim; 2380 2381 if (na == NULL) 2382 return NULL; /* no netmap support here */ 2383 if (!(na->ifp->if_capenable & IFCAP_NETMAP)) 2384 return NULL; /* nothing to reinitialize */ 2385 2386 if (tx == NR_TX) { 2387 if (n >= na->num_tx_rings) 2388 return NULL; 2389 kring = na->tx_rings + n; 2390 new_hwofs = kring->nr_hwcur - new_cur; 2391 } else { 2392 if (n >= na->num_rx_rings) 2393 return NULL; 2394 kring = na->rx_rings + n; 2395 new_hwofs = kring->nr_hwcur + kring->nr_hwavail - new_cur; 2396 } 2397 lim = kring->nkr_num_slots - 1; 2398 if (new_hwofs > lim) 2399 new_hwofs -= lim + 1; 2400 2401 /* Alwayws set the new offset value and realign the ring. */ 2402 kring->nkr_hwofs = new_hwofs; 2403 if (tx == NR_TX) 2404 kring->nr_hwavail = kring->nkr_num_slots - 1; 2405 ND(10, "new hwofs %d on %s %s[%d]", 2406 kring->nkr_hwofs, na->ifp->if_xname, 2407 tx == NR_TX ? "TX" : "RX", n); 2408 2409 #if 0 // def linux 2410 /* XXX check that the mappings are correct */ 2411 /* need ring_nr, adapter->pdev, direction */ 2412 buffer_info->dma = dma_map_single(&pdev->dev, addr, adapter->rx_buffer_len, DMA_FROM_DEVICE); 2413 if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 2414 D("error mapping rx netmap buffer %d", i); 2415 // XXX fix error handling 2416 } 2417 2418 #endif /* linux */ 2419 /* 2420 * Wakeup on the individual and global lock 2421 * We do the wakeup here, but the ring is not yet reconfigured. 2422 * However, we are under lock so there are no races. 2423 */ 2424 selwakeuppri(&kring->si, PI_NET); 2425 selwakeuppri(tx == NR_TX ? &na->tx_si : &na->rx_si, PI_NET); 2426 return kring->ring->slot; 2427 } 2428 2429 2430 /* returns the next position in the ring */ 2431 static int 2432 nm_bdg_preflush(struct netmap_adapter *na, u_int ring_nr, 2433 struct netmap_kring *kring, u_int end) 2434 { 2435 struct netmap_ring *ring = kring->ring; 2436 struct nm_bdg_fwd *ft = kring->nkr_ft; 2437 u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1; 2438 u_int ft_i = 0; /* start from 0 */ 2439 2440 for (; likely(j != end); j = unlikely(j == lim) ? 0 : j+1) { 2441 struct netmap_slot *slot = &ring->slot[j]; 2442 char *buf = NMB(slot); 2443 int len = ft[ft_i].ft_len = slot->len; 2444 2445 ft[ft_i].ft_flags = slot->flags; 2446 2447 ND("flags is 0x%x", slot->flags); 2448 /* this slot goes into a list so initialize the link field */ 2449 ft[ft_i].ft_next = NM_BDG_BATCH; /* equivalent to NULL */ 2450 if (unlikely(len < 14)) 2451 continue; 2452 buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ? 2453 *((void **)buf) : buf; 2454 prefetch(buf); 2455 if (unlikely(++ft_i == netmap_bridge)) 2456 ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr); 2457 } 2458 if (ft_i) 2459 ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr); 2460 return j; 2461 } 2462 2463 2464 /* 2465 * Pass packets from nic to the bridge. Must be called with 2466 * proper locks on the source interface. 2467 * Note, no user process can access this NIC so we can ignore 2468 * the info in the 'ring'. 2469 */ 2470 static void 2471 netmap_nic_to_bdg(struct ifnet *ifp, u_int ring_nr) 2472 { 2473 struct netmap_adapter *na = NA(ifp); 2474 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 2475 struct netmap_ring *ring = kring->ring; 2476 int j, k, lim = kring->nkr_num_slots - 1; 2477 2478 /* fetch packets that have arrived */ 2479 na->nm_rxsync(ifp, ring_nr, 0); 2480 /* XXX we don't count reserved, but it should be 0 */ 2481 j = kring->nr_hwcur; 2482 k = j + kring->nr_hwavail; 2483 if (k > lim) 2484 k -= lim + 1; 2485 if (k == j && netmap_verbose) { 2486 D("how strange, interrupt with no packets on %s", 2487 ifp->if_xname); 2488 return; 2489 } 2490 2491 j = nm_bdg_preflush(na, ring_nr, kring, k); 2492 2493 /* we consume everything, but we cannot update kring directly 2494 * because the nic may have destroyed the info in the NIC ring. 2495 * So we need to call rxsync again to restore it. 2496 */ 2497 ring->cur = j; 2498 ring->avail = 0; 2499 na->nm_rxsync(ifp, ring_nr, 0); 2500 return; 2501 } 2502 2503 2504 /* 2505 * Default functions to handle rx/tx interrupts 2506 * we have 4 cases: 2507 * 1 ring, single lock: 2508 * lock(core); wake(i=0); unlock(core) 2509 * N rings, single lock: 2510 * lock(core); wake(i); wake(N+1) unlock(core) 2511 * 1 ring, separate locks: (i=0) 2512 * lock(i); wake(i); unlock(i) 2513 * N rings, separate locks: 2514 * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core) 2515 * work_done is non-null on the RX path. 2516 * 2517 * The 'q' argument also includes flag to tell whether the queue is 2518 * already locked on enter, and whether it should remain locked on exit. 2519 * This helps adapting to different defaults in drivers and OSes. 2520 */ 2521 int 2522 netmap_rx_irq(struct ifnet *ifp, int q, int *work_done) 2523 { 2524 struct netmap_adapter *na; 2525 struct netmap_kring *r; 2526 NM_SELINFO_T *main_wq; 2527 int locktype, unlocktype, nic_to_bridge, lock; 2528 2529 if (!(ifp->if_capenable & IFCAP_NETMAP)) 2530 return 0; 2531 2532 lock = q & (NETMAP_LOCKED_ENTER | NETMAP_LOCKED_EXIT); 2533 q = q & NETMAP_RING_MASK; 2534 2535 ND(5, "received %s queue %d", work_done ? "RX" : "TX" , q); 2536 na = NA(ifp); 2537 if (na->na_flags & NAF_SKIP_INTR) { 2538 ND("use regular interrupt"); 2539 return 0; 2540 } 2541 2542 if (work_done) { /* RX path */ 2543 if (q >= na->num_rx_rings) 2544 return 0; // not a physical queue 2545 r = na->rx_rings + q; 2546 r->nr_kflags |= NKR_PENDINTR; 2547 main_wq = (na->num_rx_rings > 1) ? &na->rx_si : NULL; 2548 /* set a flag if the NIC is attached to a VALE switch */ 2549 nic_to_bridge = (na->na_bdg != NULL); 2550 locktype = NETMAP_RX_LOCK; 2551 unlocktype = NETMAP_RX_UNLOCK; 2552 } else { /* TX path */ 2553 if (q >= na->num_tx_rings) 2554 return 0; // not a physical queue 2555 r = na->tx_rings + q; 2556 main_wq = (na->num_tx_rings > 1) ? &na->tx_si : NULL; 2557 work_done = &q; /* dummy */ 2558 nic_to_bridge = 0; 2559 locktype = NETMAP_TX_LOCK; 2560 unlocktype = NETMAP_TX_UNLOCK; 2561 } 2562 if (na->separate_locks) { 2563 if (!(lock & NETMAP_LOCKED_ENTER)) 2564 na->nm_lock(ifp, locktype, q); 2565 /* If a NIC is attached to a bridge, flush packets 2566 * (and no need to wakeup anyone). Otherwise, wakeup 2567 * possible processes waiting for packets. 2568 */ 2569 if (nic_to_bridge) 2570 netmap_nic_to_bdg(ifp, q); 2571 else 2572 selwakeuppri(&r->si, PI_NET); 2573 na->nm_lock(ifp, unlocktype, q); 2574 if (main_wq && !nic_to_bridge) { 2575 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2576 selwakeuppri(main_wq, PI_NET); 2577 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 2578 } 2579 /* lock the queue again if requested */ 2580 if (lock & NETMAP_LOCKED_EXIT) 2581 na->nm_lock(ifp, locktype, q); 2582 } else { 2583 if (!(lock & NETMAP_LOCKED_ENTER)) 2584 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0); 2585 if (nic_to_bridge) 2586 netmap_nic_to_bdg(ifp, q); 2587 else { 2588 selwakeuppri(&r->si, PI_NET); 2589 if (main_wq) 2590 selwakeuppri(main_wq, PI_NET); 2591 } 2592 if (!(lock & NETMAP_LOCKED_EXIT)) 2593 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0); 2594 } 2595 *work_done = 1; /* do not fire napi again */ 2596 return 1; 2597 } 2598 2599 2600 #ifdef linux /* linux-specific routines */ 2601 2602 2603 /* 2604 * Remap linux arguments into the FreeBSD call. 2605 * - pwait is the poll table, passed as 'dev'; 2606 * If pwait == NULL someone else already woke up before. We can report 2607 * events but they are filtered upstream. 2608 * If pwait != NULL, then pwait->key contains the list of events. 2609 * - events is computed from pwait as above. 2610 * - file is passed as 'td'; 2611 */ 2612 static u_int 2613 linux_netmap_poll(struct file * file, struct poll_table_struct *pwait) 2614 { 2615 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,28) 2616 int events = POLLIN | POLLOUT; /* XXX maybe... */ 2617 #elif LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0) 2618 int events = pwait ? pwait->key : POLLIN | POLLOUT; 2619 #else /* in 3.4.0 field 'key' was renamed to '_key' */ 2620 int events = pwait ? pwait->_key : POLLIN | POLLOUT; 2621 #endif 2622 return netmap_poll((void *)pwait, events, (void *)file); 2623 } 2624 2625 2626 static int 2627 linux_netmap_mmap(struct file *f, struct vm_area_struct *vma) 2628 { 2629 int lut_skip, i, j; 2630 int user_skip = 0; 2631 struct lut_entry *l_entry; 2632 int error = 0; 2633 unsigned long off, tomap; 2634 /* 2635 * vma->vm_start: start of mapping user address space 2636 * vma->vm_end: end of the mapping user address space 2637 * vma->vm_pfoff: offset of first page in the device 2638 */ 2639 2640 // XXX security checks 2641 2642 error = netmap_get_memory(f->private_data); 2643 ND("get_memory returned %d", error); 2644 if (error) 2645 return -error; 2646 2647 off = vma->vm_pgoff << PAGE_SHIFT; /* offset in bytes */ 2648 tomap = vma->vm_end - vma->vm_start; 2649 for (i = 0; i < NETMAP_POOLS_NR; i++) { /* loop through obj_pools */ 2650 const struct netmap_obj_pool *p = &nm_mem.pools[i]; 2651 /* 2652 * In each pool memory is allocated in clusters 2653 * of size _clustsize, each containing clustentries 2654 * entries. For each object k we already store the 2655 * vtophys mapping in lut[k] so we use that, scanning 2656 * the lut[] array in steps of clustentries, 2657 * and we map each cluster (not individual pages, 2658 * it would be overkill -- XXX slow ? 20130415). 2659 */ 2660 2661 /* 2662 * We interpret vm_pgoff as an offset into the whole 2663 * netmap memory, as if all clusters where contiguous. 2664 */ 2665 for (lut_skip = 0, j = 0; j < p->_numclusters; j++, lut_skip += p->clustentries) { 2666 unsigned long paddr, mapsize; 2667 if (p->_clustsize <= off) { 2668 off -= p->_clustsize; 2669 continue; 2670 } 2671 l_entry = &p->lut[lut_skip]; /* first obj in the cluster */ 2672 paddr = l_entry->paddr + off; 2673 mapsize = p->_clustsize - off; 2674 off = 0; 2675 if (mapsize > tomap) 2676 mapsize = tomap; 2677 ND("remap_pfn_range(%lx, %lx, %lx)", 2678 vma->vm_start + user_skip, 2679 paddr >> PAGE_SHIFT, mapsize); 2680 if (remap_pfn_range(vma, vma->vm_start + user_skip, 2681 paddr >> PAGE_SHIFT, mapsize, 2682 vma->vm_page_prot)) 2683 return -EAGAIN; // XXX check return value 2684 user_skip += mapsize; 2685 tomap -= mapsize; 2686 if (tomap == 0) 2687 goto done; 2688 } 2689 } 2690 done: 2691 2692 return 0; 2693 } 2694 2695 2696 static netdev_tx_t 2697 linux_netmap_start(struct sk_buff *skb, struct net_device *dev) 2698 { 2699 netmap_start(dev, skb); 2700 return (NETDEV_TX_OK); 2701 } 2702 2703 2704 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) // XXX was 38 2705 #define LIN_IOCTL_NAME .ioctl 2706 int 2707 linux_netmap_ioctl(struct inode *inode, struct file *file, u_int cmd, u_long data /* arg */) 2708 #else 2709 #define LIN_IOCTL_NAME .unlocked_ioctl 2710 long 2711 linux_netmap_ioctl(struct file *file, u_int cmd, u_long data /* arg */) 2712 #endif 2713 { 2714 int ret; 2715 struct nmreq nmr; 2716 bzero(&nmr, sizeof(nmr)); 2717 2718 if (data && copy_from_user(&nmr, (void *)data, sizeof(nmr) ) != 0) 2719 return -EFAULT; 2720 ret = netmap_ioctl(NULL, cmd, (caddr_t)&nmr, 0, (void *)file); 2721 if (data && copy_to_user((void*)data, &nmr, sizeof(nmr) ) != 0) 2722 return -EFAULT; 2723 return -ret; 2724 } 2725 2726 2727 static int 2728 netmap_release(struct inode *inode, struct file *file) 2729 { 2730 (void)inode; /* UNUSED */ 2731 if (file->private_data) 2732 netmap_dtor(file->private_data); 2733 return (0); 2734 } 2735 2736 2737 static int 2738 linux_netmap_open(struct inode *inode, struct file *file) 2739 { 2740 struct netmap_priv_d *priv; 2741 (void)inode; /* UNUSED */ 2742 2743 priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 2744 M_NOWAIT | M_ZERO); 2745 if (priv == NULL) 2746 return -ENOMEM; 2747 2748 file->private_data = priv; 2749 2750 return (0); 2751 } 2752 2753 2754 static struct file_operations netmap_fops = { 2755 .owner = THIS_MODULE, 2756 .open = linux_netmap_open, 2757 .mmap = linux_netmap_mmap, 2758 LIN_IOCTL_NAME = linux_netmap_ioctl, 2759 .poll = linux_netmap_poll, 2760 .release = netmap_release, 2761 }; 2762 2763 2764 static struct miscdevice netmap_cdevsw = { /* same name as FreeBSD */ 2765 MISC_DYNAMIC_MINOR, 2766 "netmap", 2767 &netmap_fops, 2768 }; 2769 2770 static int netmap_init(void); 2771 static void netmap_fini(void); 2772 2773 2774 /* Errors have negative values on linux */ 2775 static int linux_netmap_init(void) 2776 { 2777 return -netmap_init(); 2778 } 2779 2780 module_init(linux_netmap_init); 2781 module_exit(netmap_fini); 2782 /* export certain symbols to other modules */ 2783 EXPORT_SYMBOL(netmap_attach); // driver attach routines 2784 EXPORT_SYMBOL(netmap_detach); // driver detach routines 2785 EXPORT_SYMBOL(netmap_ring_reinit); // ring init on error 2786 EXPORT_SYMBOL(netmap_buffer_lut); 2787 EXPORT_SYMBOL(netmap_total_buffers); // index check 2788 EXPORT_SYMBOL(netmap_buffer_base); 2789 EXPORT_SYMBOL(netmap_reset); // ring init routines 2790 EXPORT_SYMBOL(netmap_buf_size); 2791 EXPORT_SYMBOL(netmap_rx_irq); // default irq handler 2792 EXPORT_SYMBOL(netmap_no_pendintr); // XXX mitigation - should go away 2793 EXPORT_SYMBOL(netmap_bdg_ctl); // bridge configuration routine 2794 EXPORT_SYMBOL(netmap_bdg_learning); // the default lookup function 2795 2796 2797 MODULE_AUTHOR("http://info.iet.unipi.it/~luigi/netmap/"); 2798 MODULE_DESCRIPTION("The netmap packet I/O framework"); 2799 MODULE_LICENSE("Dual BSD/GPL"); /* the code here is all BSD. */ 2800 2801 #else /* __FreeBSD__ */ 2802 2803 2804 static struct cdevsw netmap_cdevsw = { 2805 .d_version = D_VERSION, 2806 .d_name = "netmap", 2807 .d_open = netmap_open, 2808 .d_mmap = netmap_mmap, 2809 .d_mmap_single = netmap_mmap_single, 2810 .d_ioctl = netmap_ioctl, 2811 .d_poll = netmap_poll, 2812 .d_close = netmap_close, 2813 }; 2814 #endif /* __FreeBSD__ */ 2815 2816 #ifdef NM_BRIDGE 2817 /* 2818 *---- support for virtual bridge ----- 2819 */ 2820 2821 /* ----- FreeBSD if_bridge hash function ------- */ 2822 2823 /* 2824 * The following hash function is adapted from "Hash Functions" by Bob Jenkins 2825 * ("Algorithm Alley", Dr. Dobbs Journal, September 1997). 2826 * 2827 * http://www.burtleburtle.net/bob/hash/spooky.html 2828 */ 2829 #define mix(a, b, c) \ 2830 do { \ 2831 a -= b; a -= c; a ^= (c >> 13); \ 2832 b -= c; b -= a; b ^= (a << 8); \ 2833 c -= a; c -= b; c ^= (b >> 13); \ 2834 a -= b; a -= c; a ^= (c >> 12); \ 2835 b -= c; b -= a; b ^= (a << 16); \ 2836 c -= a; c -= b; c ^= (b >> 5); \ 2837 a -= b; a -= c; a ^= (c >> 3); \ 2838 b -= c; b -= a; b ^= (a << 10); \ 2839 c -= a; c -= b; c ^= (b >> 15); \ 2840 } while (/*CONSTCOND*/0) 2841 2842 static __inline uint32_t 2843 nm_bridge_rthash(const uint8_t *addr) 2844 { 2845 uint32_t a = 0x9e3779b9, b = 0x9e3779b9, c = 0; // hask key 2846 2847 b += addr[5] << 8; 2848 b += addr[4]; 2849 a += addr[3] << 24; 2850 a += addr[2] << 16; 2851 a += addr[1] << 8; 2852 a += addr[0]; 2853 2854 mix(a, b, c); 2855 #define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1) 2856 return (c & BRIDGE_RTHASH_MASK); 2857 } 2858 2859 #undef mix 2860 2861 2862 static int 2863 bdg_netmap_reg(struct ifnet *ifp, int onoff) 2864 { 2865 // struct nm_bridge *b = NA(ifp)->na_bdg; 2866 2867 /* the interface is already attached to the bridge, 2868 * so we only need to toggle IFCAP_NETMAP. 2869 * Locking is not necessary (we are already under 2870 * NMA_LOCK, and the port is not in use during this call). 2871 */ 2872 /* BDG_WLOCK(b); */ 2873 if (onoff) { 2874 ifp->if_capenable |= IFCAP_NETMAP; 2875 } else { 2876 ifp->if_capenable &= ~IFCAP_NETMAP; 2877 } 2878 /* BDG_WUNLOCK(b); */ 2879 return 0; 2880 } 2881 2882 2883 /* 2884 * Lookup function for a learning bridge. 2885 * Update the hash table with the source address, 2886 * and then returns the destination port index, and the 2887 * ring in *dst_ring (at the moment, always use ring 0) 2888 */ 2889 u_int 2890 netmap_bdg_learning(char *buf, u_int len, uint8_t *dst_ring, 2891 struct netmap_adapter *na) 2892 { 2893 struct nm_hash_ent *ht = na->na_bdg->ht; 2894 uint32_t sh, dh; 2895 u_int dst, mysrc = na->bdg_port; 2896 uint64_t smac, dmac; 2897 2898 dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff; 2899 smac = le64toh(*(uint64_t *)(buf + 4)); 2900 smac >>= 16; 2901 2902 /* 2903 * The hash is somewhat expensive, there might be some 2904 * worthwhile optimizations here. 2905 */ 2906 if ((buf[6] & 1) == 0) { /* valid src */ 2907 uint8_t *s = buf+6; 2908 sh = nm_bridge_rthash(buf+6); // XXX hash of source 2909 /* update source port forwarding entry */ 2910 ht[sh].mac = smac; /* XXX expire ? */ 2911 ht[sh].ports = mysrc; 2912 if (netmap_verbose) 2913 D("src %02x:%02x:%02x:%02x:%02x:%02x on port %d", 2914 s[0], s[1], s[2], s[3], s[4], s[5], mysrc); 2915 } 2916 dst = NM_BDG_BROADCAST; 2917 if ((buf[0] & 1) == 0) { /* unicast */ 2918 dh = nm_bridge_rthash(buf); // XXX hash of dst 2919 if (ht[dh].mac == dmac) { /* found dst */ 2920 dst = ht[dh].ports; 2921 } 2922 /* XXX otherwise return NM_BDG_UNKNOWN ? */ 2923 } 2924 *dst_ring = 0; 2925 return dst; 2926 } 2927 2928 2929 /* 2930 * This flush routine supports only unicast and broadcast but a large 2931 * number of ports, and lets us replace the learn and dispatch functions. 2932 */ 2933 int 2934 nm_bdg_flush(struct nm_bdg_fwd *ft, int n, struct netmap_adapter *na, 2935 u_int ring_nr) 2936 { 2937 struct nm_bdg_q *dst_ents, *brddst; 2938 uint16_t num_dsts = 0, *dsts; 2939 struct nm_bridge *b = na->na_bdg; 2940 u_int i, me = na->bdg_port; 2941 2942 dst_ents = (struct nm_bdg_q *)(ft + NM_BDG_BATCH); 2943 dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1); 2944 2945 BDG_RLOCK(b); 2946 2947 /* first pass: find a destination */ 2948 for (i = 0; likely(i < n); i++) { 2949 uint8_t *buf = ft[i].ft_buf; 2950 uint8_t dst_ring = ring_nr; 2951 uint16_t dst_port, d_i; 2952 struct nm_bdg_q *d; 2953 2954 dst_port = b->nm_bdg_lookup(buf, ft[i].ft_len, &dst_ring, na); 2955 if (dst_port == NM_BDG_NOPORT) { 2956 continue; /* this packet is identified to be dropped */ 2957 } else if (unlikely(dst_port > NM_BDG_MAXPORTS)) { 2958 continue; 2959 } else if (dst_port == NM_BDG_BROADCAST) { 2960 dst_ring = 0; /* broadcasts always go to ring 0 */ 2961 } else if (unlikely(dst_port == me || 2962 !BDG_GET_VAR(b->bdg_ports[dst_port]))) { 2963 continue; 2964 } 2965 2966 /* get a position in the scratch pad */ 2967 d_i = dst_port * NM_BDG_MAXRINGS + dst_ring; 2968 d = dst_ents + d_i; 2969 if (d->bq_head == NM_BDG_BATCH) { /* new destination */ 2970 d->bq_head = d->bq_tail = i; 2971 /* remember this position to be scanned later */ 2972 if (dst_port != NM_BDG_BROADCAST) 2973 dsts[num_dsts++] = d_i; 2974 } else { 2975 ft[d->bq_tail].ft_next = i; 2976 d->bq_tail = i; 2977 } 2978 } 2979 2980 /* if there is a broadcast, set ring 0 of all ports to be scanned 2981 * XXX This would be optimized by recording the highest index of active 2982 * ports. 2983 */ 2984 brddst = dst_ents + NM_BDG_BROADCAST * NM_BDG_MAXRINGS; 2985 if (brddst->bq_head != NM_BDG_BATCH) { 2986 for (i = 0; likely(i < NM_BDG_MAXPORTS); i++) { 2987 uint16_t d_i = i * NM_BDG_MAXRINGS; 2988 if (unlikely(i == me) || !BDG_GET_VAR(b->bdg_ports[i])) 2989 continue; 2990 else if (dst_ents[d_i].bq_head == NM_BDG_BATCH) 2991 dsts[num_dsts++] = d_i; 2992 } 2993 } 2994 2995 /* second pass: scan destinations (XXX will be modular somehow) */ 2996 for (i = 0; i < num_dsts; i++) { 2997 struct ifnet *dst_ifp; 2998 struct netmap_adapter *dst_na; 2999 struct netmap_kring *kring; 3000 struct netmap_ring *ring; 3001 u_int dst_nr, is_vp, lim, j, sent = 0, d_i, next, brd_next; 3002 int howmany, retry = netmap_txsync_retry; 3003 struct nm_bdg_q *d; 3004 3005 d_i = dsts[i]; 3006 d = dst_ents + d_i; 3007 dst_na = BDG_GET_VAR(b->bdg_ports[d_i/NM_BDG_MAXRINGS]); 3008 /* protect from the lookup function returning an inactive 3009 * destination port 3010 */ 3011 if (unlikely(dst_na == NULL)) 3012 continue; 3013 else if (dst_na->na_flags & NAF_SW_ONLY) 3014 continue; 3015 dst_ifp = dst_na->ifp; 3016 /* 3017 * The interface may be in !netmap mode in two cases: 3018 * - when na is attached but not activated yet; 3019 * - when na is being deactivated but is still attached. 3020 */ 3021 if (unlikely(!(dst_ifp->if_capenable & IFCAP_NETMAP))) 3022 continue; 3023 3024 /* there is at least one either unicast or broadcast packet */ 3025 brd_next = brddst->bq_head; 3026 next = d->bq_head; 3027 3028 is_vp = nma_is_vp(dst_na); 3029 dst_nr = d_i & (NM_BDG_MAXRINGS-1); 3030 if (is_vp) { /* virtual port */ 3031 if (dst_nr >= dst_na->num_rx_rings) 3032 dst_nr = dst_nr % dst_na->num_rx_rings; 3033 kring = &dst_na->rx_rings[dst_nr]; 3034 ring = kring->ring; 3035 lim = kring->nkr_num_slots - 1; 3036 dst_na->nm_lock(dst_ifp, NETMAP_RX_LOCK, dst_nr); 3037 j = kring->nr_hwcur + kring->nr_hwavail; 3038 if (j > lim) 3039 j -= kring->nkr_num_slots; 3040 howmany = lim - kring->nr_hwavail; 3041 } else { /* hw or sw adapter */ 3042 if (dst_nr >= dst_na->num_tx_rings) 3043 dst_nr = dst_nr % dst_na->num_tx_rings; 3044 kring = &dst_na->tx_rings[dst_nr]; 3045 ring = kring->ring; 3046 lim = kring->nkr_num_slots - 1; 3047 dst_na->nm_lock(dst_ifp, NETMAP_TX_LOCK, dst_nr); 3048 retry: 3049 dst_na->nm_txsync(dst_ifp, dst_nr, 0); 3050 /* see nm_bdg_flush() */ 3051 j = kring->nr_hwcur; 3052 howmany = kring->nr_hwavail; 3053 } 3054 while (howmany-- > 0) { 3055 struct netmap_slot *slot; 3056 struct nm_bdg_fwd *ft_p; 3057 3058 /* our 'NULL' is always higher than valid indexes 3059 * so we never dereference it if the other list 3060 * has packets (and if both are NULL we never 3061 * get here). 3062 */ 3063 if (next < brd_next) { 3064 ft_p = ft + next; 3065 next = ft_p->ft_next; 3066 ND("j %d uni %d next %d %d", 3067 j, ft_p - ft, next, brd_next); 3068 } else { /* insert broadcast */ 3069 ft_p = ft + brd_next; 3070 brd_next = ft_p->ft_next; 3071 ND("j %d brd %d next %d %d", 3072 j, ft_p - ft, next, brd_next); 3073 } 3074 slot = &ring->slot[j]; 3075 ND("send %d %d bytes at %s:%d", i, ft_p->ft_len, dst_ifp->if_xname, j); 3076 if (ft_p->ft_flags & NS_INDIRECT) { 3077 ND("copying from INDIRECT source"); 3078 copyin(ft_p->ft_buf, NMB(slot), 3079 (ft_p->ft_len + 63) & ~63); 3080 } else { 3081 pkt_copy(ft_p->ft_buf, NMB(slot), ft_p->ft_len); 3082 } 3083 slot->len = ft_p->ft_len; 3084 j = unlikely(j == lim) ? 0: j + 1; /* XXX to be macro-ed */ 3085 sent++; 3086 /* are we done ? */ 3087 if (next == NM_BDG_BATCH && brd_next == NM_BDG_BATCH) 3088 break; 3089 } 3090 if (netmap_verbose && (howmany < 0)) 3091 D("rx ring full on %s", dst_ifp->if_xname); 3092 if (is_vp) { 3093 if (sent) { 3094 kring->nr_hwavail += sent; 3095 selwakeuppri(&kring->si, PI_NET); 3096 } 3097 dst_na->nm_lock(dst_ifp, NETMAP_RX_UNLOCK, dst_nr); 3098 } else { 3099 if (sent) { 3100 ring->avail -= sent; 3101 ring->cur = j; 3102 dst_na->nm_txsync(dst_ifp, dst_nr, 0); 3103 } 3104 /* retry to send more packets */ 3105 if (nma_is_hw(dst_na) && howmany < 0 && retry--) 3106 goto retry; 3107 dst_na->nm_lock(dst_ifp, NETMAP_TX_UNLOCK, dst_nr); 3108 } 3109 /* NM_BDG_BATCH means 'no packet' */ 3110 d->bq_head = d->bq_tail = NM_BDG_BATCH; /* cleanup */ 3111 } 3112 brddst->bq_head = brddst->bq_tail = NM_BDG_BATCH; /* cleanup */ 3113 BDG_RUNLOCK(b); 3114 return 0; 3115 } 3116 3117 3118 /* 3119 * main dispatch routine 3120 */ 3121 static int 3122 bdg_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 3123 { 3124 struct netmap_adapter *na = NA(ifp); 3125 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 3126 struct netmap_ring *ring = kring->ring; 3127 int i, j, k, lim = kring->nkr_num_slots - 1; 3128 3129 k = ring->cur; 3130 if (k > lim) 3131 return netmap_ring_reinit(kring); 3132 if (do_lock) 3133 na->nm_lock(ifp, NETMAP_TX_LOCK, ring_nr); 3134 3135 if (netmap_bridge <= 0) { /* testing only */ 3136 j = k; // used all 3137 goto done; 3138 } 3139 if (netmap_bridge > NM_BDG_BATCH) 3140 netmap_bridge = NM_BDG_BATCH; 3141 3142 j = nm_bdg_preflush(na, ring_nr, kring, k); 3143 i = k - j; 3144 if (i < 0) 3145 i += kring->nkr_num_slots; 3146 kring->nr_hwavail = kring->nkr_num_slots - 1 - i; 3147 if (j != k) 3148 D("early break at %d/ %d, avail %d", j, k, kring->nr_hwavail); 3149 3150 done: 3151 kring->nr_hwcur = j; 3152 ring->avail = kring->nr_hwavail; 3153 if (do_lock) 3154 na->nm_lock(ifp, NETMAP_TX_UNLOCK, ring_nr); 3155 3156 if (netmap_verbose) 3157 D("%s ring %d lock %d", ifp->if_xname, ring_nr, do_lock); 3158 return 0; 3159 } 3160 3161 3162 static int 3163 bdg_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 3164 { 3165 struct netmap_adapter *na = NA(ifp); 3166 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 3167 struct netmap_ring *ring = kring->ring; 3168 u_int j, lim = kring->nkr_num_slots - 1; 3169 u_int k = ring->cur, resvd = ring->reserved; 3170 int n; 3171 3172 ND("%s ring %d lock %d avail %d", 3173 ifp->if_xname, ring_nr, do_lock, kring->nr_hwavail); 3174 3175 if (k > lim) 3176 return netmap_ring_reinit(kring); 3177 if (do_lock) 3178 na->nm_lock(ifp, NETMAP_RX_LOCK, ring_nr); 3179 3180 /* skip past packets that userspace has released */ 3181 j = kring->nr_hwcur; /* netmap ring index */ 3182 if (resvd > 0) { 3183 if (resvd + ring->avail >= lim + 1) { 3184 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 3185 ring->reserved = resvd = 0; // XXX panic... 3186 } 3187 k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; 3188 } 3189 3190 if (j != k) { /* userspace has released some packets. */ 3191 n = k - j; 3192 if (n < 0) 3193 n += kring->nkr_num_slots; 3194 ND("userspace releases %d packets", n); 3195 for (n = 0; likely(j != k); n++) { 3196 struct netmap_slot *slot = &ring->slot[j]; 3197 void *addr = NMB(slot); 3198 3199 if (addr == netmap_buffer_base) { /* bad buf */ 3200 if (do_lock) 3201 na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr); 3202 return netmap_ring_reinit(kring); 3203 } 3204 /* decrease refcount for buffer */ 3205 3206 slot->flags &= ~NS_BUF_CHANGED; 3207 j = unlikely(j == lim) ? 0 : j + 1; 3208 } 3209 kring->nr_hwavail -= n; 3210 kring->nr_hwcur = k; 3211 } 3212 /* tell userspace that there are new packets */ 3213 ring->avail = kring->nr_hwavail - resvd; 3214 3215 if (do_lock) 3216 na->nm_lock(ifp, NETMAP_RX_UNLOCK, ring_nr); 3217 return 0; 3218 } 3219 3220 3221 static void 3222 bdg_netmap_attach(struct netmap_adapter *arg) 3223 { 3224 struct netmap_adapter na; 3225 3226 ND("attaching virtual bridge"); 3227 bzero(&na, sizeof(na)); 3228 3229 na.ifp = arg->ifp; 3230 na.separate_locks = 1; 3231 na.num_tx_rings = arg->num_tx_rings; 3232 na.num_rx_rings = arg->num_rx_rings; 3233 na.num_tx_desc = NM_BRIDGE_RINGSIZE; 3234 na.num_rx_desc = NM_BRIDGE_RINGSIZE; 3235 na.nm_txsync = bdg_netmap_txsync; 3236 na.nm_rxsync = bdg_netmap_rxsync; 3237 na.nm_register = bdg_netmap_reg; 3238 netmap_attach(&na, na.num_tx_rings); 3239 } 3240 3241 #endif /* NM_BRIDGE */ 3242 3243 static struct cdev *netmap_dev; /* /dev/netmap character device. */ 3244 3245 3246 /* 3247 * Module loader. 3248 * 3249 * Create the /dev/netmap device and initialize all global 3250 * variables. 3251 * 3252 * Return 0 on success, errno on failure. 3253 */ 3254 static int 3255 netmap_init(void) 3256 { 3257 int error; 3258 3259 error = netmap_memory_init(); 3260 if (error != 0) { 3261 printf("netmap: unable to initialize the memory allocator.\n"); 3262 return (error); 3263 } 3264 printf("netmap: loaded module\n"); 3265 netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660, 3266 "netmap"); 3267 3268 #ifdef NM_BRIDGE 3269 { 3270 int i; 3271 mtx_init(&netmap_bridge_mutex, "netmap_bridge_mutex", 3272 MTX_NETWORK_LOCK, MTX_DEF); 3273 bzero(nm_bridges, sizeof(struct nm_bridge) * NM_BRIDGES); /* safety */ 3274 for (i = 0; i < NM_BRIDGES; i++) 3275 rw_init(&nm_bridges[i].bdg_lock, "bdg lock"); 3276 } 3277 #endif 3278 return (error); 3279 } 3280 3281 3282 /* 3283 * Module unloader. 3284 * 3285 * Free all the memory, and destroy the ``/dev/netmap`` device. 3286 */ 3287 static void 3288 netmap_fini(void) 3289 { 3290 destroy_dev(netmap_dev); 3291 netmap_memory_fini(); 3292 printf("netmap: unloaded module.\n"); 3293 } 3294 3295 3296 #ifdef __FreeBSD__ 3297 /* 3298 * Kernel entry point. 3299 * 3300 * Initialize/finalize the module and return. 3301 * 3302 * Return 0 on success, errno on failure. 3303 */ 3304 static int 3305 netmap_loader(__unused struct module *module, int event, __unused void *arg) 3306 { 3307 int error = 0; 3308 3309 switch (event) { 3310 case MOD_LOAD: 3311 error = netmap_init(); 3312 break; 3313 3314 case MOD_UNLOAD: 3315 netmap_fini(); 3316 break; 3317 3318 default: 3319 error = EOPNOTSUPP; 3320 break; 3321 } 3322 3323 return (error); 3324 } 3325 3326 3327 DEV_MODULE(netmap, netmap_loader, NULL); 3328 #endif /* __FreeBSD__ */ 3329