1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* $FreeBSD$ */ 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 32 #include <sys/param.h> 33 #include <sys/module.h> 34 #include <sys/errno.h> 35 #include <sys/jail.h> 36 #include <sys/poll.h> /* POLLIN, POLLOUT */ 37 #include <sys/kernel.h> /* types used in module initialization */ 38 #include <sys/conf.h> /* DEV_MODULE_ORDERED */ 39 #include <sys/endian.h> 40 #include <sys/syscallsubr.h> /* kern_ioctl() */ 41 42 #include <sys/rwlock.h> 43 44 #include <vm/vm.h> /* vtophys */ 45 #include <vm/pmap.h> /* vtophys */ 46 #include <vm/vm_param.h> 47 #include <vm/vm_object.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_pager.h> 50 #include <vm/uma.h> 51 52 53 #include <sys/malloc.h> 54 #include <sys/socket.h> /* sockaddrs */ 55 #include <sys/selinfo.h> 56 #include <sys/kthread.h> /* kthread_add() */ 57 #include <sys/proc.h> /* PROC_LOCK() */ 58 #include <sys/unistd.h> /* RFNOWAIT */ 59 #include <sys/sched.h> /* sched_bind() */ 60 #include <sys/smp.h> /* mp_maxid */ 61 #include <net/if.h> 62 #include <net/if_var.h> 63 #include <net/if_types.h> /* IFT_ETHER */ 64 #include <net/ethernet.h> /* ether_ifdetach */ 65 #include <net/if_dl.h> /* LLADDR */ 66 #include <machine/bus.h> /* bus_dmamap_* */ 67 #include <netinet/in.h> /* in6_cksum_pseudo() */ 68 #include <machine/in_cksum.h> /* in_pseudo(), in_cksum_hdr() */ 69 70 #include <net/netmap.h> 71 #include <dev/netmap/netmap_kern.h> 72 #include <net/netmap_virt.h> 73 #include <dev/netmap/netmap_mem2.h> 74 75 76 /* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 77 78 void nm_os_selinfo_init(NM_SELINFO_T *si) { 79 struct mtx *m = &si->m; 80 mtx_init(m, "nm_kn_lock", NULL, MTX_DEF); 81 knlist_init_mtx(&si->si.si_note, m); 82 } 83 84 void 85 nm_os_selinfo_uninit(NM_SELINFO_T *si) 86 { 87 /* XXX kqueue(9) needed; these will mirror knlist_init. */ 88 knlist_delete(&si->si.si_note, curthread, 0 /* not locked */ ); 89 knlist_destroy(&si->si.si_note); 90 /* now we don't need the mutex anymore */ 91 mtx_destroy(&si->m); 92 } 93 94 void * 95 nm_os_malloc(size_t size) 96 { 97 return malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 98 } 99 100 void * 101 nm_os_realloc(void *addr, size_t new_size, size_t old_size __unused) 102 { 103 return realloc(addr, new_size, M_DEVBUF, M_NOWAIT | M_ZERO); 104 } 105 106 void 107 nm_os_free(void *addr) 108 { 109 free(addr, M_DEVBUF); 110 } 111 112 void 113 nm_os_ifnet_lock(void) 114 { 115 IFNET_RLOCK(); 116 } 117 118 void 119 nm_os_ifnet_unlock(void) 120 { 121 IFNET_RUNLOCK(); 122 } 123 124 static int netmap_use_count = 0; 125 126 void 127 nm_os_get_module(void) 128 { 129 netmap_use_count++; 130 } 131 132 void 133 nm_os_put_module(void) 134 { 135 netmap_use_count--; 136 } 137 138 static void 139 netmap_ifnet_arrival_handler(void *arg __unused, struct ifnet *ifp) 140 { 141 netmap_undo_zombie(ifp); 142 } 143 144 static void 145 netmap_ifnet_departure_handler(void *arg __unused, struct ifnet *ifp) 146 { 147 netmap_make_zombie(ifp); 148 } 149 150 static eventhandler_tag nm_ifnet_ah_tag; 151 static eventhandler_tag nm_ifnet_dh_tag; 152 153 int 154 nm_os_ifnet_init(void) 155 { 156 nm_ifnet_ah_tag = 157 EVENTHANDLER_REGISTER(ifnet_arrival_event, 158 netmap_ifnet_arrival_handler, 159 NULL, EVENTHANDLER_PRI_ANY); 160 nm_ifnet_dh_tag = 161 EVENTHANDLER_REGISTER(ifnet_departure_event, 162 netmap_ifnet_departure_handler, 163 NULL, EVENTHANDLER_PRI_ANY); 164 return 0; 165 } 166 167 void 168 nm_os_ifnet_fini(void) 169 { 170 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 171 nm_ifnet_ah_tag); 172 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 173 nm_ifnet_dh_tag); 174 } 175 176 unsigned 177 nm_os_ifnet_mtu(struct ifnet *ifp) 178 { 179 #if __FreeBSD_version < 1100030 180 return ifp->if_data.ifi_mtu; 181 #else /* __FreeBSD_version >= 1100030 */ 182 return ifp->if_mtu; 183 #endif 184 } 185 186 rawsum_t 187 nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) 188 { 189 /* TODO XXX please use the FreeBSD implementation for this. */ 190 uint16_t *words = (uint16_t *)data; 191 int nw = len / 2; 192 int i; 193 194 for (i = 0; i < nw; i++) 195 cur_sum += be16toh(words[i]); 196 197 if (len & 1) 198 cur_sum += (data[len-1] << 8); 199 200 return cur_sum; 201 } 202 203 /* Fold a raw checksum: 'cur_sum' is in host byte order, while the 204 * return value is in network byte order. 205 */ 206 uint16_t 207 nm_os_csum_fold(rawsum_t cur_sum) 208 { 209 /* TODO XXX please use the FreeBSD implementation for this. */ 210 while (cur_sum >> 16) 211 cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); 212 213 return htobe16((~cur_sum) & 0xFFFF); 214 } 215 216 uint16_t nm_os_csum_ipv4(struct nm_iphdr *iph) 217 { 218 #if 0 219 return in_cksum_hdr((void *)iph); 220 #else 221 return nm_os_csum_fold(nm_os_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); 222 #endif 223 } 224 225 void 226 nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 227 size_t datalen, uint16_t *check) 228 { 229 #ifdef INET 230 uint16_t pseudolen = datalen + iph->protocol; 231 232 /* Compute and insert the pseudo-header cheksum. */ 233 *check = in_pseudo(iph->saddr, iph->daddr, 234 htobe16(pseudolen)); 235 /* Compute the checksum on TCP/UDP header + payload 236 * (includes the pseudo-header). 237 */ 238 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 239 #else 240 static int notsupported = 0; 241 if (!notsupported) { 242 notsupported = 1; 243 D("inet4 segmentation not supported"); 244 } 245 #endif 246 } 247 248 void 249 nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 250 size_t datalen, uint16_t *check) 251 { 252 #ifdef INET6 253 *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); 254 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 255 #else 256 static int notsupported = 0; 257 if (!notsupported) { 258 notsupported = 1; 259 D("inet6 segmentation not supported"); 260 } 261 #endif 262 } 263 264 /* on FreeBSD we send up one packet at a time */ 265 void * 266 nm_os_send_up(struct ifnet *ifp, struct mbuf *m, struct mbuf *prev) 267 { 268 NA(ifp)->if_input(ifp, m); 269 return NULL; 270 } 271 272 int 273 nm_os_mbuf_has_csum_offld(struct mbuf *m) 274 { 275 return m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_SCTP | 276 CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | 277 CSUM_SCTP_IPV6); 278 } 279 280 int 281 nm_os_mbuf_has_seg_offld(struct mbuf *m) 282 { 283 return m->m_pkthdr.csum_flags & CSUM_TSO; 284 } 285 286 static void 287 freebsd_generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 288 { 289 int stolen; 290 291 if (!NM_NA_VALID(ifp)) { 292 RD(1, "Warning: got RX packet for invalid emulated adapter"); 293 return; 294 } 295 296 stolen = generic_rx_handler(ifp, m); 297 if (!stolen) { 298 struct netmap_generic_adapter *gna = 299 (struct netmap_generic_adapter *)NA(ifp); 300 gna->save_if_input(ifp, m); 301 } 302 } 303 304 /* 305 * Intercept the rx routine in the standard device driver. 306 * Second argument is non-zero to intercept, 0 to restore 307 */ 308 int 309 nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept) 310 { 311 struct netmap_adapter *na = &gna->up.up; 312 struct ifnet *ifp = na->ifp; 313 int ret = 0; 314 315 nm_os_ifnet_lock(); 316 if (intercept) { 317 if (gna->save_if_input) { 318 D("cannot intercept again"); 319 ret = EINVAL; /* already set */ 320 goto out; 321 } 322 gna->save_if_input = ifp->if_input; 323 ifp->if_input = freebsd_generic_rx_handler; 324 } else { 325 if (!gna->save_if_input){ 326 D("cannot restore"); 327 ret = EINVAL; /* not saved */ 328 goto out; 329 } 330 ifp->if_input = gna->save_if_input; 331 gna->save_if_input = NULL; 332 } 333 out: 334 nm_os_ifnet_unlock(); 335 336 return ret; 337 } 338 339 340 /* 341 * Intercept the packet steering routine in the tx path, 342 * so that we can decide which queue is used for an mbuf. 343 * Second argument is non-zero to intercept, 0 to restore. 344 * On freebsd we just intercept if_transmit. 345 */ 346 int 347 nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept) 348 { 349 struct netmap_adapter *na = &gna->up.up; 350 struct ifnet *ifp = netmap_generic_getifp(gna); 351 352 nm_os_ifnet_lock(); 353 if (intercept) { 354 na->if_transmit = ifp->if_transmit; 355 ifp->if_transmit = netmap_transmit; 356 } else { 357 ifp->if_transmit = na->if_transmit; 358 } 359 nm_os_ifnet_unlock(); 360 361 return 0; 362 } 363 364 365 /* 366 * Transmit routine used by generic_netmap_txsync(). Returns 0 on success 367 * and non-zero on error (which may be packet drops or other errors). 368 * addr and len identify the netmap buffer, m is the (preallocated) 369 * mbuf to use for transmissions. 370 * 371 * We should add a reference to the mbuf so the m_freem() at the end 372 * of the transmission does not consume resources. 373 * 374 * On FreeBSD, and on multiqueue cards, we can force the queue using 375 * if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 376 * i = m->m_pkthdr.flowid % adapter->num_queues; 377 * else 378 * i = curcpu % adapter->num_queues; 379 * 380 */ 381 int 382 nm_os_generic_xmit_frame(struct nm_os_gen_arg *a) 383 { 384 int ret; 385 u_int len = a->len; 386 struct ifnet *ifp = a->ifp; 387 struct mbuf *m = a->m; 388 389 #if __FreeBSD_version < 1100000 390 /* 391 * Old FreeBSD versions. The mbuf has a cluster attached, 392 * we need to copy from the cluster to the netmap buffer. 393 */ 394 if (MBUF_REFCNT(m) != 1) { 395 D("invalid refcnt %d for %p", MBUF_REFCNT(m), m); 396 panic("in generic_xmit_frame"); 397 } 398 if (m->m_ext.ext_size < len) { 399 RD(5, "size %d < len %d", m->m_ext.ext_size, len); 400 len = m->m_ext.ext_size; 401 } 402 bcopy(a->addr, m->m_data, len); 403 #else /* __FreeBSD_version >= 1100000 */ 404 /* New FreeBSD versions. Link the external storage to 405 * the netmap buffer, so that no copy is necessary. */ 406 m->m_ext.ext_buf = m->m_data = a->addr; 407 m->m_ext.ext_size = len; 408 #endif /* __FreeBSD_version >= 1100000 */ 409 410 m->m_len = m->m_pkthdr.len = len; 411 412 /* mbuf refcnt is not contended, no need to use atomic 413 * (a memory barrier is enough). */ 414 SET_MBUF_REFCNT(m, 2); 415 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 416 m->m_pkthdr.flowid = a->ring_nr; 417 m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 418 ret = NA(ifp)->if_transmit(ifp, m); 419 return ret ? -1 : 0; 420 } 421 422 423 #if __FreeBSD_version >= 1100005 424 struct netmap_adapter * 425 netmap_getna(if_t ifp) 426 { 427 return (NA((struct ifnet *)ifp)); 428 } 429 #endif /* __FreeBSD_version >= 1100005 */ 430 431 /* 432 * The following two functions are empty until we have a generic 433 * way to extract the info from the ifp 434 */ 435 int 436 nm_os_generic_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) 437 { 438 return 0; 439 } 440 441 442 void 443 nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) 444 { 445 unsigned num_rings = netmap_generic_rings ? netmap_generic_rings : 1; 446 447 *txq = num_rings; 448 *rxq = num_rings; 449 } 450 451 void 452 nm_os_generic_set_features(struct netmap_generic_adapter *gna) 453 { 454 455 gna->rxsg = 1; /* Supported through m_copydata. */ 456 gna->txqdisc = 0; /* Not supported. */ 457 } 458 459 void 460 nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na) 461 { 462 ND("called"); 463 mit->mit_pending = 0; 464 mit->mit_ring_idx = idx; 465 mit->mit_na = na; 466 } 467 468 469 void 470 nm_os_mitigation_start(struct nm_generic_mit *mit) 471 { 472 ND("called"); 473 } 474 475 476 void 477 nm_os_mitigation_restart(struct nm_generic_mit *mit) 478 { 479 ND("called"); 480 } 481 482 483 int 484 nm_os_mitigation_active(struct nm_generic_mit *mit) 485 { 486 ND("called"); 487 return 0; 488 } 489 490 491 void 492 nm_os_mitigation_cleanup(struct nm_generic_mit *mit) 493 { 494 ND("called"); 495 } 496 497 static int 498 nm_vi_dummy(struct ifnet *ifp, u_long cmd, caddr_t addr) 499 { 500 return EINVAL; 501 } 502 503 static void 504 nm_vi_start(struct ifnet *ifp) 505 { 506 panic("nm_vi_start() must not be called"); 507 } 508 509 /* 510 * Index manager of persistent virtual interfaces. 511 * It is used to decide the lowest byte of the MAC address. 512 * We use the same algorithm with management of bridge port index. 513 */ 514 #define NM_VI_MAX 255 515 static struct { 516 uint8_t index[NM_VI_MAX]; /* XXX just for a reasonable number */ 517 uint8_t active; 518 struct mtx lock; 519 } nm_vi_indices; 520 521 void 522 nm_os_vi_init_index(void) 523 { 524 int i; 525 for (i = 0; i < NM_VI_MAX; i++) 526 nm_vi_indices.index[i] = i; 527 nm_vi_indices.active = 0; 528 mtx_init(&nm_vi_indices.lock, "nm_vi_indices_lock", NULL, MTX_DEF); 529 } 530 531 /* return -1 if no index available */ 532 static int 533 nm_vi_get_index(void) 534 { 535 int ret; 536 537 mtx_lock(&nm_vi_indices.lock); 538 ret = nm_vi_indices.active == NM_VI_MAX ? -1 : 539 nm_vi_indices.index[nm_vi_indices.active++]; 540 mtx_unlock(&nm_vi_indices.lock); 541 return ret; 542 } 543 544 static void 545 nm_vi_free_index(uint8_t val) 546 { 547 int i, lim; 548 549 mtx_lock(&nm_vi_indices.lock); 550 lim = nm_vi_indices.active; 551 for (i = 0; i < lim; i++) { 552 if (nm_vi_indices.index[i] == val) { 553 /* swap index[lim-1] and j */ 554 int tmp = nm_vi_indices.index[lim-1]; 555 nm_vi_indices.index[lim-1] = val; 556 nm_vi_indices.index[i] = tmp; 557 nm_vi_indices.active--; 558 break; 559 } 560 } 561 if (lim == nm_vi_indices.active) 562 D("funny, index %u didn't found", val); 563 mtx_unlock(&nm_vi_indices.lock); 564 } 565 #undef NM_VI_MAX 566 567 /* 568 * Implementation of a netmap-capable virtual interface that 569 * registered to the system. 570 * It is based on if_tap.c and ip_fw_log.c in FreeBSD 9. 571 * 572 * Note: Linux sets refcount to 0 on allocation of net_device, 573 * then increments it on registration to the system. 574 * FreeBSD sets refcount to 1 on if_alloc(), and does not 575 * increment this refcount on if_attach(). 576 */ 577 int 578 nm_os_vi_persist(const char *name, struct ifnet **ret) 579 { 580 struct ifnet *ifp; 581 u_short macaddr_hi; 582 uint32_t macaddr_mid; 583 u_char eaddr[6]; 584 int unit = nm_vi_get_index(); /* just to decide MAC address */ 585 586 if (unit < 0) 587 return EBUSY; 588 /* 589 * We use the same MAC address generation method with tap 590 * except for the highest octet is 00:be instead of 00:bd 591 */ 592 macaddr_hi = htons(0x00be); /* XXX tap + 1 */ 593 macaddr_mid = (uint32_t) ticks; 594 bcopy(&macaddr_hi, eaddr, sizeof(short)); 595 bcopy(&macaddr_mid, &eaddr[2], sizeof(uint32_t)); 596 eaddr[5] = (uint8_t)unit; 597 598 ifp = if_alloc(IFT_ETHER); 599 if (ifp == NULL) { 600 D("if_alloc failed"); 601 return ENOMEM; 602 } 603 if_initname(ifp, name, IF_DUNIT_NONE); 604 ifp->if_mtu = 65536; 605 ifp->if_flags = IFF_UP | IFF_SIMPLEX | IFF_MULTICAST; 606 ifp->if_init = (void *)nm_vi_dummy; 607 ifp->if_ioctl = nm_vi_dummy; 608 ifp->if_start = nm_vi_start; 609 ifp->if_mtu = ETHERMTU; 610 IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); 611 ifp->if_capabilities |= IFCAP_LINKSTATE; 612 ifp->if_capenable |= IFCAP_LINKSTATE; 613 614 ether_ifattach(ifp, eaddr); 615 *ret = ifp; 616 return 0; 617 } 618 619 /* unregister from the system and drop the final refcount */ 620 void 621 nm_os_vi_detach(struct ifnet *ifp) 622 { 623 nm_vi_free_index(((char *)IF_LLADDR(ifp))[5]); 624 ether_ifdetach(ifp); 625 if_free(ifp); 626 } 627 628 #ifdef WITH_EXTMEM 629 #include <vm/vm_map.h> 630 #include <vm/vm_kern.h> 631 struct nm_os_extmem { 632 vm_object_t obj; 633 vm_offset_t kva; 634 vm_offset_t size; 635 uintptr_t scan; 636 }; 637 638 void 639 nm_os_extmem_delete(struct nm_os_extmem *e) 640 { 641 D("freeing %zx bytes", (size_t)e->size); 642 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 643 nm_os_free(e); 644 } 645 646 char * 647 nm_os_extmem_nextpage(struct nm_os_extmem *e) 648 { 649 char *rv = NULL; 650 if (e->scan < e->kva + e->size) { 651 rv = (char *)e->scan; 652 e->scan += PAGE_SIZE; 653 } 654 return rv; 655 } 656 657 int 658 nm_os_extmem_isequal(struct nm_os_extmem *e1, struct nm_os_extmem *e2) 659 { 660 return (e1->obj == e2->obj); 661 } 662 663 int 664 nm_os_extmem_nr_pages(struct nm_os_extmem *e) 665 { 666 return e->size >> PAGE_SHIFT; 667 } 668 669 struct nm_os_extmem * 670 nm_os_extmem_create(unsigned long p, struct nmreq_pools_info *pi, int *perror) 671 { 672 vm_map_t map; 673 vm_map_entry_t entry; 674 vm_object_t obj; 675 vm_prot_t prot; 676 vm_pindex_t index; 677 boolean_t wired; 678 struct nm_os_extmem *e = NULL; 679 int rv, error = 0; 680 681 e = nm_os_malloc(sizeof(*e)); 682 if (e == NULL) { 683 error = ENOMEM; 684 goto out; 685 } 686 687 map = &curthread->td_proc->p_vmspace->vm_map; 688 rv = vm_map_lookup(&map, p, VM_PROT_RW, &entry, 689 &obj, &index, &prot, &wired); 690 if (rv != KERN_SUCCESS) { 691 D("address %lx not found", p); 692 goto out_free; 693 } 694 /* check that we are given the whole vm_object ? */ 695 vm_map_lookup_done(map, entry); 696 697 // XXX can we really use obj after releasing the map lock? 698 e->obj = obj; 699 vm_object_reference(obj); 700 /* wire the memory and add the vm_object to the kernel map, 701 * to make sure that it is not fred even if the processes that 702 * are mmap()ing it all exit 703 */ 704 e->kva = vm_map_min(kernel_map); 705 e->size = obj->size << PAGE_SHIFT; 706 rv = vm_map_find(kernel_map, obj, 0, &e->kva, e->size, 0, 707 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 708 VM_PROT_READ | VM_PROT_WRITE, 0); 709 if (rv != KERN_SUCCESS) { 710 D("vm_map_find(%zx) failed", (size_t)e->size); 711 goto out_rel; 712 } 713 rv = vm_map_wire(kernel_map, e->kva, e->kva + e->size, 714 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 715 if (rv != KERN_SUCCESS) { 716 D("vm_map_wire failed"); 717 goto out_rem; 718 } 719 720 e->scan = e->kva; 721 722 return e; 723 724 out_rem: 725 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 726 e->obj = NULL; 727 out_rel: 728 vm_object_deallocate(e->obj); 729 out_free: 730 nm_os_free(e); 731 out: 732 if (perror) 733 *perror = error; 734 return NULL; 735 } 736 #endif /* WITH_EXTMEM */ 737 738 /* ======================== PTNETMAP SUPPORT ========================== */ 739 740 #ifdef WITH_PTNETMAP_GUEST 741 #include <sys/bus.h> 742 #include <sys/rman.h> 743 #include <machine/bus.h> /* bus_dmamap_* */ 744 #include <machine/resource.h> 745 #include <dev/pci/pcivar.h> 746 #include <dev/pci/pcireg.h> 747 /* 748 * ptnetmap memory device (memdev) for freebsd guest, 749 * ssed to expose host netmap memory to the guest through a PCI BAR. 750 */ 751 752 /* 753 * ptnetmap memdev private data structure 754 */ 755 struct ptnetmap_memdev { 756 device_t dev; 757 struct resource *pci_io; 758 struct resource *pci_mem; 759 struct netmap_mem_d *nm_mem; 760 }; 761 762 static int ptn_memdev_probe(device_t); 763 static int ptn_memdev_attach(device_t); 764 static int ptn_memdev_detach(device_t); 765 static int ptn_memdev_shutdown(device_t); 766 767 static device_method_t ptn_memdev_methods[] = { 768 DEVMETHOD(device_probe, ptn_memdev_probe), 769 DEVMETHOD(device_attach, ptn_memdev_attach), 770 DEVMETHOD(device_detach, ptn_memdev_detach), 771 DEVMETHOD(device_shutdown, ptn_memdev_shutdown), 772 DEVMETHOD_END 773 }; 774 775 static driver_t ptn_memdev_driver = { 776 PTNETMAP_MEMDEV_NAME, 777 ptn_memdev_methods, 778 sizeof(struct ptnetmap_memdev), 779 }; 780 781 /* We use (SI_ORDER_MIDDLE+1) here, see DEV_MODULE_ORDERED() invocation 782 * below. */ 783 static devclass_t ptnetmap_devclass; 784 DRIVER_MODULE_ORDERED(ptn_memdev, pci, ptn_memdev_driver, ptnetmap_devclass, 785 NULL, NULL, SI_ORDER_MIDDLE + 1); 786 787 /* 788 * Map host netmap memory through PCI-BAR in the guest OS, 789 * returning physical (nm_paddr) and virtual (nm_addr) addresses 790 * of the netmap memory mapped in the guest. 791 */ 792 int 793 nm_os_pt_memdev_iomap(struct ptnetmap_memdev *ptn_dev, vm_paddr_t *nm_paddr, 794 void **nm_addr, uint64_t *mem_size) 795 { 796 int rid; 797 798 D("ptn_memdev_driver iomap"); 799 800 rid = PCIR_BAR(PTNETMAP_MEM_PCI_BAR); 801 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_HI); 802 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_LO) | 803 (*mem_size << 32); 804 805 /* map memory allocator */ 806 ptn_dev->pci_mem = bus_alloc_resource(ptn_dev->dev, SYS_RES_MEMORY, 807 &rid, 0, ~0, *mem_size, RF_ACTIVE); 808 if (ptn_dev->pci_mem == NULL) { 809 *nm_paddr = 0; 810 *nm_addr = NULL; 811 return ENOMEM; 812 } 813 814 *nm_paddr = rman_get_start(ptn_dev->pci_mem); 815 *nm_addr = rman_get_virtual(ptn_dev->pci_mem); 816 817 D("=== BAR %d start %lx len %lx mem_size %lx ===", 818 PTNETMAP_MEM_PCI_BAR, 819 (unsigned long)(*nm_paddr), 820 (unsigned long)rman_get_size(ptn_dev->pci_mem), 821 (unsigned long)*mem_size); 822 return (0); 823 } 824 825 uint32_t 826 nm_os_pt_memdev_ioread(struct ptnetmap_memdev *ptn_dev, unsigned int reg) 827 { 828 return bus_read_4(ptn_dev->pci_io, reg); 829 } 830 831 /* Unmap host netmap memory. */ 832 void 833 nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *ptn_dev) 834 { 835 D("ptn_memdev_driver iounmap"); 836 837 if (ptn_dev->pci_mem) { 838 bus_release_resource(ptn_dev->dev, SYS_RES_MEMORY, 839 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 840 ptn_dev->pci_mem = NULL; 841 } 842 } 843 844 /* Device identification routine, return BUS_PROBE_DEFAULT on success, 845 * positive on failure */ 846 static int 847 ptn_memdev_probe(device_t dev) 848 { 849 char desc[256]; 850 851 if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID) 852 return (ENXIO); 853 if (pci_get_device(dev) != PTNETMAP_PCI_DEVICE_ID) 854 return (ENXIO); 855 856 snprintf(desc, sizeof(desc), "%s PCI adapter", 857 PTNETMAP_MEMDEV_NAME); 858 device_set_desc_copy(dev, desc); 859 860 return (BUS_PROBE_DEFAULT); 861 } 862 863 /* Device initialization routine. */ 864 static int 865 ptn_memdev_attach(device_t dev) 866 { 867 struct ptnetmap_memdev *ptn_dev; 868 int rid; 869 uint16_t mem_id; 870 871 D("ptn_memdev_driver attach"); 872 873 ptn_dev = device_get_softc(dev); 874 ptn_dev->dev = dev; 875 876 pci_enable_busmaster(dev); 877 878 rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 879 ptn_dev->pci_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 880 RF_ACTIVE); 881 if (ptn_dev->pci_io == NULL) { 882 device_printf(dev, "cannot map I/O space\n"); 883 return (ENXIO); 884 } 885 886 mem_id = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMID); 887 888 /* create guest allocator */ 889 ptn_dev->nm_mem = netmap_mem_pt_guest_attach(ptn_dev, mem_id); 890 if (ptn_dev->nm_mem == NULL) { 891 ptn_memdev_detach(dev); 892 return (ENOMEM); 893 } 894 netmap_mem_get(ptn_dev->nm_mem); 895 896 D("ptn_memdev_driver probe OK - host_mem_id: %d", mem_id); 897 898 return (0); 899 } 900 901 /* Device removal routine. */ 902 static int 903 ptn_memdev_detach(device_t dev) 904 { 905 struct ptnetmap_memdev *ptn_dev; 906 907 D("ptn_memdev_driver detach"); 908 ptn_dev = device_get_softc(dev); 909 910 if (ptn_dev->nm_mem) { 911 netmap_mem_put(ptn_dev->nm_mem); 912 ptn_dev->nm_mem = NULL; 913 } 914 if (ptn_dev->pci_mem) { 915 bus_release_resource(dev, SYS_RES_MEMORY, 916 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 917 ptn_dev->pci_mem = NULL; 918 } 919 if (ptn_dev->pci_io) { 920 bus_release_resource(dev, SYS_RES_IOPORT, 921 PCIR_BAR(PTNETMAP_IO_PCI_BAR), ptn_dev->pci_io); 922 ptn_dev->pci_io = NULL; 923 } 924 925 return (0); 926 } 927 928 static int 929 ptn_memdev_shutdown(device_t dev) 930 { 931 D("ptn_memdev_driver shutdown"); 932 return bus_generic_shutdown(dev); 933 } 934 935 #endif /* WITH_PTNETMAP_GUEST */ 936 937 /* 938 * In order to track whether pages are still mapped, we hook into 939 * the standard cdev_pager and intercept the constructor and 940 * destructor. 941 */ 942 943 struct netmap_vm_handle_t { 944 struct cdev *dev; 945 struct netmap_priv_d *priv; 946 }; 947 948 949 static int 950 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 951 vm_ooffset_t foff, struct ucred *cred, u_short *color) 952 { 953 struct netmap_vm_handle_t *vmh = handle; 954 955 if (netmap_verbose) 956 D("handle %p size %jd prot %d foff %jd", 957 handle, (intmax_t)size, prot, (intmax_t)foff); 958 if (color) 959 *color = 0; 960 dev_ref(vmh->dev); 961 return 0; 962 } 963 964 965 static void 966 netmap_dev_pager_dtor(void *handle) 967 { 968 struct netmap_vm_handle_t *vmh = handle; 969 struct cdev *dev = vmh->dev; 970 struct netmap_priv_d *priv = vmh->priv; 971 972 if (netmap_verbose) 973 D("handle %p", handle); 974 netmap_dtor(priv); 975 free(vmh, M_DEVBUF); 976 dev_rel(dev); 977 } 978 979 980 static int 981 netmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 982 int prot, vm_page_t *mres) 983 { 984 struct netmap_vm_handle_t *vmh = object->handle; 985 struct netmap_priv_d *priv = vmh->priv; 986 struct netmap_adapter *na = priv->np_na; 987 vm_paddr_t paddr; 988 vm_page_t page; 989 vm_memattr_t memattr; 990 vm_pindex_t pidx; 991 992 ND("object %p offset %jd prot %d mres %p", 993 object, (intmax_t)offset, prot, mres); 994 memattr = object->memattr; 995 pidx = OFF_TO_IDX(offset); 996 paddr = netmap_mem_ofstophys(na->nm_mem, offset); 997 if (paddr == 0) 998 return VM_PAGER_FAIL; 999 1000 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 1001 /* 1002 * If the passed in result page is a fake page, update it with 1003 * the new physical address. 1004 */ 1005 page = *mres; 1006 vm_page_updatefake(page, paddr, memattr); 1007 } else { 1008 /* 1009 * Replace the passed in reqpage page with our own fake page and 1010 * free up the all of the original pages. 1011 */ 1012 #ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ 1013 #define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK 1014 #define VM_OBJECT_WLOCK VM_OBJECT_LOCK 1015 #endif /* VM_OBJECT_WUNLOCK */ 1016 1017 VM_OBJECT_WUNLOCK(object); 1018 page = vm_page_getfake(paddr, memattr); 1019 VM_OBJECT_WLOCK(object); 1020 vm_page_lock(*mres); 1021 vm_page_free(*mres); 1022 vm_page_unlock(*mres); 1023 *mres = page; 1024 vm_page_insert(page, object, pidx); 1025 } 1026 page->valid = VM_PAGE_BITS_ALL; 1027 return (VM_PAGER_OK); 1028 } 1029 1030 1031 static struct cdev_pager_ops netmap_cdev_pager_ops = { 1032 .cdev_pg_ctor = netmap_dev_pager_ctor, 1033 .cdev_pg_dtor = netmap_dev_pager_dtor, 1034 .cdev_pg_fault = netmap_dev_pager_fault, 1035 }; 1036 1037 1038 static int 1039 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 1040 vm_size_t objsize, vm_object_t *objp, int prot) 1041 { 1042 int error; 1043 struct netmap_vm_handle_t *vmh; 1044 struct netmap_priv_d *priv; 1045 vm_object_t obj; 1046 1047 if (netmap_verbose) 1048 D("cdev %p foff %jd size %jd objp %p prot %d", cdev, 1049 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 1050 1051 vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 1052 M_NOWAIT | M_ZERO); 1053 if (vmh == NULL) 1054 return ENOMEM; 1055 vmh->dev = cdev; 1056 1057 NMG_LOCK(); 1058 error = devfs_get_cdevpriv((void**)&priv); 1059 if (error) 1060 goto err_unlock; 1061 if (priv->np_nifp == NULL) { 1062 error = EINVAL; 1063 goto err_unlock; 1064 } 1065 vmh->priv = priv; 1066 priv->np_refs++; 1067 NMG_UNLOCK(); 1068 1069 obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 1070 &netmap_cdev_pager_ops, objsize, prot, 1071 *foff, NULL); 1072 if (obj == NULL) { 1073 D("cdev_pager_allocate failed"); 1074 error = EINVAL; 1075 goto err_deref; 1076 } 1077 1078 *objp = obj; 1079 return 0; 1080 1081 err_deref: 1082 NMG_LOCK(); 1083 priv->np_refs--; 1084 err_unlock: 1085 NMG_UNLOCK(); 1086 // err: 1087 free(vmh, M_DEVBUF); 1088 return error; 1089 } 1090 1091 /* 1092 * On FreeBSD the close routine is only called on the last close on 1093 * the device (/dev/netmap) so we cannot do anything useful. 1094 * To track close() on individual file descriptors we pass netmap_dtor() to 1095 * devfs_set_cdevpriv() on open(). The FreeBSD kernel will call the destructor 1096 * when the last fd pointing to the device is closed. 1097 * 1098 * Note that FreeBSD does not even munmap() on close() so we also have 1099 * to track mmap() ourselves, and postpone the call to 1100 * netmap_dtor() is called when the process has no open fds and no active 1101 * memory maps on /dev/netmap, as in linux. 1102 */ 1103 static int 1104 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1105 { 1106 if (netmap_verbose) 1107 D("dev %p fflag 0x%x devtype %d td %p", 1108 dev, fflag, devtype, td); 1109 return 0; 1110 } 1111 1112 1113 static int 1114 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1115 { 1116 struct netmap_priv_d *priv; 1117 int error; 1118 1119 (void)dev; 1120 (void)oflags; 1121 (void)devtype; 1122 (void)td; 1123 1124 NMG_LOCK(); 1125 priv = netmap_priv_new(); 1126 if (priv == NULL) { 1127 error = ENOMEM; 1128 goto out; 1129 } 1130 error = devfs_set_cdevpriv(priv, netmap_dtor); 1131 if (error) { 1132 netmap_priv_delete(priv); 1133 } 1134 out: 1135 NMG_UNLOCK(); 1136 return error; 1137 } 1138 1139 /******************** kthread wrapper ****************/ 1140 #include <sys/sysproto.h> 1141 u_int 1142 nm_os_ncpus(void) 1143 { 1144 return mp_maxid + 1; 1145 } 1146 1147 struct nm_kctx_ctx { 1148 struct thread *user_td; /* thread user-space (kthread creator) to send ioctl */ 1149 struct ptnetmap_cfgentry_bhyve cfg; 1150 1151 /* worker function and parameter */ 1152 nm_kctx_worker_fn_t worker_fn; 1153 void *worker_private; 1154 1155 struct nm_kctx *nmk; 1156 1157 /* integer to manage multiple worker contexts (e.g., RX or TX on ptnetmap) */ 1158 long type; 1159 }; 1160 1161 struct nm_kctx { 1162 struct thread *worker; 1163 struct mtx worker_lock; 1164 uint64_t scheduled; /* pending wake_up request */ 1165 struct nm_kctx_ctx worker_ctx; 1166 int run; /* used to stop kthread */ 1167 int attach_user; /* kthread attached to user_process */ 1168 int affinity; 1169 }; 1170 1171 void inline 1172 nm_os_kctx_worker_wakeup(struct nm_kctx *nmk) 1173 { 1174 /* 1175 * There may be a race between FE and BE, 1176 * which call both this function, and worker kthread, 1177 * that reads nmk->scheduled. 1178 * 1179 * For us it is not important the counter value, 1180 * but simply that it has changed since the last 1181 * time the kthread saw it. 1182 */ 1183 mtx_lock(&nmk->worker_lock); 1184 nmk->scheduled++; 1185 if (nmk->worker_ctx.cfg.wchan) { 1186 wakeup((void *)(uintptr_t)nmk->worker_ctx.cfg.wchan); 1187 } 1188 mtx_unlock(&nmk->worker_lock); 1189 } 1190 1191 void inline 1192 nm_os_kctx_send_irq(struct nm_kctx *nmk) 1193 { 1194 struct nm_kctx_ctx *ctx = &nmk->worker_ctx; 1195 int err; 1196 1197 if (ctx->user_td && ctx->cfg.ioctl_fd > 0) { 1198 err = kern_ioctl(ctx->user_td, ctx->cfg.ioctl_fd, ctx->cfg.ioctl_cmd, 1199 (caddr_t)&ctx->cfg.ioctl_data); 1200 if (err) { 1201 D("kern_ioctl error: %d ioctl parameters: fd %d com %lu data %p", 1202 err, ctx->cfg.ioctl_fd, (unsigned long)ctx->cfg.ioctl_cmd, 1203 &ctx->cfg.ioctl_data); 1204 } 1205 } 1206 } 1207 1208 static void 1209 nm_kctx_worker(void *data) 1210 { 1211 struct nm_kctx *nmk = data; 1212 struct nm_kctx_ctx *ctx = &nmk->worker_ctx; 1213 uint64_t old_scheduled = nmk->scheduled; 1214 1215 if (nmk->affinity >= 0) { 1216 thread_lock(curthread); 1217 sched_bind(curthread, nmk->affinity); 1218 thread_unlock(curthread); 1219 } 1220 1221 while (nmk->run) { 1222 /* 1223 * check if the parent process dies 1224 * (when kthread is attached to user process) 1225 */ 1226 if (ctx->user_td) { 1227 PROC_LOCK(curproc); 1228 thread_suspend_check(0); 1229 PROC_UNLOCK(curproc); 1230 } else { 1231 kthread_suspend_check(); 1232 } 1233 1234 /* 1235 * if wchan is not defined, we don't have notification 1236 * mechanism and we continually execute worker_fn() 1237 */ 1238 if (!ctx->cfg.wchan) { 1239 ctx->worker_fn(ctx->worker_private, 1); /* worker body */ 1240 } else { 1241 /* checks if there is a pending notification */ 1242 mtx_lock(&nmk->worker_lock); 1243 if (likely(nmk->scheduled != old_scheduled)) { 1244 old_scheduled = nmk->scheduled; 1245 mtx_unlock(&nmk->worker_lock); 1246 1247 ctx->worker_fn(ctx->worker_private, 1); /* worker body */ 1248 1249 continue; 1250 } else if (nmk->run) { 1251 /* wait on event with one second timeout */ 1252 msleep((void *)(uintptr_t)ctx->cfg.wchan, &nmk->worker_lock, 1253 0, "nmk_ev", hz); 1254 nmk->scheduled++; 1255 } 1256 mtx_unlock(&nmk->worker_lock); 1257 } 1258 } 1259 1260 kthread_exit(); 1261 } 1262 1263 void 1264 nm_os_kctx_worker_setaff(struct nm_kctx *nmk, int affinity) 1265 { 1266 nmk->affinity = affinity; 1267 } 1268 1269 struct nm_kctx * 1270 nm_os_kctx_create(struct nm_kctx_cfg *cfg, void *opaque) 1271 { 1272 struct nm_kctx *nmk = NULL; 1273 1274 nmk = malloc(sizeof(*nmk), M_DEVBUF, M_NOWAIT | M_ZERO); 1275 if (!nmk) 1276 return NULL; 1277 1278 mtx_init(&nmk->worker_lock, "nm_kthread lock", NULL, MTX_DEF); 1279 nmk->worker_ctx.worker_fn = cfg->worker_fn; 1280 nmk->worker_ctx.worker_private = cfg->worker_private; 1281 nmk->worker_ctx.type = cfg->type; 1282 nmk->affinity = -1; 1283 1284 /* attach kthread to user process (ptnetmap) */ 1285 nmk->attach_user = cfg->attach_user; 1286 1287 /* store kick/interrupt configuration */ 1288 if (opaque) { 1289 nmk->worker_ctx.cfg = *((struct ptnetmap_cfgentry_bhyve *)opaque); 1290 } 1291 1292 return nmk; 1293 } 1294 1295 int 1296 nm_os_kctx_worker_start(struct nm_kctx *nmk) 1297 { 1298 struct proc *p = NULL; 1299 int error = 0; 1300 1301 if (nmk->worker) { 1302 return EBUSY; 1303 } 1304 1305 /* check if we want to attach kthread to user process */ 1306 if (nmk->attach_user) { 1307 nmk->worker_ctx.user_td = curthread; 1308 p = curthread->td_proc; 1309 } 1310 1311 /* enable kthread main loop */ 1312 nmk->run = 1; 1313 /* create kthread */ 1314 if((error = kthread_add(nm_kctx_worker, nmk, p, 1315 &nmk->worker, RFNOWAIT /* to be checked */, 0, "nm-kthread-%ld", 1316 nmk->worker_ctx.type))) { 1317 goto err; 1318 } 1319 1320 D("nm_kthread started td %p", nmk->worker); 1321 1322 return 0; 1323 err: 1324 D("nm_kthread start failed err %d", error); 1325 nmk->worker = NULL; 1326 return error; 1327 } 1328 1329 void 1330 nm_os_kctx_worker_stop(struct nm_kctx *nmk) 1331 { 1332 if (!nmk->worker) { 1333 return; 1334 } 1335 /* tell to kthread to exit from main loop */ 1336 nmk->run = 0; 1337 1338 /* wake up kthread if it sleeps */ 1339 kthread_resume(nmk->worker); 1340 nm_os_kctx_worker_wakeup(nmk); 1341 1342 nmk->worker = NULL; 1343 } 1344 1345 void 1346 nm_os_kctx_destroy(struct nm_kctx *nmk) 1347 { 1348 if (!nmk) 1349 return; 1350 if (nmk->worker) { 1351 nm_os_kctx_worker_stop(nmk); 1352 } 1353 1354 memset(&nmk->worker_ctx.cfg, 0, sizeof(nmk->worker_ctx.cfg)); 1355 1356 free(nmk, M_DEVBUF); 1357 } 1358 1359 /******************** kqueue support ****************/ 1360 1361 /* 1362 * nm_os_selwakeup also needs to issue a KNOTE_UNLOCKED. 1363 * We use a non-zero argument to distinguish the call from the one 1364 * in kevent_scan() which instead also needs to run netmap_poll(). 1365 * The knote uses a global mutex for the time being. We might 1366 * try to reuse the one in the si, but it is not allocated 1367 * permanently so it might be a bit tricky. 1368 * 1369 * The *kqfilter function registers one or another f_event 1370 * depending on read or write mode. 1371 * In the call to f_event() td_fpop is NULL so any child function 1372 * calling devfs_get_cdevpriv() would fail - and we need it in 1373 * netmap_poll(). As a workaround we store priv into kn->kn_hook 1374 * and pass it as first argument to netmap_poll(), which then 1375 * uses the failure to tell that we are called from f_event() 1376 * and do not need the selrecord(). 1377 */ 1378 1379 1380 void 1381 nm_os_selwakeup(struct nm_selinfo *si) 1382 { 1383 if (netmap_verbose) 1384 D("on knote %p", &si->si.si_note); 1385 selwakeuppri(&si->si, PI_NET); 1386 /* use a non-zero hint to tell the notification from the 1387 * call done in kqueue_scan() which uses 0 1388 */ 1389 KNOTE_UNLOCKED(&si->si.si_note, 0x100 /* notification */); 1390 } 1391 1392 void 1393 nm_os_selrecord(struct thread *td, struct nm_selinfo *si) 1394 { 1395 selrecord(td, &si->si); 1396 } 1397 1398 static void 1399 netmap_knrdetach(struct knote *kn) 1400 { 1401 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1402 struct selinfo *si = &priv->np_si[NR_RX]->si; 1403 1404 D("remove selinfo %p", si); 1405 knlist_remove(&si->si_note, kn, 0); 1406 } 1407 1408 static void 1409 netmap_knwdetach(struct knote *kn) 1410 { 1411 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1412 struct selinfo *si = &priv->np_si[NR_TX]->si; 1413 1414 D("remove selinfo %p", si); 1415 knlist_remove(&si->si_note, kn, 0); 1416 } 1417 1418 /* 1419 * callback from notifies (generated externally) and our 1420 * calls to kevent(). The former we just return 1 (ready) 1421 * since we do not know better. 1422 * In the latter we call netmap_poll and return 0/1 accordingly. 1423 */ 1424 static int 1425 netmap_knrw(struct knote *kn, long hint, int events) 1426 { 1427 struct netmap_priv_d *priv; 1428 int revents; 1429 1430 if (hint != 0) { 1431 ND(5, "call from notify"); 1432 return 1; /* assume we are ready */ 1433 } 1434 priv = kn->kn_hook; 1435 /* the notification may come from an external thread, 1436 * in which case we do not want to run the netmap_poll 1437 * This should be filtered above, but check just in case. 1438 */ 1439 if (curthread != priv->np_td) { /* should not happen */ 1440 RD(5, "curthread changed %p %p", curthread, priv->np_td); 1441 return 1; 1442 } else { 1443 revents = netmap_poll(priv, events, NULL); 1444 return (events & revents) ? 1 : 0; 1445 } 1446 } 1447 1448 static int 1449 netmap_knread(struct knote *kn, long hint) 1450 { 1451 return netmap_knrw(kn, hint, POLLIN); 1452 } 1453 1454 static int 1455 netmap_knwrite(struct knote *kn, long hint) 1456 { 1457 return netmap_knrw(kn, hint, POLLOUT); 1458 } 1459 1460 static struct filterops netmap_rfiltops = { 1461 .f_isfd = 1, 1462 .f_detach = netmap_knrdetach, 1463 .f_event = netmap_knread, 1464 }; 1465 1466 static struct filterops netmap_wfiltops = { 1467 .f_isfd = 1, 1468 .f_detach = netmap_knwdetach, 1469 .f_event = netmap_knwrite, 1470 }; 1471 1472 1473 /* 1474 * This is called when a thread invokes kevent() to record 1475 * a change in the configuration of the kqueue(). 1476 * The 'priv' should be the same as in the netmap device. 1477 */ 1478 static int 1479 netmap_kqfilter(struct cdev *dev, struct knote *kn) 1480 { 1481 struct netmap_priv_d *priv; 1482 int error; 1483 struct netmap_adapter *na; 1484 struct nm_selinfo *si; 1485 int ev = kn->kn_filter; 1486 1487 if (ev != EVFILT_READ && ev != EVFILT_WRITE) { 1488 D("bad filter request %d", ev); 1489 return 1; 1490 } 1491 error = devfs_get_cdevpriv((void**)&priv); 1492 if (error) { 1493 D("device not yet setup"); 1494 return 1; 1495 } 1496 na = priv->np_na; 1497 if (na == NULL) { 1498 D("no netmap adapter for this file descriptor"); 1499 return 1; 1500 } 1501 /* the si is indicated in the priv */ 1502 si = priv->np_si[(ev == EVFILT_WRITE) ? NR_TX : NR_RX]; 1503 // XXX lock(priv) ? 1504 kn->kn_fop = (ev == EVFILT_WRITE) ? 1505 &netmap_wfiltops : &netmap_rfiltops; 1506 kn->kn_hook = priv; 1507 knlist_add(&si->si.si_note, kn, 0); 1508 // XXX unlock(priv) 1509 ND("register %p %s td %p priv %p kn %p np_nifp %p kn_fp/fpop %s", 1510 na, na->ifp->if_xname, curthread, priv, kn, 1511 priv->np_nifp, 1512 kn->kn_fp == curthread->td_fpop ? "match" : "MISMATCH"); 1513 return 0; 1514 } 1515 1516 static int 1517 freebsd_netmap_poll(struct cdev *cdevi __unused, int events, struct thread *td) 1518 { 1519 struct netmap_priv_d *priv; 1520 if (devfs_get_cdevpriv((void **)&priv)) { 1521 return POLLERR; 1522 } 1523 return netmap_poll(priv, events, td); 1524 } 1525 1526 static int 1527 freebsd_netmap_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, 1528 int ffla __unused, struct thread *td) 1529 { 1530 int error; 1531 struct netmap_priv_d *priv; 1532 1533 CURVNET_SET(TD_TO_VNET(td)); 1534 error = devfs_get_cdevpriv((void **)&priv); 1535 if (error) { 1536 /* XXX ENOENT should be impossible, since the priv 1537 * is now created in the open */ 1538 if (error == ENOENT) 1539 error = ENXIO; 1540 goto out; 1541 } 1542 error = netmap_ioctl(priv, cmd, data, td, /*nr_body_is_user=*/1); 1543 out: 1544 CURVNET_RESTORE(); 1545 1546 return error; 1547 } 1548 1549 void 1550 nm_os_onattach(struct ifnet *ifp) 1551 { 1552 } 1553 1554 void 1555 nm_os_onenter(struct ifnet *ifp) 1556 { 1557 struct netmap_adapter *na = NA(ifp); 1558 1559 na->if_transmit = ifp->if_transmit; 1560 ifp->if_transmit = netmap_transmit; 1561 ifp->if_capenable |= IFCAP_NETMAP; 1562 } 1563 1564 void 1565 nm_os_onexit(struct ifnet *ifp) 1566 { 1567 struct netmap_adapter *na = NA(ifp); 1568 1569 ifp->if_transmit = na->if_transmit; 1570 ifp->if_capenable &= ~IFCAP_NETMAP; 1571 } 1572 1573 extern struct cdevsw netmap_cdevsw; /* XXX used in netmap.c, should go elsewhere */ 1574 struct cdevsw netmap_cdevsw = { 1575 .d_version = D_VERSION, 1576 .d_name = "netmap", 1577 .d_open = netmap_open, 1578 .d_mmap_single = netmap_mmap_single, 1579 .d_ioctl = freebsd_netmap_ioctl, 1580 .d_poll = freebsd_netmap_poll, 1581 .d_kqfilter = netmap_kqfilter, 1582 .d_close = netmap_close, 1583 }; 1584 /*--- end of kqueue support ----*/ 1585 1586 /* 1587 * Kernel entry point. 1588 * 1589 * Initialize/finalize the module and return. 1590 * 1591 * Return 0 on success, errno on failure. 1592 */ 1593 static int 1594 netmap_loader(__unused struct module *module, int event, __unused void *arg) 1595 { 1596 int error = 0; 1597 1598 switch (event) { 1599 case MOD_LOAD: 1600 error = netmap_init(); 1601 break; 1602 1603 case MOD_UNLOAD: 1604 /* 1605 * if some one is still using netmap, 1606 * then the module can not be unloaded. 1607 */ 1608 if (netmap_use_count) { 1609 D("netmap module can not be unloaded - netmap_use_count: %d", 1610 netmap_use_count); 1611 error = EBUSY; 1612 break; 1613 } 1614 netmap_fini(); 1615 break; 1616 1617 default: 1618 error = EOPNOTSUPP; 1619 break; 1620 } 1621 1622 return (error); 1623 } 1624 1625 #ifdef DEV_MODULE_ORDERED 1626 /* 1627 * The netmap module contains three drivers: (i) the netmap character device 1628 * driver; (ii) the ptnetmap memdev PCI device driver, (iii) the ptnet PCI 1629 * device driver. The attach() routines of both (ii) and (iii) need the 1630 * lock of the global allocator, and such lock is initialized in netmap_init(), 1631 * which is part of (i). 1632 * Therefore, we make sure that (i) is loaded before (ii) and (iii), using 1633 * the 'order' parameter of driver declaration macros. For (i), we specify 1634 * SI_ORDER_MIDDLE, while higher orders are used with the DRIVER_MODULE_ORDERED 1635 * macros for (ii) and (iii). 1636 */ 1637 DEV_MODULE_ORDERED(netmap, netmap_loader, NULL, SI_ORDER_MIDDLE); 1638 #else /* !DEV_MODULE_ORDERED */ 1639 DEV_MODULE(netmap, netmap_loader, NULL); 1640 #endif /* DEV_MODULE_ORDERED */ 1641 MODULE_DEPEND(netmap, pci, 1, 1, 1); 1642 MODULE_VERSION(netmap, 1); 1643 /* reduce conditional code */ 1644 // linux API, use for the knlist in FreeBSD 1645 /* use a private mutex for the knlist */ 1646