1 /*- 2 * Copyright (c) 1990, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from the Stanford/CMU enet packet filter, 6 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 7 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 8 * Berkeley Laboratory. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 35 */ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include "opt_bpf.h" 41 #include "opt_compat.h" 42 #include "opt_ddb.h" 43 #include "opt_netgraph.h" 44 45 #include <sys/types.h> 46 #include <sys/param.h> 47 #include <sys/lock.h> 48 #include <sys/rwlock.h> 49 #include <sys/systm.h> 50 #include <sys/conf.h> 51 #include <sys/fcntl.h> 52 #include <sys/jail.h> 53 #include <sys/malloc.h> 54 #include <sys/mbuf.h> 55 #include <sys/time.h> 56 #include <sys/priv.h> 57 #include <sys/proc.h> 58 #include <sys/signalvar.h> 59 #include <sys/filio.h> 60 #include <sys/sockio.h> 61 #include <sys/ttycom.h> 62 #include <sys/uio.h> 63 64 #include <sys/event.h> 65 #include <sys/file.h> 66 #include <sys/poll.h> 67 #include <sys/proc.h> 68 69 #include <sys/socket.h> 70 71 #ifdef DDB 72 #include <ddb/ddb.h> 73 #endif 74 75 #include <net/if.h> 76 #include <net/if_var.h> 77 #include <net/if_dl.h> 78 #include <net/bpf.h> 79 #include <net/bpf_buffer.h> 80 #ifdef BPF_JITTER 81 #include <net/bpf_jitter.h> 82 #endif 83 #include <net/bpf_zerocopy.h> 84 #include <net/bpfdesc.h> 85 #include <net/route.h> 86 #include <net/vnet.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_ether.h> 90 #include <sys/kernel.h> 91 #include <sys/sysctl.h> 92 93 #include <net80211/ieee80211_freebsd.h> 94 95 #include <security/mac/mac_framework.h> 96 97 MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); 98 99 struct bpf_if { 100 #define bif_next bif_ext.bif_next 101 #define bif_dlist bif_ext.bif_dlist 102 struct bpf_if_ext bif_ext; /* public members */ 103 u_int bif_dlt; /* link layer type */ 104 u_int bif_hdrlen; /* length of link header */ 105 struct ifnet *bif_ifp; /* corresponding interface */ 106 struct rwlock bif_lock; /* interface lock */ 107 LIST_HEAD(, bpf_d) bif_wlist; /* writer-only list */ 108 int bif_flags; /* Interface flags */ 109 struct bpf_if **bif_bpf; /* Pointer to pointer to us */ 110 }; 111 112 CTASSERT(offsetof(struct bpf_if, bif_ext) == 0); 113 114 #if defined(DEV_BPF) || defined(NETGRAPH_BPF) 115 116 #define PRINET 26 /* interruptible */ 117 118 #define SIZEOF_BPF_HDR(type) \ 119 (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen)) 120 121 #ifdef COMPAT_FREEBSD32 122 #include <sys/mount.h> 123 #include <compat/freebsd32/freebsd32.h> 124 #define BPF_ALIGNMENT32 sizeof(int32_t) 125 #define BPF_WORDALIGN32(x) roundup2(x, BPF_ALIGNMENT32) 126 127 #ifndef BURN_BRIDGES 128 /* 129 * 32-bit version of structure prepended to each packet. We use this header 130 * instead of the standard one for 32-bit streams. We mark the a stream as 131 * 32-bit the first time we see a 32-bit compat ioctl request. 132 */ 133 struct bpf_hdr32 { 134 struct timeval32 bh_tstamp; /* time stamp */ 135 uint32_t bh_caplen; /* length of captured portion */ 136 uint32_t bh_datalen; /* original length of packet */ 137 uint16_t bh_hdrlen; /* length of bpf header (this struct 138 plus alignment padding) */ 139 }; 140 #endif 141 142 struct bpf_program32 { 143 u_int bf_len; 144 uint32_t bf_insns; 145 }; 146 147 struct bpf_dltlist32 { 148 u_int bfl_len; 149 u_int bfl_list; 150 }; 151 152 #define BIOCSETF32 _IOW('B', 103, struct bpf_program32) 153 #define BIOCSRTIMEOUT32 _IOW('B', 109, struct timeval32) 154 #define BIOCGRTIMEOUT32 _IOR('B', 110, struct timeval32) 155 #define BIOCGDLTLIST32 _IOWR('B', 121, struct bpf_dltlist32) 156 #define BIOCSETWF32 _IOW('B', 123, struct bpf_program32) 157 #define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32) 158 #endif 159 160 /* 161 * bpf_iflist is a list of BPF interface structures, each corresponding to a 162 * specific DLT. The same network interface might have several BPF interface 163 * structures registered by different layers in the stack (i.e., 802.11 164 * frames, ethernet frames, etc). 165 */ 166 static LIST_HEAD(, bpf_if) bpf_iflist, bpf_freelist; 167 static struct mtx bpf_mtx; /* bpf global lock */ 168 static int bpf_bpfd_cnt; 169 170 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 171 static void bpf_detachd(struct bpf_d *); 172 static void bpf_detachd_locked(struct bpf_d *); 173 static void bpf_freed(struct bpf_d *); 174 static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, 175 struct sockaddr *, int *, struct bpf_d *); 176 static int bpf_setif(struct bpf_d *, struct ifreq *); 177 static void bpf_timed_out(void *); 178 static __inline void 179 bpf_wakeup(struct bpf_d *); 180 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 181 void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), 182 struct bintime *); 183 static void reset_d(struct bpf_d *); 184 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd); 185 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 186 static int bpf_setdlt(struct bpf_d *, u_int); 187 static void filt_bpfdetach(struct knote *); 188 static int filt_bpfread(struct knote *, long); 189 static void bpf_drvinit(void *); 190 static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS); 191 192 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl"); 193 int bpf_maxinsns = BPF_MAXINSNS; 194 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW, 195 &bpf_maxinsns, 0, "Maximum bpf program instructions"); 196 static int bpf_zerocopy_enable = 0; 197 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW, 198 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions"); 199 static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW, 200 bpf_stats_sysctl, "bpf statistics portal"); 201 202 static VNET_DEFINE(int, bpf_optimize_writers) = 0; 203 #define V_bpf_optimize_writers VNET(bpf_optimize_writers) 204 SYSCTL_INT(_net_bpf, OID_AUTO, optimize_writers, CTLFLAG_VNET | CTLFLAG_RW, 205 &VNET_NAME(bpf_optimize_writers), 0, 206 "Do not send packets until BPF program is set"); 207 208 static d_open_t bpfopen; 209 static d_read_t bpfread; 210 static d_write_t bpfwrite; 211 static d_ioctl_t bpfioctl; 212 static d_poll_t bpfpoll; 213 static d_kqfilter_t bpfkqfilter; 214 215 static struct cdevsw bpf_cdevsw = { 216 .d_version = D_VERSION, 217 .d_open = bpfopen, 218 .d_read = bpfread, 219 .d_write = bpfwrite, 220 .d_ioctl = bpfioctl, 221 .d_poll = bpfpoll, 222 .d_name = "bpf", 223 .d_kqfilter = bpfkqfilter, 224 }; 225 226 static struct filterops bpfread_filtops = { 227 .f_isfd = 1, 228 .f_detach = filt_bpfdetach, 229 .f_event = filt_bpfread, 230 }; 231 232 eventhandler_tag bpf_ifdetach_cookie = NULL; 233 234 /* 235 * LOCKING MODEL USED BY BPF: 236 * Locks: 237 * 1) global lock (BPF_LOCK). Mutex, used to protect interface addition/removal, 238 * some global counters and every bpf_if reference. 239 * 2) Interface lock. Rwlock, used to protect list of BPF descriptors and their filters. 240 * 3) Descriptor lock. Mutex, used to protect BPF buffers and various structure fields 241 * used by bpf_mtap code. 242 * 243 * Lock order: 244 * 245 * Global lock, interface lock, descriptor lock 246 * 247 * We have to acquire interface lock before descriptor main lock due to BPF_MTAP[2] 248 * working model. In many places (like bpf_detachd) we start with BPF descriptor 249 * (and we need to at least rlock it to get reliable interface pointer). This 250 * gives us potential LOR. As a result, we use global lock to protect from bpf_if 251 * change in every such place. 252 * 253 * Changing d->bd_bif is protected by 1) global lock, 2) interface lock and 254 * 3) descriptor main wlock. 255 * Reading bd_bif can be protected by any of these locks, typically global lock. 256 * 257 * Changing read/write BPF filter is protected by the same three locks, 258 * the same applies for reading. 259 * 260 * Sleeping in global lock is not allowed due to bpfdetach() using it. 261 */ 262 263 /* 264 * Wrapper functions for various buffering methods. If the set of buffer 265 * modes expands, we will probably want to introduce a switch data structure 266 * similar to protosw, et. 267 */ 268 static void 269 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 270 u_int len) 271 { 272 273 BPFD_LOCK_ASSERT(d); 274 275 switch (d->bd_bufmode) { 276 case BPF_BUFMODE_BUFFER: 277 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); 278 279 case BPF_BUFMODE_ZBUF: 280 d->bd_zcopy++; 281 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); 282 283 default: 284 panic("bpf_buf_append_bytes"); 285 } 286 } 287 288 static void 289 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 290 u_int len) 291 { 292 293 BPFD_LOCK_ASSERT(d); 294 295 switch (d->bd_bufmode) { 296 case BPF_BUFMODE_BUFFER: 297 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); 298 299 case BPF_BUFMODE_ZBUF: 300 d->bd_zcopy++; 301 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); 302 303 default: 304 panic("bpf_buf_append_mbuf"); 305 } 306 } 307 308 /* 309 * This function gets called when the free buffer is re-assigned. 310 */ 311 static void 312 bpf_buf_reclaimed(struct bpf_d *d) 313 { 314 315 BPFD_LOCK_ASSERT(d); 316 317 switch (d->bd_bufmode) { 318 case BPF_BUFMODE_BUFFER: 319 return; 320 321 case BPF_BUFMODE_ZBUF: 322 bpf_zerocopy_buf_reclaimed(d); 323 return; 324 325 default: 326 panic("bpf_buf_reclaimed"); 327 } 328 } 329 330 /* 331 * If the buffer mechanism has a way to decide that a held buffer can be made 332 * free, then it is exposed via the bpf_canfreebuf() interface. (1) is 333 * returned if the buffer can be discarded, (0) is returned if it cannot. 334 */ 335 static int 336 bpf_canfreebuf(struct bpf_d *d) 337 { 338 339 BPFD_LOCK_ASSERT(d); 340 341 switch (d->bd_bufmode) { 342 case BPF_BUFMODE_ZBUF: 343 return (bpf_zerocopy_canfreebuf(d)); 344 } 345 return (0); 346 } 347 348 /* 349 * Allow the buffer model to indicate that the current store buffer is 350 * immutable, regardless of the appearance of space. Return (1) if the 351 * buffer is writable, and (0) if not. 352 */ 353 static int 354 bpf_canwritebuf(struct bpf_d *d) 355 { 356 BPFD_LOCK_ASSERT(d); 357 358 switch (d->bd_bufmode) { 359 case BPF_BUFMODE_ZBUF: 360 return (bpf_zerocopy_canwritebuf(d)); 361 } 362 return (1); 363 } 364 365 /* 366 * Notify buffer model that an attempt to write to the store buffer has 367 * resulted in a dropped packet, in which case the buffer may be considered 368 * full. 369 */ 370 static void 371 bpf_buffull(struct bpf_d *d) 372 { 373 374 BPFD_LOCK_ASSERT(d); 375 376 switch (d->bd_bufmode) { 377 case BPF_BUFMODE_ZBUF: 378 bpf_zerocopy_buffull(d); 379 break; 380 } 381 } 382 383 /* 384 * Notify the buffer model that a buffer has moved into the hold position. 385 */ 386 void 387 bpf_bufheld(struct bpf_d *d) 388 { 389 390 BPFD_LOCK_ASSERT(d); 391 392 switch (d->bd_bufmode) { 393 case BPF_BUFMODE_ZBUF: 394 bpf_zerocopy_bufheld(d); 395 break; 396 } 397 } 398 399 static void 400 bpf_free(struct bpf_d *d) 401 { 402 403 switch (d->bd_bufmode) { 404 case BPF_BUFMODE_BUFFER: 405 return (bpf_buffer_free(d)); 406 407 case BPF_BUFMODE_ZBUF: 408 return (bpf_zerocopy_free(d)); 409 410 default: 411 panic("bpf_buf_free"); 412 } 413 } 414 415 static int 416 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) 417 { 418 419 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 420 return (EOPNOTSUPP); 421 return (bpf_buffer_uiomove(d, buf, len, uio)); 422 } 423 424 static int 425 bpf_ioctl_sblen(struct bpf_d *d, u_int *i) 426 { 427 428 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 429 return (EOPNOTSUPP); 430 return (bpf_buffer_ioctl_sblen(d, i)); 431 } 432 433 static int 434 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) 435 { 436 437 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 438 return (EOPNOTSUPP); 439 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); 440 } 441 442 static int 443 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 444 { 445 446 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 447 return (EOPNOTSUPP); 448 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); 449 } 450 451 static int 452 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 453 { 454 455 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 456 return (EOPNOTSUPP); 457 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); 458 } 459 460 /* 461 * General BPF functions. 462 */ 463 static int 464 bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp, 465 struct sockaddr *sockp, int *hdrlen, struct bpf_d *d) 466 { 467 const struct ieee80211_bpf_params *p; 468 struct ether_header *eh; 469 struct mbuf *m; 470 int error; 471 int len; 472 int hlen; 473 int slen; 474 475 /* 476 * Build a sockaddr based on the data link layer type. 477 * We do this at this level because the ethernet header 478 * is copied directly into the data field of the sockaddr. 479 * In the case of SLIP, there is no header and the packet 480 * is forwarded as is. 481 * Also, we are careful to leave room at the front of the mbuf 482 * for the link level header. 483 */ 484 switch (linktype) { 485 486 case DLT_SLIP: 487 sockp->sa_family = AF_INET; 488 hlen = 0; 489 break; 490 491 case DLT_EN10MB: 492 sockp->sa_family = AF_UNSPEC; 493 /* XXX Would MAXLINKHDR be better? */ 494 hlen = ETHER_HDR_LEN; 495 break; 496 497 case DLT_FDDI: 498 sockp->sa_family = AF_IMPLINK; 499 hlen = 0; 500 break; 501 502 case DLT_RAW: 503 sockp->sa_family = AF_UNSPEC; 504 hlen = 0; 505 break; 506 507 case DLT_NULL: 508 /* 509 * null interface types require a 4 byte pseudo header which 510 * corresponds to the address family of the packet. 511 */ 512 sockp->sa_family = AF_UNSPEC; 513 hlen = 4; 514 break; 515 516 case DLT_ATM_RFC1483: 517 /* 518 * en atm driver requires 4-byte atm pseudo header. 519 * though it isn't standard, vpi:vci needs to be 520 * specified anyway. 521 */ 522 sockp->sa_family = AF_UNSPEC; 523 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */ 524 break; 525 526 case DLT_PPP: 527 sockp->sa_family = AF_UNSPEC; 528 hlen = 4; /* This should match PPP_HDRLEN */ 529 break; 530 531 case DLT_IEEE802_11: /* IEEE 802.11 wireless */ 532 sockp->sa_family = AF_IEEE80211; 533 hlen = 0; 534 break; 535 536 case DLT_IEEE802_11_RADIO: /* IEEE 802.11 wireless w/ phy params */ 537 sockp->sa_family = AF_IEEE80211; 538 sockp->sa_len = 12; /* XXX != 0 */ 539 hlen = sizeof(struct ieee80211_bpf_params); 540 break; 541 542 default: 543 return (EIO); 544 } 545 546 len = uio->uio_resid; 547 if (len < hlen || len - hlen > ifp->if_mtu) 548 return (EMSGSIZE); 549 550 m = m_get2(len, M_WAITOK, MT_DATA, M_PKTHDR); 551 if (m == NULL) 552 return (EIO); 553 m->m_pkthdr.len = m->m_len = len; 554 *mp = m; 555 556 error = uiomove(mtod(m, u_char *), len, uio); 557 if (error) 558 goto bad; 559 560 slen = bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len); 561 if (slen == 0) { 562 error = EPERM; 563 goto bad; 564 } 565 566 /* Check for multicast destination */ 567 switch (linktype) { 568 case DLT_EN10MB: 569 eh = mtod(m, struct ether_header *); 570 if (ETHER_IS_MULTICAST(eh->ether_dhost)) { 571 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost, 572 ETHER_ADDR_LEN) == 0) 573 m->m_flags |= M_BCAST; 574 else 575 m->m_flags |= M_MCAST; 576 } 577 if (d->bd_hdrcmplt == 0) { 578 memcpy(eh->ether_shost, IF_LLADDR(ifp), 579 sizeof(eh->ether_shost)); 580 } 581 break; 582 } 583 584 /* 585 * Make room for link header, and copy it to sockaddr 586 */ 587 if (hlen != 0) { 588 if (sockp->sa_family == AF_IEEE80211) { 589 /* 590 * Collect true length from the parameter header 591 * NB: sockp is known to be zero'd so if we do a 592 * short copy unspecified parameters will be 593 * zero. 594 * NB: packet may not be aligned after stripping 595 * bpf params 596 * XXX check ibp_vers 597 */ 598 p = mtod(m, const struct ieee80211_bpf_params *); 599 hlen = p->ibp_len; 600 if (hlen > sizeof(sockp->sa_data)) { 601 error = EINVAL; 602 goto bad; 603 } 604 } 605 bcopy(mtod(m, const void *), sockp->sa_data, hlen); 606 } 607 *hdrlen = hlen; 608 609 return (0); 610 bad: 611 m_freem(m); 612 return (error); 613 } 614 615 /* 616 * Attach file to the bpf interface, i.e. make d listen on bp. 617 */ 618 static void 619 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 620 { 621 int op_w; 622 623 BPF_LOCK_ASSERT(); 624 625 /* 626 * Save sysctl value to protect from sysctl change 627 * between reads 628 */ 629 op_w = V_bpf_optimize_writers || d->bd_writer; 630 631 if (d->bd_bif != NULL) 632 bpf_detachd_locked(d); 633 /* 634 * Point d at bp, and add d to the interface's list. 635 * Since there are many applications using BPF for 636 * sending raw packets only (dhcpd, cdpd are good examples) 637 * we can delay adding d to the list of active listeners until 638 * some filter is configured. 639 */ 640 641 BPFIF_WLOCK(bp); 642 BPFD_LOCK(d); 643 644 d->bd_bif = bp; 645 646 if (op_w != 0) { 647 /* Add to writers-only list */ 648 LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); 649 /* 650 * We decrement bd_writer on every filter set operation. 651 * First BIOCSETF is done by pcap_open_live() to set up 652 * snap length. After that appliation usually sets its own filter 653 */ 654 d->bd_writer = 2; 655 } else 656 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); 657 658 BPFD_UNLOCK(d); 659 BPFIF_WUNLOCK(bp); 660 661 bpf_bpfd_cnt++; 662 663 CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list", 664 __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); 665 666 if (op_w == 0) 667 EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); 668 } 669 670 /* 671 * Check if we need to upgrade our descriptor @d from write-only mode. 672 */ 673 static int 674 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, int flen) 675 { 676 int is_snap, need_upgrade; 677 678 /* 679 * Check if we've already upgraded or new filter is empty. 680 */ 681 if (d->bd_writer == 0 || fcode == NULL) 682 return (0); 683 684 need_upgrade = 0; 685 686 /* 687 * Check if cmd looks like snaplen setting from 688 * pcap_bpf.c:pcap_open_live(). 689 * Note we're not checking .k value here: 690 * while pcap_open_live() definitely sets to to non-zero value, 691 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g. 692 * do not consider upgrading immediately 693 */ 694 if (cmd == BIOCSETF && flen == 1 && fcode[0].code == (BPF_RET | BPF_K)) 695 is_snap = 1; 696 else 697 is_snap = 0; 698 699 if (is_snap == 0) { 700 /* 701 * We're setting first filter and it doesn't look like 702 * setting snaplen. We're probably using bpf directly. 703 * Upgrade immediately. 704 */ 705 need_upgrade = 1; 706 } else { 707 /* 708 * Do not require upgrade by first BIOCSETF 709 * (used to set snaplen) by pcap_open_live(). 710 */ 711 712 if (--d->bd_writer == 0) { 713 /* 714 * First snaplen filter has already 715 * been set. This is probably catch-all 716 * filter 717 */ 718 need_upgrade = 1; 719 } 720 } 721 722 CTR5(KTR_NET, 723 "%s: filter function set by pid %d, " 724 "bd_writer counter %d, snap %d upgrade %d", 725 __func__, d->bd_pid, d->bd_writer, 726 is_snap, need_upgrade); 727 728 return (need_upgrade); 729 } 730 731 /* 732 * Add d to the list of active bp filters. 733 * Requires bpf_attachd() to be called before. 734 */ 735 static void 736 bpf_upgraded(struct bpf_d *d) 737 { 738 struct bpf_if *bp; 739 740 BPF_LOCK_ASSERT(); 741 742 bp = d->bd_bif; 743 744 /* 745 * Filter can be set several times without specifying interface. 746 * Mark d as reader and exit. 747 */ 748 if (bp == NULL) { 749 BPFD_LOCK(d); 750 d->bd_writer = 0; 751 BPFD_UNLOCK(d); 752 return; 753 } 754 755 BPFIF_WLOCK(bp); 756 BPFD_LOCK(d); 757 758 /* Remove from writers-only list */ 759 LIST_REMOVE(d, bd_next); 760 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); 761 /* Mark d as reader */ 762 d->bd_writer = 0; 763 764 BPFD_UNLOCK(d); 765 BPFIF_WUNLOCK(bp); 766 767 CTR2(KTR_NET, "%s: upgrade required by pid %d", __func__, d->bd_pid); 768 769 EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); 770 } 771 772 /* 773 * Detach a file from its interface. 774 */ 775 static void 776 bpf_detachd(struct bpf_d *d) 777 { 778 BPF_LOCK(); 779 bpf_detachd_locked(d); 780 BPF_UNLOCK(); 781 } 782 783 static void 784 bpf_detachd_locked(struct bpf_d *d) 785 { 786 int error; 787 struct bpf_if *bp; 788 struct ifnet *ifp; 789 790 CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); 791 792 BPF_LOCK_ASSERT(); 793 794 /* Check if descriptor is attached */ 795 if ((bp = d->bd_bif) == NULL) 796 return; 797 798 BPFIF_WLOCK(bp); 799 BPFD_LOCK(d); 800 801 /* Save bd_writer value */ 802 error = d->bd_writer; 803 804 /* 805 * Remove d from the interface's descriptor list. 806 */ 807 LIST_REMOVE(d, bd_next); 808 809 ifp = bp->bif_ifp; 810 d->bd_bif = NULL; 811 BPFD_UNLOCK(d); 812 BPFIF_WUNLOCK(bp); 813 814 bpf_bpfd_cnt--; 815 816 /* Call event handler iff d is attached */ 817 if (error == 0) 818 EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0); 819 820 /* 821 * Check if this descriptor had requested promiscuous mode. 822 * If so, turn it off. 823 */ 824 if (d->bd_promisc) { 825 d->bd_promisc = 0; 826 CURVNET_SET(ifp->if_vnet); 827 error = ifpromisc(ifp, 0); 828 CURVNET_RESTORE(); 829 if (error != 0 && error != ENXIO) { 830 /* 831 * ENXIO can happen if a pccard is unplugged 832 * Something is really wrong if we were able to put 833 * the driver into promiscuous mode, but can't 834 * take it out. 835 */ 836 if_printf(bp->bif_ifp, 837 "bpf_detach: ifpromisc failed (%d)\n", error); 838 } 839 } 840 } 841 842 /* 843 * Close the descriptor by detaching it from its interface, 844 * deallocating its buffers, and marking it free. 845 */ 846 static void 847 bpf_dtor(void *data) 848 { 849 struct bpf_d *d = data; 850 851 BPFD_LOCK(d); 852 if (d->bd_state == BPF_WAITING) 853 callout_stop(&d->bd_callout); 854 d->bd_state = BPF_IDLE; 855 BPFD_UNLOCK(d); 856 funsetown(&d->bd_sigio); 857 bpf_detachd(d); 858 #ifdef MAC 859 mac_bpfdesc_destroy(d); 860 #endif /* MAC */ 861 seldrain(&d->bd_sel); 862 knlist_destroy(&d->bd_sel.si_note); 863 callout_drain(&d->bd_callout); 864 bpf_freed(d); 865 free(d, M_BPF); 866 } 867 868 /* 869 * Open ethernet device. Returns ENXIO for illegal minor device number, 870 * EBUSY if file is open by another process. 871 */ 872 /* ARGSUSED */ 873 static int 874 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) 875 { 876 struct bpf_d *d; 877 int error; 878 879 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); 880 error = devfs_set_cdevpriv(d, bpf_dtor); 881 if (error != 0) { 882 free(d, M_BPF); 883 return (error); 884 } 885 886 /* 887 * For historical reasons, perform a one-time initialization call to 888 * the buffer routines, even though we're not yet committed to a 889 * particular buffer method. 890 */ 891 bpf_buffer_init(d); 892 if ((flags & FREAD) == 0) 893 d->bd_writer = 2; 894 d->bd_hbuf_in_use = 0; 895 d->bd_bufmode = BPF_BUFMODE_BUFFER; 896 d->bd_sig = SIGIO; 897 d->bd_direction = BPF_D_INOUT; 898 BPF_PID_REFRESH(d, td); 899 #ifdef MAC 900 mac_bpfdesc_init(d); 901 mac_bpfdesc_create(td->td_ucred, d); 902 #endif 903 mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); 904 callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); 905 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); 906 907 return (0); 908 } 909 910 /* 911 * bpfread - read next chunk of packets from buffers 912 */ 913 static int 914 bpfread(struct cdev *dev, struct uio *uio, int ioflag) 915 { 916 struct bpf_d *d; 917 int error; 918 int non_block; 919 int timed_out; 920 921 error = devfs_get_cdevpriv((void **)&d); 922 if (error != 0) 923 return (error); 924 925 /* 926 * Restrict application to use a buffer the same size as 927 * as kernel buffers. 928 */ 929 if (uio->uio_resid != d->bd_bufsize) 930 return (EINVAL); 931 932 non_block = ((ioflag & O_NONBLOCK) != 0); 933 934 BPFD_LOCK(d); 935 BPF_PID_REFRESH_CUR(d); 936 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { 937 BPFD_UNLOCK(d); 938 return (EOPNOTSUPP); 939 } 940 if (d->bd_state == BPF_WAITING) 941 callout_stop(&d->bd_callout); 942 timed_out = (d->bd_state == BPF_TIMED_OUT); 943 d->bd_state = BPF_IDLE; 944 while (d->bd_hbuf_in_use) { 945 error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, 946 PRINET|PCATCH, "bd_hbuf", 0); 947 if (error != 0) { 948 BPFD_UNLOCK(d); 949 return (error); 950 } 951 } 952 /* 953 * If the hold buffer is empty, then do a timed sleep, which 954 * ends when the timeout expires or when enough packets 955 * have arrived to fill the store buffer. 956 */ 957 while (d->bd_hbuf == NULL) { 958 if (d->bd_slen != 0) { 959 /* 960 * A packet(s) either arrived since the previous 961 * read or arrived while we were asleep. 962 */ 963 if (d->bd_immediate || non_block || timed_out) { 964 /* 965 * Rotate the buffers and return what's here 966 * if we are in immediate mode, non-blocking 967 * flag is set, or this descriptor timed out. 968 */ 969 ROTATE_BUFFERS(d); 970 break; 971 } 972 } 973 974 /* 975 * No data is available, check to see if the bpf device 976 * is still pointed at a real interface. If not, return 977 * ENXIO so that the userland process knows to rebind 978 * it before using it again. 979 */ 980 if (d->bd_bif == NULL) { 981 BPFD_UNLOCK(d); 982 return (ENXIO); 983 } 984 985 if (non_block) { 986 BPFD_UNLOCK(d); 987 return (EWOULDBLOCK); 988 } 989 error = msleep(d, &d->bd_lock, PRINET|PCATCH, 990 "bpf", d->bd_rtout); 991 if (error == EINTR || error == ERESTART) { 992 BPFD_UNLOCK(d); 993 return (error); 994 } 995 if (error == EWOULDBLOCK) { 996 /* 997 * On a timeout, return what's in the buffer, 998 * which may be nothing. If there is something 999 * in the store buffer, we can rotate the buffers. 1000 */ 1001 if (d->bd_hbuf) 1002 /* 1003 * We filled up the buffer in between 1004 * getting the timeout and arriving 1005 * here, so we don't need to rotate. 1006 */ 1007 break; 1008 1009 if (d->bd_slen == 0) { 1010 BPFD_UNLOCK(d); 1011 return (0); 1012 } 1013 ROTATE_BUFFERS(d); 1014 break; 1015 } 1016 } 1017 /* 1018 * At this point, we know we have something in the hold slot. 1019 */ 1020 d->bd_hbuf_in_use = 1; 1021 BPFD_UNLOCK(d); 1022 1023 /* 1024 * Move data from hold buffer into user space. 1025 * We know the entire buffer is transferred since 1026 * we checked above that the read buffer is bpf_bufsize bytes. 1027 * 1028 * We do not have to worry about simultaneous reads because 1029 * we waited for sole access to the hold buffer above. 1030 */ 1031 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); 1032 1033 BPFD_LOCK(d); 1034 KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf")); 1035 d->bd_fbuf = d->bd_hbuf; 1036 d->bd_hbuf = NULL; 1037 d->bd_hlen = 0; 1038 bpf_buf_reclaimed(d); 1039 d->bd_hbuf_in_use = 0; 1040 wakeup(&d->bd_hbuf_in_use); 1041 BPFD_UNLOCK(d); 1042 1043 return (error); 1044 } 1045 1046 /* 1047 * If there are processes sleeping on this descriptor, wake them up. 1048 */ 1049 static __inline void 1050 bpf_wakeup(struct bpf_d *d) 1051 { 1052 1053 BPFD_LOCK_ASSERT(d); 1054 if (d->bd_state == BPF_WAITING) { 1055 callout_stop(&d->bd_callout); 1056 d->bd_state = BPF_IDLE; 1057 } 1058 wakeup(d); 1059 if (d->bd_async && d->bd_sig && d->bd_sigio) 1060 pgsigio(&d->bd_sigio, d->bd_sig, 0); 1061 1062 selwakeuppri(&d->bd_sel, PRINET); 1063 KNOTE_LOCKED(&d->bd_sel.si_note, 0); 1064 } 1065 1066 static void 1067 bpf_timed_out(void *arg) 1068 { 1069 struct bpf_d *d = (struct bpf_d *)arg; 1070 1071 BPFD_LOCK_ASSERT(d); 1072 1073 if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout)) 1074 return; 1075 if (d->bd_state == BPF_WAITING) { 1076 d->bd_state = BPF_TIMED_OUT; 1077 if (d->bd_slen != 0) 1078 bpf_wakeup(d); 1079 } 1080 } 1081 1082 static int 1083 bpf_ready(struct bpf_d *d) 1084 { 1085 1086 BPFD_LOCK_ASSERT(d); 1087 1088 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) 1089 return (1); 1090 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1091 d->bd_slen != 0) 1092 return (1); 1093 return (0); 1094 } 1095 1096 static int 1097 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) 1098 { 1099 struct bpf_d *d; 1100 struct ifnet *ifp; 1101 struct mbuf *m, *mc; 1102 struct sockaddr dst; 1103 struct route ro; 1104 int error, hlen; 1105 1106 error = devfs_get_cdevpriv((void **)&d); 1107 if (error != 0) 1108 return (error); 1109 1110 BPF_PID_REFRESH_CUR(d); 1111 d->bd_wcount++; 1112 /* XXX: locking required */ 1113 if (d->bd_bif == NULL) { 1114 d->bd_wdcount++; 1115 return (ENXIO); 1116 } 1117 1118 ifp = d->bd_bif->bif_ifp; 1119 1120 if ((ifp->if_flags & IFF_UP) == 0) { 1121 d->bd_wdcount++; 1122 return (ENETDOWN); 1123 } 1124 1125 if (uio->uio_resid == 0) { 1126 d->bd_wdcount++; 1127 return (0); 1128 } 1129 1130 bzero(&dst, sizeof(dst)); 1131 m = NULL; 1132 hlen = 0; 1133 /* XXX: bpf_movein() can sleep */ 1134 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, 1135 &m, &dst, &hlen, d); 1136 if (error) { 1137 d->bd_wdcount++; 1138 return (error); 1139 } 1140 d->bd_wfcount++; 1141 if (d->bd_hdrcmplt) 1142 dst.sa_family = pseudo_AF_HDRCMPLT; 1143 1144 if (d->bd_feedback) { 1145 mc = m_dup(m, M_NOWAIT); 1146 if (mc != NULL) 1147 mc->m_pkthdr.rcvif = ifp; 1148 /* Set M_PROMISC for outgoing packets to be discarded. */ 1149 if (d->bd_direction == BPF_D_INOUT) 1150 m->m_flags |= M_PROMISC; 1151 } else 1152 mc = NULL; 1153 1154 m->m_pkthdr.len -= hlen; 1155 m->m_len -= hlen; 1156 m->m_data += hlen; /* XXX */ 1157 1158 CURVNET_SET(ifp->if_vnet); 1159 #ifdef MAC 1160 BPFD_LOCK(d); 1161 mac_bpfdesc_create_mbuf(d, m); 1162 if (mc != NULL) 1163 mac_bpfdesc_create_mbuf(d, mc); 1164 BPFD_UNLOCK(d); 1165 #endif 1166 1167 bzero(&ro, sizeof(ro)); 1168 if (hlen != 0) { 1169 ro.ro_prepend = (u_char *)&dst.sa_data; 1170 ro.ro_plen = hlen; 1171 ro.ro_flags = RT_HAS_HEADER; 1172 } 1173 1174 error = (*ifp->if_output)(ifp, m, &dst, &ro); 1175 if (error) 1176 d->bd_wdcount++; 1177 1178 if (mc != NULL) { 1179 if (error == 0) 1180 (*ifp->if_input)(ifp, mc); 1181 else 1182 m_freem(mc); 1183 } 1184 CURVNET_RESTORE(); 1185 1186 return (error); 1187 } 1188 1189 /* 1190 * Reset a descriptor by flushing its packet buffer and clearing the receive 1191 * and drop counts. This is doable for kernel-only buffers, but with 1192 * zero-copy buffers, we can't write to (or rotate) buffers that are 1193 * currently owned by userspace. It would be nice if we could encapsulate 1194 * this logic in the buffer code rather than here. 1195 */ 1196 static void 1197 reset_d(struct bpf_d *d) 1198 { 1199 1200 BPFD_LOCK_ASSERT(d); 1201 1202 while (d->bd_hbuf_in_use) 1203 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET, 1204 "bd_hbuf", 0); 1205 if ((d->bd_hbuf != NULL) && 1206 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { 1207 /* Free the hold buffer. */ 1208 d->bd_fbuf = d->bd_hbuf; 1209 d->bd_hbuf = NULL; 1210 d->bd_hlen = 0; 1211 bpf_buf_reclaimed(d); 1212 } 1213 if (bpf_canwritebuf(d)) 1214 d->bd_slen = 0; 1215 d->bd_rcount = 0; 1216 d->bd_dcount = 0; 1217 d->bd_fcount = 0; 1218 d->bd_wcount = 0; 1219 d->bd_wfcount = 0; 1220 d->bd_wdcount = 0; 1221 d->bd_zcopy = 0; 1222 } 1223 1224 /* 1225 * FIONREAD Check for read packet available. 1226 * BIOCGBLEN Get buffer len [for read()]. 1227 * BIOCSETF Set read filter. 1228 * BIOCSETFNR Set read filter without resetting descriptor. 1229 * BIOCSETWF Set write filter. 1230 * BIOCFLUSH Flush read packet buffer. 1231 * BIOCPROMISC Put interface into promiscuous mode. 1232 * BIOCGDLT Get link layer type. 1233 * BIOCGETIF Get interface name. 1234 * BIOCSETIF Set interface. 1235 * BIOCSRTIMEOUT Set read timeout. 1236 * BIOCGRTIMEOUT Get read timeout. 1237 * BIOCGSTATS Get packet stats. 1238 * BIOCIMMEDIATE Set immediate mode. 1239 * BIOCVERSION Get filter language version. 1240 * BIOCGHDRCMPLT Get "header already complete" flag 1241 * BIOCSHDRCMPLT Set "header already complete" flag 1242 * BIOCGDIRECTION Get packet direction flag 1243 * BIOCSDIRECTION Set packet direction flag 1244 * BIOCGTSTAMP Get time stamp format and resolution. 1245 * BIOCSTSTAMP Set time stamp format and resolution. 1246 * BIOCLOCK Set "locked" flag 1247 * BIOCFEEDBACK Set packet feedback mode. 1248 * BIOCSETZBUF Set current zero-copy buffer locations. 1249 * BIOCGETZMAX Get maximum zero-copy buffer size. 1250 * BIOCROTZBUF Force rotation of zero-copy buffer 1251 * BIOCSETBUFMODE Set buffer mode. 1252 * BIOCGETBUFMODE Get current buffer mode. 1253 */ 1254 /* ARGSUSED */ 1255 static int 1256 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 1257 struct thread *td) 1258 { 1259 struct bpf_d *d; 1260 int error; 1261 1262 error = devfs_get_cdevpriv((void **)&d); 1263 if (error != 0) 1264 return (error); 1265 1266 /* 1267 * Refresh PID associated with this descriptor. 1268 */ 1269 BPFD_LOCK(d); 1270 BPF_PID_REFRESH(d, td); 1271 if (d->bd_state == BPF_WAITING) 1272 callout_stop(&d->bd_callout); 1273 d->bd_state = BPF_IDLE; 1274 BPFD_UNLOCK(d); 1275 1276 if (d->bd_locked == 1) { 1277 switch (cmd) { 1278 case BIOCGBLEN: 1279 case BIOCFLUSH: 1280 case BIOCGDLT: 1281 case BIOCGDLTLIST: 1282 #ifdef COMPAT_FREEBSD32 1283 case BIOCGDLTLIST32: 1284 #endif 1285 case BIOCGETIF: 1286 case BIOCGRTIMEOUT: 1287 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1288 case BIOCGRTIMEOUT32: 1289 #endif 1290 case BIOCGSTATS: 1291 case BIOCVERSION: 1292 case BIOCGRSIG: 1293 case BIOCGHDRCMPLT: 1294 case BIOCSTSTAMP: 1295 case BIOCFEEDBACK: 1296 case FIONREAD: 1297 case BIOCLOCK: 1298 case BIOCSRTIMEOUT: 1299 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1300 case BIOCSRTIMEOUT32: 1301 #endif 1302 case BIOCIMMEDIATE: 1303 case TIOCGPGRP: 1304 case BIOCROTZBUF: 1305 break; 1306 default: 1307 return (EPERM); 1308 } 1309 } 1310 #ifdef COMPAT_FREEBSD32 1311 /* 1312 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so 1313 * that it will get 32-bit packet headers. 1314 */ 1315 switch (cmd) { 1316 case BIOCSETF32: 1317 case BIOCSETFNR32: 1318 case BIOCSETWF32: 1319 case BIOCGDLTLIST32: 1320 case BIOCGRTIMEOUT32: 1321 case BIOCSRTIMEOUT32: 1322 BPFD_LOCK(d); 1323 d->bd_compat32 = 1; 1324 BPFD_UNLOCK(d); 1325 } 1326 #endif 1327 1328 CURVNET_SET(TD_TO_VNET(td)); 1329 switch (cmd) { 1330 1331 default: 1332 error = EINVAL; 1333 break; 1334 1335 /* 1336 * Check for read packet available. 1337 */ 1338 case FIONREAD: 1339 { 1340 int n; 1341 1342 BPFD_LOCK(d); 1343 n = d->bd_slen; 1344 while (d->bd_hbuf_in_use) 1345 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, 1346 PRINET, "bd_hbuf", 0); 1347 if (d->bd_hbuf) 1348 n += d->bd_hlen; 1349 BPFD_UNLOCK(d); 1350 1351 *(int *)addr = n; 1352 break; 1353 } 1354 1355 /* 1356 * Get buffer len [for read()]. 1357 */ 1358 case BIOCGBLEN: 1359 BPFD_LOCK(d); 1360 *(u_int *)addr = d->bd_bufsize; 1361 BPFD_UNLOCK(d); 1362 break; 1363 1364 /* 1365 * Set buffer length. 1366 */ 1367 case BIOCSBLEN: 1368 error = bpf_ioctl_sblen(d, (u_int *)addr); 1369 break; 1370 1371 /* 1372 * Set link layer read filter. 1373 */ 1374 case BIOCSETF: 1375 case BIOCSETFNR: 1376 case BIOCSETWF: 1377 #ifdef COMPAT_FREEBSD32 1378 case BIOCSETF32: 1379 case BIOCSETFNR32: 1380 case BIOCSETWF32: 1381 #endif 1382 error = bpf_setf(d, (struct bpf_program *)addr, cmd); 1383 break; 1384 1385 /* 1386 * Flush read packet buffer. 1387 */ 1388 case BIOCFLUSH: 1389 BPFD_LOCK(d); 1390 reset_d(d); 1391 BPFD_UNLOCK(d); 1392 break; 1393 1394 /* 1395 * Put interface into promiscuous mode. 1396 */ 1397 case BIOCPROMISC: 1398 if (d->bd_bif == NULL) { 1399 /* 1400 * No interface attached yet. 1401 */ 1402 error = EINVAL; 1403 break; 1404 } 1405 if (d->bd_promisc == 0) { 1406 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1407 if (error == 0) 1408 d->bd_promisc = 1; 1409 } 1410 break; 1411 1412 /* 1413 * Get current data link type. 1414 */ 1415 case BIOCGDLT: 1416 BPF_LOCK(); 1417 if (d->bd_bif == NULL) 1418 error = EINVAL; 1419 else 1420 *(u_int *)addr = d->bd_bif->bif_dlt; 1421 BPF_UNLOCK(); 1422 break; 1423 1424 /* 1425 * Get a list of supported data link types. 1426 */ 1427 #ifdef COMPAT_FREEBSD32 1428 case BIOCGDLTLIST32: 1429 { 1430 struct bpf_dltlist32 *list32; 1431 struct bpf_dltlist dltlist; 1432 1433 list32 = (struct bpf_dltlist32 *)addr; 1434 dltlist.bfl_len = list32->bfl_len; 1435 dltlist.bfl_list = PTRIN(list32->bfl_list); 1436 BPF_LOCK(); 1437 if (d->bd_bif == NULL) 1438 error = EINVAL; 1439 else { 1440 error = bpf_getdltlist(d, &dltlist); 1441 if (error == 0) 1442 list32->bfl_len = dltlist.bfl_len; 1443 } 1444 BPF_UNLOCK(); 1445 break; 1446 } 1447 #endif 1448 1449 case BIOCGDLTLIST: 1450 BPF_LOCK(); 1451 if (d->bd_bif == NULL) 1452 error = EINVAL; 1453 else 1454 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); 1455 BPF_UNLOCK(); 1456 break; 1457 1458 /* 1459 * Set data link type. 1460 */ 1461 case BIOCSDLT: 1462 BPF_LOCK(); 1463 if (d->bd_bif == NULL) 1464 error = EINVAL; 1465 else 1466 error = bpf_setdlt(d, *(u_int *)addr); 1467 BPF_UNLOCK(); 1468 break; 1469 1470 /* 1471 * Get interface name. 1472 */ 1473 case BIOCGETIF: 1474 BPF_LOCK(); 1475 if (d->bd_bif == NULL) 1476 error = EINVAL; 1477 else { 1478 struct ifnet *const ifp = d->bd_bif->bif_ifp; 1479 struct ifreq *const ifr = (struct ifreq *)addr; 1480 1481 strlcpy(ifr->ifr_name, ifp->if_xname, 1482 sizeof(ifr->ifr_name)); 1483 } 1484 BPF_UNLOCK(); 1485 break; 1486 1487 /* 1488 * Set interface. 1489 */ 1490 case BIOCSETIF: 1491 { 1492 int alloc_buf, size; 1493 1494 /* 1495 * Behavior here depends on the buffering model. If 1496 * we're using kernel memory buffers, then we can 1497 * allocate them here. If we're using zero-copy, 1498 * then the user process must have registered buffers 1499 * by the time we get here. 1500 */ 1501 alloc_buf = 0; 1502 BPFD_LOCK(d); 1503 if (d->bd_bufmode == BPF_BUFMODE_BUFFER && 1504 d->bd_sbuf == NULL) 1505 alloc_buf = 1; 1506 BPFD_UNLOCK(d); 1507 if (alloc_buf) { 1508 size = d->bd_bufsize; 1509 error = bpf_buffer_ioctl_sblen(d, &size); 1510 if (error != 0) 1511 break; 1512 } 1513 BPF_LOCK(); 1514 error = bpf_setif(d, (struct ifreq *)addr); 1515 BPF_UNLOCK(); 1516 break; 1517 } 1518 1519 /* 1520 * Set read timeout. 1521 */ 1522 case BIOCSRTIMEOUT: 1523 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1524 case BIOCSRTIMEOUT32: 1525 #endif 1526 { 1527 struct timeval *tv = (struct timeval *)addr; 1528 #if defined(COMPAT_FREEBSD32) && !defined(__mips__) 1529 struct timeval32 *tv32; 1530 struct timeval tv64; 1531 1532 if (cmd == BIOCSRTIMEOUT32) { 1533 tv32 = (struct timeval32 *)addr; 1534 tv = &tv64; 1535 tv->tv_sec = tv32->tv_sec; 1536 tv->tv_usec = tv32->tv_usec; 1537 } else 1538 #endif 1539 tv = (struct timeval *)addr; 1540 1541 /* 1542 * Subtract 1 tick from tvtohz() since this isn't 1543 * a one-shot timer. 1544 */ 1545 if ((error = itimerfix(tv)) == 0) 1546 d->bd_rtout = tvtohz(tv) - 1; 1547 break; 1548 } 1549 1550 /* 1551 * Get read timeout. 1552 */ 1553 case BIOCGRTIMEOUT: 1554 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1555 case BIOCGRTIMEOUT32: 1556 #endif 1557 { 1558 struct timeval *tv; 1559 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1560 struct timeval32 *tv32; 1561 struct timeval tv64; 1562 1563 if (cmd == BIOCGRTIMEOUT32) 1564 tv = &tv64; 1565 else 1566 #endif 1567 tv = (struct timeval *)addr; 1568 1569 tv->tv_sec = d->bd_rtout / hz; 1570 tv->tv_usec = (d->bd_rtout % hz) * tick; 1571 #if defined(COMPAT_FREEBSD32) && defined(__amd64__) 1572 if (cmd == BIOCGRTIMEOUT32) { 1573 tv32 = (struct timeval32 *)addr; 1574 tv32->tv_sec = tv->tv_sec; 1575 tv32->tv_usec = tv->tv_usec; 1576 } 1577 #endif 1578 1579 break; 1580 } 1581 1582 /* 1583 * Get packet stats. 1584 */ 1585 case BIOCGSTATS: 1586 { 1587 struct bpf_stat *bs = (struct bpf_stat *)addr; 1588 1589 /* XXXCSJP overflow */ 1590 bs->bs_recv = d->bd_rcount; 1591 bs->bs_drop = d->bd_dcount; 1592 break; 1593 } 1594 1595 /* 1596 * Set immediate mode. 1597 */ 1598 case BIOCIMMEDIATE: 1599 BPFD_LOCK(d); 1600 d->bd_immediate = *(u_int *)addr; 1601 BPFD_UNLOCK(d); 1602 break; 1603 1604 case BIOCVERSION: 1605 { 1606 struct bpf_version *bv = (struct bpf_version *)addr; 1607 1608 bv->bv_major = BPF_MAJOR_VERSION; 1609 bv->bv_minor = BPF_MINOR_VERSION; 1610 break; 1611 } 1612 1613 /* 1614 * Get "header already complete" flag 1615 */ 1616 case BIOCGHDRCMPLT: 1617 BPFD_LOCK(d); 1618 *(u_int *)addr = d->bd_hdrcmplt; 1619 BPFD_UNLOCK(d); 1620 break; 1621 1622 /* 1623 * Set "header already complete" flag 1624 */ 1625 case BIOCSHDRCMPLT: 1626 BPFD_LOCK(d); 1627 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1628 BPFD_UNLOCK(d); 1629 break; 1630 1631 /* 1632 * Get packet direction flag 1633 */ 1634 case BIOCGDIRECTION: 1635 BPFD_LOCK(d); 1636 *(u_int *)addr = d->bd_direction; 1637 BPFD_UNLOCK(d); 1638 break; 1639 1640 /* 1641 * Set packet direction flag 1642 */ 1643 case BIOCSDIRECTION: 1644 { 1645 u_int direction; 1646 1647 direction = *(u_int *)addr; 1648 switch (direction) { 1649 case BPF_D_IN: 1650 case BPF_D_INOUT: 1651 case BPF_D_OUT: 1652 BPFD_LOCK(d); 1653 d->bd_direction = direction; 1654 BPFD_UNLOCK(d); 1655 break; 1656 default: 1657 error = EINVAL; 1658 } 1659 } 1660 break; 1661 1662 /* 1663 * Get packet timestamp format and resolution. 1664 */ 1665 case BIOCGTSTAMP: 1666 BPFD_LOCK(d); 1667 *(u_int *)addr = d->bd_tstamp; 1668 BPFD_UNLOCK(d); 1669 break; 1670 1671 /* 1672 * Set packet timestamp format and resolution. 1673 */ 1674 case BIOCSTSTAMP: 1675 { 1676 u_int func; 1677 1678 func = *(u_int *)addr; 1679 if (BPF_T_VALID(func)) 1680 d->bd_tstamp = func; 1681 else 1682 error = EINVAL; 1683 } 1684 break; 1685 1686 case BIOCFEEDBACK: 1687 BPFD_LOCK(d); 1688 d->bd_feedback = *(u_int *)addr; 1689 BPFD_UNLOCK(d); 1690 break; 1691 1692 case BIOCLOCK: 1693 BPFD_LOCK(d); 1694 d->bd_locked = 1; 1695 BPFD_UNLOCK(d); 1696 break; 1697 1698 case FIONBIO: /* Non-blocking I/O */ 1699 break; 1700 1701 case FIOASYNC: /* Send signal on receive packets */ 1702 BPFD_LOCK(d); 1703 d->bd_async = *(int *)addr; 1704 BPFD_UNLOCK(d); 1705 break; 1706 1707 case FIOSETOWN: 1708 /* 1709 * XXX: Add some sort of locking here? 1710 * fsetown() can sleep. 1711 */ 1712 error = fsetown(*(int *)addr, &d->bd_sigio); 1713 break; 1714 1715 case FIOGETOWN: 1716 BPFD_LOCK(d); 1717 *(int *)addr = fgetown(&d->bd_sigio); 1718 BPFD_UNLOCK(d); 1719 break; 1720 1721 /* This is deprecated, FIOSETOWN should be used instead. */ 1722 case TIOCSPGRP: 1723 error = fsetown(-(*(int *)addr), &d->bd_sigio); 1724 break; 1725 1726 /* This is deprecated, FIOGETOWN should be used instead. */ 1727 case TIOCGPGRP: 1728 *(int *)addr = -fgetown(&d->bd_sigio); 1729 break; 1730 1731 case BIOCSRSIG: /* Set receive signal */ 1732 { 1733 u_int sig; 1734 1735 sig = *(u_int *)addr; 1736 1737 if (sig >= NSIG) 1738 error = EINVAL; 1739 else { 1740 BPFD_LOCK(d); 1741 d->bd_sig = sig; 1742 BPFD_UNLOCK(d); 1743 } 1744 break; 1745 } 1746 case BIOCGRSIG: 1747 BPFD_LOCK(d); 1748 *(u_int *)addr = d->bd_sig; 1749 BPFD_UNLOCK(d); 1750 break; 1751 1752 case BIOCGETBUFMODE: 1753 BPFD_LOCK(d); 1754 *(u_int *)addr = d->bd_bufmode; 1755 BPFD_UNLOCK(d); 1756 break; 1757 1758 case BIOCSETBUFMODE: 1759 /* 1760 * Allow the buffering mode to be changed as long as we 1761 * haven't yet committed to a particular mode. Our 1762 * definition of commitment, for now, is whether or not a 1763 * buffer has been allocated or an interface attached, since 1764 * that's the point where things get tricky. 1765 */ 1766 switch (*(u_int *)addr) { 1767 case BPF_BUFMODE_BUFFER: 1768 break; 1769 1770 case BPF_BUFMODE_ZBUF: 1771 if (bpf_zerocopy_enable) 1772 break; 1773 /* FALLSTHROUGH */ 1774 1775 default: 1776 CURVNET_RESTORE(); 1777 return (EINVAL); 1778 } 1779 1780 BPFD_LOCK(d); 1781 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || 1782 d->bd_fbuf != NULL || d->bd_bif != NULL) { 1783 BPFD_UNLOCK(d); 1784 CURVNET_RESTORE(); 1785 return (EBUSY); 1786 } 1787 d->bd_bufmode = *(u_int *)addr; 1788 BPFD_UNLOCK(d); 1789 break; 1790 1791 case BIOCGETZMAX: 1792 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); 1793 break; 1794 1795 case BIOCSETZBUF: 1796 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); 1797 break; 1798 1799 case BIOCROTZBUF: 1800 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); 1801 break; 1802 } 1803 CURVNET_RESTORE(); 1804 return (error); 1805 } 1806 1807 /* 1808 * Set d's packet filter program to fp. If this file already has a filter, 1809 * free it and replace it. Returns EINVAL for bogus requests. 1810 * 1811 * Note we need global lock here to serialize bpf_setf() and bpf_setif() calls 1812 * since reading d->bd_bif can't be protected by d or interface lock due to 1813 * lock order. 1814 * 1815 * Additionally, we have to acquire interface write lock due to bpf_mtap() uses 1816 * interface read lock to read all filers. 1817 * 1818 */ 1819 static int 1820 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1821 { 1822 #ifdef COMPAT_FREEBSD32 1823 struct bpf_program fp_swab; 1824 struct bpf_program32 *fp32; 1825 #endif 1826 struct bpf_insn *fcode, *old; 1827 #ifdef BPF_JITTER 1828 bpf_jit_filter *jfunc, *ofunc; 1829 #endif 1830 size_t size; 1831 u_int flen; 1832 int need_upgrade; 1833 1834 #ifdef COMPAT_FREEBSD32 1835 switch (cmd) { 1836 case BIOCSETF32: 1837 case BIOCSETWF32: 1838 case BIOCSETFNR32: 1839 fp32 = (struct bpf_program32 *)fp; 1840 fp_swab.bf_len = fp32->bf_len; 1841 fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns; 1842 fp = &fp_swab; 1843 switch (cmd) { 1844 case BIOCSETF32: 1845 cmd = BIOCSETF; 1846 break; 1847 case BIOCSETWF32: 1848 cmd = BIOCSETWF; 1849 break; 1850 } 1851 break; 1852 } 1853 #endif 1854 1855 fcode = NULL; 1856 #ifdef BPF_JITTER 1857 jfunc = ofunc = NULL; 1858 #endif 1859 need_upgrade = 0; 1860 1861 /* 1862 * Check new filter validness before acquiring any locks. 1863 * Allocate memory for new filter, if needed. 1864 */ 1865 flen = fp->bf_len; 1866 if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0)) 1867 return (EINVAL); 1868 size = flen * sizeof(*fp->bf_insns); 1869 if (size > 0) { 1870 /* We're setting up new filter. Copy and check actual data. */ 1871 fcode = malloc(size, M_BPF, M_WAITOK); 1872 if (copyin(fp->bf_insns, fcode, size) != 0 || 1873 !bpf_validate(fcode, flen)) { 1874 free(fcode, M_BPF); 1875 return (EINVAL); 1876 } 1877 #ifdef BPF_JITTER 1878 /* Filter is copied inside fcode and is perfectly valid. */ 1879 jfunc = bpf_jitter(fcode, flen); 1880 #endif 1881 } 1882 1883 BPF_LOCK(); 1884 1885 /* 1886 * Set up new filter. 1887 * Protect filter change by interface lock. 1888 * Additionally, we are protected by global lock here. 1889 */ 1890 if (d->bd_bif != NULL) 1891 BPFIF_WLOCK(d->bd_bif); 1892 BPFD_LOCK(d); 1893 if (cmd == BIOCSETWF) { 1894 old = d->bd_wfilter; 1895 d->bd_wfilter = fcode; 1896 } else { 1897 old = d->bd_rfilter; 1898 d->bd_rfilter = fcode; 1899 #ifdef BPF_JITTER 1900 ofunc = d->bd_bfilter; 1901 d->bd_bfilter = jfunc; 1902 #endif 1903 if (cmd == BIOCSETF) 1904 reset_d(d); 1905 1906 need_upgrade = bpf_check_upgrade(cmd, d, fcode, flen); 1907 } 1908 BPFD_UNLOCK(d); 1909 if (d->bd_bif != NULL) 1910 BPFIF_WUNLOCK(d->bd_bif); 1911 if (old != NULL) 1912 free(old, M_BPF); 1913 #ifdef BPF_JITTER 1914 if (ofunc != NULL) 1915 bpf_destroy_jit_filter(ofunc); 1916 #endif 1917 1918 /* Move d to active readers list. */ 1919 if (need_upgrade != 0) 1920 bpf_upgraded(d); 1921 1922 BPF_UNLOCK(); 1923 return (0); 1924 } 1925 1926 /* 1927 * Detach a file from its current interface (if attached at all) and attach 1928 * to the interface indicated by the name stored in ifr. 1929 * Return an errno or 0. 1930 */ 1931 static int 1932 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1933 { 1934 struct bpf_if *bp; 1935 struct ifnet *theywant; 1936 1937 BPF_LOCK_ASSERT(); 1938 1939 theywant = ifunit(ifr->ifr_name); 1940 if (theywant == NULL || theywant->if_bpf == NULL) 1941 return (ENXIO); 1942 1943 bp = theywant->if_bpf; 1944 1945 /* Check if interface is not being detached from BPF */ 1946 BPFIF_RLOCK(bp); 1947 if (bp->bif_flags & BPFIF_FLAG_DYING) { 1948 BPFIF_RUNLOCK(bp); 1949 return (ENXIO); 1950 } 1951 BPFIF_RUNLOCK(bp); 1952 1953 /* 1954 * At this point, we expect the buffer is already allocated. If not, 1955 * return an error. 1956 */ 1957 switch (d->bd_bufmode) { 1958 case BPF_BUFMODE_BUFFER: 1959 case BPF_BUFMODE_ZBUF: 1960 if (d->bd_sbuf == NULL) 1961 return (EINVAL); 1962 break; 1963 1964 default: 1965 panic("bpf_setif: bufmode %d", d->bd_bufmode); 1966 } 1967 if (bp != d->bd_bif) 1968 bpf_attachd(d, bp); 1969 BPFD_LOCK(d); 1970 reset_d(d); 1971 BPFD_UNLOCK(d); 1972 return (0); 1973 } 1974 1975 /* 1976 * Support for select() and poll() system calls 1977 * 1978 * Return true iff the specific operation will not block indefinitely. 1979 * Otherwise, return false but make a note that a selwakeup() must be done. 1980 */ 1981 static int 1982 bpfpoll(struct cdev *dev, int events, struct thread *td) 1983 { 1984 struct bpf_d *d; 1985 int revents; 1986 1987 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) 1988 return (events & 1989 (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 1990 1991 /* 1992 * Refresh PID associated with this descriptor. 1993 */ 1994 revents = events & (POLLOUT | POLLWRNORM); 1995 BPFD_LOCK(d); 1996 BPF_PID_REFRESH(d, td); 1997 if (events & (POLLIN | POLLRDNORM)) { 1998 if (bpf_ready(d)) 1999 revents |= events & (POLLIN | POLLRDNORM); 2000 else { 2001 selrecord(td, &d->bd_sel); 2002 /* Start the read timeout if necessary. */ 2003 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 2004 callout_reset(&d->bd_callout, d->bd_rtout, 2005 bpf_timed_out, d); 2006 d->bd_state = BPF_WAITING; 2007 } 2008 } 2009 } 2010 BPFD_UNLOCK(d); 2011 return (revents); 2012 } 2013 2014 /* 2015 * Support for kevent() system call. Register EVFILT_READ filters and 2016 * reject all others. 2017 */ 2018 int 2019 bpfkqfilter(struct cdev *dev, struct knote *kn) 2020 { 2021 struct bpf_d *d; 2022 2023 if (devfs_get_cdevpriv((void **)&d) != 0 || 2024 kn->kn_filter != EVFILT_READ) 2025 return (1); 2026 2027 /* 2028 * Refresh PID associated with this descriptor. 2029 */ 2030 BPFD_LOCK(d); 2031 BPF_PID_REFRESH_CUR(d); 2032 kn->kn_fop = &bpfread_filtops; 2033 kn->kn_hook = d; 2034 knlist_add(&d->bd_sel.si_note, kn, 1); 2035 BPFD_UNLOCK(d); 2036 2037 return (0); 2038 } 2039 2040 static void 2041 filt_bpfdetach(struct knote *kn) 2042 { 2043 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 2044 2045 knlist_remove(&d->bd_sel.si_note, kn, 0); 2046 } 2047 2048 static int 2049 filt_bpfread(struct knote *kn, long hint) 2050 { 2051 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 2052 int ready; 2053 2054 BPFD_LOCK_ASSERT(d); 2055 ready = bpf_ready(d); 2056 if (ready) { 2057 kn->kn_data = d->bd_slen; 2058 /* 2059 * Ignore the hold buffer if it is being copied to user space. 2060 */ 2061 if (!d->bd_hbuf_in_use && d->bd_hbuf) 2062 kn->kn_data += d->bd_hlen; 2063 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 2064 callout_reset(&d->bd_callout, d->bd_rtout, 2065 bpf_timed_out, d); 2066 d->bd_state = BPF_WAITING; 2067 } 2068 2069 return (ready); 2070 } 2071 2072 #define BPF_TSTAMP_NONE 0 2073 #define BPF_TSTAMP_FAST 1 2074 #define BPF_TSTAMP_NORMAL 2 2075 #define BPF_TSTAMP_EXTERN 3 2076 2077 static int 2078 bpf_ts_quality(int tstype) 2079 { 2080 2081 if (tstype == BPF_T_NONE) 2082 return (BPF_TSTAMP_NONE); 2083 if ((tstype & BPF_T_FAST) != 0) 2084 return (BPF_TSTAMP_FAST); 2085 2086 return (BPF_TSTAMP_NORMAL); 2087 } 2088 2089 static int 2090 bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m) 2091 { 2092 struct m_tag *tag; 2093 int quality; 2094 2095 quality = bpf_ts_quality(tstype); 2096 if (quality == BPF_TSTAMP_NONE) 2097 return (quality); 2098 2099 if (m != NULL) { 2100 tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL); 2101 if (tag != NULL) { 2102 *bt = *(struct bintime *)(tag + 1); 2103 return (BPF_TSTAMP_EXTERN); 2104 } 2105 } 2106 if (quality == BPF_TSTAMP_NORMAL) 2107 binuptime(bt); 2108 else 2109 getbinuptime(bt); 2110 2111 return (quality); 2112 } 2113 2114 /* 2115 * Incoming linkage from device drivers. Process the packet pkt, of length 2116 * pktlen, which is stored in a contiguous buffer. The packet is parsed 2117 * by each process' filter, and if accepted, stashed into the corresponding 2118 * buffer. 2119 */ 2120 void 2121 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 2122 { 2123 struct bintime bt; 2124 struct bpf_d *d; 2125 #ifdef BPF_JITTER 2126 bpf_jit_filter *bf; 2127 #endif 2128 u_int slen; 2129 int gottime; 2130 2131 gottime = BPF_TSTAMP_NONE; 2132 2133 BPFIF_RLOCK(bp); 2134 2135 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 2136 /* 2137 * We are not using any locks for d here because: 2138 * 1) any filter change is protected by interface 2139 * write lock 2140 * 2) destroying/detaching d is protected by interface 2141 * write lock, too 2142 */ 2143 2144 /* XXX: Do not protect counter for the sake of performance. */ 2145 ++d->bd_rcount; 2146 /* 2147 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no 2148 * way for the caller to indiciate to us whether this packet 2149 * is inbound or outbound. In the bpf_mtap() routines, we use 2150 * the interface pointers on the mbuf to figure it out. 2151 */ 2152 #ifdef BPF_JITTER 2153 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; 2154 if (bf != NULL) 2155 slen = (*(bf->func))(pkt, pktlen, pktlen); 2156 else 2157 #endif 2158 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); 2159 if (slen != 0) { 2160 /* 2161 * Filter matches. Let's to acquire write lock. 2162 */ 2163 BPFD_LOCK(d); 2164 2165 d->bd_fcount++; 2166 if (gottime < bpf_ts_quality(d->bd_tstamp)) 2167 gottime = bpf_gettime(&bt, d->bd_tstamp, NULL); 2168 #ifdef MAC 2169 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 2170 #endif 2171 catchpacket(d, pkt, pktlen, slen, 2172 bpf_append_bytes, &bt); 2173 BPFD_UNLOCK(d); 2174 } 2175 } 2176 BPFIF_RUNLOCK(bp); 2177 } 2178 2179 #define BPF_CHECK_DIRECTION(d, r, i) \ 2180 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ 2181 ((d)->bd_direction == BPF_D_OUT && (r) == (i))) 2182 2183 /* 2184 * Incoming linkage from device drivers, when packet is in an mbuf chain. 2185 * Locking model is explained in bpf_tap(). 2186 */ 2187 void 2188 bpf_mtap(struct bpf_if *bp, struct mbuf *m) 2189 { 2190 struct bintime bt; 2191 struct bpf_d *d; 2192 #ifdef BPF_JITTER 2193 bpf_jit_filter *bf; 2194 #endif 2195 u_int pktlen, slen; 2196 int gottime; 2197 2198 /* Skip outgoing duplicate packets. */ 2199 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 2200 m->m_flags &= ~M_PROMISC; 2201 return; 2202 } 2203 2204 pktlen = m_length(m, NULL); 2205 gottime = BPF_TSTAMP_NONE; 2206 2207 BPFIF_RLOCK(bp); 2208 2209 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 2210 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 2211 continue; 2212 ++d->bd_rcount; 2213 #ifdef BPF_JITTER 2214 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; 2215 /* XXX We cannot handle multiple mbufs. */ 2216 if (bf != NULL && m->m_next == NULL) 2217 slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen); 2218 else 2219 #endif 2220 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); 2221 if (slen != 0) { 2222 BPFD_LOCK(d); 2223 2224 d->bd_fcount++; 2225 if (gottime < bpf_ts_quality(d->bd_tstamp)) 2226 gottime = bpf_gettime(&bt, d->bd_tstamp, m); 2227 #ifdef MAC 2228 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 2229 #endif 2230 catchpacket(d, (u_char *)m, pktlen, slen, 2231 bpf_append_mbuf, &bt); 2232 BPFD_UNLOCK(d); 2233 } 2234 } 2235 BPFIF_RUNLOCK(bp); 2236 } 2237 2238 /* 2239 * Incoming linkage from device drivers, when packet is in 2240 * an mbuf chain and to be prepended by a contiguous header. 2241 */ 2242 void 2243 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 2244 { 2245 struct bintime bt; 2246 struct mbuf mb; 2247 struct bpf_d *d; 2248 u_int pktlen, slen; 2249 int gottime; 2250 2251 /* Skip outgoing duplicate packets. */ 2252 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 2253 m->m_flags &= ~M_PROMISC; 2254 return; 2255 } 2256 2257 pktlen = m_length(m, NULL); 2258 /* 2259 * Craft on-stack mbuf suitable for passing to bpf_filter. 2260 * Note that we cut corners here; we only setup what's 2261 * absolutely needed--this mbuf should never go anywhere else. 2262 */ 2263 mb.m_next = m; 2264 mb.m_data = data; 2265 mb.m_len = dlen; 2266 pktlen += dlen; 2267 2268 gottime = BPF_TSTAMP_NONE; 2269 2270 BPFIF_RLOCK(bp); 2271 2272 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 2273 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 2274 continue; 2275 ++d->bd_rcount; 2276 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); 2277 if (slen != 0) { 2278 BPFD_LOCK(d); 2279 2280 d->bd_fcount++; 2281 if (gottime < bpf_ts_quality(d->bd_tstamp)) 2282 gottime = bpf_gettime(&bt, d->bd_tstamp, m); 2283 #ifdef MAC 2284 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 2285 #endif 2286 catchpacket(d, (u_char *)&mb, pktlen, slen, 2287 bpf_append_mbuf, &bt); 2288 BPFD_UNLOCK(d); 2289 } 2290 } 2291 BPFIF_RUNLOCK(bp); 2292 } 2293 2294 #undef BPF_CHECK_DIRECTION 2295 2296 #undef BPF_TSTAMP_NONE 2297 #undef BPF_TSTAMP_FAST 2298 #undef BPF_TSTAMP_NORMAL 2299 #undef BPF_TSTAMP_EXTERN 2300 2301 static int 2302 bpf_hdrlen(struct bpf_d *d) 2303 { 2304 int hdrlen; 2305 2306 hdrlen = d->bd_bif->bif_hdrlen; 2307 #ifndef BURN_BRIDGES 2308 if (d->bd_tstamp == BPF_T_NONE || 2309 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) 2310 #ifdef COMPAT_FREEBSD32 2311 if (d->bd_compat32) 2312 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32); 2313 else 2314 #endif 2315 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr); 2316 else 2317 #endif 2318 hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr); 2319 #ifdef COMPAT_FREEBSD32 2320 if (d->bd_compat32) 2321 hdrlen = BPF_WORDALIGN32(hdrlen); 2322 else 2323 #endif 2324 hdrlen = BPF_WORDALIGN(hdrlen); 2325 2326 return (hdrlen - d->bd_bif->bif_hdrlen); 2327 } 2328 2329 static void 2330 bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype) 2331 { 2332 struct bintime bt2, boottimebin; 2333 struct timeval tsm; 2334 struct timespec tsn; 2335 2336 if ((tstype & BPF_T_MONOTONIC) == 0) { 2337 bt2 = *bt; 2338 getboottimebin(&boottimebin); 2339 bintime_add(&bt2, &boottimebin); 2340 bt = &bt2; 2341 } 2342 switch (BPF_T_FORMAT(tstype)) { 2343 case BPF_T_MICROTIME: 2344 bintime2timeval(bt, &tsm); 2345 ts->bt_sec = tsm.tv_sec; 2346 ts->bt_frac = tsm.tv_usec; 2347 break; 2348 case BPF_T_NANOTIME: 2349 bintime2timespec(bt, &tsn); 2350 ts->bt_sec = tsn.tv_sec; 2351 ts->bt_frac = tsn.tv_nsec; 2352 break; 2353 case BPF_T_BINTIME: 2354 ts->bt_sec = bt->sec; 2355 ts->bt_frac = bt->frac; 2356 break; 2357 } 2358 } 2359 2360 /* 2361 * Move the packet data from interface memory (pkt) into the 2362 * store buffer. "cpfn" is the routine called to do the actual data 2363 * transfer. bcopy is passed in to copy contiguous chunks, while 2364 * bpf_append_mbuf is passed in to copy mbuf chains. In the latter case, 2365 * pkt is really an mbuf. 2366 */ 2367 static void 2368 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 2369 void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int), 2370 struct bintime *bt) 2371 { 2372 struct bpf_xhdr hdr; 2373 #ifndef BURN_BRIDGES 2374 struct bpf_hdr hdr_old; 2375 #ifdef COMPAT_FREEBSD32 2376 struct bpf_hdr32 hdr32_old; 2377 #endif 2378 #endif 2379 int caplen, curlen, hdrlen, totlen; 2380 int do_wakeup = 0; 2381 int do_timestamp; 2382 int tstype; 2383 2384 BPFD_LOCK_ASSERT(d); 2385 2386 /* 2387 * Detect whether user space has released a buffer back to us, and if 2388 * so, move it from being a hold buffer to a free buffer. This may 2389 * not be the best place to do it (for example, we might only want to 2390 * run this check if we need the space), but for now it's a reliable 2391 * spot to do it. 2392 */ 2393 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { 2394 d->bd_fbuf = d->bd_hbuf; 2395 d->bd_hbuf = NULL; 2396 d->bd_hlen = 0; 2397 bpf_buf_reclaimed(d); 2398 } 2399 2400 /* 2401 * Figure out how many bytes to move. If the packet is 2402 * greater or equal to the snapshot length, transfer that 2403 * much. Otherwise, transfer the whole packet (unless 2404 * we hit the buffer size limit). 2405 */ 2406 hdrlen = bpf_hdrlen(d); 2407 totlen = hdrlen + min(snaplen, pktlen); 2408 if (totlen > d->bd_bufsize) 2409 totlen = d->bd_bufsize; 2410 2411 /* 2412 * Round up the end of the previous packet to the next longword. 2413 * 2414 * Drop the packet if there's no room and no hope of room 2415 * If the packet would overflow the storage buffer or the storage 2416 * buffer is considered immutable by the buffer model, try to rotate 2417 * the buffer and wakeup pending processes. 2418 */ 2419 #ifdef COMPAT_FREEBSD32 2420 if (d->bd_compat32) 2421 curlen = BPF_WORDALIGN32(d->bd_slen); 2422 else 2423 #endif 2424 curlen = BPF_WORDALIGN(d->bd_slen); 2425 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { 2426 if (d->bd_fbuf == NULL) { 2427 /* 2428 * There's no room in the store buffer, and no 2429 * prospect of room, so drop the packet. Notify the 2430 * buffer model. 2431 */ 2432 bpf_buffull(d); 2433 ++d->bd_dcount; 2434 return; 2435 } 2436 KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use")); 2437 ROTATE_BUFFERS(d); 2438 do_wakeup = 1; 2439 curlen = 0; 2440 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) 2441 /* 2442 * Immediate mode is set, or the read timeout has already 2443 * expired during a select call. A packet arrived, so the 2444 * reader should be woken up. 2445 */ 2446 do_wakeup = 1; 2447 caplen = totlen - hdrlen; 2448 tstype = d->bd_tstamp; 2449 do_timestamp = tstype != BPF_T_NONE; 2450 #ifndef BURN_BRIDGES 2451 if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) { 2452 struct bpf_ts ts; 2453 if (do_timestamp) 2454 bpf_bintime2ts(bt, &ts, tstype); 2455 #ifdef COMPAT_FREEBSD32 2456 if (d->bd_compat32) { 2457 bzero(&hdr32_old, sizeof(hdr32_old)); 2458 if (do_timestamp) { 2459 hdr32_old.bh_tstamp.tv_sec = ts.bt_sec; 2460 hdr32_old.bh_tstamp.tv_usec = ts.bt_frac; 2461 } 2462 hdr32_old.bh_datalen = pktlen; 2463 hdr32_old.bh_hdrlen = hdrlen; 2464 hdr32_old.bh_caplen = caplen; 2465 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, 2466 sizeof(hdr32_old)); 2467 goto copy; 2468 } 2469 #endif 2470 bzero(&hdr_old, sizeof(hdr_old)); 2471 if (do_timestamp) { 2472 hdr_old.bh_tstamp.tv_sec = ts.bt_sec; 2473 hdr_old.bh_tstamp.tv_usec = ts.bt_frac; 2474 } 2475 hdr_old.bh_datalen = pktlen; 2476 hdr_old.bh_hdrlen = hdrlen; 2477 hdr_old.bh_caplen = caplen; 2478 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, 2479 sizeof(hdr_old)); 2480 goto copy; 2481 } 2482 #endif 2483 2484 /* 2485 * Append the bpf header. Note we append the actual header size, but 2486 * move forward the length of the header plus padding. 2487 */ 2488 bzero(&hdr, sizeof(hdr)); 2489 if (do_timestamp) 2490 bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype); 2491 hdr.bh_datalen = pktlen; 2492 hdr.bh_hdrlen = hdrlen; 2493 hdr.bh_caplen = caplen; 2494 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); 2495 2496 /* 2497 * Copy the packet data into the store buffer and update its length. 2498 */ 2499 #ifndef BURN_BRIDGES 2500 copy: 2501 #endif 2502 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); 2503 d->bd_slen = curlen + totlen; 2504 2505 if (do_wakeup) 2506 bpf_wakeup(d); 2507 } 2508 2509 /* 2510 * Free buffers currently in use by a descriptor. 2511 * Called on close. 2512 */ 2513 static void 2514 bpf_freed(struct bpf_d *d) 2515 { 2516 2517 /* 2518 * We don't need to lock out interrupts since this descriptor has 2519 * been detached from its interface and it yet hasn't been marked 2520 * free. 2521 */ 2522 bpf_free(d); 2523 if (d->bd_rfilter != NULL) { 2524 free((caddr_t)d->bd_rfilter, M_BPF); 2525 #ifdef BPF_JITTER 2526 if (d->bd_bfilter != NULL) 2527 bpf_destroy_jit_filter(d->bd_bfilter); 2528 #endif 2529 } 2530 if (d->bd_wfilter != NULL) 2531 free((caddr_t)d->bd_wfilter, M_BPF); 2532 mtx_destroy(&d->bd_lock); 2533 } 2534 2535 /* 2536 * Attach an interface to bpf. dlt is the link layer type; hdrlen is the 2537 * fixed size of the link header (variable length headers not yet supported). 2538 */ 2539 void 2540 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2541 { 2542 2543 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2544 } 2545 2546 /* 2547 * Attach an interface to bpf. ifp is a pointer to the structure 2548 * defining the interface to be attached, dlt is the link layer type, 2549 * and hdrlen is the fixed size of the link header (variable length 2550 * headers are not yet supporrted). 2551 */ 2552 void 2553 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2554 { 2555 struct bpf_if *bp; 2556 2557 bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO); 2558 if (bp == NULL) 2559 panic("bpfattach"); 2560 2561 LIST_INIT(&bp->bif_dlist); 2562 LIST_INIT(&bp->bif_wlist); 2563 bp->bif_ifp = ifp; 2564 bp->bif_dlt = dlt; 2565 rw_init(&bp->bif_lock, "bpf interface lock"); 2566 KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized")); 2567 bp->bif_bpf = driverp; 2568 *driverp = bp; 2569 2570 BPF_LOCK(); 2571 LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); 2572 BPF_UNLOCK(); 2573 2574 bp->bif_hdrlen = hdrlen; 2575 2576 if (bootverbose && IS_DEFAULT_VNET(curvnet)) 2577 if_printf(ifp, "bpf attached\n"); 2578 } 2579 2580 #ifdef VIMAGE 2581 /* 2582 * When moving interfaces between vnet instances we need a way to 2583 * query the dlt and hdrlen before detach so we can re-attch the if_bpf 2584 * after the vmove. We unfortunately have no device driver infrastructure 2585 * to query the interface for these values after creation/attach, thus 2586 * add this as a workaround. 2587 */ 2588 int 2589 bpf_get_bp_params(struct bpf_if *bp, u_int *bif_dlt, u_int *bif_hdrlen) 2590 { 2591 2592 if (bp == NULL) 2593 return (ENXIO); 2594 if (bif_dlt == NULL && bif_hdrlen == NULL) 2595 return (0); 2596 2597 if (bif_dlt != NULL) 2598 *bif_dlt = bp->bif_dlt; 2599 if (bif_hdrlen != NULL) 2600 *bif_hdrlen = bp->bif_hdrlen; 2601 2602 return (0); 2603 } 2604 #endif 2605 2606 /* 2607 * Detach bpf from an interface. This involves detaching each descriptor 2608 * associated with the interface. Notify each descriptor as it's detached 2609 * so that any sleepers wake up and get ENXIO. 2610 */ 2611 void 2612 bpfdetach(struct ifnet *ifp) 2613 { 2614 struct bpf_if *bp, *bp_temp; 2615 struct bpf_d *d; 2616 int ndetached; 2617 2618 ndetached = 0; 2619 2620 BPF_LOCK(); 2621 /* Find all bpf_if struct's which reference ifp and detach them. */ 2622 LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) { 2623 if (ifp != bp->bif_ifp) 2624 continue; 2625 2626 LIST_REMOVE(bp, bif_next); 2627 /* Add to to-be-freed list */ 2628 LIST_INSERT_HEAD(&bpf_freelist, bp, bif_next); 2629 2630 ndetached++; 2631 /* 2632 * Delay freeing bp till interface is detached 2633 * and all routes through this interface are removed. 2634 * Mark bp as detached to restrict new consumers. 2635 */ 2636 BPFIF_WLOCK(bp); 2637 bp->bif_flags |= BPFIF_FLAG_DYING; 2638 *bp->bif_bpf = NULL; 2639 BPFIF_WUNLOCK(bp); 2640 2641 CTR4(KTR_NET, "%s: sheduling free for encap %d (%p) for if %p", 2642 __func__, bp->bif_dlt, bp, ifp); 2643 2644 /* Free common descriptors */ 2645 while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) { 2646 bpf_detachd_locked(d); 2647 BPFD_LOCK(d); 2648 bpf_wakeup(d); 2649 BPFD_UNLOCK(d); 2650 } 2651 2652 /* Free writer-only descriptors */ 2653 while ((d = LIST_FIRST(&bp->bif_wlist)) != NULL) { 2654 bpf_detachd_locked(d); 2655 BPFD_LOCK(d); 2656 bpf_wakeup(d); 2657 BPFD_UNLOCK(d); 2658 } 2659 } 2660 BPF_UNLOCK(); 2661 2662 #ifdef INVARIANTS 2663 if (ndetached == 0) 2664 printf("bpfdetach: %s was not attached\n", ifp->if_xname); 2665 #endif 2666 } 2667 2668 /* 2669 * Interface departure handler. 2670 * Note departure event does not guarantee interface is going down. 2671 * Interface renaming is currently done via departure/arrival event set. 2672 * 2673 * Departure handled is called after all routes pointing to 2674 * given interface are removed and interface is in down state 2675 * restricting any packets to be sent/received. We assume it is now safe 2676 * to free data allocated by BPF. 2677 */ 2678 static void 2679 bpf_ifdetach(void *arg __unused, struct ifnet *ifp) 2680 { 2681 struct bpf_if *bp, *bp_temp; 2682 int nmatched = 0; 2683 2684 /* Ignore ifnet renaming. */ 2685 if (ifp->if_flags & IFF_RENAMING) 2686 return; 2687 2688 BPF_LOCK(); 2689 /* 2690 * Find matching entries in free list. 2691 * Nothing should be found if bpfdetach() was not called. 2692 */ 2693 LIST_FOREACH_SAFE(bp, &bpf_freelist, bif_next, bp_temp) { 2694 if (ifp != bp->bif_ifp) 2695 continue; 2696 2697 CTR3(KTR_NET, "%s: freeing BPF instance %p for interface %p", 2698 __func__, bp, ifp); 2699 2700 LIST_REMOVE(bp, bif_next); 2701 2702 rw_destroy(&bp->bif_lock); 2703 free(bp, M_BPF); 2704 2705 nmatched++; 2706 } 2707 BPF_UNLOCK(); 2708 } 2709 2710 /* 2711 * Get a list of available data link type of the interface. 2712 */ 2713 static int 2714 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2715 { 2716 struct ifnet *ifp; 2717 struct bpf_if *bp; 2718 u_int *lst; 2719 int error, n, n1; 2720 2721 BPF_LOCK_ASSERT(); 2722 2723 ifp = d->bd_bif->bif_ifp; 2724 again: 2725 n1 = 0; 2726 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2727 if (bp->bif_ifp == ifp) 2728 n1++; 2729 } 2730 if (bfl->bfl_list == NULL) { 2731 bfl->bfl_len = n1; 2732 return (0); 2733 } 2734 if (n1 > bfl->bfl_len) 2735 return (ENOMEM); 2736 BPF_UNLOCK(); 2737 lst = malloc(n1 * sizeof(u_int), M_TEMP, M_WAITOK); 2738 n = 0; 2739 BPF_LOCK(); 2740 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2741 if (bp->bif_ifp != ifp) 2742 continue; 2743 if (n >= n1) { 2744 free(lst, M_TEMP); 2745 goto again; 2746 } 2747 lst[n] = bp->bif_dlt; 2748 n++; 2749 } 2750 BPF_UNLOCK(); 2751 error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n); 2752 free(lst, M_TEMP); 2753 BPF_LOCK(); 2754 bfl->bfl_len = n; 2755 return (error); 2756 } 2757 2758 /* 2759 * Set the data link type of a BPF instance. 2760 */ 2761 static int 2762 bpf_setdlt(struct bpf_d *d, u_int dlt) 2763 { 2764 int error, opromisc; 2765 struct ifnet *ifp; 2766 struct bpf_if *bp; 2767 2768 BPF_LOCK_ASSERT(); 2769 2770 if (d->bd_bif->bif_dlt == dlt) 2771 return (0); 2772 ifp = d->bd_bif->bif_ifp; 2773 2774 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2775 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2776 break; 2777 } 2778 2779 if (bp != NULL) { 2780 opromisc = d->bd_promisc; 2781 bpf_attachd(d, bp); 2782 BPFD_LOCK(d); 2783 reset_d(d); 2784 BPFD_UNLOCK(d); 2785 if (opromisc) { 2786 error = ifpromisc(bp->bif_ifp, 1); 2787 if (error) 2788 if_printf(bp->bif_ifp, 2789 "bpf_setdlt: ifpromisc failed (%d)\n", 2790 error); 2791 else 2792 d->bd_promisc = 1; 2793 } 2794 } 2795 return (bp == NULL ? EINVAL : 0); 2796 } 2797 2798 static void 2799 bpf_drvinit(void *unused) 2800 { 2801 struct cdev *dev; 2802 2803 mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF); 2804 LIST_INIT(&bpf_iflist); 2805 LIST_INIT(&bpf_freelist); 2806 2807 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); 2808 /* For compatibility */ 2809 make_dev_alias(dev, "bpf0"); 2810 2811 /* Register interface departure handler */ 2812 bpf_ifdetach_cookie = EVENTHANDLER_REGISTER( 2813 ifnet_departure_event, bpf_ifdetach, NULL, 2814 EVENTHANDLER_PRI_ANY); 2815 } 2816 2817 /* 2818 * Zero out the various packet counters associated with all of the bpf 2819 * descriptors. At some point, we will probably want to get a bit more 2820 * granular and allow the user to specify descriptors to be zeroed. 2821 */ 2822 static void 2823 bpf_zero_counters(void) 2824 { 2825 struct bpf_if *bp; 2826 struct bpf_d *bd; 2827 2828 BPF_LOCK(); 2829 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2830 BPFIF_RLOCK(bp); 2831 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2832 BPFD_LOCK(bd); 2833 bd->bd_rcount = 0; 2834 bd->bd_dcount = 0; 2835 bd->bd_fcount = 0; 2836 bd->bd_wcount = 0; 2837 bd->bd_wfcount = 0; 2838 bd->bd_zcopy = 0; 2839 BPFD_UNLOCK(bd); 2840 } 2841 BPFIF_RUNLOCK(bp); 2842 } 2843 BPF_UNLOCK(); 2844 } 2845 2846 /* 2847 * Fill filter statistics 2848 */ 2849 static void 2850 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) 2851 { 2852 2853 bzero(d, sizeof(*d)); 2854 BPFD_LOCK_ASSERT(bd); 2855 d->bd_structsize = sizeof(*d); 2856 /* XXX: reading should be protected by global lock */ 2857 d->bd_immediate = bd->bd_immediate; 2858 d->bd_promisc = bd->bd_promisc; 2859 d->bd_hdrcmplt = bd->bd_hdrcmplt; 2860 d->bd_direction = bd->bd_direction; 2861 d->bd_feedback = bd->bd_feedback; 2862 d->bd_async = bd->bd_async; 2863 d->bd_rcount = bd->bd_rcount; 2864 d->bd_dcount = bd->bd_dcount; 2865 d->bd_fcount = bd->bd_fcount; 2866 d->bd_sig = bd->bd_sig; 2867 d->bd_slen = bd->bd_slen; 2868 d->bd_hlen = bd->bd_hlen; 2869 d->bd_bufsize = bd->bd_bufsize; 2870 d->bd_pid = bd->bd_pid; 2871 strlcpy(d->bd_ifname, 2872 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); 2873 d->bd_locked = bd->bd_locked; 2874 d->bd_wcount = bd->bd_wcount; 2875 d->bd_wdcount = bd->bd_wdcount; 2876 d->bd_wfcount = bd->bd_wfcount; 2877 d->bd_zcopy = bd->bd_zcopy; 2878 d->bd_bufmode = bd->bd_bufmode; 2879 } 2880 2881 /* 2882 * Handle `netstat -B' stats request 2883 */ 2884 static int 2885 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) 2886 { 2887 static const struct xbpf_d zerostats; 2888 struct xbpf_d *xbdbuf, *xbd, tempstats; 2889 int index, error; 2890 struct bpf_if *bp; 2891 struct bpf_d *bd; 2892 2893 /* 2894 * XXX This is not technically correct. It is possible for non 2895 * privileged users to open bpf devices. It would make sense 2896 * if the users who opened the devices were able to retrieve 2897 * the statistics for them, too. 2898 */ 2899 error = priv_check(req->td, PRIV_NET_BPF); 2900 if (error) 2901 return (error); 2902 /* 2903 * Check to see if the user is requesting that the counters be 2904 * zeroed out. Explicitly check that the supplied data is zeroed, 2905 * as we aren't allowing the user to set the counters currently. 2906 */ 2907 if (req->newptr != NULL) { 2908 if (req->newlen != sizeof(tempstats)) 2909 return (EINVAL); 2910 memset(&tempstats, 0, sizeof(tempstats)); 2911 error = SYSCTL_IN(req, &tempstats, sizeof(tempstats)); 2912 if (error) 2913 return (error); 2914 if (bcmp(&tempstats, &zerostats, sizeof(tempstats)) != 0) 2915 return (EINVAL); 2916 bpf_zero_counters(); 2917 return (0); 2918 } 2919 if (req->oldptr == NULL) 2920 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd))); 2921 if (bpf_bpfd_cnt == 0) 2922 return (SYSCTL_OUT(req, 0, 0)); 2923 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); 2924 BPF_LOCK(); 2925 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { 2926 BPF_UNLOCK(); 2927 free(xbdbuf, M_BPF); 2928 return (ENOMEM); 2929 } 2930 index = 0; 2931 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2932 BPFIF_RLOCK(bp); 2933 /* Send writers-only first */ 2934 LIST_FOREACH(bd, &bp->bif_wlist, bd_next) { 2935 xbd = &xbdbuf[index++]; 2936 BPFD_LOCK(bd); 2937 bpfstats_fill_xbpf(xbd, bd); 2938 BPFD_UNLOCK(bd); 2939 } 2940 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2941 xbd = &xbdbuf[index++]; 2942 BPFD_LOCK(bd); 2943 bpfstats_fill_xbpf(xbd, bd); 2944 BPFD_UNLOCK(bd); 2945 } 2946 BPFIF_RUNLOCK(bp); 2947 } 2948 BPF_UNLOCK(); 2949 error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); 2950 free(xbdbuf, M_BPF); 2951 return (error); 2952 } 2953 2954 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); 2955 2956 #else /* !DEV_BPF && !NETGRAPH_BPF */ 2957 /* 2958 * NOP stubs to allow bpf-using drivers to load and function. 2959 * 2960 * A 'better' implementation would allow the core bpf functionality 2961 * to be loaded at runtime. 2962 */ 2963 static struct bpf_if bp_null; 2964 2965 void 2966 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 2967 { 2968 } 2969 2970 void 2971 bpf_mtap(struct bpf_if *bp, struct mbuf *m) 2972 { 2973 } 2974 2975 void 2976 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) 2977 { 2978 } 2979 2980 void 2981 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2982 { 2983 2984 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2985 } 2986 2987 void 2988 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2989 { 2990 2991 *driverp = &bp_null; 2992 } 2993 2994 void 2995 bpfdetach(struct ifnet *ifp) 2996 { 2997 } 2998 2999 u_int 3000 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) 3001 { 3002 return -1; /* "no filter" behaviour */ 3003 } 3004 3005 int 3006 bpf_validate(const struct bpf_insn *f, int len) 3007 { 3008 return 0; /* false */ 3009 } 3010 3011 #endif /* !DEV_BPF && !NETGRAPH_BPF */ 3012 3013 #ifdef DDB 3014 static void 3015 bpf_show_bpf_if(struct bpf_if *bpf_if) 3016 { 3017 3018 if (bpf_if == NULL) 3019 return; 3020 db_printf("%p:\n", bpf_if); 3021 #define BPF_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, bpf_if->e); 3022 /* bif_ext.bif_next */ 3023 /* bif_ext.bif_dlist */ 3024 BPF_DB_PRINTF("%#x", bif_dlt); 3025 BPF_DB_PRINTF("%u", bif_hdrlen); 3026 BPF_DB_PRINTF("%p", bif_ifp); 3027 /* bif_lock */ 3028 /* bif_wlist */ 3029 BPF_DB_PRINTF("%#x", bif_flags); 3030 } 3031 3032 DB_SHOW_COMMAND(bpf_if, db_show_bpf_if) 3033 { 3034 3035 if (!have_addr) { 3036 db_printf("usage: show bpf_if <struct bpf_if *>\n"); 3037 return; 3038 } 3039 3040 bpf_show_bpf_if((struct bpf_if *)addr); 3041 } 3042 #endif 3043