Lines Matching +full:d +full:- +full:- +full:- +full:- +full:- +full:0
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
111 struct bpfd_list bif_wlist; /* writer-only list */
118 CTASSERT(offsetof(struct bpf_if, bif_ext) == 0);
125 void *buffer[0];
134 (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
144 * 32-bit version of structure prepended to each packet. We use this header
145 * instead of the standard one for 32-bit streams. We mark the a stream as
146 * 32-bit the first time we see a 32-bit compat ioctl request.
217 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
221 &bpf_maxinsns, 0, "Maximum bpf program instructions");
222 static int bpf_zerocopy_enable = 0;
224 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
228 VNET_DEFINE_STATIC(int, bpf_optimize_writers) = 0;
231 &VNET_NAME(bpf_optimize_writers), 0,
307 if_rele(bp->bif_ifp); in bpfif_free()
315 refcount_acquire(&bp->bif_refcnt); in bpfif_ref()
322 if (!refcount_release(&bp->bif_refcnt)) in bpfif_rele()
324 NET_EPOCH_CALL(bpfif_free, &bp->epoch_ctx); in bpfif_rele()
328 bpfd_ref(struct bpf_d *d) in bpfd_ref() argument
331 refcount_acquire(&d->bd_refcnt); in bpfd_ref()
335 bpfd_rele(struct bpf_d *d) in bpfd_rele() argument
338 if (!refcount_release(&d->bd_refcnt)) in bpfd_rele()
340 NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx); in bpfd_rele()
358 if (ptr->func != NULL) in bpf_program_buffer_free()
359 bpf_destroy_jit_filter(ptr->func); in bpf_program_buffer_free()
370 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_bytes() argument
374 BPFD_LOCK_ASSERT(d); in bpf_append_bytes()
376 switch (d->bd_bufmode) { in bpf_append_bytes()
378 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
381 counter_u64_add(d->bd_zcopy, 1); in bpf_append_bytes()
382 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
390 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_mbuf() argument
394 BPFD_LOCK_ASSERT(d); in bpf_append_mbuf()
396 switch (d->bd_bufmode) { in bpf_append_mbuf()
398 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
401 counter_u64_add(d->bd_zcopy, 1); in bpf_append_mbuf()
402 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
410 * This function gets called when the free buffer is re-assigned.
413 bpf_buf_reclaimed(struct bpf_d *d) in bpf_buf_reclaimed() argument
416 BPFD_LOCK_ASSERT(d); in bpf_buf_reclaimed()
418 switch (d->bd_bufmode) { in bpf_buf_reclaimed()
423 bpf_zerocopy_buf_reclaimed(d); in bpf_buf_reclaimed()
434 * returned if the buffer can be discarded, (0) is returned if it cannot.
437 bpf_canfreebuf(struct bpf_d *d) in bpf_canfreebuf() argument
440 BPFD_LOCK_ASSERT(d); in bpf_canfreebuf()
442 switch (d->bd_bufmode) { in bpf_canfreebuf()
444 return (bpf_zerocopy_canfreebuf(d)); in bpf_canfreebuf()
446 return (0); in bpf_canfreebuf()
452 * buffer is writable, and (0) if not.
455 bpf_canwritebuf(struct bpf_d *d) in bpf_canwritebuf() argument
457 BPFD_LOCK_ASSERT(d); in bpf_canwritebuf()
459 switch (d->bd_bufmode) { in bpf_canwritebuf()
461 return (bpf_zerocopy_canwritebuf(d)); in bpf_canwritebuf()
472 bpf_buffull(struct bpf_d *d) in bpf_buffull() argument
475 BPFD_LOCK_ASSERT(d); in bpf_buffull()
477 switch (d->bd_bufmode) { in bpf_buffull()
479 bpf_zerocopy_buffull(d); in bpf_buffull()
488 bpf_bufheld(struct bpf_d *d) in bpf_bufheld() argument
491 BPFD_LOCK_ASSERT(d); in bpf_bufheld()
493 switch (d->bd_bufmode) { in bpf_bufheld()
495 bpf_zerocopy_bufheld(d); in bpf_bufheld()
501 bpf_free(struct bpf_d *d) in bpf_free() argument
504 switch (d->bd_bufmode) { in bpf_free()
506 return (bpf_buffer_free(d)); in bpf_free()
509 return (bpf_zerocopy_free(d)); in bpf_free()
517 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) in bpf_uiomove() argument
520 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_uiomove()
522 return (bpf_buffer_uiomove(d, buf, len, uio)); in bpf_uiomove()
526 bpf_ioctl_sblen(struct bpf_d *d, u_int *i) in bpf_ioctl_sblen() argument
529 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_ioctl_sblen()
531 return (bpf_buffer_ioctl_sblen(d, i)); in bpf_ioctl_sblen()
535 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) in bpf_ioctl_getzmax() argument
538 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_getzmax()
540 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); in bpf_ioctl_getzmax()
544 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_rotzbuf() argument
547 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_rotzbuf()
549 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); in bpf_ioctl_rotzbuf()
553 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_setzbuf() argument
556 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_setzbuf()
558 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); in bpf_ioctl_setzbuf()
566 struct sockaddr *sockp, int *hdrlen, struct bpf_d *d) in bpf_movein() argument
587 sockp->sa_family = AF_INET; in bpf_movein()
588 hlen = 0; in bpf_movein()
592 sockp->sa_family = AF_UNSPEC; in bpf_movein()
598 sockp->sa_family = AF_IMPLINK; in bpf_movein()
599 hlen = 0; in bpf_movein()
603 sockp->sa_family = AF_UNSPEC; in bpf_movein()
604 hlen = 0; in bpf_movein()
612 sockp->sa_family = AF_UNSPEC; in bpf_movein()
618 * en atm driver requires 4-byte atm pseudo header. in bpf_movein()
622 sockp->sa_family = AF_UNSPEC; in bpf_movein()
627 sockp->sa_family = AF_UNSPEC; in bpf_movein()
632 sockp->sa_family = AF_IEEE80211; in bpf_movein()
633 hlen = 0; in bpf_movein()
637 sockp->sa_family = AF_IEEE80211; in bpf_movein()
638 sockp->sa_len = 12; /* XXX != 0 */ in bpf_movein()
646 len = uio->uio_resid; in bpf_movein()
647 if (len < hlen || len - hlen > ifp->if_mtu) in bpf_movein()
654 m->m_pkthdr.len = m->m_len = len; in bpf_movein()
661 slen = bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len); in bpf_movein()
662 if (slen == 0) { in bpf_movein()
671 if (ETHER_IS_MULTICAST(eh->ether_dhost)) { in bpf_movein()
672 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost, in bpf_movein()
673 ETHER_ADDR_LEN) == 0) in bpf_movein()
674 m->m_flags |= M_BCAST; in bpf_movein()
676 m->m_flags |= M_MCAST; in bpf_movein()
678 if (d->bd_hdrcmplt == 0) { in bpf_movein()
679 memcpy(eh->ether_shost, IF_LLADDR(ifp), in bpf_movein()
680 sizeof(eh->ether_shost)); in bpf_movein()
688 if (hlen != 0) { in bpf_movein()
689 if (sockp->sa_family == AF_IEEE80211) { in bpf_movein()
692 * NB: sockp is known to be zero'd so if we do a in bpf_movein()
700 hlen = p->ibp_len; in bpf_movein()
701 if (hlen > sizeof(sockp->sa_data)) { in bpf_movein()
706 bcopy(mtod(m, const void *), sockp->sa_data, hlen); in bpf_movein()
710 return (0); in bpf_movein()
717 * Attach descriptor to the bpf interface, i.e. make d listen on bp,
721 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) in bpf_attachd() argument
731 op_w = V_bpf_optimize_writers || d->bd_writer; in bpf_attachd()
733 if (d->bd_bif != NULL) in bpf_attachd()
734 bpf_detachd_locked(d, false); in bpf_attachd()
736 * Point d at bp, and add d to the interface's list. in bpf_attachd()
739 * we can delay adding d to the list of active listeners until in bpf_attachd()
743 BPFD_LOCK(d); in bpf_attachd()
748 d->bd_bif = bp; in bpf_attachd()
749 if (op_w != 0) { in bpf_attachd()
750 /* Add to writers-only list */ in bpf_attachd()
751 CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); in bpf_attachd()
758 d->bd_writer = 2; in bpf_attachd()
760 CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); in bpf_attachd()
762 reset_d(d); in bpf_attachd()
765 bpf_wakeup(d); in bpf_attachd()
767 BPFD_UNLOCK(d); in bpf_attachd()
770 CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list", in bpf_attachd()
771 __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); in bpf_attachd()
773 if (op_w == 0) in bpf_attachd()
774 EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); in bpf_attachd()
778 * Check if we need to upgrade our descriptor @d from write-only mode.
781 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, in bpf_check_upgrade() argument
789 if (d->bd_writer == 0 || fcode == NULL) in bpf_check_upgrade()
790 return (0); in bpf_check_upgrade()
792 need_upgrade = 0; in bpf_check_upgrade()
798 * while pcap_open_live() definitely sets to non-zero value, in bpf_check_upgrade()
799 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g. in bpf_check_upgrade()
803 fcode[0].code == (BPF_RET | BPF_K)) in bpf_check_upgrade()
806 is_snap = 0; in bpf_check_upgrade()
808 if (is_snap == 0) { in bpf_check_upgrade()
821 if (--d->bd_writer == 0) { in bpf_check_upgrade()
824 * been set. This is probably catch-all in bpf_check_upgrade()
832 "%s: filter function set by pid %d, " in bpf_check_upgrade()
833 "bd_writer counter %d, snap %d upgrade %d", in bpf_check_upgrade()
834 __func__, d->bd_pid, d->bd_writer, in bpf_check_upgrade()
844 bpf_detachd(struct bpf_d *d) in bpf_detachd() argument
847 bpf_detachd_locked(d, false); in bpf_detachd()
852 bpf_detachd_locked(struct bpf_d *d, bool detached_ifp) in bpf_detachd_locked() argument
859 CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); in bpf_detachd_locked()
862 if ((bp = d->bd_bif) == NULL) in bpf_detachd_locked()
865 BPFD_LOCK(d); in bpf_detachd_locked()
866 /* Remove d from the interface's descriptor list. */ in bpf_detachd_locked()
867 CK_LIST_REMOVE(d, bd_next); in bpf_detachd_locked()
869 error = d->bd_writer; in bpf_detachd_locked()
870 ifp = bp->bif_ifp; in bpf_detachd_locked()
871 d->bd_bif = NULL; in bpf_detachd_locked()
877 bpf_wakeup(d); in bpf_detachd_locked()
879 BPFD_UNLOCK(d); in bpf_detachd_locked()
880 bpf_bpfd_cnt--; in bpf_detachd_locked()
882 /* Call event handler iff d is attached */ in bpf_detachd_locked()
883 if (error == 0) in bpf_detachd_locked()
884 EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0); in bpf_detachd_locked()
890 if (d->bd_promisc && !detached_ifp) { in bpf_detachd_locked()
891 d->bd_promisc = 0; in bpf_detachd_locked()
892 CURVNET_SET(ifp->if_vnet); in bpf_detachd_locked()
893 error = ifpromisc(ifp, 0); in bpf_detachd_locked()
895 if (error != 0 && error != ENXIO) { in bpf_detachd_locked()
902 if_printf(bp->bif_ifp, in bpf_detachd_locked()
903 "bpf_detach: ifpromisc failed (%d)\n", error); in bpf_detachd_locked()
916 struct bpf_d *d = data; in bpf_dtor() local
918 BPFD_LOCK(d); in bpf_dtor()
919 if (d->bd_state == BPF_WAITING) in bpf_dtor()
920 callout_stop(&d->bd_callout); in bpf_dtor()
921 d->bd_state = BPF_IDLE; in bpf_dtor()
922 BPFD_UNLOCK(d); in bpf_dtor()
923 funsetown(&d->bd_sigio); in bpf_dtor()
924 bpf_detachd(d); in bpf_dtor()
926 mac_bpfdesc_destroy(d); in bpf_dtor()
928 seldrain(&d->bd_sel); in bpf_dtor()
929 knlist_destroy(&d->bd_sel.si_note); in bpf_dtor()
930 callout_drain(&d->bd_callout); in bpf_dtor()
931 bpfd_rele(d); in bpf_dtor()
942 struct bpf_d *d; in bpfopen() local
945 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); in bpfopen()
946 error = devfs_set_cdevpriv(d, bpf_dtor); in bpfopen()
947 if (error != 0) { in bpfopen()
948 free(d, M_BPF); in bpfopen()
953 d->bd_rcount = counter_u64_alloc(M_WAITOK); in bpfopen()
954 d->bd_dcount = counter_u64_alloc(M_WAITOK); in bpfopen()
955 d->bd_fcount = counter_u64_alloc(M_WAITOK); in bpfopen()
956 d->bd_wcount = counter_u64_alloc(M_WAITOK); in bpfopen()
957 d->bd_wfcount = counter_u64_alloc(M_WAITOK); in bpfopen()
958 d->bd_wdcount = counter_u64_alloc(M_WAITOK); in bpfopen()
959 d->bd_zcopy = counter_u64_alloc(M_WAITOK); in bpfopen()
962 * For historical reasons, perform a one-time initialization call to in bpfopen()
966 bpf_buffer_init(d); in bpfopen()
967 if ((flags & FREAD) == 0) in bpfopen()
968 d->bd_writer = 2; in bpfopen()
969 d->bd_hbuf_in_use = 0; in bpfopen()
970 d->bd_bufmode = BPF_BUFMODE_BUFFER; in bpfopen()
971 d->bd_sig = SIGIO; in bpfopen()
972 d->bd_direction = BPF_D_INOUT; in bpfopen()
973 refcount_init(&d->bd_refcnt, 1); in bpfopen()
974 BPF_PID_REFRESH(d, td); in bpfopen()
976 mac_bpfdesc_init(d); in bpfopen()
977 mac_bpfdesc_create(td->td_ucred, d); in bpfopen()
979 mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); in bpfopen()
980 callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); in bpfopen()
981 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); in bpfopen()
984 d->bd_pcp = 0; in bpfopen()
986 return (0); in bpfopen()
990 * bpfread - read next chunk of packets from buffers
995 struct bpf_d *d; in bpfread() local
1000 error = devfs_get_cdevpriv((void **)&d); in bpfread()
1001 if (error != 0) in bpfread()
1008 if (uio->uio_resid != d->bd_bufsize) in bpfread()
1011 non_block = ((ioflag & O_NONBLOCK) != 0); in bpfread()
1013 BPFD_LOCK(d); in bpfread()
1014 BPF_PID_REFRESH_CUR(d); in bpfread()
1015 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { in bpfread()
1016 BPFD_UNLOCK(d); in bpfread()
1019 if (d->bd_state == BPF_WAITING) in bpfread()
1020 callout_stop(&d->bd_callout); in bpfread()
1021 timed_out = (d->bd_state == BPF_TIMED_OUT); in bpfread()
1022 d->bd_state = BPF_IDLE; in bpfread()
1023 while (d->bd_hbuf_in_use) { in bpfread()
1024 error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, in bpfread()
1025 PRINET | PCATCH, "bd_hbuf", 0); in bpfread()
1026 if (error != 0) { in bpfread()
1027 BPFD_UNLOCK(d); in bpfread()
1036 while (d->bd_hbuf == NULL) { in bpfread()
1037 if (d->bd_slen != 0) { in bpfread()
1042 if (d->bd_immediate || non_block || timed_out) { in bpfread()
1045 * if we are in immediate mode, non-blocking in bpfread()
1048 ROTATE_BUFFERS(d); in bpfread()
1059 if (d->bd_bif == NULL) { in bpfread()
1060 BPFD_UNLOCK(d); in bpfread()
1065 BPFD_UNLOCK(d); in bpfread()
1068 error = msleep(d, &d->bd_lock, PRINET | PCATCH, in bpfread()
1069 "bpf", d->bd_rtout); in bpfread()
1071 BPFD_UNLOCK(d); in bpfread()
1080 if (d->bd_hbuf) in bpfread()
1088 if (d->bd_slen == 0) { in bpfread()
1089 BPFD_UNLOCK(d); in bpfread()
1090 return (0); in bpfread()
1092 ROTATE_BUFFERS(d); in bpfread()
1099 d->bd_hbuf_in_use = 1; in bpfread()
1100 BPFD_UNLOCK(d); in bpfread()
1110 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); in bpfread()
1112 BPFD_LOCK(d); in bpfread()
1113 KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf")); in bpfread()
1114 d->bd_fbuf = d->bd_hbuf; in bpfread()
1115 d->bd_hbuf = NULL; in bpfread()
1116 d->bd_hlen = 0; in bpfread()
1117 bpf_buf_reclaimed(d); in bpfread()
1118 d->bd_hbuf_in_use = 0; in bpfread()
1119 wakeup(&d->bd_hbuf_in_use); in bpfread()
1120 BPFD_UNLOCK(d); in bpfread()
1129 bpf_wakeup(struct bpf_d *d) in bpf_wakeup() argument
1132 BPFD_LOCK_ASSERT(d); in bpf_wakeup()
1133 if (d->bd_state == BPF_WAITING) { in bpf_wakeup()
1134 callout_stop(&d->bd_callout); in bpf_wakeup()
1135 d->bd_state = BPF_IDLE; in bpf_wakeup()
1137 wakeup(d); in bpf_wakeup()
1138 if (d->bd_async && d->bd_sig && d->bd_sigio) in bpf_wakeup()
1139 pgsigio(&d->bd_sigio, d->bd_sig, 0); in bpf_wakeup()
1141 selwakeuppri(&d->bd_sel, PRINET); in bpf_wakeup()
1142 KNOTE_LOCKED(&d->bd_sel.si_note, 0); in bpf_wakeup()
1148 struct bpf_d *d = (struct bpf_d *)arg; in bpf_timed_out() local
1150 BPFD_LOCK_ASSERT(d); in bpf_timed_out()
1152 if (callout_pending(&d->bd_callout) || in bpf_timed_out()
1153 !callout_active(&d->bd_callout)) in bpf_timed_out()
1155 if (d->bd_state == BPF_WAITING) { in bpf_timed_out()
1156 d->bd_state = BPF_TIMED_OUT; in bpf_timed_out()
1157 if (d->bd_slen != 0) in bpf_timed_out()
1158 bpf_wakeup(d); in bpf_timed_out()
1163 bpf_ready(struct bpf_d *d) in bpf_ready() argument
1166 BPFD_LOCK_ASSERT(d); in bpf_ready()
1168 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) in bpf_ready()
1170 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && in bpf_ready()
1171 d->bd_slen != 0) in bpf_ready()
1173 return (0); in bpf_ready()
1183 struct bpf_d *d; in bpfwrite() local
1188 error = devfs_get_cdevpriv((void **)&d); in bpfwrite()
1189 if (error != 0) in bpfwrite()
1193 BPFD_LOCK(d); in bpfwrite()
1194 BPF_PID_REFRESH_CUR(d); in bpfwrite()
1195 counter_u64_add(d->bd_wcount, 1); in bpfwrite()
1196 if ((bp = d->bd_bif) == NULL) { in bpfwrite()
1201 ifp = bp->bif_ifp; in bpfwrite()
1202 if ((ifp->if_flags & IFF_UP) == 0) { in bpfwrite()
1207 if (uio->uio_resid == 0) in bpfwrite()
1212 hlen = 0; in bpfwrite()
1215 * Take extra reference, unlock d and exit from epoch section, in bpfwrite()
1218 bpfd_ref(d); in bpfwrite()
1220 BPFD_UNLOCK(d); in bpfwrite()
1222 error = bpf_movein(uio, (int)bp->bif_dlt, ifp, in bpfwrite()
1223 &m, &dst, &hlen, d); in bpfwrite()
1225 if (error != 0) { in bpfwrite()
1226 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1227 bpfd_rele(d); in bpfwrite()
1231 BPFD_LOCK(d); in bpfwrite()
1237 if (d->bd_bif == NULL) { in bpfwrite()
1238 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1239 BPFD_UNLOCK(d); in bpfwrite()
1240 bpfd_rele(d); in bpfwrite()
1244 counter_u64_add(d->bd_wfcount, 1); in bpfwrite()
1245 if (d->bd_hdrcmplt) in bpfwrite()
1248 if (d->bd_feedback) { in bpfwrite()
1251 mc->m_pkthdr.rcvif = ifp; in bpfwrite()
1253 if (d->bd_direction == BPF_D_INOUT) in bpfwrite()
1254 m->m_flags |= M_PROMISC; in bpfwrite()
1258 m->m_pkthdr.len -= hlen; in bpfwrite()
1259 m->m_len -= hlen; in bpfwrite()
1260 m->m_data += hlen; /* XXX */ in bpfwrite()
1262 CURVNET_SET(ifp->if_vnet); in bpfwrite()
1264 mac_bpfdesc_create_mbuf(d, m); in bpfwrite()
1266 mac_bpfdesc_create_mbuf(d, mc); in bpfwrite()
1270 if (hlen != 0) { in bpfwrite()
1276 if (d->bd_pcp != 0) in bpfwrite()
1277 vlan_set_pcp(m, d->bd_pcp); in bpfwrite()
1281 BPFD_UNLOCK(d); in bpfwrite()
1282 error = (*ifp->if_output)(ifp, m, &dst, &ro); in bpfwrite()
1284 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1287 if (error == 0) in bpfwrite()
1288 (*ifp->if_input)(ifp, mc); in bpfwrite()
1294 bpfd_rele(d); in bpfwrite()
1298 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1300 BPFD_UNLOCK(d); in bpfwrite()
1306 * and drop counts. This is doable for kernel-only buffers, but with
1307 * zero-copy buffers, we can't write to (or rotate) buffers that are
1312 reset_d(struct bpf_d *d) in reset_d() argument
1315 BPFD_LOCK_ASSERT(d); in reset_d()
1317 while (d->bd_hbuf_in_use) in reset_d()
1318 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET, in reset_d()
1319 "bd_hbuf", 0); in reset_d()
1320 if ((d->bd_hbuf != NULL) && in reset_d()
1321 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { in reset_d()
1323 d->bd_fbuf = d->bd_hbuf; in reset_d()
1324 d->bd_hbuf = NULL; in reset_d()
1325 d->bd_hlen = 0; in reset_d()
1326 bpf_buf_reclaimed(d); in reset_d()
1328 if (bpf_canwritebuf(d)) in reset_d()
1329 d->bd_slen = 0; in reset_d()
1330 counter_u64_zero(d->bd_rcount); in reset_d()
1331 counter_u64_zero(d->bd_dcount); in reset_d()
1332 counter_u64_zero(d->bd_fcount); in reset_d()
1333 counter_u64_zero(d->bd_wcount); in reset_d()
1334 counter_u64_zero(d->bd_wfcount); in reset_d()
1335 counter_u64_zero(d->bd_wdcount); in reset_d()
1336 counter_u64_zero(d->bd_zcopy); in reset_d()
1363 * BIOCSETZBUF Set current zero-copy buffer locations.
1364 * BIOCGETZMAX Get maximum zero-copy buffer size.
1365 * BIOCROTZBUF Force rotation of zero-copy buffer
1375 struct bpf_d *d; in bpfioctl() local
1378 error = devfs_get_cdevpriv((void **)&d); in bpfioctl()
1379 if (error != 0) in bpfioctl()
1385 BPFD_LOCK(d); in bpfioctl()
1386 BPF_PID_REFRESH(d, td); in bpfioctl()
1387 if (d->bd_state == BPF_WAITING) in bpfioctl()
1388 callout_stop(&d->bd_callout); in bpfioctl()
1389 d->bd_state = BPF_IDLE; in bpfioctl()
1390 BPFD_UNLOCK(d); in bpfioctl()
1392 if (d->bd_locked == 1) { in bpfioctl()
1428 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so in bpfioctl()
1429 * that it will get 32-bit packet headers. in bpfioctl()
1438 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { in bpfioctl()
1439 BPFD_LOCK(d); in bpfioctl()
1440 d->bd_compat32 = 1; in bpfioctl()
1441 BPFD_UNLOCK(d); in bpfioctl()
1459 BPFD_LOCK(d); in bpfioctl()
1460 n = d->bd_slen; in bpfioctl()
1461 while (d->bd_hbuf_in_use) in bpfioctl()
1462 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, in bpfioctl()
1463 PRINET, "bd_hbuf", 0); in bpfioctl()
1464 if (d->bd_hbuf) in bpfioctl()
1465 n += d->bd_hlen; in bpfioctl()
1466 BPFD_UNLOCK(d); in bpfioctl()
1476 BPFD_LOCK(d); in bpfioctl()
1477 *(u_int *)addr = d->bd_bufsize; in bpfioctl()
1478 BPFD_UNLOCK(d); in bpfioctl()
1485 error = bpf_ioctl_sblen(d, (u_int *)addr); in bpfioctl()
1499 error = bpf_setf(d, (struct bpf_program *)addr, cmd); in bpfioctl()
1506 BPFD_LOCK(d); in bpfioctl()
1507 reset_d(d); in bpfioctl()
1508 BPFD_UNLOCK(d); in bpfioctl()
1516 if (d->bd_bif == NULL) { in bpfioctl()
1521 } else if (d->bd_promisc == 0) { in bpfioctl()
1522 error = ifpromisc(d->bd_bif->bif_ifp, 1); in bpfioctl()
1523 if (error == 0) in bpfioctl()
1524 d->bd_promisc = 1; in bpfioctl()
1534 if (d->bd_bif == NULL) in bpfioctl()
1537 *(u_int *)addr = d->bd_bif->bif_dlt; in bpfioctl()
1551 dltlist.bfl_len = list32->bfl_len; in bpfioctl()
1552 dltlist.bfl_list = PTRIN(list32->bfl_list); in bpfioctl()
1554 if (d->bd_bif == NULL) in bpfioctl()
1557 error = bpf_getdltlist(d, &dltlist); in bpfioctl()
1558 if (error == 0) in bpfioctl()
1559 list32->bfl_len = dltlist.bfl_len; in bpfioctl()
1568 if (d->bd_bif == NULL) in bpfioctl()
1571 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); in bpfioctl()
1580 if (d->bd_bif == NULL) in bpfioctl()
1583 error = bpf_setdlt(d, *(u_int *)addr); in bpfioctl()
1592 if (d->bd_bif == NULL) in bpfioctl()
1595 struct ifnet *const ifp = d->bd_bif->bif_ifp; in bpfioctl()
1598 strlcpy(ifr->ifr_name, ifp->if_xname, in bpfioctl()
1599 sizeof(ifr->ifr_name)); in bpfioctl()
1614 * allocate them here. If we're using zero-copy, in bpfioctl()
1618 alloc_buf = 0; in bpfioctl()
1619 BPFD_LOCK(d); in bpfioctl()
1620 if (d->bd_bufmode == BPF_BUFMODE_BUFFER && in bpfioctl()
1621 d->bd_sbuf == NULL) in bpfioctl()
1623 BPFD_UNLOCK(d); in bpfioctl()
1625 size = d->bd_bufsize; in bpfioctl()
1626 error = bpf_buffer_ioctl_sblen(d, &size); in bpfioctl()
1627 if (error != 0) in bpfioctl()
1631 error = bpf_setif(d, (struct ifreq *)addr); in bpfioctl()
1652 tv->tv_sec = tv32->tv_sec; in bpfioctl()
1653 tv->tv_usec = tv32->tv_usec; in bpfioctl()
1660 * a one-shot timer. in bpfioctl()
1662 if ((error = itimerfix(tv)) == 0) in bpfioctl()
1663 d->bd_rtout = tvtohz(tv) - 1; in bpfioctl()
1686 tv->tv_sec = d->bd_rtout / hz; in bpfioctl()
1687 tv->tv_usec = (d->bd_rtout % hz) * tick; in bpfioctl()
1691 tv32->tv_sec = tv->tv_sec; in bpfioctl()
1692 tv32->tv_usec = tv->tv_usec; in bpfioctl()
1707 bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount); in bpfioctl()
1708 bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount); in bpfioctl()
1716 BPFD_LOCK(d); in bpfioctl()
1717 d->bd_immediate = *(u_int *)addr; in bpfioctl()
1718 BPFD_UNLOCK(d); in bpfioctl()
1725 bv->bv_major = BPF_MAJOR_VERSION; in bpfioctl()
1726 bv->bv_minor = BPF_MINOR_VERSION; in bpfioctl()
1734 BPFD_LOCK(d); in bpfioctl()
1735 *(u_int *)addr = d->bd_hdrcmplt; in bpfioctl()
1736 BPFD_UNLOCK(d); in bpfioctl()
1743 BPFD_LOCK(d); in bpfioctl()
1744 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; in bpfioctl()
1745 BPFD_UNLOCK(d); in bpfioctl()
1752 BPFD_LOCK(d); in bpfioctl()
1753 *(u_int *)addr = d->bd_direction; in bpfioctl()
1754 BPFD_UNLOCK(d); in bpfioctl()
1769 BPFD_LOCK(d); in bpfioctl()
1770 d->bd_direction = direction; in bpfioctl()
1771 BPFD_UNLOCK(d); in bpfioctl()
1783 BPFD_LOCK(d); in bpfioctl()
1784 *(u_int *)addr = d->bd_tstamp; in bpfioctl()
1785 BPFD_UNLOCK(d); in bpfioctl()
1797 d->bd_tstamp = func; in bpfioctl()
1804 BPFD_LOCK(d); in bpfioctl()
1805 d->bd_feedback = *(u_int *)addr; in bpfioctl()
1806 BPFD_UNLOCK(d); in bpfioctl()
1810 BPFD_LOCK(d); in bpfioctl()
1811 d->bd_locked = 1; in bpfioctl()
1812 BPFD_UNLOCK(d); in bpfioctl()
1815 case FIONBIO: /* Non-blocking I/O */ in bpfioctl()
1819 BPFD_LOCK(d); in bpfioctl()
1820 d->bd_async = *(int *)addr; in bpfioctl()
1821 BPFD_UNLOCK(d); in bpfioctl()
1829 error = fsetown(*(int *)addr, &d->bd_sigio); in bpfioctl()
1833 BPFD_LOCK(d); in bpfioctl()
1834 *(int *)addr = fgetown(&d->bd_sigio); in bpfioctl()
1835 BPFD_UNLOCK(d); in bpfioctl()
1840 error = fsetown(-(*(int *)addr), &d->bd_sigio); in bpfioctl()
1845 *(int *)addr = -fgetown(&d->bd_sigio); in bpfioctl()
1857 BPFD_LOCK(d); in bpfioctl()
1858 d->bd_sig = sig; in bpfioctl()
1859 BPFD_UNLOCK(d); in bpfioctl()
1864 BPFD_LOCK(d); in bpfioctl()
1865 *(u_int *)addr = d->bd_sig; in bpfioctl()
1866 BPFD_UNLOCK(d); in bpfioctl()
1870 BPFD_LOCK(d); in bpfioctl()
1871 *(u_int *)addr = d->bd_bufmode; in bpfioctl()
1872 BPFD_UNLOCK(d); in bpfioctl()
1897 BPFD_LOCK(d); in bpfioctl()
1898 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || in bpfioctl()
1899 d->bd_fbuf != NULL || d->bd_bif != NULL) { in bpfioctl()
1900 BPFD_UNLOCK(d); in bpfioctl()
1904 d->bd_bufmode = *(u_int *)addr; in bpfioctl()
1905 BPFD_UNLOCK(d); in bpfioctl()
1909 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); in bpfioctl()
1913 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1917 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1925 if (pcp > BPF_PRIO_MAX || pcp < 0) { in bpfioctl()
1929 d->bd_pcp = pcp; in bpfioctl()
1938 * Set d's packet filter program to fp. If this file already has a filter,
1945 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) in bpf_setf() argument
1966 fp_swab.bf_len = fp32->bf_len; in bpf_setf()
1968 (struct bpf_insn *)(uintptr_t)fp32->bf_insns; in bpf_setf()
1990 flen = fp->bf_len; in bpf_setf()
1991 if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0)) in bpf_setf()
1993 size = flen * sizeof(*fp->bf_insns); in bpf_setf()
1994 if (size > 0) { in bpf_setf()
1997 filter = (struct bpf_insn *)fcode->buffer; in bpf_setf()
1998 if (copyin(fp->bf_insns, filter, size) != 0 || in bpf_setf()
2018 BPFD_LOCK(d); in bpf_setf()
2021 if (d->bd_wfilter != NULL) { in bpf_setf()
2022 fcode = __containerof((void *)d->bd_wfilter, in bpf_setf()
2025 fcode->func = NULL; in bpf_setf()
2028 d->bd_wfilter = filter; in bpf_setf()
2030 if (d->bd_rfilter != NULL) { in bpf_setf()
2031 fcode = __containerof((void *)d->bd_rfilter, in bpf_setf()
2034 fcode->func = d->bd_bfilter; in bpf_setf()
2037 d->bd_rfilter = filter; in bpf_setf()
2039 d->bd_bfilter = jfunc; in bpf_setf()
2042 reset_d(d); in bpf_setf()
2044 if (bpf_check_upgrade(cmd, d, filter, flen) != 0) { in bpf_setf()
2047 * specifying interface. In this case just mark d in bpf_setf()
2050 d->bd_writer = 0; in bpf_setf()
2051 if (d->bd_bif != NULL) { in bpf_setf()
2053 * Remove descriptor from writers-only list in bpf_setf()
2056 CK_LIST_REMOVE(d, bd_next); in bpf_setf()
2057 CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist, in bpf_setf()
2058 d, bd_next); in bpf_setf()
2060 "%s: upgrade required by pid %d", in bpf_setf()
2061 __func__, d->bd_pid); in bpf_setf()
2066 BPFD_UNLOCK(d); in bpf_setf()
2069 NET_EPOCH_CALL(bpf_program_buffer_free, &fcode->epoch_ctx); in bpf_setf()
2073 d->bd_bif->bif_ifp, d->bd_bif->bif_dlt, 1); in bpf_setf()
2076 return (0); in bpf_setf()
2082 * Return an errno or 0.
2085 bpf_setif(struct bpf_d *d, struct ifreq *ifr) in bpf_setif() argument
2092 theywant = ifunit(ifr->ifr_name); in bpf_setif()
2099 if (bp->bif_ifp == theywant && in bpf_setif()
2100 bp->bif_bpf == &theywant->if_bpf) in bpf_setif()
2106 MPASS(bp == theywant->if_bpf); in bpf_setif()
2111 switch (d->bd_bufmode) { in bpf_setif()
2114 if (d->bd_sbuf == NULL) in bpf_setif()
2119 panic("bpf_setif: bufmode %d", d->bd_bufmode); in bpf_setif()
2121 if (bp != d->bd_bif) in bpf_setif()
2122 bpf_attachd(d, bp); in bpf_setif()
2124 BPFD_LOCK(d); in bpf_setif()
2125 reset_d(d); in bpf_setif()
2126 BPFD_UNLOCK(d); in bpf_setif()
2128 return (0); in bpf_setif()
2140 struct bpf_d *d; in bpfpoll() local
2143 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) in bpfpoll()
2151 BPFD_LOCK(d); in bpfpoll()
2152 BPF_PID_REFRESH(d, td); in bpfpoll()
2154 if (bpf_ready(d)) in bpfpoll()
2157 selrecord(td, &d->bd_sel); in bpfpoll()
2159 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in bpfpoll()
2160 callout_reset(&d->bd_callout, d->bd_rtout, in bpfpoll()
2161 bpf_timed_out, d); in bpfpoll()
2162 d->bd_state = BPF_WAITING; in bpfpoll()
2166 BPFD_UNLOCK(d); in bpfpoll()
2177 struct bpf_d *d; in bpfkqfilter() local
2179 if (devfs_get_cdevpriv((void **)&d) != 0) in bpfkqfilter()
2182 switch (kn->kn_filter) { in bpfkqfilter()
2184 kn->kn_fop = &bpfread_filtops; in bpfkqfilter()
2188 kn->kn_fop = &bpfwrite_filtops; in bpfkqfilter()
2198 BPFD_LOCK(d); in bpfkqfilter()
2199 BPF_PID_REFRESH_CUR(d); in bpfkqfilter()
2200 kn->kn_hook = d; in bpfkqfilter()
2201 knlist_add(&d->bd_sel.si_note, kn, 1); in bpfkqfilter()
2202 BPFD_UNLOCK(d); in bpfkqfilter()
2204 return (0); in bpfkqfilter()
2210 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfdetach() local
2212 knlist_remove(&d->bd_sel.si_note, kn, 0); in filt_bpfdetach()
2218 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfread() local
2221 BPFD_LOCK_ASSERT(d); in filt_bpfread()
2222 ready = bpf_ready(d); in filt_bpfread()
2224 kn->kn_data = d->bd_slen; in filt_bpfread()
2228 if (!d->bd_hbuf_in_use && d->bd_hbuf) in filt_bpfread()
2229 kn->kn_data += d->bd_hlen; in filt_bpfread()
2230 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in filt_bpfread()
2231 callout_reset(&d->bd_callout, d->bd_rtout, in filt_bpfread()
2232 bpf_timed_out, d); in filt_bpfread()
2233 d->bd_state = BPF_WAITING; in filt_bpfread()
2242 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfwrite() local
2244 BPFD_LOCK_ASSERT(d); in filt_bpfwrite()
2246 if (d->bd_bif == NULL) { in filt_bpfwrite()
2247 kn->kn_data = 0; in filt_bpfwrite()
2248 return (0); in filt_bpfwrite()
2250 kn->kn_data = d->bd_bif->bif_ifp->if_mtu; in filt_bpfwrite()
2255 #define BPF_TSTAMP_NONE 0
2266 if ((tstype & BPF_T_FAST) != 0) in bpf_ts_quality()
2284 if ((m->m_flags & (M_PKTHDR | M_TSTMP)) == (M_PKTHDR | M_TSTMP)) { in bpf_gettime()
2314 struct bpf_d *d; in bpf_tap() local
2323 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_tap()
2324 counter_u64_add(d->bd_rcount, 1); in bpf_tap()
2332 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_tap()
2334 slen = (*(bf->func))(pkt, pktlen, pktlen); in bpf_tap()
2337 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); in bpf_tap()
2338 if (slen != 0) { in bpf_tap()
2342 BPFD_LOCK(d); in bpf_tap()
2343 counter_u64_add(d->bd_fcount, 1); in bpf_tap()
2344 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_tap()
2345 gottime = bpf_gettime(&bt, d->bd_tstamp, in bpf_tap()
2348 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_tap()
2350 catchpacket(d, pkt, pktlen, slen, in bpf_tap()
2352 BPFD_UNLOCK(d); in bpf_tap()
2361 if (bpf_peers_present(ifp->if_bpf)) in bpf_tap_if()
2362 bpf_tap(ifp->if_bpf, pkt, pktlen); in bpf_tap_if()
2365 #define BPF_CHECK_DIRECTION(d, r, i) \ argument
2366 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \
2367 ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
2378 struct bpf_d *d; in bpf_mtap() local
2386 if ((m->m_flags & M_PROMISC) != 0 && m_rcvif(m) == NULL) { in bpf_mtap()
2387 m->m_flags &= ~M_PROMISC; in bpf_mtap()
2395 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap()
2396 if (BPF_CHECK_DIRECTION(d, m_rcvif(m), bp->bif_ifp)) in bpf_mtap()
2398 counter_u64_add(d->bd_rcount, 1); in bpf_mtap()
2400 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_mtap()
2402 if (bf != NULL && m->m_next == NULL) in bpf_mtap()
2403 slen = (*(bf->func))(mtod(m, u_char *), pktlen, in bpf_mtap()
2407 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); in bpf_mtap()
2408 if (slen != 0) { in bpf_mtap()
2409 BPFD_LOCK(d); in bpf_mtap()
2411 counter_u64_add(d->bd_fcount, 1); in bpf_mtap()
2412 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap()
2413 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap()
2415 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_mtap()
2417 catchpacket(d, (u_char *)m, pktlen, slen, in bpf_mtap()
2419 BPFD_UNLOCK(d); in bpf_mtap()
2428 if (bpf_peers_present(ifp->if_bpf)) { in bpf_mtap_if()
2430 bpf_mtap(ifp->if_bpf, m); in bpf_mtap_if()
2444 struct bpf_d *d; in bpf_mtap2() local
2449 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { in bpf_mtap2()
2450 m->m_flags &= ~M_PROMISC; in bpf_mtap2()
2456 * Craft on-stack mbuf suitable for passing to bpf_filter. in bpf_mtap2()
2458 * absolutely needed--this mbuf should never go anywhere else. in bpf_mtap2()
2460 mb.m_flags = 0; in bpf_mtap2()
2469 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap2()
2470 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) in bpf_mtap2()
2472 counter_u64_add(d->bd_rcount, 1); in bpf_mtap2()
2473 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); in bpf_mtap2()
2474 if (slen != 0) { in bpf_mtap2()
2475 BPFD_LOCK(d); in bpf_mtap2()
2477 counter_u64_add(d->bd_fcount, 1); in bpf_mtap2()
2478 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap2()
2479 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap2()
2481 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_mtap2()
2483 catchpacket(d, (u_char *)&mb, pktlen, slen, in bpf_mtap2()
2485 BPFD_UNLOCK(d); in bpf_mtap2()
2494 if (bpf_peers_present(ifp->if_bpf)) { in bpf_mtap2_if()
2496 bpf_mtap2(ifp->if_bpf, data, dlen, m); in bpf_mtap2_if()
2507 bpf_hdrlen(struct bpf_d *d) in bpf_hdrlen() argument
2511 hdrlen = d->bd_bif->bif_hdrlen; in bpf_hdrlen()
2513 if (d->bd_tstamp == BPF_T_NONE || in bpf_hdrlen()
2514 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) in bpf_hdrlen()
2516 if (d->bd_compat32) in bpf_hdrlen()
2525 if (d->bd_compat32) in bpf_hdrlen()
2531 return (hdrlen - d->bd_bif->bif_hdrlen); in bpf_hdrlen()
2541 if ((tstype & BPF_T_MONOTONIC) == 0) { in bpf_bintime2ts()
2550 ts->bt_sec = tsm.tv_sec; in bpf_bintime2ts()
2551 ts->bt_frac = tsm.tv_usec; in bpf_bintime2ts()
2555 ts->bt_sec = tsn.tv_sec; in bpf_bintime2ts()
2556 ts->bt_frac = tsn.tv_nsec; in bpf_bintime2ts()
2559 ts->bt_sec = bt->sec; in bpf_bintime2ts()
2560 ts->bt_frac = bt->frac; in bpf_bintime2ts()
2573 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, in catchpacket() argument
2586 int do_wakeup = 0; in catchpacket()
2590 BPFD_LOCK_ASSERT(d); in catchpacket()
2591 if (d->bd_bif == NULL) { in catchpacket()
2593 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2604 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { in catchpacket()
2605 d->bd_fbuf = d->bd_hbuf; in catchpacket()
2606 d->bd_hbuf = NULL; in catchpacket()
2607 d->bd_hlen = 0; in catchpacket()
2608 bpf_buf_reclaimed(d); in catchpacket()
2617 hdrlen = bpf_hdrlen(d); in catchpacket()
2619 if (totlen > d->bd_bufsize) in catchpacket()
2620 totlen = d->bd_bufsize; in catchpacket()
2631 if (d->bd_compat32) in catchpacket()
2632 curlen = BPF_WORDALIGN32(d->bd_slen); in catchpacket()
2635 curlen = BPF_WORDALIGN(d->bd_slen); in catchpacket()
2636 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { in catchpacket()
2637 if (d->bd_fbuf == NULL) { in catchpacket()
2643 bpf_buffull(d); in catchpacket()
2644 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2647 KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use")); in catchpacket()
2648 ROTATE_BUFFERS(d); in catchpacket()
2650 curlen = 0; in catchpacket()
2652 if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { in catchpacket()
2660 pad = curlen - d->bd_slen; in catchpacket()
2661 KASSERT(pad >= 0 && pad <= sizeof(zeroes), in catchpacket()
2662 ("%s: invalid pad byte count %d", __func__, pad)); in catchpacket()
2663 if (pad > 0) { in catchpacket()
2665 bpf_append_bytes(d, d->bd_sbuf, d->bd_slen, zeroes, in catchpacket()
2670 caplen = totlen - hdrlen; in catchpacket()
2671 tstype = d->bd_tstamp; in catchpacket()
2679 if (d->bd_compat32) { in catchpacket()
2688 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, in catchpacket()
2701 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, in catchpacket()
2717 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); in catchpacket()
2725 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); in catchpacket()
2726 d->bd_slen = curlen + totlen; in catchpacket()
2729 bpf_wakeup(d); in catchpacket()
2739 struct bpf_d *d; in bpfd_free() local
2747 d = __containerof(ctx, struct bpf_d, epoch_ctx); in bpfd_free()
2748 bpf_free(d); in bpfd_free()
2749 if (d->bd_rfilter != NULL) { in bpfd_free()
2750 p = __containerof((void *)d->bd_rfilter, in bpfd_free()
2753 p->func = d->bd_bfilter; in bpfd_free()
2755 bpf_program_buffer_free(&p->epoch_ctx); in bpfd_free()
2757 if (d->bd_wfilter != NULL) { in bpfd_free()
2758 p = __containerof((void *)d->bd_wfilter, in bpfd_free()
2761 p->func = NULL; in bpfd_free()
2763 bpf_program_buffer_free(&p->epoch_ctx); in bpfd_free()
2766 mtx_destroy(&d->bd_lock); in bpfd_free()
2767 counter_u64_free(d->bd_rcount); in bpfd_free()
2768 counter_u64_free(d->bd_dcount); in bpfd_free()
2769 counter_u64_free(d->bd_fcount); in bpfd_free()
2770 counter_u64_free(d->bd_wcount); in bpfd_free()
2771 counter_u64_free(d->bd_wfcount); in bpfd_free()
2772 counter_u64_free(d->bd_wdcount); in bpfd_free()
2773 counter_u64_free(d->bd_zcopy); in bpfd_free()
2774 free(d, M_BPF); in bpfd_free()
2785 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); in bpfattach()
2805 CK_LIST_INIT(&bp->bif_dlist); in bpfattach2()
2806 CK_LIST_INIT(&bp->bif_wlist); in bpfattach2()
2807 bp->bif_ifp = ifp; in bpfattach2()
2808 bp->bif_dlt = dlt; in bpfattach2()
2809 bp->bif_hdrlen = hdrlen; in bpfattach2()
2810 bp->bif_bpf = driverp; in bpfattach2()
2811 refcount_init(&bp->bif_refcnt, 1); in bpfattach2()
2829 * query the dlt and hdrlen before detach so we can re-attch the if_bpf
2841 return (0); in bpf_get_bp_params()
2844 *bif_dlt = bp->bif_dlt; in bpf_get_bp_params()
2846 *bif_hdrlen = bp->bif_hdrlen; in bpf_get_bp_params()
2848 return (0); in bpf_get_bp_params()
2858 struct bpf_d *d; in bpf_ifdetach() local
2862 if (bp->bif_ifp != ifp) in bpf_ifdetach()
2866 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_ifdetach()
2867 bpf_detachd_locked(d, true); in bpf_ifdetach()
2870 /* Detach writer-only descriptors */ in bpf_ifdetach()
2871 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_ifdetach()
2872 bpf_detachd_locked(d, true); in bpf_ifdetach()
2888 struct bpf_d *d; in bpfdetach() local
2893 if (ifp != bp->bif_ifp) in bpfdetach()
2897 *bp->bif_bpf = __DECONST(struct bpf_if *, &dead_bpf_if); in bpfdetach()
2900 "%s: sheduling free for encap %d (%p) for if %p", in bpfdetach()
2901 __func__, bp->bif_dlt, bp, ifp); in bpfdetach()
2904 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpfdetach()
2905 bpf_detachd_locked(d, true); in bpfdetach()
2908 /* Detach writer-only descriptors */ in bpfdetach()
2909 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpfdetach()
2910 bpf_detachd_locked(d, true); in bpfdetach()
2920 return (bpf_peers_present(ifp->if_bpf)); in bpf_peers_present_if()
2927 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) in bpf_getdltlist() argument
2936 ifp = d->bd_bif->bif_ifp; in bpf_getdltlist()
2937 n1 = 0; in bpf_getdltlist()
2939 if (bp->bif_ifp == ifp) in bpf_getdltlist()
2942 if (bfl->bfl_list == NULL) { in bpf_getdltlist()
2943 bfl->bfl_len = n1; in bpf_getdltlist()
2944 return (0); in bpf_getdltlist()
2946 if (n1 > bfl->bfl_len) in bpf_getdltlist()
2950 n = 0; in bpf_getdltlist()
2952 if (bp->bif_ifp != ifp) in bpf_getdltlist()
2954 lst[n++] = bp->bif_dlt; in bpf_getdltlist()
2956 error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n); in bpf_getdltlist()
2958 bfl->bfl_len = n; in bpf_getdltlist()
2966 bpf_setdlt(struct bpf_d *d, u_int dlt) in bpf_setdlt() argument
2973 MPASS(d->bd_bif != NULL); in bpf_setdlt()
2979 if (d->bd_bif->bif_dlt == dlt) in bpf_setdlt()
2980 return (0); in bpf_setdlt()
2982 ifp = d->bd_bif->bif_ifp; in bpf_setdlt()
2984 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) in bpf_setdlt()
2990 opromisc = d->bd_promisc; in bpf_setdlt()
2991 bpf_attachd(d, bp); in bpf_setdlt()
2993 error = ifpromisc(bp->bif_ifp, 1); in bpf_setdlt()
2995 if_printf(bp->bif_ifp, "%s: ifpromisc failed (%d)\n", in bpf_setdlt()
2998 d->bd_promisc = 1; in bpf_setdlt()
3000 return (0); in bpf_setdlt()
3009 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); in bpf_drvinit()
3031 CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { in bpf_zero_counters()
3032 counter_u64_zero(bd->bd_rcount); in bpf_zero_counters()
3033 counter_u64_zero(bd->bd_dcount); in bpf_zero_counters()
3034 counter_u64_zero(bd->bd_fcount); in bpf_zero_counters()
3035 counter_u64_zero(bd->bd_wcount); in bpf_zero_counters()
3036 counter_u64_zero(bd->bd_wfcount); in bpf_zero_counters()
3037 counter_u64_zero(bd->bd_zcopy); in bpf_zero_counters()
3047 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) in bpfstats_fill_xbpf() argument
3051 bzero(d, sizeof(*d)); in bpfstats_fill_xbpf()
3052 d->bd_structsize = sizeof(*d); in bpfstats_fill_xbpf()
3053 d->bd_immediate = bd->bd_immediate; in bpfstats_fill_xbpf()
3054 d->bd_promisc = bd->bd_promisc; in bpfstats_fill_xbpf()
3055 d->bd_hdrcmplt = bd->bd_hdrcmplt; in bpfstats_fill_xbpf()
3056 d->bd_direction = bd->bd_direction; in bpfstats_fill_xbpf()
3057 d->bd_feedback = bd->bd_feedback; in bpfstats_fill_xbpf()
3058 d->bd_async = bd->bd_async; in bpfstats_fill_xbpf()
3059 d->bd_rcount = counter_u64_fetch(bd->bd_rcount); in bpfstats_fill_xbpf()
3060 d->bd_dcount = counter_u64_fetch(bd->bd_dcount); in bpfstats_fill_xbpf()
3061 d->bd_fcount = counter_u64_fetch(bd->bd_fcount); in bpfstats_fill_xbpf()
3062 d->bd_sig = bd->bd_sig; in bpfstats_fill_xbpf()
3063 d->bd_slen = bd->bd_slen; in bpfstats_fill_xbpf()
3064 d->bd_hlen = bd->bd_hlen; in bpfstats_fill_xbpf()
3065 d->bd_bufsize = bd->bd_bufsize; in bpfstats_fill_xbpf()
3066 d->bd_pid = bd->bd_pid; in bpfstats_fill_xbpf()
3067 strlcpy(d->bd_ifname, in bpfstats_fill_xbpf()
3068 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); in bpfstats_fill_xbpf()
3069 d->bd_locked = bd->bd_locked; in bpfstats_fill_xbpf()
3070 d->bd_wcount = counter_u64_fetch(bd->bd_wcount); in bpfstats_fill_xbpf()
3071 d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount); in bpfstats_fill_xbpf()
3072 d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount); in bpfstats_fill_xbpf()
3073 d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy); in bpfstats_fill_xbpf()
3074 d->bd_bufmode = bd->bd_bufmode; in bpfstats_fill_xbpf()
3078 * Handle `netstat -B' stats request
3095 error = priv_check(req->td, PRIV_NET_BPF); in bpf_stats_sysctl()
3103 if (req->newptr != NULL) { in bpf_stats_sysctl()
3104 if (req->newlen != sizeof(tempstats)) in bpf_stats_sysctl()
3106 memset(&tempstats, 0, sizeof(tempstats)); in bpf_stats_sysctl()
3110 if (bcmp(&tempstats, &zerostats, sizeof(tempstats)) != 0) in bpf_stats_sysctl()
3113 return (0); in bpf_stats_sysctl()
3115 if (req->oldptr == NULL) in bpf_stats_sysctl()
3116 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd))); in bpf_stats_sysctl()
3117 if (bpf_bpfd_cnt == 0) in bpf_stats_sysctl()
3118 return (SYSCTL_OUT(req, 0, 0)); in bpf_stats_sysctl()
3119 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); in bpf_stats_sysctl()
3121 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { in bpf_stats_sysctl()
3126 index = 0; in bpf_stats_sysctl()
3128 /* Send writers-only first */ in bpf_stats_sysctl()
3129 CK_LIST_FOREACH(bd, &bp->bif_wlist, bd_next) { in bpf_stats_sysctl()
3133 CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { in bpf_stats_sysctl()
3149 * NOP stubs to allow bpf-using drivers to load and function.
3176 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) in bpf_mtap2() argument
3189 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); in bpfattach()
3213 return (-1); /* "no filter" behaviour */ in bpf_filter()
3219 return (0); /* false */ in bpf_validate()
3232 #define BPF_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, bpf_if->e); in bpf_show_bpf_if()
3241 BPF_DB_PRINTF_RAW("%u", refcount_load(&bpf_if->bif_refcnt)); in bpf_show_bpf_if()