Lines Matching full:d
139 bif_mac_check_receive(struct bpf_if *bp, struct bpf_d *d) in bif_mac_check_receive() argument
143 d)); in bif_mac_check_receive()
154 bpf_chkdir(struct bpf_d *d, struct mbuf *m) in bpf_chkdir() argument
156 return (d->bd_bif->bif_methods->bif_chkdir(d->bd_bif->bif_softc, m, in bpf_chkdir()
157 d->bd_direction)); in bpf_chkdir()
233 static int bpf_attachd(struct bpf_d *d, struct bpf_if *);
366 bpfd_ref(struct bpf_d *d) in bpfd_ref() argument
369 refcount_acquire(&d->bd_refcnt); in bpfd_ref()
373 bpfd_rele(struct bpf_d *d) in bpfd_rele() argument
376 if (!refcount_release(&d->bd_refcnt)) in bpfd_rele()
378 NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx); in bpfd_rele()
408 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_bytes() argument
412 BPFD_LOCK_ASSERT(d); in bpf_append_bytes()
414 switch (d->bd_bufmode) { in bpf_append_bytes()
416 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
419 counter_u64_add(d->bd_zcopy, 1); in bpf_append_bytes()
420 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
428 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_mbuf() argument
432 BPFD_LOCK_ASSERT(d); in bpf_append_mbuf()
434 switch (d->bd_bufmode) { in bpf_append_mbuf()
436 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
439 counter_u64_add(d->bd_zcopy, 1); in bpf_append_mbuf()
440 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
451 bpf_buf_reclaimed(struct bpf_d *d) in bpf_buf_reclaimed() argument
454 BPFD_LOCK_ASSERT(d); in bpf_buf_reclaimed()
456 switch (d->bd_bufmode) { in bpf_buf_reclaimed()
461 bpf_zerocopy_buf_reclaimed(d); in bpf_buf_reclaimed()
475 bpf_canfreebuf(struct bpf_d *d) in bpf_canfreebuf() argument
478 BPFD_LOCK_ASSERT(d); in bpf_canfreebuf()
480 switch (d->bd_bufmode) { in bpf_canfreebuf()
482 return (bpf_zerocopy_canfreebuf(d)); in bpf_canfreebuf()
493 bpf_canwritebuf(struct bpf_d *d) in bpf_canwritebuf() argument
495 BPFD_LOCK_ASSERT(d); in bpf_canwritebuf()
497 switch (d->bd_bufmode) { in bpf_canwritebuf()
499 return (bpf_zerocopy_canwritebuf(d)); in bpf_canwritebuf()
510 bpf_buffull(struct bpf_d *d) in bpf_buffull() argument
513 BPFD_LOCK_ASSERT(d); in bpf_buffull()
515 switch (d->bd_bufmode) { in bpf_buffull()
517 bpf_zerocopy_buffull(d); in bpf_buffull()
526 bpf_bufheld(struct bpf_d *d) in bpf_bufheld() argument
529 BPFD_LOCK_ASSERT(d); in bpf_bufheld()
531 switch (d->bd_bufmode) { in bpf_bufheld()
533 bpf_zerocopy_bufheld(d); in bpf_bufheld()
539 bpf_free(struct bpf_d *d) in bpf_free() argument
542 switch (d->bd_bufmode) { in bpf_free()
544 return (bpf_buffer_free(d)); in bpf_free()
547 return (bpf_zerocopy_free(d)); in bpf_free()
555 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) in bpf_uiomove() argument
558 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_uiomove()
560 return (bpf_buffer_uiomove(d, buf, len, uio)); in bpf_uiomove()
564 bpf_ioctl_sblen(struct bpf_d *d, u_int *i) in bpf_ioctl_sblen() argument
567 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_ioctl_sblen()
569 return (bpf_buffer_ioctl_sblen(d, i)); in bpf_ioctl_sblen()
573 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) in bpf_ioctl_getzmax() argument
576 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_getzmax()
578 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); in bpf_ioctl_getzmax()
582 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_rotzbuf() argument
585 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_rotzbuf()
587 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); in bpf_ioctl_rotzbuf()
591 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_setzbuf() argument
594 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_setzbuf()
596 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); in bpf_ioctl_setzbuf()
600 * Check if we need to upgrade our descriptor @d from write-only mode.
603 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, in bpf_check_upgrade() argument
611 if (d->bd_writer == 0 || fcode == NULL) in bpf_check_upgrade()
621 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g. in bpf_check_upgrade()
643 if (--d->bd_writer == 0) { in bpf_check_upgrade()
654 "%s: filter function set by pid %d, " in bpf_check_upgrade()
655 "bd_writer counter %d, snap %d upgrade %d", in bpf_check_upgrade()
656 __func__, d->bd_pid, d->bd_writer, in bpf_check_upgrade()
666 bpf_detachd(struct bpf_d *d, bool detached_ifp) in bpf_detachd() argument
672 CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); in bpf_detachd()
675 if ((bp = d->bd_bif) == NULL) in bpf_detachd()
678 BPFD_LOCK(d); in bpf_detachd()
679 CK_LIST_REMOVE(d, bd_next); in bpf_detachd()
680 writer = (d->bd_writer > 0); in bpf_detachd()
682 d->bd_bif = NULL; in bpf_detachd()
687 bpf_wakeup(d); in bpf_detachd()
689 BPFD_UNLOCK(d); in bpf_detachd()
694 if (d->bd_promisc && !detached_ifp) { in bpf_detachd()
695 d->bd_promisc = 0; in bpf_detachd()
709 struct bpf_d *d = data; in bpf_dtor() local
711 BPFD_LOCK(d); in bpf_dtor()
712 if (d->bd_state == BPF_WAITING) in bpf_dtor()
713 callout_stop(&d->bd_callout); in bpf_dtor()
714 d->bd_state = BPF_IDLE; in bpf_dtor()
715 BPFD_UNLOCK(d); in bpf_dtor()
716 funsetown(&d->bd_sigio); in bpf_dtor()
718 bpf_detachd(d, false); in bpf_dtor()
721 mac_bpfdesc_destroy(d); in bpf_dtor()
723 seldrain(&d->bd_sel); in bpf_dtor()
724 knlist_destroy(&d->bd_sel.si_note); in bpf_dtor()
725 callout_drain(&d->bd_callout); in bpf_dtor()
726 bpfd_rele(d); in bpf_dtor()
737 struct bpf_d *d; in bpfopen() local
740 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); in bpfopen()
741 error = devfs_set_cdevpriv(d, bpf_dtor); in bpfopen()
743 free(d, M_BPF); in bpfopen()
748 d->bd_rcount = counter_u64_alloc(M_WAITOK); in bpfopen()
749 d->bd_dcount = counter_u64_alloc(M_WAITOK); in bpfopen()
750 d->bd_fcount = counter_u64_alloc(M_WAITOK); in bpfopen()
751 d->bd_wcount = counter_u64_alloc(M_WAITOK); in bpfopen()
752 d->bd_wfcount = counter_u64_alloc(M_WAITOK); in bpfopen()
753 d->bd_wdcount = counter_u64_alloc(M_WAITOK); in bpfopen()
754 d->bd_zcopy = counter_u64_alloc(M_WAITOK); in bpfopen()
761 bpf_buffer_init(d); in bpfopen()
763 d->bd_writer = 2; in bpfopen()
764 d->bd_bufmode = BPF_BUFMODE_BUFFER; in bpfopen()
765 d->bd_sig = SIGIO; in bpfopen()
766 d->bd_direction = BPF_D_INOUT; in bpfopen()
767 refcount_init(&d->bd_refcnt, 1); in bpfopen()
768 BPF_PID_REFRESH(d, td); in bpfopen()
770 mac_bpfdesc_init(d); in bpfopen()
771 mac_bpfdesc_create(td->td_ucred, d); in bpfopen()
773 mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); in bpfopen()
774 callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); in bpfopen()
775 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); in bpfopen()
778 d->bd_pcp = 0; in bpfopen()
789 struct bpf_d *d; in bpfread() local
794 error = devfs_get_cdevpriv((void **)&d); in bpfread()
802 if (uio->uio_resid != d->bd_bufsize) in bpfread()
807 BPFD_LOCK(d); in bpfread()
808 BPF_PID_REFRESH_CUR(d); in bpfread()
809 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { in bpfread()
810 BPFD_UNLOCK(d); in bpfread()
813 if (d->bd_state == BPF_WAITING) in bpfread()
814 callout_stop(&d->bd_callout); in bpfread()
815 timed_out = (d->bd_state == BPF_TIMED_OUT); in bpfread()
816 d->bd_state = BPF_IDLE; in bpfread()
817 while (d->bd_flags & BPFD_HBUF_INUSE) { in bpfread()
818 error = mtx_sleep(&d->bd_hbuf, &d->bd_lock, PRINET | PCATCH, in bpfread()
821 BPFD_UNLOCK(d); in bpfread()
830 while (d->bd_hbuf == NULL) { in bpfread()
831 if (d->bd_slen != 0) { in bpfread()
836 if ((d->bd_flags & BPFD_IMMEDIATE) || non_block || in bpfread()
843 ROTATE_BUFFERS(d); in bpfread()
854 if (d->bd_bif == NULL) { in bpfread()
855 BPFD_UNLOCK(d); in bpfread()
860 BPFD_UNLOCK(d); in bpfread()
863 error = msleep(d, &d->bd_lock, PRINET | PCATCH, in bpfread()
864 "bpf", d->bd_rtout); in bpfread()
866 BPFD_UNLOCK(d); in bpfread()
875 if (d->bd_hbuf) in bpfread()
883 if (d->bd_slen == 0) { in bpfread()
884 BPFD_UNLOCK(d); in bpfread()
887 ROTATE_BUFFERS(d); in bpfread()
894 d->bd_flags |= BPFD_HBUF_INUSE; in bpfread()
895 BPFD_UNLOCK(d); in bpfread()
905 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); in bpfread()
907 BPFD_LOCK(d); in bpfread()
908 if (d->bd_flags & BPFD_HBUF_INUSE) { in bpfread()
909 KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf")); in bpfread()
910 d->bd_fbuf = d->bd_hbuf; in bpfread()
911 d->bd_hbuf = NULL; in bpfread()
912 d->bd_hlen = 0; in bpfread()
913 bpf_buf_reclaimed(d); in bpfread()
914 d->bd_flags &= ~BPFD_HBUF_INUSE; in bpfread()
915 wakeup(&d->bd_hbuf); in bpfread()
917 BPFD_UNLOCK(d); in bpfread()
926 bpf_wakeup(struct bpf_d *d) in bpf_wakeup() argument
929 BPFD_LOCK_ASSERT(d); in bpf_wakeup()
930 if (d->bd_state == BPF_WAITING) { in bpf_wakeup()
931 callout_stop(&d->bd_callout); in bpf_wakeup()
932 d->bd_state = BPF_IDLE; in bpf_wakeup()
934 wakeup(d); in bpf_wakeup()
935 if ((d->bd_flags & BPFD_ASYNC) && d->bd_sig && d->bd_sigio) in bpf_wakeup()
936 pgsigio(&d->bd_sigio, d->bd_sig, 0); in bpf_wakeup()
938 selwakeuppri(&d->bd_sel, PRINET); in bpf_wakeup()
939 KNOTE_LOCKED(&d->bd_sel.si_note, 0); in bpf_wakeup()
945 struct bpf_d *d = (struct bpf_d *)arg; in bpf_timed_out() local
947 BPFD_LOCK_ASSERT(d); in bpf_timed_out()
949 if (callout_pending(&d->bd_callout) || in bpf_timed_out()
950 !callout_active(&d->bd_callout)) in bpf_timed_out()
952 if (d->bd_state == BPF_WAITING) { in bpf_timed_out()
953 d->bd_state = BPF_TIMED_OUT; in bpf_timed_out()
954 if (d->bd_slen != 0) in bpf_timed_out()
955 bpf_wakeup(d); in bpf_timed_out()
960 bpf_ready(struct bpf_d *d) in bpf_ready() argument
963 BPFD_LOCK_ASSERT(d); in bpf_ready()
965 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) in bpf_ready()
967 if (((d->bd_flags & BPFD_IMMEDIATE) || d->bd_state == BPF_TIMED_OUT) && in bpf_ready()
968 d->bd_slen != 0) in bpf_ready()
978 struct bpf_d *d; in bpfwrite() local
983 error = devfs_get_cdevpriv((void **)&d); in bpfwrite()
990 BPFD_LOCK(d); in bpfwrite()
991 if ((bp = d->bd_bif) == NULL) in bpfwrite()
996 BPFD_UNLOCK(d); in bpfwrite()
997 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1000 bpfd_ref(d); in bpfwrite()
1001 BPFD_UNLOCK(d); in bpfwrite()
1016 if (bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len) == 0) { in bpfwrite()
1021 if (d->bd_flags & BPFD_FEEDBACK) { in bpfwrite()
1024 if (d->bd_direction == BPF_D_INOUT) in bpfwrite()
1030 if (d->bd_pcp != 0) in bpfwrite()
1031 (void)vlan_set_pcp(m, d->bd_pcp); in bpfwrite()
1033 BPFD_LOCK(d); in bpfwrite()
1035 mac_bpfdesc_create_mbuf(d, m); in bpfwrite()
1037 mac_bpfdesc_create_mbuf(d, mc); in bpfwrite()
1043 if (__predict_false(d->bd_bif != bp)) { in bpfwrite()
1044 BPFD_UNLOCK(d); in bpfwrite()
1049 BPFD_UNLOCK(d); in bpfwrite()
1052 error = bp->bif_methods->bif_write(bp->bif_softc, m, mc, d->bd_flags); in bpfwrite()
1055 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1057 counter_u64_add(d->bd_wfcount, 1); in bpfwrite()
1058 bpfd_rele(d); in bpfwrite()
1063 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1064 bpfd_rele(d); in bpfwrite()
1077 reset_d(struct bpf_d *d) in reset_d() argument
1080 BPFD_LOCK_ASSERT(d); in reset_d()
1082 while (d->bd_flags & BPFD_HBUF_INUSE) in reset_d()
1083 mtx_sleep(&d->bd_hbuf, &d->bd_lock, PRINET, "bd_hbuf", 0); in reset_d()
1084 if ((d->bd_hbuf != NULL) && in reset_d()
1085 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { in reset_d()
1087 d->bd_fbuf = d->bd_hbuf; in reset_d()
1088 d->bd_hbuf = NULL; in reset_d()
1089 d->bd_hlen = 0; in reset_d()
1090 bpf_buf_reclaimed(d); in reset_d()
1092 if (bpf_canwritebuf(d)) in reset_d()
1093 d->bd_slen = 0; in reset_d()
1094 counter_u64_zero(d->bd_rcount); in reset_d()
1095 counter_u64_zero(d->bd_dcount); in reset_d()
1096 counter_u64_zero(d->bd_fcount); in reset_d()
1097 counter_u64_zero(d->bd_wcount); in reset_d()
1098 counter_u64_zero(d->bd_wfcount); in reset_d()
1099 counter_u64_zero(d->bd_wdcount); in reset_d()
1100 counter_u64_zero(d->bd_zcopy); in reset_d()
1140 struct bpf_d *d; in bpfioctl() local
1143 error = devfs_get_cdevpriv((void **)&d); in bpfioctl()
1150 BPFD_LOCK(d); in bpfioctl()
1151 BPF_PID_REFRESH(d, td); in bpfioctl()
1152 if (d->bd_state == BPF_WAITING) in bpfioctl()
1153 callout_stop(&d->bd_callout); in bpfioctl()
1154 d->bd_state = BPF_IDLE; in bpfioctl()
1155 BPFD_UNLOCK(d); in bpfioctl()
1157 if (d->bd_flags & BPFD_LOCKED) { in bpfioctl()
1205 BPFD_LOCK(d); in bpfioctl()
1206 d->bd_compat32 = 1; in bpfioctl()
1207 BPFD_UNLOCK(d); in bpfioctl()
1238 BPFD_LOCK(d); in bpfioctl()
1239 n = d->bd_slen; in bpfioctl()
1240 while (d->bd_flags & BPFD_HBUF_INUSE) in bpfioctl()
1241 mtx_sleep(&d->bd_hbuf, &d->bd_lock, in bpfioctl()
1243 if (d->bd_hbuf) in bpfioctl()
1244 n += d->bd_hlen; in bpfioctl()
1245 BPFD_UNLOCK(d); in bpfioctl()
1261 BPFD_LOCK(d); in bpfioctl()
1262 *(u_int *)addr = d->bd_bufsize; in bpfioctl()
1263 BPFD_UNLOCK(d); in bpfioctl()
1270 error = bpf_ioctl_sblen(d, (u_int *)addr); in bpfioctl()
1284 error = bpf_setf(d, (struct bpf_program *)addr, cmd); in bpfioctl()
1291 BPFD_LOCK(d); in bpfioctl()
1292 reset_d(d); in bpfioctl()
1293 BPFD_UNLOCK(d); in bpfioctl()
1301 if (d->bd_bif == NULL) { in bpfioctl()
1306 } else if (d->bd_promisc == 0) { in bpfioctl()
1307 struct bpf_if *bp = d->bd_bif; in bpfioctl()
1310 d->bd_promisc = 1; in bpfioctl()
1320 if (d->bd_bif == NULL) in bpfioctl()
1323 *(u_int *)addr = d->bd_bif->bif_dlt; in bpfioctl()
1340 if (d->bd_bif == NULL) in bpfioctl()
1343 error = bpf_getdltlist(d, &dltlist); in bpfioctl()
1354 if (d->bd_bif == NULL) in bpfioctl()
1357 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); in bpfioctl()
1366 if (d->bd_bif == NULL) in bpfioctl()
1369 error = bpf_setdlt(d, *(u_int *)addr); in bpfioctl()
1378 if (d->bd_bif == NULL) in bpfioctl()
1381 struct bpf_if *const bp = d->bd_bif; in bpfioctl()
1403 BPFD_LOCK(d); in bpfioctl()
1404 if (d->bd_bufmode == BPF_BUFMODE_BUFFER && in bpfioctl()
1405 d->bd_sbuf == NULL) { in bpfioctl()
1408 size = d->bd_bufsize; in bpfioctl()
1409 BPFD_UNLOCK(d); in bpfioctl()
1410 error = bpf_buffer_ioctl_sblen(d, &size); in bpfioctl()
1414 BPFD_UNLOCK(d); in bpfioctl()
1425 error = bpf_attachd(d, bp); in bpfioctl()
1454 d->bd_rtout = tvtohz(tv) - 1; in bpfioctl()
1471 tv->tv_sec = d->bd_rtout / hz; in bpfioctl()
1472 tv->tv_usec = (d->bd_rtout % hz) * tick; in bpfioctl()
1491 bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount); in bpfioctl()
1492 bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount); in bpfioctl()
1500 BPFD_LOCK(d); in bpfioctl()
1501 d->bd_flags |= *(u_int *)addr ? BPFD_IMMEDIATE : 0; in bpfioctl()
1502 BPFD_UNLOCK(d); in bpfioctl()
1518 BPFD_LOCK(d); in bpfioctl()
1519 *(u_int *)addr = d->bd_flags & BPFD_HDRCMPLT ? 1 : 0; in bpfioctl()
1520 BPFD_UNLOCK(d); in bpfioctl()
1527 BPFD_LOCK(d); in bpfioctl()
1528 d->bd_flags |= *(u_int *)addr ? BPFD_HDRCMPLT : 0; in bpfioctl()
1529 BPFD_UNLOCK(d); in bpfioctl()
1536 BPFD_LOCK(d); in bpfioctl()
1537 *(u_int *)addr = d->bd_direction; in bpfioctl()
1538 BPFD_UNLOCK(d); in bpfioctl()
1553 BPFD_LOCK(d); in bpfioctl()
1554 d->bd_direction = direction; in bpfioctl()
1555 BPFD_UNLOCK(d); in bpfioctl()
1567 BPFD_LOCK(d); in bpfioctl()
1568 *(u_int *)addr = d->bd_tstamp; in bpfioctl()
1569 BPFD_UNLOCK(d); in bpfioctl()
1581 d->bd_tstamp = func; in bpfioctl()
1588 BPFD_LOCK(d); in bpfioctl()
1589 d->bd_flags |= *(u_int *)addr ? BPFD_FEEDBACK : 0; in bpfioctl()
1590 BPFD_UNLOCK(d); in bpfioctl()
1594 BPFD_LOCK(d); in bpfioctl()
1595 d->bd_flags |= BPFD_LOCKED; in bpfioctl()
1596 BPFD_UNLOCK(d); in bpfioctl()
1603 BPFD_LOCK(d); in bpfioctl()
1604 d->bd_flags |= *(u_int *)addr ? BPFD_ASYNC : 0; in bpfioctl()
1605 BPFD_UNLOCK(d); in bpfioctl()
1613 error = fsetown(*(int *)addr, &d->bd_sigio); in bpfioctl()
1617 BPFD_LOCK(d); in bpfioctl()
1618 *(int *)addr = fgetown(&d->bd_sigio); in bpfioctl()
1619 BPFD_UNLOCK(d); in bpfioctl()
1624 error = fsetown(-(*(int *)addr), &d->bd_sigio); in bpfioctl()
1629 *(int *)addr = -fgetown(&d->bd_sigio); in bpfioctl()
1641 BPFD_LOCK(d); in bpfioctl()
1642 d->bd_sig = sig; in bpfioctl()
1643 BPFD_UNLOCK(d); in bpfioctl()
1648 BPFD_LOCK(d); in bpfioctl()
1649 *(u_int *)addr = d->bd_sig; in bpfioctl()
1650 BPFD_UNLOCK(d); in bpfioctl()
1654 BPFD_LOCK(d); in bpfioctl()
1655 *(u_int *)addr = d->bd_bufmode; in bpfioctl()
1656 BPFD_UNLOCK(d); in bpfioctl()
1681 BPFD_LOCK(d); in bpfioctl()
1682 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || in bpfioctl()
1683 d->bd_fbuf != NULL || d->bd_bif != NULL) { in bpfioctl()
1684 BPFD_UNLOCK(d); in bpfioctl()
1688 d->bd_bufmode = *(u_int *)addr; in bpfioctl()
1689 BPFD_UNLOCK(d); in bpfioctl()
1693 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); in bpfioctl()
1697 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1701 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1713 d->bd_pcp = pcp; in bpfioctl()
1775 * Set d's packet filter program to fp. If this file already has a filter,
1782 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) in bpf_setf() argument
1855 BPFD_LOCK(d); in bpf_setf()
1858 if (d->bd_wfilter != NULL) { in bpf_setf()
1859 fcode = __containerof((void *)d->bd_wfilter, in bpf_setf()
1865 d->bd_wfilter = filter; in bpf_setf()
1867 if (d->bd_rfilter != NULL) { in bpf_setf()
1868 fcode = __containerof((void *)d->bd_rfilter, in bpf_setf()
1871 fcode->func = d->bd_bfilter; in bpf_setf()
1874 d->bd_rfilter = filter; in bpf_setf()
1876 d->bd_bfilter = jfunc; in bpf_setf()
1879 reset_d(d); in bpf_setf()
1881 if (bpf_check_upgrade(cmd, d, filter, flen) != 0) { in bpf_setf()
1884 * specifying interface. In this case just mark d in bpf_setf()
1887 d->bd_writer = 0; in bpf_setf()
1888 if (d->bd_bif != NULL) { in bpf_setf()
1893 CK_LIST_REMOVE(d, bd_next); in bpf_setf()
1894 CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist, in bpf_setf()
1895 d, bd_next); in bpf_setf()
1897 "%s: upgrade required by pid %d", in bpf_setf()
1898 __func__, d->bd_pid); in bpf_setf()
1903 BPFD_UNLOCK(d); in bpf_setf()
1909 bif_attachd(d->bd_bif); in bpf_setf()
1921 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) in bpf_attachd() argument
1931 switch (d->bd_bufmode) { in bpf_attachd()
1934 if (d->bd_sbuf == NULL) in bpf_attachd()
1939 panic("%s: bufmode %d", __func__, d->bd_bufmode); in bpf_attachd()
1942 if (bp == d->bd_bif) { in bpf_attachd()
1943 BPFD_LOCK(d); in bpf_attachd()
1944 reset_d(d); in bpf_attachd()
1945 BPFD_UNLOCK(d); in bpf_attachd()
1947 } else if (d->bd_bif != NULL) in bpf_attachd()
1948 bpf_detachd(d, false); in bpf_attachd()
1953 writer = V_bpf_optimize_writers || (d->bd_writer > 0); in bpf_attachd()
1956 * Point d at bp, and add d to the interface's list. in bpf_attachd()
1959 * we can delay adding d to the list of active listeners until in bpf_attachd()
1962 BPFD_LOCK(d); in bpf_attachd()
1967 d->bd_bif = bp; in bpf_attachd()
1970 CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); in bpf_attachd()
1977 d->bd_writer = 2; in bpf_attachd()
1979 CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); in bpf_attachd()
1981 reset_d(d); in bpf_attachd()
1984 bpf_wakeup(d); in bpf_attachd()
1986 BPFD_UNLOCK(d); in bpf_attachd()
1988 CTR3(KTR_NET, "%s: called by pid %d, adding to %s list", in bpf_attachd()
1989 __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); in bpf_attachd()
2006 struct bpf_d *d; in bpfpoll() local
2009 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) in bpfpoll()
2017 BPFD_LOCK(d); in bpfpoll()
2018 BPF_PID_REFRESH(d, td); in bpfpoll()
2020 if (bpf_ready(d)) in bpfpoll()
2023 selrecord(td, &d->bd_sel); in bpfpoll()
2025 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in bpfpoll()
2026 callout_reset(&d->bd_callout, d->bd_rtout, in bpfpoll()
2027 bpf_timed_out, d); in bpfpoll()
2028 d->bd_state = BPF_WAITING; in bpfpoll()
2032 BPFD_UNLOCK(d); in bpfpoll()
2043 struct bpf_d *d; in bpfkqfilter() local
2045 if (devfs_get_cdevpriv((void **)&d) != 0) in bpfkqfilter()
2064 BPFD_LOCK(d); in bpfkqfilter()
2065 BPF_PID_REFRESH_CUR(d); in bpfkqfilter()
2066 kn->kn_hook = d; in bpfkqfilter()
2067 knlist_add(&d->bd_sel.si_note, kn, 1); in bpfkqfilter()
2068 BPFD_UNLOCK(d); in bpfkqfilter()
2076 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfdetach() local
2078 knlist_remove(&d->bd_sel.si_note, kn, 0); in filt_bpfdetach()
2084 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfread() local
2087 BPFD_LOCK_ASSERT(d); in filt_bpfread()
2088 ready = bpf_ready(d); in filt_bpfread()
2090 kn->kn_data = d->bd_slen; in filt_bpfread()
2094 if (!(d->bd_flags & BPFD_HBUF_INUSE) && d->bd_hbuf) in filt_bpfread()
2095 kn->kn_data += d->bd_hlen; in filt_bpfread()
2096 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in filt_bpfread()
2097 callout_reset(&d->bd_callout, d->bd_rtout, in filt_bpfread()
2098 bpf_timed_out, d); in filt_bpfread()
2099 d->bd_state = BPF_WAITING; in filt_bpfread()
2108 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfwrite() local
2110 BPFD_LOCK_ASSERT(d); in filt_bpfwrite()
2112 if (d->bd_bif == NULL) { in filt_bpfwrite()
2116 kn->kn_data = bif_wrsize(d->bd_bif); in filt_bpfwrite()
2180 struct bpf_d *d; in bpf_tap() local
2189 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_tap()
2190 counter_u64_add(d->bd_rcount, 1); in bpf_tap()
2198 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_tap()
2203 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); in bpf_tap()
2208 BPFD_LOCK(d); in bpf_tap()
2209 counter_u64_add(d->bd_fcount, 1); in bpf_tap()
2210 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_tap()
2211 gottime = bpf_gettime(&bt, d->bd_tstamp, in bpf_tap()
2214 if (bif_mac_check_receive(bp, d) == 0) in bpf_tap()
2216 catchpacket(d, pkt, pktlen, slen, in bpf_tap()
2218 BPFD_UNLOCK(d); in bpf_tap()
2240 struct bpf_d *d; in bpf_mtap() local
2257 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap()
2258 if (bpf_chkdir(d, m)) in bpf_mtap()
2260 counter_u64_add(d->bd_rcount, 1); in bpf_mtap()
2262 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_mtap()
2269 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); in bpf_mtap()
2271 BPFD_LOCK(d); in bpf_mtap()
2273 counter_u64_add(d->bd_fcount, 1); in bpf_mtap()
2274 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap()
2275 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap()
2277 if (bif_mac_check_receive(bp, d) == 0) in bpf_mtap()
2279 catchpacket(d, (u_char *)m, pktlen, slen, in bpf_mtap()
2281 BPFD_UNLOCK(d); in bpf_mtap()
2306 struct bpf_d *d; in bpf_mtap2() local
2331 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap2()
2332 if (bpf_chkdir(d, m)) in bpf_mtap2()
2334 counter_u64_add(d->bd_rcount, 1); in bpf_mtap2()
2335 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); in bpf_mtap2()
2337 BPFD_LOCK(d); in bpf_mtap2()
2339 counter_u64_add(d->bd_fcount, 1); in bpf_mtap2()
2340 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap2()
2341 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap2()
2343 if (bif_mac_check_receive(bp, d) == 0) in bpf_mtap2()
2345 catchpacket(d, (u_char *)&mb, pktlen, slen, in bpf_mtap2()
2347 BPFD_UNLOCK(d); in bpf_mtap2()
2368 bpf_hdrlen(struct bpf_d *d) in bpf_hdrlen() argument
2372 hdrlen = d->bd_bif->bif_hdrlen; in bpf_hdrlen()
2374 if (d->bd_tstamp == BPF_T_NONE || in bpf_hdrlen()
2375 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) in bpf_hdrlen()
2377 if (d->bd_compat32) in bpf_hdrlen()
2386 if (d->bd_compat32) in bpf_hdrlen()
2392 return (hdrlen - d->bd_bif->bif_hdrlen); in bpf_hdrlen()
2434 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, in catchpacket() argument
2451 BPFD_LOCK_ASSERT(d); in catchpacket()
2452 if (d->bd_bif == NULL) { in catchpacket()
2454 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2465 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { in catchpacket()
2466 d->bd_fbuf = d->bd_hbuf; in catchpacket()
2467 d->bd_hbuf = NULL; in catchpacket()
2468 d->bd_hlen = 0; in catchpacket()
2469 bpf_buf_reclaimed(d); in catchpacket()
2478 hdrlen = bpf_hdrlen(d); in catchpacket()
2480 if (totlen > d->bd_bufsize) in catchpacket()
2481 totlen = d->bd_bufsize; in catchpacket()
2492 if (d->bd_compat32) in catchpacket()
2493 curlen = BPF_WORDALIGN32(d->bd_slen); in catchpacket()
2496 curlen = BPF_WORDALIGN(d->bd_slen); in catchpacket()
2497 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { in catchpacket()
2498 if (d->bd_fbuf == NULL) { in catchpacket()
2504 bpf_buffull(d); in catchpacket()
2505 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2508 KASSERT(!(d->bd_flags & BPFD_HBUF_INUSE), in catchpacket()
2510 ROTATE_BUFFERS(d); in catchpacket()
2514 if ((d->bd_flags & BPFD_IMMEDIATE) || in catchpacket()
2515 d->bd_state == BPF_TIMED_OUT) { in catchpacket()
2523 pad = curlen - d->bd_slen; in catchpacket()
2525 ("%s: invalid pad byte count %d", __func__, pad)); in catchpacket()
2528 bpf_append_bytes(d, d->bd_sbuf, d->bd_slen, zeroes, in catchpacket()
2534 tstype = d->bd_tstamp; in catchpacket()
2542 if (d->bd_compat32) { in catchpacket()
2551 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, in catchpacket()
2564 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, in catchpacket()
2580 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); in catchpacket()
2588 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); in catchpacket()
2589 d->bd_slen = curlen + totlen; in catchpacket()
2592 bpf_wakeup(d); in catchpacket()
2602 struct bpf_d *d; in bpfd_free() local
2610 d = __containerof(ctx, struct bpf_d, epoch_ctx); in bpfd_free()
2611 bpf_free(d); in bpfd_free()
2612 if (d->bd_rfilter != NULL) { in bpfd_free()
2613 p = __containerof((void *)d->bd_rfilter, in bpfd_free()
2616 p->func = d->bd_bfilter; in bpfd_free()
2620 if (d->bd_wfilter != NULL) { in bpfd_free()
2621 p = __containerof((void *)d->bd_wfilter, in bpfd_free()
2629 mtx_destroy(&d->bd_lock); in bpfd_free()
2630 counter_u64_free(d->bd_rcount); in bpfd_free()
2631 counter_u64_free(d->bd_dcount); in bpfd_free()
2632 counter_u64_free(d->bd_fcount); in bpfd_free()
2633 counter_u64_free(d->bd_wcount); in bpfd_free()
2634 counter_u64_free(d->bd_wfcount); in bpfd_free()
2635 counter_u64_free(d->bd_wdcount); in bpfd_free()
2636 counter_u64_free(d->bd_zcopy); in bpfd_free()
2637 free(d, M_BPF); in bpfd_free()
2676 struct bpf_d *d; in bpf_ifdetach() local
2685 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_ifdetach()
2686 bpf_detachd(d, true); in bpf_ifdetach()
2690 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_ifdetach()
2691 bpf_detachd(d, true); in bpf_ifdetach()
2706 struct bpf_d *d; in bpf_detach() local
2711 CTR3(KTR_NET, "%s: sheduling free for encap %d for bp %p", in bpf_detach()
2715 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_detach()
2716 bpf_detachd(d, true); in bpf_detach()
2720 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_detach()
2721 bpf_detachd(d, true); in bpf_detach()
2755 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) in bpf_getdltlist() argument
2764 name = d->bd_bif->bif_name; in bpf_getdltlist()
2801 bpf_setdlt(struct bpf_d *d, u_int dlt) in bpf_setdlt() argument
2808 MPASS(d->bd_bif != NULL); in bpf_setdlt()
2814 if (d->bd_bif->bif_dlt == dlt) in bpf_setdlt()
2817 name = d->bd_bif->bif_name; in bpf_setdlt()
2825 opromisc = d->bd_promisc; in bpf_setdlt()
2826 bpf_attachd(d, bp); in bpf_setdlt()
2830 printf("%s: bif_promisc on %s failed (%d)\n", in bpf_setdlt()
2833 d->bd_promisc = 1; in bpf_setdlt()
2882 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) in bpfstats_fill_xbpf() argument
2886 bzero(d, sizeof(*d)); in bpfstats_fill_xbpf()
2887 d->bd_structsize = sizeof(*d); in bpfstats_fill_xbpf()
2888 d->bd_immediate = bd->bd_flags & BPFD_IMMEDIATE ? 1 : 0; in bpfstats_fill_xbpf()
2889 d->bd_promisc = bd->bd_promisc; in bpfstats_fill_xbpf()
2890 d->bd_hdrcmplt = bd->bd_flags & BPFD_HDRCMPLT ? 1 : 0; in bpfstats_fill_xbpf()
2891 d->bd_direction = bd->bd_direction; in bpfstats_fill_xbpf()
2892 d->bd_feedback = bd->bd_flags & BPFD_FEEDBACK ? 1 : 0; in bpfstats_fill_xbpf()
2893 d->bd_async = bd->bd_flags & BPFD_ASYNC ? 1 : 0; in bpfstats_fill_xbpf()
2894 d->bd_rcount = counter_u64_fetch(bd->bd_rcount); in bpfstats_fill_xbpf()
2895 d->bd_dcount = counter_u64_fetch(bd->bd_dcount); in bpfstats_fill_xbpf()
2896 d->bd_fcount = counter_u64_fetch(bd->bd_fcount); in bpfstats_fill_xbpf()
2897 d->bd_sig = bd->bd_sig; in bpfstats_fill_xbpf()
2898 d->bd_slen = bd->bd_slen; in bpfstats_fill_xbpf()
2899 d->bd_hlen = bd->bd_hlen; in bpfstats_fill_xbpf()
2900 d->bd_bufsize = bd->bd_bufsize; in bpfstats_fill_xbpf()
2901 d->bd_pid = bd->bd_pid; in bpfstats_fill_xbpf()
2902 strlcpy(d->bd_ifname, bd->bd_bif->bif_name, sizeof(d->bd_ifname)); in bpfstats_fill_xbpf()
2903 d->bd_locked = bd->bd_flags & BPFD_LOCKED ? 1 : 0; in bpfstats_fill_xbpf()
2904 d->bd_wcount = counter_u64_fetch(bd->bd_wcount); in bpfstats_fill_xbpf()
2905 d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount); in bpfstats_fill_xbpf()
2906 d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount); in bpfstats_fill_xbpf()
2907 d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy); in bpfstats_fill_xbpf()
2908 d->bd_bufmode = bd->bd_bufmode; in bpfstats_fill_xbpf()
3019 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) in bpf_mtap2() argument