Lines Matching full:d
139 bif_mac_check_receive(struct bpf_if *bp, struct bpf_d *d) in bif_mac_check_receive() argument
143 d)); in bif_mac_check_receive()
154 bpf_chkdir(struct bpf_d *d, struct mbuf *m) in bpf_chkdir() argument
156 return (d->bd_bif->bif_methods->bif_chkdir(d->bd_bif->bif_softc, m, in bpf_chkdir()
157 d->bd_direction)); in bpf_chkdir()
233 static int bpf_attachd(struct bpf_d *d, struct bpf_if *);
366 bpfd_ref(struct bpf_d *d) in bpfd_ref() argument
369 refcount_acquire(&d->bd_refcnt); in bpfd_ref()
373 bpfd_rele(struct bpf_d *d) in bpfd_rele() argument
376 if (!refcount_release(&d->bd_refcnt)) in bpfd_rele()
378 NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx); in bpfd_rele()
408 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_bytes() argument
412 BPFD_LOCK_ASSERT(d); in bpf_append_bytes()
414 switch (d->bd_bufmode) { in bpf_append_bytes()
416 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
419 counter_u64_add(d->bd_zcopy, 1); in bpf_append_bytes()
420 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
428 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_mbuf() argument
432 BPFD_LOCK_ASSERT(d); in bpf_append_mbuf()
434 switch (d->bd_bufmode) { in bpf_append_mbuf()
436 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
439 counter_u64_add(d->bd_zcopy, 1); in bpf_append_mbuf()
440 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
451 bpf_buf_reclaimed(struct bpf_d *d) in bpf_buf_reclaimed() argument
454 BPFD_LOCK_ASSERT(d); in bpf_buf_reclaimed()
456 switch (d->bd_bufmode) { in bpf_buf_reclaimed()
461 bpf_zerocopy_buf_reclaimed(d); in bpf_buf_reclaimed()
475 bpf_canfreebuf(struct bpf_d *d) in bpf_canfreebuf() argument
478 BPFD_LOCK_ASSERT(d); in bpf_canfreebuf()
480 switch (d->bd_bufmode) { in bpf_canfreebuf()
482 return (bpf_zerocopy_canfreebuf(d)); in bpf_canfreebuf()
493 bpf_canwritebuf(struct bpf_d *d) in bpf_canwritebuf() argument
495 BPFD_LOCK_ASSERT(d); in bpf_canwritebuf()
497 switch (d->bd_bufmode) { in bpf_canwritebuf()
499 return (bpf_zerocopy_canwritebuf(d)); in bpf_canwritebuf()
510 bpf_buffull(struct bpf_d *d) in bpf_buffull() argument
513 BPFD_LOCK_ASSERT(d); in bpf_buffull()
515 switch (d->bd_bufmode) { in bpf_buffull()
517 bpf_zerocopy_buffull(d); in bpf_buffull()
526 bpf_bufheld(struct bpf_d *d) in bpf_bufheld() argument
529 BPFD_LOCK_ASSERT(d); in bpf_bufheld()
531 switch (d->bd_bufmode) { in bpf_bufheld()
533 bpf_zerocopy_bufheld(d); in bpf_bufheld()
539 bpf_free(struct bpf_d *d) in bpf_free() argument
542 switch (d->bd_bufmode) { in bpf_free()
544 return (bpf_buffer_free(d)); in bpf_free()
547 return (bpf_zerocopy_free(d)); in bpf_free()
555 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) in bpf_uiomove() argument
558 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_uiomove()
560 return (bpf_buffer_uiomove(d, buf, len, uio)); in bpf_uiomove()
564 bpf_ioctl_sblen(struct bpf_d *d, u_int *i) in bpf_ioctl_sblen() argument
567 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_ioctl_sblen()
569 return (bpf_buffer_ioctl_sblen(d, i)); in bpf_ioctl_sblen()
573 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) in bpf_ioctl_getzmax() argument
576 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_getzmax()
578 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); in bpf_ioctl_getzmax()
582 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_rotzbuf() argument
585 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_rotzbuf()
587 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); in bpf_ioctl_rotzbuf()
591 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_setzbuf() argument
594 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_setzbuf()
596 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); in bpf_ioctl_setzbuf()
600 * Check if we need to upgrade our descriptor @d from write-only mode.
603 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, in bpf_check_upgrade() argument
611 if (d->bd_writer == 0 || fcode == NULL) in bpf_check_upgrade()
621 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g. in bpf_check_upgrade()
643 if (--d->bd_writer == 0) { in bpf_check_upgrade()
654 "%s: filter function set by pid %d, " in bpf_check_upgrade()
655 "bd_writer counter %d, snap %d upgrade %d", in bpf_check_upgrade()
656 __func__, d->bd_pid, d->bd_writer, in bpf_check_upgrade()
666 bpf_detachd(struct bpf_d *d, bool detached_ifp) in bpf_detachd() argument
672 CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); in bpf_detachd()
675 if ((bp = d->bd_bif) == NULL) in bpf_detachd()
678 BPFD_LOCK(d); in bpf_detachd()
679 CK_LIST_REMOVE(d, bd_next); in bpf_detachd()
680 writer = (d->bd_writer > 0); in bpf_detachd()
681 d->bd_bif = NULL; in bpf_detachd()
687 bpf_wakeup(d); in bpf_detachd()
689 BPFD_UNLOCK(d); in bpf_detachd()
694 if (d->bd_promisc && !detached_ifp) { in bpf_detachd()
695 d->bd_promisc = 0; in bpf_detachd()
709 struct bpf_d *d = data; in bpf_dtor() local
711 BPFD_LOCK(d); in bpf_dtor()
712 if (d->bd_state == BPF_WAITING) in bpf_dtor()
713 callout_stop(&d->bd_callout); in bpf_dtor()
714 d->bd_state = BPF_IDLE; in bpf_dtor()
715 BPFD_UNLOCK(d); in bpf_dtor()
716 funsetown(&d->bd_sigio); in bpf_dtor()
718 bpf_detachd(d, false); in bpf_dtor()
721 mac_bpfdesc_destroy(d); in bpf_dtor()
723 seldrain(&d->bd_sel); in bpf_dtor()
724 knlist_destroy(&d->bd_sel.si_note); in bpf_dtor()
725 callout_drain(&d->bd_callout); in bpf_dtor()
726 bpfd_rele(d); in bpf_dtor()
737 struct bpf_d *d; in bpfopen() local
740 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); in bpfopen()
741 error = devfs_set_cdevpriv(d, bpf_dtor); in bpfopen()
743 free(d, M_BPF); in bpfopen()
748 d->bd_rcount = counter_u64_alloc(M_WAITOK); in bpfopen()
749 d->bd_dcount = counter_u64_alloc(M_WAITOK); in bpfopen()
750 d->bd_fcount = counter_u64_alloc(M_WAITOK); in bpfopen()
751 d->bd_wcount = counter_u64_alloc(M_WAITOK); in bpfopen()
752 d->bd_wfcount = counter_u64_alloc(M_WAITOK); in bpfopen()
753 d->bd_wdcount = counter_u64_alloc(M_WAITOK); in bpfopen()
754 d->bd_zcopy = counter_u64_alloc(M_WAITOK); in bpfopen()
761 bpf_buffer_init(d); in bpfopen()
763 d->bd_writer = 2; in bpfopen()
764 d->bd_bufmode = BPF_BUFMODE_BUFFER; in bpfopen()
765 d->bd_sig = SIGIO; in bpfopen()
766 d->bd_direction = BPF_D_INOUT; in bpfopen()
767 refcount_init(&d->bd_refcnt, 1); in bpfopen()
768 BPF_PID_REFRESH(d, td); in bpfopen()
770 mac_bpfdesc_init(d); in bpfopen()
771 mac_bpfdesc_create(td->td_ucred, d); in bpfopen()
773 mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); in bpfopen()
774 callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); in bpfopen()
775 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); in bpfopen()
778 d->bd_pcp = 0; in bpfopen()
789 struct bpf_d *d; in bpfread() local
794 error = devfs_get_cdevpriv((void **)&d); in bpfread()
802 if (uio->uio_resid != d->bd_bufsize) in bpfread()
807 BPFD_LOCK(d); in bpfread()
808 BPF_PID_REFRESH_CUR(d); in bpfread()
809 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { in bpfread()
810 BPFD_UNLOCK(d); in bpfread()
813 if (d->bd_state == BPF_WAITING) in bpfread()
814 callout_stop(&d->bd_callout); in bpfread()
815 timed_out = (d->bd_state == BPF_TIMED_OUT); in bpfread()
816 d->bd_state = BPF_IDLE; in bpfread()
817 while (d->bd_flags & BPFD_HBUF_INUSE) { in bpfread()
818 error = mtx_sleep(&d->bd_hbuf, &d->bd_lock, PRINET | PCATCH, in bpfread()
821 BPFD_UNLOCK(d); in bpfread()
830 while (d->bd_hbuf == NULL) { in bpfread()
831 if (d->bd_slen != 0) { in bpfread()
836 if ((d->bd_flags & BPFD_IMMEDIATE) || non_block || in bpfread()
843 ROTATE_BUFFERS(d); in bpfread()
854 if (d->bd_bif == NULL) { in bpfread()
855 BPFD_UNLOCK(d); in bpfread()
860 BPFD_UNLOCK(d); in bpfread()
863 error = msleep(d, &d->bd_lock, PRINET | PCATCH, in bpfread()
864 "bpf", d->bd_rtout); in bpfread()
866 BPFD_UNLOCK(d); in bpfread()
875 if (d->bd_hbuf) in bpfread()
883 if (d->bd_slen == 0) { in bpfread()
884 BPFD_UNLOCK(d); in bpfread()
887 ROTATE_BUFFERS(d); in bpfread()
894 d->bd_flags |= BPFD_HBUF_INUSE; in bpfread()
895 BPFD_UNLOCK(d); in bpfread()
905 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); in bpfread()
907 BPFD_LOCK(d); in bpfread()
908 if (d->bd_flags & BPFD_HBUF_INUSE) { in bpfread()
909 KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf")); in bpfread()
910 d->bd_fbuf = d->bd_hbuf; in bpfread()
911 d->bd_hbuf = NULL; in bpfread()
912 d->bd_hlen = 0; in bpfread()
913 bpf_buf_reclaimed(d); in bpfread()
914 d->bd_flags &= ~BPFD_HBUF_INUSE; in bpfread()
915 wakeup(&d->bd_hbuf); in bpfread()
917 BPFD_UNLOCK(d); in bpfread()
926 bpf_wakeup(struct bpf_d *d) in bpf_wakeup() argument
929 BPFD_LOCK_ASSERT(d); in bpf_wakeup()
930 if (d->bd_state == BPF_WAITING) { in bpf_wakeup()
931 callout_stop(&d->bd_callout); in bpf_wakeup()
932 d->bd_state = BPF_IDLE; in bpf_wakeup()
934 wakeup(d); in bpf_wakeup()
935 if ((d->bd_flags & BPFD_ASYNC) && d->bd_sig && d->bd_sigio) in bpf_wakeup()
936 pgsigio(&d->bd_sigio, d->bd_sig, 0); in bpf_wakeup()
938 selwakeuppri(&d->bd_sel, PRINET); in bpf_wakeup()
939 KNOTE_LOCKED(&d->bd_sel.si_note, 0); in bpf_wakeup()
945 struct bpf_d *d = (struct bpf_d *)arg; in bpf_timed_out() local
947 BPFD_LOCK_ASSERT(d); in bpf_timed_out()
949 if (callout_pending(&d->bd_callout) || in bpf_timed_out()
950 !callout_active(&d->bd_callout)) in bpf_timed_out()
952 if (d->bd_state == BPF_WAITING) { in bpf_timed_out()
953 d->bd_state = BPF_TIMED_OUT; in bpf_timed_out()
954 if (d->bd_slen != 0) in bpf_timed_out()
955 bpf_wakeup(d); in bpf_timed_out()
960 bpf_ready(struct bpf_d *d) in bpf_ready() argument
963 BPFD_LOCK_ASSERT(d); in bpf_ready()
965 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) in bpf_ready()
967 if (((d->bd_flags & BPFD_IMMEDIATE) || d->bd_state == BPF_TIMED_OUT) && in bpf_ready()
968 d->bd_slen != 0) in bpf_ready()
978 struct bpf_d *d; in bpfwrite() local
983 error = devfs_get_cdevpriv((void **)&d); in bpfwrite()
990 BPFD_LOCK(d); in bpfwrite()
991 if ((bp = d->bd_bif) == NULL) in bpfwrite()
996 BPFD_UNLOCK(d); in bpfwrite()
997 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1000 bpfd_ref(d); in bpfwrite()
1001 BPFD_UNLOCK(d); in bpfwrite()
1016 if (bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len) == 0) { in bpfwrite()
1021 if (d->bd_flags & BPFD_FEEDBACK) { in bpfwrite()
1024 if (d->bd_direction == BPF_D_INOUT) in bpfwrite()
1030 if (d->bd_pcp != 0) in bpfwrite()
1031 (void)vlan_set_pcp(m, d->bd_pcp); in bpfwrite()
1033 BPFD_LOCK(d); in bpfwrite()
1035 mac_bpfdesc_create_mbuf(d, m); in bpfwrite()
1037 mac_bpfdesc_create_mbuf(d, mc); in bpfwrite()
1043 if (__predict_false(d->bd_bif != bp)) { in bpfwrite()
1044 BPFD_UNLOCK(d); in bpfwrite()
1049 BPFD_UNLOCK(d); in bpfwrite()
1052 error = bp->bif_methods->bif_write(bp->bif_softc, m, mc, d->bd_flags); in bpfwrite()
1055 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1057 counter_u64_add(d->bd_wfcount, 1); in bpfwrite()
1058 bpfd_rele(d); in bpfwrite()
1063 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1064 bpfd_rele(d); in bpfwrite()
1077 reset_d(struct bpf_d *d) in reset_d() argument
1080 BPFD_LOCK_ASSERT(d); in reset_d()
1082 while (d->bd_flags & BPFD_HBUF_INUSE) in reset_d()
1083 mtx_sleep(&d->bd_hbuf, &d->bd_lock, PRINET, "bd_hbuf", 0); in reset_d()
1084 if ((d->bd_hbuf != NULL) && in reset_d()
1085 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { in reset_d()
1087 d->bd_fbuf = d->bd_hbuf; in reset_d()
1088 d->bd_hbuf = NULL; in reset_d()
1089 d->bd_hlen = 0; in reset_d()
1090 bpf_buf_reclaimed(d); in reset_d()
1092 if (bpf_canwritebuf(d)) in reset_d()
1093 d->bd_slen = 0; in reset_d()
1094 counter_u64_zero(d->bd_rcount); in reset_d()
1095 counter_u64_zero(d->bd_dcount); in reset_d()
1096 counter_u64_zero(d->bd_fcount); in reset_d()
1097 counter_u64_zero(d->bd_wcount); in reset_d()
1098 counter_u64_zero(d->bd_wfcount); in reset_d()
1099 counter_u64_zero(d->bd_wdcount); in reset_d()
1100 counter_u64_zero(d->bd_zcopy); in reset_d()
1140 struct bpf_d *d; in bpfioctl() local
1143 error = devfs_get_cdevpriv((void **)&d); in bpfioctl()
1150 BPFD_LOCK(d); in bpfioctl()
1151 BPF_PID_REFRESH(d, td); in bpfioctl()
1152 if (d->bd_state == BPF_WAITING) in bpfioctl()
1153 callout_stop(&d->bd_callout); in bpfioctl()
1154 d->bd_state = BPF_IDLE; in bpfioctl()
1155 BPFD_UNLOCK(d); in bpfioctl()
1157 if (d->bd_flags & BPFD_LOCKED) { in bpfioctl()
1205 BPFD_LOCK(d); in bpfioctl()
1206 d->bd_compat32 = 1; in bpfioctl()
1207 BPFD_UNLOCK(d); in bpfioctl()
1225 BPFD_LOCK(d); in bpfioctl()
1226 n = d->bd_slen; in bpfioctl()
1227 while (d->bd_flags & BPFD_HBUF_INUSE) in bpfioctl()
1228 mtx_sleep(&d->bd_hbuf, &d->bd_lock, in bpfioctl()
1230 if (d->bd_hbuf) in bpfioctl()
1231 n += d->bd_hlen; in bpfioctl()
1232 BPFD_UNLOCK(d); in bpfioctl()
1248 BPFD_LOCK(d); in bpfioctl()
1249 *(u_int *)addr = d->bd_bufsize; in bpfioctl()
1250 BPFD_UNLOCK(d); in bpfioctl()
1257 error = bpf_ioctl_sblen(d, (u_int *)addr); in bpfioctl()
1271 error = bpf_setf(d, (struct bpf_program *)addr, cmd); in bpfioctl()
1278 BPFD_LOCK(d); in bpfioctl()
1279 reset_d(d); in bpfioctl()
1280 BPFD_UNLOCK(d); in bpfioctl()
1288 if (d->bd_bif == NULL) { in bpfioctl()
1293 } else if (d->bd_promisc == 0) { in bpfioctl()
1294 struct bpf_if *bp = d->bd_bif; in bpfioctl()
1297 d->bd_promisc = 1; in bpfioctl()
1307 if (d->bd_bif == NULL) in bpfioctl()
1310 *(u_int *)addr = d->bd_bif->bif_dlt; in bpfioctl()
1327 if (d->bd_bif == NULL) in bpfioctl()
1330 error = bpf_getdltlist(d, &dltlist); in bpfioctl()
1341 if (d->bd_bif == NULL) in bpfioctl()
1344 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); in bpfioctl()
1353 if (d->bd_bif == NULL) in bpfioctl()
1356 error = bpf_setdlt(d, *(u_int *)addr); in bpfioctl()
1365 if (d->bd_bif == NULL) in bpfioctl()
1368 struct bpf_if *const bp = d->bd_bif; in bpfioctl()
1390 BPFD_LOCK(d); in bpfioctl()
1391 if (d->bd_bufmode == BPF_BUFMODE_BUFFER && in bpfioctl()
1392 d->bd_sbuf == NULL) { in bpfioctl()
1395 size = d->bd_bufsize; in bpfioctl()
1396 BPFD_UNLOCK(d); in bpfioctl()
1397 error = bpf_buffer_ioctl_sblen(d, &size); in bpfioctl()
1401 BPFD_UNLOCK(d); in bpfioctl()
1412 error = bpf_attachd(d, bp); in bpfioctl()
1445 d->bd_rtout = tvtohz(tv) - 1; in bpfioctl()
1468 tv->tv_sec = d->bd_rtout / hz; in bpfioctl()
1469 tv->tv_usec = (d->bd_rtout % hz) * tick; in bpfioctl()
1489 bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount); in bpfioctl()
1490 bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount); in bpfioctl()
1498 BPFD_LOCK(d); in bpfioctl()
1499 d->bd_flags |= *(u_int *)addr ? BPFD_IMMEDIATE : 0; in bpfioctl()
1500 BPFD_UNLOCK(d); in bpfioctl()
1516 BPFD_LOCK(d); in bpfioctl()
1517 *(u_int *)addr = d->bd_flags & BPFD_HDRCMPLT ? 1 : 0; in bpfioctl()
1518 BPFD_UNLOCK(d); in bpfioctl()
1525 BPFD_LOCK(d); in bpfioctl()
1526 d->bd_flags |= *(u_int *)addr ? BPFD_HDRCMPLT : 0; in bpfioctl()
1527 BPFD_UNLOCK(d); in bpfioctl()
1534 BPFD_LOCK(d); in bpfioctl()
1535 *(u_int *)addr = d->bd_direction; in bpfioctl()
1536 BPFD_UNLOCK(d); in bpfioctl()
1551 BPFD_LOCK(d); in bpfioctl()
1552 d->bd_direction = direction; in bpfioctl()
1553 BPFD_UNLOCK(d); in bpfioctl()
1565 BPFD_LOCK(d); in bpfioctl()
1566 *(u_int *)addr = d->bd_tstamp; in bpfioctl()
1567 BPFD_UNLOCK(d); in bpfioctl()
1579 d->bd_tstamp = func; in bpfioctl()
1586 BPFD_LOCK(d); in bpfioctl()
1587 d->bd_flags |= *(u_int *)addr ? BPFD_FEEDBACK : 0; in bpfioctl()
1588 BPFD_UNLOCK(d); in bpfioctl()
1592 BPFD_LOCK(d); in bpfioctl()
1593 d->bd_flags |= BPFD_LOCKED; in bpfioctl()
1594 BPFD_UNLOCK(d); in bpfioctl()
1601 BPFD_LOCK(d); in bpfioctl()
1602 d->bd_flags |= *(u_int *)addr ? BPFD_ASYNC : 0; in bpfioctl()
1603 BPFD_UNLOCK(d); in bpfioctl()
1611 error = fsetown(*(int *)addr, &d->bd_sigio); in bpfioctl()
1615 BPFD_LOCK(d); in bpfioctl()
1616 *(int *)addr = fgetown(&d->bd_sigio); in bpfioctl()
1617 BPFD_UNLOCK(d); in bpfioctl()
1622 error = fsetown(-(*(int *)addr), &d->bd_sigio); in bpfioctl()
1627 *(int *)addr = -fgetown(&d->bd_sigio); in bpfioctl()
1639 BPFD_LOCK(d); in bpfioctl()
1640 d->bd_sig = sig; in bpfioctl()
1641 BPFD_UNLOCK(d); in bpfioctl()
1646 BPFD_LOCK(d); in bpfioctl()
1647 *(u_int *)addr = d->bd_sig; in bpfioctl()
1648 BPFD_UNLOCK(d); in bpfioctl()
1652 BPFD_LOCK(d); in bpfioctl()
1653 *(u_int *)addr = d->bd_bufmode; in bpfioctl()
1654 BPFD_UNLOCK(d); in bpfioctl()
1679 BPFD_LOCK(d); in bpfioctl()
1680 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || in bpfioctl()
1681 d->bd_fbuf != NULL || d->bd_bif != NULL) { in bpfioctl()
1682 BPFD_UNLOCK(d); in bpfioctl()
1686 d->bd_bufmode = *(u_int *)addr; in bpfioctl()
1687 BPFD_UNLOCK(d); in bpfioctl()
1691 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); in bpfioctl()
1695 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1699 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1711 d->bd_pcp = pcp; in bpfioctl()
1773 * Set d's packet filter program to fp. If this file already has a filter,
1780 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) in bpf_setf() argument
1853 BPFD_LOCK(d); in bpf_setf()
1856 if (d->bd_wfilter != NULL) { in bpf_setf()
1857 fcode = __containerof((void *)d->bd_wfilter, in bpf_setf()
1863 d->bd_wfilter = filter; in bpf_setf()
1865 if (d->bd_rfilter != NULL) { in bpf_setf()
1866 fcode = __containerof((void *)d->bd_rfilter, in bpf_setf()
1869 fcode->func = d->bd_bfilter; in bpf_setf()
1872 d->bd_rfilter = filter; in bpf_setf()
1874 d->bd_bfilter = jfunc; in bpf_setf()
1877 reset_d(d); in bpf_setf()
1879 if (bpf_check_upgrade(cmd, d, filter, flen) != 0) { in bpf_setf()
1882 * specifying interface. In this case just mark d in bpf_setf()
1885 d->bd_writer = 0; in bpf_setf()
1886 if (d->bd_bif != NULL) { in bpf_setf()
1891 CK_LIST_REMOVE(d, bd_next); in bpf_setf()
1892 CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist, in bpf_setf()
1893 d, bd_next); in bpf_setf()
1895 "%s: upgrade required by pid %d", in bpf_setf()
1896 __func__, d->bd_pid); in bpf_setf()
1901 BPFD_UNLOCK(d); in bpf_setf()
1907 bif_attachd(d->bd_bif); in bpf_setf()
1919 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) in bpf_attachd() argument
1929 switch (d->bd_bufmode) { in bpf_attachd()
1932 if (d->bd_sbuf == NULL) in bpf_attachd()
1937 panic("%s: bufmode %d", __func__, d->bd_bufmode); in bpf_attachd()
1940 if (bp == d->bd_bif) { in bpf_attachd()
1941 BPFD_LOCK(d); in bpf_attachd()
1942 reset_d(d); in bpf_attachd()
1943 BPFD_UNLOCK(d); in bpf_attachd()
1945 } else if (d->bd_bif != NULL) in bpf_attachd()
1946 bpf_detachd(d, false); in bpf_attachd()
1951 writer = V_bpf_optimize_writers || (d->bd_writer > 0); in bpf_attachd()
1954 * Point d at bp, and add d to the interface's list. in bpf_attachd()
1957 * we can delay adding d to the list of active listeners until in bpf_attachd()
1960 BPFD_LOCK(d); in bpf_attachd()
1965 d->bd_bif = bp; in bpf_attachd()
1968 CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); in bpf_attachd()
1975 d->bd_writer = 2; in bpf_attachd()
1977 CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); in bpf_attachd()
1979 reset_d(d); in bpf_attachd()
1982 bpf_wakeup(d); in bpf_attachd()
1984 BPFD_UNLOCK(d); in bpf_attachd()
1986 CTR3(KTR_NET, "%s: called by pid %d, adding to %s list", in bpf_attachd()
1987 __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); in bpf_attachd()
2004 struct bpf_d *d; in bpfpoll() local
2007 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) in bpfpoll()
2015 BPFD_LOCK(d); in bpfpoll()
2016 BPF_PID_REFRESH(d, td); in bpfpoll()
2018 if (bpf_ready(d)) in bpfpoll()
2021 selrecord(td, &d->bd_sel); in bpfpoll()
2023 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in bpfpoll()
2024 callout_reset(&d->bd_callout, d->bd_rtout, in bpfpoll()
2025 bpf_timed_out, d); in bpfpoll()
2026 d->bd_state = BPF_WAITING; in bpfpoll()
2030 BPFD_UNLOCK(d); in bpfpoll()
2041 struct bpf_d *d; in bpfkqfilter() local
2043 if (devfs_get_cdevpriv((void **)&d) != 0) in bpfkqfilter()
2062 BPFD_LOCK(d); in bpfkqfilter()
2063 BPF_PID_REFRESH_CUR(d); in bpfkqfilter()
2064 kn->kn_hook = d; in bpfkqfilter()
2065 knlist_add(&d->bd_sel.si_note, kn, 1); in bpfkqfilter()
2066 BPFD_UNLOCK(d); in bpfkqfilter()
2074 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfdetach() local
2076 knlist_remove(&d->bd_sel.si_note, kn, 0); in filt_bpfdetach()
2082 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfread() local
2085 BPFD_LOCK_ASSERT(d); in filt_bpfread()
2086 ready = bpf_ready(d); in filt_bpfread()
2088 kn->kn_data = d->bd_slen; in filt_bpfread()
2092 if (!(d->bd_flags & BPFD_HBUF_INUSE) && d->bd_hbuf) in filt_bpfread()
2093 kn->kn_data += d->bd_hlen; in filt_bpfread()
2094 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in filt_bpfread()
2095 callout_reset(&d->bd_callout, d->bd_rtout, in filt_bpfread()
2096 bpf_timed_out, d); in filt_bpfread()
2097 d->bd_state = BPF_WAITING; in filt_bpfread()
2106 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfwrite() local
2108 BPFD_LOCK_ASSERT(d); in filt_bpfwrite()
2110 if (d->bd_bif == NULL) { in filt_bpfwrite()
2114 kn->kn_data = bif_wrsize(d->bd_bif); in filt_bpfwrite()
2178 struct bpf_d *d; in bpf_tap() local
2187 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_tap()
2188 counter_u64_add(d->bd_rcount, 1); in bpf_tap()
2196 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_tap()
2201 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); in bpf_tap()
2206 BPFD_LOCK(d); in bpf_tap()
2207 counter_u64_add(d->bd_fcount, 1); in bpf_tap()
2208 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_tap()
2209 gottime = bpf_gettime(&bt, d->bd_tstamp, in bpf_tap()
2212 if (bif_mac_check_receive(bp, d) == 0) in bpf_tap()
2214 catchpacket(d, pkt, pktlen, slen, in bpf_tap()
2216 BPFD_UNLOCK(d); in bpf_tap()
2238 struct bpf_d *d; in bpf_mtap() local
2255 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap()
2256 if (bpf_chkdir(d, m)) in bpf_mtap()
2258 counter_u64_add(d->bd_rcount, 1); in bpf_mtap()
2260 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_mtap()
2267 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); in bpf_mtap()
2269 BPFD_LOCK(d); in bpf_mtap()
2271 counter_u64_add(d->bd_fcount, 1); in bpf_mtap()
2272 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap()
2273 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap()
2275 if (bif_mac_check_receive(bp, d) == 0) in bpf_mtap()
2277 catchpacket(d, (u_char *)m, pktlen, slen, in bpf_mtap()
2279 BPFD_UNLOCK(d); in bpf_mtap()
2304 struct bpf_d *d; in bpf_mtap2() local
2329 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap2()
2330 if (bpf_chkdir(d, m)) in bpf_mtap2()
2332 counter_u64_add(d->bd_rcount, 1); in bpf_mtap2()
2333 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); in bpf_mtap2()
2335 BPFD_LOCK(d); in bpf_mtap2()
2337 counter_u64_add(d->bd_fcount, 1); in bpf_mtap2()
2338 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap2()
2339 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap2()
2341 if (bif_mac_check_receive(bp, d) == 0) in bpf_mtap2()
2343 catchpacket(d, (u_char *)&mb, pktlen, slen, in bpf_mtap2()
2345 BPFD_UNLOCK(d); in bpf_mtap2()
2366 bpf_hdrlen(struct bpf_d *d) in bpf_hdrlen() argument
2370 hdrlen = d->bd_bif->bif_hdrlen; in bpf_hdrlen()
2372 if (d->bd_tstamp == BPF_T_NONE || in bpf_hdrlen()
2373 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) in bpf_hdrlen()
2375 if (d->bd_compat32) in bpf_hdrlen()
2384 if (d->bd_compat32) in bpf_hdrlen()
2390 return (hdrlen - d->bd_bif->bif_hdrlen); in bpf_hdrlen()
2432 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, in catchpacket() argument
2449 BPFD_LOCK_ASSERT(d); in catchpacket()
2450 if (d->bd_bif == NULL) { in catchpacket()
2452 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2463 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { in catchpacket()
2464 d->bd_fbuf = d->bd_hbuf; in catchpacket()
2465 d->bd_hbuf = NULL; in catchpacket()
2466 d->bd_hlen = 0; in catchpacket()
2467 bpf_buf_reclaimed(d); in catchpacket()
2476 hdrlen = bpf_hdrlen(d); in catchpacket()
2478 if (totlen > d->bd_bufsize) in catchpacket()
2479 totlen = d->bd_bufsize; in catchpacket()
2490 if (d->bd_compat32) in catchpacket()
2491 curlen = BPF_WORDALIGN32(d->bd_slen); in catchpacket()
2494 curlen = BPF_WORDALIGN(d->bd_slen); in catchpacket()
2495 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { in catchpacket()
2496 if (d->bd_fbuf == NULL) { in catchpacket()
2502 bpf_buffull(d); in catchpacket()
2503 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2506 KASSERT(!(d->bd_flags & BPFD_HBUF_INUSE), in catchpacket()
2508 ROTATE_BUFFERS(d); in catchpacket()
2512 if ((d->bd_flags & BPFD_IMMEDIATE) || in catchpacket()
2513 d->bd_state == BPF_TIMED_OUT) { in catchpacket()
2521 pad = curlen - d->bd_slen; in catchpacket()
2523 ("%s: invalid pad byte count %d", __func__, pad)); in catchpacket()
2526 bpf_append_bytes(d, d->bd_sbuf, d->bd_slen, zeroes, in catchpacket()
2532 tstype = d->bd_tstamp; in catchpacket()
2540 if (d->bd_compat32) { in catchpacket()
2549 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, in catchpacket()
2562 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, in catchpacket()
2578 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); in catchpacket()
2586 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); in catchpacket()
2587 d->bd_slen = curlen + totlen; in catchpacket()
2590 bpf_wakeup(d); in catchpacket()
2600 struct bpf_d *d; in bpfd_free() local
2608 d = __containerof(ctx, struct bpf_d, epoch_ctx); in bpfd_free()
2609 bpf_free(d); in bpfd_free()
2610 if (d->bd_rfilter != NULL) { in bpfd_free()
2611 p = __containerof((void *)d->bd_rfilter, in bpfd_free()
2614 p->func = d->bd_bfilter; in bpfd_free()
2618 if (d->bd_wfilter != NULL) { in bpfd_free()
2619 p = __containerof((void *)d->bd_wfilter, in bpfd_free()
2627 mtx_destroy(&d->bd_lock); in bpfd_free()
2628 counter_u64_free(d->bd_rcount); in bpfd_free()
2629 counter_u64_free(d->bd_dcount); in bpfd_free()
2630 counter_u64_free(d->bd_fcount); in bpfd_free()
2631 counter_u64_free(d->bd_wcount); in bpfd_free()
2632 counter_u64_free(d->bd_wfcount); in bpfd_free()
2633 counter_u64_free(d->bd_wdcount); in bpfd_free()
2634 counter_u64_free(d->bd_zcopy); in bpfd_free()
2635 free(d, M_BPF); in bpfd_free()
2674 struct bpf_d *d; in bpf_ifdetach() local
2683 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_ifdetach()
2684 bpf_detachd(d, true); in bpf_ifdetach()
2688 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_ifdetach()
2689 bpf_detachd(d, true); in bpf_ifdetach()
2704 struct bpf_d *d; in bpf_detach() local
2709 CTR3(KTR_NET, "%s: sheduling free for encap %d for bp %p", in bpf_detach()
2713 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_detach()
2714 bpf_detachd(d, true); in bpf_detach()
2718 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_detach()
2719 bpf_detachd(d, true); in bpf_detach()
2753 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) in bpf_getdltlist() argument
2762 name = d->bd_bif->bif_name; in bpf_getdltlist()
2799 bpf_setdlt(struct bpf_d *d, u_int dlt) in bpf_setdlt() argument
2806 MPASS(d->bd_bif != NULL); in bpf_setdlt()
2812 if (d->bd_bif->bif_dlt == dlt) in bpf_setdlt()
2815 name = d->bd_bif->bif_name; in bpf_setdlt()
2823 opromisc = d->bd_promisc; in bpf_setdlt()
2824 bpf_detachd(d, false); in bpf_setdlt()
2825 bpf_attachd(d, bp); in bpf_setdlt()
2829 printf("%s: bif_promisc on %s failed (%d)\n", in bpf_setdlt()
2832 d->bd_promisc = 1; in bpf_setdlt()
2881 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) in bpfstats_fill_xbpf() argument
2885 bzero(d, sizeof(*d)); in bpfstats_fill_xbpf()
2886 d->bd_structsize = sizeof(*d); in bpfstats_fill_xbpf()
2887 d->bd_immediate = bd->bd_flags & BPFD_IMMEDIATE ? 1 : 0; in bpfstats_fill_xbpf()
2888 d->bd_promisc = bd->bd_promisc; in bpfstats_fill_xbpf()
2889 d->bd_hdrcmplt = bd->bd_flags & BPFD_HDRCMPLT ? 1 : 0; in bpfstats_fill_xbpf()
2890 d->bd_direction = bd->bd_direction; in bpfstats_fill_xbpf()
2891 d->bd_feedback = bd->bd_flags & BPFD_FEEDBACK ? 1 : 0; in bpfstats_fill_xbpf()
2892 d->bd_async = bd->bd_flags & BPFD_ASYNC ? 1 : 0; in bpfstats_fill_xbpf()
2893 d->bd_rcount = counter_u64_fetch(bd->bd_rcount); in bpfstats_fill_xbpf()
2894 d->bd_dcount = counter_u64_fetch(bd->bd_dcount); in bpfstats_fill_xbpf()
2895 d->bd_fcount = counter_u64_fetch(bd->bd_fcount); in bpfstats_fill_xbpf()
2896 d->bd_sig = bd->bd_sig; in bpfstats_fill_xbpf()
2897 d->bd_slen = bd->bd_slen; in bpfstats_fill_xbpf()
2898 d->bd_hlen = bd->bd_hlen; in bpfstats_fill_xbpf()
2899 d->bd_bufsize = bd->bd_bufsize; in bpfstats_fill_xbpf()
2900 d->bd_pid = bd->bd_pid; in bpfstats_fill_xbpf()
2901 strlcpy(d->bd_ifname, bd->bd_bif->bif_name, sizeof(d->bd_ifname)); in bpfstats_fill_xbpf()
2902 d->bd_locked = bd->bd_flags & BPFD_LOCKED ? 1 : 0; in bpfstats_fill_xbpf()
2903 d->bd_wcount = counter_u64_fetch(bd->bd_wcount); in bpfstats_fill_xbpf()
2904 d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount); in bpfstats_fill_xbpf()
2905 d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount); in bpfstats_fill_xbpf()
2906 d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy); in bpfstats_fill_xbpf()
2907 d->bd_bufmode = bd->bd_bufmode; in bpfstats_fill_xbpf()
3018 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) in bpf_mtap2() argument