Lines Matching refs:d
328 bpfd_ref(struct bpf_d *d) in bpfd_ref() argument
331 refcount_acquire(&d->bd_refcnt); in bpfd_ref()
335 bpfd_rele(struct bpf_d *d) in bpfd_rele() argument
338 if (!refcount_release(&d->bd_refcnt)) in bpfd_rele()
340 NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx); in bpfd_rele()
370 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_bytes() argument
374 BPFD_LOCK_ASSERT(d); in bpf_append_bytes()
376 switch (d->bd_bufmode) { in bpf_append_bytes()
378 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
381 counter_u64_add(d->bd_zcopy, 1); in bpf_append_bytes()
382 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); in bpf_append_bytes()
390 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, in bpf_append_mbuf() argument
394 BPFD_LOCK_ASSERT(d); in bpf_append_mbuf()
396 switch (d->bd_bufmode) { in bpf_append_mbuf()
398 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
401 counter_u64_add(d->bd_zcopy, 1); in bpf_append_mbuf()
402 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); in bpf_append_mbuf()
413 bpf_buf_reclaimed(struct bpf_d *d) in bpf_buf_reclaimed() argument
416 BPFD_LOCK_ASSERT(d); in bpf_buf_reclaimed()
418 switch (d->bd_bufmode) { in bpf_buf_reclaimed()
423 bpf_zerocopy_buf_reclaimed(d); in bpf_buf_reclaimed()
437 bpf_canfreebuf(struct bpf_d *d) in bpf_canfreebuf() argument
440 BPFD_LOCK_ASSERT(d); in bpf_canfreebuf()
442 switch (d->bd_bufmode) { in bpf_canfreebuf()
444 return (bpf_zerocopy_canfreebuf(d)); in bpf_canfreebuf()
455 bpf_canwritebuf(struct bpf_d *d) in bpf_canwritebuf() argument
457 BPFD_LOCK_ASSERT(d); in bpf_canwritebuf()
459 switch (d->bd_bufmode) { in bpf_canwritebuf()
461 return (bpf_zerocopy_canwritebuf(d)); in bpf_canwritebuf()
472 bpf_buffull(struct bpf_d *d) in bpf_buffull() argument
475 BPFD_LOCK_ASSERT(d); in bpf_buffull()
477 switch (d->bd_bufmode) { in bpf_buffull()
479 bpf_zerocopy_buffull(d); in bpf_buffull()
488 bpf_bufheld(struct bpf_d *d) in bpf_bufheld() argument
491 BPFD_LOCK_ASSERT(d); in bpf_bufheld()
493 switch (d->bd_bufmode) { in bpf_bufheld()
495 bpf_zerocopy_bufheld(d); in bpf_bufheld()
501 bpf_free(struct bpf_d *d) in bpf_free() argument
504 switch (d->bd_bufmode) { in bpf_free()
506 return (bpf_buffer_free(d)); in bpf_free()
509 return (bpf_zerocopy_free(d)); in bpf_free()
517 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) in bpf_uiomove() argument
520 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_uiomove()
522 return (bpf_buffer_uiomove(d, buf, len, uio)); in bpf_uiomove()
526 bpf_ioctl_sblen(struct bpf_d *d, u_int *i) in bpf_ioctl_sblen() argument
529 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) in bpf_ioctl_sblen()
531 return (bpf_buffer_ioctl_sblen(d, i)); in bpf_ioctl_sblen()
535 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) in bpf_ioctl_getzmax() argument
538 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_getzmax()
540 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); in bpf_ioctl_getzmax()
544 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_rotzbuf() argument
547 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_rotzbuf()
549 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); in bpf_ioctl_rotzbuf()
553 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) in bpf_ioctl_setzbuf() argument
556 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) in bpf_ioctl_setzbuf()
558 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); in bpf_ioctl_setzbuf()
566 struct sockaddr *sockp, int *hdrlen, struct bpf_d *d) in bpf_movein() argument
661 slen = bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len); in bpf_movein()
678 if (d->bd_hdrcmplt == 0) { in bpf_movein()
721 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) in bpf_attachd() argument
731 op_w = V_bpf_optimize_writers || d->bd_writer; in bpf_attachd()
733 if (d->bd_bif != NULL) in bpf_attachd()
734 bpf_detachd_locked(d, false); in bpf_attachd()
743 BPFD_LOCK(d); in bpf_attachd()
748 d->bd_bif = bp; in bpf_attachd()
751 CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); in bpf_attachd()
758 d->bd_writer = 2; in bpf_attachd()
760 CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); in bpf_attachd()
762 reset_d(d); in bpf_attachd()
765 bpf_wakeup(d); in bpf_attachd()
767 BPFD_UNLOCK(d); in bpf_attachd()
771 __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); in bpf_attachd()
781 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, in bpf_check_upgrade() argument
789 if (d->bd_writer == 0 || fcode == NULL) in bpf_check_upgrade()
821 if (--d->bd_writer == 0) { in bpf_check_upgrade()
834 __func__, d->bd_pid, d->bd_writer, in bpf_check_upgrade()
844 bpf_detachd(struct bpf_d *d) in bpf_detachd() argument
847 bpf_detachd_locked(d, false); in bpf_detachd()
852 bpf_detachd_locked(struct bpf_d *d, bool detached_ifp) in bpf_detachd_locked() argument
859 CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); in bpf_detachd_locked()
862 if ((bp = d->bd_bif) == NULL) in bpf_detachd_locked()
865 BPFD_LOCK(d); in bpf_detachd_locked()
867 CK_LIST_REMOVE(d, bd_next); in bpf_detachd_locked()
869 error = d->bd_writer; in bpf_detachd_locked()
871 d->bd_bif = NULL; in bpf_detachd_locked()
877 bpf_wakeup(d); in bpf_detachd_locked()
879 BPFD_UNLOCK(d); in bpf_detachd_locked()
890 if (d->bd_promisc && !detached_ifp) { in bpf_detachd_locked()
891 d->bd_promisc = 0; in bpf_detachd_locked()
916 struct bpf_d *d = data; in bpf_dtor() local
918 BPFD_LOCK(d); in bpf_dtor()
919 if (d->bd_state == BPF_WAITING) in bpf_dtor()
920 callout_stop(&d->bd_callout); in bpf_dtor()
921 d->bd_state = BPF_IDLE; in bpf_dtor()
922 BPFD_UNLOCK(d); in bpf_dtor()
923 funsetown(&d->bd_sigio); in bpf_dtor()
924 bpf_detachd(d); in bpf_dtor()
926 mac_bpfdesc_destroy(d); in bpf_dtor()
928 seldrain(&d->bd_sel); in bpf_dtor()
929 knlist_destroy(&d->bd_sel.si_note); in bpf_dtor()
930 callout_drain(&d->bd_callout); in bpf_dtor()
931 bpfd_rele(d); in bpf_dtor()
942 struct bpf_d *d; in bpfopen() local
945 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); in bpfopen()
946 error = devfs_set_cdevpriv(d, bpf_dtor); in bpfopen()
948 free(d, M_BPF); in bpfopen()
953 d->bd_rcount = counter_u64_alloc(M_WAITOK); in bpfopen()
954 d->bd_dcount = counter_u64_alloc(M_WAITOK); in bpfopen()
955 d->bd_fcount = counter_u64_alloc(M_WAITOK); in bpfopen()
956 d->bd_wcount = counter_u64_alloc(M_WAITOK); in bpfopen()
957 d->bd_wfcount = counter_u64_alloc(M_WAITOK); in bpfopen()
958 d->bd_wdcount = counter_u64_alloc(M_WAITOK); in bpfopen()
959 d->bd_zcopy = counter_u64_alloc(M_WAITOK); in bpfopen()
966 bpf_buffer_init(d); in bpfopen()
968 d->bd_writer = 2; in bpfopen()
969 d->bd_hbuf_in_use = 0; in bpfopen()
970 d->bd_bufmode = BPF_BUFMODE_BUFFER; in bpfopen()
971 d->bd_sig = SIGIO; in bpfopen()
972 d->bd_direction = BPF_D_INOUT; in bpfopen()
973 refcount_init(&d->bd_refcnt, 1); in bpfopen()
974 BPF_PID_REFRESH(d, td); in bpfopen()
976 mac_bpfdesc_init(d); in bpfopen()
977 mac_bpfdesc_create(td->td_ucred, d); in bpfopen()
979 mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); in bpfopen()
980 callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); in bpfopen()
981 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); in bpfopen()
984 d->bd_pcp = 0; in bpfopen()
995 struct bpf_d *d; in bpfread() local
1000 error = devfs_get_cdevpriv((void **)&d); in bpfread()
1008 if (uio->uio_resid != d->bd_bufsize) in bpfread()
1013 BPFD_LOCK(d); in bpfread()
1014 BPF_PID_REFRESH_CUR(d); in bpfread()
1015 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { in bpfread()
1016 BPFD_UNLOCK(d); in bpfread()
1019 if (d->bd_state == BPF_WAITING) in bpfread()
1020 callout_stop(&d->bd_callout); in bpfread()
1021 timed_out = (d->bd_state == BPF_TIMED_OUT); in bpfread()
1022 d->bd_state = BPF_IDLE; in bpfread()
1023 while (d->bd_hbuf_in_use) { in bpfread()
1024 error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, in bpfread()
1027 BPFD_UNLOCK(d); in bpfread()
1036 while (d->bd_hbuf == NULL) { in bpfread()
1037 if (d->bd_slen != 0) { in bpfread()
1042 if (d->bd_immediate || non_block || timed_out) { in bpfread()
1048 ROTATE_BUFFERS(d); in bpfread()
1059 if (d->bd_bif == NULL) { in bpfread()
1060 BPFD_UNLOCK(d); in bpfread()
1065 BPFD_UNLOCK(d); in bpfread()
1068 error = msleep(d, &d->bd_lock, PRINET | PCATCH, in bpfread()
1069 "bpf", d->bd_rtout); in bpfread()
1071 BPFD_UNLOCK(d); in bpfread()
1080 if (d->bd_hbuf) in bpfread()
1088 if (d->bd_slen == 0) { in bpfread()
1089 BPFD_UNLOCK(d); in bpfread()
1092 ROTATE_BUFFERS(d); in bpfread()
1099 d->bd_hbuf_in_use = 1; in bpfread()
1100 BPFD_UNLOCK(d); in bpfread()
1110 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); in bpfread()
1112 BPFD_LOCK(d); in bpfread()
1113 KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf")); in bpfread()
1114 d->bd_fbuf = d->bd_hbuf; in bpfread()
1115 d->bd_hbuf = NULL; in bpfread()
1116 d->bd_hlen = 0; in bpfread()
1117 bpf_buf_reclaimed(d); in bpfread()
1118 d->bd_hbuf_in_use = 0; in bpfread()
1119 wakeup(&d->bd_hbuf_in_use); in bpfread()
1120 BPFD_UNLOCK(d); in bpfread()
1129 bpf_wakeup(struct bpf_d *d) in bpf_wakeup() argument
1132 BPFD_LOCK_ASSERT(d); in bpf_wakeup()
1133 if (d->bd_state == BPF_WAITING) { in bpf_wakeup()
1134 callout_stop(&d->bd_callout); in bpf_wakeup()
1135 d->bd_state = BPF_IDLE; in bpf_wakeup()
1137 wakeup(d); in bpf_wakeup()
1138 if (d->bd_async && d->bd_sig && d->bd_sigio) in bpf_wakeup()
1139 pgsigio(&d->bd_sigio, d->bd_sig, 0); in bpf_wakeup()
1141 selwakeuppri(&d->bd_sel, PRINET); in bpf_wakeup()
1142 KNOTE_LOCKED(&d->bd_sel.si_note, 0); in bpf_wakeup()
1148 struct bpf_d *d = (struct bpf_d *)arg; in bpf_timed_out() local
1150 BPFD_LOCK_ASSERT(d); in bpf_timed_out()
1152 if (callout_pending(&d->bd_callout) || in bpf_timed_out()
1153 !callout_active(&d->bd_callout)) in bpf_timed_out()
1155 if (d->bd_state == BPF_WAITING) { in bpf_timed_out()
1156 d->bd_state = BPF_TIMED_OUT; in bpf_timed_out()
1157 if (d->bd_slen != 0) in bpf_timed_out()
1158 bpf_wakeup(d); in bpf_timed_out()
1163 bpf_ready(struct bpf_d *d) in bpf_ready() argument
1166 BPFD_LOCK_ASSERT(d); in bpf_ready()
1168 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) in bpf_ready()
1170 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && in bpf_ready()
1171 d->bd_slen != 0) in bpf_ready()
1183 struct bpf_d *d; in bpfwrite() local
1188 error = devfs_get_cdevpriv((void **)&d); in bpfwrite()
1193 BPFD_LOCK(d); in bpfwrite()
1194 BPF_PID_REFRESH_CUR(d); in bpfwrite()
1195 counter_u64_add(d->bd_wcount, 1); in bpfwrite()
1196 if ((bp = d->bd_bif) == NULL) { in bpfwrite()
1218 bpfd_ref(d); in bpfwrite()
1220 BPFD_UNLOCK(d); in bpfwrite()
1223 &m, &dst, &hlen, d); in bpfwrite()
1226 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1227 bpfd_rele(d); in bpfwrite()
1231 BPFD_LOCK(d); in bpfwrite()
1237 if (d->bd_bif == NULL) { in bpfwrite()
1238 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1239 BPFD_UNLOCK(d); in bpfwrite()
1240 bpfd_rele(d); in bpfwrite()
1244 counter_u64_add(d->bd_wfcount, 1); in bpfwrite()
1245 if (d->bd_hdrcmplt) in bpfwrite()
1248 if (d->bd_feedback) { in bpfwrite()
1253 if (d->bd_direction == BPF_D_INOUT) in bpfwrite()
1264 mac_bpfdesc_create_mbuf(d, m); in bpfwrite()
1266 mac_bpfdesc_create_mbuf(d, mc); in bpfwrite()
1276 if (d->bd_pcp != 0) in bpfwrite()
1277 vlan_set_pcp(m, d->bd_pcp); in bpfwrite()
1281 BPFD_UNLOCK(d); in bpfwrite()
1284 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1294 bpfd_rele(d); in bpfwrite()
1298 counter_u64_add(d->bd_wdcount, 1); in bpfwrite()
1300 BPFD_UNLOCK(d); in bpfwrite()
1312 reset_d(struct bpf_d *d) in reset_d() argument
1315 BPFD_LOCK_ASSERT(d); in reset_d()
1317 while (d->bd_hbuf_in_use) in reset_d()
1318 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET, in reset_d()
1320 if ((d->bd_hbuf != NULL) && in reset_d()
1321 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { in reset_d()
1323 d->bd_fbuf = d->bd_hbuf; in reset_d()
1324 d->bd_hbuf = NULL; in reset_d()
1325 d->bd_hlen = 0; in reset_d()
1326 bpf_buf_reclaimed(d); in reset_d()
1328 if (bpf_canwritebuf(d)) in reset_d()
1329 d->bd_slen = 0; in reset_d()
1330 counter_u64_zero(d->bd_rcount); in reset_d()
1331 counter_u64_zero(d->bd_dcount); in reset_d()
1332 counter_u64_zero(d->bd_fcount); in reset_d()
1333 counter_u64_zero(d->bd_wcount); in reset_d()
1334 counter_u64_zero(d->bd_wfcount); in reset_d()
1335 counter_u64_zero(d->bd_wdcount); in reset_d()
1336 counter_u64_zero(d->bd_zcopy); in reset_d()
1375 struct bpf_d *d; in bpfioctl() local
1378 error = devfs_get_cdevpriv((void **)&d); in bpfioctl()
1385 BPFD_LOCK(d); in bpfioctl()
1386 BPF_PID_REFRESH(d, td); in bpfioctl()
1387 if (d->bd_state == BPF_WAITING) in bpfioctl()
1388 callout_stop(&d->bd_callout); in bpfioctl()
1389 d->bd_state = BPF_IDLE; in bpfioctl()
1390 BPFD_UNLOCK(d); in bpfioctl()
1392 if (d->bd_locked == 1) { in bpfioctl()
1439 BPFD_LOCK(d); in bpfioctl()
1440 d->bd_compat32 = 1; in bpfioctl()
1441 BPFD_UNLOCK(d); in bpfioctl()
1459 BPFD_LOCK(d); in bpfioctl()
1460 n = d->bd_slen; in bpfioctl()
1461 while (d->bd_hbuf_in_use) in bpfioctl()
1462 mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, in bpfioctl()
1464 if (d->bd_hbuf) in bpfioctl()
1465 n += d->bd_hlen; in bpfioctl()
1466 BPFD_UNLOCK(d); in bpfioctl()
1476 BPFD_LOCK(d); in bpfioctl()
1477 *(u_int *)addr = d->bd_bufsize; in bpfioctl()
1478 BPFD_UNLOCK(d); in bpfioctl()
1485 error = bpf_ioctl_sblen(d, (u_int *)addr); in bpfioctl()
1499 error = bpf_setf(d, (struct bpf_program *)addr, cmd); in bpfioctl()
1506 BPFD_LOCK(d); in bpfioctl()
1507 reset_d(d); in bpfioctl()
1508 BPFD_UNLOCK(d); in bpfioctl()
1516 if (d->bd_bif == NULL) { in bpfioctl()
1521 } else if (d->bd_promisc == 0) { in bpfioctl()
1522 error = ifpromisc(d->bd_bif->bif_ifp, 1); in bpfioctl()
1524 d->bd_promisc = 1; in bpfioctl()
1534 if (d->bd_bif == NULL) in bpfioctl()
1537 *(u_int *)addr = d->bd_bif->bif_dlt; in bpfioctl()
1554 if (d->bd_bif == NULL) in bpfioctl()
1557 error = bpf_getdltlist(d, &dltlist); in bpfioctl()
1568 if (d->bd_bif == NULL) in bpfioctl()
1571 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); in bpfioctl()
1580 if (d->bd_bif == NULL) in bpfioctl()
1583 error = bpf_setdlt(d, *(u_int *)addr); in bpfioctl()
1592 if (d->bd_bif == NULL) in bpfioctl()
1595 struct ifnet *const ifp = d->bd_bif->bif_ifp; in bpfioctl()
1619 BPFD_LOCK(d); in bpfioctl()
1620 if (d->bd_bufmode == BPF_BUFMODE_BUFFER && in bpfioctl()
1621 d->bd_sbuf == NULL) in bpfioctl()
1623 BPFD_UNLOCK(d); in bpfioctl()
1625 size = d->bd_bufsize; in bpfioctl()
1626 error = bpf_buffer_ioctl_sblen(d, &size); in bpfioctl()
1631 error = bpf_setif(d, (struct ifreq *)addr); in bpfioctl()
1663 d->bd_rtout = tvtohz(tv) - 1; in bpfioctl()
1686 tv->tv_sec = d->bd_rtout / hz; in bpfioctl()
1687 tv->tv_usec = (d->bd_rtout % hz) * tick; in bpfioctl()
1707 bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount); in bpfioctl()
1708 bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount); in bpfioctl()
1716 BPFD_LOCK(d); in bpfioctl()
1717 d->bd_immediate = *(u_int *)addr; in bpfioctl()
1718 BPFD_UNLOCK(d); in bpfioctl()
1734 BPFD_LOCK(d); in bpfioctl()
1735 *(u_int *)addr = d->bd_hdrcmplt; in bpfioctl()
1736 BPFD_UNLOCK(d); in bpfioctl()
1743 BPFD_LOCK(d); in bpfioctl()
1744 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; in bpfioctl()
1745 BPFD_UNLOCK(d); in bpfioctl()
1752 BPFD_LOCK(d); in bpfioctl()
1753 *(u_int *)addr = d->bd_direction; in bpfioctl()
1754 BPFD_UNLOCK(d); in bpfioctl()
1769 BPFD_LOCK(d); in bpfioctl()
1770 d->bd_direction = direction; in bpfioctl()
1771 BPFD_UNLOCK(d); in bpfioctl()
1783 BPFD_LOCK(d); in bpfioctl()
1784 *(u_int *)addr = d->bd_tstamp; in bpfioctl()
1785 BPFD_UNLOCK(d); in bpfioctl()
1797 d->bd_tstamp = func; in bpfioctl()
1804 BPFD_LOCK(d); in bpfioctl()
1805 d->bd_feedback = *(u_int *)addr; in bpfioctl()
1806 BPFD_UNLOCK(d); in bpfioctl()
1810 BPFD_LOCK(d); in bpfioctl()
1811 d->bd_locked = 1; in bpfioctl()
1812 BPFD_UNLOCK(d); in bpfioctl()
1819 BPFD_LOCK(d); in bpfioctl()
1820 d->bd_async = *(int *)addr; in bpfioctl()
1821 BPFD_UNLOCK(d); in bpfioctl()
1829 error = fsetown(*(int *)addr, &d->bd_sigio); in bpfioctl()
1833 BPFD_LOCK(d); in bpfioctl()
1834 *(int *)addr = fgetown(&d->bd_sigio); in bpfioctl()
1835 BPFD_UNLOCK(d); in bpfioctl()
1840 error = fsetown(-(*(int *)addr), &d->bd_sigio); in bpfioctl()
1845 *(int *)addr = -fgetown(&d->bd_sigio); in bpfioctl()
1857 BPFD_LOCK(d); in bpfioctl()
1858 d->bd_sig = sig; in bpfioctl()
1859 BPFD_UNLOCK(d); in bpfioctl()
1864 BPFD_LOCK(d); in bpfioctl()
1865 *(u_int *)addr = d->bd_sig; in bpfioctl()
1866 BPFD_UNLOCK(d); in bpfioctl()
1870 BPFD_LOCK(d); in bpfioctl()
1871 *(u_int *)addr = d->bd_bufmode; in bpfioctl()
1872 BPFD_UNLOCK(d); in bpfioctl()
1897 BPFD_LOCK(d); in bpfioctl()
1898 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || in bpfioctl()
1899 d->bd_fbuf != NULL || d->bd_bif != NULL) { in bpfioctl()
1900 BPFD_UNLOCK(d); in bpfioctl()
1904 d->bd_bufmode = *(u_int *)addr; in bpfioctl()
1905 BPFD_UNLOCK(d); in bpfioctl()
1909 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); in bpfioctl()
1913 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1917 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); in bpfioctl()
1929 d->bd_pcp = pcp; in bpfioctl()
1945 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) in bpf_setf() argument
2018 BPFD_LOCK(d); in bpf_setf()
2021 if (d->bd_wfilter != NULL) { in bpf_setf()
2022 fcode = __containerof((void *)d->bd_wfilter, in bpf_setf()
2028 d->bd_wfilter = filter; in bpf_setf()
2030 if (d->bd_rfilter != NULL) { in bpf_setf()
2031 fcode = __containerof((void *)d->bd_rfilter, in bpf_setf()
2034 fcode->func = d->bd_bfilter; in bpf_setf()
2037 d->bd_rfilter = filter; in bpf_setf()
2039 d->bd_bfilter = jfunc; in bpf_setf()
2042 reset_d(d); in bpf_setf()
2044 if (bpf_check_upgrade(cmd, d, filter, flen) != 0) { in bpf_setf()
2050 d->bd_writer = 0; in bpf_setf()
2051 if (d->bd_bif != NULL) { in bpf_setf()
2056 CK_LIST_REMOVE(d, bd_next); in bpf_setf()
2057 CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist, in bpf_setf()
2058 d, bd_next); in bpf_setf()
2061 __func__, d->bd_pid); in bpf_setf()
2066 BPFD_UNLOCK(d); in bpf_setf()
2073 d->bd_bif->bif_ifp, d->bd_bif->bif_dlt, 1); in bpf_setf()
2085 bpf_setif(struct bpf_d *d, struct ifreq *ifr) in bpf_setif() argument
2111 switch (d->bd_bufmode) { in bpf_setif()
2114 if (d->bd_sbuf == NULL) in bpf_setif()
2119 panic("bpf_setif: bufmode %d", d->bd_bufmode); in bpf_setif()
2121 if (bp != d->bd_bif) in bpf_setif()
2122 bpf_attachd(d, bp); in bpf_setif()
2124 BPFD_LOCK(d); in bpf_setif()
2125 reset_d(d); in bpf_setif()
2126 BPFD_UNLOCK(d); in bpf_setif()
2140 struct bpf_d *d; in bpfpoll() local
2143 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) in bpfpoll()
2151 BPFD_LOCK(d); in bpfpoll()
2152 BPF_PID_REFRESH(d, td); in bpfpoll()
2154 if (bpf_ready(d)) in bpfpoll()
2157 selrecord(td, &d->bd_sel); in bpfpoll()
2159 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in bpfpoll()
2160 callout_reset(&d->bd_callout, d->bd_rtout, in bpfpoll()
2161 bpf_timed_out, d); in bpfpoll()
2162 d->bd_state = BPF_WAITING; in bpfpoll()
2166 BPFD_UNLOCK(d); in bpfpoll()
2177 struct bpf_d *d; in bpfkqfilter() local
2179 if (devfs_get_cdevpriv((void **)&d) != 0) in bpfkqfilter()
2198 BPFD_LOCK(d); in bpfkqfilter()
2199 BPF_PID_REFRESH_CUR(d); in bpfkqfilter()
2200 kn->kn_hook = d; in bpfkqfilter()
2201 knlist_add(&d->bd_sel.si_note, kn, 1); in bpfkqfilter()
2202 BPFD_UNLOCK(d); in bpfkqfilter()
2210 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfdetach() local
2212 knlist_remove(&d->bd_sel.si_note, kn, 0); in filt_bpfdetach()
2218 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfread() local
2221 BPFD_LOCK_ASSERT(d); in filt_bpfread()
2222 ready = bpf_ready(d); in filt_bpfread()
2224 kn->kn_data = d->bd_slen; in filt_bpfread()
2228 if (!d->bd_hbuf_in_use && d->bd_hbuf) in filt_bpfread()
2229 kn->kn_data += d->bd_hlen; in filt_bpfread()
2230 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { in filt_bpfread()
2231 callout_reset(&d->bd_callout, d->bd_rtout, in filt_bpfread()
2232 bpf_timed_out, d); in filt_bpfread()
2233 d->bd_state = BPF_WAITING; in filt_bpfread()
2242 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; in filt_bpfwrite() local
2244 BPFD_LOCK_ASSERT(d); in filt_bpfwrite()
2246 if (d->bd_bif == NULL) { in filt_bpfwrite()
2250 kn->kn_data = d->bd_bif->bif_ifp->if_mtu; in filt_bpfwrite()
2314 struct bpf_d *d; in bpf_tap() local
2323 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_tap()
2324 counter_u64_add(d->bd_rcount, 1); in bpf_tap()
2332 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_tap()
2337 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); in bpf_tap()
2342 BPFD_LOCK(d); in bpf_tap()
2343 counter_u64_add(d->bd_fcount, 1); in bpf_tap()
2344 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_tap()
2345 gottime = bpf_gettime(&bt, d->bd_tstamp, in bpf_tap()
2348 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_tap()
2350 catchpacket(d, pkt, pktlen, slen, in bpf_tap()
2352 BPFD_UNLOCK(d); in bpf_tap()
2365 #define BPF_CHECK_DIRECTION(d, r, i) \ argument
2366 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \
2367 ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
2378 struct bpf_d *d; in bpf_mtap() local
2395 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap()
2396 if (BPF_CHECK_DIRECTION(d, m_rcvif(m), bp->bif_ifp)) in bpf_mtap()
2398 counter_u64_add(d->bd_rcount, 1); in bpf_mtap()
2400 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; in bpf_mtap()
2407 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); in bpf_mtap()
2409 BPFD_LOCK(d); in bpf_mtap()
2411 counter_u64_add(d->bd_fcount, 1); in bpf_mtap()
2412 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap()
2413 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap()
2415 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_mtap()
2417 catchpacket(d, (u_char *)m, pktlen, slen, in bpf_mtap()
2419 BPFD_UNLOCK(d); in bpf_mtap()
2444 struct bpf_d *d; in bpf_mtap2() local
2469 CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { in bpf_mtap2()
2470 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) in bpf_mtap2()
2472 counter_u64_add(d->bd_rcount, 1); in bpf_mtap2()
2473 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); in bpf_mtap2()
2475 BPFD_LOCK(d); in bpf_mtap2()
2477 counter_u64_add(d->bd_fcount, 1); in bpf_mtap2()
2478 if (gottime < bpf_ts_quality(d->bd_tstamp)) in bpf_mtap2()
2479 gottime = bpf_gettime(&bt, d->bd_tstamp, m); in bpf_mtap2()
2481 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) in bpf_mtap2()
2483 catchpacket(d, (u_char *)&mb, pktlen, slen, in bpf_mtap2()
2485 BPFD_UNLOCK(d); in bpf_mtap2()
2507 bpf_hdrlen(struct bpf_d *d) in bpf_hdrlen() argument
2511 hdrlen = d->bd_bif->bif_hdrlen; in bpf_hdrlen()
2513 if (d->bd_tstamp == BPF_T_NONE || in bpf_hdrlen()
2514 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) in bpf_hdrlen()
2516 if (d->bd_compat32) in bpf_hdrlen()
2525 if (d->bd_compat32) in bpf_hdrlen()
2531 return (hdrlen - d->bd_bif->bif_hdrlen); in bpf_hdrlen()
2573 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, in catchpacket() argument
2590 BPFD_LOCK_ASSERT(d); in catchpacket()
2591 if (d->bd_bif == NULL) { in catchpacket()
2593 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2604 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { in catchpacket()
2605 d->bd_fbuf = d->bd_hbuf; in catchpacket()
2606 d->bd_hbuf = NULL; in catchpacket()
2607 d->bd_hlen = 0; in catchpacket()
2608 bpf_buf_reclaimed(d); in catchpacket()
2617 hdrlen = bpf_hdrlen(d); in catchpacket()
2619 if (totlen > d->bd_bufsize) in catchpacket()
2620 totlen = d->bd_bufsize; in catchpacket()
2631 if (d->bd_compat32) in catchpacket()
2632 curlen = BPF_WORDALIGN32(d->bd_slen); in catchpacket()
2635 curlen = BPF_WORDALIGN(d->bd_slen); in catchpacket()
2636 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { in catchpacket()
2637 if (d->bd_fbuf == NULL) { in catchpacket()
2643 bpf_buffull(d); in catchpacket()
2644 counter_u64_add(d->bd_dcount, 1); in catchpacket()
2647 KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use")); in catchpacket()
2648 ROTATE_BUFFERS(d); in catchpacket()
2652 if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { in catchpacket()
2660 pad = curlen - d->bd_slen; in catchpacket()
2665 bpf_append_bytes(d, d->bd_sbuf, d->bd_slen, zeroes, in catchpacket()
2671 tstype = d->bd_tstamp; in catchpacket()
2679 if (d->bd_compat32) { in catchpacket()
2688 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, in catchpacket()
2701 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, in catchpacket()
2717 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); in catchpacket()
2725 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); in catchpacket()
2726 d->bd_slen = curlen + totlen; in catchpacket()
2729 bpf_wakeup(d); in catchpacket()
2739 struct bpf_d *d; in bpfd_free() local
2747 d = __containerof(ctx, struct bpf_d, epoch_ctx); in bpfd_free()
2748 bpf_free(d); in bpfd_free()
2749 if (d->bd_rfilter != NULL) { in bpfd_free()
2750 p = __containerof((void *)d->bd_rfilter, in bpfd_free()
2753 p->func = d->bd_bfilter; in bpfd_free()
2757 if (d->bd_wfilter != NULL) { in bpfd_free()
2758 p = __containerof((void *)d->bd_wfilter, in bpfd_free()
2766 mtx_destroy(&d->bd_lock); in bpfd_free()
2767 counter_u64_free(d->bd_rcount); in bpfd_free()
2768 counter_u64_free(d->bd_dcount); in bpfd_free()
2769 counter_u64_free(d->bd_fcount); in bpfd_free()
2770 counter_u64_free(d->bd_wcount); in bpfd_free()
2771 counter_u64_free(d->bd_wfcount); in bpfd_free()
2772 counter_u64_free(d->bd_wdcount); in bpfd_free()
2773 counter_u64_free(d->bd_zcopy); in bpfd_free()
2774 free(d, M_BPF); in bpfd_free()
2858 struct bpf_d *d; in bpf_ifdetach() local
2866 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpf_ifdetach()
2867 bpf_detachd_locked(d, true); in bpf_ifdetach()
2871 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpf_ifdetach()
2872 bpf_detachd_locked(d, true); in bpf_ifdetach()
2888 struct bpf_d *d; in bpfdetach() local
2904 while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { in bpfdetach()
2905 bpf_detachd_locked(d, true); in bpfdetach()
2909 while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { in bpfdetach()
2910 bpf_detachd_locked(d, true); in bpfdetach()
2927 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) in bpf_getdltlist() argument
2936 ifp = d->bd_bif->bif_ifp; in bpf_getdltlist()
2966 bpf_setdlt(struct bpf_d *d, u_int dlt) in bpf_setdlt() argument
2973 MPASS(d->bd_bif != NULL); in bpf_setdlt()
2979 if (d->bd_bif->bif_dlt == dlt) in bpf_setdlt()
2982 ifp = d->bd_bif->bif_ifp; in bpf_setdlt()
2990 opromisc = d->bd_promisc; in bpf_setdlt()
2991 bpf_attachd(d, bp); in bpf_setdlt()
2998 d->bd_promisc = 1; in bpf_setdlt()
3047 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) in bpfstats_fill_xbpf() argument
3051 bzero(d, sizeof(*d)); in bpfstats_fill_xbpf()
3052 d->bd_structsize = sizeof(*d); in bpfstats_fill_xbpf()
3053 d->bd_immediate = bd->bd_immediate; in bpfstats_fill_xbpf()
3054 d->bd_promisc = bd->bd_promisc; in bpfstats_fill_xbpf()
3055 d->bd_hdrcmplt = bd->bd_hdrcmplt; in bpfstats_fill_xbpf()
3056 d->bd_direction = bd->bd_direction; in bpfstats_fill_xbpf()
3057 d->bd_feedback = bd->bd_feedback; in bpfstats_fill_xbpf()
3058 d->bd_async = bd->bd_async; in bpfstats_fill_xbpf()
3059 d->bd_rcount = counter_u64_fetch(bd->bd_rcount); in bpfstats_fill_xbpf()
3060 d->bd_dcount = counter_u64_fetch(bd->bd_dcount); in bpfstats_fill_xbpf()
3061 d->bd_fcount = counter_u64_fetch(bd->bd_fcount); in bpfstats_fill_xbpf()
3062 d->bd_sig = bd->bd_sig; in bpfstats_fill_xbpf()
3063 d->bd_slen = bd->bd_slen; in bpfstats_fill_xbpf()
3064 d->bd_hlen = bd->bd_hlen; in bpfstats_fill_xbpf()
3065 d->bd_bufsize = bd->bd_bufsize; in bpfstats_fill_xbpf()
3066 d->bd_pid = bd->bd_pid; in bpfstats_fill_xbpf()
3067 strlcpy(d->bd_ifname, in bpfstats_fill_xbpf()
3069 d->bd_locked = bd->bd_locked; in bpfstats_fill_xbpf()
3070 d->bd_wcount = counter_u64_fetch(bd->bd_wcount); in bpfstats_fill_xbpf()
3071 d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount); in bpfstats_fill_xbpf()
3072 d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount); in bpfstats_fill_xbpf()
3073 d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy); in bpfstats_fill_xbpf()
3074 d->bd_bufmode = bd->bd_bufmode; in bpfstats_fill_xbpf()
3176 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) in bpf_mtap2() argument