Lines Matching +full:- +full:m
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
74 static void sbcompress_ktls_rx(struct sockbuf *sb, struct mbuf *m,
84 sbm_clrprotoflags(struct mbuf *m, int flags) in sbm_clrprotoflags() argument
91 while (m) { in sbm_clrprotoflags()
92 m->m_flags &= mask; in sbm_clrprotoflags()
93 m = m->m_next; in sbm_clrprotoflags()
109 struct mbuf *m, *n; in sbready_compress() local
114 if ((sb->sb_flags & SB_NOCOALESCE) != 0) in sbready_compress()
117 for (m = m0; m != end; m = m->m_next) { in sbready_compress()
118 MPASS((m->m_flags & M_NOTREADY) == 0); in sbready_compress()
121 * socket buffer and 'm' is the new mbuf being copied in sbready_compress()
123 * are reversed and 'n' is the next mbuf after 'm' in sbready_compress()
125 * 'm'. in sbready_compress()
127 n = m->m_next; in sbready_compress()
130 if ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 && in sbready_compress()
131 (m->m_flags & M_EXTPG) && in sbready_compress()
132 (n->m_flags & M_EXTPG) && in sbready_compress()
133 !mbuf_has_tls_session(m) && in sbready_compress()
137 hdr_len = n->m_epg_hdrlen; in sbready_compress()
138 trail_len = m->m_epg_trllen; in sbready_compress()
141 /* copy n's header to m's trailer */ in sbready_compress()
142 memcpy(&m->m_epg_trail[trail_len], in sbready_compress()
143 n->m_epg_hdr, hdr_len); in sbready_compress()
144 m->m_epg_trllen += hdr_len; in sbready_compress()
145 m->m_len += hdr_len; in sbready_compress()
146 n->m_epg_hdrlen = 0; in sbready_compress()
147 n->m_len -= hdr_len; in sbready_compress()
153 if ((m->m_flags & M_EXTPG) && m->m_len <= MLEN && in sbready_compress()
154 !mbuf_has_tls_session(m)) { in sbready_compress()
155 ext_size = m->m_ext.ext_size; in sbready_compress()
156 if (mb_unmapped_compress(m) == 0) in sbready_compress()
157 sb->sb_mbcnt -= ext_size; in sbready_compress()
160 while ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 && in sbready_compress()
161 M_WRITABLE(m) && in sbready_compress()
162 (m->m_flags & M_EXTPG) == 0 && in sbready_compress()
164 !mbuf_has_tls_session(m) && in sbready_compress()
165 n->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ in sbready_compress()
166 n->m_len <= M_TRAILINGSPACE(m) && in sbready_compress()
167 m->m_type == n->m_type) { in sbready_compress()
168 KASSERT(sb->sb_lastrecord != n, in sbready_compress()
170 __func__, n, m)); in sbready_compress()
171 m_copydata(n, 0, n->m_len, mtodo(m, m->m_len)); in sbready_compress()
172 m->m_len += n->m_len; in sbready_compress()
173 m->m_next = n->m_next; in sbready_compress()
174 m->m_flags |= n->m_flags & M_EOR; in sbready_compress()
175 if (sb->sb_mbtail == n) in sbready_compress()
176 sb->sb_mbtail = m; in sbready_compress()
178 sb->sb_mbcnt -= MSIZE; in sbready_compress()
179 if (n->m_flags & M_EXT) in sbready_compress()
180 sb->sb_mbcnt -= n->m_ext.ext_size; in sbready_compress()
182 n = m->m_next; in sbready_compress()
190 * Mark ready "count" units of I/O starting with "m". Most mbufs
197 struct mbuf *m; in sbready() local
201 KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); in sbready()
204 m = m0; in sbready()
205 blocker = (sb->sb_fnrdy == m); in sbready()
208 KASSERT(m->m_flags & M_NOTREADY, in sbready()
209 ("%s: m %p !M_NOTREADY", __func__, m)); in sbready()
210 if ((m->m_flags & M_EXTPG) != 0 && m->m_epg_npgs != 0) { in sbready()
211 if (count < m->m_epg_nrdy) { in sbready()
212 m->m_epg_nrdy -= count; in sbready()
216 count -= m->m_epg_nrdy; in sbready()
217 m->m_epg_nrdy = 0; in sbready()
219 count--; in sbready()
220 m->m_flags &= ~M_NOTREADY; in sbready()
222 sb->sb_acc += m->m_len; in sbready()
223 m = m->m_next; in sbready()
231 if (m0 == m) { in sbready()
232 MPASS(m->m_flags & M_NOTREADY); in sbready()
237 sbready_compress(sb, m0, m); in sbready()
242 for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) in sbready()
243 sb->sb_acc += m->m_len; in sbready()
245 sb->sb_fnrdy = m; in sbready()
246 sbready_compress(sb, m0, m); in sbready()
252 * Adjust sockbuf state reflecting allocation of m.
255 sballoc(struct sockbuf *sb, struct mbuf *m) in sballoc() argument
260 sb->sb_ccc += m->m_len; in sballoc()
262 if (sb->sb_fnrdy == NULL) { in sballoc()
263 if (m->m_flags & M_NOTREADY) in sballoc()
264 sb->sb_fnrdy = m; in sballoc()
266 sb->sb_acc += m->m_len; in sballoc()
269 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) in sballoc()
270 sb->sb_ctl += m->m_len; in sballoc()
272 sb->sb_mbcnt += MSIZE; in sballoc()
274 if (m->m_flags & M_EXT) in sballoc()
275 sb->sb_mbcnt += m->m_ext.ext_size; in sballoc()
279 * Adjust sockbuf state reflecting freeing of m.
282 sbfree(struct sockbuf *sb, struct mbuf *m) in sbfree() argument
289 sb->sb_ccc -= m->m_len; in sbfree()
291 if (m == sb->sb_fnrdy) { in sbfree()
292 KASSERT(m->m_flags & M_NOTREADY, in sbfree()
293 ("%s: m %p !M_NOTREADY", __func__, m)); in sbfree()
295 n = m->m_next; in sbfree()
296 while (n != NULL && !(n->m_flags & M_NOTREADY)) { in sbfree()
297 sb->sb_acc += n->m_len; in sbfree()
298 n = n->m_next; in sbfree()
300 sb->sb_fnrdy = n; in sbfree()
303 for (n = sb->sb_fnrdy; n != NULL; n = n->m_next) in sbfree()
304 KASSERT(n != m, ("%s: sb %p freeing %p behind sb_fnrdy", in sbfree()
305 __func__, sb, m)); in sbfree()
306 sb->sb_acc -= m->m_len; in sbfree()
309 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) in sbfree()
310 sb->sb_ctl -= m->m_len; in sbfree()
312 sb->sb_mbcnt -= MSIZE; in sbfree()
313 if (m->m_flags & M_EXT) in sbfree()
314 sb->sb_mbcnt -= m->m_ext.ext_size; in sbfree()
316 if (sb->sb_sndptr == m) { in sbfree()
317 sb->sb_sndptr = NULL; in sbfree()
318 sb->sb_sndptroff = 0; in sbfree()
320 if (sb->sb_sndptroff != 0) in sbfree()
321 sb->sb_sndptroff -= m->m_len; in sbfree()
331 sballoc_ktls_rx(struct sockbuf *sb, struct mbuf *m) in sballoc_ktls_rx() argument
336 sb->sb_ccc += m->m_len; in sballoc_ktls_rx()
337 sb->sb_tlscc += m->m_len; in sballoc_ktls_rx()
339 sb->sb_mbcnt += MSIZE; in sballoc_ktls_rx()
341 if (m->m_flags & M_EXT) in sballoc_ktls_rx()
342 sb->sb_mbcnt += m->m_ext.ext_size; in sballoc_ktls_rx()
346 sbfree_ktls_rx(struct sockbuf *sb, struct mbuf *m) in sbfree_ktls_rx() argument
353 sb->sb_ccc -= m->m_len; in sbfree_ktls_rx()
354 sb->sb_tlscc -= m->m_len; in sbfree_ktls_rx()
356 sb->sb_mbcnt -= MSIZE; in sbfree_ktls_rx()
358 if (m->m_flags & M_EXT) in sbfree_ktls_rx()
359 sb->sb_mbcnt -= m->m_ext.ext_size; in sbfree_ktls_rx()
378 so->so_snd.sb_state |= SBS_CANTSENDMORE; in socantsendmore_locked()
398 so->so_rcv.sb_state |= SBS_CANTRCVMORE; in socantrcvmore_locked()
400 if (so->so_rcv.sb_flags & SB_TLS_RX) in socantrcvmore_locked()
401 ktls_check_rx(&so->so_rcv); in socantrcvmore_locked()
422 if (so->so_options & SO_RERROR) { in soroverflow_locked()
423 so->so_rerror = ENOBUFS; in soroverflow_locked()
451 sb->sb_flags |= SB_WAIT; in sbwait()
452 return (msleep_sbt(&sb->sb_acc, soeventmtx(so, which), in sbwait()
453 PSOCK | PCATCH, "sbwait", sb->sb_timeo, 0, 0)); in sbwait()
477 selwakeuppri(sb->sb_sel, PSOCK); in sowakeup()
478 if (!SEL_WAITING(sb->sb_sel)) in sowakeup()
479 sb->sb_flags &= ~SB_SEL; in sowakeup()
480 if (sb->sb_flags & SB_WAIT) { in sowakeup()
481 sb->sb_flags &= ~SB_WAIT; in sowakeup()
482 wakeup(&sb->sb_acc); in sowakeup()
484 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); in sowakeup()
485 if (sb->sb_upcall != NULL) { in sowakeup()
486 ret = sb->sb_upcall(so, sb->sb_upcallarg, M_NOWAIT); in sowakeup()
488 KASSERT(sb == &so->so_rcv, in sowakeup()
494 if (sb->sb_flags & SB_AIO) in sowakeup()
499 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) in sowakeup()
500 pgsigio(&so->so_sigio, SIGIO, 0); in sowakeup()
511 sp = so->so_splice; in splice_push()
512 mtx_lock(&sp->mtx); in splice_push()
524 sp = so->so_splice_back; in splice_pull()
525 mtx_lock(&sp->mtx); in splice_pull()
536 return ((sb->sb_flags & (SB_WAIT | SB_SEL | SB_ASYNC | in sb_notify()
544 if (so->so_rcv.sb_flags & SB_SPLICED) in sorwakeup_locked()
546 else if (sb_notify(&so->so_rcv)) in sorwakeup_locked()
556 if (so->so_snd.sb_flags & SB_SPLICED) in sowwakeup_locked()
558 else if (sb_notify(&so->so_snd)) in sowwakeup_locked()
606 if (so->so_rcv.sb_lowat == 0) in soreserve()
607 so->so_rcv.sb_lowat = 1; in soreserve()
608 if (so->so_snd.sb_lowat == 0) in soreserve()
609 so->so_snd.sb_lowat = MCLBYTES; in soreserve()
610 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) in soreserve()
611 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; in soreserve()
630 if (error || !req->newptr) in sysctl_handle_sb_max()
665 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, in sbreserve_locked_limit()
668 sb->sb_mbmax = min(cc * sb_efficiency, buf_max); in sbreserve_locked_limit()
669 if (sb->sb_lowat > sb->sb_hiwat) in sbreserve_locked_limit()
670 sb->sb_lowat = sb->sb_hiwat; in sbreserve_locked_limit()
688 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, in sbunreserve_locked()
690 sb->sb_mbmax = 0; in sbunreserve_locked()
717 switch (sopt->sopt_name) { in sbsetopt()
720 lowat = &so->sol_sbsnd_lowat; in sbsetopt()
721 hiwat = &so->sol_sbsnd_hiwat; in sbsetopt()
722 flags = &so->sol_sbsnd_flags; in sbsetopt()
726 lowat = &so->sol_sbrcv_lowat; in sbsetopt()
727 hiwat = &so->sol_sbrcv_hiwat; in sbsetopt()
728 flags = &so->sol_sbrcv_flags; in sbsetopt()
732 switch (sopt->sopt_name) { in sbsetopt()
735 sb = &so->so_snd; in sbsetopt()
740 sb = &so->so_rcv; in sbsetopt()
744 flags = &sb->sb_flags; in sbsetopt()
745 hiwat = &sb->sb_hiwat; in sbsetopt()
746 lowat = &sb->sb_lowat; in sbsetopt()
751 switch (sopt->sopt_name) { in sbsetopt()
772 * Make sure the low-water is never greater than the in sbsetopt()
773 * high-water. in sbsetopt()
815 if (sb->sb_tls_info != NULL) in sbdestroy()
816 ktls_free(sb->sb_tls_info); in sbdestroy()
817 sb->sb_tls_info = NULL; in sbdestroy()
848 struct mbuf *m = sb->sb_mb; in sblastrecordchk() local
852 while (m && m->m_nextpkt) in sblastrecordchk()
853 m = m->m_nextpkt; in sblastrecordchk()
855 if (m != sb->sb_lastrecord) { in sblastrecordchk()
857 __func__, sb->sb_mb, sb->sb_lastrecord, m); in sblastrecordchk()
859 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) in sblastrecordchk()
860 printf("\t%p\n", m); in sblastrecordchk()
868 struct mbuf *m = sb->sb_mb; in sblastmbufchk() local
873 while (m && m->m_nextpkt) in sblastmbufchk()
874 m = m->m_nextpkt; in sblastmbufchk()
876 while (m && m->m_next) in sblastmbufchk()
877 m = m->m_next; in sblastmbufchk()
879 if (m != sb->sb_mbtail) { in sblastmbufchk()
881 __func__, sb->sb_mb, sb->sb_mbtail, m); in sblastmbufchk()
883 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { in sblastmbufchk()
885 for (n = m; n != NULL; n = n->m_next) in sblastmbufchk()
893 m = sb->sb_mtls; in sblastmbufchk()
894 while (m && m->m_next) in sblastmbufchk()
895 m = m->m_next; in sblastmbufchk()
897 if (m != sb->sb_mtlstail) { in sblastmbufchk()
899 __func__, sb->sb_mtls, sb->sb_mtlstail, m); in sblastmbufchk()
902 for (m = sb->sb_mtls; m != NULL; m = m->m_next) { in sblastmbufchk()
903 printf("%p ", m); in sblastmbufchk()
914 if ((sb)->sb_lastrecord != NULL) \
915 (sb)->sb_lastrecord->m_nextpkt = (m0); \
917 (sb)->sb_mb = (m0); \
918 (sb)->sb_lastrecord = (m0); \
922 * Append mbuf chain m to the last record in the socket buffer sb. The
927 sbappend_locked(struct sockbuf *sb, struct mbuf *m, int flags) in sbappend_locked() argument
933 if (m == NULL) in sbappend_locked()
935 kmsan_check_mbuf(m, "sbappend"); in sbappend_locked()
936 sbm_clrprotoflags(m, flags); in sbappend_locked()
938 n = sb->sb_mb; in sbappend_locked()
940 while (n->m_nextpkt) in sbappend_locked()
941 n = n->m_nextpkt; in sbappend_locked()
943 if (n->m_flags & M_EOR) { in sbappend_locked()
944 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ in sbappend_locked()
947 } while (n->m_next && (n = n->m_next)); in sbappend_locked()
954 if ((n = sb->sb_lastrecord) != NULL) { in sbappend_locked()
956 if (n->m_flags & M_EOR) { in sbappend_locked()
957 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ in sbappend_locked()
960 } while (n->m_next && (n = n->m_next)); in sbappend_locked()
966 sb->sb_lastrecord = m; in sbappend_locked()
969 sbcompress(sb, m, n); in sbappend_locked()
974 * Append mbuf chain m to the last record in the socket buffer sb. The
979 sbappend(struct sockbuf *sb, struct mbuf *m, int flags) in sbappend() argument
983 sbappend_locked(sb, m, flags); in sbappend()
994 sbappend_ktls_rx(struct sockbuf *sb, struct mbuf *m) in sbappend_ktls_rx() argument
1006 MPASS((m->m_flags & M_PKTHDR) != 0); in sbappend_ktls_rx()
1009 for (n = m; n != NULL; n = n->m_next) { in sbappend_ktls_rx()
1010 if (n->m_flags & M_PKTHDR) { in sbappend_ktls_rx()
1011 ifp = m->m_pkthdr.leaf_rcvif; in sbappend_ktls_rx()
1012 if ((n->m_pkthdr.csum_flags & CSUM_TLS_MASK) == in sbappend_ktls_rx()
1022 n->m_flags &= M_DEMOTEFLAGS; in sbappend_ktls_rx()
1023 n->m_flags |= flags; in sbappend_ktls_rx()
1025 MPASS((n->m_flags & M_NOTREADY) != 0); in sbappend_ktls_rx()
1028 sbcompress_ktls_rx(sb, m, sb->sb_mtlstail); in sbappend_ktls_rx()
1032 if (ifp != NULL && sb->sb_tls_info->rx_ifp != NULL && in sbappend_ktls_rx()
1033 sb->sb_tls_info->rx_ifp != ifp) in sbappend_ktls_rx()
1044 sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags) in sbappendstream_locked() argument
1048 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); in sbappendstream_locked()
1050 kmsan_check_mbuf(m, "sbappend"); in sbappendstream_locked()
1058 if (sb->sb_flags & SB_TLS_RX) { in sbappendstream_locked()
1059 sbappend_ktls_rx(sb, m); in sbappendstream_locked()
1064 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); in sbappendstream_locked()
1069 if (sb->sb_tls_info != NULL) in sbappendstream_locked()
1070 ktls_seq(sb, m); in sbappendstream_locked()
1074 m_demote(m, 1, flags & PRUS_NOTREADY ? M_NOTREADY : 0); in sbappendstream_locked()
1076 sbcompress(sb, m, sb->sb_mbtail); in sbappendstream_locked()
1078 sb->sb_lastrecord = sb->sb_mb; in sbappendstream_locked()
1088 sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags) in sbappendstream() argument
1092 sbappendstream_locked(sb, m, flags); in sbappendstream()
1100 struct mbuf *m, *n, *fnrdy; in sbcheck() local
1111 for (m = sb->sb_mb; m; m = n) { in sbcheck()
1112 n = m->m_nextpkt; in sbcheck()
1113 for (; m; m = m->m_next) { in sbcheck()
1114 if (m->m_len == 0) { in sbcheck()
1115 printf("sb %p empty mbuf %p\n", sb, m); in sbcheck()
1118 if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) { in sbcheck()
1119 if (m != sb->sb_fnrdy) { in sbcheck()
1120 printf("sb %p: fnrdy %p != m %p\n", in sbcheck()
1121 sb, sb->sb_fnrdy, m); in sbcheck()
1124 fnrdy = m; in sbcheck()
1127 acc += m->m_len; in sbcheck()
1128 ccc += m->m_len; in sbcheck()
1130 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ in sbcheck()
1131 mbcnt += m->m_ext.ext_size; in sbcheck()
1142 ccc += sb->sb_tlsdcc; in sbcheck()
1145 for (m = sb->sb_mtls; m; m = m->m_next) { in sbcheck()
1146 if (m->m_nextpkt != NULL) { in sbcheck()
1147 printf("sb %p TLS mbuf %p with nextpkt\n", sb, m); in sbcheck()
1150 if ((m->m_flags & M_NOTREADY) == 0) { in sbcheck()
1151 printf("sb %p TLS mbuf %p ready\n", sb, m); in sbcheck()
1154 tlscc += m->m_len; in sbcheck()
1155 ccc += m->m_len; in sbcheck()
1157 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ in sbcheck()
1158 mbcnt += m->m_ext.ext_size; in sbcheck()
1161 if (sb->sb_tlscc != tlscc) { in sbcheck()
1162 printf("tlscc %ld/%u dcc %u\n", tlscc, sb->sb_tlscc, in sbcheck()
1163 sb->sb_tlsdcc); in sbcheck()
1167 if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) { in sbcheck()
1169 acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt); in sbcheck()
1171 printf("tlscc %ld/%u dcc %u\n", tlscc, sb->sb_tlscc, in sbcheck()
1172 sb->sb_tlsdcc); in sbcheck()
1188 struct mbuf *m; in sbappendrecord_locked() local
1205 sb->sb_mbtail = m0; in sbappendrecord_locked()
1206 m = m0->m_next; in sbappendrecord_locked()
1207 m0->m_next = 0; in sbappendrecord_locked()
1208 if (m && (m0->m_flags & M_EOR)) { in sbappendrecord_locked()
1209 m0->m_flags &= ~M_EOR; in sbappendrecord_locked()
1210 m->m_flags |= M_EOR; in sbappendrecord_locked()
1213 sbcompress(sb, m, m0); in sbappendrecord_locked()
1233 struct mbuf *m, *n, *nlast; in sbappendaddr_locked_internal() local
1241 if (asa->sa_len > MLEN) in sbappendaddr_locked_internal()
1244 m = m_get(M_NOWAIT, MT_SONAME); in sbappendaddr_locked_internal()
1245 if (m == NULL) in sbappendaddr_locked_internal()
1247 m->m_len = asa->sa_len; in sbappendaddr_locked_internal()
1248 bcopy(asa, mtod(m, caddr_t), asa->sa_len); in sbappendaddr_locked_internal()
1258 m0->m_pkthdr.rcvif = NULL; in sbappendaddr_locked_internal()
1259 m0->m_pkthdr.flowid = 0; in sbappendaddr_locked_internal()
1260 m0->m_pkthdr.csum_flags = 0; in sbappendaddr_locked_internal()
1261 m0->m_pkthdr.fibnum = 0; in sbappendaddr_locked_internal()
1262 m0->m_pkthdr.rsstype = 0; in sbappendaddr_locked_internal()
1265 ctrl_last->m_next = m0; /* concatenate data to control */ in sbappendaddr_locked_internal()
1268 m->m_next = control; in sbappendaddr_locked_internal()
1269 for (n = m; n->m_next != NULL; n = n->m_next) in sbappendaddr_locked_internal()
1273 SBLINKRECORD(sb, m); in sbappendaddr_locked_internal()
1275 sb->sb_mbtail = nlast; in sbappendaddr_locked_internal()
1293 int space = asa->sa_len; in sbappendaddr_locked()
1297 if (m0 && (m0->m_flags & M_PKTHDR) == 0) in sbappendaddr_locked()
1300 space += m0->m_pkthdr.len; in sbappendaddr_locked()
1348 struct mbuf *m, *mlast; in sbappendcontrol_locked() local
1355 m_last(control)->m_next = m0; in sbappendcontrol_locked()
1359 for (m = control; m->m_next; m = m->m_next) in sbappendcontrol_locked()
1360 sballoc(sb, m); in sbappendcontrol_locked()
1361 sballoc(sb, m); in sbappendcontrol_locked()
1362 mlast = m; in sbappendcontrol_locked()
1365 sb->sb_mbtail = mlast; in sbappendcontrol_locked()
1382 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf
1391 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into
1399 * end-of-record.
1402 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) in sbcompress() argument
1409 while (m) { in sbcompress()
1410 eor |= m->m_flags & M_EOR; in sbcompress()
1411 if (m->m_len == 0 && in sbcompress()
1413 (((o = m->m_next) || (o = n)) && in sbcompress()
1414 o->m_type == m->m_type))) { in sbcompress()
1415 if (sb->sb_lastrecord == m) in sbcompress()
1416 sb->sb_lastrecord = m->m_next; in sbcompress()
1417 m = m_free(m); in sbcompress()
1420 if (n && (n->m_flags & M_EOR) == 0 && in sbcompress()
1422 ((sb->sb_flags & SB_NOCOALESCE) == 0) && in sbcompress()
1423 !(m->m_flags & M_NOTREADY) && in sbcompress()
1424 !(n->m_flags & (M_NOTREADY | M_EXTPG)) && in sbcompress()
1425 !mbuf_has_tls_session(m) && in sbcompress()
1427 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ in sbcompress()
1428 m->m_len <= M_TRAILINGSPACE(n) && in sbcompress()
1429 n->m_type == m->m_type) { in sbcompress()
1430 m_copydata(m, 0, m->m_len, mtodo(n, n->m_len)); in sbcompress()
1431 n->m_len += m->m_len; in sbcompress()
1432 sb->sb_ccc += m->m_len; in sbcompress()
1433 if (sb->sb_fnrdy == NULL) in sbcompress()
1434 sb->sb_acc += m->m_len; in sbcompress()
1435 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) in sbcompress()
1437 sb->sb_ctl += m->m_len; in sbcompress()
1438 m = m_free(m); in sbcompress()
1441 if (m->m_len <= MLEN && (m->m_flags & M_EXTPG) && in sbcompress()
1442 (m->m_flags & M_NOTREADY) == 0 && in sbcompress()
1443 !mbuf_has_tls_session(m)) in sbcompress()
1444 (void)mb_unmapped_compress(m); in sbcompress()
1446 n->m_next = m; in sbcompress()
1448 sb->sb_mb = m; in sbcompress()
1449 sb->sb_mbtail = m; in sbcompress()
1450 sballoc(sb, m); in sbcompress()
1451 n = m; in sbcompress()
1452 m->m_flags &= ~M_EOR; in sbcompress()
1453 m = m->m_next; in sbcompress()
1454 n->m_next = 0; in sbcompress()
1458 n->m_flags |= eor; in sbcompress()
1470 sbcompress_ktls_rx(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) in sbcompress_ktls_rx() argument
1475 while (m) { in sbcompress_ktls_rx()
1476 KASSERT((m->m_flags & M_EOR) == 0, in sbcompress_ktls_rx()
1477 ("TLS RX mbuf %p with EOR", m)); in sbcompress_ktls_rx()
1478 KASSERT(m->m_type == MT_DATA, in sbcompress_ktls_rx()
1479 ("TLS RX mbuf %p is not MT_DATA", m)); in sbcompress_ktls_rx()
1480 KASSERT((m->m_flags & M_NOTREADY) != 0, in sbcompress_ktls_rx()
1481 ("TLS RX mbuf %p ready", m)); in sbcompress_ktls_rx()
1482 KASSERT((m->m_flags & M_EXTPG) == 0, in sbcompress_ktls_rx()
1483 ("TLS RX mbuf %p unmapped", m)); in sbcompress_ktls_rx()
1485 if (m->m_len == 0) { in sbcompress_ktls_rx()
1486 m = m_free(m); in sbcompress_ktls_rx()
1491 * Even though both 'n' and 'm' are NOTREADY, it's ok in sbcompress_ktls_rx()
1496 ((sb->sb_flags & SB_NOCOALESCE) == 0) && in sbcompress_ktls_rx()
1497 !((m->m_flags ^ n->m_flags) & M_DECRYPTED) && in sbcompress_ktls_rx()
1498 !(n->m_flags & M_EXTPG) && in sbcompress_ktls_rx()
1499 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ in sbcompress_ktls_rx()
1500 m->m_len <= M_TRAILINGSPACE(n)) { in sbcompress_ktls_rx()
1501 m_copydata(m, 0, m->m_len, mtodo(n, n->m_len)); in sbcompress_ktls_rx()
1502 n->m_len += m->m_len; in sbcompress_ktls_rx()
1503 sb->sb_ccc += m->m_len; in sbcompress_ktls_rx()
1504 sb->sb_tlscc += m->m_len; in sbcompress_ktls_rx()
1505 m = m_free(m); in sbcompress_ktls_rx()
1509 n->m_next = m; in sbcompress_ktls_rx()
1511 sb->sb_mtls = m; in sbcompress_ktls_rx()
1512 sb->sb_mtlstail = m; in sbcompress_ktls_rx()
1513 sballoc_ktls_rx(sb, m); in sbcompress_ktls_rx()
1514 n = m; in sbcompress_ktls_rx()
1515 m = m->m_next; in sbcompress_ktls_rx()
1516 n->m_next = NULL; in sbcompress_ktls_rx()
1531 while (sb->sb_mbcnt || sb->sb_tlsdcc) { in sbflush_locked()
1533 * Don't call sbcut(sb, 0) if the leading mbuf is non-empty: in sbflush_locked()
1536 if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len)) in sbflush_locked()
1538 m_freem(sbcut_internal(sb, (int)sb->sb_ccc)); in sbflush_locked()
1540 KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0, in sbflush_locked()
1542 sb->sb_ccc, (void *)sb->sb_mb, sb->sb_mbcnt)); in sbflush_locked()
1560 struct mbuf *m, *next, *mfree; in sbcut_internal() local
1565 KASSERT(len <= sb->sb_ccc, ("%s: len: %d is > ccc: %u", in sbcut_internal()
1566 __func__, len, sb->sb_ccc)); in sbcut_internal()
1568 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; in sbcut_internal()
1573 if (m == NULL) { in sbcut_internal()
1576 if (sb->sb_tlsdcc != 0) { in sbcut_internal()
1577 MPASS(len >= sb->sb_tlsdcc); in sbcut_internal()
1578 len -= sb->sb_tlsdcc; in sbcut_internal()
1579 sb->sb_ccc -= sb->sb_tlsdcc; in sbcut_internal()
1580 sb->sb_tlsdcc = 0; in sbcut_internal()
1584 next = sb->sb_mtls; in sbcut_internal()
1589 m = next; in sbcut_internal()
1590 next = m->m_nextpkt; in sbcut_internal()
1592 if (m->m_len > len) { in sbcut_internal()
1593 KASSERT(!(m->m_flags & M_NOTREADY), in sbcut_internal()
1594 ("%s: m %p M_NOTREADY", __func__, m)); in sbcut_internal()
1595 m->m_len -= len; in sbcut_internal()
1596 m->m_data += len; in sbcut_internal()
1597 sb->sb_ccc -= len; in sbcut_internal()
1598 sb->sb_acc -= len; in sbcut_internal()
1599 if (sb->sb_sndptroff != 0) in sbcut_internal()
1600 sb->sb_sndptroff -= len; in sbcut_internal()
1601 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) in sbcut_internal()
1602 sb->sb_ctl -= len; in sbcut_internal()
1605 len -= m->m_len; in sbcut_internal()
1608 sbfree_ktls_rx(sb, m); in sbcut_internal()
1611 sbfree(sb, m); in sbcut_internal()
1616 if (m->m_flags & M_NOTREADY && !is_tls) in sbcut_internal()
1617 m = m->m_next; in sbcut_internal()
1621 n = m->m_next; in sbcut_internal()
1622 m->m_next = mfree; in sbcut_internal()
1623 mfree = m; in sbcut_internal()
1624 m = n; in sbcut_internal()
1628 * Free any zero-length mbufs from the buffer. in sbcut_internal()
1633 while (m && m->m_len == 0) { in sbcut_internal()
1636 sbfree(sb, m); in sbcut_internal()
1637 n = m->m_next; in sbcut_internal()
1638 m->m_next = mfree; in sbcut_internal()
1639 mfree = m; in sbcut_internal()
1640 m = n; in sbcut_internal()
1644 sb->sb_mb = NULL; in sbcut_internal()
1645 sb->sb_mtls = m; in sbcut_internal()
1646 if (m == NULL) in sbcut_internal()
1647 sb->sb_mtlstail = NULL; in sbcut_internal()
1650 if (m) { in sbcut_internal()
1651 sb->sb_mb = m; in sbcut_internal()
1652 m->m_nextpkt = next; in sbcut_internal()
1654 sb->sb_mb = next; in sbcut_internal()
1657 * sb_lastrecord is up-to-date if we dropped part of the last record. in sbcut_internal()
1659 m = sb->sb_mb; in sbcut_internal()
1660 if (m == NULL) { in sbcut_internal()
1661 sb->sb_mbtail = NULL; in sbcut_internal()
1662 sb->sb_lastrecord = NULL; in sbcut_internal()
1663 } else if (m->m_nextpkt == NULL) { in sbcut_internal()
1664 sb->sb_lastrecord = m; in sbcut_internal()
1708 struct mbuf *m; in sbsndptr_noadv() local
1710 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); in sbsndptr_noadv()
1711 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { in sbsndptr_noadv()
1713 if (sb->sb_sndptr == NULL) { in sbsndptr_noadv()
1714 sb->sb_sndptr = sb->sb_mb; in sbsndptr_noadv()
1715 sb->sb_sndptroff = 0; in sbsndptr_noadv()
1717 return (sb->sb_mb); in sbsndptr_noadv()
1719 m = sb->sb_sndptr; in sbsndptr_noadv()
1720 off -= sb->sb_sndptroff; in sbsndptr_noadv()
1723 return (m); in sbsndptr_noadv()
1733 struct mbuf *m; in sbsndptr_adv() local
1735 if (mb != sb->sb_sndptr) { in sbsndptr_adv()
1739 m = mb; in sbsndptr_adv()
1740 while (m && (len > 0)) { in sbsndptr_adv()
1741 if (len >= m->m_len) { in sbsndptr_adv()
1742 len -= m->m_len; in sbsndptr_adv()
1743 if (m->m_next) { in sbsndptr_adv()
1744 sb->sb_sndptroff += m->m_len; in sbsndptr_adv()
1745 sb->sb_sndptr = m->m_next; in sbsndptr_adv()
1747 m = m->m_next; in sbsndptr_adv()
1761 struct mbuf *m; in sbsndmbuf() local
1763 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); in sbsndmbuf()
1769 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { in sbsndmbuf()
1770 m = sb->sb_mb; in sbsndmbuf()
1772 m = sb->sb_sndptr; in sbsndmbuf()
1773 off -= sb->sb_sndptroff; in sbsndmbuf()
1775 while (off > 0 && m != NULL) { in sbsndmbuf()
1776 if (off < m->m_len) in sbsndmbuf()
1778 off -= m->m_len; in sbsndmbuf()
1779 m = m->m_next; in sbsndmbuf()
1782 return (m); in sbsndmbuf()
1792 struct mbuf *m; in sbdroprecord_locked() local
1796 m = sb->sb_mb; in sbdroprecord_locked()
1797 if (m) { in sbdroprecord_locked()
1798 sb->sb_mb = m->m_nextpkt; in sbdroprecord_locked()
1800 sbfree(sb, m); in sbdroprecord_locked()
1801 m = m_free(m); in sbdroprecord_locked()
1802 } while (m); in sbdroprecord_locked()
1828 struct mbuf *m; in sbcreatecontrol() local
1840 m = m_getcl(wait, MT_CONTROL, 0); in sbcreatecontrol()
1842 m = m_get(wait, MT_CONTROL); in sbcreatecontrol()
1843 if (m == NULL) in sbcreatecontrol()
1846 KASSERT(CMSG_SPACE(size) <= M_TRAILINGSPACE(m), in sbcreatecontrol()
1850 * cmsg data and the padding after the cmsg data un-initialized. in sbcreatecontrol()
1852 cp = mtod(m, struct cmsghdr *); in sbcreatecontrol()
1856 m->m_len = CMSG_SPACE(size); in sbcreatecontrol()
1857 cp->cmsg_len = CMSG_LEN(size); in sbcreatecontrol()
1858 cp->cmsg_level = level; in sbcreatecontrol()
1859 cp->cmsg_type = type; in sbcreatecontrol()
1860 return (m); in sbcreatecontrol()
1865 * generate an user-format data structure describing the socket buffer. Note
1874 xsb->sb_cc = sb->sb_ccc; in sbtoxsockbuf()
1875 xsb->sb_hiwat = sb->sb_hiwat; in sbtoxsockbuf()
1876 xsb->sb_mbcnt = sb->sb_mbcnt; in sbtoxsockbuf()
1877 xsb->sb_mbmax = sb->sb_mbmax; in sbtoxsockbuf()
1878 xsb->sb_lowat = sb->sb_lowat; in sbtoxsockbuf()
1879 xsb->sb_flags = sb->sb_flags; in sbtoxsockbuf()
1880 xsb->sb_timeo = sb->sb_timeo; in sbtoxsockbuf()