Lines Matching +full:min +full:- +full:len
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
183 * to compiler-induced padding and alignment artifacts.
185 CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
186 CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
189 * mbuf data storage should be 64-bit aligned regardless of architectural
196 * While the specific values here don't matter too much (i.e., +/- a few
199 * network-protocol and device-driver modules encode these layouts, and must
214 /* PowerPC booke has 64-bit physical pointers. */
242 KASSERT(m->m_flags & (M_EXT | M_EXTPG), in mb_dupcl()
244 KASSERT(!(n->m_flags & (M_EXT | M_EXTPG)), in mb_dupcl()
256 * special - it needs full copy of m_ext into each mbuf, since any in mb_dupcl()
259 if (m->m_flags & M_EXTPG) { in mb_dupcl()
260 bcopy(&m->m_epg_startcopy, &n->m_epg_startcopy, in mb_dupcl()
262 bcopy(&m->m_ext, &n->m_ext, m_epg_ext_copylen); in mb_dupcl()
263 } else if (m->m_ext.ext_type == EXT_EXTREF) in mb_dupcl()
264 bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext)); in mb_dupcl()
266 bcopy(&m->m_ext, &n->m_ext, m_ext_copylen); in mb_dupcl()
268 n->m_flags |= m->m_flags & (M_RDONLY | M_EXT | M_EXTPG); in mb_dupcl()
271 if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { in mb_dupcl()
272 refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count; in mb_dupcl()
273 n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF; in mb_dupcl()
275 KASSERT(m->m_ext.ext_cnt != NULL, in mb_dupcl()
277 refcnt = m->m_ext.ext_cnt; in mb_dupcl()
294 m->m_flags &= ~M_PKTHDR; in m_demote_pkthdr()
295 bzero(&m->m_pkthdr, sizeof(struct pkthdr)); in m_demote_pkthdr()
310 for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { in m_demote()
311 KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", in m_demote()
313 if (m->m_flags & M_PKTHDR) in m_demote()
315 m->m_flags &= flags; in m_demote()
339 for (m = m0; m != NULL; m = m->m_next) { in m_sanity()
347 if ((caddr_t)m->m_data < a) in m_sanity()
349 if ((caddr_t)m->m_data > b) in m_sanity()
351 if ((caddr_t)m->m_data + m->m_len > b) in m_sanity()
354 /* m->m_nextpkt may only be set on first mbuf in chain. */ in m_sanity()
355 if (m != m0 && m->m_nextpkt != NULL) { in m_sanity()
357 m_freem(m->m_nextpkt); in m_sanity()
358 m->m_nextpkt = (struct mbuf *)0xDEADC0DE; in m_sanity()
360 M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); in m_sanity()
364 if (m0->m_flags & M_PKTHDR) in m_sanity()
365 pktlen += m->m_len; in m_sanity()
368 if (m != m0 && m->m_flags & M_PKTHDR && in m_sanity()
369 !SLIST_EMPTY(&m->m_pkthdr.tags)) { in m_sanity()
374 M_SANITY_ACTION("m_tags on in-chain mbuf"); in m_sanity()
378 if (m != m0 && m->m_flags & M_PKTHDR) { in m_sanity()
380 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); in m_sanity()
381 m->m_flags &= ~M_PKTHDR; in m_sanity()
384 M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); in m_sanity()
388 if (pktlen && pktlen != m->m_pkthdr.len) { in m_sanity()
390 m->m_pkthdr.len = 0; in m_sanity()
392 M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); in m_sanity()
400 * Non-inlined part of m_init().
408 m->m_data = m->m_pktdat; in m_pkthdr_init()
409 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); in m_pkthdr_init()
411 m->m_pkthdr.numa_domain = M_NODOM; in m_pkthdr_init()
435 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), in m_move_pkthdr()
440 * XXXMAC: It could be this should also occur for non-MAC? in m_move_pkthdr()
442 if (to->m_flags & M_PKTHDR) in m_move_pkthdr()
445 to->m_flags = (from->m_flags & M_COPYFLAGS) | in m_move_pkthdr()
446 (to->m_flags & (M_EXT | M_EXTPG)); in m_move_pkthdr()
447 if ((to->m_flags & M_EXT) == 0) in m_move_pkthdr()
448 to->m_data = to->m_pktdat; in m_move_pkthdr()
449 to->m_pkthdr = from->m_pkthdr; /* especially tags */ in m_move_pkthdr()
450 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ in m_move_pkthdr()
451 from->m_flags &= ~M_PKTHDR; in m_move_pkthdr()
452 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_move_pkthdr()
453 from->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; in m_move_pkthdr()
454 from->m_pkthdr.snd_tag = NULL; in m_move_pkthdr()
477 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); in m_dup_pkthdr()
481 if (to->m_flags & M_PKTHDR) in m_dup_pkthdr()
484 to->m_flags = (from->m_flags & M_COPYFLAGS) | in m_dup_pkthdr()
485 (to->m_flags & (M_EXT | M_EXTPG)); in m_dup_pkthdr()
486 if ((to->m_flags & M_EXT) == 0) in m_dup_pkthdr()
487 to->m_data = to->m_pktdat; in m_dup_pkthdr()
488 to->m_pkthdr = from->m_pkthdr; in m_dup_pkthdr()
489 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) in m_dup_pkthdr()
490 m_snd_tag_ref(from->m_pkthdr.snd_tag); in m_dup_pkthdr()
491 SLIST_INIT(&to->m_pkthdr.tags); in m_dup_pkthdr()
496 * Lesser-used path for M_PREPEND:
501 m_prepend(struct mbuf *m, int len, int how) in m_prepend() argument
505 if (m->m_flags & M_PKTHDR) in m_prepend()
506 mn = m_gethdr(how, m->m_type); in m_prepend()
508 mn = m_get(how, m->m_type); in m_prepend()
513 if (m->m_flags & M_PKTHDR) in m_prepend()
515 mn->m_next = m; in m_prepend()
517 if (len < M_SIZE(m)) in m_prepend()
518 M_ALIGN(m, len); in m_prepend()
519 m->m_len = len; in m_prepend()
525 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
527 * Note that the copy is read-only, because clusters are not copied,
531 m_copym(struct mbuf *m, int off0, int len, int wait) in m_copym() argument
539 KASSERT(len >= 0, ("m_copym, negative len %d", len)); in m_copym()
541 if (off == 0 && m->m_flags & M_PKTHDR) in m_copym()
545 if (off < m->m_len) in m_copym()
547 off -= m->m_len; in m_copym()
548 m = m->m_next; in m_copym()
552 while (len > 0) { in m_copym()
554 KASSERT(len == M_COPYALL, in m_copym()
559 n = m_gethdr(wait, m->m_type); in m_copym()
561 n = m_get(wait, m->m_type); in m_copym()
568 if (len == M_COPYALL) in m_copym()
569 n->m_pkthdr.len -= off0; in m_copym()
571 n->m_pkthdr.len = len; in m_copym()
574 n->m_len = min(len, m->m_len - off); in m_copym()
575 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copym()
576 n->m_data = m->m_data + off; in m_copym()
580 (u_int)n->m_len); in m_copym()
581 if (len != M_COPYALL) in m_copym()
582 len -= n->m_len; in m_copym()
584 m = m->m_next; in m_copym()
585 np = &n->m_next; in m_copym()
597 * Note that the copy is read-only, because clusters are not copied,
609 n = m_get(how, m->m_type); in m_copypacket()
616 n->m_len = m->m_len; in m_copypacket()
617 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copypacket()
618 n->m_data = m->m_data; in m_copypacket()
621 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); in m_copypacket()
622 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); in m_copypacket()
625 m = m->m_next; in m_copypacket()
627 o = m_get(how, m->m_type); in m_copypacket()
631 n->m_next = o; in m_copypacket()
632 n = n->m_next; in m_copypacket()
634 n->m_len = m->m_len; in m_copypacket()
635 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copypacket()
636 n->m_data = m->m_data; in m_copypacket()
639 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); in m_copypacket()
642 m = m->m_next; in m_copypacket()
651 m_copyfromunmapped(const struct mbuf *m, int off, int len, caddr_t cp) in m_copyfromunmapped() argument
658 KASSERT(len >= 0, ("m_copyfromunmapped: negative len %d", len)); in m_copyfromunmapped()
659 KASSERT(off < m->m_len, in m_copyfromunmapped()
660 ("m_copyfromunmapped: len exceeds mbuf length")); in m_copyfromunmapped()
662 iov.iov_len = len; in m_copyfromunmapped()
663 uio.uio_resid = len; in m_copyfromunmapped()
669 error = m_unmapped_uiomove(m, off, &uio, len); in m_copyfromunmapped()
670 KASSERT(error == 0, ("m_unmapped_uiomove failed: off %d, len %d", off, in m_copyfromunmapped()
671 len)); in m_copyfromunmapped()
676 * continuing for "len" bytes, into the indicated buffer.
679 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) in m_copydata() argument
684 KASSERT(len >= 0, ("m_copydata, negative len %d", len)); in m_copydata()
687 if (off < m->m_len) in m_copydata()
689 off -= m->m_len; in m_copydata()
690 m = m->m_next; in m_copydata()
692 while (len > 0) { in m_copydata()
694 count = min(m->m_len - off, len); in m_copydata()
695 if ((m->m_flags & M_EXTPG) != 0) in m_copydata()
699 len -= count; in m_copydata()
702 m = m->m_next; in m_copydata()
724 remain = m->m_pkthdr.len; in m_dup()
727 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ in m_dup()
732 n = m_getcl(how, m->m_type, 0); in m_dup()
735 n = m_get(how, m->m_type); in m_dup()
746 if ((n->m_flags & M_EXT) == 0) in m_dup()
748 n->m_flags &= ~M_RDONLY; in m_dup()
750 n->m_len = 0; in m_dup()
754 p = &n->m_next; in m_dup()
757 while (n->m_len < nsize && m != NULL) { in m_dup()
758 int chunk = min(nsize - n->m_len, m->m_len - moff); in m_dup()
760 m_copydata(m, moff, chunk, n->m_data + n->m_len); in m_dup()
762 n->m_len += chunk; in m_dup()
763 remain -= chunk; in m_dup()
764 if (moff == m->m_len) { in m_dup()
765 m = m->m_next; in m_dup()
772 ("%s: bogus m_pkthdr.len", __func__)); in m_dup()
789 while (m->m_next) in m_cat()
790 m = m->m_next; in m_cat()
793 (n->m_flags & M_EXTPG) != 0 || in m_cat()
794 M_TRAILINGSPACE(m) < n->m_len) { in m_cat()
796 m->m_next = n; in m_cat()
800 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, in m_cat()
801 (u_int)n->m_len); in m_cat()
802 m->m_len += n->m_len; in m_cat()
817 m->m_pkthdr.len += n->m_pkthdr.len; in m_catpkt()
826 int len = req_len; in m_adj() local
832 if (len >= 0) { in m_adj()
836 while (m != NULL && len > 0) { in m_adj()
837 if (m->m_len <= len) { in m_adj()
838 len -= m->m_len; in m_adj()
839 m->m_len = 0; in m_adj()
840 m = m->m_next; in m_adj()
842 m->m_len -= len; in m_adj()
843 m->m_data += len; in m_adj()
844 len = 0; in m_adj()
847 if (mp->m_flags & M_PKTHDR) in m_adj()
848 mp->m_pkthdr.len -= (req_len - len); in m_adj()
857 len = -len; in m_adj()
860 count += m->m_len; in m_adj()
861 if (m->m_next == (struct mbuf *)0) in m_adj()
863 m = m->m_next; in m_adj()
865 if (m->m_len >= len) { in m_adj()
866 m->m_len -= len; in m_adj()
867 if (mp->m_flags & M_PKTHDR) in m_adj()
868 mp->m_pkthdr.len -= len; in m_adj()
871 count -= len; in m_adj()
880 if (m->m_flags & M_PKTHDR) in m_adj()
881 m->m_pkthdr.len = count; in m_adj()
882 for (; m; m = m->m_next) { in m_adj()
883 if (m->m_len >= count) { in m_adj()
884 m->m_len = count; in m_adj()
885 if (m->m_next != NULL) { in m_adj()
886 m_freem(m->m_next); in m_adj()
887 m->m_next = NULL; in m_adj()
891 count -= m->m_len; in m_adj()
897 m_adj_decap(struct mbuf *mp, int len) in m_adj_decap() argument
901 m_adj(mp, len); in m_adj_decap()
902 if ((mp->m_flags & M_PKTHDR) != 0) { in m_adj_decap()
910 rsstype = mp->m_pkthdr.rsstype; in m_adj_decap()
920 * Rearange an mbuf chain so that len bytes are contiguous
922 * for a structure of size len). Returns the resulting
924 * If there is room, it will add up to max_protohdr-len extra bytes to the
928 m_pullup(struct mbuf *n, int len) in m_pullup() argument
934 KASSERT((n->m_flags & M_EXTPG) == 0, in m_pullup()
938 * If first mbuf has no cluster, and has room for len bytes in m_pullup()
942 if ((n->m_flags & M_EXT) == 0 && in m_pullup()
943 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { in m_pullup()
944 if (n->m_len >= len) in m_pullup()
947 n = n->m_next; in m_pullup()
948 len -= m->m_len; in m_pullup()
950 if (len > MHLEN) in m_pullup()
952 m = m_get(M_NOWAIT, n->m_type); in m_pullup()
955 if (n->m_flags & M_PKTHDR) in m_pullup()
958 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); in m_pullup()
960 count = min(min(max(len, max_protohdr), space), n->m_len); in m_pullup()
961 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, in m_pullup()
963 len -= count; in m_pullup()
964 m->m_len += count; in m_pullup()
965 n->m_len -= count; in m_pullup()
966 space -= count; in m_pullup()
967 if (n->m_len) in m_pullup()
968 n->m_data += count; in m_pullup()
971 } while (len > 0 && n); in m_pullup()
972 if (len > 0) { in m_pullup()
976 m->m_next = n; in m_pullup()
989 m_copyup(struct mbuf *n, int len, int dstoff) in m_copyup() argument
994 if (len > (MHLEN - dstoff)) in m_copyup()
996 m = m_get(M_NOWAIT, n->m_type); in m_copyup()
999 if (n->m_flags & M_PKTHDR) in m_copyup()
1001 m->m_data += dstoff; in m_copyup()
1002 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); in m_copyup()
1004 count = min(min(max(len, max_protohdr), space), n->m_len); in m_copyup()
1005 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), in m_copyup()
1007 len -= count; in m_copyup()
1008 m->m_len += count; in m_copyup()
1009 n->m_len -= count; in m_copyup()
1010 space -= count; in m_copyup()
1011 if (n->m_len) in m_copyup()
1012 n->m_data += count; in m_copyup()
1015 } while (len > 0 && n); in m_copyup()
1016 if (len > 0) { in m_copyup()
1020 m->m_next = n; in m_copyup()
1028 * Partition an mbuf chain in two pieces, returning the tail --
1032 * Note that the resulting mbufs might be read-only, because the new
1041 u_int len = len0, remain; in m_split() local
1044 for (m = m0; m && len > m->m_len; m = m->m_next) in m_split()
1045 len -= m->m_len; in m_split()
1048 remain = m->m_len - len; in m_split()
1049 if (m0->m_flags & M_PKTHDR && remain == 0) { in m_split()
1050 n = m_gethdr(wait, m0->m_type); in m_split()
1053 n->m_next = m->m_next; in m_split()
1054 m->m_next = NULL; in m_split()
1055 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_split()
1056 n->m_pkthdr.snd_tag = in m_split()
1057 m_snd_tag_ref(m0->m_pkthdr.snd_tag); in m_split()
1058 n->m_pkthdr.csum_flags |= CSUM_SND_TAG; in m_split()
1060 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; in m_split()
1061 n->m_pkthdr.len = m0->m_pkthdr.len - len0; in m_split()
1062 m0->m_pkthdr.len = len0; in m_split()
1064 } else if (m0->m_flags & M_PKTHDR) { in m_split()
1065 n = m_gethdr(wait, m0->m_type); in m_split()
1068 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_split()
1069 n->m_pkthdr.snd_tag = in m_split()
1070 m_snd_tag_ref(m0->m_pkthdr.snd_tag); in m_split()
1071 n->m_pkthdr.csum_flags |= CSUM_SND_TAG; in m_split()
1073 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; in m_split()
1074 n->m_pkthdr.len = m0->m_pkthdr.len - len0; in m_split()
1075 m0->m_pkthdr.len = len0; in m_split()
1076 if (m->m_flags & (M_EXT | M_EXTPG)) in m_split()
1081 n->m_next = m_split(m, len, wait); in m_split()
1082 if (n->m_next == NULL) { in m_split()
1086 n->m_len = 0; in m_split()
1092 n = m->m_next; in m_split()
1093 m->m_next = NULL; in m_split()
1096 n = m_get(wait, m->m_type); in m_split()
1102 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_split()
1103 n->m_data = m->m_data + len; in m_split()
1106 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); in m_split()
1108 n->m_len = remain; in m_split()
1109 m->m_len = len; in m_split()
1110 n->m_next = m->m_next; in m_split()
1111 m->m_next = NULL; in m_split()
1119 * Resulting mbufs might be read-only.
1125 u_int len, mlen, remain; in mc_split() local
1127 MPASS(!(mc_first(head)->m_flags & M_PKTHDR)); in mc_split()
1131 len = len0; in mc_split()
1132 STAILQ_FOREACH(m, &head->mc_q, m_stailq) { in mc_split()
1134 if (m->m_flags & M_EXT) in mc_split()
1135 mlen += m->m_ext.ext_size; in mc_split()
1136 if (len > m->m_len) in mc_split()
1137 len -= m->m_len; in mc_split()
1145 remain = m->m_len - len; in mc_split()
1147 if (__predict_false((n = m_get(wait, m->m_type)) == NULL)) in mc_split()
1150 if (m->m_flags & M_EXT) { in mc_split()
1151 n->m_data = m->m_data + len; in mc_split()
1154 bcopy(mtod(m, char *) + len, mtod(n, char *), remain); in mc_split()
1158 STAILQ_FIRST(&tail->mc_q) = STAILQ_NEXT(m, m_stailq); in mc_split()
1159 tail->mc_q.stqh_last = head->mc_q.stqh_last; in mc_split()
1160 tail->mc_len = head->mc_len - len0; in mc_split()
1161 tail->mc_mlen = head->mc_mlen - mlen; in mc_split()
1163 MPASS(n->m_len == 0); in mc_split()
1165 n->m_len = remain; in mc_split()
1166 m->m_len -= remain; in mc_split()
1167 if (m->m_flags & M_EOR) { in mc_split()
1168 m->m_flags &= ~M_EOR; in mc_split()
1169 n->m_flags |= M_EOR; in mc_split()
1172 head->mc_q.stqh_last = &STAILQ_NEXT(m, m_stailq); in mc_split()
1174 head->mc_len = len0; in mc_split()
1175 head->mc_mlen = mlen; in mc_split()
1187 void (*copy)(char *from, caddr_t to, u_int len)) in m_devget() argument
1191 int len; in m_devget() local
1200 len = MCLBYTES; in m_devget()
1203 len = MHLEN; in m_devget()
1207 m->m_data += max_linkhdr; in m_devget()
1208 len -= max_linkhdr; in m_devget()
1213 m->m_pkthdr.rcvif = ifp; in m_devget()
1214 m->m_pkthdr.len = totlen; in m_devget()
1218 len = MCLBYTES; in m_devget()
1221 len = MLEN; in m_devget()
1229 m->m_data += off; in m_devget()
1230 len -= off; in m_devget()
1233 m->m_len = len = min(totlen, len); in m_devget()
1235 copy(buf, mtod(m, caddr_t), (u_int)len); in m_devget()
1237 bcopy(buf, mtod(m, caddr_t), (u_int)len); in m_devget()
1238 buf += len; in m_devget()
1240 mp = &m->m_next; in m_devget()
1241 totlen -= len; in m_devget()
1247 m_copytounmapped(const struct mbuf *m, int off, int len, c_caddr_t cp) in m_copytounmapped() argument
1254 KASSERT(len >= 0, ("m_copytounmapped: negative len %d", len)); in m_copytounmapped()
1255 KASSERT(off < m->m_len, ("m_copytounmapped: len exceeds mbuf length")); in m_copytounmapped()
1257 iov.iov_len = len; in m_copytounmapped()
1258 uio.uio_resid = len; in m_copytounmapped()
1264 error = m_unmapped_uiomove(m, off, &uio, len); in m_copytounmapped()
1265 KASSERT(error == 0, ("m_unmapped_uiomove failed: off %d, len %d", off, in m_copytounmapped()
1266 len)); in m_copytounmapped()
1275 m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) in m_copyback() argument
1283 while (off > (mlen = m->m_len)) { in m_copyback()
1284 off -= mlen; in m_copyback()
1286 if (m->m_next == NULL) { in m_copyback()
1287 n = m_get(M_NOWAIT, m->m_type); in m_copyback()
1291 n->m_len = min(MLEN, len + off); in m_copyback()
1292 m->m_next = n; in m_copyback()
1294 m = m->m_next; in m_copyback()
1296 while (len > 0) { in m_copyback()
1297 if (m->m_next == NULL && (len > m->m_len - off)) { in m_copyback()
1298 m->m_len += min(len - (m->m_len - off), in m_copyback()
1301 mlen = min (m->m_len - off, len); in m_copyback()
1302 if ((m->m_flags & M_EXTPG) != 0) in m_copyback()
1307 len -= mlen; in m_copyback()
1311 if (len == 0) in m_copyback()
1313 if (m->m_next == NULL) { in m_copyback()
1314 n = m_get(M_NOWAIT, m->m_type); in m_copyback()
1317 n->m_len = min(MLEN, len); in m_copyback()
1318 m->m_next = n; in m_copyback()
1320 m = m->m_next; in m_copyback()
1322 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) in m_copyback()
1323 m->m_pkthdr.len = totlen; in m_copyback()
1334 m_append(struct mbuf *m0, int len, c_caddr_t cp) in m_append() argument
1339 for (m = m0; m->m_next != NULL; m = m->m_next) in m_append()
1341 remainder = len; in m_append()
1349 bcopy(cp, mtod(m, caddr_t) + m->m_len, space); in m_append()
1350 m->m_len += space; in m_append()
1351 cp += space, remainder -= space; in m_append()
1358 n = m_get(M_NOWAIT, m->m_type); in m_append()
1361 n->m_len = min(MLEN, remainder); in m_append()
1362 bcopy(cp, mtod(n, caddr_t), n->m_len); in m_append()
1363 cp += n->m_len, remainder -= n->m_len; in m_append()
1364 m->m_next = n; in m_append()
1367 if (m0->m_flags & M_PKTHDR) in m_append()
1368 m0->m_pkthdr.len += len - remainder; in m_append()
1373 m_apply_extpg_one(struct mbuf *m, int off, int len, in m_apply_extpg_one() argument
1383 if (off < m->m_epg_hdrlen) { in m_apply_extpg_one()
1384 count = min(m->m_epg_hdrlen - off, len); in m_apply_extpg_one()
1385 rval = f(arg, m->m_epg_hdr + off, count); in m_apply_extpg_one()
1388 len -= count; in m_apply_extpg_one()
1391 off -= m->m_epg_hdrlen; in m_apply_extpg_one()
1392 pgoff = m->m_epg_1st_off; in m_apply_extpg_one()
1393 for (i = 0; i < m->m_epg_npgs && len > 0; i++) { in m_apply_extpg_one()
1396 count = min(pglen - off, len); in m_apply_extpg_one()
1397 p = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff + off); in m_apply_extpg_one()
1401 len -= count; in m_apply_extpg_one()
1404 off -= pglen; in m_apply_extpg_one()
1407 if (len > 0) { in m_apply_extpg_one()
1408 KASSERT(off < m->m_epg_trllen, in m_apply_extpg_one()
1410 KASSERT(len <= m->m_epg_trllen - off, in m_apply_extpg_one()
1412 return (f(arg, m->m_epg_trail + off, len)); in m_apply_extpg_one()
1419 m_apply_one(struct mbuf *m, int off, int len, in m_apply_one() argument
1422 if ((m->m_flags & M_EXTPG) != 0) in m_apply_one()
1423 return (m_apply_extpg_one(m, off, len, f, arg)); in m_apply_one()
1425 return (f(arg, mtod(m, caddr_t) + off, len)); in m_apply_one()
1430 * the beginning, continuing for "len" bytes.
1433 m_apply(struct mbuf *m, int off, int len, in m_apply() argument
1440 KASSERT(len >= 0, ("m_apply, negative len %d", len)); in m_apply()
1444 if (off < m->m_len) in m_apply()
1446 off -= m->m_len; in m_apply()
1447 m = m->m_next; in m_apply()
1449 while (len > 0) { in m_apply()
1451 "(%d extra)", len)); in m_apply()
1452 count = min(m->m_len - off, len); in m_apply()
1456 len -= count; in m_apply()
1458 m = m->m_next; in m_apply()
1472 if (m->m_len > loc) { in m_getptr()
1476 loc -= m->m_len; in m_getptr()
1477 if (m->m_next == NULL) { in m_getptr()
1480 *off = m->m_len; in m_getptr()
1485 m = m->m_next; in m_getptr()
1494 int len; in m_print() local
1503 if (m->m_flags & M_PKTHDR) in m_print()
1504 len = m->m_pkthdr.len; in m_print()
1506 len = -1; in m_print()
1508 while (m2 != NULL && (len == -1 || len)) { in m_print()
1509 pdata = m2->m_len; in m_print()
1510 if (maxlen != -1 && pdata > maxlen) in m_print()
1512 printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, in m_print()
1513 m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" in m_print()
1517 printf(", %*D\n", pdata, (u_char *)m2->m_data, "-"); in m_print()
1518 if (len != -1) in m_print()
1519 len -= m2->m_len; in m_print()
1520 m2 = m2->m_next; in m_print()
1522 if (len > 0) in m_print()
1523 printf("%d bytes unaccounted for.\n", len); in m_print()
1530 u_int len; in m_fixhdr() local
1532 len = m_length(m0, NULL); in m_fixhdr()
1533 m0->m_pkthdr.len = len; in m_fixhdr()
1534 return (len); in m_fixhdr()
1541 u_int len; in m_length() local
1543 len = 0; in m_length()
1544 for (m = m0; m != NULL; m = m->m_next) { in m_length()
1545 len += m->m_len; in m_length()
1546 if (m->m_next == NULL) in m_length()
1551 return (len); in m_length()
1562 * If a non-packet header is passed in, the original
1572 if (!(m0->m_flags & M_PKTHDR)) in m_defrag()
1585 if (m0->m_pkthdr.len > MHLEN) in m_defrag()
1598 while (progress < m0->m_pkthdr.len) { in m_defrag()
1599 length = m0->m_pkthdr.len - progress; in m_defrag()
1614 m_new->m_len = length; in m_defrag()
1620 if (m0->m_next == NULL) in m_defrag()
1627 m_defragbytes += m0->m_pkthdr.len; in m_defrag()
1652 if ((m->m_flags & M_EXTPG) == 0) in frags_per_mbuf()
1663 if (m->m_epg_hdrlen != 0) in frags_per_mbuf()
1665 frags += m->m_epg_npgs; in frags_per_mbuf()
1666 if (m->m_epg_trllen != 0) in frags_per_mbuf()
1691 for (m = m0; m != NULL; m = m->m_next) in m_collapse()
1702 n = m->m_next; in m_collapse()
1706 n->m_len < M_TRAILINGSPACE(m)) { in m_collapse()
1707 m_copydata(n, 0, n->m_len, in m_collapse()
1708 mtod(m, char *) + m->m_len); in m_collapse()
1709 m->m_len += n->m_len; in m_collapse()
1710 m->m_next = n->m_next; in m_collapse()
1711 curfrags -= frags_per_mbuf(n); in m_collapse()
1723 prev = &m0->m_next; /* NB: not the first mbuf */ in m_collapse()
1725 if ((n2 = n->m_next) != NULL && in m_collapse()
1726 n->m_len + n2->m_len < MCLBYTES) { in m_collapse()
1730 m_copydata(n, 0, n->m_len, mtod(m, char *)); in m_collapse()
1731 m_copydata(n2, 0, n2->m_len, in m_collapse()
1732 mtod(m, char *) + n->m_len); in m_collapse()
1733 m->m_len = n->m_len + n2->m_len; in m_collapse()
1734 m->m_next = n2->m_next; in m_collapse()
1737 curfrags -= frags_per_mbuf(n); in m_collapse()
1738 curfrags -= frags_per_mbuf(n2); in m_collapse()
1749 prev = &n->m_next; in m_collapse()
1775 * -1 each fragment will be the same random value in length
1776 * -2 each fragment's length will be entirely random
1785 if (!(m0->m_flags & M_PKTHDR)) in m_fragment()
1788 if (length == 0 || length < -2) in m_fragment()
1794 if (length == -1) in m_fragment()
1810 while (progress < m0->m_pkthdr.len) { in m_fragment()
1811 if (length == -2) in m_fragment()
1813 if (fraglen > m0->m_pkthdr.len - progress) in m_fragment()
1814 fraglen = m0->m_pkthdr.len - progress; in m_fragment()
1821 m_last->m_next = m_new; in m_fragment()
1827 m_last->m_len = fraglen; in m_fragment()
1852 for (int i = 0; i < m->m_epg_npgs; i++) { in mb_free_mext_pgs()
1853 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); in mb_free_mext_pgs()
1860 m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags) in m_uiotombuf_nomap() argument
1872 * len can be zero or an arbitrary large value bound by in m_uiotombuf_nomap()
1875 if (len > 0) in m_uiotombuf_nomap()
1876 total = MIN(uio->uio_resid, len); in m_uiotombuf_nomap()
1878 total = uio->uio_resid; in m_uiotombuf_nomap()
1886 * a countermeasure against the known-IV weakness in CBC in m_uiotombuf_nomap()
1893 mb->m_epg_flags = EPG_FLAG_ANON; in m_uiotombuf_nomap()
1908 prev->m_next = mb; in m_uiotombuf_nomap()
1910 mb->m_epg_flags = EPG_FLAG_ANON; in m_uiotombuf_nomap()
1911 needed = length = MIN(maxseg, total); in m_uiotombuf_nomap()
1912 for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) { in m_uiotombuf_nomap()
1923 mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]); in m_uiotombuf_nomap()
1924 mb->m_epg_npgs++; in m_uiotombuf_nomap()
1926 mb->m_epg_last_len = length - PAGE_SIZE * (mb->m_epg_npgs - 1); in m_uiotombuf_nomap()
1928 total -= length; in m_uiotombuf_nomap()
1932 mb->m_len = length; in m_uiotombuf_nomap()
1933 mb->m_ext.ext_size += PAGE_SIZE * mb->m_epg_npgs; in m_uiotombuf_nomap()
1935 m->m_pkthdr.len += length; in m_uiotombuf_nomap()
1950 m_uiotombuf(struct uio *uio, int how, int len, int lspace, int flags) in m_uiotombuf() argument
1955 return (m_uiotombuf_nomap(uio, how, len, lspace, flags)); in m_uiotombuf()
1956 } else if (__predict_false(uio->uio_resid == 0)) { in m_uiotombuf()
1965 m->m_pkthdr.memlen = MSIZE; in m_uiotombuf()
1969 m->m_data += lspace; in m_uiotombuf()
1975 error = mc_uiotomc(&mc, uio, len, lspace, how, flags); in m_uiotombuf()
1978 mc_first(&mc)->m_pkthdr.len = mc.mc_len; in m_uiotombuf()
1979 mc_first(&mc)->m_pkthdr.memlen = mc.mc_mlen; in m_uiotombuf()
2001 MPASS(UINT_MAX - lspace >= length); in mc_uiotomc()
2002 MPASS(uio->uio_rw == UIO_WRITE); in mc_uiotomc()
2003 MPASS(uio->uio_resid >= 0); in mc_uiotomc()
2006 if (uio->uio_resid > length) { in mc_uiotomc()
2010 total = uio->uio_resid; in mc_uiotomc()
2011 } else if (__predict_false(uio->uio_resid + lspace > UINT_MAX)) in mc_uiotomc()
2014 total = uio->uio_resid; in mc_uiotomc()
2024 mc_first(mc)->m_data += lspace; in mc_uiotomc()
2027 STAILQ_FOREACH(mb, &mc->mc_q, m_stailq) { in mc_uiotomc()
2030 mlen = min(M_TRAILINGSPACE(mb), total - mc->mc_len); in mc_uiotomc()
2037 mb->m_len = mlen; in mc_uiotomc()
2038 mc->mc_len += mlen; in mc_uiotomc()
2040 MPASS(mc->mc_len == total); in mc_uiotomc()
2046 * Copy data to/from an unmapped mbuf into a uio limited by len if set.
2049 m_unmapped_uiomove(const struct mbuf *m, int m_off, struct uio *uio, int len) in m_unmapped_uiomove() argument
2061 if (m->m_epg_hdrlen != 0) { in m_unmapped_uiomove()
2062 if (off >= m->m_epg_hdrlen) { in m_unmapped_uiomove()
2063 off -= m->m_epg_hdrlen; in m_unmapped_uiomove()
2065 seglen = m->m_epg_hdrlen - off; in m_unmapped_uiomove()
2067 seglen = min(seglen, len); in m_unmapped_uiomove()
2069 len -= seglen; in m_unmapped_uiomove()
2071 &m->m_epg_hdr[segoff]), seglen, uio); in m_unmapped_uiomove()
2074 pgoff = m->m_epg_1st_off; in m_unmapped_uiomove()
2075 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) { in m_unmapped_uiomove()
2078 off -= pglen; in m_unmapped_uiomove()
2082 seglen = pglen - off; in m_unmapped_uiomove()
2085 seglen = min(seglen, len); in m_unmapped_uiomove()
2086 len -= seglen; in m_unmapped_uiomove()
2087 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); in m_unmapped_uiomove()
2091 if (len != 0 && error == 0) { in m_unmapped_uiomove()
2092 KASSERT((off + len) <= m->m_epg_trllen, in m_unmapped_uiomove()
2093 ("off + len > trail (%d + %d > %d, m_off = %d)", off, len, in m_unmapped_uiomove()
2094 m->m_epg_trllen, m_off)); in m_unmapped_uiomove()
2095 error = uiomove(__DECONST(void *, &m->m_epg_trail[off]), in m_unmapped_uiomove()
2096 len, uio); in m_unmapped_uiomove()
2102 * Copy an mbuf chain into a uio limited by len if set.
2105 m_mbuftouio(struct uio *uio, const struct mbuf *m, int len) in m_mbuftouio() argument
2110 if (len > 0) in m_mbuftouio()
2111 total = min(uio->uio_resid, len); in m_mbuftouio()
2113 total = uio->uio_resid; in m_mbuftouio()
2116 for (; m != NULL; m = m->m_next) { in m_mbuftouio()
2117 length = min(m->m_len, total - progress); in m_mbuftouio()
2119 if ((m->m_flags & M_EXTPG) != 0) in m_mbuftouio()
2146 int len, off; in m_unshare() local
2149 for (m = m0; m != NULL; m = mprev->m_next) { in m_unshare()
2155 * mbuf+cluster -> cluster). This optimization is heavily in m_unshare()
2162 if ((m->m_flags & M_EXT) == 0) { in m_unshare()
2163 if (mprev && (mprev->m_flags & M_EXT) && in m_unshare()
2164 m->m_len <= M_TRAILINGSPACE(mprev)) { in m_unshare()
2166 memcpy(mtod(mprev, caddr_t) + mprev->m_len, in m_unshare()
2167 mtod(m, caddr_t), m->m_len); in m_unshare()
2168 mprev->m_len += m->m_len; in m_unshare()
2169 mprev->m_next = m->m_next; /* unlink from chain */ in m_unshare()
2190 KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags)); in m_unshare()
2192 if (mprev != NULL && (mprev->m_flags & M_EXT) && in m_unshare()
2193 m->m_len <= M_TRAILINGSPACE(mprev)) { in m_unshare()
2195 memcpy(mtod(mprev, caddr_t) + mprev->m_len, in m_unshare()
2196 mtod(m, caddr_t), m->m_len); in m_unshare()
2197 mprev->m_len += m->m_len; in m_unshare()
2198 mprev->m_next = m->m_next; /* unlink from chain */ in m_unshare()
2208 * don't know how to break up the non-contiguous memory when in m_unshare()
2211 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); in m_unshare()
2216 if (m->m_flags & M_PKTHDR) { in m_unshare()
2221 len = m->m_len; in m_unshare()
2226 int cc = min(len, MCLBYTES); in m_unshare()
2228 n->m_len = cc; in m_unshare()
2230 mlast->m_next = n; in m_unshare()
2236 len -= cc; in m_unshare()
2237 if (len <= 0) in m_unshare()
2241 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); in m_unshare()
2248 n->m_next = m->m_next; in m_unshare()
2252 mprev->m_next = mfirst; /* replace old mbuf */ in m_unshare()
2277 used += m->m_len; in m_profile()
2278 if (m->m_flags & M_EXT) { in m_profile()
2279 wasted += MHLEN - sizeof(m->m_ext) + in m_profile()
2280 m->m_ext.ext_size - m->m_len; in m_profile()
2282 if (m->m_flags & M_PKTHDR) in m_profile()
2283 wasted += MHLEN - m->m_len; in m_profile()
2285 wasted += MLEN - m->m_len; in m_profile()
2287 m = m->m_next; in m_profile()
2290 if (segments > MP_BUCKETS - 1) in m_profile()
2291 segments = MP_BUCKETS - 1; in m_profile()
2371 if (error || !req->newptr) in mbprof_clr_handler()