Lines Matching +full:- +full:m
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
183 * to compiler-induced padding and alignment artifacts.
185 CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
186 CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
189 * mbuf data storage should be 64-bit aligned regardless of architectural
196 * While the specific values here don't matter too much (i.e., +/- a few
199 * network-protocol and device-driver modules encode these layouts, and must
214 /* PowerPC booke has 64-bit physical pointers. */
234 * Attach the cluster from *m to *n, set up m_ext in *n
238 mb_dupcl(struct mbuf *n, struct mbuf *m) in mb_dupcl() argument
242 KASSERT(m->m_flags & (M_EXT | M_EXTPG), in mb_dupcl()
243 ("%s: M_EXT | M_EXTPG not set on %p", __func__, m)); in mb_dupcl()
244 KASSERT(!(n->m_flags & (M_EXT | M_EXTPG)), in mb_dupcl()
256 * special - it needs full copy of m_ext into each mbuf, since any in mb_dupcl()
259 if (m->m_flags & M_EXTPG) { in mb_dupcl()
260 bcopy(&m->m_epg_startcopy, &n->m_epg_startcopy, in mb_dupcl()
262 bcopy(&m->m_ext, &n->m_ext, m_epg_ext_copylen); in mb_dupcl()
263 } else if (m->m_ext.ext_type == EXT_EXTREF) in mb_dupcl()
264 bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext)); in mb_dupcl()
266 bcopy(&m->m_ext, &n->m_ext, m_ext_copylen); in mb_dupcl()
268 n->m_flags |= m->m_flags & (M_RDONLY | M_EXT | M_EXTPG); in mb_dupcl()
271 if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { in mb_dupcl()
272 refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count; in mb_dupcl()
273 n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF; in mb_dupcl()
275 KASSERT(m->m_ext.ext_cnt != NULL, in mb_dupcl()
276 ("%s: no refcounting pointer on %p", __func__, m)); in mb_dupcl()
277 refcnt = m->m_ext.ext_cnt; in mb_dupcl()
287 m_demote_pkthdr(struct mbuf *m) in m_demote_pkthdr() argument
290 M_ASSERTPKTHDR(m); in m_demote_pkthdr()
291 M_ASSERT_NO_SND_TAG(m); in m_demote_pkthdr()
293 m_tag_delete_chain(m, NULL); in m_demote_pkthdr()
294 m->m_flags &= ~M_PKTHDR; in m_demote_pkthdr()
295 bzero(&m->m_pkthdr, sizeof(struct pkthdr)); in m_demote_pkthdr()
306 struct mbuf *m; in m_demote() local
310 for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { in m_demote()
311 KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", in m_demote()
312 __func__, m, m0)); in m_demote()
313 if (m->m_flags & M_PKTHDR) in m_demote()
314 m_demote_pkthdr(m); in m_demote()
315 m->m_flags &= flags; in m_demote()
329 struct mbuf *m; in m_sanity() local
334 #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) in m_sanity()
336 #define M_SANITY_ACTION(s) printf("mbuf %p: " s, m) in m_sanity()
339 for (m = m0; m != NULL; m = m->m_next) { in m_sanity()
345 a = M_START(m); in m_sanity()
346 b = a + M_SIZE(m); in m_sanity()
347 if ((caddr_t)m->m_data < a) in m_sanity()
349 if ((caddr_t)m->m_data > b) in m_sanity()
351 if ((caddr_t)m->m_data + m->m_len > b) in m_sanity()
354 /* m->m_nextpkt may only be set on first mbuf in chain. */ in m_sanity()
355 if (m != m0 && m->m_nextpkt != NULL) { in m_sanity()
357 m_freem(m->m_nextpkt); in m_sanity()
358 m->m_nextpkt = (struct mbuf *)0xDEADC0DE; in m_sanity()
360 M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); in m_sanity()
364 if (m0->m_flags & M_PKTHDR) in m_sanity()
365 pktlen += m->m_len; in m_sanity()
368 if (m != m0 && m->m_flags & M_PKTHDR && in m_sanity()
369 !SLIST_EMPTY(&m->m_pkthdr.tags)) { in m_sanity()
371 m_tag_delete_chain(m, NULL); in m_sanity()
374 M_SANITY_ACTION("m_tags on in-chain mbuf"); in m_sanity()
378 if (m != m0 && m->m_flags & M_PKTHDR) { in m_sanity()
380 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); in m_sanity()
381 m->m_flags &= ~M_PKTHDR; in m_sanity()
384 M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); in m_sanity()
387 m = m0; in m_sanity()
388 if (pktlen && pktlen != m->m_pkthdr.len) { in m_sanity()
390 m->m_pkthdr.len = 0; in m_sanity()
400 * Non-inlined part of m_init().
403 m_pkthdr_init(struct mbuf *m, int how) in m_pkthdr_init() argument
408 m->m_data = m->m_pktdat; in m_pkthdr_init()
409 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); in m_pkthdr_init()
411 m->m_pkthdr.numa_domain = M_NODOM; in m_pkthdr_init()
415 error = mac_mbuf_init(m, how); in m_pkthdr_init()
435 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), in m_move_pkthdr()
440 * XXXMAC: It could be this should also occur for non-MAC? in m_move_pkthdr()
442 if (to->m_flags & M_PKTHDR) in m_move_pkthdr()
445 to->m_flags = (from->m_flags & M_COPYFLAGS) | in m_move_pkthdr()
446 (to->m_flags & (M_EXT | M_EXTPG)); in m_move_pkthdr()
447 if ((to->m_flags & M_EXT) == 0) in m_move_pkthdr()
448 to->m_data = to->m_pktdat; in m_move_pkthdr()
449 to->m_pkthdr = from->m_pkthdr; /* especially tags */ in m_move_pkthdr()
450 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ in m_move_pkthdr()
451 from->m_flags &= ~M_PKTHDR; in m_move_pkthdr()
452 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_move_pkthdr()
453 from->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; in m_move_pkthdr()
454 from->m_pkthdr.snd_tag = NULL; in m_move_pkthdr()
477 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); in m_dup_pkthdr()
481 if (to->m_flags & M_PKTHDR) in m_dup_pkthdr()
484 to->m_flags = (from->m_flags & M_COPYFLAGS) | in m_dup_pkthdr()
485 (to->m_flags & (M_EXT | M_EXTPG)); in m_dup_pkthdr()
486 if ((to->m_flags & M_EXT) == 0) in m_dup_pkthdr()
487 to->m_data = to->m_pktdat; in m_dup_pkthdr()
488 to->m_pkthdr = from->m_pkthdr; in m_dup_pkthdr()
489 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) in m_dup_pkthdr()
490 m_snd_tag_ref(from->m_pkthdr.snd_tag); in m_dup_pkthdr()
491 SLIST_INIT(&to->m_pkthdr.tags); in m_dup_pkthdr()
496 * Lesser-used path for M_PREPEND:
501 m_prepend(struct mbuf *m, int len, int how) in m_prepend() argument
505 if (m->m_flags & M_PKTHDR) in m_prepend()
506 mn = m_gethdr(how, m->m_type); in m_prepend()
508 mn = m_get(how, m->m_type); in m_prepend()
510 m_freem(m); in m_prepend()
513 if (m->m_flags & M_PKTHDR) in m_prepend()
514 m_move_pkthdr(mn, m); in m_prepend()
515 mn->m_next = m; in m_prepend()
516 m = mn; in m_prepend()
517 if (len < M_SIZE(m)) in m_prepend()
518 M_ALIGN(m, len); in m_prepend()
519 m->m_len = len; in m_prepend()
520 return (m); in m_prepend()
527 * Note that the copy is read-only, because clusters are not copied,
531 m_copym(struct mbuf *m, int off0, int len, int wait) in m_copym() argument
541 if (off == 0 && m->m_flags & M_PKTHDR) in m_copym()
544 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); in m_copym()
545 if (off < m->m_len) in m_copym()
547 off -= m->m_len; in m_copym()
548 m = m->m_next; in m_copym()
553 if (m == NULL) { in m_copym()
559 n = m_gethdr(wait, m->m_type); in m_copym()
561 n = m_get(wait, m->m_type); in m_copym()
566 if (!m_dup_pkthdr(n, m, wait)) in m_copym()
569 n->m_pkthdr.len -= off0; in m_copym()
571 n->m_pkthdr.len = len; in m_copym()
574 n->m_len = min(len, m->m_len - off); in m_copym()
575 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copym()
576 n->m_data = m->m_data + off; in m_copym()
577 mb_dupcl(n, m); in m_copym()
579 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), in m_copym()
580 (u_int)n->m_len); in m_copym()
582 len -= n->m_len; in m_copym()
584 m = m->m_next; in m_copym()
585 np = &n->m_next; in m_copym()
596 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
597 * Note that the copy is read-only, because clusters are not copied,
604 m_copypacket(struct mbuf *m, int how) in m_copypacket() argument
609 n = m_get(how, m->m_type); in m_copypacket()
614 if (!m_dup_pkthdr(n, m, how)) in m_copypacket()
616 n->m_len = m->m_len; in m_copypacket()
617 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copypacket()
618 n->m_data = m->m_data; in m_copypacket()
619 mb_dupcl(n, m); in m_copypacket()
621 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); in m_copypacket()
622 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); in m_copypacket()
625 m = m->m_next; in m_copypacket()
626 while (m) { in m_copypacket()
627 o = m_get(how, m->m_type); in m_copypacket()
631 n->m_next = o; in m_copypacket()
632 n = n->m_next; in m_copypacket()
634 n->m_len = m->m_len; in m_copypacket()
635 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_copypacket()
636 n->m_data = m->m_data; in m_copypacket()
637 mb_dupcl(n, m); in m_copypacket()
639 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); in m_copypacket()
642 m = m->m_next; in m_copypacket()
651 m_copyfromunmapped(const struct mbuf *m, int off, int len, caddr_t cp) in m_copyfromunmapped() argument
659 KASSERT(off < m->m_len, in m_copyfromunmapped()
669 error = m_unmapped_uiomove(m, off, &uio, len); in m_copyfromunmapped()
679 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) in m_copydata() argument
686 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); in m_copydata()
687 if (off < m->m_len) in m_copydata()
689 off -= m->m_len; in m_copydata()
690 m = m->m_next; in m_copydata()
693 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); in m_copydata()
694 count = min(m->m_len - off, len); in m_copydata()
695 if ((m->m_flags & M_EXTPG) != 0) in m_copydata()
696 m_copyfromunmapped(m, off, count, cp); in m_copydata()
698 bcopy(mtod(m, caddr_t) + off, cp, count); in m_copydata()
699 len -= count; in m_copydata()
702 m = m->m_next; in m_copydata()
712 m_dup(const struct mbuf *m, int how) in m_dup() argument
719 if (m == NULL) in m_dup()
721 M_ASSERTPKTHDR(m); in m_dup()
724 remain = m->m_pkthdr.len; in m_dup()
727 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ in m_dup()
732 n = m_getcl(how, m->m_type, 0); in m_dup()
735 n = m_get(how, m->m_type); in m_dup()
742 if (!m_dup_pkthdr(n, m, how)) { in m_dup()
746 if ((n->m_flags & M_EXT) == 0) in m_dup()
748 n->m_flags &= ~M_RDONLY; in m_dup()
750 n->m_len = 0; in m_dup()
754 p = &n->m_next; in m_dup()
757 while (n->m_len < nsize && m != NULL) { in m_dup()
758 int chunk = min(nsize - n->m_len, m->m_len - moff); in m_dup()
760 m_copydata(m, moff, chunk, n->m_data + n->m_len); in m_dup()
762 n->m_len += chunk; in m_dup()
763 remain -= chunk; in m_dup()
764 if (moff == m->m_len) { in m_dup()
765 m = m->m_next; in m_dup()
771 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), in m_dup()
782 * Concatenate mbuf chain n to m.
787 m_cat(struct mbuf *m, struct mbuf *n) in m_cat() argument
789 while (m->m_next) in m_cat()
790 m = m->m_next; in m_cat()
792 if (!M_WRITABLE(m) || in m_cat()
793 (n->m_flags & M_EXTPG) != 0 || in m_cat()
794 M_TRAILINGSPACE(m) < n->m_len) { in m_cat()
796 m->m_next = n; in m_cat()
800 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, in m_cat()
801 (u_int)n->m_len); in m_cat()
802 m->m_len += n->m_len; in m_cat()
811 m_catpkt(struct mbuf *m, struct mbuf *n) in m_catpkt() argument
814 M_ASSERTPKTHDR(m); in m_catpkt()
817 m->m_pkthdr.len += n->m_pkthdr.len; in m_catpkt()
820 m_cat(m, n); in m_catpkt()
827 struct mbuf *m; in m_adj() local
830 if ((m = mp) == NULL) in m_adj()
836 while (m != NULL && len > 0) { in m_adj()
837 if (m->m_len <= len) { in m_adj()
838 len -= m->m_len; in m_adj()
839 m->m_len = 0; in m_adj()
840 m = m->m_next; in m_adj()
842 m->m_len -= len; in m_adj()
843 m->m_data += len; in m_adj()
847 if (mp->m_flags & M_PKTHDR) in m_adj()
848 mp->m_pkthdr.len -= (req_len - len); in m_adj()
857 len = -len; in m_adj()
860 count += m->m_len; in m_adj()
861 if (m->m_next == (struct mbuf *)0) in m_adj()
863 m = m->m_next; in m_adj()
865 if (m->m_len >= len) { in m_adj()
866 m->m_len -= len; in m_adj()
867 if (mp->m_flags & M_PKTHDR) in m_adj()
868 mp->m_pkthdr.len -= len; in m_adj()
871 count -= len; in m_adj()
879 m = mp; in m_adj()
880 if (m->m_flags & M_PKTHDR) in m_adj()
881 m->m_pkthdr.len = count; in m_adj()
882 for (; m; m = m->m_next) { in m_adj()
883 if (m->m_len >= count) { in m_adj()
884 m->m_len = count; in m_adj()
885 if (m->m_next != NULL) { in m_adj()
886 m_freem(m->m_next); in m_adj()
887 m->m_next = NULL; in m_adj()
891 count -= m->m_len; in m_adj()
902 if ((mp->m_flags & M_PKTHDR) != 0) { in m_adj_decap()
910 rsstype = mp->m_pkthdr.rsstype; in m_adj_decap()
924 * If there is room, it will add up to max_protohdr-len extra bytes to the
930 struct mbuf *m; in m_pullup() local
934 KASSERT((n->m_flags & M_EXTPG) == 0, in m_pullup()
942 if ((n->m_flags & M_EXT) == 0 && in m_pullup()
943 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { in m_pullup()
944 if (n->m_len >= len) in m_pullup()
946 m = n; in m_pullup()
947 n = n->m_next; in m_pullup()
948 len -= m->m_len; in m_pullup()
952 m = m_get(M_NOWAIT, n->m_type); in m_pullup()
953 if (m == NULL) in m_pullup()
955 if (n->m_flags & M_PKTHDR) in m_pullup()
956 m_move_pkthdr(m, n); in m_pullup()
958 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); in m_pullup()
960 count = min(min(max(len, max_protohdr), space), n->m_len); in m_pullup()
961 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, in m_pullup()
963 len -= count; in m_pullup()
964 m->m_len += count; in m_pullup()
965 n->m_len -= count; in m_pullup()
966 space -= count; in m_pullup()
967 if (n->m_len) in m_pullup()
968 n->m_data += count; in m_pullup()
973 (void) m_free(m); in m_pullup()
976 m->m_next = n; in m_pullup()
977 return (m); in m_pullup()
991 struct mbuf *m; in m_copyup() local
994 if (len > (MHLEN - dstoff)) in m_copyup()
996 m = m_get(M_NOWAIT, n->m_type); in m_copyup()
997 if (m == NULL) in m_copyup()
999 if (n->m_flags & M_PKTHDR) in m_copyup()
1000 m_move_pkthdr(m, n); in m_copyup()
1001 m->m_data += dstoff; in m_copyup()
1002 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); in m_copyup()
1004 count = min(min(max(len, max_protohdr), space), n->m_len); in m_copyup()
1005 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), in m_copyup()
1007 len -= count; in m_copyup()
1008 m->m_len += count; in m_copyup()
1009 n->m_len -= count; in m_copyup()
1010 space -= count; in m_copyup()
1011 if (n->m_len) in m_copyup()
1012 n->m_data += count; in m_copyup()
1017 (void) m_free(m); in m_copyup()
1020 m->m_next = n; in m_copyup()
1021 return (m); in m_copyup()
1028 * Partition an mbuf chain in two pieces, returning the tail --
1032 * Note that the resulting mbufs might be read-only, because the new
1040 struct mbuf *m, *n; in m_split() local
1044 for (m = m0; m && len > m->m_len; m = m->m_next) in m_split()
1045 len -= m->m_len; in m_split()
1046 if (m == NULL) in m_split()
1048 remain = m->m_len - len; in m_split()
1049 if (m0->m_flags & M_PKTHDR && remain == 0) { in m_split()
1050 n = m_gethdr(wait, m0->m_type); in m_split()
1053 n->m_next = m->m_next; in m_split()
1054 m->m_next = NULL; in m_split()
1055 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_split()
1056 n->m_pkthdr.snd_tag = in m_split()
1057 m_snd_tag_ref(m0->m_pkthdr.snd_tag); in m_split()
1058 n->m_pkthdr.csum_flags |= CSUM_SND_TAG; in m_split()
1060 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; in m_split()
1061 n->m_pkthdr.len = m0->m_pkthdr.len - len0; in m_split()
1062 m0->m_pkthdr.len = len0; in m_split()
1064 } else if (m0->m_flags & M_PKTHDR) { in m_split()
1065 n = m_gethdr(wait, m0->m_type); in m_split()
1068 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) { in m_split()
1069 n->m_pkthdr.snd_tag = in m_split()
1070 m_snd_tag_ref(m0->m_pkthdr.snd_tag); in m_split()
1071 n->m_pkthdr.csum_flags |= CSUM_SND_TAG; in m_split()
1073 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; in m_split()
1074 n->m_pkthdr.len = m0->m_pkthdr.len - len0; in m_split()
1075 m0->m_pkthdr.len = len0; in m_split()
1076 if (m->m_flags & (M_EXT | M_EXTPG)) in m_split()
1079 /* m can't be the lead packet */ in m_split()
1081 n->m_next = m_split(m, len, wait); in m_split()
1082 if (n->m_next == NULL) { in m_split()
1086 n->m_len = 0; in m_split()
1092 n = m->m_next; in m_split()
1093 m->m_next = NULL; in m_split()
1096 n = m_get(wait, m->m_type); in m_split()
1102 if (m->m_flags & (M_EXT | M_EXTPG)) { in m_split()
1103 n->m_data = m->m_data + len; in m_split()
1104 mb_dupcl(n, m); in m_split()
1106 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); in m_split()
1108 n->m_len = remain; in m_split()
1109 m->m_len = len; in m_split()
1110 n->m_next = m->m_next; in m_split()
1111 m->m_next = NULL; in m_split()
1119 * Resulting mbufs might be read-only.
1124 struct mbuf *m, *n; in mc_split() local
1127 MPASS(!(mc_first(head)->m_flags & M_PKTHDR)); in mc_split()
1132 STAILQ_FOREACH(m, &head->mc_q, m_stailq) { in mc_split()
1134 if (m->m_flags & M_EXT) in mc_split()
1135 mlen += m->m_ext.ext_size; in mc_split()
1136 if (len > m->m_len) in mc_split()
1137 len -= m->m_len; in mc_split()
1141 if (__predict_false(m == NULL)) { in mc_split()
1145 remain = m->m_len - len; in mc_split()
1147 if (__predict_false((n = m_get(wait, m->m_type)) == NULL)) in mc_split()
1150 if (m->m_flags & M_EXT) { in mc_split()
1151 n->m_data = m->m_data + len; in mc_split()
1152 mb_dupcl(n, m); in mc_split()
1154 bcopy(mtod(m, char *) + len, mtod(n, char *), remain); in mc_split()
1158 STAILQ_FIRST(&tail->mc_q) = STAILQ_NEXT(m, m_stailq); in mc_split()
1159 tail->mc_q.stqh_last = head->mc_q.stqh_last; in mc_split()
1160 tail->mc_len = head->mc_len - len0; in mc_split()
1161 tail->mc_mlen = head->mc_mlen - mlen; in mc_split()
1163 MPASS(n->m_len == 0); in mc_split()
1165 n->m_len = remain; in mc_split()
1166 m->m_len -= remain; in mc_split()
1167 if (m->m_flags & M_EOR) { in mc_split()
1168 m->m_flags &= ~M_EOR; in mc_split()
1169 n->m_flags |= M_EOR; in mc_split()
1172 head->mc_q.stqh_last = &STAILQ_NEXT(m, m_stailq); in mc_split()
1173 STAILQ_NEXT(m, m_stailq) = NULL; in mc_split()
1174 head->mc_len = len0; in mc_split()
1175 head->mc_mlen = mlen; in mc_split()
1189 struct mbuf *m; in m_devget() local
1199 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); in m_devget()
1202 m = m_gethdr(M_NOWAIT, MT_DATA); in m_devget()
1206 if (m && totlen + off + max_linkhdr <= MHLEN) { in m_devget()
1207 m->m_data += max_linkhdr; in m_devget()
1208 len -= max_linkhdr; in m_devget()
1211 if (m == NULL) in m_devget()
1213 m->m_pkthdr.rcvif = ifp; in m_devget()
1214 m->m_pkthdr.len = totlen; in m_devget()
1217 m = m_getcl(M_NOWAIT, MT_DATA, 0); in m_devget()
1220 m = m_get(M_NOWAIT, MT_DATA); in m_devget()
1223 if (m == NULL) { in m_devget()
1229 m->m_data += off; in m_devget()
1230 len -= off; in m_devget()
1233 m->m_len = len = min(totlen, len); in m_devget()
1235 copy(buf, mtod(m, caddr_t), (u_int)len); in m_devget()
1237 bcopy(buf, mtod(m, caddr_t), (u_int)len); in m_devget()
1239 *mp = m; in m_devget()
1240 mp = &m->m_next; in m_devget()
1241 totlen -= len; in m_devget()
1247 m_copytounmapped(const struct mbuf *m, int off, int len, c_caddr_t cp) in m_copytounmapped() argument
1255 KASSERT(off < m->m_len, ("m_copytounmapped: len exceeds mbuf length")); in m_copytounmapped()
1264 error = m_unmapped_uiomove(m, off, &uio, len); in m_copytounmapped()
1278 struct mbuf *m = m0, *n; in m_copyback() local
1283 while (off > (mlen = m->m_len)) { in m_copyback()
1284 off -= mlen; in m_copyback()
1286 if (m->m_next == NULL) { in m_copyback()
1287 n = m_get(M_NOWAIT, m->m_type); in m_copyback()
1291 n->m_len = min(MLEN, len + off); in m_copyback()
1292 m->m_next = n; in m_copyback()
1294 m = m->m_next; in m_copyback()
1297 if (m->m_next == NULL && (len > m->m_len - off)) { in m_copyback()
1298 m->m_len += min(len - (m->m_len - off), in m_copyback()
1299 M_TRAILINGSPACE(m)); in m_copyback()
1301 mlen = min (m->m_len - off, len); in m_copyback()
1302 if ((m->m_flags & M_EXTPG) != 0) in m_copyback()
1303 m_copytounmapped(m, off, mlen, cp); in m_copyback()
1305 bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen); in m_copyback()
1307 len -= mlen; in m_copyback()
1313 if (m->m_next == NULL) { in m_copyback()
1314 n = m_get(M_NOWAIT, m->m_type); in m_copyback()
1317 n->m_len = min(MLEN, len); in m_copyback()
1318 m->m_next = n; in m_copyback()
1320 m = m->m_next; in m_copyback()
1322 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) in m_copyback()
1323 m->m_pkthdr.len = totlen; in m_copyback()
1336 struct mbuf *m, *n; in m_append() local
1339 for (m = m0; m->m_next != NULL; m = m->m_next) in m_append()
1342 space = M_TRAILINGSPACE(m); in m_append()
1349 bcopy(cp, mtod(m, caddr_t) + m->m_len, space); in m_append()
1350 m->m_len += space; in m_append()
1351 cp += space, remainder -= space; in m_append()
1358 n = m_get(M_NOWAIT, m->m_type); in m_append()
1361 n->m_len = min(MLEN, remainder); in m_append()
1362 bcopy(cp, mtod(n, caddr_t), n->m_len); in m_append()
1363 cp += n->m_len, remainder -= n->m_len; in m_append()
1364 m->m_next = n; in m_append()
1365 m = n; in m_append()
1367 if (m0->m_flags & M_PKTHDR) in m_append()
1368 m0->m_pkthdr.len += len - remainder; in m_append()
1373 m_apply_extpg_one(struct mbuf *m, int off, int len, in m_apply_extpg_one() argument
1382 off += mtod(m, vm_offset_t); in m_apply_extpg_one()
1383 if (off < m->m_epg_hdrlen) { in m_apply_extpg_one()
1384 count = min(m->m_epg_hdrlen - off, len); in m_apply_extpg_one()
1385 rval = f(arg, m->m_epg_hdr + off, count); in m_apply_extpg_one()
1388 len -= count; in m_apply_extpg_one()
1391 off -= m->m_epg_hdrlen; in m_apply_extpg_one()
1392 pgoff = m->m_epg_1st_off; in m_apply_extpg_one()
1393 for (i = 0; i < m->m_epg_npgs && len > 0; i++) { in m_apply_extpg_one()
1394 pglen = m_epg_pagelen(m, i, pgoff); in m_apply_extpg_one()
1396 count = min(pglen - off, len); in m_apply_extpg_one()
1397 p = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff + off); in m_apply_extpg_one()
1401 len -= count; in m_apply_extpg_one()
1404 off -= pglen; in m_apply_extpg_one()
1408 KASSERT(off < m->m_epg_trllen, in m_apply_extpg_one()
1410 KASSERT(len <= m->m_epg_trllen - off, in m_apply_extpg_one()
1412 return (f(arg, m->m_epg_trail + off, len)); in m_apply_extpg_one()
1419 m_apply_one(struct mbuf *m, int off, int len, in m_apply_one() argument
1422 if ((m->m_flags & M_EXTPG) != 0) in m_apply_one()
1423 return (m_apply_extpg_one(m, off, len, f, arg)); in m_apply_one()
1425 return (f(arg, mtod(m, caddr_t) + off, len)); in m_apply_one()
1433 m_apply(struct mbuf *m, int off, int len, in m_apply() argument
1442 KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain " in m_apply()
1444 if (off < m->m_len) in m_apply()
1446 off -= m->m_len; in m_apply()
1447 m = m->m_next; in m_apply()
1450 KASSERT(m != NULL, ("m_apply, length > size of mbuf chain " in m_apply()
1452 count = min(m->m_len - off, len); in m_apply()
1453 rval = m_apply_one(m, off, count, f, arg); in m_apply()
1456 len -= count; in m_apply()
1458 m = m->m_next; in m_apply()
1467 m_getptr(struct mbuf *m, int loc, int *off) in m_getptr() argument
1472 if (m->m_len > loc) { in m_getptr()
1474 return (m); in m_getptr()
1476 loc -= m->m_len; in m_getptr()
1477 if (m->m_next == NULL) { in m_getptr()
1480 *off = m->m_len; in m_getptr()
1481 return (m); in m_getptr()
1485 m = m->m_next; in m_getptr()
1492 m_print(const struct mbuf *m, int maxlen) in m_print() argument
1498 if (m == NULL) { in m_print()
1499 printf("mbuf: %p\n", m); in m_print()
1503 if (m->m_flags & M_PKTHDR) in m_print()
1504 len = m->m_pkthdr.len; in m_print()
1506 len = -1; in m_print()
1507 m2 = m; in m_print()
1508 while (m2 != NULL && (len == -1 || len)) { in m_print()
1509 pdata = m2->m_len; in m_print()
1510 if (maxlen != -1 && pdata > maxlen) in m_print()
1512 printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, in m_print()
1513 m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" in m_print()
1517 printf(", %*D\n", pdata, (u_char *)m2->m_data, "-"); in m_print()
1518 if (len != -1) in m_print()
1519 len -= m2->m_len; in m_print()
1520 m2 = m2->m_next; in m_print()
1533 m0->m_pkthdr.len = len; in m_fixhdr()
1540 struct mbuf *m; in m_length() local
1544 for (m = m0; m != NULL; m = m->m_next) { in m_length()
1545 len += m->m_len; in m_length()
1546 if (m->m_next == NULL) in m_length()
1550 *last = m; in m_length()
1562 * If a non-packet header is passed in, the original
1572 if (!(m0->m_flags & M_PKTHDR)) in m_defrag()
1585 if (m0->m_pkthdr.len > MHLEN) in m_defrag()
1598 while (progress < m0->m_pkthdr.len) { in m_defrag()
1599 length = m0->m_pkthdr.len - progress; in m_defrag()
1614 m_new->m_len = length; in m_defrag()
1620 if (m0->m_next == NULL) in m_defrag()
1627 m_defragbytes += m0->m_pkthdr.len; in m_defrag()
1648 frags_per_mbuf(struct mbuf *m) in frags_per_mbuf() argument
1652 if ((m->m_flags & M_EXTPG) == 0) in frags_per_mbuf()
1663 if (m->m_epg_hdrlen != 0) in frags_per_mbuf()
1665 frags += m->m_epg_npgs; in frags_per_mbuf()
1666 if (m->m_epg_trllen != 0) in frags_per_mbuf()
1684 struct mbuf *m, *n, *n2, **prev; in m_collapse() local
1691 for (m = m0; m != NULL; m = m->m_next) in m_collapse()
1692 curfrags += frags_per_mbuf(m); in m_collapse()
1699 m = m0; in m_collapse()
1702 n = m->m_next; in m_collapse()
1705 if (M_WRITABLE(m) && in m_collapse()
1706 n->m_len < M_TRAILINGSPACE(m)) { in m_collapse()
1707 m_copydata(n, 0, n->m_len, in m_collapse()
1708 mtod(m, char *) + m->m_len); in m_collapse()
1709 m->m_len += n->m_len; in m_collapse()
1710 m->m_next = n->m_next; in m_collapse()
1711 curfrags -= frags_per_mbuf(n); in m_collapse()
1716 m = n; in m_collapse()
1723 prev = &m0->m_next; /* NB: not the first mbuf */ in m_collapse()
1725 if ((n2 = n->m_next) != NULL && in m_collapse()
1726 n->m_len + n2->m_len < MCLBYTES) { in m_collapse()
1727 m = m_getcl(how, MT_DATA, 0); in m_collapse()
1728 if (m == NULL) in m_collapse()
1730 m_copydata(n, 0, n->m_len, mtod(m, char *)); in m_collapse()
1731 m_copydata(n2, 0, n2->m_len, in m_collapse()
1732 mtod(m, char *) + n->m_len); in m_collapse()
1733 m->m_len = n->m_len + n2->m_len; in m_collapse()
1734 m->m_next = n2->m_next; in m_collapse()
1735 *prev = m; in m_collapse()
1737 curfrags -= frags_per_mbuf(n); in m_collapse()
1738 curfrags -= frags_per_mbuf(n2); in m_collapse()
1749 prev = &n->m_next; in m_collapse()
1775 * -1 each fragment will be the same random value in length
1776 * -2 each fragment's length will be entirely random
1785 if (!(m0->m_flags & M_PKTHDR)) in m_fragment()
1788 if (length == 0 || length < -2) in m_fragment()
1794 if (length == -1) in m_fragment()
1810 while (progress < m0->m_pkthdr.len) { in m_fragment()
1811 if (length == -2) in m_fragment()
1813 if (fraglen > m0->m_pkthdr.len - progress) in m_fragment()
1814 fraglen = m0->m_pkthdr.len - progress; in m_fragment()
1821 m_last->m_next = m_new; in m_fragment()
1827 m_last->m_len = fraglen; in m_fragment()
1847 mb_free_mext_pgs(struct mbuf *m) in mb_free_mext_pgs() argument
1851 M_ASSERTEXTPG(m); in mb_free_mext_pgs()
1852 for (int i = 0; i < m->m_epg_npgs; i++) { in mb_free_mext_pgs()
1853 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); in mb_free_mext_pgs()
1862 struct mbuf *m, *mb, *prev; in m_uiotombuf_nomap() local
1876 total = MIN(uio->uio_resid, len); in m_uiotombuf_nomap()
1878 total = uio->uio_resid; in m_uiotombuf_nomap()
1886 * a countermeasure against the known-IV weakness in CBC in m_uiotombuf_nomap()
1893 mb->m_epg_flags = EPG_FLAG_ANON; in m_uiotombuf_nomap()
1900 m = NULL; in m_uiotombuf_nomap()
1905 if (m == NULL) in m_uiotombuf_nomap()
1906 m = mb; in m_uiotombuf_nomap()
1908 prev->m_next = mb; in m_uiotombuf_nomap()
1910 mb->m_epg_flags = EPG_FLAG_ANON; in m_uiotombuf_nomap()
1912 for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) { in m_uiotombuf_nomap()
1923 mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]); in m_uiotombuf_nomap()
1924 mb->m_epg_npgs++; in m_uiotombuf_nomap()
1926 mb->m_epg_last_len = length - PAGE_SIZE * (mb->m_epg_npgs - 1); in m_uiotombuf_nomap()
1928 total -= length; in m_uiotombuf_nomap()
1932 mb->m_len = length; in m_uiotombuf_nomap()
1933 mb->m_ext.ext_size += PAGE_SIZE * mb->m_epg_npgs; in m_uiotombuf_nomap()
1935 m->m_pkthdr.len += length; in m_uiotombuf_nomap()
1937 return (m); in m_uiotombuf_nomap()
1940 m_freem(m); in m_uiotombuf_nomap()
1956 } else if (__predict_false(uio->uio_resid == 0)) { in m_uiotombuf()
1957 struct mbuf *m; in m_uiotombuf() local
1964 m = m_gethdr(how, MT_DATA); in m_uiotombuf()
1965 m->m_pkthdr.memlen = MSIZE; in m_uiotombuf()
1967 m = m_get(how, MT_DATA); in m_uiotombuf()
1968 if (m != NULL) in m_uiotombuf()
1969 m->m_data += lspace; in m_uiotombuf()
1970 return (m); in m_uiotombuf()
1978 mc_first(&mc)->m_pkthdr.len = mc.mc_len; in m_uiotombuf()
1979 mc_first(&mc)->m_pkthdr.memlen = mc.mc_mlen; in m_uiotombuf()
2001 MPASS(UINT_MAX - lspace >= length); in mc_uiotomc()
2002 MPASS(uio->uio_rw == UIO_WRITE); in mc_uiotomc()
2003 MPASS(uio->uio_resid >= 0); in mc_uiotomc()
2006 if (uio->uio_resid > length) { in mc_uiotomc()
2010 total = uio->uio_resid; in mc_uiotomc()
2011 } else if (__predict_false(uio->uio_resid + lspace > UINT_MAX)) in mc_uiotomc()
2014 total = uio->uio_resid; in mc_uiotomc()
2024 mc_first(mc)->m_data += lspace; in mc_uiotomc()
2027 STAILQ_FOREACH(mb, &mc->mc_q, m_stailq) { in mc_uiotomc()
2030 mlen = min(M_TRAILINGSPACE(mb), total - mc->mc_len); in mc_uiotomc()
2037 mb->m_len = mlen; in mc_uiotomc()
2038 mc->mc_len += mlen; in mc_uiotomc()
2040 MPASS(mc->mc_len == total); in mc_uiotomc()
2049 m_unmapped_uiomove(const struct mbuf *m, int m_off, struct uio *uio, int len) in m_unmapped_uiomove() argument
2054 M_ASSERTEXTPG(m); in m_unmapped_uiomove()
2058 off = mtod(m, vm_offset_t); in m_unmapped_uiomove()
2061 if (m->m_epg_hdrlen != 0) { in m_unmapped_uiomove()
2062 if (off >= m->m_epg_hdrlen) { in m_unmapped_uiomove()
2063 off -= m->m_epg_hdrlen; in m_unmapped_uiomove()
2065 seglen = m->m_epg_hdrlen - off; in m_unmapped_uiomove()
2069 len -= seglen; in m_unmapped_uiomove()
2071 &m->m_epg_hdr[segoff]), seglen, uio); in m_unmapped_uiomove()
2074 pgoff = m->m_epg_1st_off; in m_unmapped_uiomove()
2075 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) { in m_unmapped_uiomove()
2076 pglen = m_epg_pagelen(m, i, pgoff); in m_unmapped_uiomove()
2078 off -= pglen; in m_unmapped_uiomove()
2082 seglen = pglen - off; in m_unmapped_uiomove()
2086 len -= seglen; in m_unmapped_uiomove()
2087 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); in m_unmapped_uiomove()
2092 KASSERT((off + len) <= m->m_epg_trllen, in m_unmapped_uiomove()
2094 m->m_epg_trllen, m_off)); in m_unmapped_uiomove()
2095 error = uiomove(__DECONST(void *, &m->m_epg_trail[off]), in m_unmapped_uiomove()
2105 m_mbuftouio(struct uio *uio, const struct mbuf *m, int len) in m_mbuftouio() argument
2111 total = min(uio->uio_resid, len); in m_mbuftouio()
2113 total = uio->uio_resid; in m_mbuftouio()
2116 for (; m != NULL; m = m->m_next) { in m_mbuftouio()
2117 length = min(m->m_len, total - progress); in m_mbuftouio()
2119 if ((m->m_flags & M_EXTPG) != 0) in m_mbuftouio()
2120 error = m_unmapped_uiomove(m, 0, uio, length); in m_mbuftouio()
2122 error = uiomove(mtod(m, void *), length, uio); in m_mbuftouio()
2144 struct mbuf *m, *mprev; in m_unshare() local
2149 for (m = m0; m != NULL; m = mprev->m_next) { in m_unshare()
2155 * mbuf+cluster -> cluster). This optimization is heavily in m_unshare()
2162 if ((m->m_flags & M_EXT) == 0) { in m_unshare()
2163 if (mprev && (mprev->m_flags & M_EXT) && in m_unshare()
2164 m->m_len <= M_TRAILINGSPACE(mprev)) { in m_unshare()
2166 memcpy(mtod(mprev, caddr_t) + mprev->m_len, in m_unshare()
2167 mtod(m, caddr_t), m->m_len); in m_unshare()
2168 mprev->m_len += m->m_len; in m_unshare()
2169 mprev->m_next = m->m_next; /* unlink from chain */ in m_unshare()
2170 m_free(m); /* reclaim mbuf */ in m_unshare()
2172 mprev = m; in m_unshare()
2179 if (M_WRITABLE(m)) { in m_unshare()
2180 mprev = m; in m_unshare()
2190 KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags)); in m_unshare()
2192 if (mprev != NULL && (mprev->m_flags & M_EXT) && in m_unshare()
2193 m->m_len <= M_TRAILINGSPACE(mprev)) { in m_unshare()
2195 memcpy(mtod(mprev, caddr_t) + mprev->m_len, in m_unshare()
2196 mtod(m, caddr_t), m->m_len); in m_unshare()
2197 mprev->m_len += m->m_len; in m_unshare()
2198 mprev->m_next = m->m_next; /* unlink from chain */ in m_unshare()
2199 m_free(m); /* reclaim mbuf */ in m_unshare()
2208 * don't know how to break up the non-contiguous memory when in m_unshare()
2211 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); in m_unshare()
2216 if (m->m_flags & M_PKTHDR) { in m_unshare()
2217 KASSERT(mprev == NULL, ("%s: m0 %p, m %p has M_PKTHDR", in m_unshare()
2218 __func__, m0, m)); in m_unshare()
2219 m_move_pkthdr(n, m); in m_unshare()
2221 len = m->m_len; in m_unshare()
2227 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc); in m_unshare()
2228 n->m_len = cc; in m_unshare()
2230 mlast->m_next = n; in m_unshare()
2236 len -= cc; in m_unshare()
2241 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); in m_unshare()
2248 n->m_next = m->m_next; in m_unshare()
2252 mprev->m_next = mfirst; /* replace old mbuf */ in m_unshare()
2253 m_free(m); /* release old mbuf */ in m_unshare()
2269 m_profile(struct mbuf *m) in m_profile() argument
2275 while (m) { in m_profile()
2277 used += m->m_len; in m_profile()
2278 if (m->m_flags & M_EXT) { in m_profile()
2279 wasted += MHLEN - sizeof(m->m_ext) + in m_profile()
2280 m->m_ext.ext_size - m->m_len; in m_profile()
2282 if (m->m_flags & M_PKTHDR) in m_profile()
2283 wasted += MHLEN - m->m_len; in m_profile()
2285 wasted += MLEN - m->m_len; in m_profile()
2287 m = m->m_next; in m_profile()
2290 if (segments > MP_BUCKETS - 1) in m_profile()
2291 segments = MP_BUCKETS - 1; in m_profile()
2371 if (error || !req->newptr) in mbprof_clr_handler()