xref: /freebsd/sys/kern/uipc_mbuf.c (revision 17ee9d00bc1ae1e598c38f25826f861e4bc6c3ce)
1 /*
2  * Copyright (c) 1982, 1986, 1988, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  *
33  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
34  * $Id: uipc_mbuf.c,v 1.6 1994/10/09 07:34:59 davidg Exp $
35  */
36 
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/proc.h>
40 #include <sys/malloc.h>
41 #define MBTYPES
42 #include <sys/mbuf.h>
43 #include <sys/kernel.h>
44 #include <sys/syslog.h>
45 #include <sys/domain.h>
46 #include <sys/protosw.h>
47 
48 #include <vm/vm.h>
49 
50 extern	vm_map_t mb_map;
51 struct	mbuf *mbutl;
52 char	*mclrefcnt;
53 
54 void
55 mbinit()
56 {
57 	int s;
58 
59 #if CLBYTES < 4096
60 #define NCL_INIT	(4096/CLBYTES)
61 #else
62 #define NCL_INIT	1
63 #endif
64 	s = splimp();
65 	if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
66 		goto bad;
67 	splx(s);
68 	return;
69 bad:
70 	panic("mbinit");
71 }
72 
73 /*
74  * Allocate some number of mbuf clusters
75  * and place on cluster free list.
76  * Must be called at splimp.
77  */
78 /* ARGSUSED */
79 int
80 m_clalloc(ncl, nowait)
81 	register int ncl;
82 	int nowait;
83 {
84 	static int logged;
85 	register caddr_t p;
86 	register int i;
87 	int npg;
88 
89 	npg = ncl * CLSIZE;
90 	p = (caddr_t)kmem_malloc(mb_map, ctob(npg),
91 				 nowait ? M_NOWAIT : M_WAITOK);
92 	if (p == NULL) {
93 		if (logged == 0) {
94 			logged++;
95 			log(LOG_ERR, "mb_map full\n");
96 		}
97 		return (0);
98 	}
99 	ncl = ncl * CLBYTES / MCLBYTES;
100 	for (i = 0; i < ncl; i++) {
101 		((union mcluster *)p)->mcl_next = mclfree;
102 		mclfree = (union mcluster *)p;
103 		p += MCLBYTES;
104 		mbstat.m_clfree++;
105 	}
106 	mbstat.m_clusters += ncl;
107 	return (1);
108 }
109 
110 /*
111  * When MGET failes, ask protocols to free space when short of memory,
112  * then re-attempt to allocate an mbuf.
113  */
114 struct mbuf *
115 m_retry(i, t)
116 	int i, t;
117 {
118 	register struct mbuf *m;
119 
120 	m_reclaim();
121 #define m_retry(i, t)	(struct mbuf *)0
122 	MGET(m, i, t);
123 #undef m_retry
124 	return (m);
125 }
126 
127 /*
128  * As above; retry an MGETHDR.
129  */
130 struct mbuf *
131 m_retryhdr(i, t)
132 	int i, t;
133 {
134 	register struct mbuf *m;
135 
136 	m_reclaim();
137 #define m_retryhdr(i, t) (struct mbuf *)0
138 	MGETHDR(m, i, t);
139 #undef m_retryhdr
140 	return (m);
141 }
142 
143 void
144 m_reclaim()
145 {
146 	register struct domain *dp;
147 	register struct protosw *pr;
148 	int s = splimp();
149 
150 	for (dp = domains; dp; dp = dp->dom_next)
151 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
152 			if (pr->pr_drain)
153 				(*pr->pr_drain)();
154 	splx(s);
155 	mbstat.m_drain++;
156 }
157 
158 /*
159  * Space allocation routines.
160  * These are also available as macros
161  * for critical paths.
162  */
163 struct mbuf *
164 m_get(nowait, type)
165 	int nowait, type;
166 {
167 	register struct mbuf *m;
168 
169 	MGET(m, nowait, type);
170 	return (m);
171 }
172 
173 struct mbuf *
174 m_gethdr(nowait, type)
175 	int nowait, type;
176 {
177 	register struct mbuf *m;
178 
179 	MGETHDR(m, nowait, type);
180 	return (m);
181 }
182 
183 struct mbuf *
184 m_getclr(nowait, type)
185 	int nowait, type;
186 {
187 	register struct mbuf *m;
188 
189 	MGET(m, nowait, type);
190 	if (m == 0)
191 		return (0);
192 	bzero(mtod(m, caddr_t), MLEN);
193 	return (m);
194 }
195 
196 struct mbuf *
197 m_free(m)
198 	struct mbuf *m;
199 {
200 	register struct mbuf *n;
201 
202 	MFREE(m, n);
203 	return (n);
204 }
205 
206 void
207 m_freem(m)
208 	register struct mbuf *m;
209 {
210 	register struct mbuf *n;
211 
212 	if (m == NULL)
213 		return;
214 	do {
215 		MFREE(m, n);
216 		m = n;
217 	} while (m);
218 }
219 
220 /*
221  * Mbuffer utility routines.
222  */
223 
224 /*
225  * Lesser-used path for M_PREPEND:
226  * allocate new mbuf to prepend to chain,
227  * copy junk along.
228  */
229 struct mbuf *
230 m_prepend(m, len, how)
231 	register struct mbuf *m;
232 	int len, how;
233 {
234 	struct mbuf *mn;
235 
236 	MGET(mn, how, m->m_type);
237 	if (mn == (struct mbuf *)NULL) {
238 		m_freem(m);
239 		return ((struct mbuf *)NULL);
240 	}
241 	if (m->m_flags & M_PKTHDR) {
242 		M_COPY_PKTHDR(mn, m);
243 		m->m_flags &= ~M_PKTHDR;
244 	}
245 	mn->m_next = m;
246 	m = mn;
247 	if (len < MHLEN)
248 		MH_ALIGN(m, len);
249 	m->m_len = len;
250 	return (m);
251 }
252 
253 /*
254  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
255  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
256  * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
257  */
258 int MCFail;
259 
260 struct mbuf *
261 m_copym(m, off0, len, wait)
262 	register struct mbuf *m;
263 	int off0, wait;
264 	register int len;
265 {
266 	register struct mbuf *n, **np;
267 	register int off = off0;
268 	struct mbuf *top;
269 	int copyhdr = 0;
270 
271 	if (off < 0 || len < 0)
272 		panic("m_copym");
273 	if (off == 0 && m->m_flags & M_PKTHDR)
274 		copyhdr = 1;
275 	while (off > 0) {
276 		if (m == 0)
277 			panic("m_copym");
278 		if (off < m->m_len)
279 			break;
280 		off -= m->m_len;
281 		m = m->m_next;
282 	}
283 	np = &top;
284 	top = 0;
285 	while (len > 0) {
286 		if (m == 0) {
287 			if (len != M_COPYALL)
288 				panic("m_copym");
289 			break;
290 		}
291 		MGET(n, wait, m->m_type);
292 		*np = n;
293 		if (n == 0)
294 			goto nospace;
295 		if (copyhdr) {
296 			M_COPY_PKTHDR(n, m);
297 			if (len == M_COPYALL)
298 				n->m_pkthdr.len -= off0;
299 			else
300 				n->m_pkthdr.len = len;
301 			copyhdr = 0;
302 		}
303 		n->m_len = min(len, m->m_len - off);
304 		if (m->m_flags & M_EXT) {
305 			n->m_data = m->m_data + off;
306 			mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
307 			n->m_ext = m->m_ext;
308 			n->m_flags |= M_EXT;
309 		} else
310 			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
311 			    (unsigned)n->m_len);
312 		if (len != M_COPYALL)
313 			len -= n->m_len;
314 		off = 0;
315 		m = m->m_next;
316 		np = &n->m_next;
317 	}
318 	if (top == 0)
319 		MCFail++;
320 	return (top);
321 nospace:
322 	m_freem(top);
323 	MCFail++;
324 	return (0);
325 }
326 
327 /*
328  * Copy data from an mbuf chain starting "off" bytes from the beginning,
329  * continuing for "len" bytes, into the indicated buffer.
330  */
331 void
332 m_copydata(m, off, len, cp)
333 	register struct mbuf *m;
334 	register int off;
335 	register int len;
336 	caddr_t cp;
337 {
338 	register unsigned count;
339 
340 	if (off < 0 || len < 0)
341 		panic("m_copydata");
342 	while (off > 0) {
343 		if (m == 0)
344 			panic("m_copydata");
345 		if (off < m->m_len)
346 			break;
347 		off -= m->m_len;
348 		m = m->m_next;
349 	}
350 	while (len > 0) {
351 		if (m == 0)
352 			panic("m_copydata");
353 		count = min(m->m_len - off, len);
354 		bcopy(mtod(m, caddr_t) + off, cp, count);
355 		len -= count;
356 		cp += count;
357 		off = 0;
358 		m = m->m_next;
359 	}
360 }
361 
362 /*
363  * Concatenate mbuf chain n to m.
364  * Both chains must be of the same type (e.g. MT_DATA).
365  * Any m_pkthdr is not updated.
366  */
367 void
368 m_cat(m, n)
369 	register struct mbuf *m, *n;
370 {
371 	while (m->m_next)
372 		m = m->m_next;
373 	while (n) {
374 		if (m->m_flags & M_EXT ||
375 		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
376 			/* just join the two chains */
377 			m->m_next = n;
378 			return;
379 		}
380 		/* splat the data from one into the other */
381 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
382 		    (u_int)n->m_len);
383 		m->m_len += n->m_len;
384 		n = m_free(n);
385 	}
386 }
387 
388 void
389 m_adj(mp, req_len)
390 	struct mbuf *mp;
391 	int req_len;
392 {
393 	register int len = req_len;
394 	register struct mbuf *m;
395 	register count;
396 
397 	if ((m = mp) == NULL)
398 		return;
399 	if (len >= 0) {
400 		/*
401 		 * Trim from head.
402 		 */
403 		while (m != NULL && len > 0) {
404 			if (m->m_len <= len) {
405 				len -= m->m_len;
406 				m->m_len = 0;
407 				m = m->m_next;
408 			} else {
409 				m->m_len -= len;
410 				m->m_data += len;
411 				len = 0;
412 			}
413 		}
414 		m = mp;
415 		if (mp->m_flags & M_PKTHDR)
416 			m->m_pkthdr.len -= (req_len - len);
417 	} else {
418 		/*
419 		 * Trim from tail.  Scan the mbuf chain,
420 		 * calculating its length and finding the last mbuf.
421 		 * If the adjustment only affects this mbuf, then just
422 		 * adjust and return.  Otherwise, rescan and truncate
423 		 * after the remaining size.
424 		 */
425 		len = -len;
426 		count = 0;
427 		for (;;) {
428 			count += m->m_len;
429 			if (m->m_next == (struct mbuf *)0)
430 				break;
431 			m = m->m_next;
432 		}
433 		if (m->m_len >= len) {
434 			m->m_len -= len;
435 			if (mp->m_flags & M_PKTHDR)
436 				mp->m_pkthdr.len -= len;
437 			return;
438 		}
439 		count -= len;
440 		if (count < 0)
441 			count = 0;
442 		/*
443 		 * Correct length for chain is "count".
444 		 * Find the mbuf with last data, adjust its length,
445 		 * and toss data from remaining mbufs on chain.
446 		 */
447 		m = mp;
448 		if (m->m_flags & M_PKTHDR)
449 			m->m_pkthdr.len = count;
450 		for (; m; m = m->m_next) {
451 			if (m->m_len >= count) {
452 				m->m_len = count;
453 				break;
454 			}
455 			count -= m->m_len;
456 		}
457 		while (m->m_next)
458 			(m = m->m_next) ->m_len = 0;
459 	}
460 }
461 
462 /*
463  * Rearange an mbuf chain so that len bytes are contiguous
464  * and in the data area of an mbuf (so that mtod and dtom
465  * will work for a structure of size len).  Returns the resulting
466  * mbuf chain on success, frees it and returns null on failure.
467  * If there is room, it will add up to max_protohdr-len extra bytes to the
468  * contiguous region in an attempt to avoid being called next time.
469  */
470 int MPFail;
471 
472 struct mbuf *
473 m_pullup(n, len)
474 	register struct mbuf *n;
475 	int len;
476 {
477 	register struct mbuf *m;
478 	register int count;
479 	int space;
480 
481 	/*
482 	 * If first mbuf has no cluster, and has room for len bytes
483 	 * without shifting current data, pullup into it,
484 	 * otherwise allocate a new mbuf to prepend to the chain.
485 	 */
486 	if ((n->m_flags & M_EXT) == 0 &&
487 	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
488 		if (n->m_len >= len)
489 			return (n);
490 		m = n;
491 		n = n->m_next;
492 		len -= m->m_len;
493 	} else {
494 		if (len > MHLEN)
495 			goto bad;
496 		MGET(m, M_DONTWAIT, n->m_type);
497 		if (m == 0)
498 			goto bad;
499 		m->m_len = 0;
500 		if (n->m_flags & M_PKTHDR) {
501 			M_COPY_PKTHDR(m, n);
502 			n->m_flags &= ~M_PKTHDR;
503 		}
504 	}
505 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
506 	do {
507 		count = min(min(max(len, max_protohdr), space), n->m_len);
508 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
509 		  (unsigned)count);
510 		len -= count;
511 		m->m_len += count;
512 		n->m_len -= count;
513 		space -= count;
514 		if (n->m_len)
515 			n->m_data += count;
516 		else
517 			n = m_free(n);
518 	} while (len > 0 && n);
519 	if (len > 0) {
520 		(void) m_free(m);
521 		goto bad;
522 	}
523 	m->m_next = n;
524 	return (m);
525 bad:
526 	m_freem(n);
527 	MPFail++;
528 	return (0);
529 }
530 
531 /*
532  * Partition an mbuf chain in two pieces, returning the tail --
533  * all but the first len0 bytes.  In case of failure, it returns NULL and
534  * attempts to restore the chain to its original state.
535  */
536 struct mbuf *
537 m_split(m0, len0, wait)
538 	register struct mbuf *m0;
539 	int len0, wait;
540 {
541 	register struct mbuf *m, *n;
542 	unsigned len = len0, remain;
543 
544 	for (m = m0; m && len > m->m_len; m = m->m_next)
545 		len -= m->m_len;
546 	if (m == 0)
547 		return (0);
548 	remain = m->m_len - len;
549 	if (m0->m_flags & M_PKTHDR) {
550 		MGETHDR(n, wait, m0->m_type);
551 		if (n == 0)
552 			return (0);
553 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
554 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
555 		m0->m_pkthdr.len = len0;
556 		if (m->m_flags & M_EXT)
557 			goto extpacket;
558 		if (remain > MHLEN) {
559 			/* m can't be the lead packet */
560 			MH_ALIGN(n, 0);
561 			n->m_next = m_split(m, len, wait);
562 			if (n->m_next == 0) {
563 				(void) m_free(n);
564 				return (0);
565 			} else
566 				return (n);
567 		} else
568 			MH_ALIGN(n, remain);
569 	} else if (remain == 0) {
570 		n = m->m_next;
571 		m->m_next = 0;
572 		return (n);
573 	} else {
574 		MGET(n, wait, m->m_type);
575 		if (n == 0)
576 			return (0);
577 		M_ALIGN(n, remain);
578 	}
579 extpacket:
580 	if (m->m_flags & M_EXT) {
581 		n->m_flags |= M_EXT;
582 		n->m_ext = m->m_ext;
583 		mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
584 		m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */
585 		n->m_data = m->m_data + len;
586 	} else {
587 		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
588 	}
589 	n->m_len = remain;
590 	m->m_len = len;
591 	n->m_next = m->m_next;
592 	m->m_next = 0;
593 	return (n);
594 }
595 /*
596  * Routine to copy from device local memory into mbufs.
597  */
598 struct mbuf *
599 m_devget(buf, totlen, off0, ifp, copy)
600 	char *buf;
601 	int totlen, off0;
602 	struct ifnet *ifp;
603 	void (*copy)();
604 {
605 	register struct mbuf *m;
606 	struct mbuf *top = 0, **mp = &top;
607 	register int off = off0, len;
608 	register char *cp;
609 	char *epkt;
610 
611 	cp = buf;
612 	epkt = cp + totlen;
613 	if (off) {
614 		cp += off + 2 * sizeof(u_short);
615 		totlen -= 2 * sizeof(u_short);
616 	}
617 	MGETHDR(m, M_DONTWAIT, MT_DATA);
618 	if (m == 0)
619 		return (0);
620 	m->m_pkthdr.rcvif = ifp;
621 	m->m_pkthdr.len = totlen;
622 	m->m_len = MHLEN;
623 
624 	while (totlen > 0) {
625 		if (top) {
626 			MGET(m, M_DONTWAIT, MT_DATA);
627 			if (m == 0) {
628 				m_freem(top);
629 				return (0);
630 			}
631 			m->m_len = MLEN;
632 		}
633 		len = min(totlen, epkt - cp);
634 		if (len >= MINCLSIZE) {
635 			MCLGET(m, M_DONTWAIT);
636 			if (m->m_flags & M_EXT)
637 				m->m_len = len = min(len, MCLBYTES);
638 			else
639 				len = m->m_len;
640 		} else {
641 			/*
642 			 * Place initial small packet/header at end of mbuf.
643 			 */
644 			if (len < m->m_len) {
645 				if (top == 0 && len + max_linkhdr <= m->m_len)
646 					m->m_data += max_linkhdr;
647 				m->m_len = len;
648 			} else
649 				len = m->m_len;
650 		}
651 		if (copy)
652 			copy(cp, mtod(m, caddr_t), (unsigned)len);
653 		else
654 			bcopy(cp, mtod(m, caddr_t), (unsigned)len);
655 		cp += len;
656 		*mp = m;
657 		mp = &m->m_next;
658 		totlen -= len;
659 		if (cp == epkt)
660 			cp = buf;
661 	}
662 	return (top);
663 }
664 
665 /*
666  * Copy data from a buffer back into the indicated mbuf chain,
667  * starting "off" bytes from the beginning, extending the mbuf
668  * chain if necessary.
669  */
670 void
671 m_copyback(m0, off, len, cp)
672 	struct	mbuf *m0;
673 	register int off;
674 	register int len;
675 	caddr_t cp;
676 {
677 	register int mlen;
678 	register struct mbuf *m = m0, *n;
679 	int totlen = 0;
680 
681 	if (m0 == 0)
682 		return;
683 	while (off > (mlen = m->m_len)) {
684 		off -= mlen;
685 		totlen += mlen;
686 		if (m->m_next == 0) {
687 			n = m_getclr(M_DONTWAIT, m->m_type);
688 			if (n == 0)
689 				goto out;
690 			n->m_len = min(MLEN, len + off);
691 			m->m_next = n;
692 		}
693 		m = m->m_next;
694 	}
695 	while (len > 0) {
696 		mlen = min (m->m_len - off, len);
697 		bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
698 		cp += mlen;
699 		len -= mlen;
700 		mlen += off;
701 		off = 0;
702 		totlen += mlen;
703 		if (len == 0)
704 			break;
705 		if (m->m_next == 0) {
706 			n = m_get(M_DONTWAIT, m->m_type);
707 			if (n == 0)
708 				break;
709 			n->m_len = min(MLEN, len);
710 			m->m_next = n;
711 		}
712 		m = m->m_next;
713 	}
714 out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
715 		m->m_pkthdr.len = totlen;
716 }
717