xref: /freebsd/sys/sys/mbuf.h (revision 1e413cf93298b5b97441a21d9a50fdcd0ee9945e)
1 /*-
2  * Copyright (c) 1982, 1986, 1988, 1993
3  *	The Regents of the University of California.
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. Neither the name of the University nor the names of its contributors
15  *    may be used to endorse or promote products derived from this software
16  *    without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  *	@(#)mbuf.h	8.5 (Berkeley) 2/19/95
31  * $FreeBSD$
32  */
33 
34 #ifndef _SYS_MBUF_H_
35 #define	_SYS_MBUF_H_
36 
37 /* XXX: These includes suck. Sorry! */
38 #include <sys/queue.h>
39 #ifdef _KERNEL
40 #include <sys/systm.h>
41 #include <vm/uma.h>
42 #ifdef WITNESS
43 #include <sys/lock.h>
44 #endif
45 #endif
46 
47 /*
48  * Mbufs are of a single size, MSIZE (sys/param.h), which includes overhead.
49  * An mbuf may add a single "mbuf cluster" of size MCLBYTES (also in
50  * sys/param.h), which has no additional overhead and is used instead of the
51  * internal data area; this is done when at least MINCLSIZE of data must be
52  * stored.  Additionally, it is possible to allocate a separate buffer
53  * externally and attach it to the mbuf in a way similar to that of mbuf
54  * clusters.
55  */
56 #define	MLEN		(MSIZE - sizeof(struct m_hdr))	/* normal data len */
57 #define	MHLEN		(MLEN - sizeof(struct pkthdr))	/* data len w/pkthdr */
58 #define	MINCLSIZE	(MHLEN + 1)	/* smallest amount to put in cluster */
59 #define	M_MAXCOMPRESS	(MHLEN / 2)	/* max amount to copy for compression */
60 
61 #ifdef _KERNEL
62 /*-
63  * Macros for type conversion:
64  * mtod(m, t)	-- Convert mbuf pointer to data pointer of correct type.
65  * dtom(x)	-- Convert data pointer within mbuf to mbuf pointer (XXX).
66  */
67 #define	mtod(m, t)	((t)((m)->m_data))
68 #define	dtom(x)		((struct mbuf *)((intptr_t)(x) & ~(MSIZE-1)))
69 
70 /*
71  * Argument structure passed to UMA routines during mbuf and packet
72  * allocations.
73  */
74 struct mb_args {
75 	int	flags;	/* Flags for mbuf being allocated */
76 	short	type;	/* Type of mbuf being allocated */
77 };
78 #endif /* _KERNEL */
79 
80 #if defined(__LP64__)
81 #define M_HDR_PAD    6
82 #else
83 #define M_HDR_PAD    2
84 #endif
85 
86 /*
87  * Header present at the beginning of every mbuf.
88  */
89 struct m_hdr {
90 	struct mbuf	*mh_next;	/* next buffer in chain */
91 	struct mbuf	*mh_nextpkt;	/* next chain in queue/record */
92 	caddr_t		 mh_data;	/* location of data */
93 	int		 mh_len;	/* amount of data in this mbuf */
94 	int		 mh_flags;	/* flags; see below */
95 	short		 mh_type;	/* type of data in this mbuf */
96 	uint8_t          pad[M_HDR_PAD];/* word align                  */
97 };
98 
99 /*
100  * Packet tag structure (see below for details).
101  */
102 struct m_tag {
103 	SLIST_ENTRY(m_tag)	m_tag_link;	/* List of packet tags */
104 	u_int16_t		m_tag_id;	/* Tag ID */
105 	u_int16_t		m_tag_len;	/* Length of data */
106 	u_int32_t		m_tag_cookie;	/* ABI/Module ID */
107 	void			(*m_tag_free)(struct m_tag *);
108 };
109 
110 /*
111  * Record/packet header in first mbuf of chain; valid only if M_PKTHDR is set.
112  */
113 struct pkthdr {
114 	struct ifnet	*rcvif;		/* rcv interface */
115 	/* variables for ip and tcp reassembly */
116 	void		*header;	/* pointer to packet header */
117 	int		 len;		/* total packet length */
118 	/* variables for hardware checksum */
119 	int		 csum_flags;	/* flags regarding checksum */
120 	int		 csum_data;	/* data field used by csum routines */
121 	u_int16_t	 tso_segsz;	/* TSO segment size */
122 	u_int16_t	 ether_vtag;	/* Ethernet 802.1p+q vlan tag */
123 	SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
124 };
125 
126 /*
127  * Description of external storage mapped into mbuf; valid only if M_EXT is
128  * set.
129  */
130 struct m_ext {
131 	caddr_t		 ext_buf;	/* start of buffer */
132 	void		(*ext_free)	/* free routine if not the usual */
133 			    (void *, void *);
134 	void		*ext_args;	/* optional argument pointer */
135 	u_int		 ext_size;	/* size of buffer, for ext_free */
136 	volatile u_int	*ref_cnt;	/* pointer to ref count info */
137 	int		 ext_type;	/* type of external storage */
138 };
139 
140 /*
141  * The core of the mbuf object along with some shortcut defines for practical
142  * purposes.
143  */
144 struct mbuf {
145 	struct m_hdr	m_hdr;
146 	union {
147 		struct {
148 			struct pkthdr	MH_pkthdr;	/* M_PKTHDR set */
149 			union {
150 				struct m_ext	MH_ext;	/* M_EXT set */
151 				char		MH_databuf[MHLEN];
152 			} MH_dat;
153 		} MH;
154 		char	M_databuf[MLEN];		/* !M_PKTHDR, !M_EXT */
155 	} M_dat;
156 };
157 #define	m_next		m_hdr.mh_next
158 #define	m_len		m_hdr.mh_len
159 #define	m_data		m_hdr.mh_data
160 #define	m_type		m_hdr.mh_type
161 #define	m_flags		m_hdr.mh_flags
162 #define	m_nextpkt	m_hdr.mh_nextpkt
163 #define	m_act		m_nextpkt
164 #define	m_pkthdr	M_dat.MH.MH_pkthdr
165 #define	m_ext		M_dat.MH.MH_dat.MH_ext
166 #define	m_pktdat	M_dat.MH.MH_dat.MH_databuf
167 #define	m_dat		M_dat.M_databuf
168 
169 /*
170  * mbuf flags.
171  */
172 #define	M_EXT		0x0001	/* has associated external storage */
173 #define	M_PKTHDR	0x0002	/* start of record */
174 #define	M_EOR		0x0004	/* end of record */
175 #define	M_RDONLY	0x0008	/* associated data is marked read-only */
176 #define	M_PROTO1	0x0010	/* protocol-specific */
177 #define	M_PROTO2	0x0020	/* protocol-specific */
178 #define	M_PROTO3	0x0040	/* protocol-specific */
179 #define	M_PROTO4	0x0080	/* protocol-specific */
180 #define	M_PROTO5	0x0100	/* protocol-specific */
181 #define	M_NOTIFICATION	M_PROTO5/* SCTP notification */
182 #define	M_SKIP_FIREWALL	0x4000	/* skip firewall processing */
183 #define	M_FREELIST	0x8000	/* mbuf is on the free list */
184 
185 /*
186  * mbuf pkthdr flags (also stored in m_flags).
187  */
188 #define	M_BCAST		0x0200	/* send/received as link-level broadcast */
189 #define	M_MCAST		0x0400	/* send/received as link-level multicast */
190 #define	M_FRAG		0x0800	/* packet is a fragment of a larger packet */
191 #define	M_FIRSTFRAG	0x1000	/* packet is first fragment */
192 #define	M_LASTFRAG	0x2000	/* packet is last fragment */
193 #define	M_VLANTAG	0x10000	/* ether_vtag is valid */
194 #define	M_PROMISC	0x20000	/* packet was not for us */
195 #define	M_NOFREE	0x40000	/* do not free mbuf - it is embedded in the cluster */
196 
197 /*
198  * External buffer types: identify ext_buf type.
199  */
200 #define	EXT_CLUSTER	1	/* mbuf cluster */
201 #define	EXT_SFBUF	2	/* sendfile(2)'s sf_bufs */
202 #define	EXT_JUMBOP	3	/* jumbo cluster 4096 bytes */
203 #define	EXT_JUMBO9	4	/* jumbo cluster 9216 bytes */
204 #define	EXT_JUMBO16	5	/* jumbo cluster 16184 bytes */
205 #define	EXT_PACKET	6	/* mbuf+cluster from packet zone */
206 #define	EXT_MBUF	7	/* external mbuf reference (M_IOVEC) */
207 #define	EXT_NET_DRV	100	/* custom ext_buf provided by net driver(s) */
208 #define	EXT_MOD_TYPE	200	/* custom module's ext_buf type */
209 #define	EXT_DISPOSABLE	300	/* can throw this buffer away w/page flipping */
210 #define	EXT_EXTREF	400	/* has externally maintained ref_cnt ptr */
211 
212 /*
213  * Flags copied when copying m_pkthdr.
214  */
215 #define	M_COPYFLAGS	(M_PKTHDR|M_EOR|M_RDONLY|M_PROTO1|M_PROTO1|M_PROTO2|\
216 			    M_PROTO3|M_PROTO4|M_PROTO5|M_SKIP_FIREWALL|\
217 			    M_BCAST|M_MCAST|M_FRAG|M_FIRSTFRAG|M_LASTFRAG|\
218 			    M_VLANTAG|M_PROMISC)
219 
220 /*
221  * Flags to purge when crossing layers.
222  */
223 #define	M_PROTOFLAGS	(M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO4|M_PROTO5)
224 
225 /*
226  * Flags indicating hw checksum support and sw checksum requirements.  This
227  * field can be directly tested against if_data.ifi_hwassist.
228  */
229 #define	CSUM_IP			0x0001		/* will csum IP */
230 #define	CSUM_TCP		0x0002		/* will csum TCP */
231 #define	CSUM_UDP		0x0004		/* will csum UDP */
232 #define	CSUM_IP_FRAGS		0x0008		/* will csum IP fragments */
233 #define	CSUM_FRAGMENT		0x0010		/* will do IP fragmentation */
234 #define	CSUM_TSO		0x0020		/* will do TSO */
235 
236 #define	CSUM_IP_CHECKED		0x0100		/* did csum IP */
237 #define	CSUM_IP_VALID		0x0200		/*   ... the csum is valid */
238 #define	CSUM_DATA_VALID		0x0400		/* csum_data field is valid */
239 #define	CSUM_PSEUDO_HDR		0x0800		/* csum_data has pseudo hdr */
240 
241 #define	CSUM_DELAY_DATA		(CSUM_TCP | CSUM_UDP)
242 #define	CSUM_DELAY_IP		(CSUM_IP)	/* XXX add ipv6 here too? */
243 
244 /*
245  * mbuf types.
246  */
247 #define	MT_NOTMBUF	0	/* USED INTERNALLY ONLY! Object is not mbuf */
248 #define	MT_DATA		1	/* dynamic (data) allocation */
249 #define	MT_HEADER	MT_DATA	/* packet header, use M_PKTHDR instead */
250 #define	MT_SONAME	8	/* socket name */
251 #define	MT_CONTROL	14	/* extra-data protocol message */
252 #define	MT_OOBDATA	15	/* expedited data  */
253 #define	MT_NTYPES	16	/* number of mbuf types for mbtypes[] */
254 
255 #define	MT_NOINIT	255	/* Not a type but a flag to allocate
256 				   a non-initialized mbuf */
257 
258 #define MB_NOTAGS	0x1UL	/* no tags attached to mbuf */
259 
260 /*
261  * General mbuf allocator statistics structure.
262  *
263  * Many of these statistics are no longer used; we instead track many
264  * allocator statistics through UMA's built in statistics mechanism.
265  */
266 struct mbstat {
267 	u_long	m_mbufs;	/* XXX */
268 	u_long	m_mclusts;	/* XXX */
269 
270 	u_long	m_drain;	/* times drained protocols for space */
271 	u_long	m_mcfail;	/* XXX: times m_copym failed */
272 	u_long	m_mpfail;	/* XXX: times m_pullup failed */
273 	u_long	m_msize;	/* length of an mbuf */
274 	u_long	m_mclbytes;	/* length of an mbuf cluster */
275 	u_long	m_minclsize;	/* min length of data to allocate a cluster */
276 	u_long	m_mlen;		/* length of data in an mbuf */
277 	u_long	m_mhlen;	/* length of data in a header mbuf */
278 
279 	/* Number of mbtypes (gives # elems in mbtypes[] array: */
280 	short	m_numtypes;
281 
282 	/* XXX: Sendfile stats should eventually move to their own struct */
283 	u_long	sf_iocnt;	/* times sendfile had to do disk I/O */
284 	u_long	sf_allocfail;	/* times sfbuf allocation failed */
285 	u_long	sf_allocwait;	/* times sfbuf allocation had to wait */
286 };
287 
288 /*
289  * Flags specifying how an allocation should be made.
290  *
291  * The flag to use is as follows:
292  * - M_DONTWAIT or M_NOWAIT from an interrupt handler to not block allocation.
293  * - M_WAIT or M_WAITOK or M_TRYWAIT from wherever it is safe to block.
294  *
295  * M_DONTWAIT/M_NOWAIT means that we will not block the thread explicitly and
296  * if we cannot allocate immediately we may return NULL, whereas
297  * M_WAIT/M_WAITOK/M_TRYWAIT means that if we cannot allocate resources we
298  * will block until they are available, and thus never return NULL.
299  *
300  * XXX Eventually just phase this out to use M_WAITOK/M_NOWAIT.
301  */
302 #define	MBTOM(how)	(how)
303 #define	M_DONTWAIT	M_NOWAIT
304 #define	M_TRYWAIT	M_WAITOK
305 #define	M_WAIT		M_WAITOK
306 
307 /*
308  * String names of mbuf-related UMA(9) and malloc(9) types.  Exposed to
309  * !_KERNEL so that monitoring tools can look up the zones with
310  * libmemstat(3).
311  */
312 #define	MBUF_MEM_NAME		"mbuf"
313 #define	MBUF_CLUSTER_MEM_NAME	"mbuf_cluster"
314 #define	MBUF_PACKET_MEM_NAME	"mbuf_packet"
315 #define	MBUF_JUMBOP_MEM_NAME	"mbuf_jumbo_page"
316 #define	MBUF_JUMBO9_MEM_NAME	"mbuf_jumbo_9k"
317 #define	MBUF_JUMBO16_MEM_NAME	"mbuf_jumbo_16k"
318 #define	MBUF_TAG_MEM_NAME	"mbuf_tag"
319 #define	MBUF_EXTREFCNT_MEM_NAME	"mbuf_ext_refcnt"
320 
321 #ifdef _KERNEL
322 
323 #ifdef WITNESS
324 #define	MBUF_CHECKSLEEP(how) do {					\
325 	if (how == M_WAITOK)						\
326 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,		\
327 		    "Sleeping in \"%s\"", __func__);			\
328 } while (0)
329 #else
330 #define	MBUF_CHECKSLEEP(how)
331 #endif
332 
333 /*
334  * Network buffer allocation API
335  *
336  * The rest of it is defined in kern/kern_mbuf.c
337  */
338 
339 extern uma_zone_t	zone_mbuf;
340 extern uma_zone_t	zone_clust;
341 extern uma_zone_t	zone_pack;
342 extern uma_zone_t	zone_jumbop;
343 extern uma_zone_t	zone_jumbo9;
344 extern uma_zone_t	zone_jumbo16;
345 extern uma_zone_t	zone_ext_refcnt;
346 
347 static __inline struct mbuf	*m_getcl(int how, short type, int flags);
348 static __inline struct mbuf	*m_get(int how, short type);
349 static __inline struct mbuf	*m_gethdr(int how, short type);
350 static __inline struct mbuf	*m_getjcl(int how, short type, int flags,
351 				    int size);
352 static __inline struct mbuf	*m_getclr(int how, short type);	/* XXX */
353 static __inline struct mbuf	*m_free(struct mbuf *m);
354 static __inline void		 m_clget(struct mbuf *m, int how);
355 static __inline void		*m_cljget(struct mbuf *m, int how, int size);
356 static __inline void		 m_chtype(struct mbuf *m, short new_type);
357 void				 mb_free_ext(struct mbuf *);
358 static __inline struct mbuf	*m_last(struct mbuf *m);
359 
360 static __inline int
361 m_gettype(int size)
362 {
363 	int type;
364 
365 	switch (size) {
366 	case MSIZE:
367 		type = EXT_MBUF;
368 		break;
369 	case MCLBYTES:
370 		type = EXT_CLUSTER;
371 		break;
372 #if MJUMPAGESIZE != MCLBYTES
373 	case MJUMPAGESIZE:
374 		type = EXT_JUMBOP;
375 		break;
376 #endif
377 	case MJUM9BYTES:
378 		type = EXT_JUMBO9;
379 		break;
380 	case MJUM16BYTES:
381 		type = EXT_JUMBO16;
382 		break;
383 	default:
384 		panic("%s: m_getjcl: invalid cluster size", __func__);
385 	}
386 
387 	return (type);
388 }
389 
390 static __inline uma_zone_t
391 m_getzone(int size)
392 {
393 	uma_zone_t zone;
394 
395 	switch (size) {
396 	case MSIZE:
397 		zone = zone_mbuf;
398 		break;
399 	case MCLBYTES:
400 		zone = zone_clust;
401 		break;
402 #if MJUMPAGESIZE != MCLBYTES
403 	case MJUMPAGESIZE:
404 		zone = zone_jumbop;
405 		break;
406 #endif
407 	case MJUM9BYTES:
408 		zone = zone_jumbo9;
409 		break;
410 	case MJUM16BYTES:
411 		zone = zone_jumbo16;
412 		break;
413 	default:
414 		panic("%s: m_getjcl: invalid cluster type", __func__);
415 	}
416 
417 	return (zone);
418 }
419 
420 static __inline struct mbuf *
421 m_get(int how, short type)
422 {
423 	struct mb_args args;
424 
425 	args.flags = 0;
426 	args.type = type;
427 	return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how)));
428 }
429 
430 /*
431  * XXX This should be deprecated, very little use.
432  */
433 static __inline struct mbuf *
434 m_getclr(int how, short type)
435 {
436 	struct mbuf *m;
437 	struct mb_args args;
438 
439 	args.flags = 0;
440 	args.type = type;
441 	m = uma_zalloc_arg(zone_mbuf, &args, how);
442 	if (m != NULL)
443 		bzero(m->m_data, MLEN);
444 	return (m);
445 }
446 
447 static __inline struct mbuf *
448 m_gethdr(int how, short type)
449 {
450 	struct mb_args args;
451 
452 	args.flags = M_PKTHDR;
453 	args.type = type;
454 	return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how)));
455 }
456 
457 static __inline struct mbuf *
458 m_getcl(int how, short type, int flags)
459 {
460 	struct mb_args args;
461 
462 	args.flags = flags;
463 	args.type = type;
464 	return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how)));
465 }
466 
467 /*
468  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
469  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
470  *
471  * XXX: This is rather large, should be real function maybe.
472  */
473 static __inline struct mbuf *
474 m_getjcl(int how, short type, int flags, int size)
475 {
476 	struct mb_args args;
477 	struct mbuf *m, *n;
478 	uma_zone_t zone;
479 
480 	args.flags = flags;
481 	args.type = type;
482 
483 	m = uma_zalloc_arg(zone_mbuf, &args, how);
484 	if (m == NULL)
485 		return (NULL);
486 
487 	zone = m_getzone(size);
488 	n = uma_zalloc_arg(zone, m, how);
489 	if (n == NULL) {
490 		uma_zfree(zone_mbuf, m);
491 		return (NULL);
492 	}
493 	return (m);
494 }
495 
496 static __inline void
497 m_free_fast(struct mbuf *m)
498 {
499 #ifdef INVARIANTS
500 	if (m->m_flags & M_PKTHDR)
501 		KASSERT(SLIST_EMPTY(&m->m_pkthdr.tags), ("doing fast free of mbuf with tags"));
502 #endif
503 
504 	uma_zfree_arg(zone_mbuf, m, (void *)MB_NOTAGS);
505 }
506 
507 static __inline struct mbuf *
508 m_free(struct mbuf *m)
509 {
510 	struct mbuf *n = m->m_next;
511 
512 	if (m->m_flags & M_EXT)
513 		mb_free_ext(m);
514 	else if ((m->m_flags & M_NOFREE) == 0)
515 		uma_zfree(zone_mbuf, m);
516 	return (n);
517 }
518 
519 static __inline void
520 m_clget(struct mbuf *m, int how)
521 {
522 
523 	if (m->m_flags & M_EXT)
524 		printf("%s: %p mbuf already has cluster\n", __func__, m);
525 	m->m_ext.ext_buf = (char *)NULL;
526 	uma_zalloc_arg(zone_clust, m, how);
527 	/*
528 	 * On a cluster allocation failure, drain the packet zone and retry,
529 	 * we might be able to loosen a few clusters up on the drain.
530 	 */
531 	if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
532 		zone_drain(zone_pack);
533 		uma_zalloc_arg(zone_clust, m, how);
534 	}
535 }
536 
537 /*
538  * m_cljget() is different from m_clget() as it can allocate clusters without
539  * attaching them to an mbuf.  In that case the return value is the pointer
540  * to the cluster of the requested size.  If an mbuf was specified, it gets
541  * the cluster attached to it and the return value can be safely ignored.
542  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
543  */
544 static __inline void *
545 m_cljget(struct mbuf *m, int how, int size)
546 {
547 	uma_zone_t zone;
548 
549 	if (m && m->m_flags & M_EXT)
550 		printf("%s: %p mbuf already has cluster\n", __func__, m);
551 	if (m != NULL)
552 		m->m_ext.ext_buf = NULL;
553 
554 	zone = m_getzone(size);
555 	return (uma_zalloc_arg(zone, m, how));
556 }
557 
558 static __inline void
559 m_cljset(struct mbuf *m, void *cl, int type)
560 {
561 	uma_zone_t zone;
562 	int size;
563 
564 	switch (type) {
565 	case EXT_CLUSTER:
566 		size = MCLBYTES;
567 		zone = zone_clust;
568 		break;
569 #if MJUMPAGESIZE != MCLBYTES
570 	case EXT_JUMBOP:
571 		size = MJUMPAGESIZE;
572 		zone = zone_jumbop;
573 		break;
574 #endif
575 	case EXT_JUMBO9:
576 		size = MJUM9BYTES;
577 		zone = zone_jumbo9;
578 		break;
579 	case EXT_JUMBO16:
580 		size = MJUM16BYTES;
581 		zone = zone_jumbo16;
582 		break;
583 	default:
584 		panic("unknown cluster type");
585 		break;
586 	}
587 
588 	m->m_data = m->m_ext.ext_buf = cl;
589 	m->m_ext.ext_free = m->m_ext.ext_args = NULL;
590 	m->m_ext.ext_size = size;
591 	m->m_ext.ext_type = type;
592 	m->m_ext.ref_cnt = uma_find_refcnt(zone, cl);
593 	m->m_flags |= M_EXT;
594 
595 }
596 
597 static __inline void
598 m_chtype(struct mbuf *m, short new_type)
599 {
600 
601 	m->m_type = new_type;
602 }
603 
604 static __inline struct mbuf *
605 m_last(struct mbuf *m)
606 {
607 
608 	while (m->m_next)
609 		m = m->m_next;
610 	return (m);
611 }
612 
613 /*
614  * mbuf, cluster, and external object allocation macros (for compatibility
615  * purposes).
616  */
617 #define	M_MOVE_PKTHDR(to, from)	m_move_pkthdr((to), (from))
618 #define	MGET(m, how, type)	((m) = m_get((how), (type)))
619 #define	MGETHDR(m, how, type)	((m) = m_gethdr((how), (type)))
620 #define	MCLGET(m, how)		m_clget((m), (how))
621 #define	MEXTADD(m, buf, size, free, args, flags, type) 			\
622     m_extadd((m), (caddr_t)(buf), (size), (free), (args), (flags), (type))
623 #define	m_getm(m, len, how, type)					\
624     m_getm2((m), (len), (how), (type), M_PKTHDR)
625 
626 /*
627  * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
628  * be both the local data payload, or an external buffer area, depending on
629  * whether M_EXT is set).
630  */
631 #define	M_WRITABLE(m)	(!((m)->m_flags & M_RDONLY) &&			\
632 			 (!(((m)->m_flags & M_EXT)) ||			\
633 			 (*((m)->m_ext.ref_cnt) == 1)) )		\
634 
635 /* Check if the supplied mbuf has a packet header, or else panic. */
636 #define	M_ASSERTPKTHDR(m)						\
637 	KASSERT((m) != NULL && (m)->m_flags & M_PKTHDR,			\
638 	    ("%s: no mbuf packet header!", __func__))
639 
640 /*
641  * Ensure that the supplied mbuf is a valid, non-free mbuf.
642  *
643  * XXX: Broken at the moment.  Need some UMA magic to make it work again.
644  */
645 #define	M_ASSERTVALID(m)						\
646 	KASSERT((((struct mbuf *)m)->m_flags & 0) == 0,			\
647 	    ("%s: attempted use of a free mbuf!", __func__))
648 
649 /*
650  * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place an
651  * object of the specified size at the end of the mbuf, longword aligned.
652  */
653 #define	M_ALIGN(m, len) do {						\
654 	KASSERT(!((m)->m_flags & (M_PKTHDR|M_EXT)),			\
655 		("%s: M_ALIGN not normal mbuf", __func__));		\
656 	KASSERT((m)->m_data == (m)->m_dat,				\
657 		("%s: M_ALIGN not a virgin mbuf", __func__));		\
658 	(m)->m_data += (MLEN - (len)) & ~(sizeof(long) - 1);		\
659 } while (0)
660 
661 /*
662  * As above, for mbufs allocated with m_gethdr/MGETHDR or initialized by
663  * M_DUP/MOVE_PKTHDR.
664  */
665 #define	MH_ALIGN(m, len) do {						\
666 	KASSERT((m)->m_flags & M_PKTHDR && !((m)->m_flags & M_EXT),	\
667 		("%s: MH_ALIGN not PKTHDR mbuf", __func__));		\
668 	KASSERT((m)->m_data == (m)->m_pktdat,				\
669 		("%s: MH_ALIGN not a virgin mbuf", __func__));		\
670 	(m)->m_data += (MHLEN - (len)) & ~(sizeof(long) - 1);		\
671 } while (0)
672 
673 /*
674  * Compute the amount of space available before the current start of data in
675  * an mbuf.
676  *
677  * The M_WRITABLE() is a temporary, conservative safety measure: the burden
678  * of checking writability of the mbuf data area rests solely with the caller.
679  */
680 #define	M_LEADINGSPACE(m)						\
681 	((m)->m_flags & M_EXT ?						\
682 	    (M_WRITABLE(m) ? (m)->m_data - (m)->m_ext.ext_buf : 0):	\
683 	    (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat :	\
684 	    (m)->m_data - (m)->m_dat)
685 
686 /*
687  * Compute the amount of space available after the end of data in an mbuf.
688  *
689  * The M_WRITABLE() is a temporary, conservative safety measure: the burden
690  * of checking writability of the mbuf data area rests solely with the caller.
691  */
692 #define	M_TRAILINGSPACE(m)						\
693 	((m)->m_flags & M_EXT ?						\
694 	    (M_WRITABLE(m) ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size	\
695 		- ((m)->m_data + (m)->m_len) : 0) :			\
696 	    &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len))
697 
698 /*
699  * Arrange to prepend space of size plen to mbuf m.  If a new mbuf must be
700  * allocated, how specifies whether to wait.  If the allocation fails, the
701  * original mbuf chain is freed and m is set to NULL.
702  */
703 #define	M_PREPEND(m, plen, how) do {					\
704 	struct mbuf **_mmp = &(m);					\
705 	struct mbuf *_mm = *_mmp;					\
706 	int _mplen = (plen);						\
707 	int __mhow = (how);						\
708 									\
709 	MBUF_CHECKSLEEP(how);						\
710 	if (M_LEADINGSPACE(_mm) >= _mplen) {				\
711 		_mm->m_data -= _mplen;					\
712 		_mm->m_len += _mplen;					\
713 	} else								\
714 		_mm = m_prepend(_mm, _mplen, __mhow);			\
715 	if (_mm != NULL && _mm->m_flags & M_PKTHDR)			\
716 		_mm->m_pkthdr.len += _mplen;				\
717 	*_mmp = _mm;							\
718 } while (0)
719 
720 /*
721  * Change mbuf to new type.  This is a relatively expensive operation and
722  * should be avoided.
723  */
724 #define	MCHTYPE(m, t)	m_chtype((m), (t))
725 
726 /* Length to m_copy to copy all. */
727 #define	M_COPYALL	1000000000
728 
729 /* Compatibility with 4.3. */
730 #define	m_copy(m, o, l)	m_copym((m), (o), (l), M_DONTWAIT)
731 
732 extern int		max_datalen;	/* MHLEN - max_hdr */
733 extern int		max_hdr;	/* Largest link + protocol header */
734 extern int		max_linkhdr;	/* Largest link-level header */
735 extern int		max_protohdr;	/* Largest protocol header */
736 extern struct mbstat	mbstat;		/* General mbuf stats/infos */
737 extern int		nmbclusters;	/* Maximum number of clusters */
738 
739 struct uio;
740 
741 void		 m_adj(struct mbuf *, int);
742 void		 m_align(struct mbuf *, int);
743 int		 m_apply(struct mbuf *, int, int,
744 		    int (*)(void *, void *, u_int), void *);
745 int		 m_append(struct mbuf *, int, c_caddr_t);
746 void		 m_cat(struct mbuf *, struct mbuf *);
747 void		 m_extadd(struct mbuf *, caddr_t, u_int,
748 		    void (*)(void *, void *), void *, int, int);
749 void		 m_copyback(struct mbuf *, int, int, c_caddr_t);
750 void		 m_copydata(const struct mbuf *, int, int, caddr_t);
751 struct mbuf	*m_copym(struct mbuf *, int, int, int);
752 struct mbuf	*m_copymdata(struct mbuf *, struct mbuf *,
753 		    int, int, int, int);
754 struct mbuf	*m_copypacket(struct mbuf *, int);
755 void		 m_copy_pkthdr(struct mbuf *, struct mbuf *);
756 struct mbuf	*m_copyup(struct mbuf *n, int len, int dstoff);
757 struct mbuf	*m_defrag(struct mbuf *, int);
758 void		 m_demote(struct mbuf *, int);
759 struct mbuf	*m_devget(char *, int, int, struct ifnet *,
760 		    void (*)(char *, caddr_t, u_int));
761 struct mbuf	*m_dup(struct mbuf *, int);
762 int		 m_dup_pkthdr(struct mbuf *, struct mbuf *, int);
763 u_int		 m_fixhdr(struct mbuf *);
764 struct mbuf	*m_fragment(struct mbuf *, int, int);
765 void		 m_freem(struct mbuf *);
766 struct mbuf	*m_getm2(struct mbuf *, int, int, short, int);
767 struct mbuf	*m_getptr(struct mbuf *, int, int *);
768 u_int		 m_length(struct mbuf *, struct mbuf **);
769 void		 m_move_pkthdr(struct mbuf *, struct mbuf *);
770 struct mbuf	*m_prepend(struct mbuf *, int, int);
771 void		 m_print(const struct mbuf *, int);
772 struct mbuf	*m_pulldown(struct mbuf *, int, int, int *);
773 struct mbuf	*m_pullup(struct mbuf *, int);
774 int		m_sanity(struct mbuf *, int);
775 struct mbuf	*m_split(struct mbuf *, int, int);
776 struct mbuf	*m_uiotombuf(struct uio *, int, int, int, int);
777 struct mbuf	*m_unshare(struct mbuf *, int how);
778 
779 /*-
780  * Network packets may have annotations attached by affixing a list of
781  * "packet tags" to the pkthdr structure.  Packet tags are dynamically
782  * allocated semi-opaque data structures that have a fixed header
783  * (struct m_tag) that specifies the size of the memory block and a
784  * <cookie,type> pair that identifies it.  The cookie is a 32-bit unique
785  * unsigned value used to identify a module or ABI.  By convention this value
786  * is chosen as the date+time that the module is created, expressed as the
787  * number of seconds since the epoch (e.g., using date -u +'%s').  The type
788  * value is an ABI/module-specific value that identifies a particular
789  * annotation and is private to the module.  For compatibility with systems
790  * like OpenBSD that define packet tags w/o an ABI/module cookie, the value
791  * PACKET_ABI_COMPAT is used to implement m_tag_get and m_tag_find
792  * compatibility shim functions and several tag types are defined below.
793  * Users that do not require compatibility should use a private cookie value
794  * so that packet tag-related definitions can be maintained privately.
795  *
796  * Note that the packet tag returned by m_tag_alloc has the default memory
797  * alignment implemented by malloc.  To reference private data one can use a
798  * construct like:
799  *
800  *	struct m_tag *mtag = m_tag_alloc(...);
801  *	struct foo *p = (struct foo *)(mtag+1);
802  *
803  * if the alignment of struct m_tag is sufficient for referencing members of
804  * struct foo.  Otherwise it is necessary to embed struct m_tag within the
805  * private data structure to insure proper alignment; e.g.,
806  *
807  *	struct foo {
808  *		struct m_tag	tag;
809  *		...
810  *	};
811  *	struct foo *p = (struct foo *) m_tag_alloc(...);
812  *	struct m_tag *mtag = &p->tag;
813  */
814 
815 /*
816  * Persistent tags stay with an mbuf until the mbuf is reclaimed.  Otherwise
817  * tags are expected to ``vanish'' when they pass through a network
818  * interface.  For most interfaces this happens normally as the tags are
819  * reclaimed when the mbuf is free'd.  However in some special cases
820  * reclaiming must be done manually.  An example is packets that pass through
821  * the loopback interface.  Also, one must be careful to do this when
822  * ``turning around'' packets (e.g., icmp_reflect).
823  *
824  * To mark a tag persistent bit-or this flag in when defining the tag id.
825  * The tag will then be treated as described above.
826  */
827 #define	MTAG_PERSISTENT				0x800
828 
829 #define	PACKET_TAG_NONE				0  /* Nadda */
830 
831 /* Packet tags for use with PACKET_ABI_COMPAT. */
832 #define	PACKET_TAG_IPSEC_IN_DONE		1  /* IPsec applied, in */
833 #define	PACKET_TAG_IPSEC_OUT_DONE		2  /* IPsec applied, out */
834 #define	PACKET_TAG_IPSEC_IN_CRYPTO_DONE		3  /* NIC IPsec crypto done */
835 #define	PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED	4  /* NIC IPsec crypto req'ed */
836 #define	PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO	5  /* NIC notifies IPsec */
837 #define	PACKET_TAG_IPSEC_PENDING_TDB		6  /* Reminder to do IPsec */
838 #define	PACKET_TAG_BRIDGE			7  /* Bridge processing done */
839 #define	PACKET_TAG_GIF				8  /* GIF processing done */
840 #define	PACKET_TAG_GRE				9  /* GRE processing done */
841 #define	PACKET_TAG_IN_PACKET_CHECKSUM		10 /* NIC checksumming done */
842 #define	PACKET_TAG_ENCAP			11 /* Encap.  processing */
843 #define	PACKET_TAG_IPSEC_SOCKET			12 /* IPSEC socket ref */
844 #define	PACKET_TAG_IPSEC_HISTORY		13 /* IPSEC history */
845 #define	PACKET_TAG_IPV6_INPUT			14 /* IPV6 input processing */
846 #define	PACKET_TAG_DUMMYNET			15 /* dummynet info */
847 #define	PACKET_TAG_DIVERT			17 /* divert info */
848 #define	PACKET_TAG_IPFORWARD			18 /* ipforward info */
849 #define	PACKET_TAG_MACLABEL	(19 | MTAG_PERSISTENT) /* MAC label */
850 #define	PACKET_TAG_PF				21 /* PF + ALTQ information */
851 #define	PACKET_TAG_RTSOCKFAM			25 /* rtsock sa family */
852 #define	PACKET_TAG_IPOPTIONS			27 /* Saved IP options */
853 #define	PACKET_TAG_CARP                         28 /* CARP info */
854 
855 /* Specific cookies and tags. */
856 
857 /* Packet tag routines. */
858 struct m_tag	*m_tag_alloc(u_int32_t, int, int, int);
859 void		 m_tag_delete(struct mbuf *, struct m_tag *);
860 void		 m_tag_delete_chain(struct mbuf *, struct m_tag *);
861 void		 m_tag_free_default(struct m_tag *);
862 struct m_tag	*m_tag_locate(struct mbuf *, u_int32_t, int, struct m_tag *);
863 struct m_tag	*m_tag_copy(struct m_tag *, int);
864 int		 m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
865 void		 m_tag_delete_nonpersistent(struct mbuf *);
866 
867 /*
868  * Initialize the list of tags associated with an mbuf.
869  */
870 static __inline void
871 m_tag_init(struct mbuf *m)
872 {
873 
874 	SLIST_INIT(&m->m_pkthdr.tags);
875 }
876 
877 /*
878  * Set up the contents of a tag.  Note that this does not fill in the free
879  * method; the caller is expected to do that.
880  *
881  * XXX probably should be called m_tag_init, but that was already taken.
882  */
883 static __inline void
884 m_tag_setup(struct m_tag *t, u_int32_t cookie, int type, int len)
885 {
886 
887 	t->m_tag_id = type;
888 	t->m_tag_len = len;
889 	t->m_tag_cookie = cookie;
890 }
891 
892 /*
893  * Reclaim resources associated with a tag.
894  */
895 static __inline void
896 m_tag_free(struct m_tag *t)
897 {
898 
899 	(*t->m_tag_free)(t);
900 }
901 
902 /*
903  * Return the first tag associated with an mbuf.
904  */
905 static __inline struct m_tag *
906 m_tag_first(struct mbuf *m)
907 {
908 
909 	return (SLIST_FIRST(&m->m_pkthdr.tags));
910 }
911 
912 /*
913  * Return the next tag in the list of tags associated with an mbuf.
914  */
915 static __inline struct m_tag *
916 m_tag_next(struct mbuf *m, struct m_tag *t)
917 {
918 
919 	return (SLIST_NEXT(t, m_tag_link));
920 }
921 
922 /*
923  * Prepend a tag to the list of tags associated with an mbuf.
924  */
925 static __inline void
926 m_tag_prepend(struct mbuf *m, struct m_tag *t)
927 {
928 
929 	SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
930 }
931 
932 /*
933  * Unlink a tag from the list of tags associated with an mbuf.
934  */
935 static __inline void
936 m_tag_unlink(struct mbuf *m, struct m_tag *t)
937 {
938 
939 	SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
940 }
941 
942 /* These are for OpenBSD compatibility. */
943 #define	MTAG_ABI_COMPAT		0		/* compatibility ABI */
944 
945 static __inline struct m_tag *
946 m_tag_get(int type, int length, int wait)
947 {
948 	return (m_tag_alloc(MTAG_ABI_COMPAT, type, length, wait));
949 }
950 
951 static __inline struct m_tag *
952 m_tag_find(struct mbuf *m, int type, struct m_tag *start)
953 {
954 	return (SLIST_EMPTY(&m->m_pkthdr.tags) ? (struct m_tag *)NULL :
955 	    m_tag_locate(m, MTAG_ABI_COMPAT, type, start));
956 }
957 
958 #endif /* _KERNEL */
959 
960 #endif /* !_SYS_MBUF_H_ */
961