xref: /freebsd/sys/netgraph/ng_ppp.c (revision b601c69bdbe8755d26570261d7fd4c02ee4eff74)
1 
2 /*
3  * ng_ppp.c
4  *
5  * Copyright (c) 1996-2000 Whistle Communications, Inc.
6  * All rights reserved.
7  *
8  * Subject to the following obligations and disclaimer of warranty, use and
9  * redistribution of this software, in source or object code forms, with or
10  * without modifications are expressly permitted by Whistle Communications;
11  * provided, however, that:
12  * 1. Any and all reproductions of the source or object code must include the
13  *    copyright notice above and the following disclaimer of warranties; and
14  * 2. No rights are granted, in any manner or form, to use Whistle
15  *    Communications, Inc. trademarks, including the mark "WHISTLE
16  *    COMMUNICATIONS" on advertising, endorsements, or otherwise except as
17  *    such appears in the above copyright notice or in the software.
18  *
19  * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
20  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
21  * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
22  * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
24  * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
25  * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
26  * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
27  * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
28  * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
29  * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
30  * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
31  * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34  * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
35  * OF SUCH DAMAGE.
36  *
37  * Author: Archie Cobbs <archie@whistle.com>
38  *
39  * $FreeBSD$
40  * $Whistle: ng_ppp.c,v 1.24 1999/11/01 09:24:52 julian Exp $
41  */
42 
43 /*
44  * PPP node type.
45  */
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/time.h>
51 #include <sys/mbuf.h>
52 #include <sys/malloc.h>
53 #include <sys/errno.h>
54 #include <sys/syslog.h>
55 #include <sys/ctype.h>
56 
57 #include <machine/limits.h>
58 
59 #include <netgraph/ng_message.h>
60 #include <netgraph/netgraph.h>
61 #include <netgraph/ng_parse.h>
62 #include <netgraph/ng_ppp.h>
63 #include <netgraph/ng_vjc.h>
64 
65 #define PROT_VALID(p)		(((p) & 0x0101) == 0x0001)
66 #define PROT_COMPRESSABLE(p)	(((p) & 0xff00) == 0x0000)
67 
68 /* Some PPP protocol numbers we're interested in */
69 #define PROT_APPLETALK		0x0029
70 #define PROT_COMPD		0x00fd
71 #define PROT_CRYPTD		0x0053
72 #define PROT_IP			0x0021
73 #define PROT_IPV6		0x0057
74 #define PROT_IPX		0x002b
75 #define PROT_LCP		0xc021
76 #define PROT_MP			0x003d
77 #define PROT_VJCOMP		0x002d
78 #define PROT_VJUNCOMP		0x002f
79 
80 /* Multilink PPP definitions */
81 #define MP_MIN_MRRU		1500		/* per RFC 1990 */
82 #define MP_INITIAL_SEQ		0		/* per RFC 1990 */
83 #define MP_MIN_LINK_MRU		32
84 
85 #define MP_SHORT_SEQ_MASK	0x00000fff	/* short seq # mask */
86 #define MP_SHORT_SEQ_HIBIT	0x00000800	/* short seq # high bit */
87 #define MP_SHORT_FIRST_FLAG	0x00008000	/* first fragment in frame */
88 #define MP_SHORT_LAST_FLAG	0x00004000	/* last fragment in frame */
89 
90 #define MP_LONG_SEQ_MASK	0x00ffffff	/* long seq # mask */
91 #define MP_LONG_SEQ_HIBIT	0x00800000	/* long seq # high bit */
92 #define MP_LONG_FIRST_FLAG	0x80000000	/* first fragment in frame */
93 #define MP_LONG_LAST_FLAG	0x40000000	/* last fragment in frame */
94 
95 #define MP_NOSEQ		INT_MAX		/* impossible sequence number */
96 
97 #define MP_SEQ_MASK(priv)	((priv)->conf.recvShortSeq ? \
98 				    MP_SHORT_SEQ_MASK : MP_LONG_SEQ_MASK)
99 
100 /* Sign extension of MP sequence numbers */
101 #define MP_SHORT_EXTEND(s)	(((s) & MP_SHORT_SEQ_HIBIT) ? \
102 				    ((s) | ~MP_SHORT_SEQ_MASK) : (s))
103 #define MP_LONG_EXTEND(s)	(((s) & MP_LONG_SEQ_HIBIT) ? \
104 				    ((s) | ~MP_LONG_SEQ_MASK) : (s))
105 
106 /* Comparision of MP sequence numbers */
107 #define MP_SHORT_SEQ_DIFF(x,y)	(MP_SHORT_EXTEND(x) - MP_SHORT_EXTEND(y))
108 #define MP_LONG_SEQ_DIFF(x,y)	(MP_LONG_EXTEND(x) - MP_LONG_EXTEND(y))
109 
110 #define MP_SEQ_DIFF(priv,x,y)	((priv)->conf.recvShortSeq ? \
111 				    MP_SHORT_SEQ_DIFF((x), (y)) : \
112 				    MP_LONG_SEQ_DIFF((x), (y)))
113 
114 #define MP_NEXT_SEQ(priv,seq)	(((seq) + 1) & MP_SEQ_MASK(priv))
115 #define MP_PREV_SEQ(priv,seq)	(((seq) - 1) & MP_SEQ_MASK(priv))
116 
117 /* Don't fragment transmitted packets smaller than this */
118 #define MP_MIN_FRAG_LEN		6
119 
120 /* Maximum fragment reasssembly queue length */
121 #define MP_MAX_QUEUE_LEN	128
122 
123 /* Fragment queue scanner period */
124 #define MP_FRAGTIMER_INTERVAL	(hz/2)
125 
126 /* We store incoming fragments this way */
127 struct ng_ppp_frag {
128 	int				seq;		/* fragment seq# */
129 	u_char				first;		/* First in packet? */
130 	u_char				last;		/* Last in packet? */
131 	struct timeval			timestamp;	/* time of reception */
132 	struct mbuf			*data;		/* Fragment data */
133 	meta_p				meta;		/* Fragment meta */
134 	CIRCLEQ_ENTRY(ng_ppp_frag)	f_qent;		/* Fragment queue */
135 };
136 
137 /* We use integer indicies to refer to the non-link hooks */
138 static const char *const ng_ppp_hook_names[] = {
139 	NG_PPP_HOOK_ATALK,
140 #define HOOK_INDEX_ATALK		0
141 	NG_PPP_HOOK_BYPASS,
142 #define HOOK_INDEX_BYPASS		1
143 	NG_PPP_HOOK_COMPRESS,
144 #define HOOK_INDEX_COMPRESS		2
145 	NG_PPP_HOOK_ENCRYPT,
146 #define HOOK_INDEX_ENCRYPT		3
147 	NG_PPP_HOOK_DECOMPRESS,
148 #define HOOK_INDEX_DECOMPRESS		4
149 	NG_PPP_HOOK_DECRYPT,
150 #define HOOK_INDEX_DECRYPT		5
151 	NG_PPP_HOOK_INET,
152 #define HOOK_INDEX_INET			6
153 	NG_PPP_HOOK_IPX,
154 #define HOOK_INDEX_IPX			7
155 	NG_PPP_HOOK_VJC_COMP,
156 #define HOOK_INDEX_VJC_COMP		8
157 	NG_PPP_HOOK_VJC_IP,
158 #define HOOK_INDEX_VJC_IP		9
159 	NG_PPP_HOOK_VJC_UNCOMP,
160 #define HOOK_INDEX_VJC_UNCOMP		10
161 	NG_PPP_HOOK_VJC_VJIP,
162 #define HOOK_INDEX_VJC_VJIP		11
163 	NG_PPP_HOOK_IPV6,
164 #define HOOK_INDEX_IPV6			12
165 	NULL
166 #define HOOK_INDEX_MAX			13
167 };
168 
169 /* We store index numbers in the hook private pointer. The HOOK_INDEX()
170    for a hook is either the index (above) for normal hooks, or the ones
171    complement of the link number for link hooks. */
172 #define HOOK_INDEX(hook)	(*((int16_t *) &(hook)->private))
173 
174 /* Per-link private information */
175 struct ng_ppp_link {
176 	struct ng_ppp_link_conf	conf;		/* link configuration */
177 	hook_p			hook;		/* connection to link data */
178 	int			seq;		/* highest rec'd seq# - MSEQ */
179 	struct timeval		lastWrite;	/* time of last write */
180 	int			bytesInQueue;	/* bytes in the output queue */
181 	struct ng_ppp_link_stat	stats;		/* Link stats */
182 };
183 
184 /* Total per-node private information */
185 struct ng_ppp_private {
186 	struct ng_ppp_bund_conf	conf;			/* bundle config */
187 	struct ng_ppp_link_stat	bundleStats;		/* bundle stats */
188 	struct ng_ppp_link	links[NG_PPP_MAX_LINKS];/* per-link info */
189 	int			xseq;			/* next out MP seq # */
190 	int			mseq;			/* min links[i].seq */
191 	u_char			vjCompHooked;		/* VJ comp hooked up? */
192 	u_char			allLinksEqual;		/* all xmit the same? */
193 	u_char			timerActive;		/* frag timer active? */
194 	u_int			numActiveLinks;		/* how many links up */
195 	int			activeLinks[NG_PPP_MAX_LINKS];	/* indicies */
196 	u_int			lastLink;		/* for round robin */
197 	hook_p			hooks[HOOK_INDEX_MAX];	/* non-link hooks */
198 	CIRCLEQ_HEAD(ng_ppp_fraglist, ng_ppp_frag)	/* fragment queue */
199 				frags;
200 	int			qlen;			/* fraq queue length */
201 	struct callout_handle	fragTimer;		/* fraq queue check */
202 };
203 typedef struct ng_ppp_private *priv_p;
204 
205 /* Netgraph node methods */
206 static ng_constructor_t	ng_ppp_constructor;
207 static ng_rcvmsg_t	ng_ppp_rcvmsg;
208 static ng_shutdown_t	ng_ppp_rmnode;
209 static ng_newhook_t	ng_ppp_newhook;
210 static ng_rcvdata_t	ng_ppp_rcvdata;
211 static ng_disconnect_t	ng_ppp_disconnect;
212 
213 /* Helper functions */
214 static int	ng_ppp_input(node_p node, int bypass,
215 			int linkNum, struct mbuf *m, meta_p meta);
216 static int	ng_ppp_output(node_p node, int bypass, int proto,
217 			int linkNum, struct mbuf *m, meta_p meta);
218 static int	ng_ppp_mp_input(node_p node, int linkNum,
219 			struct mbuf *m, meta_p meta);
220 static int	ng_ppp_check_packet(node_p node);
221 static void	ng_ppp_get_packet(node_p node, struct mbuf **mp, meta_p *metap);
222 static int	ng_ppp_frag_process(node_p node);
223 static int	ng_ppp_frag_trim(node_p node);
224 static void	ng_ppp_frag_timeout(void *arg);
225 static void	ng_ppp_frag_checkstale(node_p node);
226 static void	ng_ppp_frag_reset(node_p node);
227 static int	ng_ppp_mp_output(node_p node, struct mbuf *m, meta_p meta);
228 static void	ng_ppp_mp_strategy(node_p node, int len, int *distrib);
229 static int	ng_ppp_intcmp(const void *v1, const void *v2);
230 static struct	mbuf *ng_ppp_addproto(struct mbuf *m, int proto, int compOK);
231 static struct	mbuf *ng_ppp_prepend(struct mbuf *m, const void *buf, int len);
232 static int	ng_ppp_config_valid(node_p node,
233 			const struct ng_ppp_node_conf *newConf);
234 static void	ng_ppp_update(node_p node, int newConf);
235 static void	ng_ppp_start_frag_timer(node_p node);
236 static void	ng_ppp_stop_frag_timer(node_p node);
237 
238 /* Parse type for struct ng_ppp_link_conf */
239 static const struct ng_parse_struct_info
240 	ng_ppp_link_type_info = NG_PPP_LINK_TYPE_INFO;
241 static const struct ng_parse_type ng_ppp_link_type = {
242 	&ng_parse_struct_type,
243 	&ng_ppp_link_type_info,
244 };
245 
246 /* Parse type for struct ng_ppp_bund_conf */
247 static const struct ng_parse_struct_info
248 	ng_ppp_bund_type_info = NG_PPP_BUND_TYPE_INFO;
249 static const struct ng_parse_type ng_ppp_bund_type = {
250 	&ng_parse_struct_type,
251 	&ng_ppp_bund_type_info,
252 };
253 
254 /* Parse type for struct ng_ppp_node_conf */
255 struct ng_parse_fixedarray_info ng_ppp_array_info = {
256 	&ng_ppp_link_type,
257 	NG_PPP_MAX_LINKS
258 };
259 static const struct ng_parse_type ng_ppp_link_array_type = {
260 	&ng_parse_fixedarray_type,
261 	&ng_ppp_array_info,
262 };
263 static const struct ng_parse_struct_info ng_ppp_conf_type_info
264 	= NG_PPP_CONFIG_TYPE_INFO(&ng_ppp_bund_type, &ng_ppp_link_array_type);
265 static const struct ng_parse_type ng_ppp_conf_type = {
266 	&ng_parse_struct_type,
267 	&ng_ppp_conf_type_info
268 };
269 
270 /* Parse type for struct ng_ppp_link_stat */
271 static const struct ng_parse_struct_info
272 	ng_ppp_stats_type_info = NG_PPP_STATS_TYPE_INFO;
273 static const struct ng_parse_type ng_ppp_stats_type = {
274 	&ng_parse_struct_type,
275 	&ng_ppp_stats_type_info
276 };
277 
278 /* List of commands and how to convert arguments to/from ASCII */
279 static const struct ng_cmdlist ng_ppp_cmds[] = {
280 	{
281 	  NGM_PPP_COOKIE,
282 	  NGM_PPP_SET_CONFIG,
283 	  "setconfig",
284 	  &ng_ppp_conf_type,
285 	  NULL
286 	},
287 	{
288 	  NGM_PPP_COOKIE,
289 	  NGM_PPP_GET_CONFIG,
290 	  "getconfig",
291 	  NULL,
292 	  &ng_ppp_conf_type
293 	},
294 	{
295 	  NGM_PPP_COOKIE,
296 	  NGM_PPP_GET_LINK_STATS,
297 	  "getstats",
298 	  &ng_parse_int16_type,
299 	  &ng_ppp_stats_type
300 	},
301 	{
302 	  NGM_PPP_COOKIE,
303 	  NGM_PPP_CLR_LINK_STATS,
304 	  "clrstats",
305 	  &ng_parse_int16_type,
306 	  NULL
307 	},
308 	{
309 	  NGM_PPP_COOKIE,
310 	  NGM_PPP_GETCLR_LINK_STATS,
311 	  "getclrstats",
312 	  &ng_parse_int16_type,
313 	  &ng_ppp_stats_type
314 	},
315 	{ 0 }
316 };
317 
318 /* Node type descriptor */
319 static struct ng_type ng_ppp_typestruct = {
320 	NG_VERSION,
321 	NG_PPP_NODE_TYPE,
322 	NULL,
323 	ng_ppp_constructor,
324 	ng_ppp_rcvmsg,
325 	ng_ppp_rmnode,
326 	ng_ppp_newhook,
327 	NULL,
328 	NULL,
329 	ng_ppp_rcvdata,
330 	ng_ppp_rcvdata,
331 	ng_ppp_disconnect,
332 	ng_ppp_cmds
333 };
334 NETGRAPH_INIT(ppp, &ng_ppp_typestruct);
335 
336 static int *compareLatencies;			/* hack for ng_ppp_intcmp() */
337 
338 /* Address and control field header */
339 static const u_char ng_ppp_acf[2] = { 0xff, 0x03 };
340 
341 /* Maximum time we'll let a complete incoming packet sit in the queue */
342 static const struct timeval ng_ppp_max_staleness = { 2, 0 };	/* 2 seconds */
343 
344 #define ERROUT(x)	do { error = (x); goto done; } while (0)
345 
346 /************************************************************************
347 			NETGRAPH NODE STUFF
348  ************************************************************************/
349 
350 /*
351  * Node type constructor
352  */
353 static int
354 ng_ppp_constructor(node_p *nodep)
355 {
356 	priv_p priv;
357 	int i, error;
358 
359 	/* Allocate private structure */
360 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_WAITOK);
361 	if (priv == NULL)
362 		return (ENOMEM);
363 	bzero(priv, sizeof(*priv));
364 
365 	/* Call generic node constructor */
366 	if ((error = ng_make_node_common(&ng_ppp_typestruct, nodep))) {
367 		FREE(priv, M_NETGRAPH);
368 		return (error);
369 	}
370 	(*nodep)->private = priv;
371 
372 	/* Initialize state */
373 	CIRCLEQ_INIT(&priv->frags);
374 	for (i = 0; i < NG_PPP_MAX_LINKS; i++)
375 		priv->links[i].seq = MP_NOSEQ;
376 	callout_handle_init(&priv->fragTimer);
377 
378 	/* Done */
379 	return (0);
380 }
381 
382 /*
383  * Give our OK for a hook to be added
384  */
385 static int
386 ng_ppp_newhook(node_p node, hook_p hook, const char *name)
387 {
388 	const priv_p priv = node->private;
389 	int linkNum = -1;
390 	hook_p *hookPtr = NULL;
391 	int hookIndex = -1;
392 
393 	/* Figure out which hook it is */
394 	if (strncmp(name, NG_PPP_HOOK_LINK_PREFIX,	/* a link hook? */
395 	    strlen(NG_PPP_HOOK_LINK_PREFIX)) == 0) {
396 		const char *cp;
397 		char *eptr;
398 
399 		cp = name + strlen(NG_PPP_HOOK_LINK_PREFIX);
400 		if (!isdigit(*cp) || (cp[0] == '0' && cp[1] != '\0'))
401 			return (EINVAL);
402 		linkNum = (int)strtoul(cp, &eptr, 10);
403 		if (*eptr != '\0' || linkNum < 0 || linkNum >= NG_PPP_MAX_LINKS)
404 			return (EINVAL);
405 		hookPtr = &priv->links[linkNum].hook;
406 		hookIndex = ~linkNum;
407 	} else {				/* must be a non-link hook */
408 		int i;
409 
410 		for (i = 0; ng_ppp_hook_names[i] != NULL; i++) {
411 			if (strcmp(name, ng_ppp_hook_names[i]) == 0) {
412 				hookPtr = &priv->hooks[i];
413 				hookIndex = i;
414 				break;
415 			}
416 		}
417 		if (ng_ppp_hook_names[i] == NULL)
418 			return (EINVAL);	/* no such hook */
419 	}
420 
421 	/* See if hook is already connected */
422 	if (*hookPtr != NULL)
423 		return (EISCONN);
424 
425 	/* Disallow more than one link unless multilink is enabled */
426 	if (linkNum != -1 && priv->links[linkNum].conf.enableLink
427 	    && !priv->conf.enableMultilink && priv->numActiveLinks >= 1)
428 		return (ENODEV);
429 
430 	/* OK */
431 	*hookPtr = hook;
432 	HOOK_INDEX(hook) = hookIndex;
433 	ng_ppp_update(node, 0);
434 	return (0);
435 }
436 
437 /*
438  * Receive a control message
439  */
440 static int
441 ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
442 	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
443 {
444 	const priv_p priv = node->private;
445 	struct ng_mesg *resp = NULL;
446 	int error = 0;
447 
448 	switch (msg->header.typecookie) {
449 	case NGM_PPP_COOKIE:
450 		switch (msg->header.cmd) {
451 		case NGM_PPP_SET_CONFIG:
452 		    {
453 			struct ng_ppp_node_conf *const conf =
454 			    (struct ng_ppp_node_conf *)msg->data;
455 			int i;
456 
457 			/* Check for invalid or illegal config */
458 			if (msg->header.arglen != sizeof(*conf))
459 				ERROUT(EINVAL);
460 			if (!ng_ppp_config_valid(node, conf))
461 				ERROUT(EINVAL);
462 
463 			/* Copy config */
464 			priv->conf = conf->bund;
465 			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
466 				priv->links[i].conf = conf->links[i];
467 			ng_ppp_update(node, 1);
468 			break;
469 		    }
470 		case NGM_PPP_GET_CONFIG:
471 		    {
472 			struct ng_ppp_node_conf *conf;
473 			int i;
474 
475 			NG_MKRESPONSE(resp, msg, sizeof(*conf), M_NOWAIT);
476 			if (resp == NULL)
477 				ERROUT(ENOMEM);
478 			conf = (struct ng_ppp_node_conf *)resp->data;
479 			conf->bund = priv->conf;
480 			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
481 				conf->links[i] = priv->links[i].conf;
482 			break;
483 		    }
484 		case NGM_PPP_GET_LINK_STATS:
485 		case NGM_PPP_CLR_LINK_STATS:
486 		case NGM_PPP_GETCLR_LINK_STATS:
487 		    {
488 			struct ng_ppp_link_stat *stats;
489 			u_int16_t linkNum;
490 
491 			if (msg->header.arglen != sizeof(u_int16_t))
492 				ERROUT(EINVAL);
493 			linkNum = *((u_int16_t *) msg->data);
494 			if (linkNum >= NG_PPP_MAX_LINKS
495 			    && linkNum != NG_PPP_BUNDLE_LINKNUM)
496 				ERROUT(EINVAL);
497 			stats = (linkNum == NG_PPP_BUNDLE_LINKNUM) ?
498 			    &priv->bundleStats : &priv->links[linkNum].stats;
499 			if (msg->header.cmd != NGM_PPP_CLR_LINK_STATS) {
500 				NG_MKRESPONSE(resp, msg,
501 				    sizeof(struct ng_ppp_link_stat), M_NOWAIT);
502 				if (resp == NULL)
503 					ERROUT(ENOMEM);
504 				bcopy(stats, resp->data, sizeof(*stats));
505 			}
506 			if (msg->header.cmd != NGM_PPP_GET_LINK_STATS)
507 				bzero(stats, sizeof(*stats));
508 			break;
509 		    }
510 		default:
511 			error = EINVAL;
512 			break;
513 		}
514 		break;
515 	case NGM_VJC_COOKIE:
516 	    {
517 		char path[NG_PATHLEN + 1];
518 		node_p origNode;
519 
520 		if ((error = ng_path2node(node,
521 		    raddr, &origNode, NULL, NULL)) != 0)
522 			ERROUT(error);
523 		snprintf(path, sizeof(path), "[%lx]:%s",
524 		    (long)node, NG_PPP_HOOK_VJC_IP);
525 		return ng_send_msg(origNode, msg, path, rptr);
526 	    }
527 	default:
528 		error = EINVAL;
529 		break;
530 	}
531 	if (rptr)
532 		*rptr = resp;
533 	else if (resp)
534 		FREE(resp, M_NETGRAPH);
535 
536 done:
537 	FREE(msg, M_NETGRAPH);
538 	return (error);
539 }
540 
541 /*
542  * Receive data on a hook
543  */
544 static int
545 ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
546 		struct mbuf **ret_m, meta_p *ret_meta)
547 {
548 	const node_p node = hook->node;
549 	const priv_p priv = node->private;
550 	const int index = HOOK_INDEX(hook);
551 	u_int16_t linkNum = NG_PPP_BUNDLE_LINKNUM;
552 	hook_p outHook = NULL;
553 	int proto = 0, error;
554 
555 	/* Did it come from a link hook? */
556 	if (index < 0) {
557 		struct ng_ppp_link *link;
558 
559 		/* Convert index into a link number */
560 		linkNum = (u_int16_t)~index;
561 		KASSERT(linkNum < NG_PPP_MAX_LINKS,
562 		    ("%s: bogus index 0x%x", __FUNCTION__, index));
563 		link = &priv->links[linkNum];
564 
565 		/* Stats */
566 		link->stats.recvFrames++;
567 		link->stats.recvOctets += m->m_pkthdr.len;
568 
569 		/* Strip address and control fields, if present */
570 		if (m->m_pkthdr.len >= 2) {
571 			if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL) {
572 				NG_FREE_DATA(m, meta);
573 				return (ENOBUFS);
574 			}
575 			if (bcmp(mtod(m, u_char *), &ng_ppp_acf, 2) == 0)
576 				m_adj(m, 2);
577 		}
578 
579 		/* Dispatch incoming frame (if not enabled, to bypass) */
580 		return ng_ppp_input(node,
581 		    !link->conf.enableLink, linkNum, m, meta);
582 	}
583 
584 	/* Get protocol & check if data allowed from this hook */
585 	switch (index) {
586 
587 	/* Outgoing data */
588 	case HOOK_INDEX_ATALK:
589 		if (!priv->conf.enableAtalk) {
590 			NG_FREE_DATA(m, meta);
591 			return (ENXIO);
592 		}
593 		proto = PROT_APPLETALK;
594 		break;
595 	case HOOK_INDEX_IPX:
596 		if (!priv->conf.enableIPX) {
597 			NG_FREE_DATA(m, meta);
598 			return (ENXIO);
599 		}
600 		proto = PROT_IPX;
601 		break;
602 	case HOOK_INDEX_IPV6:
603 		if (!priv->conf.enableIPv6) {
604 			NG_FREE_DATA(m, meta);
605 			return (ENXIO);
606 		}
607 		proto = PROT_IPV6;
608 		break;
609 	case HOOK_INDEX_INET:
610 	case HOOK_INDEX_VJC_VJIP:
611 		if (!priv->conf.enableIP) {
612 			NG_FREE_DATA(m, meta);
613 			return (ENXIO);
614 		}
615 		proto = PROT_IP;
616 		break;
617 	case HOOK_INDEX_VJC_COMP:
618 		if (!priv->conf.enableVJCompression) {
619 			NG_FREE_DATA(m, meta);
620 			return (ENXIO);
621 		}
622 		proto = PROT_VJCOMP;
623 		break;
624 	case HOOK_INDEX_VJC_UNCOMP:
625 		if (!priv->conf.enableVJCompression) {
626 			NG_FREE_DATA(m, meta);
627 			return (ENXIO);
628 		}
629 		proto = PROT_VJUNCOMP;
630 		break;
631 	case HOOK_INDEX_COMPRESS:
632 		if (!priv->conf.enableCompression) {
633 			NG_FREE_DATA(m, meta);
634 			return (ENXIO);
635 		}
636 		proto = PROT_COMPD;
637 		break;
638 	case HOOK_INDEX_ENCRYPT:
639 		if (!priv->conf.enableEncryption) {
640 			NG_FREE_DATA(m, meta);
641 			return (ENXIO);
642 		}
643 		proto = PROT_CRYPTD;
644 		break;
645 	case HOOK_INDEX_BYPASS:
646 		if (m->m_pkthdr.len < 4) {
647 			NG_FREE_DATA(m, meta);
648 			return (EINVAL);
649 		}
650 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
651 			NG_FREE_META(meta);
652 			return (ENOBUFS);
653 		}
654 		linkNum = ntohs(mtod(m, u_int16_t *)[0]);
655 		proto = ntohs(mtod(m, u_int16_t *)[1]);
656 		m_adj(m, 4);
657 		if (linkNum >= NG_PPP_MAX_LINKS
658 		    && linkNum != NG_PPP_BUNDLE_LINKNUM) {
659 			NG_FREE_DATA(m, meta);
660 			return (EINVAL);
661 		}
662 		break;
663 
664 	/* Incoming data */
665 	case HOOK_INDEX_VJC_IP:
666 		if (!priv->conf.enableIP || !priv->conf.enableVJDecompression) {
667 			NG_FREE_DATA(m, meta);
668 			return (ENXIO);
669 		}
670 		break;
671 	case HOOK_INDEX_DECOMPRESS:
672 		if (!priv->conf.enableDecompression) {
673 			NG_FREE_DATA(m, meta);
674 			return (ENXIO);
675 		}
676 		break;
677 	case HOOK_INDEX_DECRYPT:
678 		if (!priv->conf.enableDecryption) {
679 			NG_FREE_DATA(m, meta);
680 			return (ENXIO);
681 		}
682 		break;
683 	default:
684 		panic("%s: bogus index 0x%x", __FUNCTION__, index);
685 	}
686 
687 	/* Now figure out what to do with the frame */
688 	switch (index) {
689 
690 	/* Outgoing data */
691 	case HOOK_INDEX_INET:
692 		if (priv->conf.enableVJCompression && priv->vjCompHooked) {
693 			outHook = priv->hooks[HOOK_INDEX_VJC_IP];
694 			break;
695 		}
696 		/* FALLTHROUGH */
697 	case HOOK_INDEX_ATALK:
698 	case HOOK_INDEX_IPV6:
699 	case HOOK_INDEX_IPX:
700 	case HOOK_INDEX_VJC_COMP:
701 	case HOOK_INDEX_VJC_UNCOMP:
702 	case HOOK_INDEX_VJC_VJIP:
703 		if (priv->conf.enableCompression
704 		    && priv->hooks[HOOK_INDEX_COMPRESS] != NULL) {
705 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
706 				NG_FREE_META(meta);
707 				return (ENOBUFS);
708 			}
709 			outHook = priv->hooks[HOOK_INDEX_COMPRESS];
710 			break;
711 		}
712 		/* FALLTHROUGH */
713 	case HOOK_INDEX_COMPRESS:
714 		if (priv->conf.enableEncryption
715 		    && priv->hooks[HOOK_INDEX_ENCRYPT] != NULL) {
716 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
717 				NG_FREE_META(meta);
718 				return (ENOBUFS);
719 			}
720 			outHook = priv->hooks[HOOK_INDEX_ENCRYPT];
721 			break;
722 		}
723 		/* FALLTHROUGH */
724 	case HOOK_INDEX_ENCRYPT:
725 		return ng_ppp_output(node, 0,
726 		    proto, NG_PPP_BUNDLE_LINKNUM, m, meta);
727 
728 	case HOOK_INDEX_BYPASS:
729 		return ng_ppp_output(node, 1, proto, linkNum, m, meta);
730 
731 	/* Incoming data */
732 	case HOOK_INDEX_DECRYPT:
733 	case HOOK_INDEX_DECOMPRESS:
734 		return ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
735 
736 	case HOOK_INDEX_VJC_IP:
737 		outHook = priv->hooks[HOOK_INDEX_INET];
738 		break;
739 	}
740 
741 	/* Send packet out hook */
742 	NG_SEND_DATA_RET(error, outHook, m, meta);
743 	if (m != NULL || meta != NULL)
744 		return ng_ppp_rcvdata(outHook, m, meta, NULL, NULL);
745 	return (error);
746 }
747 
748 /*
749  * Destroy node
750  */
751 static int
752 ng_ppp_rmnode(node_p node)
753 {
754 	const priv_p priv = node->private;
755 
756 	/* Stop fragment queue timer */
757 	ng_ppp_stop_frag_timer(node);
758 
759 	/* Take down netgraph node */
760 	node->flags |= NG_INVALID;
761 	ng_cutlinks(node);
762 	ng_unname(node);
763 	ng_ppp_frag_reset(node);
764 	bzero(priv, sizeof(*priv));
765 	FREE(priv, M_NETGRAPH);
766 	node->private = NULL;
767 	ng_unref(node);		/* let the node escape */
768 	return (0);
769 }
770 
771 /*
772  * Hook disconnection
773  */
774 static int
775 ng_ppp_disconnect(hook_p hook)
776 {
777 	const node_p node = hook->node;
778 	const priv_p priv = node->private;
779 	const int index = HOOK_INDEX(hook);
780 
781 	/* Zero out hook pointer */
782 	if (index < 0)
783 		priv->links[~index].hook = NULL;
784 	else
785 		priv->hooks[index] = NULL;
786 
787 	/* Update derived info (or go away if no hooks left) */
788 	if (node->numhooks > 0)
789 		ng_ppp_update(node, 0);
790 	else
791 		ng_rmnode(node);
792 	return (0);
793 }
794 
795 /************************************************************************
796 			HELPER STUFF
797  ************************************************************************/
798 
799 /*
800  * Handle an incoming frame.  Extract the PPP protocol number
801  * and dispatch accordingly.
802  */
803 static int
804 ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
805 {
806 	const priv_p priv = node->private;
807 	hook_p outHook = NULL;
808 	int proto, error;
809 
810 	/* Extract protocol number */
811 	for (proto = 0; !PROT_VALID(proto) && m->m_pkthdr.len > 0; ) {
812 		if (m->m_len < 1 && (m = m_pullup(m, 1)) == NULL) {
813 			NG_FREE_META(meta);
814 			return (ENOBUFS);
815 		}
816 		proto = (proto << 8) + *mtod(m, u_char *);
817 		m_adj(m, 1);
818 	}
819 	if (!PROT_VALID(proto)) {
820 		if (linkNum == NG_PPP_BUNDLE_LINKNUM)
821 			priv->bundleStats.badProtos++;
822 		else
823 			priv->links[linkNum].stats.badProtos++;
824 		NG_FREE_DATA(m, meta);
825 		return (EINVAL);
826 	}
827 
828 	/* Bypass frame? */
829 	if (bypass)
830 		goto bypass;
831 
832 	/* Check protocol */
833 	switch (proto) {
834 	case PROT_COMPD:
835 		if (priv->conf.enableDecompression)
836 			outHook = priv->hooks[HOOK_INDEX_DECOMPRESS];
837 		break;
838 	case PROT_CRYPTD:
839 		if (priv->conf.enableDecryption)
840 			outHook = priv->hooks[HOOK_INDEX_DECRYPT];
841 		break;
842 	case PROT_VJCOMP:
843 		if (priv->conf.enableVJDecompression && priv->vjCompHooked)
844 			outHook = priv->hooks[HOOK_INDEX_VJC_COMP];
845 		break;
846 	case PROT_VJUNCOMP:
847 		if (priv->conf.enableVJDecompression && priv->vjCompHooked)
848 			outHook = priv->hooks[HOOK_INDEX_VJC_UNCOMP];
849 		break;
850 	case PROT_MP:
851 		if (priv->conf.enableMultilink
852 		    && linkNum != NG_PPP_BUNDLE_LINKNUM)
853 			return ng_ppp_mp_input(node, linkNum, m, meta);
854 		break;
855 	case PROT_APPLETALK:
856 		if (priv->conf.enableAtalk)
857 			outHook = priv->hooks[HOOK_INDEX_ATALK];
858 		break;
859 	case PROT_IPX:
860 		if (priv->conf.enableIPX)
861 			outHook = priv->hooks[HOOK_INDEX_IPX];
862 		break;
863 	case PROT_IP:
864 		if (priv->conf.enableIP)
865 			outHook = priv->hooks[HOOK_INDEX_INET];
866 		break;
867 	case PROT_IPV6:
868 		if (priv->conf.enableIPv6)
869 			outHook = priv->hooks[HOOK_INDEX_IPV6];
870 		break;
871 	}
872 
873 bypass:
874 	/* For unknown/inactive protocols, forward out the bypass hook */
875 	if (outHook == NULL) {
876 		u_int16_t hdr[2];
877 
878 		hdr[0] = htons(linkNum);
879 		hdr[1] = htons((u_int16_t)proto);
880 		if ((m = ng_ppp_prepend(m, &hdr, 4)) == NULL) {
881 			NG_FREE_META(meta);
882 			return (ENOBUFS);
883 		}
884 		outHook = priv->hooks[HOOK_INDEX_BYPASS];
885 	}
886 
887 	/* Forward frame */
888 	NG_SEND_DATA(error, outHook, m, meta);
889 	return (error);
890 }
891 
892 /*
893  * Deliver a frame out a link, either a real one or NG_PPP_BUNDLE_LINKNUM
894  * If the link is not enabled then ENXIO is returned, unless "bypass" is != 0.
895  */
896 static int
897 ng_ppp_output(node_p node, int bypass,
898 	int proto, int linkNum, struct mbuf *m, meta_p meta)
899 {
900 	const priv_p priv = node->private;
901 	struct ng_ppp_link *link;
902 	int len, error;
903 
904 	/* If not doing MP, map bundle virtual link to (the only) link */
905 	if (linkNum == NG_PPP_BUNDLE_LINKNUM && !priv->conf.enableMultilink)
906 		linkNum = priv->activeLinks[0];
907 
908 	/* Get link pointer (optimization) */
909 	link = (linkNum != NG_PPP_BUNDLE_LINKNUM) ?
910 	    &priv->links[linkNum] : NULL;
911 
912 	/* Check link status (if real) */
913 	if (linkNum != NG_PPP_BUNDLE_LINKNUM) {
914 		if (!bypass && !link->conf.enableLink) {
915 			NG_FREE_DATA(m, meta);
916 			return (ENXIO);
917 		}
918 		if (link->hook == NULL) {
919 			NG_FREE_DATA(m, meta);
920 			return (ENETDOWN);
921 		}
922 	}
923 
924 	/* Prepend protocol number, possibly compressed */
925 	if ((m = ng_ppp_addproto(m, proto,
926 	    linkNum == NG_PPP_BUNDLE_LINKNUM
927 	      || link->conf.enableProtoComp)) == NULL) {
928 		NG_FREE_META(meta);
929 		return (ENOBUFS);
930 	}
931 
932 	/* Special handling for the MP virtual link */
933 	if (linkNum == NG_PPP_BUNDLE_LINKNUM)
934 		return ng_ppp_mp_output(node, m, meta);
935 
936 	/* Prepend address and control field (unless compressed) */
937 	if (proto == PROT_LCP || !link->conf.enableACFComp) {
938 		if ((m = ng_ppp_prepend(m, &ng_ppp_acf, 2)) == NULL) {
939 			NG_FREE_META(meta);
940 			return (ENOBUFS);
941 		}
942 	}
943 
944 	/* Deliver frame */
945 	len = m->m_pkthdr.len;
946 	NG_SEND_DATA(error, link->hook, m, meta);
947 
948 	/* Update stats and 'bytes in queue' counter */
949 	if (error == 0) {
950 		link->stats.xmitFrames++;
951 		link->stats.xmitOctets += len;
952 		link->bytesInQueue += len;
953 		getmicrouptime(&link->lastWrite);
954 	}
955 	return error;
956 }
957 
958 /*
959  * Handle an incoming multi-link fragment
960  *
961  * The fragment reassembly algorithm is somewhat complex. This is mainly
962  * because we are required not to reorder the reconstructed packets, yet
963  * fragments are only guaranteed to arrive in order on a per-link basis.
964  * In other words, when we have a complete packet ready, but the previous
965  * packet is still incomplete, we have to decide between delivering the
966  * complete packet and throwing away the incomplete one, or waiting to
967  * see if the remainder of the incomplete one arrives, at which time we
968  * can deliver both packets, in order.
969  *
970  * This problem is exacerbated by "sequence number slew", which is when
971  * the sequence numbers coming in from different links are far apart from
972  * each other. In particular, certain unnamed equipment (*cough* Ascend)
973  * has been seen to generate sequence number slew of up to 10 on an ISDN
974  * 2B-channel MP link. There is nothing invalid about sequence number slew
975  * but it makes the reasssembly process have to work harder.
976  *
977  * However, the peer is required to transmit fragments in order on each
978  * link. That means if we define MSEQ as the minimum over all links of
979  * the highest sequence number received on that link, then we can always
980  * give up any hope of receiving a fragment with sequence number < MSEQ in
981  * the future (all of this using 'wraparound' sequence number space).
982  * Therefore we can always immediately throw away incomplete packets
983  * missing fragments with sequence numbers < MSEQ.
984  *
985  * Here is an overview of our algorithm:
986  *
987  *    o Received fragments are inserted into a queue, for which we
988  *	maintain these invariants between calls to this function:
989  *
990  *	- Fragments are ordered in the queue by sequence number
991  *	- If a complete packet is at the head of the queue, then
992  *	  the first fragment in the packet has seq# > MSEQ + 1
993  *	  (otherwise, we could deliver it immediately)
994  *	- If any fragments have seq# < MSEQ, then they are necessarily
995  *	  part of a packet whose missing seq#'s are all > MSEQ (otherwise,
996  *	  we can throw them away because they'll never be completed)
997  *	- The queue contains at most MP_MAX_QUEUE_LEN fragments
998  *
999  *    o We have a periodic timer that checks the queue for the first
1000  *	complete packet that has been sitting in the queue "too long".
1001  *	When one is detected, all previous (incomplete) fragments are
1002  *	discarded, their missing fragments are declared lost and MSEQ
1003  *	is increased.
1004  *
1005  *    o If we recieve a fragment with seq# < MSEQ, we throw it away
1006  *	because we've already delcared it lost.
1007  *
1008  * This assumes linkNum != NG_PPP_BUNDLE_LINKNUM.
1009  */
1010 static int
1011 ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
1012 {
1013 	const priv_p priv = node->private;
1014 	struct ng_ppp_link *const link = &priv->links[linkNum];
1015 	struct ng_ppp_frag frag0, *frag = &frag0;
1016 	struct ng_ppp_frag *qent;
1017 	int i, diff, inserted;
1018 
1019 	/* Stats */
1020 	priv->bundleStats.recvFrames++;
1021 	priv->bundleStats.recvOctets += m->m_pkthdr.len;
1022 
1023 	/* Extract fragment information from MP header */
1024 	if (priv->conf.recvShortSeq) {
1025 		u_int16_t shdr;
1026 
1027 		if (m->m_pkthdr.len < 2) {
1028 			link->stats.runts++;
1029 			NG_FREE_DATA(m, meta);
1030 			return (EINVAL);
1031 		}
1032 		if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL) {
1033 			NG_FREE_META(meta);
1034 			return (ENOBUFS);
1035 		}
1036 		shdr = ntohs(*mtod(m, u_int16_t *));
1037 		frag->seq = shdr & MP_SHORT_SEQ_MASK;
1038 		frag->first = (shdr & MP_SHORT_FIRST_FLAG) != 0;
1039 		frag->last = (shdr & MP_SHORT_LAST_FLAG) != 0;
1040 		diff = MP_SHORT_SEQ_DIFF(frag->seq, priv->mseq);
1041 		m_adj(m, 2);
1042 	} else {
1043 		u_int32_t lhdr;
1044 
1045 		if (m->m_pkthdr.len < 4) {
1046 			link->stats.runts++;
1047 			NG_FREE_DATA(m, meta);
1048 			return (EINVAL);
1049 		}
1050 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
1051 			NG_FREE_META(meta);
1052 			return (ENOBUFS);
1053 		}
1054 		lhdr = ntohl(*mtod(m, u_int32_t *));
1055 		frag->seq = lhdr & MP_LONG_SEQ_MASK;
1056 		frag->first = (lhdr & MP_LONG_FIRST_FLAG) != 0;
1057 		frag->last = (lhdr & MP_LONG_LAST_FLAG) != 0;
1058 		diff = MP_LONG_SEQ_DIFF(frag->seq, priv->mseq);
1059 		m_adj(m, 4);
1060 	}
1061 	frag->data = m;
1062 	frag->meta = meta;
1063 	getmicrouptime(&frag->timestamp);
1064 
1065 	/* If sequence number is < MSEQ, we've already declared this
1066 	   fragment as lost, so we have no choice now but to drop it */
1067 	if (diff < 0) {
1068 		link->stats.dropFragments++;
1069 		NG_FREE_DATA(m, meta);
1070 		return (0);
1071 	}
1072 
1073 	/* Update highest received sequence number on this link and MSEQ */
1074 	priv->mseq = link->seq = frag->seq;
1075 	for (i = 0; i < priv->numActiveLinks; i++) {
1076 		struct ng_ppp_link *const alink =
1077 		    &priv->links[priv->activeLinks[i]];
1078 
1079 		if (MP_SEQ_DIFF(priv, alink->seq, priv->mseq) < 0)
1080 			priv->mseq = alink->seq;
1081 	}
1082 
1083 	/* Allocate a new frag struct for the queue */
1084 	MALLOC(frag, struct ng_ppp_frag *, sizeof(*frag), M_NETGRAPH, M_NOWAIT);
1085 	if (frag == NULL) {
1086 		NG_FREE_DATA(m, meta);
1087 		ng_ppp_frag_process(node);
1088 		return (ENOMEM);
1089 	}
1090 	*frag = frag0;
1091 
1092 	/* Add fragment to queue, which is sorted by sequence number */
1093 	inserted = 0;
1094 	CIRCLEQ_FOREACH_REVERSE(qent, &priv->frags, f_qent) {
1095 		diff = MP_SEQ_DIFF(priv, frag->seq, qent->seq);
1096 		if (diff > 0) {
1097 			CIRCLEQ_INSERT_AFTER(&priv->frags, qent, frag, f_qent);
1098 			inserted = 1;
1099 			break;
1100 		} else if (diff == 0) {	     /* should never happen! */
1101 			link->stats.dupFragments++;
1102 			NG_FREE_DATA(frag->data, frag->meta);
1103 			FREE(frag, M_NETGRAPH);
1104 			return (EINVAL);
1105 		}
1106 	}
1107 	if (!inserted)
1108 		CIRCLEQ_INSERT_HEAD(&priv->frags, frag, f_qent);
1109 	priv->qlen++;
1110 
1111 	/* Process the queue */
1112 	return ng_ppp_frag_process(node);
1113 }
1114 
1115 /*
1116  * Examine our list of fragments, and determine if there is a
1117  * complete and deliverable packet at the head of the list.
1118  * Return 1 if so, zero otherwise.
1119  */
1120 static int
1121 ng_ppp_check_packet(node_p node)
1122 {
1123 	const priv_p priv = node->private;
1124 	struct ng_ppp_frag *qent, *qnext;
1125 
1126 	/* Check for empty queue */
1127 	if (CIRCLEQ_EMPTY(&priv->frags))
1128 		return (0);
1129 
1130 	/* Check first fragment is the start of a deliverable packet */
1131 	qent = CIRCLEQ_FIRST(&priv->frags);
1132 	if (!qent->first || MP_SEQ_DIFF(priv, qent->seq, priv->mseq) > 1)
1133 		return (0);
1134 
1135 	/* Check that all the fragments are there */
1136 	while (!qent->last) {
1137 		qnext = CIRCLEQ_NEXT(qent, f_qent);
1138 		if (qnext == (void *)&priv->frags)	/* end of queue */
1139 			return (0);
1140 		if (qnext->seq != MP_NEXT_SEQ(priv, qent->seq))
1141 			return (0);
1142 		qent = qnext;
1143 	}
1144 
1145 	/* Got one */
1146 	return (1);
1147 }
1148 
1149 /*
1150  * Pull a completed packet off the head of the incoming fragment queue.
1151  * This assumes there is a completed packet there to pull off.
1152  */
1153 static void
1154 ng_ppp_get_packet(node_p node, struct mbuf **mp, meta_p *metap)
1155 {
1156 	const priv_p priv = node->private;
1157 	struct ng_ppp_frag *qent, *qnext;
1158 	struct mbuf *m = NULL, *tail;
1159 
1160 	qent = CIRCLEQ_FIRST(&priv->frags);
1161 	KASSERT(!CIRCLEQ_EMPTY(&priv->frags) && qent->first,
1162 	    ("%s: no packet", __FUNCTION__));
1163 	for (tail = NULL; qent != NULL; qent = qnext) {
1164 		qnext = CIRCLEQ_NEXT(qent, f_qent);
1165 		KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
1166 		    ("%s: empty q", __FUNCTION__));
1167 		CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
1168 		if (tail == NULL) {
1169 			tail = m = qent->data;
1170 			*metap = qent->meta;	/* inherit first frag's meta */
1171 		} else {
1172 			m->m_pkthdr.len += qent->data->m_pkthdr.len;
1173 			tail->m_next = qent->data;
1174 			NG_FREE_META(qent->meta); /* drop other frags' metas */
1175 		}
1176 		while (tail->m_next != NULL)
1177 			tail = tail->m_next;
1178 		if (qent->last)
1179 			qnext = NULL;
1180 		FREE(qent, M_NETGRAPH);
1181 		priv->qlen--;
1182 	}
1183 	*mp = m;
1184 }
1185 
1186 /*
1187  * Trim fragments from the queue whose packets can never be completed.
1188  * This assumes a complete packet is NOT at the beginning of the queue.
1189  * Returns 1 if fragments were removed, zero otherwise.
1190  */
1191 static int
1192 ng_ppp_frag_trim(node_p node)
1193 {
1194 	const priv_p priv = node->private;
1195 	struct ng_ppp_frag *qent, *qnext = NULL;
1196 	int removed = 0;
1197 
1198 	/* Scan for "dead" fragments and remove them */
1199 	while (1) {
1200 		int dead = 0;
1201 
1202 		/* If queue is empty, we're done */
1203 		if (CIRCLEQ_EMPTY(&priv->frags))
1204 			break;
1205 
1206 		/* Determine whether first fragment can ever be completed */
1207 		CIRCLEQ_FOREACH(qent, &priv->frags, f_qent) {
1208 			if (MP_SEQ_DIFF(priv, qent->seq, priv->mseq) >= 0)
1209 				break;
1210 			qnext = CIRCLEQ_NEXT(qent, f_qent);
1211 			KASSERT(qnext != (void*)&priv->frags,
1212 			    ("%s: last frag < MSEQ?", __FUNCTION__));
1213 			if (qnext->seq != MP_NEXT_SEQ(priv, qent->seq)
1214 			    || qent->last || qnext->first) {
1215 				dead = 1;
1216 				break;
1217 			}
1218 		}
1219 		if (!dead)
1220 			break;
1221 
1222 		/* Remove fragment and all others in the same packet */
1223 		while ((qent = CIRCLEQ_FIRST(&priv->frags)) != qnext) {
1224 			KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
1225 			    ("%s: empty q", __FUNCTION__));
1226 			priv->bundleStats.dropFragments++;
1227 			CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
1228 			NG_FREE_DATA(qent->data, qent->meta);
1229 			FREE(qent, M_NETGRAPH);
1230 			priv->qlen--;
1231 			removed = 1;
1232 		}
1233 	}
1234 	return (removed);
1235 }
1236 
1237 /*
1238  * Run the queue, restoring the queue invariants
1239  */
1240 static int
1241 ng_ppp_frag_process(node_p node)
1242 {
1243 	const priv_p priv = node->private;
1244 	struct mbuf *m;
1245 	meta_p meta;
1246 
1247 	/* Deliver any deliverable packets */
1248 	while (ng_ppp_check_packet(node)) {
1249 		ng_ppp_get_packet(node, &m, &meta);
1250 		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
1251 	}
1252 
1253 	/* Delete dead fragments and try again */
1254 	if (ng_ppp_frag_trim(node)) {
1255 		while (ng_ppp_check_packet(node)) {
1256 			ng_ppp_get_packet(node, &m, &meta);
1257 			ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
1258 		}
1259 	}
1260 
1261 	/* Check for stale fragments while we're here */
1262 	ng_ppp_frag_checkstale(node);
1263 
1264 	/* Check queue length */
1265 	if (priv->qlen > MP_MAX_QUEUE_LEN) {
1266 		struct ng_ppp_frag *qent;
1267 		int i;
1268 
1269 		/* Get oldest fragment */
1270 		KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
1271 		    ("%s: empty q", __FUNCTION__));
1272 		qent = CIRCLEQ_FIRST(&priv->frags);
1273 
1274 		/* Bump MSEQ if necessary */
1275 		if (MP_SEQ_DIFF(priv, priv->mseq, qent->seq) < 0) {
1276 			priv->mseq = qent->seq;
1277 			for (i = 0; i < priv->numActiveLinks; i++) {
1278 				struct ng_ppp_link *const alink =
1279 				    &priv->links[priv->activeLinks[i]];
1280 
1281 				if (MP_SEQ_DIFF(priv,
1282 				    alink->seq, priv->mseq) < 0)
1283 					alink->seq = priv->mseq;
1284 			}
1285 		}
1286 
1287 		/* Drop it */
1288 		priv->bundleStats.dropFragments++;
1289 		CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
1290 		NG_FREE_DATA(qent->data, qent->meta);
1291 		FREE(qent, M_NETGRAPH);
1292 		priv->qlen--;
1293 
1294 		/* Process queue again */
1295 		return ng_ppp_frag_process(node);
1296 	}
1297 
1298 	/* Done */
1299 	return (0);
1300 }
1301 
1302 /*
1303  * Check for 'stale' completed packets that need to be delivered
1304  *
1305  * If a link goes down or has a temporary failure, MSEQ can get
1306  * "stuck", because no new incoming fragments appear on that link.
1307  * This can cause completed packets to never get delivered if
1308  * their sequence numbers are all > MSEQ + 1.
1309  *
1310  * This routine checks how long all of the completed packets have
1311  * been sitting in the queue, and if too long, removes fragments
1312  * from the queue and increments MSEQ to allow them to be delivered.
1313  */
1314 static void
1315 ng_ppp_frag_checkstale(node_p node)
1316 {
1317 	const priv_p priv = node->private;
1318 	struct ng_ppp_frag *qent, *beg, *end;
1319 	struct timeval now, age;
1320 	struct mbuf *m;
1321 	meta_p meta;
1322 	int i, seq;
1323 
1324 	now.tv_sec = 0;			/* uninitialized state */
1325 	while (1) {
1326 
1327 		/* If queue is empty, we're done */
1328 		if (CIRCLEQ_EMPTY(&priv->frags))
1329 			break;
1330 
1331 		/* Find the first complete packet in the queue */
1332 		beg = end = NULL;
1333 		seq = CIRCLEQ_FIRST(&priv->frags)->seq;
1334 		CIRCLEQ_FOREACH(qent, &priv->frags, f_qent) {
1335 			if (qent->first)
1336 				beg = qent;
1337 			else if (qent->seq != seq)
1338 				beg = NULL;
1339 			if (beg != NULL && qent->last) {
1340 				end = qent;
1341 				break;
1342 			}
1343 			seq = MP_NEXT_SEQ(priv, seq);
1344 		}
1345 
1346 		/* If none found, exit */
1347 		if (end == NULL)
1348 			break;
1349 
1350 		/* Get current time (we assume we've been up for >= 1 second) */
1351 		if (now.tv_sec == 0)
1352 			getmicrouptime(&now);
1353 
1354 		/* Check if packet has been queued too long */
1355 		age = now;
1356 		timevalsub(&age, &beg->timestamp);
1357 		if (timevalcmp(&age, &ng_ppp_max_staleness, < ))
1358 			break;
1359 
1360 		/* Throw away junk fragments in front of the completed packet */
1361 		while ((qent = CIRCLEQ_FIRST(&priv->frags)) != beg) {
1362 			KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
1363 			    ("%s: empty q", __FUNCTION__));
1364 			priv->bundleStats.dropFragments++;
1365 			CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
1366 			NG_FREE_DATA(qent->data, qent->meta);
1367 			FREE(qent, M_NETGRAPH);
1368 			priv->qlen--;
1369 		}
1370 
1371 		/* Extract completed packet */
1372 		ng_ppp_get_packet(node, &m, &meta);
1373 
1374 		/* Bump MSEQ if necessary */
1375 		if (MP_SEQ_DIFF(priv, priv->mseq, end->seq) < 0) {
1376 			priv->mseq = end->seq;
1377 			for (i = 0; i < priv->numActiveLinks; i++) {
1378 				struct ng_ppp_link *const alink =
1379 				    &priv->links[priv->activeLinks[i]];
1380 
1381 				if (MP_SEQ_DIFF(priv,
1382 				    alink->seq, priv->mseq) < 0)
1383 					alink->seq = priv->mseq;
1384 			}
1385 		}
1386 
1387 		/* Deliver packet */
1388 		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
1389 	}
1390 }
1391 
1392 /*
1393  * Periodically call ng_ppp_frag_checkstale()
1394  */
1395 static void
1396 ng_ppp_frag_timeout(void *arg)
1397 {
1398 	const node_p node = arg;
1399 	const priv_p priv = node->private;
1400 	int s = splnet();
1401 
1402 	/* Handle the race where shutdown happens just before splnet() above */
1403 	if ((node->flags & NG_INVALID) != 0) {
1404 		ng_unref(node);
1405 		splx(s);
1406 		return;
1407 	}
1408 
1409 	/* Reset timer state after timeout */
1410 	KASSERT(priv->timerActive, ("%s: !timerActive", __FUNCTION__));
1411 	priv->timerActive = 0;
1412 	KASSERT(node->refs > 1, ("%s: refs=%d", __FUNCTION__, node->refs));
1413 	ng_unref(node);
1414 
1415 	/* Start timer again */
1416 	ng_ppp_start_frag_timer(node);
1417 
1418 	/* Scan the fragment queue */
1419 	ng_ppp_frag_checkstale(node);
1420 	splx(s);
1421 }
1422 
1423 /*
1424  * Deliver a frame out on the bundle, i.e., figure out how to fragment
1425  * the frame across the individual PPP links and do so.
1426  */
1427 static int
1428 ng_ppp_mp_output(node_p node, struct mbuf *m, meta_p meta)
1429 {
1430 	const priv_p priv = node->private;
1431 	int distrib[NG_PPP_MAX_LINKS];
1432 	int firstFragment;
1433 	int activeLinkNum;
1434 
1435 	/* At least one link must be active */
1436 	if (priv->numActiveLinks == 0) {
1437 		NG_FREE_DATA(m, meta);
1438 		return (ENETDOWN);
1439 	}
1440 
1441 	/* Round-robin strategy */
1442 	if (priv->conf.enableRoundRobin || m->m_pkthdr.len < MP_MIN_FRAG_LEN) {
1443 		activeLinkNum = priv->lastLink++ % priv->numActiveLinks;
1444 		bzero(&distrib, priv->numActiveLinks * sizeof(distrib[0]));
1445 		distrib[activeLinkNum] = m->m_pkthdr.len;
1446 		goto deliver;
1447 	}
1448 
1449 	/* Strategy when all links are equivalent (optimize the common case) */
1450 	if (priv->allLinksEqual) {
1451 		const int fraction = m->m_pkthdr.len / priv->numActiveLinks;
1452 		int i, remain;
1453 
1454 		for (i = 0; i < priv->numActiveLinks; i++)
1455 			distrib[priv->lastLink++ % priv->numActiveLinks]
1456 			    = fraction;
1457 		remain = m->m_pkthdr.len - (fraction * priv->numActiveLinks);
1458 		while (remain > 0) {
1459 			distrib[priv->lastLink++ % priv->numActiveLinks]++;
1460 			remain--;
1461 		}
1462 		goto deliver;
1463 	}
1464 
1465 	/* Strategy when all links are not equivalent */
1466 	ng_ppp_mp_strategy(node, m->m_pkthdr.len, distrib);
1467 
1468 deliver:
1469 	/* Update stats */
1470 	priv->bundleStats.xmitFrames++;
1471 	priv->bundleStats.xmitOctets += m->m_pkthdr.len;
1472 
1473 	/* Send alloted portions of frame out on the link(s) */
1474 	for (firstFragment = 1, activeLinkNum = priv->numActiveLinks - 1;
1475 	    activeLinkNum >= 0; activeLinkNum--) {
1476 		const int linkNum = priv->activeLinks[activeLinkNum];
1477 		struct ng_ppp_link *const link = &priv->links[linkNum];
1478 
1479 		/* Deliver fragment(s) out the next link */
1480 		for ( ; distrib[activeLinkNum] > 0; firstFragment = 0) {
1481 			int len, lastFragment, error;
1482 			struct mbuf *m2;
1483 			meta_p meta2;
1484 
1485 			/* Calculate fragment length; don't exceed link MTU */
1486 			len = distrib[activeLinkNum];
1487 			if (len > link->conf.mru)
1488 				len = link->conf.mru;
1489 			distrib[activeLinkNum] -= len;
1490 			lastFragment = (len == m->m_pkthdr.len);
1491 
1492 			/* Split off next fragment as "m2" */
1493 			m2 = m;
1494 			if (!lastFragment) {
1495 				struct mbuf *n = m_split(m, len, M_NOWAIT);
1496 
1497 				if (n == NULL) {
1498 					NG_FREE_DATA(m, meta);
1499 					return (ENOMEM);
1500 				}
1501 				m = n;
1502 			}
1503 
1504 			/* Prepend MP header */
1505 			if (priv->conf.xmitShortSeq) {
1506 				u_int16_t shdr;
1507 
1508 				shdr = priv->xseq;
1509 				priv->xseq =
1510 				    (priv->xseq + 1) % MP_SHORT_SEQ_MASK;
1511 				if (firstFragment)
1512 					shdr |= MP_SHORT_FIRST_FLAG;
1513 				if (lastFragment)
1514 					shdr |= MP_SHORT_LAST_FLAG;
1515 				shdr = htons(shdr);
1516 				m2 = ng_ppp_prepend(m2, &shdr, 2);
1517 			} else {
1518 				u_int32_t lhdr;
1519 
1520 				lhdr = priv->xseq;
1521 				priv->xseq =
1522 				    (priv->xseq + 1) % MP_LONG_SEQ_MASK;
1523 				if (firstFragment)
1524 					lhdr |= MP_LONG_FIRST_FLAG;
1525 				if (lastFragment)
1526 					lhdr |= MP_LONG_LAST_FLAG;
1527 				lhdr = htonl(lhdr);
1528 				m2 = ng_ppp_prepend(m2, &lhdr, 4);
1529 			}
1530 			if (m2 == NULL) {
1531 				if (!lastFragment)
1532 					m_freem(m);
1533 				NG_FREE_META(meta);
1534 				return (ENOBUFS);
1535 			}
1536 
1537 			/* Copy the meta information, if any */
1538 			meta2 = lastFragment ? meta : ng_copy_meta(meta);
1539 
1540 			/* Send fragment */
1541 			error = ng_ppp_output(node, 0,
1542 			    PROT_MP, linkNum, m2, meta2);
1543 			if (error != 0) {
1544 				if (!lastFragment)
1545 					NG_FREE_DATA(m, meta);
1546 				return (error);
1547 			}
1548 		}
1549 	}
1550 
1551 	/* Done */
1552 	return (0);
1553 }
1554 
1555 /*
1556  * Computing the optimal fragmentation
1557  * -----------------------------------
1558  *
1559  * This routine tries to compute the optimal fragmentation pattern based
1560  * on each link's latency, bandwidth, and calculated additional latency.
1561  * The latter quantity is the additional latency caused by previously
1562  * written data that has not been transmitted yet.
1563  *
1564  * This algorithm is only useful when not all of the links have the
1565  * same latency and bandwidth values.
1566  *
1567  * The essential idea is to make the last bit of each fragment of the
1568  * frame arrive at the opposite end at the exact same time. This greedy
1569  * algorithm is optimal, in that no other scheduling could result in any
1570  * packet arriving any sooner unless packets are delivered out of order.
1571  *
1572  * Suppose link i has bandwidth b_i (in tens of bytes per milisecond) and
1573  * latency l_i (in miliseconds). Consider the function function f_i(t)
1574  * which is equal to the number of bytes that will have arrived at
1575  * the peer after t miliseconds if we start writing continuously at
1576  * time t = 0. Then f_i(t) = b_i * (t - l_i) = ((b_i * t) - (l_i * b_i).
1577  * That is, f_i(t) is a line with slope b_i and y-intersect -(l_i * b_i).
1578  * Note that the y-intersect is always <= zero because latency can't be
1579  * negative.  Note also that really the function is f_i(t) except when
1580  * f_i(t) is negative, in which case the function is zero.  To take
1581  * care of this, let Q_i(t) = { if (f_i(t) > 0) return 1; else return 0; }.
1582  * So the actual number of bytes that will have arrived at the peer after
1583  * t miliseconds is f_i(t) * Q_i(t).
1584  *
1585  * At any given time, each link has some additional latency a_i >= 0
1586  * due to previously written fragment(s) which are still in the queue.
1587  * This value is easily computed from the time since last transmission,
1588  * the previous latency value, the number of bytes written, and the
1589  * link's bandwidth.
1590  *
1591  * Assume that l_i includes any a_i already, and that the links are
1592  * sorted by latency, so that l_i <= l_{i+1}.
1593  *
1594  * Let N be the total number of bytes in the current frame we are sending.
1595  *
1596  * Suppose we were to start writing bytes at time t = 0 on all links
1597  * simultaneously, which is the most we can possibly do.  Then let
1598  * F(t) be equal to the total number of bytes received by the peer
1599  * after t miliseconds. Then F(t) = Sum_i (f_i(t) * Q_i(t)).
1600  *
1601  * Our goal is simply this: fragment the frame across the links such
1602  * that the peer is able to reconstruct the completed frame as soon as
1603  * possible, i.e., at the least possible value of t. Call this value t_0.
1604  *
1605  * Then it follows that F(t_0) = N. Our strategy is first to find the value
1606  * of t_0, and then deduce how many bytes to write to each link.
1607  *
1608  * Rewriting F(t_0):
1609  *
1610  *   t_0 = ( N + Sum_i ( l_i * b_i * Q_i(t_0) ) ) / Sum_i ( b_i * Q_i(t_0) )
1611  *
1612  * Now, we note that Q_i(t) is constant for l_i <= t <= l_{i+1}. t_0 will
1613  * lie in one of these ranges.  To find it, we just need to find the i such
1614  * that F(l_i) <= N <= F(l_{i+1}).  Then we compute all the constant values
1615  * for Q_i() in this range, plug in the remaining values, solving for t_0.
1616  *
1617  * Once t_0 is known, then the number of bytes to send on link i is
1618  * just f_i(t_0) * Q_i(t_0).
1619  *
1620  * In other words, we start allocating bytes to the links one at a time.
1621  * We keep adding links until the frame is completely sent.  Some links
1622  * may not get any bytes because their latency is too high.
1623  *
1624  * Is all this work really worth the trouble?  Depends on the situation.
1625  * The bigger the ratio of computer speed to link speed, and the more
1626  * important total bundle latency is (e.g., for interactive response time),
1627  * the more it's worth it.  There is however the cost of calling this
1628  * function for every frame.  The running time is O(n^2) where n is the
1629  * number of links that receive a non-zero number of bytes.
1630  *
1631  * Since latency is measured in miliseconds, the "resolution" of this
1632  * algorithm is one milisecond.
1633  *
1634  * To avoid this algorithm altogether, configure all links to have the
1635  * same latency and bandwidth.
1636  */
1637 static void
1638 ng_ppp_mp_strategy(node_p node, int len, int *distrib)
1639 {
1640 	const priv_p priv = node->private;
1641 	int latency[NG_PPP_MAX_LINKS];
1642 	int sortByLatency[NG_PPP_MAX_LINKS];
1643 	int activeLinkNum;
1644 	int t0, total, topSum, botSum;
1645 	struct timeval now;
1646 	int i, numFragments;
1647 
1648 	/* If only one link, this gets real easy */
1649 	if (priv->numActiveLinks == 1) {
1650 		distrib[0] = len;
1651 		return;
1652 	}
1653 
1654 	/* Get current time */
1655 	getmicrouptime(&now);
1656 
1657 	/* Compute latencies for each link at this point in time */
1658 	for (activeLinkNum = 0;
1659 	    activeLinkNum < priv->numActiveLinks; activeLinkNum++) {
1660 		struct ng_ppp_link *alink;
1661 		struct timeval diff;
1662 		int xmitBytes;
1663 
1664 		/* Start with base latency value */
1665 		alink = &priv->links[priv->activeLinks[activeLinkNum]];
1666 		latency[activeLinkNum] = alink->conf.latency;
1667 		sortByLatency[activeLinkNum] = activeLinkNum;	/* see below */
1668 
1669 		/* Any additional latency? */
1670 		if (alink->bytesInQueue == 0)
1671 			continue;
1672 
1673 		/* Compute time delta since last write */
1674 		diff = now;
1675 		timevalsub(&diff, &alink->lastWrite);
1676 		if (now.tv_sec < 0 || diff.tv_sec >= 10) {	/* sanity */
1677 			alink->bytesInQueue = 0;
1678 			continue;
1679 		}
1680 
1681 		/* How many bytes could have transmitted since last write? */
1682 		xmitBytes = (alink->conf.bandwidth * diff.tv_sec)
1683 		    + (alink->conf.bandwidth * (diff.tv_usec / 1000)) / 100;
1684 		alink->bytesInQueue -= xmitBytes;
1685 		if (alink->bytesInQueue < 0)
1686 			alink->bytesInQueue = 0;
1687 		else
1688 			latency[activeLinkNum] +=
1689 			    (100 * alink->bytesInQueue) / alink->conf.bandwidth;
1690 	}
1691 
1692 	/* Sort active links by latency */
1693 	compareLatencies = latency;
1694 	qsort(sortByLatency,
1695 	    priv->numActiveLinks, sizeof(*sortByLatency), ng_ppp_intcmp);
1696 	compareLatencies = NULL;
1697 
1698 	/* Find the interval we need (add links in sortByLatency[] order) */
1699 	for (numFragments = 1;
1700 	    numFragments < priv->numActiveLinks; numFragments++) {
1701 		for (total = i = 0; i < numFragments; i++) {
1702 			int flowTime;
1703 
1704 			flowTime = latency[sortByLatency[numFragments]]
1705 			    - latency[sortByLatency[i]];
1706 			total += ((flowTime * priv->links[
1707 			    priv->activeLinks[sortByLatency[i]]].conf.bandwidth)
1708 			    	+ 99) / 100;
1709 		}
1710 		if (total >= len)
1711 			break;
1712 	}
1713 
1714 	/* Solve for t_0 in that interval */
1715 	for (topSum = botSum = i = 0; i < numFragments; i++) {
1716 		int bw = priv->links[
1717 		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
1718 
1719 		topSum += latency[sortByLatency[i]] * bw;	/* / 100 */
1720 		botSum += bw;					/* / 100 */
1721 	}
1722 	t0 = ((len * 100) + topSum + botSum / 2) / botSum;
1723 
1724 	/* Compute f_i(t_0) all i */
1725 	bzero(distrib, priv->numActiveLinks * sizeof(*distrib));
1726 	for (total = i = 0; i < numFragments; i++) {
1727 		int bw = priv->links[
1728 		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
1729 
1730 		distrib[sortByLatency[i]] =
1731 		    (bw * (t0 - latency[sortByLatency[i]]) + 50) / 100;
1732 		total += distrib[sortByLatency[i]];
1733 	}
1734 
1735 	/* Deal with any rounding error */
1736 	if (total < len) {
1737 		struct ng_ppp_link *fastLink =
1738 		    &priv->links[priv->activeLinks[sortByLatency[0]]];
1739 		int fast = 0;
1740 
1741 		/* Find the fastest link */
1742 		for (i = 1; i < numFragments; i++) {
1743 			struct ng_ppp_link *const link =
1744 			    &priv->links[priv->activeLinks[sortByLatency[i]]];
1745 
1746 			if (link->conf.bandwidth > fastLink->conf.bandwidth) {
1747 				fast = i;
1748 				fastLink = link;
1749 			}
1750 		}
1751 		distrib[sortByLatency[fast]] += len - total;
1752 	} else while (total > len) {
1753 		struct ng_ppp_link *slowLink =
1754 		    &priv->links[priv->activeLinks[sortByLatency[0]]];
1755 		int delta, slow = 0;
1756 
1757 		/* Find the slowest link that still has bytes to remove */
1758 		for (i = 1; i < numFragments; i++) {
1759 			struct ng_ppp_link *const link =
1760 			    &priv->links[priv->activeLinks[sortByLatency[i]]];
1761 
1762 			if (distrib[sortByLatency[slow]] == 0
1763 			  || (distrib[sortByLatency[i]] > 0
1764 			    && link->conf.bandwidth <
1765 			      slowLink->conf.bandwidth)) {
1766 				slow = i;
1767 				slowLink = link;
1768 			}
1769 		}
1770 		delta = total - len;
1771 		if (delta > distrib[sortByLatency[slow]])
1772 			delta = distrib[sortByLatency[slow]];
1773 		distrib[sortByLatency[slow]] -= delta;
1774 		total -= delta;
1775 	}
1776 }
1777 
1778 /*
1779  * Compare two integers
1780  */
1781 static int
1782 ng_ppp_intcmp(const void *v1, const void *v2)
1783 {
1784 	const int index1 = *((const int *) v1);
1785 	const int index2 = *((const int *) v2);
1786 
1787 	return compareLatencies[index1] - compareLatencies[index2];
1788 }
1789 
1790 /*
1791  * Prepend a possibly compressed PPP protocol number in front of a frame
1792  */
1793 static struct mbuf *
1794 ng_ppp_addproto(struct mbuf *m, int proto, int compOK)
1795 {
1796 	if (compOK && PROT_COMPRESSABLE(proto)) {
1797 		u_char pbyte = (u_char)proto;
1798 
1799 		return ng_ppp_prepend(m, &pbyte, 1);
1800 	} else {
1801 		u_int16_t pword = htons((u_int16_t)proto);
1802 
1803 		return ng_ppp_prepend(m, &pword, 2);
1804 	}
1805 }
1806 
1807 /*
1808  * Prepend some bytes to an mbuf
1809  */
1810 static struct mbuf *
1811 ng_ppp_prepend(struct mbuf *m, const void *buf, int len)
1812 {
1813 	M_PREPEND(m, len, M_NOWAIT);
1814 	if (m == NULL || (m->m_len < len && (m = m_pullup(m, len)) == NULL))
1815 		return (NULL);
1816 	bcopy(buf, mtod(m, u_char *), len);
1817 	return (m);
1818 }
1819 
1820 /*
1821  * Update private information that is derived from other private information
1822  */
1823 static void
1824 ng_ppp_update(node_p node, int newConf)
1825 {
1826 	const priv_p priv = node->private;
1827 	int i;
1828 
1829 	/* Update active status for VJ Compression */
1830 	priv->vjCompHooked = priv->hooks[HOOK_INDEX_VJC_IP] != NULL
1831 	    && priv->hooks[HOOK_INDEX_VJC_COMP] != NULL
1832 	    && priv->hooks[HOOK_INDEX_VJC_UNCOMP] != NULL
1833 	    && priv->hooks[HOOK_INDEX_VJC_VJIP] != NULL;
1834 
1835 	/* Increase latency for each link an amount equal to one MP header */
1836 	if (newConf) {
1837 		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1838 			int hdrBytes;
1839 
1840 			hdrBytes = (priv->links[i].conf.enableACFComp ? 0 : 2)
1841 			    + (priv->links[i].conf.enableProtoComp ? 1 : 2)
1842 			    + (priv->conf.xmitShortSeq ? 2 : 4);
1843 			priv->links[i].conf.latency +=
1844 			    ((hdrBytes * priv->links[i].conf.bandwidth) + 50)
1845 				/ 100;
1846 		}
1847 	}
1848 
1849 	/* Update list of active links */
1850 	bzero(&priv->activeLinks, sizeof(priv->activeLinks));
1851 	priv->numActiveLinks = 0;
1852 	priv->allLinksEqual = 1;
1853 	for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1854 		struct ng_ppp_link *const link = &priv->links[i];
1855 
1856 		/* Is link active? */
1857 		if (link->conf.enableLink && link->hook != NULL) {
1858 			struct ng_ppp_link *link0;
1859 
1860 			/* Add link to list of active links */
1861 			priv->activeLinks[priv->numActiveLinks++] = i;
1862 			link0 = &priv->links[priv->activeLinks[0]];
1863 
1864 			/* Determine if all links are still equal */
1865 			if (link->conf.latency != link0->conf.latency
1866 			  || link->conf.bandwidth != link0->conf.bandwidth)
1867 				priv->allLinksEqual = 0;
1868 
1869 			/* Initialize rec'd sequence number */
1870 			if (link->seq == MP_NOSEQ) {
1871 				link->seq = (link == link0) ?
1872 				    MP_INITIAL_SEQ : link0->seq;
1873 			}
1874 		} else
1875 			link->seq = MP_NOSEQ;
1876 	}
1877 
1878 	/* Update MP state as multi-link is active or not */
1879 	if (priv->conf.enableMultilink && priv->numActiveLinks > 0)
1880 		ng_ppp_start_frag_timer(node);
1881 	else {
1882 		ng_ppp_stop_frag_timer(node);
1883 		ng_ppp_frag_reset(node);
1884 		priv->xseq = MP_INITIAL_SEQ;
1885 		priv->mseq = MP_INITIAL_SEQ;
1886 		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1887 			struct ng_ppp_link *const link = &priv->links[i];
1888 
1889 			bzero(&link->lastWrite, sizeof(link->lastWrite));
1890 			link->bytesInQueue = 0;
1891 			link->seq = MP_NOSEQ;
1892 		}
1893 	}
1894 }
1895 
1896 /*
1897  * Determine if a new configuration would represent a valid change
1898  * from the current configuration and link activity status.
1899  */
1900 static int
1901 ng_ppp_config_valid(node_p node, const struct ng_ppp_node_conf *newConf)
1902 {
1903 	const priv_p priv = node->private;
1904 	int i, newNumLinksActive;
1905 
1906 	/* Check per-link config and count how many links would be active */
1907 	for (newNumLinksActive = i = 0; i < NG_PPP_MAX_LINKS; i++) {
1908 		if (newConf->links[i].enableLink && priv->links[i].hook != NULL)
1909 			newNumLinksActive++;
1910 		if (!newConf->links[i].enableLink)
1911 			continue;
1912 		if (newConf->links[i].mru < MP_MIN_LINK_MRU)
1913 			return (0);
1914 		if (newConf->links[i].bandwidth == 0)
1915 			return (0);
1916 		if (newConf->links[i].bandwidth > NG_PPP_MAX_BANDWIDTH)
1917 			return (0);
1918 		if (newConf->links[i].latency > NG_PPP_MAX_LATENCY)
1919 			return (0);
1920 	}
1921 
1922 	/* Check bundle parameters */
1923 	if (newConf->bund.enableMultilink && newConf->bund.mrru < MP_MIN_MRRU)
1924 		return (0);
1925 
1926 	/* Disallow changes to multi-link configuration while MP is active */
1927 	if (priv->numActiveLinks > 0 && newNumLinksActive > 0) {
1928 		if (!priv->conf.enableMultilink
1929 				!= !newConf->bund.enableMultilink
1930 		    || !priv->conf.xmitShortSeq != !newConf->bund.xmitShortSeq
1931 		    || !priv->conf.recvShortSeq != !newConf->bund.recvShortSeq)
1932 			return (0);
1933 	}
1934 
1935 	/* At most one link can be active unless multi-link is enabled */
1936 	if (!newConf->bund.enableMultilink && newNumLinksActive > 1)
1937 		return (0);
1938 
1939 	/* Configuration change would be valid */
1940 	return (1);
1941 }
1942 
1943 /*
1944  * Free all entries in the fragment queue
1945  */
1946 static void
1947 ng_ppp_frag_reset(node_p node)
1948 {
1949 	const priv_p priv = node->private;
1950 	struct ng_ppp_frag *qent, *qnext;
1951 
1952 	for (qent = CIRCLEQ_FIRST(&priv->frags);
1953 	    qent != (void *)&priv->frags; qent = qnext) {
1954 		qnext = CIRCLEQ_NEXT(qent, f_qent);
1955 		NG_FREE_DATA(qent->data, qent->meta);
1956 		FREE(qent, M_NETGRAPH);
1957 	}
1958 	CIRCLEQ_INIT(&priv->frags);
1959 	priv->qlen = 0;
1960 }
1961 
1962 /*
1963  * Start fragment queue timer
1964  */
1965 static void
1966 ng_ppp_start_frag_timer(node_p node)
1967 {
1968 	const priv_p priv = node->private;
1969 
1970 	if (!priv->timerActive) {
1971 		priv->fragTimer = timeout(ng_ppp_frag_timeout,
1972 		    node, MP_FRAGTIMER_INTERVAL);
1973 		priv->timerActive = 1;
1974 		node->refs++;
1975 	}
1976 }
1977 
1978 /*
1979  * Stop fragment queue timer
1980  */
1981 static void
1982 ng_ppp_stop_frag_timer(node_p node)
1983 {
1984 	const priv_p priv = node->private;
1985 
1986 	if (priv->timerActive) {
1987 		untimeout(ng_ppp_frag_timeout, node, priv->fragTimer);
1988 		priv->timerActive = 0;
1989 		KASSERT(node->refs > 1,
1990 		    ("%s: refs=%d", __FUNCTION__, node->refs));
1991 		ng_unref(node);
1992 	}
1993 }
1994 
1995