xref: /freebsd/sys/netgraph/ng_ppp.c (revision 6fd05b64b5b65dd4ba9b86482a0634a5f0b96c29)
1 
2 /*
3  * ng_ppp.c
4  *
5  * Copyright (c) 1996-2000 Whistle Communications, Inc.
6  * All rights reserved.
7  *
8  * Subject to the following obligations and disclaimer of warranty, use and
9  * redistribution of this software, in source or object code forms, with or
10  * without modifications are expressly permitted by Whistle Communications;
11  * provided, however, that:
12  * 1. Any and all reproductions of the source or object code must include the
13  *    copyright notice above and the following disclaimer of warranties; and
14  * 2. No rights are granted, in any manner or form, to use Whistle
15  *    Communications, Inc. trademarks, including the mark "WHISTLE
16  *    COMMUNICATIONS" on advertising, endorsements, or otherwise except as
17  *    such appears in the above copyright notice or in the software.
18  *
19  * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
20  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
21  * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
22  * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
24  * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
25  * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
26  * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
27  * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
28  * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
29  * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
30  * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
31  * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34  * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
35  * OF SUCH DAMAGE.
36  *
37  * Author: Archie Cobbs <archie@freebsd.org>
38  *
39  * $FreeBSD$
40  * $Whistle: ng_ppp.c,v 1.24 1999/11/01 09:24:52 julian Exp $
41  */
42 
43 /*
44  * PPP node type.
45  */
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/limits.h>
51 #include <sys/time.h>
52 #include <sys/mbuf.h>
53 #include <sys/malloc.h>
54 #include <sys/errno.h>
55 #include <sys/ctype.h>
56 
57 #include <netgraph/ng_message.h>
58 #include <netgraph/netgraph.h>
59 #include <netgraph/ng_parse.h>
60 #include <netgraph/ng_ppp.h>
61 #include <netgraph/ng_vjc.h>
62 
63 #ifdef NG_SEPARATE_MALLOC
64 MALLOC_DEFINE(M_NETGRAPH_PPP, "netgraph_ppp", "netgraph ppp node");
65 #else
66 #define M_NETGRAPH_PPP M_NETGRAPH
67 #endif
68 
69 #define PROT_VALID(p)		(((p) & 0x0101) == 0x0001)
70 #define PROT_COMPRESSABLE(p)	(((p) & 0xff00) == 0x0000)
71 
72 /* Some PPP protocol numbers we're interested in */
73 #define PROT_APPLETALK		0x0029
74 #define PROT_COMPD		0x00fd
75 #define PROT_CRYPTD		0x0053
76 #define PROT_IP			0x0021
77 #define PROT_IPV6		0x0057
78 #define PROT_IPX		0x002b
79 #define PROT_LCP		0xc021
80 #define PROT_MP			0x003d
81 #define PROT_VJCOMP		0x002d
82 #define PROT_VJUNCOMP		0x002f
83 
84 /* Multilink PPP definitions */
85 #define MP_MIN_MRRU		1500		/* per RFC 1990 */
86 #define MP_INITIAL_SEQ		0		/* per RFC 1990 */
87 #define MP_MIN_LINK_MRU		32
88 
89 #define MP_SHORT_SEQ_MASK	0x00000fff	/* short seq # mask */
90 #define MP_SHORT_SEQ_HIBIT	0x00000800	/* short seq # high bit */
91 #define MP_SHORT_FIRST_FLAG	0x00008000	/* first fragment in frame */
92 #define MP_SHORT_LAST_FLAG	0x00004000	/* last fragment in frame */
93 
94 #define MP_LONG_SEQ_MASK	0x00ffffff	/* long seq # mask */
95 #define MP_LONG_SEQ_HIBIT	0x00800000	/* long seq # high bit */
96 #define MP_LONG_FIRST_FLAG	0x80000000	/* first fragment in frame */
97 #define MP_LONG_LAST_FLAG	0x40000000	/* last fragment in frame */
98 
99 #define MP_NOSEQ		0x7fffffff	/* impossible sequence number */
100 
101 /* Sign extension of MP sequence numbers */
102 #define MP_SHORT_EXTEND(s)	(((s) & MP_SHORT_SEQ_HIBIT) ?		\
103 				    ((s) | ~MP_SHORT_SEQ_MASK)		\
104 				    : ((s) & MP_SHORT_SEQ_MASK))
105 #define MP_LONG_EXTEND(s)	(((s) & MP_LONG_SEQ_HIBIT) ?		\
106 				    ((s) | ~MP_LONG_SEQ_MASK)		\
107 				    : ((s) & MP_LONG_SEQ_MASK))
108 
109 /* Comparision of MP sequence numbers. Note: all sequence numbers
110    except priv->xseq are stored with the sign bit extended. */
111 #define MP_SHORT_SEQ_DIFF(x,y)	MP_SHORT_EXTEND((x) - (y))
112 #define MP_LONG_SEQ_DIFF(x,y)	MP_LONG_EXTEND((x) - (y))
113 
114 #define MP_RECV_SEQ_DIFF(priv,x,y)					\
115 				((priv)->conf.recvShortSeq ?		\
116 				    MP_SHORT_SEQ_DIFF((x), (y)) :	\
117 				    MP_LONG_SEQ_DIFF((x), (y)))
118 
119 /* Increment receive sequence number */
120 #define MP_NEXT_RECV_SEQ(priv,seq)					\
121 				((priv)->conf.recvShortSeq ?		\
122 				    MP_SHORT_EXTEND((seq) + 1) :	\
123 				    MP_LONG_EXTEND((seq) + 1))
124 
125 /* Don't fragment transmitted packets smaller than this */
126 #define MP_MIN_FRAG_LEN		6
127 
128 /* Maximum fragment reasssembly queue length */
129 #define MP_MAX_QUEUE_LEN	128
130 
131 /* Fragment queue scanner period */
132 #define MP_FRAGTIMER_INTERVAL	(hz/2)
133 
134 /* We store incoming fragments this way */
135 struct ng_ppp_frag {
136 	int				seq;		/* fragment seq# */
137 	u_char				first;		/* First in packet? */
138 	u_char				last;		/* Last in packet? */
139 	struct timeval			timestamp;	/* time of reception */
140 	struct mbuf			*data;		/* Fragment data */
141 	TAILQ_ENTRY(ng_ppp_frag)	f_qent;		/* Fragment queue */
142 };
143 
144 /* We use integer indicies to refer to the non-link hooks */
145 static const char *const ng_ppp_hook_names[] = {
146 	NG_PPP_HOOK_ATALK,
147 #define HOOK_INDEX_ATALK		0
148 	NG_PPP_HOOK_BYPASS,
149 #define HOOK_INDEX_BYPASS		1
150 	NG_PPP_HOOK_COMPRESS,
151 #define HOOK_INDEX_COMPRESS		2
152 	NG_PPP_HOOK_ENCRYPT,
153 #define HOOK_INDEX_ENCRYPT		3
154 	NG_PPP_HOOK_DECOMPRESS,
155 #define HOOK_INDEX_DECOMPRESS		4
156 	NG_PPP_HOOK_DECRYPT,
157 #define HOOK_INDEX_DECRYPT		5
158 	NG_PPP_HOOK_INET,
159 #define HOOK_INDEX_INET			6
160 	NG_PPP_HOOK_IPX,
161 #define HOOK_INDEX_IPX			7
162 	NG_PPP_HOOK_VJC_COMP,
163 #define HOOK_INDEX_VJC_COMP		8
164 	NG_PPP_HOOK_VJC_IP,
165 #define HOOK_INDEX_VJC_IP		9
166 	NG_PPP_HOOK_VJC_UNCOMP,
167 #define HOOK_INDEX_VJC_UNCOMP		10
168 	NG_PPP_HOOK_VJC_VJIP,
169 #define HOOK_INDEX_VJC_VJIP		11
170 	NG_PPP_HOOK_IPV6,
171 #define HOOK_INDEX_IPV6			12
172 	NULL
173 #define HOOK_INDEX_MAX			13
174 };
175 
176 /* We store index numbers in the hook private pointer. The HOOK_INDEX()
177    for a hook is either the index (above) for normal hooks, or the ones
178    complement of the link number for link hooks.
179 XXX Not any more.. (what a hack)
180 #define HOOK_INDEX(hook)	(*((int16_t *) &(hook)->private))
181 */
182 
183 /* Per-link private information */
184 struct ng_ppp_link {
185 	struct ng_ppp_link_conf	conf;		/* link configuration */
186 	hook_p			hook;		/* connection to link data */
187 	int32_t			seq;		/* highest rec'd seq# - MSEQ */
188 	struct timeval		lastWrite;	/* time of last write */
189 	int			bytesInQueue;	/* bytes in the output queue */
190 	struct ng_ppp_link_stat	stats;		/* Link stats */
191 };
192 
193 /* Total per-node private information */
194 struct ng_ppp_private {
195 	struct ng_ppp_bund_conf	conf;			/* bundle config */
196 	struct ng_ppp_link_stat	bundleStats;		/* bundle stats */
197 	struct ng_ppp_link	links[NG_PPP_MAX_LINKS];/* per-link info */
198 	int32_t			xseq;			/* next out MP seq # */
199 	int32_t			mseq;			/* min links[i].seq */
200 	u_char			vjCompHooked;		/* VJ comp hooked up? */
201 	u_char			allLinksEqual;		/* all xmit the same? */
202 	u_char			timerActive;		/* frag timer active? */
203 	u_int			numActiveLinks;		/* how many links up */
204 	int			activeLinks[NG_PPP_MAX_LINKS];	/* indicies */
205 	u_int			lastLink;		/* for round robin */
206 	hook_p			hooks[HOOK_INDEX_MAX];	/* non-link hooks */
207 	TAILQ_HEAD(ng_ppp_fraglist, ng_ppp_frag)	/* fragment queue */
208 				frags;
209 	int			qlen;			/* fraq queue length */
210 	struct callout_handle	fragTimer;		/* fraq queue check */
211 };
212 typedef struct ng_ppp_private *priv_p;
213 
214 /* Netgraph node methods */
215 static ng_constructor_t	ng_ppp_constructor;
216 static ng_rcvmsg_t	ng_ppp_rcvmsg;
217 static ng_shutdown_t	ng_ppp_shutdown;
218 static ng_newhook_t	ng_ppp_newhook;
219 static ng_rcvdata_t	ng_ppp_rcvdata;
220 static ng_disconnect_t	ng_ppp_disconnect;
221 
222 /* Helper functions */
223 static int	ng_ppp_input(node_p node, int bypass,
224 			int linkNum, item_p item);
225 static int	ng_ppp_output(node_p node, int bypass, int proto,
226 			int linkNum, item_p item);
227 static int	ng_ppp_mp_input(node_p node, int linkNum, item_p item);
228 static int	ng_ppp_check_packet(node_p node);
229 static void	ng_ppp_get_packet(node_p node, struct mbuf **mp);
230 static int	ng_ppp_frag_process(node_p node);
231 static int	ng_ppp_frag_trim(node_p node);
232 static void	ng_ppp_frag_timeout(void *arg);
233 static void	ng_ppp_frag_checkstale(node_p node);
234 static void	ng_ppp_frag_reset(node_p node);
235 static int	ng_ppp_mp_output(node_p node, struct mbuf *m);
236 static void	ng_ppp_mp_strategy(node_p node, int len, int *distrib);
237 static int	ng_ppp_intcmp(void *latency, const void *v1, const void *v2);
238 static struct	mbuf *ng_ppp_addproto(struct mbuf *m, int proto, int compOK);
239 static struct	mbuf *ng_ppp_prepend(struct mbuf *m, const void *buf, int len);
240 static int	ng_ppp_config_valid(node_p node,
241 			const struct ng_ppp_node_conf *newConf);
242 static void	ng_ppp_update(node_p node, int newConf);
243 static void	ng_ppp_start_frag_timer(node_p node);
244 static void	ng_ppp_stop_frag_timer(node_p node);
245 
246 /* Parse type for struct ng_ppp_mp_state_type */
247 static const struct ng_parse_fixedarray_info ng_ppp_rseq_array_info = {
248 	&ng_parse_hint32_type,
249 	NG_PPP_MAX_LINKS
250 };
251 static const struct ng_parse_type ng_ppp_rseq_array_type = {
252 	&ng_parse_fixedarray_type,
253 	&ng_ppp_rseq_array_info,
254 };
255 static const struct ng_parse_struct_field ng_ppp_mp_state_type_fields[]
256 	= NG_PPP_MP_STATE_TYPE_INFO(&ng_ppp_rseq_array_type);
257 static const struct ng_parse_type ng_ppp_mp_state_type = {
258 	&ng_parse_struct_type,
259 	&ng_ppp_mp_state_type_fields
260 };
261 
262 /* Parse type for struct ng_ppp_link_conf */
263 static const struct ng_parse_struct_field ng_ppp_link_type_fields[]
264 	= NG_PPP_LINK_TYPE_INFO;
265 static const struct ng_parse_type ng_ppp_link_type = {
266 	&ng_parse_struct_type,
267 	&ng_ppp_link_type_fields
268 };
269 
270 /* Parse type for struct ng_ppp_bund_conf */
271 static const struct ng_parse_struct_field ng_ppp_bund_type_fields[]
272 	= NG_PPP_BUND_TYPE_INFO;
273 static const struct ng_parse_type ng_ppp_bund_type = {
274 	&ng_parse_struct_type,
275 	&ng_ppp_bund_type_fields
276 };
277 
278 /* Parse type for struct ng_ppp_node_conf */
279 static const struct ng_parse_fixedarray_info ng_ppp_array_info = {
280 	&ng_ppp_link_type,
281 	NG_PPP_MAX_LINKS
282 };
283 static const struct ng_parse_type ng_ppp_link_array_type = {
284 	&ng_parse_fixedarray_type,
285 	&ng_ppp_array_info,
286 };
287 static const struct ng_parse_struct_field ng_ppp_conf_type_fields[]
288 	= NG_PPP_CONFIG_TYPE_INFO(&ng_ppp_bund_type, &ng_ppp_link_array_type);
289 static const struct ng_parse_type ng_ppp_conf_type = {
290 	&ng_parse_struct_type,
291 	&ng_ppp_conf_type_fields
292 };
293 
294 /* Parse type for struct ng_ppp_link_stat */
295 static const struct ng_parse_struct_field ng_ppp_stats_type_fields[]
296 	= NG_PPP_STATS_TYPE_INFO;
297 static const struct ng_parse_type ng_ppp_stats_type = {
298 	&ng_parse_struct_type,
299 	&ng_ppp_stats_type_fields
300 };
301 
302 /* List of commands and how to convert arguments to/from ASCII */
303 static const struct ng_cmdlist ng_ppp_cmds[] = {
304 	{
305 	  NGM_PPP_COOKIE,
306 	  NGM_PPP_SET_CONFIG,
307 	  "setconfig",
308 	  &ng_ppp_conf_type,
309 	  NULL
310 	},
311 	{
312 	  NGM_PPP_COOKIE,
313 	  NGM_PPP_GET_CONFIG,
314 	  "getconfig",
315 	  NULL,
316 	  &ng_ppp_conf_type
317 	},
318 	{
319 	  NGM_PPP_COOKIE,
320 	  NGM_PPP_GET_MP_STATE,
321 	  "getmpstate",
322 	  NULL,
323 	  &ng_ppp_mp_state_type
324 	},
325 	{
326 	  NGM_PPP_COOKIE,
327 	  NGM_PPP_GET_LINK_STATS,
328 	  "getstats",
329 	  &ng_parse_int16_type,
330 	  &ng_ppp_stats_type
331 	},
332 	{
333 	  NGM_PPP_COOKIE,
334 	  NGM_PPP_CLR_LINK_STATS,
335 	  "clrstats",
336 	  &ng_parse_int16_type,
337 	  NULL
338 	},
339 	{
340 	  NGM_PPP_COOKIE,
341 	  NGM_PPP_GETCLR_LINK_STATS,
342 	  "getclrstats",
343 	  &ng_parse_int16_type,
344 	  &ng_ppp_stats_type
345 	},
346 	{ 0 }
347 };
348 
349 /* Node type descriptor */
350 static struct ng_type ng_ppp_typestruct = {
351 	.version =	NG_ABI_VERSION,
352 	.name =		NG_PPP_NODE_TYPE,
353 	.constructor =	ng_ppp_constructor,
354 	.rcvmsg =	ng_ppp_rcvmsg,
355 	.shutdown =	ng_ppp_shutdown,
356 	.newhook =	ng_ppp_newhook,
357 	.rcvdata =	ng_ppp_rcvdata,
358 	.disconnect =	ng_ppp_disconnect,
359 	.cmdlist =	ng_ppp_cmds,
360 };
361 NETGRAPH_INIT(ppp, &ng_ppp_typestruct);
362 
363 /* Address and control field header */
364 static const u_char ng_ppp_acf[2] = { 0xff, 0x03 };
365 
366 /* Maximum time we'll let a complete incoming packet sit in the queue */
367 static const struct timeval ng_ppp_max_staleness = { 2, 0 };	/* 2 seconds */
368 
369 #define ERROUT(x)	do { error = (x); goto done; } while (0)
370 
371 /************************************************************************
372 			NETGRAPH NODE STUFF
373  ************************************************************************/
374 
375 /*
376  * Node type constructor
377  */
378 static int
379 ng_ppp_constructor(node_p node)
380 {
381 	priv_p priv;
382 	int i;
383 
384 	/* Allocate private structure */
385 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH_PPP, M_NOWAIT | M_ZERO);
386 	if (priv == NULL)
387 		return (ENOMEM);
388 
389 	NG_NODE_SET_PRIVATE(node, priv);
390 
391 	/* Initialize state */
392 	TAILQ_INIT(&priv->frags);
393 	for (i = 0; i < NG_PPP_MAX_LINKS; i++)
394 		priv->links[i].seq = MP_NOSEQ;
395 	callout_handle_init(&priv->fragTimer);
396 
397 	/* Done */
398 	return (0);
399 }
400 
401 /*
402  * Give our OK for a hook to be added
403  */
404 static int
405 ng_ppp_newhook(node_p node, hook_p hook, const char *name)
406 {
407 	const priv_p priv = NG_NODE_PRIVATE(node);
408 	int linkNum = -1;
409 	hook_p *hookPtr = NULL;
410 	int hookIndex = -1;
411 
412 	/* Figure out which hook it is */
413 	if (strncmp(name, NG_PPP_HOOK_LINK_PREFIX,	/* a link hook? */
414 	    strlen(NG_PPP_HOOK_LINK_PREFIX)) == 0) {
415 		const char *cp;
416 		char *eptr;
417 
418 		cp = name + strlen(NG_PPP_HOOK_LINK_PREFIX);
419 		if (!isdigit(*cp) || (cp[0] == '0' && cp[1] != '\0'))
420 			return (EINVAL);
421 		linkNum = (int)strtoul(cp, &eptr, 10);
422 		if (*eptr != '\0' || linkNum < 0 || linkNum >= NG_PPP_MAX_LINKS)
423 			return (EINVAL);
424 		hookPtr = &priv->links[linkNum].hook;
425 		hookIndex = ~linkNum;
426 	} else {				/* must be a non-link hook */
427 		int i;
428 
429 		for (i = 0; ng_ppp_hook_names[i] != NULL; i++) {
430 			if (strcmp(name, ng_ppp_hook_names[i]) == 0) {
431 				hookPtr = &priv->hooks[i];
432 				hookIndex = i;
433 				break;
434 			}
435 		}
436 		if (ng_ppp_hook_names[i] == NULL)
437 			return (EINVAL);	/* no such hook */
438 	}
439 
440 	/* See if hook is already connected */
441 	if (*hookPtr != NULL)
442 		return (EISCONN);
443 
444 	/* Disallow more than one link unless multilink is enabled */
445 	if (linkNum != -1 && priv->links[linkNum].conf.enableLink
446 	    && !priv->conf.enableMultilink && priv->numActiveLinks >= 1)
447 		return (ENODEV);
448 
449 	/* OK */
450 	*hookPtr = hook;
451 	NG_HOOK_SET_PRIVATE(hook, (void *)(intptr_t)hookIndex);
452 	ng_ppp_update(node, 0);
453 	return (0);
454 }
455 
456 /*
457  * Receive a control message
458  */
459 static int
460 ng_ppp_rcvmsg(node_p node, item_p item, hook_p lasthook)
461 {
462 	const priv_p priv = NG_NODE_PRIVATE(node);
463 	struct ng_mesg *resp = NULL;
464 	int error = 0;
465 	struct ng_mesg *msg;
466 
467 	NGI_GET_MSG(item, msg);
468 	switch (msg->header.typecookie) {
469 	case NGM_PPP_COOKIE:
470 		switch (msg->header.cmd) {
471 		case NGM_PPP_SET_CONFIG:
472 		    {
473 			struct ng_ppp_node_conf *const conf =
474 			    (struct ng_ppp_node_conf *)msg->data;
475 			int i;
476 
477 			/* Check for invalid or illegal config */
478 			if (msg->header.arglen != sizeof(*conf))
479 				ERROUT(EINVAL);
480 			if (!ng_ppp_config_valid(node, conf))
481 				ERROUT(EINVAL);
482 
483 			/* Copy config */
484 			priv->conf = conf->bund;
485 			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
486 				priv->links[i].conf = conf->links[i];
487 			ng_ppp_update(node, 1);
488 			break;
489 		    }
490 		case NGM_PPP_GET_CONFIG:
491 		    {
492 			struct ng_ppp_node_conf *conf;
493 			int i;
494 
495 			NG_MKRESPONSE(resp, msg, sizeof(*conf), M_NOWAIT);
496 			if (resp == NULL)
497 				ERROUT(ENOMEM);
498 			conf = (struct ng_ppp_node_conf *)resp->data;
499 			conf->bund = priv->conf;
500 			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
501 				conf->links[i] = priv->links[i].conf;
502 			break;
503 		    }
504 		case NGM_PPP_GET_MP_STATE:
505 		    {
506 			struct ng_ppp_mp_state *info;
507 			int i;
508 
509 			NG_MKRESPONSE(resp, msg, sizeof(*info), M_NOWAIT);
510 			if (resp == NULL)
511 				ERROUT(ENOMEM);
512 			info = (struct ng_ppp_mp_state *)resp->data;
513 			bzero(info, sizeof(*info));
514 			for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
515 				if (priv->links[i].seq != MP_NOSEQ)
516 					info->rseq[i] = priv->links[i].seq;
517 			}
518 			info->mseq = priv->mseq;
519 			info->xseq = priv->xseq;
520 			break;
521 		    }
522 		case NGM_PPP_GET_LINK_STATS:
523 		case NGM_PPP_CLR_LINK_STATS:
524 		case NGM_PPP_GETCLR_LINK_STATS:
525 		    {
526 			struct ng_ppp_link_stat *stats;
527 			u_int16_t linkNum;
528 
529 			if (msg->header.arglen != sizeof(u_int16_t))
530 				ERROUT(EINVAL);
531 			linkNum = *((u_int16_t *) msg->data);
532 			if (linkNum >= NG_PPP_MAX_LINKS
533 			    && linkNum != NG_PPP_BUNDLE_LINKNUM)
534 				ERROUT(EINVAL);
535 			stats = (linkNum == NG_PPP_BUNDLE_LINKNUM) ?
536 			    &priv->bundleStats : &priv->links[linkNum].stats;
537 			if (msg->header.cmd != NGM_PPP_CLR_LINK_STATS) {
538 				NG_MKRESPONSE(resp, msg,
539 				    sizeof(struct ng_ppp_link_stat), M_NOWAIT);
540 				if (resp == NULL)
541 					ERROUT(ENOMEM);
542 				bcopy(stats, resp->data, sizeof(*stats));
543 			}
544 			if (msg->header.cmd != NGM_PPP_GET_LINK_STATS)
545 				bzero(stats, sizeof(*stats));
546 			break;
547 		    }
548 		default:
549 			error = EINVAL;
550 			break;
551 		}
552 		break;
553 	case NGM_VJC_COOKIE:
554 	    {
555 		/*
556 		 * Forward it to the vjc node. leave the
557 		 * old return address alone.
558 		 * If we have no hook, let NG_RESPOND_MSG
559 		 * clean up any remaining resources.
560 		 * Because we have no resp, the item will be freed
561 		 * along with anything it references. Don't
562 		 * let msg be freed twice.
563 		 */
564 		NGI_MSG(item) = msg;	/* put it back in the item */
565 		msg = NULL;
566 		if ((lasthook = priv->links[HOOK_INDEX_VJC_IP].hook)) {
567 			NG_FWD_ITEM_HOOK(error, item, lasthook);
568 		}
569 		return (error);
570 	    }
571 	default:
572 		error = EINVAL;
573 		break;
574 	}
575 done:
576 	NG_RESPOND_MSG(error, node, item, resp);
577 	NG_FREE_MSG(msg);
578 	return (error);
579 }
580 
581 /*
582  * Receive data on a hook
583  */
584 static int
585 ng_ppp_rcvdata(hook_p hook, item_p item)
586 {
587 	const node_p node = NG_HOOK_NODE(hook);
588 	const priv_p priv = NG_NODE_PRIVATE(node);
589 	const int index = (intptr_t)NG_HOOK_PRIVATE(hook);
590 	u_int16_t linkNum = NG_PPP_BUNDLE_LINKNUM;
591 	hook_p outHook = NULL;
592 	int proto = 0, error;
593 	struct mbuf *m;
594 
595 	NGI_GET_M(item, m);
596 	/* Did it come from a link hook? */
597 	if (index < 0) {
598 		struct ng_ppp_link *link;
599 
600 		/* Convert index into a link number */
601 		linkNum = (u_int16_t)~index;
602 		KASSERT(linkNum < NG_PPP_MAX_LINKS,
603 		    ("%s: bogus index 0x%x", __func__, index));
604 		link = &priv->links[linkNum];
605 
606 		/* Stats */
607 		link->stats.recvFrames++;
608 		link->stats.recvOctets += m->m_pkthdr.len;
609 
610 		/* Strip address and control fields, if present */
611 		if (m->m_pkthdr.len >= 2) {
612 			if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL) {
613 				NG_FREE_ITEM(item);
614 				return (ENOBUFS);
615 			}
616 			if (bcmp(mtod(m, u_char *), &ng_ppp_acf, 2) == 0)
617 				m_adj(m, 2);
618 		}
619 
620 		/* Dispatch incoming frame (if not enabled, to bypass) */
621 		NGI_M(item) = m; 	/* put changed m back in item */
622 		return ng_ppp_input(node,
623 		    !link->conf.enableLink, linkNum, item);
624 	}
625 
626 	/* Get protocol & check if data allowed from this hook */
627 	NGI_M(item) = m; 	/* put possibly changed m back in item */
628 	switch (index) {
629 
630 	/* Outgoing data */
631 	case HOOK_INDEX_ATALK:
632 		if (!priv->conf.enableAtalk) {
633 			NG_FREE_ITEM(item);
634 			return (ENXIO);
635 		}
636 		proto = PROT_APPLETALK;
637 		break;
638 	case HOOK_INDEX_IPX:
639 		if (!priv->conf.enableIPX) {
640 			NG_FREE_ITEM(item);
641 			return (ENXIO);
642 		}
643 		proto = PROT_IPX;
644 		break;
645 	case HOOK_INDEX_IPV6:
646 		if (!priv->conf.enableIPv6) {
647 			NG_FREE_ITEM(item);
648 			return (ENXIO);
649 		}
650 		proto = PROT_IPV6;
651 		break;
652 	case HOOK_INDEX_INET:
653 	case HOOK_INDEX_VJC_VJIP:
654 		if (!priv->conf.enableIP) {
655 			NG_FREE_ITEM(item);
656 			return (ENXIO);
657 		}
658 		proto = PROT_IP;
659 		break;
660 	case HOOK_INDEX_VJC_COMP:
661 		if (!priv->conf.enableVJCompression) {
662 			NG_FREE_ITEM(item);
663 			return (ENXIO);
664 		}
665 		proto = PROT_VJCOMP;
666 		break;
667 	case HOOK_INDEX_VJC_UNCOMP:
668 		if (!priv->conf.enableVJCompression) {
669 			NG_FREE_ITEM(item);
670 			return (ENXIO);
671 		}
672 		proto = PROT_VJUNCOMP;
673 		break;
674 	case HOOK_INDEX_COMPRESS:
675 		if (!priv->conf.enableCompression) {
676 			NG_FREE_ITEM(item);
677 			return (ENXIO);
678 		}
679 		proto = PROT_COMPD;
680 		break;
681 	case HOOK_INDEX_ENCRYPT:
682 		if (!priv->conf.enableEncryption) {
683 			NG_FREE_ITEM(item);
684 			return (ENXIO);
685 		}
686 		proto = PROT_CRYPTD;
687 		break;
688 	case HOOK_INDEX_BYPASS:
689 		if (m->m_pkthdr.len < 4) {
690 			NG_FREE_ITEM(item);
691 			return (EINVAL);
692 		}
693 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
694 			NGI_M(item) = NULL; /* don't free twice */
695 			NG_FREE_ITEM(item);
696 			return (ENOBUFS);
697 		}
698 		NGI_M(item) = m; /* m may have changed */
699 		linkNum = ntohs(mtod(m, u_int16_t *)[0]);
700 		proto = ntohs(mtod(m, u_int16_t *)[1]);
701 		m_adj(m, 4);
702 		if (linkNum >= NG_PPP_MAX_LINKS
703 		    && linkNum != NG_PPP_BUNDLE_LINKNUM) {
704 			NG_FREE_ITEM(item);
705 			return (EINVAL);
706 		}
707 		break;
708 
709 	/* Incoming data */
710 	case HOOK_INDEX_VJC_IP:
711 		if (!priv->conf.enableIP || !priv->conf.enableVJDecompression) {
712 			NG_FREE_ITEM(item);
713 			return (ENXIO);
714 		}
715 		break;
716 	case HOOK_INDEX_DECOMPRESS:
717 		if (!priv->conf.enableDecompression) {
718 			NG_FREE_ITEM(item);
719 			return (ENXIO);
720 		}
721 		break;
722 	case HOOK_INDEX_DECRYPT:
723 		if (!priv->conf.enableDecryption) {
724 			NG_FREE_ITEM(item);
725 			return (ENXIO);
726 		}
727 		break;
728 	default:
729 		panic("%s: bogus index 0x%x", __func__, index);
730 	}
731 
732 	/* Now figure out what to do with the frame */
733 	switch (index) {
734 
735 	/* Outgoing data */
736 	case HOOK_INDEX_INET:
737 		if (priv->conf.enableVJCompression && priv->vjCompHooked) {
738 			outHook = priv->hooks[HOOK_INDEX_VJC_IP];
739 			break;
740 		}
741 		/* FALLTHROUGH */
742 	case HOOK_INDEX_ATALK:
743 	case HOOK_INDEX_IPV6:
744 	case HOOK_INDEX_IPX:
745 	case HOOK_INDEX_VJC_COMP:
746 	case HOOK_INDEX_VJC_UNCOMP:
747 	case HOOK_INDEX_VJC_VJIP:
748 		if (priv->conf.enableCompression
749 		    && priv->hooks[HOOK_INDEX_COMPRESS] != NULL) {
750 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
751 				NGI_M(item) = NULL;
752 				NG_FREE_ITEM(item);
753 				return (ENOBUFS);
754 			}
755 			NGI_M(item) = m; /* m may have changed */
756 			outHook = priv->hooks[HOOK_INDEX_COMPRESS];
757 			break;
758 		}
759 		/* FALLTHROUGH */
760 	case HOOK_INDEX_COMPRESS:
761 		if (priv->conf.enableEncryption
762 		    && priv->hooks[HOOK_INDEX_ENCRYPT] != NULL) {
763 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
764 				NGI_M(item) = NULL;
765 				NG_FREE_ITEM(item);
766 				return (ENOBUFS);
767 			}
768 			NGI_M(item) = m; /* m may have changed */
769 			outHook = priv->hooks[HOOK_INDEX_ENCRYPT];
770 			break;
771 		}
772 		/* FALLTHROUGH */
773 	case HOOK_INDEX_ENCRYPT:
774 		return ng_ppp_output(node, 0, proto, NG_PPP_BUNDLE_LINKNUM, item);
775 
776 	case HOOK_INDEX_BYPASS:
777 		return ng_ppp_output(node, 1, proto, linkNum, item);
778 
779 	/* Incoming data */
780 	case HOOK_INDEX_DECRYPT:
781 	case HOOK_INDEX_DECOMPRESS:
782 		return ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
783 
784 	case HOOK_INDEX_VJC_IP:
785 		outHook = priv->hooks[HOOK_INDEX_INET];
786 		break;
787 	}
788 
789 	/* Send packet out hook */
790 	NG_FWD_ITEM_HOOK(error, item, outHook);
791 	return (error);
792 }
793 
794 /*
795  * Destroy node
796  */
797 static int
798 ng_ppp_shutdown(node_p node)
799 {
800 	const priv_p priv = NG_NODE_PRIVATE(node);
801 
802 	/* Stop fragment queue timer */
803 	ng_ppp_stop_frag_timer(node);
804 
805 	/* Take down netgraph node */
806 	ng_ppp_frag_reset(node);
807 	bzero(priv, sizeof(*priv));
808 	FREE(priv, M_NETGRAPH_PPP);
809 	NG_NODE_SET_PRIVATE(node, NULL);
810 	NG_NODE_UNREF(node);		/* let the node escape */
811 	return (0);
812 }
813 
814 /*
815  * Hook disconnection
816  */
817 static int
818 ng_ppp_disconnect(hook_p hook)
819 {
820 	const node_p node = NG_HOOK_NODE(hook);
821 	const priv_p priv = NG_NODE_PRIVATE(node);
822 	const int index = (intptr_t)NG_HOOK_PRIVATE(hook);
823 
824 	/* Zero out hook pointer */
825 	if (index < 0)
826 		priv->links[~index].hook = NULL;
827 	else
828 		priv->hooks[index] = NULL;
829 
830 	/* Update derived info (or go away if no hooks left) */
831 	if (NG_NODE_NUMHOOKS(node) > 0) {
832 		ng_ppp_update(node, 0);
833 	} else {
834 		if (NG_NODE_IS_VALID(node)) {
835 			ng_rmnode_self(node);
836 		}
837 	}
838 	return (0);
839 }
840 
841 /************************************************************************
842 			HELPER STUFF
843  ************************************************************************/
844 
845 /*
846  * Handle an incoming frame.  Extract the PPP protocol number
847  * and dispatch accordingly.
848  */
849 static int
850 ng_ppp_input(node_p node, int bypass, int linkNum, item_p item)
851 {
852 	const priv_p priv = NG_NODE_PRIVATE(node);
853 	hook_p outHook = NULL;
854 	int proto, error;
855 	struct mbuf *m;
856 
857 
858 	NGI_GET_M(item, m);
859 	/* Extract protocol number */
860 	for (proto = 0; !PROT_VALID(proto) && m->m_pkthdr.len > 0; ) {
861 		if (m->m_len < 1 && (m = m_pullup(m, 1)) == NULL) {
862 			NG_FREE_ITEM(item);
863 			return (ENOBUFS);
864 		}
865 		proto = (proto << 8) + *mtod(m, u_char *);
866 		m_adj(m, 1);
867 	}
868 	if (!PROT_VALID(proto)) {
869 		if (linkNum == NG_PPP_BUNDLE_LINKNUM)
870 			priv->bundleStats.badProtos++;
871 		else
872 			priv->links[linkNum].stats.badProtos++;
873 		NG_FREE_ITEM(item);
874 		NG_FREE_M(m);
875 		return (EINVAL);
876 	}
877 
878 	/* Bypass frame? */
879 	if (bypass)
880 		goto bypass;
881 
882 	/* Check protocol */
883 	switch (proto) {
884 	case PROT_COMPD:
885 		if (priv->conf.enableDecompression)
886 			outHook = priv->hooks[HOOK_INDEX_DECOMPRESS];
887 		break;
888 	case PROT_CRYPTD:
889 		if (priv->conf.enableDecryption)
890 			outHook = priv->hooks[HOOK_INDEX_DECRYPT];
891 		break;
892 	case PROT_VJCOMP:
893 		if (priv->conf.enableVJDecompression && priv->vjCompHooked)
894 			outHook = priv->hooks[HOOK_INDEX_VJC_COMP];
895 		break;
896 	case PROT_VJUNCOMP:
897 		if (priv->conf.enableVJDecompression && priv->vjCompHooked)
898 			outHook = priv->hooks[HOOK_INDEX_VJC_UNCOMP];
899 		break;
900 	case PROT_MP:
901 		if (priv->conf.enableMultilink
902 		    && linkNum != NG_PPP_BUNDLE_LINKNUM) {
903 			NGI_M(item) = m;
904 			return ng_ppp_mp_input(node, linkNum, item);
905 		}
906 		break;
907 	case PROT_APPLETALK:
908 		if (priv->conf.enableAtalk)
909 			outHook = priv->hooks[HOOK_INDEX_ATALK];
910 		break;
911 	case PROT_IPX:
912 		if (priv->conf.enableIPX)
913 			outHook = priv->hooks[HOOK_INDEX_IPX];
914 		break;
915 	case PROT_IP:
916 		if (priv->conf.enableIP)
917 			outHook = priv->hooks[HOOK_INDEX_INET];
918 		break;
919 	case PROT_IPV6:
920 		if (priv->conf.enableIPv6)
921 			outHook = priv->hooks[HOOK_INDEX_IPV6];
922 		break;
923 	}
924 
925 bypass:
926 	/* For unknown/inactive protocols, forward out the bypass hook */
927 	if (outHook == NULL) {
928 		u_int16_t hdr[2];
929 
930 		hdr[0] = htons(linkNum);
931 		hdr[1] = htons((u_int16_t)proto);
932 		if ((m = ng_ppp_prepend(m, &hdr, 4)) == NULL) {
933 			NG_FREE_ITEM(item);
934 			return (ENOBUFS);
935 		}
936 		outHook = priv->hooks[HOOK_INDEX_BYPASS];
937 	}
938 
939 	/* Forward frame */
940 	NG_FWD_NEW_DATA(error, item, outHook, m);
941 	return (error);
942 }
943 
944 /*
945  * Deliver a frame out a link, either a real one or NG_PPP_BUNDLE_LINKNUM.
946  * If the link is not enabled then ENXIO is returned, unless "bypass" is != 0.
947  *
948  * If the frame is too big for the particular link, return EMSGSIZE.
949  */
950 static int
951 ng_ppp_output(node_p node, int bypass,
952 	int proto, int linkNum, item_p item)
953 {
954 	const priv_p priv = NG_NODE_PRIVATE(node);
955 	struct ng_ppp_link *link;
956 	int len, error;
957 	struct mbuf *m;
958 	u_int16_t mru;
959 
960 	/* Extract mbuf */
961 	NGI_GET_M(item, m);
962 
963 	/* If not doing MP, map bundle virtual link to (the only) link */
964 	if (linkNum == NG_PPP_BUNDLE_LINKNUM && !priv->conf.enableMultilink)
965 		linkNum = priv->activeLinks[0];
966 
967 	/* Get link pointer (optimization) */
968 	link = (linkNum != NG_PPP_BUNDLE_LINKNUM) ?
969 	    &priv->links[linkNum] : NULL;
970 
971 	/* Check link status (if real) */
972 	if (linkNum != NG_PPP_BUNDLE_LINKNUM) {
973 		if (!bypass && !link->conf.enableLink) {
974 			NG_FREE_M(m);
975 			NG_FREE_ITEM(item);
976 			return (ENXIO);
977 		}
978 		if (link->hook == NULL) {
979 			NG_FREE_M(m);
980 			NG_FREE_ITEM(item);
981 			return (ENETDOWN);
982 		}
983 	}
984 
985 	/* Check peer's MRU for this link */
986 	mru = (link != NULL) ? link->conf.mru : priv->conf.mrru;
987 	if (mru != 0 && m->m_pkthdr.len > mru) {
988 		NG_FREE_M(m);
989 		NG_FREE_ITEM(item);
990 		return (EMSGSIZE);
991 	}
992 
993 	/* Prepend protocol number, possibly compressed */
994 	if ((m = ng_ppp_addproto(m, proto,
995 	    linkNum == NG_PPP_BUNDLE_LINKNUM
996 	      || link->conf.enableProtoComp)) == NULL) {
997 		NG_FREE_ITEM(item);
998 		return (ENOBUFS);
999 	}
1000 
1001 	/* Special handling for the MP virtual link */
1002 	if (linkNum == NG_PPP_BUNDLE_LINKNUM) {
1003 		/* discard the queue item */
1004 		NG_FREE_ITEM(item);
1005 		return ng_ppp_mp_output(node, m);
1006 	}
1007 
1008 	/* Prepend address and control field (unless compressed) */
1009 	if (proto == PROT_LCP || !link->conf.enableACFComp) {
1010 		if ((m = ng_ppp_prepend(m, &ng_ppp_acf, 2)) == NULL) {
1011 			NG_FREE_ITEM(item);
1012 			return (ENOBUFS);
1013 		}
1014 	}
1015 
1016 	/* Deliver frame */
1017 	len = m->m_pkthdr.len;
1018 	NG_FWD_NEW_DATA(error, item,  link->hook, m);
1019 
1020 	/* Update stats and 'bytes in queue' counter */
1021 	if (error == 0) {
1022 		link->stats.xmitFrames++;
1023 		link->stats.xmitOctets += len;
1024 		link->bytesInQueue += len;
1025 		getmicrouptime(&link->lastWrite);
1026 	}
1027 	return error;
1028 }
1029 
1030 /*
1031  * Handle an incoming multi-link fragment
1032  *
1033  * The fragment reassembly algorithm is somewhat complex. This is mainly
1034  * because we are required not to reorder the reconstructed packets, yet
1035  * fragments are only guaranteed to arrive in order on a per-link basis.
1036  * In other words, when we have a complete packet ready, but the previous
1037  * packet is still incomplete, we have to decide between delivering the
1038  * complete packet and throwing away the incomplete one, or waiting to
1039  * see if the remainder of the incomplete one arrives, at which time we
1040  * can deliver both packets, in order.
1041  *
1042  * This problem is exacerbated by "sequence number slew", which is when
1043  * the sequence numbers coming in from different links are far apart from
1044  * each other. In particular, certain unnamed equipment (*cough* Ascend)
1045  * has been seen to generate sequence number slew of up to 10 on an ISDN
1046  * 2B-channel MP link. There is nothing invalid about sequence number slew
1047  * but it makes the reasssembly process have to work harder.
1048  *
1049  * However, the peer is required to transmit fragments in order on each
1050  * link. That means if we define MSEQ as the minimum over all links of
1051  * the highest sequence number received on that link, then we can always
1052  * give up any hope of receiving a fragment with sequence number < MSEQ in
1053  * the future (all of this using 'wraparound' sequence number space).
1054  * Therefore we can always immediately throw away incomplete packets
1055  * missing fragments with sequence numbers < MSEQ.
1056  *
1057  * Here is an overview of our algorithm:
1058  *
1059  *    o Received fragments are inserted into a queue, for which we
1060  *	maintain these invariants between calls to this function:
1061  *
1062  *	- Fragments are ordered in the queue by sequence number
1063  *	- If a complete packet is at the head of the queue, then
1064  *	  the first fragment in the packet has seq# > MSEQ + 1
1065  *	  (otherwise, we could deliver it immediately)
1066  *	- If any fragments have seq# < MSEQ, then they are necessarily
1067  *	  part of a packet whose missing seq#'s are all > MSEQ (otherwise,
1068  *	  we can throw them away because they'll never be completed)
1069  *	- The queue contains at most MP_MAX_QUEUE_LEN fragments
1070  *
1071  *    o We have a periodic timer that checks the queue for the first
1072  *	complete packet that has been sitting in the queue "too long".
1073  *	When one is detected, all previous (incomplete) fragments are
1074  *	discarded, their missing fragments are declared lost and MSEQ
1075  *	is increased.
1076  *
1077  *    o If we recieve a fragment with seq# < MSEQ, we throw it away
1078  *	because we've already delcared it lost.
1079  *
1080  * This assumes linkNum != NG_PPP_BUNDLE_LINKNUM.
1081  */
1082 static int
1083 ng_ppp_mp_input(node_p node, int linkNum, item_p item)
1084 {
1085 	const priv_p priv = NG_NODE_PRIVATE(node);
1086 	struct ng_ppp_link *const link = &priv->links[linkNum];
1087 	struct ng_ppp_frag frag0, *frag = &frag0;
1088 	struct ng_ppp_frag *qent;
1089 	int i, diff, inserted;
1090 	struct mbuf *m;
1091 
1092 	NGI_GET_M(item, m);
1093 	NG_FREE_ITEM(item);
1094 	/* Stats */
1095 	priv->bundleStats.recvFrames++;
1096 	priv->bundleStats.recvOctets += m->m_pkthdr.len;
1097 
1098 	/* Extract fragment information from MP header */
1099 	if (priv->conf.recvShortSeq) {
1100 		u_int16_t shdr;
1101 
1102 		if (m->m_pkthdr.len < 2) {
1103 			link->stats.runts++;
1104 			NG_FREE_M(m);
1105 			return (EINVAL);
1106 		}
1107 		if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL)
1108 			return (ENOBUFS);
1109 
1110 		shdr = ntohs(*mtod(m, u_int16_t *));
1111 		frag->seq = MP_SHORT_EXTEND(shdr);
1112 		frag->first = (shdr & MP_SHORT_FIRST_FLAG) != 0;
1113 		frag->last = (shdr & MP_SHORT_LAST_FLAG) != 0;
1114 		diff = MP_SHORT_SEQ_DIFF(frag->seq, priv->mseq);
1115 		m_adj(m, 2);
1116 	} else {
1117 		u_int32_t lhdr;
1118 
1119 		if (m->m_pkthdr.len < 4) {
1120 			link->stats.runts++;
1121 			NG_FREE_M(m);
1122 			return (EINVAL);
1123 		}
1124 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL)
1125 			return (ENOBUFS);
1126 
1127 		lhdr = ntohl(*mtod(m, u_int32_t *));
1128 		frag->seq = MP_LONG_EXTEND(lhdr);
1129 		frag->first = (lhdr & MP_LONG_FIRST_FLAG) != 0;
1130 		frag->last = (lhdr & MP_LONG_LAST_FLAG) != 0;
1131 		diff = MP_LONG_SEQ_DIFF(frag->seq, priv->mseq);
1132 		m_adj(m, 4);
1133 	}
1134 	frag->data = m;
1135 	getmicrouptime(&frag->timestamp);
1136 
1137 	/* If sequence number is < MSEQ, we've already declared this
1138 	   fragment as lost, so we have no choice now but to drop it */
1139 	if (diff < 0) {
1140 		link->stats.dropFragments++;
1141 		NG_FREE_M(m);
1142 		return (0);
1143 	}
1144 
1145 	/* Update highest received sequence number on this link and MSEQ */
1146 	priv->mseq = link->seq = frag->seq;
1147 	for (i = 0; i < priv->numActiveLinks; i++) {
1148 		struct ng_ppp_link *const alink =
1149 		    &priv->links[priv->activeLinks[i]];
1150 
1151 		if (MP_RECV_SEQ_DIFF(priv, alink->seq, priv->mseq) < 0)
1152 			priv->mseq = alink->seq;
1153 	}
1154 
1155 	/* Allocate a new frag struct for the queue */
1156 	MALLOC(frag, struct ng_ppp_frag *, sizeof(*frag), M_NETGRAPH_PPP, M_NOWAIT);
1157 	if (frag == NULL) {
1158 		NG_FREE_M(m);
1159 		ng_ppp_frag_process(node);
1160 		return (ENOMEM);
1161 	}
1162 	*frag = frag0;
1163 
1164 	/* Add fragment to queue, which is sorted by sequence number */
1165 	inserted = 0;
1166 	TAILQ_FOREACH_REVERSE(qent, &priv->frags, ng_ppp_fraglist, f_qent) {
1167 		diff = MP_RECV_SEQ_DIFF(priv, frag->seq, qent->seq);
1168 		if (diff > 0) {
1169 			TAILQ_INSERT_AFTER(&priv->frags, qent, frag, f_qent);
1170 			inserted = 1;
1171 			break;
1172 		} else if (diff == 0) {	     /* should never happen! */
1173 			link->stats.dupFragments++;
1174 			NG_FREE_M(frag->data);
1175 			FREE(frag, M_NETGRAPH_PPP);
1176 			return (EINVAL);
1177 		}
1178 	}
1179 	if (!inserted)
1180 		TAILQ_INSERT_HEAD(&priv->frags, frag, f_qent);
1181 	priv->qlen++;
1182 
1183 	/* Process the queue */
1184 	return ng_ppp_frag_process(node);
1185 }
1186 
1187 /*
1188  * Examine our list of fragments, and determine if there is a
1189  * complete and deliverable packet at the head of the list.
1190  * Return 1 if so, zero otherwise.
1191  */
1192 static int
1193 ng_ppp_check_packet(node_p node)
1194 {
1195 	const priv_p priv = NG_NODE_PRIVATE(node);
1196 	struct ng_ppp_frag *qent, *qnext;
1197 
1198 	/* Check for empty queue */
1199 	if (TAILQ_EMPTY(&priv->frags))
1200 		return (0);
1201 
1202 	/* Check first fragment is the start of a deliverable packet */
1203 	qent = TAILQ_FIRST(&priv->frags);
1204 	if (!qent->first || MP_RECV_SEQ_DIFF(priv, qent->seq, priv->mseq) > 1)
1205 		return (0);
1206 
1207 	/* Check that all the fragments are there */
1208 	while (!qent->last) {
1209 		qnext = TAILQ_NEXT(qent, f_qent);
1210 		if (qnext == NULL)	/* end of queue */
1211 			return (0);
1212 		if (qnext->seq != MP_NEXT_RECV_SEQ(priv, qent->seq))
1213 			return (0);
1214 		qent = qnext;
1215 	}
1216 
1217 	/* Got one */
1218 	return (1);
1219 }
1220 
1221 /*
1222  * Pull a completed packet off the head of the incoming fragment queue.
1223  * This assumes there is a completed packet there to pull off.
1224  */
1225 static void
1226 ng_ppp_get_packet(node_p node, struct mbuf **mp)
1227 {
1228 	const priv_p priv = NG_NODE_PRIVATE(node);
1229 	struct ng_ppp_frag *qent, *qnext;
1230 	struct mbuf *m = NULL, *tail;
1231 
1232 	qent = TAILQ_FIRST(&priv->frags);
1233 	KASSERT(!TAILQ_EMPTY(&priv->frags) && qent->first,
1234 	    ("%s: no packet", __func__));
1235 	for (tail = NULL; qent != NULL; qent = qnext) {
1236 		qnext = TAILQ_NEXT(qent, f_qent);
1237 		KASSERT(!TAILQ_EMPTY(&priv->frags),
1238 		    ("%s: empty q", __func__));
1239 		TAILQ_REMOVE(&priv->frags, qent, f_qent);
1240 		if (tail == NULL)
1241 			tail = m = qent->data;
1242 		else {
1243 			m->m_pkthdr.len += qent->data->m_pkthdr.len;
1244 			tail->m_next = qent->data;
1245 		}
1246 		while (tail->m_next != NULL)
1247 			tail = tail->m_next;
1248 		if (qent->last)
1249 			qnext = NULL;
1250 		FREE(qent, M_NETGRAPH_PPP);
1251 		priv->qlen--;
1252 	}
1253 	*mp = m;
1254 }
1255 
1256 /*
1257  * Trim fragments from the queue whose packets can never be completed.
1258  * This assumes a complete packet is NOT at the beginning of the queue.
1259  * Returns 1 if fragments were removed, zero otherwise.
1260  */
1261 static int
1262 ng_ppp_frag_trim(node_p node)
1263 {
1264 	const priv_p priv = NG_NODE_PRIVATE(node);
1265 	struct ng_ppp_frag *qent, *qnext = NULL;
1266 	int removed = 0;
1267 
1268 	/* Scan for "dead" fragments and remove them */
1269 	while (1) {
1270 		int dead = 0;
1271 
1272 		/* If queue is empty, we're done */
1273 		if (TAILQ_EMPTY(&priv->frags))
1274 			break;
1275 
1276 		/* Determine whether first fragment can ever be completed */
1277 		TAILQ_FOREACH(qent, &priv->frags, f_qent) {
1278 			if (MP_RECV_SEQ_DIFF(priv, qent->seq, priv->mseq) >= 0)
1279 				break;
1280 			qnext = TAILQ_NEXT(qent, f_qent);
1281 			KASSERT(qnext != NULL,
1282 			    ("%s: last frag < MSEQ?", __func__));
1283 			if (qnext->seq != MP_NEXT_RECV_SEQ(priv, qent->seq)
1284 			    || qent->last || qnext->first) {
1285 				dead = 1;
1286 				break;
1287 			}
1288 		}
1289 		if (!dead)
1290 			break;
1291 
1292 		/* Remove fragment and all others in the same packet */
1293 		while ((qent = TAILQ_FIRST(&priv->frags)) != qnext) {
1294 			KASSERT(!TAILQ_EMPTY(&priv->frags),
1295 			    ("%s: empty q", __func__));
1296 			priv->bundleStats.dropFragments++;
1297 			TAILQ_REMOVE(&priv->frags, qent, f_qent);
1298 			NG_FREE_M(qent->data);
1299 			FREE(qent, M_NETGRAPH_PPP);
1300 			priv->qlen--;
1301 			removed = 1;
1302 		}
1303 	}
1304 	return (removed);
1305 }
1306 
1307 /*
1308  * Run the queue, restoring the queue invariants
1309  */
1310 static int
1311 ng_ppp_frag_process(node_p node)
1312 {
1313 	const priv_p priv = NG_NODE_PRIVATE(node);
1314 	struct mbuf *m;
1315 	item_p item;
1316 
1317 	/* Deliver any deliverable packets */
1318 	while (ng_ppp_check_packet(node)) {
1319 		ng_ppp_get_packet(node, &m);
1320 		item = ng_package_data(m, NULL);
1321 		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
1322 	}
1323 
1324 	/* Delete dead fragments and try again */
1325 	if (ng_ppp_frag_trim(node)) {
1326 		while (ng_ppp_check_packet(node)) {
1327 			ng_ppp_get_packet(node, &m);
1328 			item = ng_package_data(m, NULL);
1329 			ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
1330 		}
1331 	}
1332 
1333 	/* Check for stale fragments while we're here */
1334 	ng_ppp_frag_checkstale(node);
1335 
1336 	/* Check queue length */
1337 	if (priv->qlen > MP_MAX_QUEUE_LEN) {
1338 		struct ng_ppp_frag *qent;
1339 		int i;
1340 
1341 		/* Get oldest fragment */
1342 		KASSERT(!TAILQ_EMPTY(&priv->frags),
1343 		    ("%s: empty q", __func__));
1344 		qent = TAILQ_FIRST(&priv->frags);
1345 
1346 		/* Bump MSEQ if necessary */
1347 		if (MP_RECV_SEQ_DIFF(priv, priv->mseq, qent->seq) < 0) {
1348 			priv->mseq = qent->seq;
1349 			for (i = 0; i < priv->numActiveLinks; i++) {
1350 				struct ng_ppp_link *const alink =
1351 				    &priv->links[priv->activeLinks[i]];
1352 
1353 				if (MP_RECV_SEQ_DIFF(priv,
1354 				    alink->seq, priv->mseq) < 0)
1355 					alink->seq = priv->mseq;
1356 			}
1357 		}
1358 
1359 		/* Drop it */
1360 		priv->bundleStats.dropFragments++;
1361 		TAILQ_REMOVE(&priv->frags, qent, f_qent);
1362 		NG_FREE_M(qent->data);
1363 		FREE(qent, M_NETGRAPH_PPP);
1364 		priv->qlen--;
1365 
1366 		/* Process queue again */
1367 		return ng_ppp_frag_process(node);
1368 	}
1369 
1370 	/* Done */
1371 	return (0);
1372 }
1373 
1374 /*
1375  * Check for 'stale' completed packets that need to be delivered
1376  *
1377  * If a link goes down or has a temporary failure, MSEQ can get
1378  * "stuck", because no new incoming fragments appear on that link.
1379  * This can cause completed packets to never get delivered if
1380  * their sequence numbers are all > MSEQ + 1.
1381  *
1382  * This routine checks how long all of the completed packets have
1383  * been sitting in the queue, and if too long, removes fragments
1384  * from the queue and increments MSEQ to allow them to be delivered.
1385  */
1386 static void
1387 ng_ppp_frag_checkstale(node_p node)
1388 {
1389 	const priv_p priv = NG_NODE_PRIVATE(node);
1390 	struct ng_ppp_frag *qent, *beg, *end;
1391 	struct timeval now, age;
1392 	struct mbuf *m;
1393 	int i, seq;
1394 	item_p item;
1395 	int endseq;
1396 
1397 	now.tv_sec = 0;			/* uninitialized state */
1398 	while (1) {
1399 
1400 		/* If queue is empty, we're done */
1401 		if (TAILQ_EMPTY(&priv->frags))
1402 			break;
1403 
1404 		/* Find the first complete packet in the queue */
1405 		beg = end = NULL;
1406 		seq = TAILQ_FIRST(&priv->frags)->seq;
1407 		TAILQ_FOREACH(qent, &priv->frags, f_qent) {
1408 			if (qent->first)
1409 				beg = qent;
1410 			else if (qent->seq != seq)
1411 				beg = NULL;
1412 			if (beg != NULL && qent->last) {
1413 				end = qent;
1414 				break;
1415 			}
1416 			seq = MP_NEXT_RECV_SEQ(priv, seq);
1417 		}
1418 
1419 		/* If none found, exit */
1420 		if (end == NULL)
1421 			break;
1422 
1423 		/* Get current time (we assume we've been up for >= 1 second) */
1424 		if (now.tv_sec == 0)
1425 			getmicrouptime(&now);
1426 
1427 		/* Check if packet has been queued too long */
1428 		age = now;
1429 		timevalsub(&age, &beg->timestamp);
1430 		if (timevalcmp(&age, &ng_ppp_max_staleness, < ))
1431 			break;
1432 
1433 		/* Throw away junk fragments in front of the completed packet */
1434 		while ((qent = TAILQ_FIRST(&priv->frags)) != beg) {
1435 			KASSERT(!TAILQ_EMPTY(&priv->frags),
1436 			    ("%s: empty q", __func__));
1437 			priv->bundleStats.dropFragments++;
1438 			TAILQ_REMOVE(&priv->frags, qent, f_qent);
1439 			NG_FREE_M(qent->data);
1440 			FREE(qent, M_NETGRAPH_PPP);
1441 			priv->qlen--;
1442 		}
1443 
1444 		/* Extract completed packet */
1445 		endseq = end->seq;
1446 		ng_ppp_get_packet(node, &m);
1447 
1448 		/* Bump MSEQ if necessary */
1449 		if (MP_RECV_SEQ_DIFF(priv, priv->mseq, endseq) < 0) {
1450 			priv->mseq = endseq;
1451 			for (i = 0; i < priv->numActiveLinks; i++) {
1452 				struct ng_ppp_link *const alink =
1453 				    &priv->links[priv->activeLinks[i]];
1454 
1455 				if (MP_RECV_SEQ_DIFF(priv,
1456 				    alink->seq, priv->mseq) < 0)
1457 					alink->seq = priv->mseq;
1458 			}
1459 		}
1460 
1461 		/* Deliver packet */
1462 		item = ng_package_data(m, NULL);
1463 		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
1464 	}
1465 }
1466 
1467 /*
1468  * Periodically call ng_ppp_frag_checkstale()
1469  */
1470 static void
1471 ng_ppp_frag_timeout(void *arg)
1472 {
1473 	const node_p node = arg;
1474 	const priv_p priv = NG_NODE_PRIVATE(node);
1475 	int s = splnet();
1476 
1477 	/* Handle the race where shutdown happens just before splnet() above */
1478 	if (NG_NODE_NOT_VALID(node)) {
1479 		NG_NODE_UNREF(node);
1480 		splx(s);
1481 		return;
1482 	}
1483 
1484 	/* Reset timer state after timeout */
1485 	KASSERT(priv->timerActive, ("%s: !timerActive", __func__));
1486 	priv->timerActive = 0;
1487 	KASSERT(node->nd_refs > 1, ("%s: nd_refs=%d", __func__, node->nd_refs));
1488 	NG_NODE_UNREF(node);
1489 
1490 	/* Start timer again */
1491 	ng_ppp_start_frag_timer(node);
1492 
1493 	/* Scan the fragment queue */
1494 	ng_ppp_frag_checkstale(node);
1495 	splx(s);
1496 }
1497 
1498 /*
1499  * Deliver a frame out on the bundle, i.e., figure out how to fragment
1500  * the frame across the individual PPP links and do so.
1501  */
1502 static int
1503 ng_ppp_mp_output(node_p node, struct mbuf *m)
1504 {
1505 	const priv_p priv = NG_NODE_PRIVATE(node);
1506 	const int hdr_len = priv->conf.xmitShortSeq ? 2 : 4;
1507 	int distrib[NG_PPP_MAX_LINKS];
1508 	int firstFragment;
1509 	int activeLinkNum;
1510 	item_p item;
1511 
1512 	/* At least one link must be active */
1513 	if (priv->numActiveLinks == 0) {
1514 		NG_FREE_M(m);
1515 		return (ENETDOWN);
1516 	}
1517 
1518 	/* Round-robin strategy */
1519 	if (priv->conf.enableRoundRobin || m->m_pkthdr.len < MP_MIN_FRAG_LEN) {
1520 		activeLinkNum = priv->lastLink++ % priv->numActiveLinks;
1521 		bzero(&distrib, priv->numActiveLinks * sizeof(distrib[0]));
1522 		distrib[activeLinkNum] = m->m_pkthdr.len;
1523 		goto deliver;
1524 	}
1525 
1526 	/* Strategy when all links are equivalent (optimize the common case) */
1527 	if (priv->allLinksEqual) {
1528 		const int fraction = m->m_pkthdr.len / priv->numActiveLinks;
1529 		int i, remain;
1530 
1531 		for (i = 0; i < priv->numActiveLinks; i++)
1532 			distrib[priv->lastLink++ % priv->numActiveLinks]
1533 			    = fraction;
1534 		remain = m->m_pkthdr.len - (fraction * priv->numActiveLinks);
1535 		while (remain > 0) {
1536 			distrib[priv->lastLink++ % priv->numActiveLinks]++;
1537 			remain--;
1538 		}
1539 		goto deliver;
1540 	}
1541 
1542 	/* Strategy when all links are not equivalent */
1543 	ng_ppp_mp_strategy(node, m->m_pkthdr.len, distrib);
1544 
1545 deliver:
1546 	/* Update stats */
1547 	priv->bundleStats.xmitFrames++;
1548 	priv->bundleStats.xmitOctets += m->m_pkthdr.len;
1549 
1550 	/* Send alloted portions of frame out on the link(s) */
1551 	for (firstFragment = 1, activeLinkNum = priv->numActiveLinks - 1;
1552 	    activeLinkNum >= 0; activeLinkNum--) {
1553 		const int linkNum = priv->activeLinks[activeLinkNum];
1554 		struct ng_ppp_link *const link = &priv->links[linkNum];
1555 
1556 		/* Deliver fragment(s) out the next link */
1557 		for ( ; distrib[activeLinkNum] > 0; firstFragment = 0) {
1558 			int len, lastFragment, error;
1559 			struct mbuf *m2;
1560 
1561 			/* Calculate fragment length; don't exceed link MTU */
1562 			len = distrib[activeLinkNum];
1563 			if (len > link->conf.mru - hdr_len)
1564 				len = link->conf.mru - hdr_len;
1565 			distrib[activeLinkNum] -= len;
1566 			lastFragment = (len == m->m_pkthdr.len);
1567 
1568 			/* Split off next fragment as "m2" */
1569 			m2 = m;
1570 			if (!lastFragment) {
1571 				struct mbuf *n = m_split(m, len, M_DONTWAIT);
1572 
1573 				if (n == NULL) {
1574 					NG_FREE_M(m);
1575 					return (ENOMEM);
1576 				}
1577 				m = n;
1578 			}
1579 
1580 			/* Prepend MP header */
1581 			if (priv->conf.xmitShortSeq) {
1582 				u_int16_t shdr;
1583 
1584 				shdr = priv->xseq;
1585 				priv->xseq =
1586 				    (priv->xseq + 1) & MP_SHORT_SEQ_MASK;
1587 				if (firstFragment)
1588 					shdr |= MP_SHORT_FIRST_FLAG;
1589 				if (lastFragment)
1590 					shdr |= MP_SHORT_LAST_FLAG;
1591 				shdr = htons(shdr);
1592 				m2 = ng_ppp_prepend(m2, &shdr, 2);
1593 			} else {
1594 				u_int32_t lhdr;
1595 
1596 				lhdr = priv->xseq;
1597 				priv->xseq =
1598 				    (priv->xseq + 1) & MP_LONG_SEQ_MASK;
1599 				if (firstFragment)
1600 					lhdr |= MP_LONG_FIRST_FLAG;
1601 				if (lastFragment)
1602 					lhdr |= MP_LONG_LAST_FLAG;
1603 				lhdr = htonl(lhdr);
1604 				m2 = ng_ppp_prepend(m2, &lhdr, 4);
1605 			}
1606 			if (m2 == NULL) {
1607 				if (!lastFragment)
1608 					m_freem(m);
1609 				return (ENOBUFS);
1610 			}
1611 
1612 			/* Send fragment */
1613 			item = ng_package_data(m2, NULL);
1614 			error = ng_ppp_output(node, 0, PROT_MP, linkNum, item);
1615 			if (error != 0) {
1616 				if (!lastFragment)
1617 					NG_FREE_M(m);
1618 				return (error);
1619 			}
1620 		}
1621 	}
1622 
1623 	/* Done */
1624 	return (0);
1625 }
1626 
1627 /*
1628  * Computing the optimal fragmentation
1629  * -----------------------------------
1630  *
1631  * This routine tries to compute the optimal fragmentation pattern based
1632  * on each link's latency, bandwidth, and calculated additional latency.
1633  * The latter quantity is the additional latency caused by previously
1634  * written data that has not been transmitted yet.
1635  *
1636  * This algorithm is only useful when not all of the links have the
1637  * same latency and bandwidth values.
1638  *
1639  * The essential idea is to make the last bit of each fragment of the
1640  * frame arrive at the opposite end at the exact same time. This greedy
1641  * algorithm is optimal, in that no other scheduling could result in any
1642  * packet arriving any sooner unless packets are delivered out of order.
1643  *
1644  * Suppose link i has bandwidth b_i (in tens of bytes per milisecond) and
1645  * latency l_i (in miliseconds). Consider the function function f_i(t)
1646  * which is equal to the number of bytes that will have arrived at
1647  * the peer after t miliseconds if we start writing continuously at
1648  * time t = 0. Then f_i(t) = b_i * (t - l_i) = ((b_i * t) - (l_i * b_i).
1649  * That is, f_i(t) is a line with slope b_i and y-intersect -(l_i * b_i).
1650  * Note that the y-intersect is always <= zero because latency can't be
1651  * negative.  Note also that really the function is f_i(t) except when
1652  * f_i(t) is negative, in which case the function is zero.  To take
1653  * care of this, let Q_i(t) = { if (f_i(t) > 0) return 1; else return 0; }.
1654  * So the actual number of bytes that will have arrived at the peer after
1655  * t miliseconds is f_i(t) * Q_i(t).
1656  *
1657  * At any given time, each link has some additional latency a_i >= 0
1658  * due to previously written fragment(s) which are still in the queue.
1659  * This value is easily computed from the time since last transmission,
1660  * the previous latency value, the number of bytes written, and the
1661  * link's bandwidth.
1662  *
1663  * Assume that l_i includes any a_i already, and that the links are
1664  * sorted by latency, so that l_i <= l_{i+1}.
1665  *
1666  * Let N be the total number of bytes in the current frame we are sending.
1667  *
1668  * Suppose we were to start writing bytes at time t = 0 on all links
1669  * simultaneously, which is the most we can possibly do.  Then let
1670  * F(t) be equal to the total number of bytes received by the peer
1671  * after t miliseconds. Then F(t) = Sum_i (f_i(t) * Q_i(t)).
1672  *
1673  * Our goal is simply this: fragment the frame across the links such
1674  * that the peer is able to reconstruct the completed frame as soon as
1675  * possible, i.e., at the least possible value of t. Call this value t_0.
1676  *
1677  * Then it follows that F(t_0) = N. Our strategy is first to find the value
1678  * of t_0, and then deduce how many bytes to write to each link.
1679  *
1680  * Rewriting F(t_0):
1681  *
1682  *   t_0 = ( N + Sum_i ( l_i * b_i * Q_i(t_0) ) ) / Sum_i ( b_i * Q_i(t_0) )
1683  *
1684  * Now, we note that Q_i(t) is constant for l_i <= t <= l_{i+1}. t_0 will
1685  * lie in one of these ranges.  To find it, we just need to find the i such
1686  * that F(l_i) <= N <= F(l_{i+1}).  Then we compute all the constant values
1687  * for Q_i() in this range, plug in the remaining values, solving for t_0.
1688  *
1689  * Once t_0 is known, then the number of bytes to send on link i is
1690  * just f_i(t_0) * Q_i(t_0).
1691  *
1692  * In other words, we start allocating bytes to the links one at a time.
1693  * We keep adding links until the frame is completely sent.  Some links
1694  * may not get any bytes because their latency is too high.
1695  *
1696  * Is all this work really worth the trouble?  Depends on the situation.
1697  * The bigger the ratio of computer speed to link speed, and the more
1698  * important total bundle latency is (e.g., for interactive response time),
1699  * the more it's worth it.  There is however the cost of calling this
1700  * function for every frame.  The running time is O(n^2) where n is the
1701  * number of links that receive a non-zero number of bytes.
1702  *
1703  * Since latency is measured in miliseconds, the "resolution" of this
1704  * algorithm is one milisecond.
1705  *
1706  * To avoid this algorithm altogether, configure all links to have the
1707  * same latency and bandwidth.
1708  */
1709 static void
1710 ng_ppp_mp_strategy(node_p node, int len, int *distrib)
1711 {
1712 	const priv_p priv = NG_NODE_PRIVATE(node);
1713 	int latency[NG_PPP_MAX_LINKS];
1714 	int sortByLatency[NG_PPP_MAX_LINKS];
1715 	int activeLinkNum;
1716 	int t0, total, topSum, botSum;
1717 	struct timeval now;
1718 	int i, numFragments;
1719 
1720 	/* If only one link, this gets real easy */
1721 	if (priv->numActiveLinks == 1) {
1722 		distrib[0] = len;
1723 		return;
1724 	}
1725 
1726 	/* Get current time */
1727 	getmicrouptime(&now);
1728 
1729 	/* Compute latencies for each link at this point in time */
1730 	for (activeLinkNum = 0;
1731 	    activeLinkNum < priv->numActiveLinks; activeLinkNum++) {
1732 		struct ng_ppp_link *alink;
1733 		struct timeval diff;
1734 		int xmitBytes;
1735 
1736 		/* Start with base latency value */
1737 		alink = &priv->links[priv->activeLinks[activeLinkNum]];
1738 		latency[activeLinkNum] = alink->conf.latency;
1739 		sortByLatency[activeLinkNum] = activeLinkNum;	/* see below */
1740 
1741 		/* Any additional latency? */
1742 		if (alink->bytesInQueue == 0)
1743 			continue;
1744 
1745 		/* Compute time delta since last write */
1746 		diff = now;
1747 		timevalsub(&diff, &alink->lastWrite);
1748 		if (now.tv_sec < 0 || diff.tv_sec >= 10) {	/* sanity */
1749 			alink->bytesInQueue = 0;
1750 			continue;
1751 		}
1752 
1753 		/* How many bytes could have transmitted since last write? */
1754 		xmitBytes = (alink->conf.bandwidth * diff.tv_sec)
1755 		    + (alink->conf.bandwidth * (diff.tv_usec / 1000)) / 100;
1756 		alink->bytesInQueue -= xmitBytes;
1757 		if (alink->bytesInQueue < 0)
1758 			alink->bytesInQueue = 0;
1759 		else
1760 			latency[activeLinkNum] +=
1761 			    (100 * alink->bytesInQueue) / alink->conf.bandwidth;
1762 	}
1763 
1764 	/* Sort active links by latency */
1765 	qsort_r(sortByLatency,
1766 	    priv->numActiveLinks, sizeof(*sortByLatency), latency, ng_ppp_intcmp);
1767 
1768 	/* Find the interval we need (add links in sortByLatency[] order) */
1769 	for (numFragments = 1;
1770 	    numFragments < priv->numActiveLinks; numFragments++) {
1771 		for (total = i = 0; i < numFragments; i++) {
1772 			int flowTime;
1773 
1774 			flowTime = latency[sortByLatency[numFragments]]
1775 			    - latency[sortByLatency[i]];
1776 			total += ((flowTime * priv->links[
1777 			    priv->activeLinks[sortByLatency[i]]].conf.bandwidth)
1778 			    	+ 99) / 100;
1779 		}
1780 		if (total >= len)
1781 			break;
1782 	}
1783 
1784 	/* Solve for t_0 in that interval */
1785 	for (topSum = botSum = i = 0; i < numFragments; i++) {
1786 		int bw = priv->links[
1787 		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
1788 
1789 		topSum += latency[sortByLatency[i]] * bw;	/* / 100 */
1790 		botSum += bw;					/* / 100 */
1791 	}
1792 	t0 = ((len * 100) + topSum + botSum / 2) / botSum;
1793 
1794 	/* Compute f_i(t_0) all i */
1795 	bzero(distrib, priv->numActiveLinks * sizeof(*distrib));
1796 	for (total = i = 0; i < numFragments; i++) {
1797 		int bw = priv->links[
1798 		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
1799 
1800 		distrib[sortByLatency[i]] =
1801 		    (bw * (t0 - latency[sortByLatency[i]]) + 50) / 100;
1802 		total += distrib[sortByLatency[i]];
1803 	}
1804 
1805 	/* Deal with any rounding error */
1806 	if (total < len) {
1807 		struct ng_ppp_link *fastLink =
1808 		    &priv->links[priv->activeLinks[sortByLatency[0]]];
1809 		int fast = 0;
1810 
1811 		/* Find the fastest link */
1812 		for (i = 1; i < numFragments; i++) {
1813 			struct ng_ppp_link *const link =
1814 			    &priv->links[priv->activeLinks[sortByLatency[i]]];
1815 
1816 			if (link->conf.bandwidth > fastLink->conf.bandwidth) {
1817 				fast = i;
1818 				fastLink = link;
1819 			}
1820 		}
1821 		distrib[sortByLatency[fast]] += len - total;
1822 	} else while (total > len) {
1823 		struct ng_ppp_link *slowLink =
1824 		    &priv->links[priv->activeLinks[sortByLatency[0]]];
1825 		int delta, slow = 0;
1826 
1827 		/* Find the slowest link that still has bytes to remove */
1828 		for (i = 1; i < numFragments; i++) {
1829 			struct ng_ppp_link *const link =
1830 			    &priv->links[priv->activeLinks[sortByLatency[i]]];
1831 
1832 			if (distrib[sortByLatency[slow]] == 0
1833 			  || (distrib[sortByLatency[i]] > 0
1834 			    && link->conf.bandwidth <
1835 			      slowLink->conf.bandwidth)) {
1836 				slow = i;
1837 				slowLink = link;
1838 			}
1839 		}
1840 		delta = total - len;
1841 		if (delta > distrib[sortByLatency[slow]])
1842 			delta = distrib[sortByLatency[slow]];
1843 		distrib[sortByLatency[slow]] -= delta;
1844 		total -= delta;
1845 	}
1846 }
1847 
1848 /*
1849  * Compare two integers
1850  */
1851 static int
1852 ng_ppp_intcmp(void *latency, const void *v1, const void *v2)
1853 {
1854 	const int index1 = *((const int *) v1);
1855 	const int index2 = *((const int *) v2);
1856 
1857 	return ((int *)latency)[index1] - ((int *)latency)[index2];
1858 }
1859 
1860 /*
1861  * Prepend a possibly compressed PPP protocol number in front of a frame
1862  */
1863 static struct mbuf *
1864 ng_ppp_addproto(struct mbuf *m, int proto, int compOK)
1865 {
1866 	if (compOK && PROT_COMPRESSABLE(proto)) {
1867 		u_char pbyte = (u_char)proto;
1868 
1869 		return ng_ppp_prepend(m, &pbyte, 1);
1870 	} else {
1871 		u_int16_t pword = htons((u_int16_t)proto);
1872 
1873 		return ng_ppp_prepend(m, &pword, 2);
1874 	}
1875 }
1876 
1877 /*
1878  * Prepend some bytes to an mbuf
1879  */
1880 static struct mbuf *
1881 ng_ppp_prepend(struct mbuf *m, const void *buf, int len)
1882 {
1883 	M_PREPEND(m, len, M_DONTWAIT);
1884 	if (m == NULL || (m->m_len < len && (m = m_pullup(m, len)) == NULL))
1885 		return (NULL);
1886 	bcopy(buf, mtod(m, u_char *), len);
1887 	return (m);
1888 }
1889 
1890 /*
1891  * Update private information that is derived from other private information
1892  */
1893 static void
1894 ng_ppp_update(node_p node, int newConf)
1895 {
1896 	const priv_p priv = NG_NODE_PRIVATE(node);
1897 	int i;
1898 
1899 	/* Update active status for VJ Compression */
1900 	priv->vjCompHooked = priv->hooks[HOOK_INDEX_VJC_IP] != NULL
1901 	    && priv->hooks[HOOK_INDEX_VJC_COMP] != NULL
1902 	    && priv->hooks[HOOK_INDEX_VJC_UNCOMP] != NULL
1903 	    && priv->hooks[HOOK_INDEX_VJC_VJIP] != NULL;
1904 
1905 	/* Increase latency for each link an amount equal to one MP header */
1906 	if (newConf) {
1907 		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1908 			int hdrBytes;
1909 
1910 			hdrBytes = (priv->links[i].conf.enableACFComp ? 0 : 2)
1911 			    + (priv->links[i].conf.enableProtoComp ? 1 : 2)
1912 			    + (priv->conf.xmitShortSeq ? 2 : 4);
1913 			priv->links[i].conf.latency +=
1914 			    ((hdrBytes * priv->links[i].conf.bandwidth) + 50)
1915 				/ 100;
1916 		}
1917 	}
1918 
1919 	/* Update list of active links */
1920 	bzero(&priv->activeLinks, sizeof(priv->activeLinks));
1921 	priv->numActiveLinks = 0;
1922 	priv->allLinksEqual = 1;
1923 	for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1924 		struct ng_ppp_link *const link = &priv->links[i];
1925 
1926 		/* Is link active? */
1927 		if (link->conf.enableLink && link->hook != NULL) {
1928 			struct ng_ppp_link *link0;
1929 
1930 			/* Add link to list of active links */
1931 			priv->activeLinks[priv->numActiveLinks++] = i;
1932 			link0 = &priv->links[priv->activeLinks[0]];
1933 
1934 			/* Determine if all links are still equal */
1935 			if (link->conf.latency != link0->conf.latency
1936 			  || link->conf.bandwidth != link0->conf.bandwidth)
1937 				priv->allLinksEqual = 0;
1938 
1939 			/* Initialize rec'd sequence number */
1940 			if (link->seq == MP_NOSEQ) {
1941 				link->seq = (link == link0) ?
1942 				    MP_INITIAL_SEQ : link0->seq;
1943 			}
1944 		} else
1945 			link->seq = MP_NOSEQ;
1946 	}
1947 
1948 	/* Update MP state as multi-link is active or not */
1949 	if (priv->conf.enableMultilink && priv->numActiveLinks > 0)
1950 		ng_ppp_start_frag_timer(node);
1951 	else {
1952 		ng_ppp_stop_frag_timer(node);
1953 		ng_ppp_frag_reset(node);
1954 		priv->xseq = MP_INITIAL_SEQ;
1955 		priv->mseq = MP_INITIAL_SEQ;
1956 		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
1957 			struct ng_ppp_link *const link = &priv->links[i];
1958 
1959 			bzero(&link->lastWrite, sizeof(link->lastWrite));
1960 			link->bytesInQueue = 0;
1961 			link->seq = MP_NOSEQ;
1962 		}
1963 	}
1964 }
1965 
1966 /*
1967  * Determine if a new configuration would represent a valid change
1968  * from the current configuration and link activity status.
1969  */
1970 static int
1971 ng_ppp_config_valid(node_p node, const struct ng_ppp_node_conf *newConf)
1972 {
1973 	const priv_p priv = NG_NODE_PRIVATE(node);
1974 	int i, newNumLinksActive;
1975 
1976 	/* Check per-link config and count how many links would be active */
1977 	for (newNumLinksActive = i = 0; i < NG_PPP_MAX_LINKS; i++) {
1978 		if (newConf->links[i].enableLink && priv->links[i].hook != NULL)
1979 			newNumLinksActive++;
1980 		if (!newConf->links[i].enableLink)
1981 			continue;
1982 		if (newConf->links[i].mru < MP_MIN_LINK_MRU)
1983 			return (0);
1984 		if (newConf->links[i].bandwidth == 0)
1985 			return (0);
1986 		if (newConf->links[i].bandwidth > NG_PPP_MAX_BANDWIDTH)
1987 			return (0);
1988 		if (newConf->links[i].latency > NG_PPP_MAX_LATENCY)
1989 			return (0);
1990 	}
1991 
1992 	/* Check bundle parameters */
1993 	if (newConf->bund.enableMultilink && newConf->bund.mrru < MP_MIN_MRRU)
1994 		return (0);
1995 
1996 	/* Disallow changes to multi-link configuration while MP is active */
1997 	if (priv->numActiveLinks > 0 && newNumLinksActive > 0) {
1998 		if (!priv->conf.enableMultilink
1999 				!= !newConf->bund.enableMultilink
2000 		    || !priv->conf.xmitShortSeq != !newConf->bund.xmitShortSeq
2001 		    || !priv->conf.recvShortSeq != !newConf->bund.recvShortSeq)
2002 			return (0);
2003 	}
2004 
2005 	/* At most one link can be active unless multi-link is enabled */
2006 	if (!newConf->bund.enableMultilink && newNumLinksActive > 1)
2007 		return (0);
2008 
2009 	/* Configuration change would be valid */
2010 	return (1);
2011 }
2012 
2013 /*
2014  * Free all entries in the fragment queue
2015  */
2016 static void
2017 ng_ppp_frag_reset(node_p node)
2018 {
2019 	const priv_p priv = NG_NODE_PRIVATE(node);
2020 	struct ng_ppp_frag *qent, *qnext;
2021 
2022 	for (qent = TAILQ_FIRST(&priv->frags); qent; qent = qnext) {
2023 		qnext = TAILQ_NEXT(qent, f_qent);
2024 		NG_FREE_M(qent->data);
2025 		FREE(qent, M_NETGRAPH_PPP);
2026 	}
2027 	TAILQ_INIT(&priv->frags);
2028 	priv->qlen = 0;
2029 }
2030 
2031 /*
2032  * Start fragment queue timer
2033  */
2034 static void
2035 ng_ppp_start_frag_timer(node_p node)
2036 {
2037 	const priv_p priv = NG_NODE_PRIVATE(node);
2038 
2039 	if (!priv->timerActive) {
2040 		priv->fragTimer = timeout(ng_ppp_frag_timeout,
2041 		    node, MP_FRAGTIMER_INTERVAL);
2042 		priv->timerActive = 1;
2043 		NG_NODE_REF(node);
2044 	}
2045 }
2046 
2047 /*
2048  * Stop fragment queue timer
2049  */
2050 static void
2051 ng_ppp_stop_frag_timer(node_p node)
2052 {
2053 	const priv_p priv = NG_NODE_PRIVATE(node);
2054 
2055 	if (priv->timerActive) {
2056 		untimeout(ng_ppp_frag_timeout, node, priv->fragTimer);
2057 		priv->timerActive = 0;
2058 		KASSERT(node->nd_refs > 1,
2059 		    ("%s: nd_refs=%d", __func__, node->nd_refs));
2060 		NG_NODE_UNREF(node);
2061 	}
2062 }
2063 
2064