xref: /freebsd/sys/netgraph/ng_base.c (revision c68159a6d8eede11766cf13896d0f7670dbd51aa)
1 /*
2  * ng_base.c
3  *
4  * Copyright (c) 1996-1999 Whistle Communications, Inc.
5  * All rights reserved.
6  *
7  * Subject to the following obligations and disclaimer of warranty, use and
8  * redistribution of this software, in source or object code forms, with or
9  * without modifications are expressly permitted by Whistle Communications;
10  * provided, however, that:
11  * 1. Any and all reproductions of the source or object code must include the
12  *    copyright notice above and the following disclaimer of warranties; and
13  * 2. No rights are granted, in any manner or form, to use Whistle
14  *    Communications, Inc. trademarks, including the mark "WHISTLE
15  *    COMMUNICATIONS" on advertising, endorsements, or otherwise except as
16  *    such appears in the above copyright notice or in the software.
17  *
18  * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
19  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
20  * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
21  * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
23  * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
24  * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
25  * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
26  * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
27  * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
28  * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
29  * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
30  * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
31  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33  * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
34  * OF SUCH DAMAGE.
35  *
36  * Authors: Julian Elischer <julian@freebsd.org>
37  *          Archie Cobbs <archie@freebsd.org>
38  *
39  * $FreeBSD$
40  * $Whistle: ng_base.c,v 1.39 1999/01/28 23:54:53 julian Exp $
41  */
42 
43 /*
44  * This file implements the base netgraph code.
45  */
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/errno.h>
50 #include <sys/kernel.h>
51 #include <sys/malloc.h>
52 #include <sys/syslog.h>
53 #include <sys/sysctl.h>
54 #include <sys/linker.h>
55 #include <sys/queue.h>
56 #include <sys/mbuf.h>
57 #include <sys/ctype.h>
58 #include <machine/limits.h>
59 
60 #include <net/netisr.h>
61 
62 #include <netgraph/ng_message.h>
63 #include <netgraph/netgraph.h>
64 #include <netgraph/ng_parse.h>
65 
66 MODULE_VERSION(netgraph, 1);
67 
68 /* List of all nodes */
69 static LIST_HEAD(, ng_node) ng_nodelist;
70 static struct mtx	ng_nodelist_mtx;
71 
72 /* NETISR queue */
73 /* List nodes with unallocated work */
74 static TAILQ_HEAD(, ng_node) ng_worklist = TAILQ_HEAD_INITIALIZER(ng_worklist);
75 static struct mtx	ng_worklist_mtx;
76 
77 /* List of installed types */
78 static LIST_HEAD(, ng_type) ng_typelist;
79 static struct mtx	ng_typelist_mtx;
80 
81 /* Hash related definitions */
82 /* Don't nead to initialise them because it's a LIST */
83 #define ID_HASH_SIZE 32 /* most systems wont need even this many */
84 static LIST_HEAD(, ng_node) ng_ID_hash[ID_HASH_SIZE];
85 static struct mtx	ng_idhash_mtx;
86 
87 /* Mutex that protects the free queue item list */
88 static volatile item_p		ngqfree;	/* free ones */
89 static struct mtx	ngq_mtx;
90 
91 /* Internal functions */
92 static int	ng_add_hook(node_p node, const char *name, hook_p * hookp);
93 static int	ng_connect(hook_p hook1, hook_p hook2);
94 static void	ng_disconnect_hook(hook_p hook);
95 static int	ng_generic_msg(node_p here, item_p item, hook_p lasthook);
96 static ng_ID_t	ng_decodeidname(const char *name);
97 static int	ngb_mod_event(module_t mod, int event, void *data);
98 static void	ng_worklist_remove(node_p node);
99 static void	ngintr(void);
100 static int	ng_apply_item(node_p node, item_p item);
101 static void	ng_flush_input_queue(struct ng_queue * ngq);
102 static void	ng_setisr(node_p node);
103 static node_p	ng_ID2noderef(ng_ID_t ID);
104 
105 /* imported */
106 int	ng_bypass(hook_p hook1, hook_p hook2);
107 void	ng_cutlinks(node_p node);
108 int	ng_con_nodes(node_p node, const char *name, node_p node2,
109 	const char *name2);
110 void	ng_destroy_hook(hook_p hook);
111 node_p	ng_name2noderef(node_p node, const char *name);
112 int	ng_path2noderef(node_p here, const char *path,
113 	node_p *dest, hook_p *lasthook);
114 struct	ng_type *ng_findtype(const char *type);
115 int	ng_make_node(const char *type, node_p *nodepp);
116 int	ng_mkpeer(node_p node, const char *name, const char *name2, char *type);
117 int	ng_path_parse(char *addr, char **node, char **path, char **hook);
118 void	ng_rmnode(node_p node);
119 
120 
121 /* Our own netgraph malloc type */
122 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages");
123 MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", "netgraph hook structures");
124 MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", "netgraph node structures");
125 MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", "netgraph item structures");
126 MALLOC_DEFINE(M_NETGRAPH_META, "netgraph_meta", "netgraph name storage");
127 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage");
128 
129 /* Should not be visible outside this file */
130 #define NG_FREE_HOOK(hook) do { FREE((hook), M_NETGRAPH_HOOK); } while (0)
131 #define NG_FREE_NODE(node) do { FREE((node), M_NETGRAPH_NODE); } while (0)
132 #define NG_FREE_NAME(name) do { FREE((name), M_NETGRAPH_NAME); } while (0)
133 /* Warning: Generally use NG_FREE_ITEM() instead */
134 #define NG_FREE_ITEM_REAL(item) do { FREE((item), M_NETGRAPH_ITEM); } while (0)
135 
136 
137 /* Set this to Debugger("X") to catch all errors as they occur */
138 #ifndef TRAP_ERROR
139 #define TRAP_ERROR
140 #endif
141 
142 static	ng_ID_t nextID = 1;
143 
144 #ifdef INVARIANTS
145 #define CHECK_DATA_MBUF(m)	do {					\
146 		struct mbuf *n;						\
147 		int total;						\
148 									\
149 		if (((m)->m_flags & M_PKTHDR) == 0)			\
150 			panic("%s: !PKTHDR", __FUNCTION__);		\
151 		for (total = 0, n = (m); n != NULL; n = n->m_next)	\
152 			total += n->m_len;				\
153 		if ((m)->m_pkthdr.len != total) {			\
154 			panic("%s: %d != %d",				\
155 			    __FUNCTION__, (m)->m_pkthdr.len, total);	\
156 		}							\
157 	} while (0)
158 #else
159 #define CHECK_DATA_MBUF(m)
160 #endif
161 
162 
163 /************************************************************************
164 	Parse type definitions for generic messages
165 ************************************************************************/
166 
167 /* Handy structure parse type defining macro */
168 #define DEFINE_PARSE_STRUCT_TYPE(lo, up, args)				\
169 static const struct ng_parse_struct_info				\
170 	ng_ ## lo ## _type_info = NG_GENERIC_ ## up ## _INFO args;	\
171 static const struct ng_parse_type ng_generic_ ## lo ## _type = {	\
172 	&ng_parse_struct_type,						\
173 	&ng_ ## lo ## _type_info					\
174 }
175 
176 DEFINE_PARSE_STRUCT_TYPE(mkpeer, MKPEER, ());
177 DEFINE_PARSE_STRUCT_TYPE(connect, CONNECT, ());
178 DEFINE_PARSE_STRUCT_TYPE(name, NAME, ());
179 DEFINE_PARSE_STRUCT_TYPE(rmhook, RMHOOK, ());
180 DEFINE_PARSE_STRUCT_TYPE(nodeinfo, NODEINFO, ());
181 DEFINE_PARSE_STRUCT_TYPE(typeinfo, TYPEINFO, ());
182 DEFINE_PARSE_STRUCT_TYPE(linkinfo, LINKINFO, (&ng_generic_nodeinfo_type));
183 
184 /* Get length of an array when the length is stored as a 32 bit
185    value immediately preceeding the array -- as with struct namelist
186    and struct typelist. */
187 static int
188 ng_generic_list_getLength(const struct ng_parse_type *type,
189 	const u_char *start, const u_char *buf)
190 {
191 	return *((const u_int32_t *)(buf - 4));
192 }
193 
194 /* Get length of the array of struct linkinfo inside a struct hooklist */
195 static int
196 ng_generic_linkinfo_getLength(const struct ng_parse_type *type,
197 	const u_char *start, const u_char *buf)
198 {
199 	const struct hooklist *hl = (const struct hooklist *)start;
200 
201 	return hl->nodeinfo.hooks;
202 }
203 
204 /* Array type for a variable length array of struct namelist */
205 static const struct ng_parse_array_info ng_nodeinfoarray_type_info = {
206 	&ng_generic_nodeinfo_type,
207 	&ng_generic_list_getLength
208 };
209 static const struct ng_parse_type ng_generic_nodeinfoarray_type = {
210 	&ng_parse_array_type,
211 	&ng_nodeinfoarray_type_info
212 };
213 
214 /* Array type for a variable length array of struct typelist */
215 static const struct ng_parse_array_info ng_typeinfoarray_type_info = {
216 	&ng_generic_typeinfo_type,
217 	&ng_generic_list_getLength
218 };
219 static const struct ng_parse_type ng_generic_typeinfoarray_type = {
220 	&ng_parse_array_type,
221 	&ng_typeinfoarray_type_info
222 };
223 
224 /* Array type for array of struct linkinfo in struct hooklist */
225 static const struct ng_parse_array_info ng_generic_linkinfo_array_type_info = {
226 	&ng_generic_linkinfo_type,
227 	&ng_generic_linkinfo_getLength
228 };
229 static const struct ng_parse_type ng_generic_linkinfo_array_type = {
230 	&ng_parse_array_type,
231 	&ng_generic_linkinfo_array_type_info
232 };
233 
234 DEFINE_PARSE_STRUCT_TYPE(typelist, TYPELIST, (&ng_generic_nodeinfoarray_type));
235 DEFINE_PARSE_STRUCT_TYPE(hooklist, HOOKLIST,
236 	(&ng_generic_nodeinfo_type, &ng_generic_linkinfo_array_type));
237 DEFINE_PARSE_STRUCT_TYPE(listnodes, LISTNODES,
238 	(&ng_generic_nodeinfoarray_type));
239 
240 /* List of commands and how to convert arguments to/from ASCII */
241 static const struct ng_cmdlist ng_generic_cmds[] = {
242 	{
243 	  NGM_GENERIC_COOKIE,
244 	  NGM_SHUTDOWN,
245 	  "shutdown",
246 	  NULL,
247 	  NULL
248 	},
249 	{
250 	  NGM_GENERIC_COOKIE,
251 	  NGM_MKPEER,
252 	  "mkpeer",
253 	  &ng_generic_mkpeer_type,
254 	  NULL
255 	},
256 	{
257 	  NGM_GENERIC_COOKIE,
258 	  NGM_CONNECT,
259 	  "connect",
260 	  &ng_generic_connect_type,
261 	  NULL
262 	},
263 	{
264 	  NGM_GENERIC_COOKIE,
265 	  NGM_NAME,
266 	  "name",
267 	  &ng_generic_name_type,
268 	  NULL
269 	},
270 	{
271 	  NGM_GENERIC_COOKIE,
272 	  NGM_RMHOOK,
273 	  "rmhook",
274 	  &ng_generic_rmhook_type,
275 	  NULL
276 	},
277 	{
278 	  NGM_GENERIC_COOKIE,
279 	  NGM_NODEINFO,
280 	  "nodeinfo",
281 	  NULL,
282 	  &ng_generic_nodeinfo_type
283 	},
284 	{
285 	  NGM_GENERIC_COOKIE,
286 	  NGM_LISTHOOKS,
287 	  "listhooks",
288 	  NULL,
289 	  &ng_generic_hooklist_type
290 	},
291 	{
292 	  NGM_GENERIC_COOKIE,
293 	  NGM_LISTNAMES,
294 	  "listnames",
295 	  NULL,
296 	  &ng_generic_listnodes_type	/* same as NGM_LISTNODES */
297 	},
298 	{
299 	  NGM_GENERIC_COOKIE,
300 	  NGM_LISTNODES,
301 	  "listnodes",
302 	  NULL,
303 	  &ng_generic_listnodes_type
304 	},
305 	{
306 	  NGM_GENERIC_COOKIE,
307 	  NGM_LISTTYPES,
308 	  "listtypes",
309 	  NULL,
310 	  &ng_generic_typeinfo_type
311 	},
312 	{
313 	  NGM_GENERIC_COOKIE,
314 	  NGM_TEXT_CONFIG,
315 	  "textconfig",
316 	  NULL,
317 	  &ng_parse_string_type
318 	},
319 	{
320 	  NGM_GENERIC_COOKIE,
321 	  NGM_TEXT_STATUS,
322 	  "textstatus",
323 	  NULL,
324 	  &ng_parse_string_type
325 	},
326 	{
327 	  NGM_GENERIC_COOKIE,
328 	  NGM_ASCII2BINARY,
329 	  "ascii2binary",
330 	  &ng_parse_ng_mesg_type,
331 	  &ng_parse_ng_mesg_type
332 	},
333 	{
334 	  NGM_GENERIC_COOKIE,
335 	  NGM_BINARY2ASCII,
336 	  "binary2ascii",
337 	  &ng_parse_ng_mesg_type,
338 	  &ng_parse_ng_mesg_type
339 	},
340 	{ 0 }
341 };
342 
343 /************************************************************************
344 			Node routines
345 ************************************************************************/
346 
347 /*
348  * Instantiate a node of the requested type
349  */
350 int
351 ng_make_node(const char *typename, node_p *nodepp)
352 {
353 	struct ng_type *type;
354 	int	error;
355 
356 	/* Check that the type makes sense */
357 	if (typename == NULL) {
358 		TRAP_ERROR;
359 		return (EINVAL);
360 	}
361 
362 	/* Locate the node type */
363 	if ((type = ng_findtype(typename)) == NULL) {
364 		char filename[NG_TYPELEN + 4];
365 		linker_file_t lf;
366 		int error;
367 
368 		/* Not found, try to load it as a loadable module */
369 		snprintf(filename, sizeof(filename), "ng_%s", typename);
370 		error = linker_load_file(filename, &lf);
371 		if (error != 0)
372 			return (error);
373 		lf->userrefs++;		/* pretend loaded by the syscall */
374 
375 		/* Try again, as now the type should have linked itself in */
376 		if ((type = ng_findtype(typename)) == NULL)
377 			return (ENXIO);
378 	}
379 
380 	/*
381 	 * If we have a constructor, then make the node and
382 	 * call the constructor to do type specific initialisation.
383 	 */
384 	if (type->constructor != NULL) {
385 		if ((error = ng_make_node_common(type, nodepp)) == 0) {
386 			if ((error = ((*type->constructor)(*nodepp)) != 0)) {
387 				ng_unref(*nodepp);
388 			}
389 		}
390 	} else {
391 		/*
392 		 * Node has no constructor. We cannot ask for one
393 		 * to be made. It must be brought into existance by
394 		 * some external agency. The external acency should
395 		 * call ng_make_node_common() directly to get the
396 		 * netgraph part initialised.
397 		 */
398 		error = EINVAL;
399 	}
400 	return (error);
401 }
402 
403 /*
404  * Generic node creation. Called by node initialisation for externally
405  * instantiated nodes (e.g. hardware, sockets, etc ).
406  * The returned node has a reference count of 1.
407  */
408 int
409 ng_make_node_common(struct ng_type *type, node_p *nodepp)
410 {
411 	node_p node;
412 
413 	/* Require the node type to have been already installed */
414 	if (ng_findtype(type->name) == NULL) {
415 		TRAP_ERROR;
416 		return (EINVAL);
417 	}
418 
419 	/* Make a node and try attach it to the type */
420 	MALLOC(node, node_p, sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO);
421 	if (node == NULL) {
422 		TRAP_ERROR;
423 		return (ENOMEM);
424 	}
425 	node->type = type;
426 	node->refs++;				/* note reference */
427 	type->refs++;
428 
429 	mtx_init(&node->input_queue.q_mtx, "netgraph node mutex", 0);
430 	node->input_queue.queue = NULL;
431 	node->input_queue.last = &node->input_queue.queue;
432 	node->input_queue.q_flags = 0;
433 	node->input_queue.q_node = node;
434 
435 	/* Initialize hook list for new node */
436 	LIST_INIT(&node->hooks);
437 
438 	/* Link us into the node linked list */
439 	mtx_enter(&ng_nodelist_mtx, MTX_DEF);
440 	LIST_INSERT_HEAD(&ng_nodelist, node, nodes);
441 	mtx_exit(&ng_nodelist_mtx, MTX_DEF);
442 
443 
444 	/* get an ID and put us in the hash chain */
445 	mtx_enter(&ng_idhash_mtx, MTX_DEF);
446 	do { /* wrap protection, even if silly */
447 		node_p node2 = NULL;
448 		node->ID = nextID++; /* 137 per second for 1 year before wrap */
449 		if ((node->ID == 0) || (node2 = ng_ID2noderef(node->ID))) {
450 			if (node2) {
451 				ng_unref(node2);
452 				node2 = NULL;
453 			}
454 			continue;	/* try again */
455 		}
456 	} while (0);
457 	LIST_INSERT_HEAD(&ng_ID_hash[node->ID % ID_HASH_SIZE], node, idnodes);
458 	mtx_exit(&ng_idhash_mtx, MTX_DEF);
459 
460 	/* Done */
461 	*nodepp = node;
462 	return (0);
463 }
464 
465 /*
466  * Forceably start the shutdown process on a node. Either call
467  * it's shutdown method, or do the default shutdown if there is
468  * no type-specific method.
469  *
470  * We can only be called form a shutdown message, so we know we have
471  * a writer lock, and therefore exclusive access.
472  *
473  * Persistent node types must have a type-specific method which
474  * Allocates a new node. This one is irretrievably going away.
475  */
476 void
477 ng_rmnode(node_p node)
478 {
479 	/* Check if it's already shutting down */
480 	if ((node->flags & NG_CLOSING) != 0)
481 		return;
482 
483 	/* Add an extra reference so it doesn't go away during this */
484 	node->refs++;
485 
486 	/* Mark it invalid so any newcomers know not to try use it */
487 	node->flags |= NG_INVALID|NG_CLOSING;
488 
489 	ng_unname(node);
490 	ng_cutlinks(node);
491 	/*
492 	 * Drain the input queue forceably.
493 	 */
494 	ng_flush_input_queue(&node->input_queue);
495 
496 	/*
497 	 * Take us off the work queue if we are there.
498 	 */
499 	ng_worklist_remove(node);
500 
501 
502 	/* Ask the type if it has anything to do in this case */
503 	if (node->type && node->type->shutdown) {
504 		(*node->type->shutdown)(node);
505 	} else {				/* do the default thing */
506 		ng_unref(node); /* XXX hmmmmm check this */
507 	}
508 
509 	/* Remove extra reference, possibly the last */
510 	ng_unref(node);
511 }
512 
513 /*
514  * Called by the destructor to remove any STANDARD external references
515  */
516 void
517 ng_cutlinks(node_p node)
518 {
519 	hook_p  hook;
520 
521 	/* Make sure that this is set to stop infinite loops */
522 	node->flags |= NG_INVALID;
523 
524 	/*
525 	 * Drain the input queue forceably.
526 	 * We also do this in ng_rmnode
527 	 * to make sure we get all code paths.
528 	 */
529 	ng_flush_input_queue(&node->input_queue);
530 
531 	/* Notify all remaining connected nodes to disconnect */
532 	while ((hook = LIST_FIRST(&node->hooks)) != NULL)
533 		ng_destroy_hook(hook);
534 }
535 
536 /*
537  * Remove a reference to the node, possibly the last
538  */
539 void
540 ng_unref(node_p node)
541 {
542 	int	s;
543 
544 	s = splhigh();
545 /* XXX not atomic.. fix */
546 	if (--node->refs <= 0) {
547 
548 		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
549 		node->type->refs--; /* XXX maybe should get types lock? */
550 		LIST_REMOVE(node, nodes);
551 		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
552 
553 		mtx_enter(&ng_idhash_mtx, MTX_DEF);
554 		LIST_REMOVE(node, idnodes);
555 		mtx_exit(&ng_idhash_mtx, MTX_DEF);
556 
557 		NG_FREE_NODE(node);
558 	}
559 	splx(s);
560 }
561 
562 /************************************************************************
563 			Node ID handling
564 ************************************************************************/
565 static node_p
566 ng_ID2noderef(ng_ID_t ID)
567 {
568 	node_p np;
569 	mtx_enter(&ng_idhash_mtx, MTX_DEF);
570 	LIST_FOREACH(np, &ng_ID_hash[ID % ID_HASH_SIZE], idnodes) {
571 		if (np->ID == ID)
572 			break;
573 	}
574 	if(np)
575 		np->refs++;
576 	mtx_exit(&ng_idhash_mtx, MTX_DEF);
577 	return(np);
578 }
579 
580 ng_ID_t
581 ng_node2ID(node_p node)
582 {
583 	return (node?node->ID:0);
584 }
585 
586 /************************************************************************
587 			Node name handling
588 ************************************************************************/
589 
590 /*
591  * Assign a node a name. Once assigned, the name cannot be changed.
592  */
593 int
594 ng_name_node(node_p node, const char *name)
595 {
596 	int i;
597 	node_p node2;
598 
599 	/* Check the name is valid */
600 	for (i = 0; i < NG_NODELEN + 1; i++) {
601 		if (name[i] == '\0' || name[i] == '.' || name[i] == ':')
602 			break;
603 	}
604 	if (i == 0 || name[i] != '\0') {
605 		TRAP_ERROR;
606 		return (EINVAL);
607 	}
608 	if (ng_decodeidname(name) != 0) { /* valid IDs not allowed here */
609 		TRAP_ERROR;
610 		return (EINVAL);
611 	}
612 
613 	/* Check the name isn't already being used */
614 	if ((node2 = ng_name2noderef(node, name)) != NULL) {
615 		ng_unref(node2);
616 		TRAP_ERROR;
617 		return (EADDRINUSE);
618 	}
619 
620 	/* copy it */
621 	strcpy(node->name, name);
622 
623 	return (0);
624 }
625 
626 /*
627  * Find a node by absolute name. The name should NOT end with ':'
628  * The name "." means "this node" and "[xxx]" means "the node
629  * with ID (ie, at address) xxx".
630  *
631  * Returns the node if found, else NULL.
632  * Eventually should add something faster than a sequential search.
633  * Note it holds a reference on the node so you an be sure it's still there.
634  */
635 node_p
636 ng_name2noderef(node_p here, const char *name)
637 {
638 	node_p node;
639 	ng_ID_t temp;
640 
641 	/* "." means "this node" */
642 	if (strcmp(name, ".") == 0) {
643 		here->refs++;
644 		return(here);
645 	}
646 
647 	/* Check for name-by-ID */
648 	if ((temp = ng_decodeidname(name)) != 0) {
649 		return (ng_ID2noderef(temp));
650 	}
651 
652 	/* Find node by name */
653 	mtx_enter(&ng_nodelist_mtx, MTX_DEF);
654 	LIST_FOREACH(node, &ng_nodelist, nodes) {
655 		if (node->name[0] != '\0' && strcmp(node->name, name) == 0)
656 			break;
657 	}
658 	if (node)
659 		node->refs++;
660 	mtx_exit(&ng_nodelist_mtx, MTX_DEF);
661 	return (node);
662 }
663 
664 /*
665  * Decode a ID name, eg. "[f03034de]". Returns 0 if the
666  * string is not valid, otherwise returns the value.
667  */
668 static ng_ID_t
669 ng_decodeidname(const char *name)
670 {
671 	const int len = strlen(name);
672 	char *eptr;
673 	u_long val;
674 
675 	/* Check for proper length, brackets, no leading junk */
676 	if (len < 3 || name[0] != '[' || name[len - 1] != ']'
677 	    || !isxdigit(name[1]))
678 		return (0);
679 
680 	/* Decode number */
681 	val = strtoul(name + 1, &eptr, 16);
682 	if (eptr - name != len - 1 || val == ULONG_MAX || val == 0)
683 		return ((ng_ID_t)0);
684 	return (ng_ID_t)val;
685 }
686 
687 /*
688  * Remove a name from a node. This should only be called
689  * when shutting down and removing the node.
690  */
691 void
692 ng_unname(node_p node)
693 {
694 	bzero(node->name, NG_NODELEN);
695 }
696 
697 /************************************************************************
698 			Hook routines
699  Names are not optional. Hooks are always connected, except for a
700  brief moment within these routines.
701 ************************************************************************/
702 
703 /*
704  * Remove a hook reference
705  */
706 static void
707 ng_unref_hook(hook_p hook)
708 {
709 	int	s;
710 
711 	s = splhigh();
712 /* XXX not atomic.. fix */
713 	if (--hook->refs == 0) {
714 		if (hook->node) {
715 			ng_unref(hook->node);
716 			hook->node = NULL;
717 		}
718 		NG_FREE_HOOK(hook);
719 	}
720 	splx(s);
721 }
722 
723 /*
724  * Add an unconnected hook to a node. Only used internally.
725  */
726 static int
727 ng_add_hook(node_p node, const char *name, hook_p *hookp)
728 {
729 	hook_p hook;
730 	int error = 0;
731 
732 	/* Check that the given name is good */
733 	if (name == NULL) {
734 		TRAP_ERROR;
735 		return (EINVAL);
736 	}
737 	if (ng_findhook(node, name) != NULL) {
738 		TRAP_ERROR;
739 		return (EEXIST);
740 	}
741 
742 	/* Allocate the hook and link it up */
743 	MALLOC(hook, hook_p, sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO);
744 	if (hook == NULL) {
745 		TRAP_ERROR;
746 		return (ENOMEM);
747 	}
748 	hook->refs = 1;
749 	hook->flags = HK_INVALID;
750 	hook->node = node;
751 	node->refs++;		/* each hook counts as a reference */
752 
753 	/* Check if the node type code has something to say about it */
754 	if (node->type->newhook != NULL)
755 		if ((error = (*node->type->newhook)(node, hook, name)) != 0) {
756 			ng_unref_hook(hook);	/* this frees the hook */
757 			return (error);
758 		}
759 
760 	/*
761 	 * The 'type' agrees so far, so go ahead and link it in.
762 	 * We'll ask again later when we actually connect the hooks.
763 	 * The reference we have is for this linkage.
764 	 */
765 	LIST_INSERT_HEAD(&node->hooks, hook, hooks);
766 	node->numhooks++;
767 
768 	/* Set hook name */
769 	strcpy(hook->name, name);
770 	if (hookp)
771 		*hookp = hook;
772 	return (error);
773 }
774 
775 /*
776  * Connect a pair of hooks. Only used internally.
777  */
778 static int
779 ng_connect(hook_p hook1, hook_p hook2)
780 {
781 	int     error;
782 
783 	hook1->peer = hook2;
784 	hook2->peer = hook1;
785 
786 	/* Give each node the opportunity to veto the impending connection */
787 	if (hook1->node->type->connect) {
788 		if ((error = (*hook1->node->type->connect) (hook1))) {
789 			ng_destroy_hook(hook1);	/* also zaps hook2 */
790 			return (error);
791 		}
792 	}
793 	if (hook2->node->type->connect) {
794 		if ((error = (*hook2->node->type->connect) (hook2))) {
795 			ng_destroy_hook(hook2);	/* also zaps hook1 */
796 			return (error);
797 		}
798 	}
799 	hook1->flags &= ~HK_INVALID;
800 	hook2->flags &= ~HK_INVALID;
801 	return (0);
802 }
803 
804 /*
805  * Find a hook
806  *
807  * Node types may supply their own optimized routines for finding
808  * hooks.  If none is supplied, we just do a linear search.
809  */
810 hook_p
811 ng_findhook(node_p node, const char *name)
812 {
813 	hook_p hook;
814 
815 	if (node->type->findhook != NULL)
816 		return (*node->type->findhook)(node, name);
817 	LIST_FOREACH(hook, &node->hooks, hooks) {
818 		if (strcmp(hook->name, name) == 0)
819 			return (hook);
820 	}
821 	return (NULL);
822 }
823 
824 /*
825  * Destroy a hook
826  *
827  * As hooks are always attached, this really destroys two hooks.
828  * The one given, and the one attached to it. Disconnect the hooks
829  * from each other first.
830  */
831 void
832 ng_destroy_hook(hook_p hook)
833 {
834 	hook_p peer = hook->peer;
835 
836 	hook->flags |= HK_INVALID;		/* as soon as possible */
837 	if (peer) {
838 		peer->flags |= HK_INVALID;	/* as soon as possible */
839 		hook->peer = NULL;
840 		peer->peer = NULL;
841 		ng_disconnect_hook(peer);
842 	}
843 	ng_disconnect_hook(hook);
844 }
845 
846 /*
847  * Notify the node of the hook's demise. This may result in more actions
848  * (e.g. shutdown) but we don't do that ourselves and don't know what
849  * happens there. If there is no appropriate handler, then just remove it
850  * (and decrement the reference count of it's node which in turn might
851  * make something happen).
852  */
853 static void
854 ng_disconnect_hook(hook_p hook)
855 {
856 	node_p node = hook->node;
857 
858 	/*
859 	 * Remove the hook from the node's list to avoid possible recursion
860 	 * in case the disconnection results in node shutdown.
861 	 */
862 	LIST_REMOVE(hook, hooks);
863 	node->numhooks--;
864 	if (node->type->disconnect) {
865 		/*
866 		 * The type handler may elect to destroy the peer so don't
867 		 * trust its existance after this point.
868 		 */
869 		(*node->type->disconnect) (hook);
870 	}
871 	ng_unref_hook(hook);
872 }
873 
874 /*
875  * Take two hooks on a node and merge the connection so that the given node
876  * is effectively bypassed.
877  */
878 int
879 ng_bypass(hook_p hook1, hook_p hook2)
880 {
881 	if (hook1->node != hook2->node)
882 		return (EINVAL);
883 	hook1->peer->peer = hook2->peer;
884 	hook2->peer->peer = hook1->peer;
885 
886 	/* XXX If we ever cache methods on hooks update them as well */
887 	hook1->peer = NULL;
888 	hook2->peer = NULL;
889 	ng_destroy_hook(hook1);
890 	ng_destroy_hook(hook2);
891 	return (0);
892 }
893 
894 /*
895  * Install a new netgraph type
896  */
897 int
898 ng_newtype(struct ng_type *tp)
899 {
900 	const size_t namelen = strlen(tp->name);
901 
902 	/* Check version and type name fields */
903 	if ((tp->version != NG_ABI_VERSION)
904 	|| (namelen == 0)
905 	|| (namelen > NG_TYPELEN)) {
906 		TRAP_ERROR;
907 		return (EINVAL);
908 	}
909 
910 	/* Check for name collision */
911 	if (ng_findtype(tp->name) != NULL) {
912 		TRAP_ERROR;
913 		return (EEXIST);
914 	}
915 
916 	tp->refs = 0;
917 
918 	/* Link in new type */
919 	mtx_enter(&ng_typelist_mtx, MTX_DEF);
920 	LIST_INSERT_HEAD(&ng_typelist, tp, types);
921 	mtx_exit(&ng_typelist_mtx, MTX_DEF);
922 	return (0);
923 }
924 
925 /*
926  * Look for a type of the name given
927  */
928 struct ng_type *
929 ng_findtype(const char *typename)
930 {
931 	struct ng_type *type;
932 
933 	mtx_enter(&ng_typelist_mtx, MTX_DEF);
934 	LIST_FOREACH(type, &ng_typelist, types) {
935 		if (strcmp(type->name, typename) == 0)
936 			break;
937 	}
938 	mtx_exit(&ng_typelist_mtx, MTX_DEF);
939 	return (type);
940 }
941 
942 /************************************************************************
943 			Composite routines
944 ************************************************************************/
945 
946 /*
947  * Make a peer and connect. The order is arranged to minimise
948  * the work needed to back out in case of error.
949  */
950 int
951 ng_mkpeer(node_p node, const char *name, const char *name2, char *type)
952 {
953 	node_p  node2;
954 	hook_p  hook;
955 	hook_p  hook2;
956 	int     error;
957 
958 	if ((error = ng_add_hook(node, name, &hook)))
959 		return (error);
960 	if ((error = ng_make_node(type, &node2))) {
961 		ng_destroy_hook(hook);
962 		return (error);
963 	}
964 	if ((error = ng_add_hook(node2, name2, &hook2))) {
965 		ng_rmnode(node2);
966 		ng_destroy_hook(hook);
967 		return (error);
968 	}
969 
970 	/*
971 	 * Actually link the two hooks together.. on failure they are
972 	 * destroyed so we don't have to do that here.
973 	 */
974 	if ((error = ng_connect(hook, hook2)))
975 		ng_rmnode(node2);
976 	return (error);
977 }
978 
979 /*
980  * Connect two nodes using the specified hooks
981  */
982 int
983 ng_con_nodes(node_p node, const char *name, node_p node2, const char *name2)
984 {
985 	int     error;
986 	hook_p  hook;
987 	hook_p  hook2;
988 
989 	if ((error = ng_add_hook(node, name, &hook)))
990 		return (error);
991 	if ((error = ng_add_hook(node2, name2, &hook2))) {
992 		ng_destroy_hook(hook);
993 		return (error);
994 	}
995 	return (ng_connect(hook, hook2));
996 }
997 /************************************************************************
998 		Utility routines to send self messages
999 ************************************************************************/
1000 /*
1001  * Static version of shutdown message. we don't want to need resources
1002  * to shut down (we may be doing it to release resources because we ran out.
1003  */
1004 static struct	ng_mesg  ng_msg_shutdown = {
1005 	{NG_VERSION,		/* u_char */
1006 	0,			/* u_char spare */
1007 	0,			/* u_int16_t arglen */
1008 	NGF_STATIC,		/* u_int32_t flags */
1009 	0,			/* u_int32_t token */
1010 	NGM_GENERIC_COOKIE,	/* u_int32_t */
1011 	NGM_SHUTDOWN,		/* u_int32_t */
1012 	"shutdown"}		/* u_char[16] */
1013 };
1014 
1015 int
1016 ng_rmnode_self(node_p here)
1017 {
1018 	item_p	item;
1019 	struct	ng_mesg	*msg;
1020 
1021 	/*
1022 	 * Use the static version to avoid needing
1023 	 * memory allocation to succeed.
1024 	 * The message is never written to and always the same.
1025 	 */
1026 	msg = &ng_msg_shutdown;
1027 
1028 	/*
1029 	 * Try get a queue item to send it with.
1030 	 * Hopefully since it has a reserve, we can get one.
1031 	 * If we can't we are screwed anyhow.
1032 	 * Increase the chances by flushing our queue first.
1033 	 * We may free an item, (if we were the hog).
1034 	 * Work in progress is allowed to complete.
1035 	 * We also pretty much ensure that we come straight
1036 	 * back in to do the shutdown. It may be a good idea
1037 	 * to hold a reference actually to stop it from all
1038 	 * going up in smoke.
1039 	 */
1040 /*	ng_flush_input_queue(&here->input_queue); will mask problem  */
1041 	item = ng_package_msg_self(here, NULL, msg);
1042 	if (item == NULL) { /* it would have freed the msg except static */
1043 		/* try again after flushing our queue */
1044 		ng_flush_input_queue(&here->input_queue);
1045 		item = ng_package_msg_self(here, NULL, msg);
1046 		if (item == NULL) {
1047 			printf("failed to free node 0x%x\n", ng_node2ID(here));
1048 			return (ENOMEM);
1049 		}
1050 	}
1051 	return (ng_snd_item(item, 0));
1052 }
1053 
1054 /***********************************************************************
1055  * Parse and verify a string of the form:  <NODE:><PATH>
1056  *
1057  * Such a string can refer to a specific node or a specific hook
1058  * on a specific node, depending on how you look at it. In the
1059  * latter case, the PATH component must not end in a dot.
1060  *
1061  * Both <NODE:> and <PATH> are optional. The <PATH> is a string
1062  * of hook names separated by dots. This breaks out the original
1063  * string, setting *nodep to "NODE" (or NULL if none) and *pathp
1064  * to "PATH" (or NULL if degenerate). Also, *hookp will point to
1065  * the final hook component of <PATH>, if any, otherwise NULL.
1066  *
1067  * This returns -1 if the path is malformed. The char ** are optional.
1068  ***********************************************************************/
1069 int
1070 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp)
1071 {
1072 	char   *node, *path, *hook;
1073 	int     k;
1074 
1075 	/*
1076 	 * Extract absolute NODE, if any
1077 	 */
1078 	for (path = addr; *path && *path != ':'; path++);
1079 	if (*path) {
1080 		node = addr;	/* Here's the NODE */
1081 		*path++ = '\0';	/* Here's the PATH */
1082 
1083 		/* Node name must not be empty */
1084 		if (!*node)
1085 			return -1;
1086 
1087 		/* A name of "." is OK; otherwise '.' not allowed */
1088 		if (strcmp(node, ".") != 0) {
1089 			for (k = 0; node[k]; k++)
1090 				if (node[k] == '.')
1091 					return -1;
1092 		}
1093 	} else {
1094 		node = NULL;	/* No absolute NODE */
1095 		path = addr;	/* Here's the PATH */
1096 	}
1097 
1098 	/* Snoop for illegal characters in PATH */
1099 	for (k = 0; path[k]; k++)
1100 		if (path[k] == ':')
1101 			return -1;
1102 
1103 	/* Check for no repeated dots in PATH */
1104 	for (k = 0; path[k]; k++)
1105 		if (path[k] == '.' && path[k + 1] == '.')
1106 			return -1;
1107 
1108 	/* Remove extra (degenerate) dots from beginning or end of PATH */
1109 	if (path[0] == '.')
1110 		path++;
1111 	if (*path && path[strlen(path) - 1] == '.')
1112 		path[strlen(path) - 1] = 0;
1113 
1114 	/* If PATH has a dot, then we're not talking about a hook */
1115 	if (*path) {
1116 		for (hook = path, k = 0; path[k]; k++)
1117 			if (path[k] == '.') {
1118 				hook = NULL;
1119 				break;
1120 			}
1121 	} else
1122 		path = hook = NULL;
1123 
1124 	/* Done */
1125 	if (nodep)
1126 		*nodep = node;
1127 	if (pathp)
1128 		*pathp = path;
1129 	if (hookp)
1130 		*hookp = hook;
1131 	return (0);
1132 }
1133 
1134 /*
1135  * Given a path, which may be absolute or relative, and a starting node,
1136  * return the destination node.
1137  */
1138 int
1139 ng_path2noderef(node_p here, const char *address,
1140 				node_p *destp, hook_p *lasthook)
1141 {
1142 	char    fullpath[NG_PATHLEN + 1];
1143 	char   *nodename, *path, pbuf[2];
1144 	node_p  node, oldnode;
1145 	char   *cp;
1146 	hook_p hook = NULL;
1147 
1148 	/* Initialize */
1149 	if (destp == NULL)
1150 		return EINVAL;
1151 	*destp = NULL;
1152 
1153 	/* Make a writable copy of address for ng_path_parse() */
1154 	strncpy(fullpath, address, sizeof(fullpath) - 1);
1155 	fullpath[sizeof(fullpath) - 1] = '\0';
1156 
1157 	/* Parse out node and sequence of hooks */
1158 	if (ng_path_parse(fullpath, &nodename, &path, NULL) < 0) {
1159 		TRAP_ERROR;
1160 		return EINVAL;
1161 	}
1162 	if (path == NULL) {
1163 		pbuf[0] = '.';	/* Needs to be writable */
1164 		pbuf[1] = '\0';
1165 		path = pbuf;
1166 	}
1167 
1168 	/*
1169 	 * For an absolute address, jump to the starting node.
1170 	 * Note that this holds a reference on the node for us.
1171 	 * Don't forget to drop the reference if we don't need it.
1172 	 */
1173 	if (nodename) {
1174 		node = ng_name2noderef(here, nodename);
1175 		if (node == NULL) {
1176 			TRAP_ERROR;
1177 			return (ENOENT);
1178 		}
1179 	} else {
1180 		if (here == NULL) {
1181 			TRAP_ERROR
1182 			return (EINVAL);
1183 		}
1184 		node = here;
1185 		node->refs++;
1186 	}
1187 
1188 	/*
1189 	 * Now follow the sequence of hooks
1190 	 * XXX
1191 	 * We actually cannot guarantee that the sequence
1192 	 * is not being demolished as we crawl along it
1193 	 * without extra-ordinary locking etc.
1194 	 * So this is a bit dodgy to say the least.
1195 	 * We can probably hold up some things by holding
1196 	 * the nodelist mutex for the time of this
1197 	 * crawl if we wanted.. At least that way we wouldn't have to
1198 	 * worry about the nodes dissappearing, but the hooks would still
1199 	 * be a problem.
1200 	 */
1201 	for (cp = path; node != NULL && *cp != '\0'; ) {
1202 		char *segment;
1203 
1204 		/*
1205 		 * Break out the next path segment. Replace the dot we just
1206 		 * found with a NUL; "cp" points to the next segment (or the
1207 		 * NUL at the end).
1208 		 */
1209 		for (segment = cp; *cp != '\0'; cp++) {
1210 			if (*cp == '.') {
1211 				*cp++ = '\0';
1212 				break;
1213 			}
1214 		}
1215 
1216 		/* Empty segment */
1217 		if (*segment == '\0')
1218 			continue;
1219 
1220 		/* We have a segment, so look for a hook by that name */
1221 		hook = ng_findhook(node, segment);
1222 
1223 		/* Can't get there from here... */
1224 		if (hook == NULL
1225 		    || hook->peer == NULL
1226 		    || (hook->flags & HK_INVALID) != 0
1227 		    || (hook->peer->flags & HK_INVALID) != 0) {
1228 			TRAP_ERROR;
1229 			ng_unref(node);
1230 			return (ENOENT);
1231 		}
1232 
1233 		/*
1234 		 * Hop on over to the next node
1235 		 * XXX
1236 		 * Big race conditions here as hooks and nodes go away
1237 		 * *** Idea.. store an ng_ID_t in each hook and use that
1238 		 * instead of the direct hook in this crawl?
1239 		 */
1240 		oldnode = node;
1241 		if ((node = hook->peer->node))
1242 			node->refs++;	/* XXX RACE */
1243 		ng_unref(oldnode);	/* XXX another race */
1244 		if (node->flags & NG_INVALID) {
1245 			ng_unref(node);	/* XXX more races */
1246 			node = NULL;
1247 		}
1248 	}
1249 
1250 	/* If node somehow missing, fail here (probably this is not needed) */
1251 	if (node == NULL) {
1252 		TRAP_ERROR;
1253 		return (ENXIO);
1254 	}
1255 
1256 	/* Done */
1257 	*destp = node;
1258 	if (lasthook != NULL)
1259 		*lasthook = (hook ? hook->peer : NULL);
1260 	return (0);
1261 }
1262 
1263 /***************************************************************\
1264 * Input queue handling.
1265 * All activities are submitted to the node via the input queue
1266 * which implements a multiple-reader/single-writer gate.
1267 * Items which cannot be handled immeditly are queued.
1268 *
1269 * read-write queue locking inline functions			*
1270 \***************************************************************/
1271 
1272 static __inline item_p ng_dequeue(struct ng_queue * ngq);
1273 static __inline item_p ng_acquire_read(struct ng_queue * ngq,
1274 					item_p  item);
1275 static __inline item_p ng_acquire_write(struct ng_queue * ngq,
1276 					item_p  item);
1277 static __inline void	ng_leave_read(struct ng_queue * ngq);
1278 static __inline void	ng_leave_write(struct ng_queue * ngq);
1279 static __inline void	ng_queue_rw(struct ng_queue * ngq,
1280 					item_p  item, int rw);
1281 
1282 /*
1283  * Definition of the bits fields in the ng_queue flag word.
1284  * Defined here rather than in netgraph.h because no-one should fiddle
1285  * with them.
1286  *
1287  * The ordering here is important! don't shuffle these. If you add
1288  * READ_PENDING to the word when it has READ_PENDING already set, you
1289  * generate a carry into the reader count, this you atomically add a reader,
1290  * and remove the pending reader count! Similarly for the pending writer
1291  * flag, adding WRITE_PENDING generates a carry and sets the WRITER_ACTIVE
1292  * flag, while clearing WRITE_PENDING. When 'SINGLE_THREAD_ONLY' is set, then
1293  * it is only permitted to do WRITER operations. Reader operations will
1294  * result in errors.
1295  * But that "hack" is unnecessary: "cpp" can do the math for us!
1296  */
1297 /*-
1298  Safety Barrier--------+ (adjustable to suit taste) (not used yet)
1299                        |
1300                        V
1301 +-------+-------+-------+-------+-------+-------+-------+-------+
1302 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
1303 |A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |R|A|W|S|
1304 | | | | | | | | | | | | | | | | | | | | | | | | | | | | |P|W|P|T|
1305 +-------+-------+-------+-------+-------+-------+-------+-------+
1306 \_________________________ ____________________________/ | | | |
1307                           V                              | | | |
1308                 [active reader count]                    | | | |
1309                                                          | | | |
1310         Read Pending ------------------------------------+ | | |
1311                                                            | | |
1312         Active Writer -------------------------------------+ | |
1313                                                              | |
1314         Write Pending ---------------------------------------+ |
1315                                                                |
1316         Single Threading Only ---------------------------------+
1317 */
1318 #define	SINGLE_THREAD_ONLY 0x00000001	/* if set, even reads single thread */
1319 #define WRITE_PENDING	0x00000002
1320 #define	WRITER_ACTIVE	0x00000004
1321 #define READ_PENDING	0x00000008
1322 #define	READER_INCREMENT 0x00000010
1323 #define	READER_MASK	0xfffffff0	/* Not valid if WRITER_ACTIVE is set */
1324 #define SAFETY_BARRIER	0x00100000	/* 64K items queued should be enough */
1325 /*
1326  * Taking into account the current state of the queue and node, possibly take
1327  * the next entry off the queue and return it. Return NULL if there was
1328  * nothing we could return, either because there really was nothing there, or
1329  * because the node was in a state where it cannot yet process the next item
1330  * on the queue.
1331  *
1332  * This MUST MUST MUST be called with the mutex held.
1333  */
1334 static __inline item_p
1335 ng_dequeue(struct ng_queue *ngq)
1336 {
1337 	item_p item;
1338 	u_int		add_arg;
1339 	/*
1340 	 * If there is a writer, then the answer is "no". Everything else
1341 	 * stops when there is a WRITER.
1342 	 */
1343 	if (ngq->q_flags & WRITER_ACTIVE) {
1344 		return (NULL);
1345 	}
1346 	/* Now take a look at what's on the queue and what's running */
1347 	if ((ngq->q_flags & ~(READER_MASK | SINGLE_THREAD_ONLY)) == READ_PENDING) {
1348 		/*
1349 		 * It was a reader and we have no write active. We don't care
1350 		 * how many readers are already active. Adjust the count for
1351 		 * the item we are about to dequeue. Adding READ_PENDING to
1352 		 * the exisiting READ_PENDING clears it and generates a carry
1353 		 * into the reader count.
1354 		 */
1355 		add_arg = READ_PENDING;
1356 	} else if ((ngq->q_flags & ~SINGLE_THREAD_ONLY) == WRITE_PENDING) {
1357 		/*
1358 		 * There is a pending write, no readers and no active writer.
1359 		 * This means we can go ahead with the pending writer. Note
1360 		 * the fact that we now have a writer, ready for when we take
1361 		 * it off the queue.
1362 		 *
1363 		 * We don't need to worry about a possible collision with the
1364 		 * fasttrack reader.
1365 		 *
1366 		 * The fasttrack thread may take a long time to discover that we
1367 		 * are running so we would have an inconsistent state in the
1368 		 * flags for a while. Since we ignore the reader count
1369 		 * entirely when the WRITER_ACTIVE flag is set, this should
1370 		 * not matter (in fact it is defined that way). If it tests
1371 		 * the flag before this operation, the WRITE_PENDING flag
1372 		 * will make it fail, and if it tests it later, the
1373 		 * ACTIVE_WRITER flag will do the same. If it is SO slow that
1374 		 * we have actually completed the operation, and neither flag
1375 		 * is set (nor the READ_PENDING) by the time that it tests
1376 		 * the flags, then it is actually ok for it to continue. If
1377 		 * it completes and we've finished and the read pending is
1378 		 * set it still fails.
1379 		 *
1380 		 * So we can just ignore it,  as long as we can ensure that the
1381 		 * transition from WRITE_PENDING state to the WRITER_ACTIVE
1382 		 * state is atomic.
1383 		 *
1384 		 * After failing, first it will be held back by the mutex, then
1385 		 * when it can proceed, it will queue its request, then it
1386 		 * would arrive at this function. Usually it will have to
1387 		 * leave empty handed because the ACTIVE WRITER bit wil be
1388 		 * set.
1389 		 */
1390 		/*
1391 		 * Adjust the flags for the item we are about to dequeue.
1392 		 * Adding WRITE_PENDING to the exisiting WRITE_PENDING clears
1393 		 * it and generates a carry into the WRITER_ACTIVE flag, all
1394 		 * atomically.
1395 		 */
1396 		add_arg = WRITE_PENDING;
1397 		/*
1398 		 * We want to write "active writer, no readers " Now go make
1399 		 * it true. In fact there may be a number in the readers
1400 		 * count but we know it is not true and will be fixed soon.
1401 		 * We will fix the flags for the next pending entry in a
1402 		 * moment.
1403 		 */
1404 	} else {
1405 		/*
1406 		 * We can't dequeue anything.. return and say so. Probably we
1407 		 * have a write pending and the readers count is non zero. If
1408 		 * we got here because a reader hit us just at the wrong
1409 		 * moment with the fasttrack code, and put us in a strange
1410 		 * state, then it will be through in just a moment, (as soon
1411 		 * as we release the mutex) and keep things moving.
1412 		 */
1413 		return (0);
1414 	}
1415 
1416 	/*
1417 	 * Now we dequeue the request (whatever it may be) and correct the
1418 	 * pending flags and the next and last pointers.
1419 	 */
1420 	item = ngq->queue;
1421 	ngq->queue = item->el_next;
1422 	if (ngq->last == &(item->el_next)) {
1423 		/*
1424 		 * that was the last entry in the queue so set the 'last
1425 		 * pointer up correctly and make sure the pending flags are
1426 		 * clear.
1427 		 */
1428 		ngq->last = &(ngq->queue);
1429 		/*
1430 		 * Whatever flag was set is cleared and the carry sets the
1431 		 * correct new active state/count. So we don't need to change
1432 		 * add_arg.
1433 		 */
1434 	} else {
1435 		if ((ngq->queue->el_flags & NGQF_TYPE) == NGQF_READER) {
1436 			/*
1437 			 * If we had a READ_PENDING and have another one, we
1438 			 * just want to add READ_PENDING twice (the same as
1439 			 * adding READER_INCREMENT). If we had WRITE_PENDING,
1440 			 * we want to add READ_PENDING + WRITE_PENDING to
1441 			 * clear the old WRITE_PENDING, set ACTIVE_WRITER,
1442 			 * and set READ_PENDING. Either way we just add
1443 			 * READ_PENDING to whatever we already had.
1444 			 */
1445 			add_arg += READ_PENDING;
1446 		} else {
1447 			/*
1448 			 * If we had a WRITE_PENDING and have another one, we
1449 			 * just want to add WRITE_PENDING twice (the same as
1450 			 * adding ACTIVE_WRITER). If we had READ_PENDING, we
1451 			 * want to add READ_PENDING + WRITE_PENDING to clear
1452 			 * the old READ_PENDING, increment the readers, and
1453 			 * set WRITE_PENDING. Either way we just add
1454 			 * WRITE_PENDING to whatever we already had.
1455 			 */
1456 			add_arg += WRITE_PENDING;
1457 		}
1458 	}
1459 	atomic_add_long(&ngq->q_flags, add_arg);
1460 	/*
1461 	 * We have successfully cleared the old pending flag, set the new one
1462 	 * if it is needed, and incremented the appropriate active field.
1463 	 * (all in one atomic addition.. wow)
1464 	 */
1465 	return (item);
1466 }
1467 
1468 /*
1469  * Queue a packet to be picked up by someone else.
1470  * We really don't care who, but we can't or don't want to hang around
1471  * to process it ourselves. We are probably an interrupt routine..
1472  * 1 = writer, 0 = reader
1473  * We should set something to indicate NETISR requested
1474  * If it's the first item queued.
1475  */
1476 #define NGQRW_R 0
1477 #define NGQRW_W 1
1478 static __inline void
1479 ng_queue_rw(struct ng_queue * ngq, item_p  item, int rw)
1480 {
1481 	item->el_next = NULL;	/* maybe not needed */
1482 	*ngq->last = item;
1483 	/*
1484 	 * If it was the first item in the queue then we need to
1485 	 * set the last pointer and the type flags.
1486 	 */
1487 	if (ngq->last == &(ngq->queue)) {
1488 		/*
1489 		 * When called with constants for rw, the optimiser will
1490 		 * remove the unneeded branch below.
1491 		 */
1492 		if (rw == NGQRW_W) {
1493 			atomic_add_long(&ngq->q_flags, WRITE_PENDING);
1494 		} else {
1495 			atomic_add_long(&ngq->q_flags, READ_PENDING);
1496 		}
1497 	}
1498 	ngq->last = &(item->el_next);
1499 }
1500 
1501 
1502 /*
1503  * This function 'cheats' in that it first tries to 'grab' the use of the
1504  * node, without going through the mutex. We can do this becasue of the
1505  * semantics of the lock. The semantics include a clause that says that the
1506  * value of the readers count is invalid if the WRITER_ACTIVE flag is set. It
1507  * also says that the WRITER_ACTIVE flag cannot be set if the readers count
1508  * is not zero. Note that this talks about what is valid to SET the
1509  * WRITER_ACTIVE flag, because from the moment it is set, the value if the
1510  * reader count is immaterial, and not valid. The two 'pending' flags have a
1511  * similar effect, in that If they are orthogonal to the two active fields in
1512  * how they are set, but if either is set, the attempted 'grab' need to be
1513  * backed out because there is earlier work, and we maintain ordering in the
1514  * queue. The result of this is that the reader request can try obtain use of
1515  * the node with only a single atomic addition, and without any of the mutex
1516  * overhead. If this fails the operation degenerates to the same as for other
1517  * cases.
1518  *
1519  */
1520 static __inline item_p
1521 ng_acquire_read(struct ng_queue *ngq, item_p item)
1522 {
1523 
1524 	/* ######### Hack alert ######### */
1525 	atomic_add_long(&ngq->q_flags, READER_INCREMENT);
1526 	if ((ngq->q_flags & (~READER_MASK)) == 0) {
1527 		/* Successfully grabbed node */
1528 		return (item);
1529 	}
1530 	/* undo the damage if we didn't succeed */
1531 	atomic_subtract_long(&ngq->q_flags, READER_INCREMENT);
1532 
1533 	/* ######### End Hack alert ######### */
1534 	mtx_enter((&ngq->q_mtx), MTX_SPIN);
1535 	/*
1536 	 * Try again. Another processor (or interrupt for that matter) may
1537 	 * have removed the last queued item that was stopping us from
1538 	 * running, between the previous test, and the moment that we took
1539 	 * the mutex. (Or maybe a writer completed.)
1540 	 */
1541 	if ((ngq->q_flags & (~READER_MASK)) == 0) {
1542 		atomic_add_long(&ngq->q_flags, READER_INCREMENT);
1543 		mtx_exit((&ngq->q_mtx), MTX_SPIN);
1544 		return (item);
1545 	}
1546 
1547 	/*
1548 	 * Quick check that we are doing things right.
1549 	 */
1550 	if (ngq->q_flags & SINGLE_THREAD_ONLY) {
1551 		panic("Calling single treaded queue incorrectly");
1552 	}
1553 
1554 	/*
1555 	 * and queue the request for later.
1556 	 */
1557 	item->el_flags |= NGQF_TYPE;
1558 	ng_queue_rw(ngq, item, NGQRW_R);
1559 
1560 	/*
1561 	 * Ok, so that's the item successfully queued for later. So now we
1562 	 * see if we can dequeue something to run instead.
1563 	 */
1564 	item = ng_dequeue(ngq);
1565 	mtx_exit(&(ngq->q_mtx), MTX_SPIN);
1566 	return (item);
1567 }
1568 
1569 static __inline item_p
1570 ng_acquire_write(struct ng_queue *ngq, item_p item)
1571 {
1572 restart:
1573 	mtx_enter(&(ngq->q_mtx), MTX_SPIN);
1574 	/*
1575 	 * If there are no readers, no writer, and no pending packets, then
1576 	 * we can just go ahead. In all other situations we need to queue the
1577 	 * request
1578 	 */
1579 	if ((ngq->q_flags & (~SINGLE_THREAD_ONLY)) == 0) {
1580 		atomic_add_long(&ngq->q_flags, WRITER_ACTIVE);
1581 		mtx_exit((&ngq->q_mtx), MTX_SPIN);
1582 		if (ngq->q_flags & READER_MASK) {
1583 			/* Collision with fast-track reader */
1584 			atomic_add_long(&ngq->q_flags, -WRITER_ACTIVE);
1585 			goto restart;
1586 		}
1587 
1588 		return (item);
1589 	}
1590 
1591 	/*
1592 	 * and queue the request for later.
1593 	 */
1594 	item->el_flags &= ~NGQF_TYPE;
1595 	ng_queue_rw(ngq, item, NGQRW_W);
1596 
1597 	/*
1598 	 * Ok, so that's the item successfully queued for later. So now we
1599 	 * see if we can dequeue something to run instead.
1600 	 */
1601 	item = ng_dequeue(ngq);
1602 	mtx_exit(&(ngq->q_mtx), MTX_SPIN);
1603 	return (item);
1604 }
1605 
1606 static __inline void
1607 ng_leave_read(struct ng_queue *ngq)
1608 {
1609 	atomic_subtract_long(&ngq->q_flags, READER_INCREMENT);
1610 }
1611 
1612 static __inline void
1613 ng_leave_write(struct ng_queue *ngq)
1614 {
1615 	atomic_subtract_long(&ngq->q_flags, WRITER_ACTIVE);
1616 }
1617 
1618 static void
1619 ng_flush_input_queue(struct ng_queue * ngq)
1620 {
1621 	item_p item;
1622 	u_int		add_arg;
1623 	mtx_enter(&ngq->q_mtx, MTX_SPIN);
1624 	for (;;) {
1625 		/* Now take a look at what's on the queue */
1626 		if (ngq->q_flags & READ_PENDING) {
1627 			add_arg = -READ_PENDING;
1628 		} else if (ngq->q_flags & WRITE_PENDING) {
1629 			add_arg = -WRITE_PENDING;
1630 		} else {
1631 			break;
1632 		}
1633 
1634 		item = ngq->queue;
1635 		ngq->queue = item->el_next;
1636 		if (ngq->last == &(item->el_next)) {
1637 			ngq->last = &(ngq->queue);
1638 		} else {
1639 			if ((ngq->queue->el_flags & NGQF_TYPE) == NGQF_READER) {
1640 				add_arg += READ_PENDING;
1641 			} else {
1642 				add_arg += WRITE_PENDING;
1643 			}
1644 		}
1645 		atomic_add_long(&ngq->q_flags, add_arg);
1646 
1647 		mtx_exit(&ngq->q_mtx, MTX_SPIN);
1648 		NG_FREE_ITEM(item);
1649 		mtx_enter(&ngq->q_mtx, MTX_SPIN);
1650 	}
1651 	mtx_exit(&ngq->q_mtx, MTX_SPIN);
1652 }
1653 
1654 /***********************************************************************
1655 * Externally visible method for sending or queueing messages or data.
1656 ***********************************************************************/
1657 
1658 /*
1659  * MACRO WILL DO THE JOB OF CALLING ng_package_msg IN CALLER
1660  * before we are called. The user code should have filled out the item
1661  * correctly by this stage:
1662  * Common:
1663  *    reference to destination node.
1664  *    Reference to destination rcv hook if relevant.
1665  * Data:
1666  *    pointer to mbuf
1667  *    pointer to metadata
1668  * Control_Message:
1669  *    pointer to msg.
1670  *    ID of original sender node. (return address)
1671  *
1672  * The nodes have several routines and macros to help with this task:
1673  * ng_package_msg()
1674  * ng_package_data() do much of the work.
1675  * ng_retarget_msg
1676  * ng_retarget_data
1677  */
1678 
1679 int
1680 ng_snd_item(item_p item, int queue)
1681 {
1682 	hook_p hook = item->el_hook;
1683 	node_p dest = item->el_dest;
1684 	int rw;
1685 	int error = 0, ierror;
1686 	item_p	oitem;
1687 	struct ng_queue * ngq = &dest->input_queue;
1688 
1689 #ifdef	ITEM_DEBUG
1690         _ngi_check(item, __FILE__, __LINE__);
1691 #endif
1692 
1693 	if (item == NULL) {
1694 		return (EINVAL);	/* failed to get queue element */
1695 	}
1696 	if (dest == NULL) {
1697 		NG_FREE_ITEM(item);
1698 		return (EINVAL);	/* No address */
1699 	}
1700 	if ((item->el_flags & NGQF_D_M) == NGQF_DATA) {
1701 		/*
1702 		 * DATA MESSAGE
1703 		 * Delivered to a node via a non-optional hook.
1704 		 * Both should be present in the item even though
1705 		 * the node is derivable from the hook.
1706 		 * References are held on both by the item.
1707 		 */
1708 #ifdef	ITEM_DEBUG
1709         _ngi_check(item, __FILE__, __LINE__);
1710 #endif
1711 		CHECK_DATA_MBUF(NGI_M(item));
1712 		if (hook == NULL) {
1713 			NG_FREE_ITEM(item);
1714 			return(EINVAL);
1715 		}
1716 		if (((hook->flags & HK_INVALID) != 0)
1717 		|| ((hook->node->flags & NG_INVALID) != 0)) {
1718 			TRAP_ERROR;
1719 			NG_FREE_ITEM(item);
1720 			return (ENOTCONN);
1721 		}
1722 		if ((hook->flags & HK_QUEUE)) {
1723 			queue = 1;
1724 		}
1725 		/* By default data is a reader in the locking scheme */
1726 		item->el_flags |= NGQF_READER;
1727 		rw = NGQRW_R;
1728 	} else {
1729 		/*
1730 		 * CONTROL MESSAGE
1731 		 * Delivered to a node.
1732 		 * Hook is optional.
1733 		 * References are held by the item on the node and
1734 		 * the hook if it is present.
1735 		 */
1736 		if (hook && (hook->flags & HK_QUEUE)) {
1737 			queue = 1;
1738 		}
1739 		/* Data messages count as writers unles explicitly exempted */
1740 		if (NGI_MSG(item)->header.cmd & NGM_READONLY) {
1741 			item->el_flags |= NGQF_READER;
1742 			rw = NGQRW_R;
1743 		} else {
1744 			item->el_flags &= ~NGQF_TYPE;
1745 			rw = NGQRW_W;
1746 		}
1747 	}
1748 	/*
1749 	 * If the node specifies single threading, force writer semantics
1750 	 * Similarly the node may say one hook always produces writers.
1751 	 * These are over-rides.
1752 	 */
1753 	if ((ngq->q_flags & SINGLE_THREAD_ONLY)
1754 	|| (dest->flags & NG_FORCE_WRITER)
1755 	|| (hook && (hook->flags & HK_FORCE_WRITER))) {
1756 			rw = NGQRW_W;
1757 			item->el_flags &= ~NGQF_TYPE;
1758 	}
1759 	if (queue) {
1760 		/* Put it on the queue for that node*/
1761 #ifdef	ITEM_DEBUG
1762         _ngi_check(item, __FILE__, __LINE__);
1763 #endif
1764 		mtx_enter(&(ngq->q_mtx), MTX_SPIN);
1765 		ng_queue_rw(ngq, item, rw);
1766 		mtx_exit(&(ngq->q_mtx), MTX_SPIN);
1767 		/*
1768 		 * If there are active elements then we can rely on
1769 		 * them. if not we should not rely on another packet
1770 		 * coming here by another path,
1771 		 * so it is best to put us in the netisr list.
1772 		 */
1773 		if ((ngq->q_flags & (READER_MASK|WRITER_ACTIVE)) == 0) {
1774 			ng_setisr(ngq->q_node);
1775 		}
1776 		return (0);
1777 	}
1778 	/*
1779 	 * Take a queue item and a node and see if we can apply the item to
1780 	 * the node. We may end up getting a different item to apply instead.
1781 	 * Will allow for a piggyback reply only in the case where
1782 	 * there is no queueing.
1783 	 */
1784 
1785 	oitem = item;
1786 	/*
1787 	 * We already decided how we will be queueud or treated.
1788 	 * Try get the appropriate operating permission.
1789 	 */
1790  	if (rw == NGQRW_R) {
1791 		item = ng_acquire_read(ngq, item);
1792 	} else {
1793 		item = ng_acquire_write(ngq, item);
1794 	}
1795 
1796 	/*
1797 	 * May have come back with a different item.
1798 	 * or maybe none at all. The one we started with will
1799 	 * have been queued in thises cases.
1800 	 */
1801 	if (item == NULL) {
1802 		return (0);
1803 	}
1804 
1805 #ifdef	ITEM_DEBUG
1806         _ngi_check(item, __FILE__, __LINE__);
1807 #endif
1808 	ierror = ng_apply_item(dest, item); /* drops r/w lock when done */
1809 
1810 	/* only return an error if it was our initial item.. (compat hack) */
1811 	if (oitem == item) {
1812 		error = ierror;
1813 	}
1814 
1815 	/*
1816 	 * Now we've handled the packet we brought, (or a friend of it) let's
1817 	 * look for any other packets that may have been queued up. We hold
1818 	 * no locks, so if someone puts something in the queue after
1819 	 * we check that it is empty, it is their problem
1820 	 * to ensure it is processed. If we have the netisr thread cme in here
1821 	 * while we still say we have stuff to do, we may get a boost
1822 	 * in SMP systems. :-)
1823 	 */
1824 	for (;;) {
1825 		/* quick hack to save all that mutex stuff */
1826 		if ((ngq->q_flags & (WRITE_PENDING | READ_PENDING)) == 0) {
1827 			if (dest->flags & NG_WORKQ)
1828 				ng_worklist_remove(dest);
1829 			return (0);
1830 		}
1831 		/*
1832 		 * dequeue acquires and adjusts the input_queue as it dequeues
1833 		 * packets. It acquires the rw lock as needed.
1834 		 */
1835 		mtx_enter(&ngq->q_mtx, MTX_SPIN);
1836 		item = ng_dequeue(ngq);
1837 		mtx_exit(&ngq->q_mtx, MTX_SPIN);
1838 		if (!item) {
1839 			/*
1840 			 * If we have no work to do
1841 			 * then we certainly don't need to be
1842 			 * on the worklist.
1843 			 */
1844 			if (dest->flags & NG_WORKQ)
1845 				ng_worklist_remove(dest);
1846 			return (0);
1847 		}
1848 #ifdef	ITEM_DEBUG
1849         _ngi_check(item, __FILE__, __LINE__);
1850 #endif
1851 
1852 		/*
1853 		 * We have the appropriate lock, so run the item.
1854 		 * When finished it will drop the lock accordingly
1855 		 */
1856 
1857 		ierror = ng_apply_item(dest, item);
1858 		/*
1859 		 * only return an error if it was our initial
1860 		 * item.. (compat hack)
1861 		 */
1862 		if (oitem == item) {
1863 			error = ierror;
1864 		}
1865 	}
1866 	return (0);
1867 }
1868 
1869 /*
1870  * We have an item that was possibly queued somewhere.
1871  * It should contain all the information needed
1872  * to run it on the appropriate node/hook.
1873  */
1874 static int
1875 ng_apply_item(node_p node, item_p item)
1876 {
1877 	hook_p  hook;
1878 	int was_reader = ((item->el_flags & NGQF_TYPE));
1879 	int error = 0;
1880 	ng_rcvdata_t *rcvdata;
1881 
1882 	hook = item->el_hook;
1883 	item->el_hook = NULL;	/* so NG_FREE_ITEM doesn't ng_unref_hook() */
1884 	/* We already have the node.. assume responsibility */
1885 	/* And the reference */
1886 	/* node = item->el_dest; */
1887 	item->el_dest = NULL;	/* same as for the hook above */
1888 #ifdef	ITEM_DEBUG
1889         _ngi_check(item, __FILE__, __LINE__);
1890 #endif
1891 
1892 	switch (item->el_flags & NGQF_D_M) {
1893 	case NGQF_DATA:
1894 		/*
1895 		 * Check things are still ok as when we were queued.
1896 		 */
1897 
1898 		if ((hook == NULL)
1899 		|| ((hook->flags & HK_INVALID) != 0)
1900 		|| ((hook->node->flags & NG_INVALID) != 0)
1901 		|| ((rcvdata = hook->node->type->rcvdata) == NULL)) {
1902 			error = EIO;
1903 			NG_FREE_ITEM(item);
1904 		} else {
1905 			error = (*rcvdata)(hook, item);
1906 		}
1907 		break;
1908 	case NGQF_MESG:
1909 
1910 		if (hook) {
1911 			item->el_hook = NULL;
1912 			if ((hook->flags & HK_INVALID) != 0) {
1913 			/*
1914 			 * If the hook has been zapped then we can't use it.
1915 			 * Immediatly drop its reference.
1916 			 * The message may not need it.
1917 			 */
1918 				ng_unref_hook(hook);
1919 				hook = NULL;
1920 			}
1921 		}
1922 		/*
1923 		 * Similarly, if the node is a zombie there is
1924 		 * nothing we can do with it, drop everything.
1925 		 */
1926 		if (node->flags & NG_INVALID) {
1927 			error = EINVAL;
1928 			NG_FREE_ITEM(item);
1929 		} else {
1930 			/*
1931 			 * Call the appropriate message handler for the object.
1932 			 * It is up to the message handler to free the message.
1933 			 * If it's a generic message, handle it generically,
1934 			 * otherwise call the type's message handler
1935 			 * (if it exists)
1936 			 * XXX (race). Remember that a queued message may
1937 			 * reference a node or hook that has just been
1938 			 * invalidated. It will exist as the queue code
1939 			 * is holding a reference, but..
1940 			 */
1941 
1942 			struct ng_mesg *msg = NGI_MSG(item);
1943 
1944 			if ((msg->header.typecookie == NGM_GENERIC_COOKIE)
1945 			&& ((msg->header.flags & NGF_RESP) == 0)) {
1946 				error = ng_generic_msg(node, item, hook);
1947 			} else {
1948 				if ((node)->type->rcvmsg != NULL) {
1949 					error = (*(node)->type->rcvmsg)((node),
1950 						(item), (hook));
1951 				} else {
1952 					TRAP_ERROR;
1953 					error = EINVAL; /* XXX */
1954 					NG_FREE_ITEM(item);
1955 				}
1956 			}
1957 			/* item is now invalid */
1958 		}
1959 		break;
1960 	}
1961 	/*
1962 	 * We held references on some of the resources
1963 	 * that we took from the item. Now that we have
1964 	 * finished doing everything, drop those references.
1965 	 */
1966 	if (hook) {
1967 		ng_unref_hook(hook);
1968 	}
1969 
1970 	if (was_reader) {
1971 		ng_leave_read(&node->input_queue);
1972 	} else {
1973 		ng_leave_write(&node->input_queue);
1974 	}
1975 	ng_unref(node);
1976 	return (error);
1977 }
1978 
1979 /***********************************************************************
1980  * Implement the 'generic' control messages
1981  ***********************************************************************/
1982 static int
1983 ng_generic_msg(node_p here, item_p item, hook_p lasthook)
1984 {
1985 	int error = 0;
1986 	struct ng_mesg *msg;
1987 	struct ng_mesg *resp = NULL;
1988 
1989 	NGI_GET_MSG(item, msg);
1990 	if (msg->header.typecookie != NGM_GENERIC_COOKIE) {
1991 		error = EINVAL;
1992 		goto out;
1993 	}
1994 	switch (msg->header.cmd) {
1995 	case NGM_SHUTDOWN:
1996 		ng_rmnode(here);
1997 		break;
1998 	case NGM_MKPEER:
1999 	    {
2000 		struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data;
2001 
2002 		if (msg->header.arglen != sizeof(*mkp)) {
2003 			error = EINVAL;
2004 			break;
2005 		}
2006 		mkp->type[sizeof(mkp->type) - 1] = '\0';
2007 		mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0';
2008 		mkp->peerhook[sizeof(mkp->peerhook) - 1] = '\0';
2009 		error = ng_mkpeer(here, mkp->ourhook, mkp->peerhook, mkp->type);
2010 		break;
2011 	    }
2012 	case NGM_CONNECT:
2013 	    {
2014 		struct ngm_connect *const con =
2015 			(struct ngm_connect *) msg->data;
2016 		node_p node2;
2017 
2018 		if (msg->header.arglen != sizeof(*con)) {
2019 			error = EINVAL;
2020 			break;
2021 		}
2022 		con->path[sizeof(con->path) - 1] = '\0';
2023 		con->ourhook[sizeof(con->ourhook) - 1] = '\0';
2024 		con->peerhook[sizeof(con->peerhook) - 1] = '\0';
2025 		/* Don't forget we get a reference.. */
2026 		error = ng_path2noderef(here, con->path, &node2, NULL);
2027 		if (error)
2028 			break;
2029 		error = ng_con_nodes(here, con->ourhook, node2, con->peerhook);
2030 		ng_unref(node2);
2031 		break;
2032 	    }
2033 	case NGM_NAME:
2034 	    {
2035 		struct ngm_name *const nam = (struct ngm_name *) msg->data;
2036 
2037 		if (msg->header.arglen != sizeof(*nam)) {
2038 			error = EINVAL;
2039 			break;
2040 		}
2041 		nam->name[sizeof(nam->name) - 1] = '\0';
2042 		error = ng_name_node(here, nam->name);
2043 		break;
2044 	    }
2045 	case NGM_RMHOOK:
2046 	    {
2047 		struct ngm_rmhook *const rmh = (struct ngm_rmhook *) msg->data;
2048 		hook_p hook;
2049 
2050 		if (msg->header.arglen != sizeof(*rmh)) {
2051 			error = EINVAL;
2052 			break;
2053 		}
2054 		rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0';
2055 		if ((hook = ng_findhook(here, rmh->ourhook)) != NULL)
2056 			ng_destroy_hook(hook);
2057 		break;
2058 	    }
2059 	case NGM_NODEINFO:
2060 	    {
2061 		struct nodeinfo *ni;
2062 
2063 		NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT);
2064 		if (resp == NULL) {
2065 			error = ENOMEM;
2066 			break;
2067 		}
2068 
2069 		/* Fill in node info */
2070 		ni = (struct nodeinfo *) resp->data;
2071 		if (here->name != NULL)
2072 			strncpy(ni->name, here->name, NG_NODELEN);
2073 		strncpy(ni->type, here->type->name, NG_TYPELEN);
2074 		ni->id = ng_node2ID(here);
2075 		ni->hooks = here->numhooks;
2076 		break;
2077 	    }
2078 	case NGM_LISTHOOKS:
2079 	    {
2080 		const int nhooks = here->numhooks;
2081 		struct hooklist *hl;
2082 		struct nodeinfo *ni;
2083 		hook_p hook;
2084 
2085 		/* Get response struct */
2086 		NG_MKRESPONSE(resp, msg, sizeof(*hl)
2087 		    + (nhooks * sizeof(struct linkinfo)), M_NOWAIT);
2088 		if (resp == NULL) {
2089 			error = ENOMEM;
2090 			break;
2091 		}
2092 		hl = (struct hooklist *) resp->data;
2093 		ni = &hl->nodeinfo;
2094 
2095 		/* Fill in node info */
2096 		if (here->name)
2097 			strncpy(ni->name, here->name, NG_NODELEN);
2098 		strncpy(ni->type, here->type->name, NG_TYPELEN);
2099 		ni->id = ng_node2ID(here);
2100 
2101 		/* Cycle through the linked list of hooks */
2102 		ni->hooks = 0;
2103 		LIST_FOREACH(hook, &here->hooks, hooks) {
2104 			struct linkinfo *const link = &hl->link[ni->hooks];
2105 
2106 			if (ni->hooks >= nhooks) {
2107 				log(LOG_ERR, "%s: number of %s changed\n",
2108 				    __FUNCTION__, "hooks");
2109 				break;
2110 			}
2111 			if ((hook->flags & HK_INVALID) != 0)
2112 				continue;
2113 			strncpy(link->ourhook, hook->name, NG_HOOKLEN);
2114 			strncpy(link->peerhook, hook->peer->name, NG_HOOKLEN);
2115 			if (hook->peer->node->name[0] != '\0')
2116 				strncpy(link->nodeinfo.name,
2117 				    hook->peer->node->name, NG_NODELEN);
2118 			strncpy(link->nodeinfo.type,
2119 			   hook->peer->node->type->name, NG_TYPELEN);
2120 			link->nodeinfo.id = ng_node2ID(hook->peer->node);
2121 			link->nodeinfo.hooks = hook->peer->node->numhooks;
2122 			ni->hooks++;
2123 		}
2124 		break;
2125 	    }
2126 
2127 	case NGM_LISTNAMES:
2128 	case NGM_LISTNODES:
2129 	    {
2130 		const int unnamed = (msg->header.cmd == NGM_LISTNODES);
2131 		struct namelist *nl;
2132 		node_p node;
2133 		int num = 0;
2134 
2135 		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
2136 		/* Count number of nodes */
2137 		LIST_FOREACH(node, &ng_nodelist, nodes) {
2138 			if (unnamed || node->name[0] != '\0')
2139 				num++;
2140 		}
2141 		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
2142 
2143 		/* Get response struct */
2144 		NG_MKRESPONSE(resp, msg, sizeof(*nl)
2145 		    + (num * sizeof(struct nodeinfo)), M_NOWAIT);
2146 		if (resp == NULL) {
2147 			error = ENOMEM;
2148 			break;
2149 		}
2150 		nl = (struct namelist *) resp->data;
2151 
2152 		/* Cycle through the linked list of nodes */
2153 		nl->numnames = 0;
2154 		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
2155 		LIST_FOREACH(node, &ng_nodelist, nodes) {
2156 			struct nodeinfo *const np = &nl->nodeinfo[nl->numnames];
2157 
2158 			if (nl->numnames >= num) {
2159 				log(LOG_ERR, "%s: number of %s changed\n",
2160 				    __FUNCTION__, "nodes");
2161 				break;
2162 			}
2163 			if ((node->flags & NG_INVALID) != 0)
2164 				continue;
2165 			if (!unnamed && node->name[0] == '\0')
2166 				continue;
2167 			if (node->name[0] != '\0')
2168 				strncpy(np->name, node->name, NG_NODELEN);
2169 			strncpy(np->type, node->type->name, NG_TYPELEN);
2170 			np->id = ng_node2ID(node);
2171 			np->hooks = node->numhooks;
2172 			nl->numnames++;
2173 		}
2174 		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
2175 		break;
2176 	    }
2177 
2178 	case NGM_LISTTYPES:
2179 	    {
2180 		struct typelist *tl;
2181 		struct ng_type *type;
2182 		int num = 0;
2183 
2184 		mtx_enter(&ng_typelist_mtx, MTX_DEF);
2185 		/* Count number of types */
2186 		LIST_FOREACH(type, &ng_typelist, types)
2187 			num++;
2188 		mtx_exit(&ng_typelist_mtx, MTX_DEF);
2189 
2190 		/* Get response struct */
2191 		NG_MKRESPONSE(resp, msg, sizeof(*tl)
2192 		    + (num * sizeof(struct typeinfo)), M_NOWAIT);
2193 		if (resp == NULL) {
2194 			error = ENOMEM;
2195 			break;
2196 		}
2197 		tl = (struct typelist *) resp->data;
2198 
2199 		/* Cycle through the linked list of types */
2200 		tl->numtypes = 0;
2201 		mtx_enter(&ng_typelist_mtx, MTX_DEF);
2202 		LIST_FOREACH(type, &ng_typelist, types) {
2203 			struct typeinfo *const tp = &tl->typeinfo[tl->numtypes];
2204 
2205 			if (tl->numtypes >= num) {
2206 				log(LOG_ERR, "%s: number of %s changed\n",
2207 				    __FUNCTION__, "types");
2208 				break;
2209 			}
2210 			strncpy(tp->type_name, type->name, NG_TYPELEN);
2211 			tp->numnodes = type->refs;
2212 			tl->numtypes++;
2213 		}
2214 		mtx_exit(&ng_typelist_mtx, MTX_DEF);
2215 		break;
2216 	    }
2217 
2218 	case NGM_BINARY2ASCII:
2219 	    {
2220 		int bufSize = 20 * 1024;	/* XXX hard coded constant */
2221 		const struct ng_parse_type *argstype;
2222 		const struct ng_cmdlist *c;
2223 		struct ng_mesg *binary, *ascii;
2224 
2225 		/* Data area must contain a valid netgraph message */
2226 		binary = (struct ng_mesg *)msg->data;
2227 		if (msg->header.arglen < sizeof(struct ng_mesg)
2228 		    || msg->header.arglen - sizeof(struct ng_mesg)
2229 		      < binary->header.arglen) {
2230 			error = EINVAL;
2231 			break;
2232 		}
2233 
2234 		/* Get a response message with lots of room */
2235 		NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT);
2236 		if (resp == NULL) {
2237 			error = ENOMEM;
2238 			break;
2239 		}
2240 		ascii = (struct ng_mesg *)resp->data;
2241 
2242 		/* Copy binary message header to response message payload */
2243 		bcopy(binary, ascii, sizeof(*binary));
2244 
2245 		/* Find command by matching typecookie and command number */
2246 		for (c = here->type->cmdlist;
2247 		    c != NULL && c->name != NULL; c++) {
2248 			if (binary->header.typecookie == c->cookie
2249 			    && binary->header.cmd == c->cmd)
2250 				break;
2251 		}
2252 		if (c == NULL || c->name == NULL) {
2253 			for (c = ng_generic_cmds; c->name != NULL; c++) {
2254 				if (binary->header.typecookie == c->cookie
2255 				    && binary->header.cmd == c->cmd)
2256 					break;
2257 			}
2258 			if (c->name == NULL) {
2259 				NG_FREE_MSG(resp);
2260 				error = ENOSYS;
2261 				break;
2262 			}
2263 		}
2264 
2265 		/* Convert command name to ASCII */
2266 		snprintf(ascii->header.cmdstr, sizeof(ascii->header.cmdstr),
2267 		    "%s", c->name);
2268 
2269 		/* Convert command arguments to ASCII */
2270 		argstype = (binary->header.flags & NGF_RESP) ?
2271 		    c->respType : c->mesgType;
2272 		if (argstype == NULL)
2273 			*ascii->data = '\0';
2274 		else {
2275 			if ((error = ng_unparse(argstype,
2276 			    (u_char *)binary->data,
2277 			    ascii->data, bufSize)) != 0) {
2278 				NG_FREE_MSG(resp);
2279 				break;
2280 			}
2281 		}
2282 
2283 		/* Return the result as struct ng_mesg plus ASCII string */
2284 		bufSize = strlen(ascii->data) + 1;
2285 		ascii->header.arglen = bufSize;
2286 		resp->header.arglen = sizeof(*ascii) + bufSize;
2287 		break;
2288 	    }
2289 
2290 	case NGM_ASCII2BINARY:
2291 	    {
2292 		int bufSize = 2000;	/* XXX hard coded constant */
2293 		const struct ng_cmdlist *c;
2294 		const struct ng_parse_type *argstype;
2295 		struct ng_mesg *ascii, *binary;
2296 		int off = 0;
2297 
2298 		/* Data area must contain at least a struct ng_mesg + '\0' */
2299 		ascii = (struct ng_mesg *)msg->data;
2300 		if (msg->header.arglen < sizeof(*ascii) + 1
2301 		    || ascii->header.arglen < 1
2302 		    || msg->header.arglen
2303 		      < sizeof(*ascii) + ascii->header.arglen) {
2304 			error = EINVAL;
2305 			break;
2306 		}
2307 		ascii->data[ascii->header.arglen - 1] = '\0';
2308 
2309 		/* Get a response message with lots of room */
2310 		NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT);
2311 		if (resp == NULL) {
2312 			error = ENOMEM;
2313 			break;
2314 		}
2315 		binary = (struct ng_mesg *)resp->data;
2316 
2317 		/* Copy ASCII message header to response message payload */
2318 		bcopy(ascii, binary, sizeof(*ascii));
2319 
2320 		/* Find command by matching ASCII command string */
2321 		for (c = here->type->cmdlist;
2322 		    c != NULL && c->name != NULL; c++) {
2323 			if (strcmp(ascii->header.cmdstr, c->name) == 0)
2324 				break;
2325 		}
2326 		if (c == NULL || c->name == NULL) {
2327 			for (c = ng_generic_cmds; c->name != NULL; c++) {
2328 				if (strcmp(ascii->header.cmdstr, c->name) == 0)
2329 					break;
2330 			}
2331 			if (c->name == NULL) {
2332 				NG_FREE_MSG(resp);
2333 				error = ENOSYS;
2334 				break;
2335 			}
2336 		}
2337 
2338 		/* Convert command name to binary */
2339 		binary->header.cmd = c->cmd;
2340 		binary->header.typecookie = c->cookie;
2341 
2342 		/* Convert command arguments to binary */
2343 		argstype = (binary->header.flags & NGF_RESP) ?
2344 		    c->respType : c->mesgType;
2345 		if (argstype == NULL)
2346 			bufSize = 0;
2347 		else {
2348 			if ((error = ng_parse(argstype, ascii->data,
2349 			    &off, (u_char *)binary->data, &bufSize)) != 0) {
2350 				NG_FREE_MSG(resp);
2351 				break;
2352 			}
2353 		}
2354 
2355 		/* Return the result */
2356 		binary->header.arglen = bufSize;
2357 		resp->header.arglen = sizeof(*binary) + bufSize;
2358 		break;
2359 	    }
2360 
2361 	case NGM_TEXT_CONFIG:
2362 	case NGM_TEXT_STATUS:
2363 		/*
2364 		 * This one is tricky as it passes the command down to the
2365 		 * actual node, even though it is a generic type command.
2366 		 * This means we must assume that the item/msg is already freed
2367 		 * when control passes back to us.
2368 		 */
2369 		if (here->type->rcvmsg != NULL) {
2370 			NGI_MSG(item) = msg; /* put it back as we found it */
2371 			return((*here->type->rcvmsg)(here, item, lasthook));
2372 		}
2373 		/* Fall through if rcvmsg not supported */
2374 	default:
2375 		TRAP_ERROR;
2376 		error = EINVAL;
2377 	}
2378 	/*
2379 	 * Sometimes a generic message may be statically allocated
2380 	 * to avoid problems with allocating when in tight memeory situations.
2381 	 * Don't free it if it is so.
2382 	 * I break them appart here, because erros may cause a free if the item
2383 	 * in which case we'd be doing it twice.
2384 	 * they are kept together above, to simplify freeing.
2385 	 */
2386 out:
2387 	NG_RESPOND_MSG(error, here, item, resp);
2388 	if ( msg && ((msg->header.flags & NGF_STATIC) == 0))
2389 		NG_FREE_MSG(msg);
2390 	return (error);
2391 }
2392 
2393 /*
2394  * Copy a 'meta'.
2395  *
2396  * Returns new meta, or NULL if original meta is NULL or ENOMEM.
2397  */
2398 meta_p
2399 ng_copy_meta(meta_p meta)
2400 {
2401 	meta_p meta2;
2402 
2403 	if (meta == NULL)
2404 		return (NULL);
2405 	MALLOC(meta2, meta_p, meta->used_len, M_NETGRAPH_META, M_NOWAIT);
2406 	if (meta2 == NULL)
2407 		return (NULL);
2408 	meta2->allocated_len = meta->used_len;
2409 	bcopy(meta, meta2, meta->used_len);
2410 	return (meta2);
2411 }
2412 
2413 /************************************************************************
2414 			Module routines
2415 ************************************************************************/
2416 
2417 /*
2418  * Handle the loading/unloading of a netgraph node type module
2419  */
2420 int
2421 ng_mod_event(module_t mod, int event, void *data)
2422 {
2423 	struct ng_type *const type = data;
2424 	int s, error = 0;
2425 
2426 	switch (event) {
2427 	case MOD_LOAD:
2428 
2429 		/* Register new netgraph node type */
2430 		s = splnet();
2431 		if ((error = ng_newtype(type)) != 0) {
2432 			splx(s);
2433 			break;
2434 		}
2435 
2436 		/* Call type specific code */
2437 		if (type->mod_event != NULL)
2438 			if ((error = (*type->mod_event)(mod, event, data))) {
2439 				mtx_enter(&ng_typelist_mtx, MTX_DEF);
2440 				LIST_REMOVE(type, types);
2441 				mtx_exit(&ng_typelist_mtx, MTX_DEF);
2442 			}
2443 		splx(s);
2444 		break;
2445 
2446 	case MOD_UNLOAD:
2447 		s = splnet();
2448 		if (type->refs != 0)		/* make sure no nodes exist! */
2449 			error = EBUSY;
2450 		else {
2451 			if (type->mod_event != NULL) {	/* check with type */
2452 				error = (*type->mod_event)(mod, event, data);
2453 				if (error != 0) {	/* type refuses.. */
2454 					splx(s);
2455 					break;
2456 				}
2457 			}
2458 			mtx_enter(&ng_typelist_mtx, MTX_DEF);
2459 			LIST_REMOVE(type, types);
2460 			mtx_exit(&ng_typelist_mtx, MTX_DEF);
2461 		}
2462 		splx(s);
2463 		break;
2464 
2465 	default:
2466 		if (type->mod_event != NULL)
2467 			error = (*type->mod_event)(mod, event, data);
2468 		else
2469 			error = 0;		/* XXX ? */
2470 		break;
2471 	}
2472 	return (error);
2473 }
2474 
2475 /*
2476  * Handle loading and unloading for this code.
2477  * The only thing we need to link into is the NETISR strucure.
2478  */
2479 static int
2480 ngb_mod_event(module_t mod, int event, void *data)
2481 {
2482 	int s, error = 0;
2483 
2484 	switch (event) {
2485 	case MOD_LOAD:
2486 		/* Register line discipline */
2487 		mtx_init(&ng_worklist_mtx, "netgraph worklist mutex", 0);
2488 		mtx_init(&ng_typelist_mtx, "netgraph types mutex", 0);
2489 		mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", 0);
2490 		mtx_init(&ng_idhash_mtx, "netgraph idhash mutex", 0);
2491 		mtx_init(&ngq_mtx, "netgraph netisr mutex", 0);
2492 		s = splimp();
2493 		error = register_netisr(NETISR_NETGRAPH, ngintr);
2494 		splx(s);
2495 		break;
2496 	case MOD_UNLOAD:
2497 		/* You cant unload it because an interface may be using it.  */
2498 		error = EBUSY;
2499 		break;
2500 	default:
2501 		error = EOPNOTSUPP;
2502 		break;
2503 	}
2504 	return (error);
2505 }
2506 
2507 static moduledata_t netgraph_mod = {
2508 	"netgraph",
2509 	ngb_mod_event,
2510 	(NULL)
2511 };
2512 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2513 
2514 /************************************************************************
2515 			Queue element get/free routines
2516 ************************************************************************/
2517 
2518 
2519 static int			allocated;	/* number of items malloc'd */
2520 static int			maxalloc = 128;	/* limit the damage of a leak */
2521 static const int		ngqfreemax = 64;/* cache at most this many */
2522 static const int		ngqfreelow = 4; /* try malloc if free < this */
2523 static volatile int		ngqfreesize;	/* number of cached entries */
2524 #ifdef	ITEM_DEBUG
2525 static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist);
2526 #endif
2527 /*
2528  * Get a queue entry
2529  * This is usually called when a packet first enters netgraph.
2530  * By definition, this is usually from an interrupt, or from a user.
2531  * Users are not so important, but try be quick for the times that it's
2532  * an interrupt. Use atomic operations to cope with collisions
2533  * with interrupts and other processors. Assumes MALLOC is SMP safe.
2534  * XXX If reserve is low, we should try to get 2 from malloc as this
2535  * would indicate it often fails.
2536  */
2537 static item_p
2538 ng_getqblk(void)
2539 {
2540 	item_p item = NULL;
2541 
2542 	/*
2543 	 * Try get a cached queue block, or else allocate a new one
2544 	 * If we are less than our reserve, try malloc. If malloc
2545 	 * fails, then that's what the reserve is for...
2546 	 * Don't completely trust ngqfreesize, as it is subject
2547 	 * to races.. (it'll eventually catch up but may be out by one or two
2548 	 * for brief moments(under SMP or interrupts).
2549 	 * ngqfree is the final arbiter. We have our little reserve
2550 	 * because we use M_NOWAIT for malloc. This just helps us
2551 	 * avoid dropping packets while not increasing the time
2552 	 * we take to service the interrupt (on average) (we hope).
2553 	 */
2554 	for (;;) {
2555 		if ((ngqfreesize < ngqfreelow) || (ngqfree == NULL)) {
2556 			if (allocated < maxalloc) {  /* don't leak forever */
2557 				MALLOC(item, item_p ,
2558 				    sizeof(*item), M_NETGRAPH_ITEM,
2559 				    (M_NOWAIT | M_ZERO));
2560 				if (item) {
2561 #ifdef	ITEM_DEBUG
2562 					TAILQ_INSERT_TAIL(&ng_itemlist,
2563 								item, all);
2564 #endif	/* ITEM_DEBUG */
2565 					atomic_add_int(&allocated, 1);
2566 					break;
2567 				}
2568 			}
2569 		}
2570 
2571 		/*
2572 		 * We didn't or couldn't malloc.
2573 		 * try get one from our cache.
2574 		 * item must be NULL to get here.
2575 		 */
2576 		if ((item = ngqfree) != NULL) {
2577 			/*
2578 			 * Atomically try grab the first item
2579 			 * and put it's successor in its place.
2580 			 * If we fail, just try again.. someone else
2581 			 * beat us to this one or freed one.
2582 			 * Don't worry about races with ngqfreesize.
2583 			 * Close enough is good enough..
2584 			 */
2585 			if (atomic_cmpset_ptr(&ngqfree, item, item->el_next)) {
2586 				atomic_subtract_int(&ngqfreesize, 1);
2587 				break;
2588 			}
2589 			item = NULL;
2590 		} else {
2591 			/* We really ran out */
2592 			break;
2593 		}
2594 	}
2595 	item->el_flags &= ~NGQF_FREE;
2596 	return (item);
2597 }
2598 
2599 /*
2600  * Release a queue entry
2601  */
2602 void
2603 ng_free_item(item_p item)
2604 {
2605 
2606 	/*
2607 	 * The item may hold resources on it's own. We need to free
2608 	 * these before we can free the item. What they are depends upon
2609 	 * what kind of item it is. it is important that nodes zero
2610 	 * out pointers to resources that they remove from the item
2611 	 * or we release them again here.
2612 	 */
2613 	if (item->el_flags & NGQF_FREE) {
2614 		panic(" Freeing free queue item");
2615 	}
2616 	switch (item->el_flags & NGQF_D_M) {
2617 	case NGQF_DATA:
2618 		/* If we have an mbuf and metadata still attached.. */
2619 		NG_FREE_M(_NGI_M(item));
2620 		NG_FREE_META(_NGI_META(item));
2621 		break;
2622 	case NGQF_MESG:
2623 		_NGI_RETADDR(item) = NULL;
2624 		NG_FREE_MSG(_NGI_MSG(item));
2625 		break;
2626 	}
2627 		/* If we still have a node or hook referenced... */
2628 	if (item->el_dest) {
2629 		ng_unref(item->el_dest);
2630 		item->el_dest = NULL;
2631 	}
2632 	if (item->el_hook) {
2633 		ng_unref_hook(item->el_hook);
2634 		item->el_hook = NULL;
2635 	}
2636 	item->el_flags |= NGQF_FREE;
2637 
2638 	/*
2639 	 * We have freed any resources held by the item.
2640 	 * now we can free the item itself.
2641 	 */
2642 	if (ngqfreesize < ngqfreemax) { /* don't worry about races */
2643 		for (;;) {
2644 			item->el_next = ngqfree;
2645 			if (atomic_cmpset_ptr(&ngqfree, item->el_next, item)) {
2646 				break;
2647 			}
2648 		}
2649 		atomic_add_int(&ngqfreesize, 1);
2650 	} else {
2651 		/* This is the only place that should use this Macro */
2652 #ifdef	ITEM_DEBUG
2653 		TAILQ_REMOVE(&ng_itemlist, item, all);
2654 #endif	/* ITEM_DEBUG */
2655 		NG_FREE_ITEM_REAL(item);
2656 		atomic_subtract_int(&allocated, 1);
2657 	}
2658 }
2659 
2660 #ifdef	ITEM_DEBUG
2661 void
2662 dumpitem(item_p item, char *file, int line)
2663 {
2664 	if (item->el_flags & NGQF_FREE) {
2665 		printf(" Free item, freed at %s, line %d\n",
2666 			item->lastfile, item->lastline);
2667 	} else {
2668 		printf(" ACTIVE item, last used at %s, line %d",
2669 			item->lastfile, item->lastline);
2670 		if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {
2671 			printf(" - retaddr[%d]:\n", _NGI_RETADDR(item));
2672 		} else {
2673 			printf(" - [data]\n");
2674 		}
2675 	}
2676 	printf(" problem discovered at file %s, line %d\n", file, line);
2677 	if (item->el_dest)
2678 		printf("node %X ([%x])\n",
2679 			item->el_dest, ng_node2ID(item->el_dest));
2680 }
2681 
2682 static int
2683 sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS)
2684 {
2685 	int error;
2686 	int val;
2687 	item_p item;
2688 	int i;
2689 
2690 	val = allocated;
2691 	i = 1;
2692 	error = sysctl_handle_int(oidp, &val, sizeof(int), req);
2693 	TAILQ_FOREACH(item, &ng_itemlist, all) {
2694 		if (item->el_flags & NGQF_FREE) {
2695 			printf("[%d] free item, freed at %s, line %d\n",
2696 				i++, item->lastfile, item->lastline);
2697 		} else {
2698 			printf("[%d] ACTIVE item, last used at %s, line %d",
2699 				i++, item->lastfile, item->lastline);
2700 			if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {
2701 				printf(" - retaddr[%d]:\n", _NGI_RETADDR(item));
2702 			} else {
2703 				printf(" - [data]\n");
2704 			}
2705 		}
2706 		if (item->el_dest) {
2707 			printf("node %X ([%x])",
2708 				item->el_dest, ng_node2ID(item->el_dest));
2709 			printf("<%X>\n",item->el_dest->input_queue.q_flags);
2710 		}
2711 	}
2712 	return error;
2713 }
2714 
2715 SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, CTLTYPE_INT | CTLFLAG_RD,
2716     0, 0, sysctl_debug_ng_dump_items, "I", "Number of allocated items");
2717 #endif	/* ITEM_DEBUG */
2718 
2719 
2720 /***********************************************************************
2721 * Worklist routines
2722 **********************************************************************/
2723 /* NETISR thread enters here */
2724 /*
2725  * Pick a node off the list of nodes with work,
2726  * try get an item to process off it.
2727  * If there are no more, remove the node from the list.
2728  */
2729 static void
2730 ngintr(void)
2731 {
2732 	item_p item;
2733 	node_p  node = NULL;
2734 
2735 	for (;;) {
2736 		mtx_enter(&ng_worklist_mtx, MTX_SPIN);
2737 		node = TAILQ_FIRST(&ng_worklist);
2738 		if (!node) {
2739 			mtx_exit(&ng_worklist_mtx, MTX_SPIN);
2740 			break;
2741 		}
2742 		TAILQ_REMOVE(&ng_worklist, node, work);
2743 		mtx_exit(&ng_worklist_mtx, MTX_SPIN);
2744 		/*
2745 		 * We have the node. We also take over the reference
2746 		 * that the list had on it.
2747 		 * Now process as much as you can, until it won't
2748 		 * let you have another item off the queue.
2749 		 * All this time, keep the reference
2750 		 * that lets us be sure that the node still exists.
2751 		 * Let the reference go at the last minute.
2752 		 */
2753 		for (;;) {
2754 			mtx_enter(&node->input_queue.q_mtx, MTX_SPIN);
2755 			item = ng_dequeue(&node->input_queue);
2756 			if (item == NULL) {
2757 				/*
2758 				 * Say we are on the queue as long as
2759 				 * we are processing it here.
2760 				 * it probably wouldn't come here while we
2761 				 * are processing anyhow.
2762 				 */
2763 				node->flags &= ~NG_WORKQ;
2764 				mtx_exit(&node->input_queue.q_mtx, MTX_SPIN);
2765 				ng_unref(node);
2766 				break; /* go look for another node */
2767 			} else {
2768 				mtx_exit(&node->input_queue.q_mtx, MTX_SPIN);
2769 #ifdef	ITEM_DEBUG
2770         _ngi_check(item, __FILE__, __LINE__);
2771 #endif
2772 				ng_apply_item(node, item);
2773 			}
2774 		}
2775 	}
2776 }
2777 
2778 static void
2779 ng_worklist_remove(node_p node)
2780 {
2781 	mtx_enter(&ng_worklist_mtx, MTX_SPIN);
2782 	if (node->flags & NG_WORKQ) {
2783 		TAILQ_REMOVE(&ng_worklist, node, work);
2784 		ng_unref(node);
2785 	}
2786 	node->flags &= ~NG_WORKQ;
2787 	mtx_exit(&ng_worklist_mtx, MTX_SPIN);
2788 }
2789 
2790 static void
2791 ng_setisr(node_p node)
2792 {
2793 	mtx_enter(&ng_worklist_mtx, MTX_SPIN);
2794 	if ((node->flags & NG_WORKQ) == 0) {
2795 		/*
2796 		 * If we are not already on the work queue,
2797 		 * then put us on.
2798 		 */
2799 		node->flags |= NG_WORKQ;
2800 		TAILQ_INSERT_TAIL(&ng_worklist, node, work);
2801 		node->refs++;
2802 	}
2803 	mtx_exit(&ng_worklist_mtx, MTX_SPIN);
2804 	schednetisr(NETISR_NETGRAPH);
2805 }
2806 
2807 
2808 /***********************************************************************
2809 * Externally useable functions to set up a queue item ready for sending
2810 ***********************************************************************/
2811 
2812 #ifdef	ITEM_DEBUG
2813 #define	DEBUG_CHECKS							\
2814 	do {								\
2815 		if (item->el_dest ) {					\
2816 			printf("item already has node");		\
2817 			Debugger("has node");				\
2818 			ng_unref(item->el_dest);			\
2819 			item->el_dest = NULL;				\
2820 		}							\
2821 		if (item->el_hook ) {					\
2822 			printf("item already has hook");		\
2823 			Debugger("has hook");				\
2824 			ng_unref_hook(item->el_hook);			\
2825 			item->el_hook = NULL;				\
2826 		}							\
2827 	} while (0)
2828 #else
2829 #define DEBUG_CHECKS
2830 #endif
2831 
2832 /*
2833  * Put elements into the item.
2834  * Hook and node references will be removed when the item is dequeued.
2835  * (or equivalent)
2836  * (XXX) Unsafe because no reference held by peer on remote node.
2837  * remote node might go away in this timescale.
2838  * We know the hooks can't go away because that would require getting
2839  * a writer item on both nodes and we must have at least a  reader
2840  * here to eb able to do this.
2841  * Note that the hook loaded is the REMOTE hook.
2842  *
2843  * This is possibly in the critical path for new data.
2844  */
2845 item_p
2846 ng_package_data(struct mbuf *m, meta_p meta)
2847 {
2848 	item_p item;
2849 
2850 	if ((item = ng_getqblk()) == NULL) {
2851 		NG_FREE_M(m);
2852 		NG_FREE_META(meta);
2853 		return (NULL);
2854 	}
2855 	DEBUG_CHECKS;
2856 	item->el_flags = NGQF_DATA;
2857 	item->el_next = NULL;
2858 	NGI_M(item) = m;
2859 	NGI_META(item) = meta;
2860 	return (item);
2861 }
2862 
2863 /*
2864  * Allocate a queue item and put items into it..
2865  * Evaluate the address as this will be needed to queue it and
2866  * to work out what some of the fields should be.
2867  * Hook and node references will be removed when the item is dequeued.
2868  * (or equivalent)
2869  */
2870 item_p
2871 ng_package_msg(struct ng_mesg *msg)
2872 {
2873 	item_p item;
2874 
2875 	if ((item = ng_getqblk()) == NULL) {
2876 		if ((msg->header.flags & NGF_STATIC) == 0) {
2877 			NG_FREE_MSG(msg);
2878 		}
2879 		return (NULL);
2880 	}
2881 	DEBUG_CHECKS;
2882 	item->el_flags = NGQF_MESG;
2883 	item->el_next = NULL;
2884 	/*
2885 	 * Set the current lasthook into the queue item
2886 	 */
2887 	NGI_MSG(item) = msg;
2888 	NGI_RETADDR(item) = NULL;
2889 	return (item);
2890 }
2891 
2892 
2893 
2894 #define SET_RETADDR							\
2895 	do {	/* Data items don't have retaddrs */			\
2896 		if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {		\
2897 			if (retaddr) {					\
2898 				NGI_RETADDR(item) = retaddr;		\
2899 			} else {					\
2900 				/*					\
2901 				 * The old return address should be ok.	\
2902 				 * If there isn't one, use the address	\
2903 				 * here.				\
2904 				 */					\
2905 				if (NGI_RETADDR(item) == 0) {		\
2906 					NGI_RETADDR(item)		\
2907 						= ng_node2ID(here);	\
2908 				}					\
2909 			}						\
2910 		}							\
2911 	} while (0)
2912 
2913 int
2914 ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr)
2915 {
2916 	DEBUG_CHECKS;
2917 	/*
2918 	 * Quick sanity check..
2919 	 */
2920 	if ((hook == NULL)
2921 	|| ((hook->flags & HK_INVALID) != 0)
2922 	|| (hook->peer == NULL)
2923 	|| ((hook->peer->flags & HK_INVALID) != 0)
2924 	|| ((hook->peer->node->flags & NG_INVALID) != 0)) {
2925 		NG_FREE_ITEM(item);
2926 		return (EINVAL);
2927 	}
2928 
2929 	/*
2930 	 * Transfer our interest to the other (peer) end.
2931 	 * note sleazy use of 'hook'.
2932 	 */
2933 	item->el_hook = hook->peer;
2934 	item->el_hook->refs++;	/* don't let it go away while on the queue */
2935 	item->el_dest = hook->peer->node; /* sleaze */
2936 	item->el_dest->refs++; /* XXX dangerous, not atomic */
2937 	SET_RETADDR;
2938 	return (0);
2939 }
2940 
2941 int
2942 ng_address_path(node_p here, item_p item, char *address, ng_ID_t retaddr)
2943 {
2944 	node_p  dest = NULL;
2945 	hook_p	hook = NULL;
2946 	int     error;
2947 
2948 	DEBUG_CHECKS;
2949 	/*
2950 	 * Note that ng_path2noderef increments the reference count
2951 	 * on the node for us if it finds one. So we don't have to.
2952 	 */
2953 	error = ng_path2noderef(here, address, &dest, &hook);
2954 	if (error) {
2955 		NG_FREE_ITEM(item);
2956 		return (EINVAL);
2957 	}
2958 	item->el_dest = dest;
2959 	if (( item->el_hook = hook))
2960 		hook->refs++;	/* don't let it go away while on the queue */
2961 	SET_RETADDR;
2962 	return (0);
2963 }
2964 
2965 int
2966 ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr)
2967 {
2968 	node_p dest;
2969 
2970 	DEBUG_CHECKS;
2971 	/*
2972 	 * Find the target node.
2973 	 */
2974 	dest = ng_ID2noderef(ID); /* GETS REFERENCE! */
2975 	if (dest == NULL) {
2976 		NG_FREE_ITEM(item);
2977 		return(EINVAL);
2978 	}
2979 	/* Fill out the contents */
2980 	item->el_flags = NGQF_MESG;
2981 	item->el_next = NULL;
2982 	item->el_dest = dest;
2983 	item->el_hook = NULL;
2984 	/* NGI_RETADDR(item) = ng_node2ID(here); not sure why its here XXX */
2985 	SET_RETADDR;
2986 	return (0);
2987 }
2988 
2989 /*
2990  * special case to send a message to self (e.g. destroy node)
2991  * Possibly indicate an arrival hook too.
2992  * Useful for removing that hook :-)
2993  */
2994 item_p
2995 ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg)
2996 {
2997 	item_p item;
2998 
2999 	/*
3000 	 * Find the target node.
3001 	 * If there is a HOOK argument, then use that in preference
3002 	 * to the address.
3003 	 */
3004 	if ((item = ng_getqblk()) == NULL) {
3005 		if ((msg->header.flags & NGF_STATIC) == 0) {
3006 			NG_FREE_MSG(msg);
3007 		}
3008 		return (NULL);
3009 	}
3010 
3011 	/* Fill out the contents */
3012 	item->el_flags = NGQF_MESG;
3013 	item->el_next = NULL;
3014 	item->el_dest = here;
3015 	here->refs++; /* XXX  not atomic, + May have other races */
3016 	item->el_hook = hook;
3017 	if (hook)
3018 		hook->refs++;
3019 	NGI_MSG(item) = msg;
3020 	NGI_RETADDR(item) = ng_node2ID(here);
3021 	return (item);
3022 }
3023 
3024 /*
3025  * Set the address, if none given, give the node here.
3026  */
3027 void
3028 ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr)
3029 {
3030 	if (retaddr) {
3031 		NGI_RETADDR(item) = retaddr;
3032 	} else {
3033 		/*
3034 		 * The old return address should be ok.
3035 		 * If there isn't one, use the address here.
3036 		 */
3037 		NGI_RETADDR(item) = ng_node2ID(here);
3038 	}
3039 }
3040 
3041 #define TESTING
3042 #ifdef TESTING
3043 /* just test all the macros */
3044 void
3045 ng_macro_test(item_p item);
3046 void
3047 ng_macro_test(item_p item)
3048 {
3049 	node_p node = NULL;
3050 	hook_p hook = NULL;
3051 	struct mbuf *m;
3052 	meta_p meta;
3053 	struct ng_mesg *msg;
3054 	ng_ID_t retaddr;
3055 	int	error;
3056 
3057 	NGI_GET_M(item, m);
3058 	NGI_GET_META(item, meta);
3059 	NGI_GET_MSG(item, msg);
3060 	retaddr = NGI_RETADDR(item);
3061 	NG_SEND_DATA(error, hook, m, meta);
3062 	NG_SEND_DATA_ONLY(error, hook, m);
3063 	NG_FWD_NEW_DATA(error, item, hook, m);
3064 	NG_FWD_DATA(error, item, hook);
3065 	NG_SEND_MSG_HOOK(error, node, msg, hook, retaddr);
3066 	NG_SEND_MSG_ID(error, node, msg, retaddr, retaddr);
3067 	NG_SEND_MSG_PATH(error, node, msg, ".:", retaddr);
3068 	NG_QUEUE_MSG(error, node, msg, ".:", retaddr);
3069 	NG_FWD_MSG_HOOK(error, node, item, hook, retaddr);
3070 }
3071 #endif /* TESTING */
3072 
3073