1 /* 2 * ng_base.c 3 */ 4 5 /*- 6 * Copyright (c) 1996-1999 Whistle Communications, Inc. 7 * All rights reserved. 8 * 9 * Subject to the following obligations and disclaimer of warranty, use and 10 * redistribution of this software, in source or object code forms, with or 11 * without modifications are expressly permitted by Whistle Communications; 12 * provided, however, that: 13 * 1. Any and all reproductions of the source or object code must include the 14 * copyright notice above and the following disclaimer of warranties; and 15 * 2. No rights are granted, in any manner or form, to use Whistle 16 * Communications, Inc. trademarks, including the mark "WHISTLE 17 * COMMUNICATIONS" on advertising, endorsements, or otherwise except as 18 * such appears in the above copyright notice or in the software. 19 * 20 * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND 21 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO 22 * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, 23 * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF 24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 25 * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY 26 * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS 27 * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE. 28 * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES 29 * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING 30 * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 31 * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR 32 * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 35 * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY 36 * OF SUCH DAMAGE. 37 * 38 * Authors: Julian Elischer <julian@freebsd.org> 39 * Archie Cobbs <archie@freebsd.org> 40 * 41 * $FreeBSD$ 42 * $Whistle: ng_base.c,v 1.39 1999/01/28 23:54:53 julian Exp $ 43 */ 44 45 /* 46 * This file implements the base netgraph code. 47 */ 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/ctype.h> 52 #include <sys/errno.h> 53 #include <sys/kdb.h> 54 #include <sys/kernel.h> 55 #include <sys/ktr.h> 56 #include <sys/limits.h> 57 #include <sys/malloc.h> 58 #include <sys/mbuf.h> 59 #include <sys/queue.h> 60 #include <sys/sysctl.h> 61 #include <sys/syslog.h> 62 #include <sys/refcount.h> 63 #include <sys/proc.h> 64 #include <sys/vimage.h> 65 #include <sys/unistd.h> 66 #include <sys/kthread.h> 67 #include <sys/smp.h> 68 #include <machine/cpu.h> 69 70 #include <net/netisr.h> 71 72 #include <netgraph/ng_message.h> 73 #include <netgraph/netgraph.h> 74 #include <netgraph/ng_parse.h> 75 76 MODULE_VERSION(netgraph, NG_ABI_VERSION); 77 78 #ifndef VIMAGE 79 #ifndef VIMAGE_GLOBALS 80 struct vnet_netgraph vnet_netgraph_0; 81 #endif 82 #endif 83 84 /* Mutex to protect topology events. */ 85 static struct mtx ng_topo_mtx; 86 87 static vnet_attach_fn vnet_netgraph_iattach; 88 #ifdef VIMAGE 89 static vnet_detach_fn vnet_netgraph_idetach; 90 #endif 91 92 #ifdef NETGRAPH_DEBUG 93 static struct mtx ng_nodelist_mtx; /* protects global node/hook lists */ 94 static struct mtx ngq_mtx; /* protects the queue item list */ 95 96 static SLIST_HEAD(, ng_node) ng_allnodes; 97 static LIST_HEAD(, ng_node) ng_freenodes; /* in debug, we never free() them */ 98 static SLIST_HEAD(, ng_hook) ng_allhooks; 99 static LIST_HEAD(, ng_hook) ng_freehooks; /* in debug, we never free() them */ 100 101 static void ng_dumpitems(void); 102 static void ng_dumpnodes(void); 103 static void ng_dumphooks(void); 104 105 #endif /* NETGRAPH_DEBUG */ 106 /* 107 * DEAD versions of the structures. 108 * In order to avoid races, it is sometimes neccesary to point 109 * at SOMETHING even though theoretically, the current entity is 110 * INVALID. Use these to avoid these races. 111 */ 112 struct ng_type ng_deadtype = { 113 NG_ABI_VERSION, 114 "dead", 115 NULL, /* modevent */ 116 NULL, /* constructor */ 117 NULL, /* rcvmsg */ 118 NULL, /* shutdown */ 119 NULL, /* newhook */ 120 NULL, /* findhook */ 121 NULL, /* connect */ 122 NULL, /* rcvdata */ 123 NULL, /* disconnect */ 124 NULL, /* cmdlist */ 125 }; 126 127 struct ng_node ng_deadnode = { 128 "dead", 129 &ng_deadtype, 130 NGF_INVALID, 131 0, /* numhooks */ 132 NULL, /* private */ 133 0, /* ID */ 134 LIST_HEAD_INITIALIZER(ng_deadnode.hooks), 135 {}, /* all_nodes list entry */ 136 {}, /* id hashtable list entry */ 137 { 0, 138 0, 139 {}, /* should never use! (should hang) */ 140 {}, /* workqueue entry */ 141 STAILQ_HEAD_INITIALIZER(ng_deadnode.nd_input_queue.queue), 142 }, 143 1, /* refs */ 144 NULL, /* vnet */ 145 #ifdef NETGRAPH_DEBUG 146 ND_MAGIC, 147 __FILE__, 148 __LINE__, 149 {NULL} 150 #endif /* NETGRAPH_DEBUG */ 151 }; 152 153 struct ng_hook ng_deadhook = { 154 "dead", 155 NULL, /* private */ 156 HK_INVALID | HK_DEAD, 157 0, /* undefined data link type */ 158 &ng_deadhook, /* Peer is self */ 159 &ng_deadnode, /* attached to deadnode */ 160 {}, /* hooks list */ 161 NULL, /* override rcvmsg() */ 162 NULL, /* override rcvdata() */ 163 1, /* refs always >= 1 */ 164 #ifdef NETGRAPH_DEBUG 165 HK_MAGIC, 166 __FILE__, 167 __LINE__, 168 {NULL} 169 #endif /* NETGRAPH_DEBUG */ 170 }; 171 172 /* 173 * END DEAD STRUCTURES 174 */ 175 /* List nodes with unallocated work */ 176 static STAILQ_HEAD(, ng_node) ng_worklist = STAILQ_HEAD_INITIALIZER(ng_worklist); 177 static struct mtx ng_worklist_mtx; /* MUST LOCK NODE FIRST */ 178 179 /* List of installed types */ 180 static LIST_HEAD(, ng_type) ng_typelist; 181 static struct mtx ng_typelist_mtx; 182 183 /* Hash related definitions */ 184 /* XXX Don't need to initialise them because it's a LIST */ 185 #ifdef VIMAGE_GLOBALS 186 static LIST_HEAD(, ng_node) ng_ID_hash[NG_ID_HASH_SIZE]; 187 #endif 188 static struct mtx ng_idhash_mtx; 189 /* Method to find a node.. used twice so do it here */ 190 #define NG_IDHASH_FN(ID) ((ID) % (NG_ID_HASH_SIZE)) 191 #define NG_IDHASH_FIND(ID, node) \ 192 do { \ 193 mtx_assert(&ng_idhash_mtx, MA_OWNED); \ 194 LIST_FOREACH(node, &V_ng_ID_hash[NG_IDHASH_FN(ID)], \ 195 nd_idnodes) { \ 196 if (NG_NODE_IS_VALID(node) \ 197 && (NG_NODE_ID(node) == ID)) { \ 198 break; \ 199 } \ 200 } \ 201 } while (0) 202 203 #ifdef VIMAGE_GLOBALS 204 static LIST_HEAD(, ng_node) ng_name_hash[NG_NAME_HASH_SIZE]; 205 #endif 206 static struct mtx ng_namehash_mtx; 207 #define NG_NAMEHASH(NAME, HASH) \ 208 do { \ 209 u_char h = 0; \ 210 const u_char *c; \ 211 for (c = (const u_char*)(NAME); *c; c++)\ 212 h += *c; \ 213 (HASH) = h % (NG_NAME_HASH_SIZE); \ 214 } while (0) 215 216 217 /* Internal functions */ 218 static int ng_add_hook(node_p node, const char *name, hook_p * hookp); 219 static int ng_generic_msg(node_p here, item_p item, hook_p lasthook); 220 static ng_ID_t ng_decodeidname(const char *name); 221 static int ngb_mod_event(module_t mod, int event, void *data); 222 static void ng_worklist_add(node_p node); 223 static void ngthread(void *); 224 static int ng_apply_item(node_p node, item_p item, int rw); 225 static void ng_flush_input_queue(node_p node); 226 static node_p ng_ID2noderef(ng_ID_t ID); 227 static int ng_con_nodes(item_p item, node_p node, const char *name, 228 node_p node2, const char *name2); 229 static int ng_con_part2(node_p node, item_p item, hook_p hook); 230 static int ng_con_part3(node_p node, item_p item, hook_p hook); 231 static int ng_mkpeer(node_p node, const char *name, 232 const char *name2, char *type); 233 234 /* Imported, these used to be externally visible, some may go back. */ 235 void ng_destroy_hook(hook_p hook); 236 int ng_path2noderef(node_p here, const char *path, 237 node_p *dest, hook_p *lasthook); 238 int ng_make_node(const char *type, node_p *nodepp); 239 int ng_path_parse(char *addr, char **node, char **path, char **hook); 240 void ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3); 241 void ng_unname(node_p node); 242 243 244 /* Our own netgraph malloc type */ 245 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages"); 246 MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", "netgraph hook structures"); 247 MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", "netgraph node structures"); 248 MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", "netgraph item structures"); 249 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage"); 250 251 /* Should not be visible outside this file */ 252 253 #define _NG_ALLOC_HOOK(hook) \ 254 hook = malloc(sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO) 255 #define _NG_ALLOC_NODE(node) \ 256 node = malloc(sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO) 257 258 #define NG_QUEUE_LOCK_INIT(n) \ 259 mtx_init(&(n)->q_mtx, "ng_node", NULL, MTX_DEF) 260 #define NG_QUEUE_LOCK(n) \ 261 mtx_lock(&(n)->q_mtx) 262 #define NG_QUEUE_UNLOCK(n) \ 263 mtx_unlock(&(n)->q_mtx) 264 #define NG_WORKLIST_LOCK_INIT() \ 265 mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_DEF) 266 #define NG_WORKLIST_LOCK() \ 267 mtx_lock(&ng_worklist_mtx) 268 #define NG_WORKLIST_UNLOCK() \ 269 mtx_unlock(&ng_worklist_mtx) 270 #define NG_WORKLIST_SLEEP() \ 271 mtx_sleep(&ng_worklist, &ng_worklist_mtx, PI_NET, "sleep", 0) 272 #define NG_WORKLIST_WAKEUP() \ 273 wakeup_one(&ng_worklist) 274 275 #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ 276 /* 277 * In debug mode: 278 * In an attempt to help track reference count screwups 279 * we do not free objects back to the malloc system, but keep them 280 * in a local cache where we can examine them and keep information safely 281 * after they have been freed. 282 * We use this scheme for nodes and hooks, and to some extent for items. 283 */ 284 static __inline hook_p 285 ng_alloc_hook(void) 286 { 287 hook_p hook; 288 SLIST_ENTRY(ng_hook) temp; 289 mtx_lock(&ng_nodelist_mtx); 290 hook = LIST_FIRST(&ng_freehooks); 291 if (hook) { 292 LIST_REMOVE(hook, hk_hooks); 293 bcopy(&hook->hk_all, &temp, sizeof(temp)); 294 bzero(hook, sizeof(struct ng_hook)); 295 bcopy(&temp, &hook->hk_all, sizeof(temp)); 296 mtx_unlock(&ng_nodelist_mtx); 297 hook->hk_magic = HK_MAGIC; 298 } else { 299 mtx_unlock(&ng_nodelist_mtx); 300 _NG_ALLOC_HOOK(hook); 301 if (hook) { 302 hook->hk_magic = HK_MAGIC; 303 mtx_lock(&ng_nodelist_mtx); 304 SLIST_INSERT_HEAD(&ng_allhooks, hook, hk_all); 305 mtx_unlock(&ng_nodelist_mtx); 306 } 307 } 308 return (hook); 309 } 310 311 static __inline node_p 312 ng_alloc_node(void) 313 { 314 node_p node; 315 SLIST_ENTRY(ng_node) temp; 316 mtx_lock(&ng_nodelist_mtx); 317 node = LIST_FIRST(&ng_freenodes); 318 if (node) { 319 LIST_REMOVE(node, nd_nodes); 320 bcopy(&node->nd_all, &temp, sizeof(temp)); 321 bzero(node, sizeof(struct ng_node)); 322 bcopy(&temp, &node->nd_all, sizeof(temp)); 323 mtx_unlock(&ng_nodelist_mtx); 324 node->nd_magic = ND_MAGIC; 325 } else { 326 mtx_unlock(&ng_nodelist_mtx); 327 _NG_ALLOC_NODE(node); 328 if (node) { 329 node->nd_magic = ND_MAGIC; 330 mtx_lock(&ng_nodelist_mtx); 331 SLIST_INSERT_HEAD(&ng_allnodes, node, nd_all); 332 mtx_unlock(&ng_nodelist_mtx); 333 } 334 } 335 return (node); 336 } 337 338 #define NG_ALLOC_HOOK(hook) do { (hook) = ng_alloc_hook(); } while (0) 339 #define NG_ALLOC_NODE(node) do { (node) = ng_alloc_node(); } while (0) 340 341 342 #define NG_FREE_HOOK(hook) \ 343 do { \ 344 mtx_lock(&ng_nodelist_mtx); \ 345 LIST_INSERT_HEAD(&ng_freehooks, hook, hk_hooks); \ 346 hook->hk_magic = 0; \ 347 mtx_unlock(&ng_nodelist_mtx); \ 348 } while (0) 349 350 #define NG_FREE_NODE(node) \ 351 do { \ 352 mtx_lock(&ng_nodelist_mtx); \ 353 LIST_INSERT_HEAD(&ng_freenodes, node, nd_nodes); \ 354 node->nd_magic = 0; \ 355 mtx_unlock(&ng_nodelist_mtx); \ 356 } while (0) 357 358 #else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 359 360 #define NG_ALLOC_HOOK(hook) _NG_ALLOC_HOOK(hook) 361 #define NG_ALLOC_NODE(node) _NG_ALLOC_NODE(node) 362 363 #define NG_FREE_HOOK(hook) do { free((hook), M_NETGRAPH_HOOK); } while (0) 364 #define NG_FREE_NODE(node) do { free((node), M_NETGRAPH_NODE); } while (0) 365 366 #endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 367 368 /* Set this to kdb_enter("X") to catch all errors as they occur */ 369 #ifndef TRAP_ERROR 370 #define TRAP_ERROR() 371 #endif 372 373 #ifdef VIMAGE_GLOBALS 374 static ng_ID_t nextID; 375 #endif 376 377 #ifdef INVARIANTS 378 #define CHECK_DATA_MBUF(m) do { \ 379 struct mbuf *n; \ 380 int total; \ 381 \ 382 M_ASSERTPKTHDR(m); \ 383 for (total = 0, n = (m); n != NULL; n = n->m_next) { \ 384 total += n->m_len; \ 385 if (n->m_nextpkt != NULL) \ 386 panic("%s: m_nextpkt", __func__); \ 387 } \ 388 \ 389 if ((m)->m_pkthdr.len != total) { \ 390 panic("%s: %d != %d", \ 391 __func__, (m)->m_pkthdr.len, total); \ 392 } \ 393 } while (0) 394 #else 395 #define CHECK_DATA_MBUF(m) 396 #endif 397 398 #define ERROUT(x) do { error = (x); goto done; } while (0) 399 400 /************************************************************************ 401 Parse type definitions for generic messages 402 ************************************************************************/ 403 404 /* Handy structure parse type defining macro */ 405 #define DEFINE_PARSE_STRUCT_TYPE(lo, up, args) \ 406 static const struct ng_parse_struct_field \ 407 ng_ ## lo ## _type_fields[] = NG_GENERIC_ ## up ## _INFO args; \ 408 static const struct ng_parse_type ng_generic_ ## lo ## _type = { \ 409 &ng_parse_struct_type, \ 410 &ng_ ## lo ## _type_fields \ 411 } 412 413 DEFINE_PARSE_STRUCT_TYPE(mkpeer, MKPEER, ()); 414 DEFINE_PARSE_STRUCT_TYPE(connect, CONNECT, ()); 415 DEFINE_PARSE_STRUCT_TYPE(name, NAME, ()); 416 DEFINE_PARSE_STRUCT_TYPE(rmhook, RMHOOK, ()); 417 DEFINE_PARSE_STRUCT_TYPE(nodeinfo, NODEINFO, ()); 418 DEFINE_PARSE_STRUCT_TYPE(typeinfo, TYPEINFO, ()); 419 DEFINE_PARSE_STRUCT_TYPE(linkinfo, LINKINFO, (&ng_generic_nodeinfo_type)); 420 421 /* Get length of an array when the length is stored as a 32 bit 422 value immediately preceding the array -- as with struct namelist 423 and struct typelist. */ 424 static int 425 ng_generic_list_getLength(const struct ng_parse_type *type, 426 const u_char *start, const u_char *buf) 427 { 428 return *((const u_int32_t *)(buf - 4)); 429 } 430 431 /* Get length of the array of struct linkinfo inside a struct hooklist */ 432 static int 433 ng_generic_linkinfo_getLength(const struct ng_parse_type *type, 434 const u_char *start, const u_char *buf) 435 { 436 const struct hooklist *hl = (const struct hooklist *)start; 437 438 return hl->nodeinfo.hooks; 439 } 440 441 /* Array type for a variable length array of struct namelist */ 442 static const struct ng_parse_array_info ng_nodeinfoarray_type_info = { 443 &ng_generic_nodeinfo_type, 444 &ng_generic_list_getLength 445 }; 446 static const struct ng_parse_type ng_generic_nodeinfoarray_type = { 447 &ng_parse_array_type, 448 &ng_nodeinfoarray_type_info 449 }; 450 451 /* Array type for a variable length array of struct typelist */ 452 static const struct ng_parse_array_info ng_typeinfoarray_type_info = { 453 &ng_generic_typeinfo_type, 454 &ng_generic_list_getLength 455 }; 456 static const struct ng_parse_type ng_generic_typeinfoarray_type = { 457 &ng_parse_array_type, 458 &ng_typeinfoarray_type_info 459 }; 460 461 /* Array type for array of struct linkinfo in struct hooklist */ 462 static const struct ng_parse_array_info ng_generic_linkinfo_array_type_info = { 463 &ng_generic_linkinfo_type, 464 &ng_generic_linkinfo_getLength 465 }; 466 static const struct ng_parse_type ng_generic_linkinfo_array_type = { 467 &ng_parse_array_type, 468 &ng_generic_linkinfo_array_type_info 469 }; 470 471 DEFINE_PARSE_STRUCT_TYPE(typelist, TYPELIST, (&ng_generic_nodeinfoarray_type)); 472 DEFINE_PARSE_STRUCT_TYPE(hooklist, HOOKLIST, 473 (&ng_generic_nodeinfo_type, &ng_generic_linkinfo_array_type)); 474 DEFINE_PARSE_STRUCT_TYPE(listnodes, LISTNODES, 475 (&ng_generic_nodeinfoarray_type)); 476 477 /* List of commands and how to convert arguments to/from ASCII */ 478 static const struct ng_cmdlist ng_generic_cmds[] = { 479 { 480 NGM_GENERIC_COOKIE, 481 NGM_SHUTDOWN, 482 "shutdown", 483 NULL, 484 NULL 485 }, 486 { 487 NGM_GENERIC_COOKIE, 488 NGM_MKPEER, 489 "mkpeer", 490 &ng_generic_mkpeer_type, 491 NULL 492 }, 493 { 494 NGM_GENERIC_COOKIE, 495 NGM_CONNECT, 496 "connect", 497 &ng_generic_connect_type, 498 NULL 499 }, 500 { 501 NGM_GENERIC_COOKIE, 502 NGM_NAME, 503 "name", 504 &ng_generic_name_type, 505 NULL 506 }, 507 { 508 NGM_GENERIC_COOKIE, 509 NGM_RMHOOK, 510 "rmhook", 511 &ng_generic_rmhook_type, 512 NULL 513 }, 514 { 515 NGM_GENERIC_COOKIE, 516 NGM_NODEINFO, 517 "nodeinfo", 518 NULL, 519 &ng_generic_nodeinfo_type 520 }, 521 { 522 NGM_GENERIC_COOKIE, 523 NGM_LISTHOOKS, 524 "listhooks", 525 NULL, 526 &ng_generic_hooklist_type 527 }, 528 { 529 NGM_GENERIC_COOKIE, 530 NGM_LISTNAMES, 531 "listnames", 532 NULL, 533 &ng_generic_listnodes_type /* same as NGM_LISTNODES */ 534 }, 535 { 536 NGM_GENERIC_COOKIE, 537 NGM_LISTNODES, 538 "listnodes", 539 NULL, 540 &ng_generic_listnodes_type 541 }, 542 { 543 NGM_GENERIC_COOKIE, 544 NGM_LISTTYPES, 545 "listtypes", 546 NULL, 547 &ng_generic_typeinfo_type 548 }, 549 { 550 NGM_GENERIC_COOKIE, 551 NGM_TEXT_CONFIG, 552 "textconfig", 553 NULL, 554 &ng_parse_string_type 555 }, 556 { 557 NGM_GENERIC_COOKIE, 558 NGM_TEXT_STATUS, 559 "textstatus", 560 NULL, 561 &ng_parse_string_type 562 }, 563 { 564 NGM_GENERIC_COOKIE, 565 NGM_ASCII2BINARY, 566 "ascii2binary", 567 &ng_parse_ng_mesg_type, 568 &ng_parse_ng_mesg_type 569 }, 570 { 571 NGM_GENERIC_COOKIE, 572 NGM_BINARY2ASCII, 573 "binary2ascii", 574 &ng_parse_ng_mesg_type, 575 &ng_parse_ng_mesg_type 576 }, 577 { 0 } 578 }; 579 580 /************************************************************************ 581 Node routines 582 ************************************************************************/ 583 584 /* 585 * Instantiate a node of the requested type 586 */ 587 int 588 ng_make_node(const char *typename, node_p *nodepp) 589 { 590 struct ng_type *type; 591 int error; 592 593 /* Check that the type makes sense */ 594 if (typename == NULL) { 595 TRAP_ERROR(); 596 return (EINVAL); 597 } 598 599 /* Locate the node type. If we fail we return. Do not try to load 600 * module. 601 */ 602 if ((type = ng_findtype(typename)) == NULL) 603 return (ENXIO); 604 605 /* 606 * If we have a constructor, then make the node and 607 * call the constructor to do type specific initialisation. 608 */ 609 if (type->constructor != NULL) { 610 if ((error = ng_make_node_common(type, nodepp)) == 0) { 611 if ((error = ((*type->constructor)(*nodepp)) != 0)) { 612 NG_NODE_UNREF(*nodepp); 613 } 614 } 615 } else { 616 /* 617 * Node has no constructor. We cannot ask for one 618 * to be made. It must be brought into existence by 619 * some external agency. The external agency should 620 * call ng_make_node_common() directly to get the 621 * netgraph part initialised. 622 */ 623 TRAP_ERROR(); 624 error = EINVAL; 625 } 626 return (error); 627 } 628 629 /* 630 * Generic node creation. Called by node initialisation for externally 631 * instantiated nodes (e.g. hardware, sockets, etc ). 632 * The returned node has a reference count of 1. 633 */ 634 int 635 ng_make_node_common(struct ng_type *type, node_p *nodepp) 636 { 637 INIT_VNET_NETGRAPH(curvnet); 638 node_p node; 639 640 /* Require the node type to have been already installed */ 641 if (ng_findtype(type->name) == NULL) { 642 TRAP_ERROR(); 643 return (EINVAL); 644 } 645 646 /* Make a node and try attach it to the type */ 647 NG_ALLOC_NODE(node); 648 if (node == NULL) { 649 TRAP_ERROR(); 650 return (ENOMEM); 651 } 652 node->nd_type = type; 653 #ifdef VIMAGE 654 node->nd_vnet = curvnet; 655 #endif 656 NG_NODE_REF(node); /* note reference */ 657 type->refs++; 658 659 NG_QUEUE_LOCK_INIT(&node->nd_input_queue); 660 STAILQ_INIT(&node->nd_input_queue.queue); 661 node->nd_input_queue.q_flags = 0; 662 663 /* Initialize hook list for new node */ 664 LIST_INIT(&node->nd_hooks); 665 666 /* Link us into the name hash. */ 667 mtx_lock(&ng_namehash_mtx); 668 LIST_INSERT_HEAD(&V_ng_name_hash[0], node, nd_nodes); 669 mtx_unlock(&ng_namehash_mtx); 670 671 /* get an ID and put us in the hash chain */ 672 mtx_lock(&ng_idhash_mtx); 673 for (;;) { /* wrap protection, even if silly */ 674 node_p node2 = NULL; 675 node->nd_ID = V_nextID++; /* 137/sec for 1 year before wrap */ 676 677 /* Is there a problem with the new number? */ 678 NG_IDHASH_FIND(node->nd_ID, node2); /* already taken? */ 679 if ((node->nd_ID != 0) && (node2 == NULL)) { 680 break; 681 } 682 } 683 LIST_INSERT_HEAD(&V_ng_ID_hash[NG_IDHASH_FN(node->nd_ID)], 684 node, nd_idnodes); 685 mtx_unlock(&ng_idhash_mtx); 686 687 /* Done */ 688 *nodepp = node; 689 return (0); 690 } 691 692 /* 693 * Forceably start the shutdown process on a node. Either call 694 * its shutdown method, or do the default shutdown if there is 695 * no type-specific method. 696 * 697 * We can only be called from a shutdown message, so we know we have 698 * a writer lock, and therefore exclusive access. It also means 699 * that we should not be on the work queue, but we check anyhow. 700 * 701 * Persistent node types must have a type-specific method which 702 * allocates a new node in which case, this one is irretrievably going away, 703 * or cleans up anything it needs, and just makes the node valid again, 704 * in which case we allow the node to survive. 705 * 706 * XXX We need to think of how to tell a persistent node that we 707 * REALLY need to go away because the hardware has gone or we 708 * are rebooting.... etc. 709 */ 710 void 711 ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3) 712 { 713 hook_p hook; 714 715 /* Check if it's already shutting down */ 716 if ((node->nd_flags & NGF_CLOSING) != 0) 717 return; 718 719 if (node == &ng_deadnode) { 720 printf ("shutdown called on deadnode\n"); 721 return; 722 } 723 724 /* Add an extra reference so it doesn't go away during this */ 725 NG_NODE_REF(node); 726 727 /* 728 * Mark it invalid so any newcomers know not to try use it 729 * Also add our own mark so we can't recurse 730 * note that NGF_INVALID does not do this as it's also set during 731 * creation 732 */ 733 node->nd_flags |= NGF_INVALID|NGF_CLOSING; 734 735 /* If node has its pre-shutdown method, then call it first*/ 736 if (node->nd_type && node->nd_type->close) 737 (*node->nd_type->close)(node); 738 739 /* Notify all remaining connected nodes to disconnect */ 740 while ((hook = LIST_FIRST(&node->nd_hooks)) != NULL) 741 ng_destroy_hook(hook); 742 743 /* 744 * Drain the input queue forceably. 745 * it has no hooks so what's it going to do, bleed on someone? 746 * Theoretically we came here from a queue entry that was added 747 * Just before the queue was closed, so it should be empty anyway. 748 * Also removes us from worklist if needed. 749 */ 750 ng_flush_input_queue(node); 751 752 /* Ask the type if it has anything to do in this case */ 753 if (node->nd_type && node->nd_type->shutdown) { 754 (*node->nd_type->shutdown)(node); 755 if (NG_NODE_IS_VALID(node)) { 756 /* 757 * Well, blow me down if the node code hasn't declared 758 * that it doesn't want to die. 759 * Presumably it is a persistant node. 760 * If we REALLY want it to go away, 761 * e.g. hardware going away, 762 * Our caller should set NGF_REALLY_DIE in nd_flags. 763 */ 764 node->nd_flags &= ~(NGF_INVALID|NGF_CLOSING); 765 NG_NODE_UNREF(node); /* Assume they still have theirs */ 766 return; 767 } 768 } else { /* do the default thing */ 769 NG_NODE_UNREF(node); 770 } 771 772 ng_unname(node); /* basically a NOP these days */ 773 774 /* 775 * Remove extra reference, possibly the last 776 * Possible other holders of references may include 777 * timeout callouts, but theoretically the node's supposed to 778 * have cancelled them. Possibly hardware dependencies may 779 * force a driver to 'linger' with a reference. 780 */ 781 NG_NODE_UNREF(node); 782 } 783 784 /* 785 * Remove a reference to the node, possibly the last. 786 * deadnode always acts as it it were the last. 787 */ 788 int 789 ng_unref_node(node_p node) 790 { 791 int v; 792 793 if (node == &ng_deadnode) { 794 return (0); 795 } 796 797 v = atomic_fetchadd_int(&node->nd_refs, -1); 798 799 if (v == 1) { /* we were the last */ 800 801 mtx_lock(&ng_namehash_mtx); 802 node->nd_type->refs--; /* XXX maybe should get types lock? */ 803 LIST_REMOVE(node, nd_nodes); 804 mtx_unlock(&ng_namehash_mtx); 805 806 mtx_lock(&ng_idhash_mtx); 807 LIST_REMOVE(node, nd_idnodes); 808 mtx_unlock(&ng_idhash_mtx); 809 810 mtx_destroy(&node->nd_input_queue.q_mtx); 811 NG_FREE_NODE(node); 812 } 813 return (v - 1); 814 } 815 816 /************************************************************************ 817 Node ID handling 818 ************************************************************************/ 819 static node_p 820 ng_ID2noderef(ng_ID_t ID) 821 { 822 INIT_VNET_NETGRAPH(curvnet); 823 node_p node; 824 mtx_lock(&ng_idhash_mtx); 825 NG_IDHASH_FIND(ID, node); 826 if(node) 827 NG_NODE_REF(node); 828 mtx_unlock(&ng_idhash_mtx); 829 return(node); 830 } 831 832 ng_ID_t 833 ng_node2ID(node_p node) 834 { 835 return (node ? NG_NODE_ID(node) : 0); 836 } 837 838 /************************************************************************ 839 Node name handling 840 ************************************************************************/ 841 842 /* 843 * Assign a node a name. Once assigned, the name cannot be changed. 844 */ 845 int 846 ng_name_node(node_p node, const char *name) 847 { 848 INIT_VNET_NETGRAPH(curvnet); 849 int i, hash; 850 node_p node2; 851 852 /* Check the name is valid */ 853 for (i = 0; i < NG_NODESIZ; i++) { 854 if (name[i] == '\0' || name[i] == '.' || name[i] == ':') 855 break; 856 } 857 if (i == 0 || name[i] != '\0') { 858 TRAP_ERROR(); 859 return (EINVAL); 860 } 861 if (ng_decodeidname(name) != 0) { /* valid IDs not allowed here */ 862 TRAP_ERROR(); 863 return (EINVAL); 864 } 865 866 /* Check the name isn't already being used */ 867 if ((node2 = ng_name2noderef(node, name)) != NULL) { 868 NG_NODE_UNREF(node2); 869 TRAP_ERROR(); 870 return (EADDRINUSE); 871 } 872 873 /* copy it */ 874 strlcpy(NG_NODE_NAME(node), name, NG_NODESIZ); 875 876 /* Update name hash. */ 877 NG_NAMEHASH(name, hash); 878 mtx_lock(&ng_namehash_mtx); 879 LIST_REMOVE(node, nd_nodes); 880 LIST_INSERT_HEAD(&V_ng_name_hash[hash], node, nd_nodes); 881 mtx_unlock(&ng_namehash_mtx); 882 883 return (0); 884 } 885 886 /* 887 * Find a node by absolute name. The name should NOT end with ':' 888 * The name "." means "this node" and "[xxx]" means "the node 889 * with ID (ie, at address) xxx". 890 * 891 * Returns the node if found, else NULL. 892 * Eventually should add something faster than a sequential search. 893 * Note it acquires a reference on the node so you can be sure it's still 894 * there. 895 */ 896 node_p 897 ng_name2noderef(node_p here, const char *name) 898 { 899 INIT_VNET_NETGRAPH(curvnet); 900 node_p node; 901 ng_ID_t temp; 902 int hash; 903 904 /* "." means "this node" */ 905 if (strcmp(name, ".") == 0) { 906 NG_NODE_REF(here); 907 return(here); 908 } 909 910 /* Check for name-by-ID */ 911 if ((temp = ng_decodeidname(name)) != 0) { 912 return (ng_ID2noderef(temp)); 913 } 914 915 /* Find node by name */ 916 NG_NAMEHASH(name, hash); 917 mtx_lock(&ng_namehash_mtx); 918 LIST_FOREACH(node, &V_ng_name_hash[hash], nd_nodes) { 919 if (NG_NODE_IS_VALID(node) && 920 (strcmp(NG_NODE_NAME(node), name) == 0)) { 921 break; 922 } 923 } 924 if (node) 925 NG_NODE_REF(node); 926 mtx_unlock(&ng_namehash_mtx); 927 return (node); 928 } 929 930 /* 931 * Decode an ID name, eg. "[f03034de]". Returns 0 if the 932 * string is not valid, otherwise returns the value. 933 */ 934 static ng_ID_t 935 ng_decodeidname(const char *name) 936 { 937 const int len = strlen(name); 938 char *eptr; 939 u_long val; 940 941 /* Check for proper length, brackets, no leading junk */ 942 if ((len < 3) 943 || (name[0] != '[') 944 || (name[len - 1] != ']') 945 || (!isxdigit(name[1]))) { 946 return ((ng_ID_t)0); 947 } 948 949 /* Decode number */ 950 val = strtoul(name + 1, &eptr, 16); 951 if ((eptr - name != len - 1) 952 || (val == ULONG_MAX) 953 || (val == 0)) { 954 return ((ng_ID_t)0); 955 } 956 return (ng_ID_t)val; 957 } 958 959 /* 960 * Remove a name from a node. This should only be called 961 * when shutting down and removing the node. 962 * IF we allow name changing this may be more resurrected. 963 */ 964 void 965 ng_unname(node_p node) 966 { 967 } 968 969 /************************************************************************ 970 Hook routines 971 Names are not optional. Hooks are always connected, except for a 972 brief moment within these routines. On invalidation or during creation 973 they are connected to the 'dead' hook. 974 ************************************************************************/ 975 976 /* 977 * Remove a hook reference 978 */ 979 void 980 ng_unref_hook(hook_p hook) 981 { 982 int v; 983 984 if (hook == &ng_deadhook) { 985 return; 986 } 987 988 v = atomic_fetchadd_int(&hook->hk_refs, -1); 989 990 if (v == 1) { /* we were the last */ 991 if (_NG_HOOK_NODE(hook)) /* it'll probably be ng_deadnode */ 992 _NG_NODE_UNREF((_NG_HOOK_NODE(hook))); 993 NG_FREE_HOOK(hook); 994 } 995 } 996 997 /* 998 * Add an unconnected hook to a node. Only used internally. 999 * Assumes node is locked. (XXX not yet true ) 1000 */ 1001 static int 1002 ng_add_hook(node_p node, const char *name, hook_p *hookp) 1003 { 1004 hook_p hook; 1005 int error = 0; 1006 1007 /* Check that the given name is good */ 1008 if (name == NULL) { 1009 TRAP_ERROR(); 1010 return (EINVAL); 1011 } 1012 if (ng_findhook(node, name) != NULL) { 1013 TRAP_ERROR(); 1014 return (EEXIST); 1015 } 1016 1017 /* Allocate the hook and link it up */ 1018 NG_ALLOC_HOOK(hook); 1019 if (hook == NULL) { 1020 TRAP_ERROR(); 1021 return (ENOMEM); 1022 } 1023 hook->hk_refs = 1; /* add a reference for us to return */ 1024 hook->hk_flags = HK_INVALID; 1025 hook->hk_peer = &ng_deadhook; /* start off this way */ 1026 hook->hk_node = node; 1027 NG_NODE_REF(node); /* each hook counts as a reference */ 1028 1029 /* Set hook name */ 1030 strlcpy(NG_HOOK_NAME(hook), name, NG_HOOKSIZ); 1031 1032 /* 1033 * Check if the node type code has something to say about it 1034 * If it fails, the unref of the hook will also unref the node. 1035 */ 1036 if (node->nd_type->newhook != NULL) { 1037 if ((error = (*node->nd_type->newhook)(node, hook, name))) { 1038 NG_HOOK_UNREF(hook); /* this frees the hook */ 1039 return (error); 1040 } 1041 } 1042 /* 1043 * The 'type' agrees so far, so go ahead and link it in. 1044 * We'll ask again later when we actually connect the hooks. 1045 */ 1046 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1047 node->nd_numhooks++; 1048 NG_HOOK_REF(hook); /* one for the node */ 1049 1050 if (hookp) 1051 *hookp = hook; 1052 return (0); 1053 } 1054 1055 /* 1056 * Find a hook 1057 * 1058 * Node types may supply their own optimized routines for finding 1059 * hooks. If none is supplied, we just do a linear search. 1060 * XXX Possibly we should add a reference to the hook? 1061 */ 1062 hook_p 1063 ng_findhook(node_p node, const char *name) 1064 { 1065 hook_p hook; 1066 1067 if (node->nd_type->findhook != NULL) 1068 return (*node->nd_type->findhook)(node, name); 1069 LIST_FOREACH(hook, &node->nd_hooks, hk_hooks) { 1070 if (NG_HOOK_IS_VALID(hook) 1071 && (strcmp(NG_HOOK_NAME(hook), name) == 0)) 1072 return (hook); 1073 } 1074 return (NULL); 1075 } 1076 1077 /* 1078 * Destroy a hook 1079 * 1080 * As hooks are always attached, this really destroys two hooks. 1081 * The one given, and the one attached to it. Disconnect the hooks 1082 * from each other first. We reconnect the peer hook to the 'dead' 1083 * hook so that it can still exist after we depart. We then 1084 * send the peer its own destroy message. This ensures that we only 1085 * interact with the peer's structures when it is locked processing that 1086 * message. We hold a reference to the peer hook so we are guaranteed that 1087 * the peer hook and node are still going to exist until 1088 * we are finished there as the hook holds a ref on the node. 1089 * We run this same code again on the peer hook, but that time it is already 1090 * attached to the 'dead' hook. 1091 * 1092 * This routine is called at all stages of hook creation 1093 * on error detection and must be able to handle any such stage. 1094 */ 1095 void 1096 ng_destroy_hook(hook_p hook) 1097 { 1098 hook_p peer; 1099 node_p node; 1100 1101 if (hook == &ng_deadhook) { /* better safe than sorry */ 1102 printf("ng_destroy_hook called on deadhook\n"); 1103 return; 1104 } 1105 1106 /* 1107 * Protect divorce process with mutex, to avoid races on 1108 * simultaneous disconnect. 1109 */ 1110 mtx_lock(&ng_topo_mtx); 1111 1112 hook->hk_flags |= HK_INVALID; 1113 1114 peer = NG_HOOK_PEER(hook); 1115 node = NG_HOOK_NODE(hook); 1116 1117 if (peer && (peer != &ng_deadhook)) { 1118 /* 1119 * Set the peer to point to ng_deadhook 1120 * from this moment on we are effectively independent it. 1121 * send it an rmhook message of it's own. 1122 */ 1123 peer->hk_peer = &ng_deadhook; /* They no longer know us */ 1124 hook->hk_peer = &ng_deadhook; /* Nor us, them */ 1125 if (NG_HOOK_NODE(peer) == &ng_deadnode) { 1126 /* 1127 * If it's already divorced from a node, 1128 * just free it. 1129 */ 1130 mtx_unlock(&ng_topo_mtx); 1131 } else { 1132 mtx_unlock(&ng_topo_mtx); 1133 ng_rmhook_self(peer); /* Send it a surprise */ 1134 } 1135 NG_HOOK_UNREF(peer); /* account for peer link */ 1136 NG_HOOK_UNREF(hook); /* account for peer link */ 1137 } else 1138 mtx_unlock(&ng_topo_mtx); 1139 1140 mtx_assert(&ng_topo_mtx, MA_NOTOWNED); 1141 1142 /* 1143 * Remove the hook from the node's list to avoid possible recursion 1144 * in case the disconnection results in node shutdown. 1145 */ 1146 if (node == &ng_deadnode) { /* happens if called from ng_con_nodes() */ 1147 return; 1148 } 1149 LIST_REMOVE(hook, hk_hooks); 1150 node->nd_numhooks--; 1151 if (node->nd_type->disconnect) { 1152 /* 1153 * The type handler may elect to destroy the node so don't 1154 * trust its existence after this point. (except 1155 * that we still hold a reference on it. (which we 1156 * inherrited from the hook we are destroying) 1157 */ 1158 (*node->nd_type->disconnect) (hook); 1159 } 1160 1161 /* 1162 * Note that because we will point to ng_deadnode, the original node 1163 * is not decremented automatically so we do that manually. 1164 */ 1165 _NG_HOOK_NODE(hook) = &ng_deadnode; 1166 NG_NODE_UNREF(node); /* We no longer point to it so adjust count */ 1167 NG_HOOK_UNREF(hook); /* Account for linkage (in list) to node */ 1168 } 1169 1170 /* 1171 * Take two hooks on a node and merge the connection so that the given node 1172 * is effectively bypassed. 1173 */ 1174 int 1175 ng_bypass(hook_p hook1, hook_p hook2) 1176 { 1177 if (hook1->hk_node != hook2->hk_node) { 1178 TRAP_ERROR(); 1179 return (EINVAL); 1180 } 1181 hook1->hk_peer->hk_peer = hook2->hk_peer; 1182 hook2->hk_peer->hk_peer = hook1->hk_peer; 1183 1184 hook1->hk_peer = &ng_deadhook; 1185 hook2->hk_peer = &ng_deadhook; 1186 1187 NG_HOOK_UNREF(hook1); 1188 NG_HOOK_UNREF(hook2); 1189 1190 /* XXX If we ever cache methods on hooks update them as well */ 1191 ng_destroy_hook(hook1); 1192 ng_destroy_hook(hook2); 1193 return (0); 1194 } 1195 1196 /* 1197 * Install a new netgraph type 1198 */ 1199 int 1200 ng_newtype(struct ng_type *tp) 1201 { 1202 const size_t namelen = strlen(tp->name); 1203 1204 /* Check version and type name fields */ 1205 if ((tp->version != NG_ABI_VERSION) 1206 || (namelen == 0) 1207 || (namelen >= NG_TYPESIZ)) { 1208 TRAP_ERROR(); 1209 if (tp->version != NG_ABI_VERSION) { 1210 printf("Netgraph: Node type rejected. ABI mismatch. Suggest recompile\n"); 1211 } 1212 return (EINVAL); 1213 } 1214 1215 /* Check for name collision */ 1216 if (ng_findtype(tp->name) != NULL) { 1217 TRAP_ERROR(); 1218 return (EEXIST); 1219 } 1220 1221 1222 /* Link in new type */ 1223 mtx_lock(&ng_typelist_mtx); 1224 LIST_INSERT_HEAD(&ng_typelist, tp, types); 1225 tp->refs = 1; /* first ref is linked list */ 1226 mtx_unlock(&ng_typelist_mtx); 1227 return (0); 1228 } 1229 1230 /* 1231 * unlink a netgraph type 1232 * If no examples exist 1233 */ 1234 int 1235 ng_rmtype(struct ng_type *tp) 1236 { 1237 /* Check for name collision */ 1238 if (tp->refs != 1) { 1239 TRAP_ERROR(); 1240 return (EBUSY); 1241 } 1242 1243 /* Unlink type */ 1244 mtx_lock(&ng_typelist_mtx); 1245 LIST_REMOVE(tp, types); 1246 mtx_unlock(&ng_typelist_mtx); 1247 return (0); 1248 } 1249 1250 /* 1251 * Look for a type of the name given 1252 */ 1253 struct ng_type * 1254 ng_findtype(const char *typename) 1255 { 1256 struct ng_type *type; 1257 1258 mtx_lock(&ng_typelist_mtx); 1259 LIST_FOREACH(type, &ng_typelist, types) { 1260 if (strcmp(type->name, typename) == 0) 1261 break; 1262 } 1263 mtx_unlock(&ng_typelist_mtx); 1264 return (type); 1265 } 1266 1267 /************************************************************************ 1268 Composite routines 1269 ************************************************************************/ 1270 /* 1271 * Connect two nodes using the specified hooks, using queued functions. 1272 */ 1273 static int 1274 ng_con_part3(node_p node, item_p item, hook_p hook) 1275 { 1276 int error = 0; 1277 1278 /* 1279 * When we run, we know that the node 'node' is locked for us. 1280 * Our caller has a reference on the hook. 1281 * Our caller has a reference on the node. 1282 * (In this case our caller is ng_apply_item() ). 1283 * The peer hook has a reference on the hook. 1284 * We are all set up except for the final call to the node, and 1285 * the clearing of the INVALID flag. 1286 */ 1287 if (NG_HOOK_NODE(hook) == &ng_deadnode) { 1288 /* 1289 * The node must have been freed again since we last visited 1290 * here. ng_destry_hook() has this effect but nothing else does. 1291 * We should just release our references and 1292 * free anything we can think of. 1293 * Since we know it's been destroyed, and it's our caller 1294 * that holds the references, just return. 1295 */ 1296 ERROUT(ENOENT); 1297 } 1298 if (hook->hk_node->nd_type->connect) { 1299 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1300 ng_destroy_hook(hook); /* also zaps peer */ 1301 printf("failed in ng_con_part3()\n"); 1302 ERROUT(error); 1303 } 1304 } 1305 /* 1306 * XXX this is wrong for SMP. Possibly we need 1307 * to separate out 'create' and 'invalid' flags. 1308 * should only set flags on hooks we have locked under our node. 1309 */ 1310 hook->hk_flags &= ~HK_INVALID; 1311 done: 1312 NG_FREE_ITEM(item); 1313 return (error); 1314 } 1315 1316 static int 1317 ng_con_part2(node_p node, item_p item, hook_p hook) 1318 { 1319 hook_p peer; 1320 int error = 0; 1321 1322 /* 1323 * When we run, we know that the node 'node' is locked for us. 1324 * Our caller has a reference on the hook. 1325 * Our caller has a reference on the node. 1326 * (In this case our caller is ng_apply_item() ). 1327 * The peer hook has a reference on the hook. 1328 * our node pointer points to the 'dead' node. 1329 * First check the hook name is unique. 1330 * Should not happen because we checked before queueing this. 1331 */ 1332 if (ng_findhook(node, NG_HOOK_NAME(hook)) != NULL) { 1333 TRAP_ERROR(); 1334 ng_destroy_hook(hook); /* should destroy peer too */ 1335 printf("failed in ng_con_part2()\n"); 1336 ERROUT(EEXIST); 1337 } 1338 /* 1339 * Check if the node type code has something to say about it 1340 * If it fails, the unref of the hook will also unref the attached node, 1341 * however since that node is 'ng_deadnode' this will do nothing. 1342 * The peer hook will also be destroyed. 1343 */ 1344 if (node->nd_type->newhook != NULL) { 1345 if ((error = (*node->nd_type->newhook)(node, hook, 1346 hook->hk_name))) { 1347 ng_destroy_hook(hook); /* should destroy peer too */ 1348 printf("failed in ng_con_part2()\n"); 1349 ERROUT(error); 1350 } 1351 } 1352 1353 /* 1354 * The 'type' agrees so far, so go ahead and link it in. 1355 * We'll ask again later when we actually connect the hooks. 1356 */ 1357 hook->hk_node = node; /* just overwrite ng_deadnode */ 1358 NG_NODE_REF(node); /* each hook counts as a reference */ 1359 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1360 node->nd_numhooks++; 1361 NG_HOOK_REF(hook); /* one for the node */ 1362 1363 /* 1364 * We now have a symmetrical situation, where both hooks have been 1365 * linked to their nodes, the newhook methods have been called 1366 * And the references are all correct. The hooks are still marked 1367 * as invalid, as we have not called the 'connect' methods 1368 * yet. 1369 * We can call the local one immediately as we have the 1370 * node locked, but we need to queue the remote one. 1371 */ 1372 if (hook->hk_node->nd_type->connect) { 1373 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1374 ng_destroy_hook(hook); /* also zaps peer */ 1375 printf("failed in ng_con_part2(A)\n"); 1376 ERROUT(error); 1377 } 1378 } 1379 1380 /* 1381 * Acquire topo mutex to avoid race with ng_destroy_hook(). 1382 */ 1383 mtx_lock(&ng_topo_mtx); 1384 peer = hook->hk_peer; 1385 if (peer == &ng_deadhook) { 1386 mtx_unlock(&ng_topo_mtx); 1387 printf("failed in ng_con_part2(B)\n"); 1388 ng_destroy_hook(hook); 1389 ERROUT(ENOENT); 1390 } 1391 mtx_unlock(&ng_topo_mtx); 1392 1393 if ((error = ng_send_fn2(peer->hk_node, peer, item, &ng_con_part3, 1394 NULL, 0, NG_REUSE_ITEM))) { 1395 printf("failed in ng_con_part2(C)\n"); 1396 ng_destroy_hook(hook); /* also zaps peer */ 1397 return (error); /* item was consumed. */ 1398 } 1399 hook->hk_flags &= ~HK_INVALID; /* need both to be able to work */ 1400 return (0); /* item was consumed. */ 1401 done: 1402 NG_FREE_ITEM(item); 1403 return (error); 1404 } 1405 1406 /* 1407 * Connect this node with another node. We assume that this node is 1408 * currently locked, as we are only called from an NGM_CONNECT message. 1409 */ 1410 static int 1411 ng_con_nodes(item_p item, node_p node, const char *name, 1412 node_p node2, const char *name2) 1413 { 1414 int error; 1415 hook_p hook; 1416 hook_p hook2; 1417 1418 if (ng_findhook(node2, name2) != NULL) { 1419 return(EEXIST); 1420 } 1421 if ((error = ng_add_hook(node, name, &hook))) /* gives us a ref */ 1422 return (error); 1423 /* Allocate the other hook and link it up */ 1424 NG_ALLOC_HOOK(hook2); 1425 if (hook2 == NULL) { 1426 TRAP_ERROR(); 1427 ng_destroy_hook(hook); /* XXX check ref counts so far */ 1428 NG_HOOK_UNREF(hook); /* including our ref */ 1429 return (ENOMEM); 1430 } 1431 hook2->hk_refs = 1; /* start with a reference for us. */ 1432 hook2->hk_flags = HK_INVALID; 1433 hook2->hk_peer = hook; /* Link the two together */ 1434 hook->hk_peer = hook2; 1435 NG_HOOK_REF(hook); /* Add a ref for the peer to each*/ 1436 NG_HOOK_REF(hook2); 1437 hook2->hk_node = &ng_deadnode; 1438 strlcpy(NG_HOOK_NAME(hook2), name2, NG_HOOKSIZ); 1439 1440 /* 1441 * Queue the function above. 1442 * Procesing continues in that function in the lock context of 1443 * the other node. 1444 */ 1445 if ((error = ng_send_fn2(node2, hook2, item, &ng_con_part2, NULL, 0, 1446 NG_NOFLAGS))) { 1447 printf("failed in ng_con_nodes(): %d\n", error); 1448 ng_destroy_hook(hook); /* also zaps peer */ 1449 } 1450 1451 NG_HOOK_UNREF(hook); /* Let each hook go if it wants to */ 1452 NG_HOOK_UNREF(hook2); 1453 return (error); 1454 } 1455 1456 /* 1457 * Make a peer and connect. 1458 * We assume that the local node is locked. 1459 * The new node probably doesn't need a lock until 1460 * it has a hook, because it cannot really have any work until then, 1461 * but we should think about it a bit more. 1462 * 1463 * The problem may come if the other node also fires up 1464 * some hardware or a timer or some other source of activation, 1465 * also it may already get a command msg via it's ID. 1466 * 1467 * We could use the same method as ng_con_nodes() but we'd have 1468 * to add ability to remove the node when failing. (Not hard, just 1469 * make arg1 point to the node to remove). 1470 * Unless of course we just ignore failure to connect and leave 1471 * an unconnected node? 1472 */ 1473 static int 1474 ng_mkpeer(node_p node, const char *name, const char *name2, char *type) 1475 { 1476 node_p node2; 1477 hook_p hook1, hook2; 1478 int error; 1479 1480 if ((error = ng_make_node(type, &node2))) { 1481 return (error); 1482 } 1483 1484 if ((error = ng_add_hook(node, name, &hook1))) { /* gives us a ref */ 1485 ng_rmnode(node2, NULL, NULL, 0); 1486 return (error); 1487 } 1488 1489 if ((error = ng_add_hook(node2, name2, &hook2))) { 1490 ng_rmnode(node2, NULL, NULL, 0); 1491 ng_destroy_hook(hook1); 1492 NG_HOOK_UNREF(hook1); 1493 return (error); 1494 } 1495 1496 /* 1497 * Actually link the two hooks together. 1498 */ 1499 hook1->hk_peer = hook2; 1500 hook2->hk_peer = hook1; 1501 1502 /* Each hook is referenced by the other */ 1503 NG_HOOK_REF(hook1); 1504 NG_HOOK_REF(hook2); 1505 1506 /* Give each node the opportunity to veto the pending connection */ 1507 if (hook1->hk_node->nd_type->connect) { 1508 error = (*hook1->hk_node->nd_type->connect) (hook1); 1509 } 1510 1511 if ((error == 0) && hook2->hk_node->nd_type->connect) { 1512 error = (*hook2->hk_node->nd_type->connect) (hook2); 1513 1514 } 1515 1516 /* 1517 * drop the references we were holding on the two hooks. 1518 */ 1519 if (error) { 1520 ng_destroy_hook(hook2); /* also zaps hook1 */ 1521 ng_rmnode(node2, NULL, NULL, 0); 1522 } else { 1523 /* As a last act, allow the hooks to be used */ 1524 hook1->hk_flags &= ~HK_INVALID; 1525 hook2->hk_flags &= ~HK_INVALID; 1526 } 1527 NG_HOOK_UNREF(hook1); 1528 NG_HOOK_UNREF(hook2); 1529 return (error); 1530 } 1531 1532 /************************************************************************ 1533 Utility routines to send self messages 1534 ************************************************************************/ 1535 1536 /* Shut this node down as soon as everyone is clear of it */ 1537 /* Should add arg "immediately" to jump the queue */ 1538 int 1539 ng_rmnode_self(node_p node) 1540 { 1541 int error; 1542 1543 if (node == &ng_deadnode) 1544 return (0); 1545 node->nd_flags |= NGF_INVALID; 1546 if (node->nd_flags & NGF_CLOSING) 1547 return (0); 1548 1549 error = ng_send_fn(node, NULL, &ng_rmnode, NULL, 0); 1550 return (error); 1551 } 1552 1553 static void 1554 ng_rmhook_part2(node_p node, hook_p hook, void *arg1, int arg2) 1555 { 1556 ng_destroy_hook(hook); 1557 return ; 1558 } 1559 1560 int 1561 ng_rmhook_self(hook_p hook) 1562 { 1563 int error; 1564 node_p node = NG_HOOK_NODE(hook); 1565 1566 if (node == &ng_deadnode) 1567 return (0); 1568 1569 error = ng_send_fn(node, hook, &ng_rmhook_part2, NULL, 0); 1570 return (error); 1571 } 1572 1573 /*********************************************************************** 1574 * Parse and verify a string of the form: <NODE:><PATH> 1575 * 1576 * Such a string can refer to a specific node or a specific hook 1577 * on a specific node, depending on how you look at it. In the 1578 * latter case, the PATH component must not end in a dot. 1579 * 1580 * Both <NODE:> and <PATH> are optional. The <PATH> is a string 1581 * of hook names separated by dots. This breaks out the original 1582 * string, setting *nodep to "NODE" (or NULL if none) and *pathp 1583 * to "PATH" (or NULL if degenerate). Also, *hookp will point to 1584 * the final hook component of <PATH>, if any, otherwise NULL. 1585 * 1586 * This returns -1 if the path is malformed. The char ** are optional. 1587 ***********************************************************************/ 1588 int 1589 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp) 1590 { 1591 char *node, *path, *hook; 1592 int k; 1593 1594 /* 1595 * Extract absolute NODE, if any 1596 */ 1597 for (path = addr; *path && *path != ':'; path++); 1598 if (*path) { 1599 node = addr; /* Here's the NODE */ 1600 *path++ = '\0'; /* Here's the PATH */ 1601 1602 /* Node name must not be empty */ 1603 if (!*node) 1604 return -1; 1605 1606 /* A name of "." is OK; otherwise '.' not allowed */ 1607 if (strcmp(node, ".") != 0) { 1608 for (k = 0; node[k]; k++) 1609 if (node[k] == '.') 1610 return -1; 1611 } 1612 } else { 1613 node = NULL; /* No absolute NODE */ 1614 path = addr; /* Here's the PATH */ 1615 } 1616 1617 /* Snoop for illegal characters in PATH */ 1618 for (k = 0; path[k]; k++) 1619 if (path[k] == ':') 1620 return -1; 1621 1622 /* Check for no repeated dots in PATH */ 1623 for (k = 0; path[k]; k++) 1624 if (path[k] == '.' && path[k + 1] == '.') 1625 return -1; 1626 1627 /* Remove extra (degenerate) dots from beginning or end of PATH */ 1628 if (path[0] == '.') 1629 path++; 1630 if (*path && path[strlen(path) - 1] == '.') 1631 path[strlen(path) - 1] = 0; 1632 1633 /* If PATH has a dot, then we're not talking about a hook */ 1634 if (*path) { 1635 for (hook = path, k = 0; path[k]; k++) 1636 if (path[k] == '.') { 1637 hook = NULL; 1638 break; 1639 } 1640 } else 1641 path = hook = NULL; 1642 1643 /* Done */ 1644 if (nodep) 1645 *nodep = node; 1646 if (pathp) 1647 *pathp = path; 1648 if (hookp) 1649 *hookp = hook; 1650 return (0); 1651 } 1652 1653 /* 1654 * Given a path, which may be absolute or relative, and a starting node, 1655 * return the destination node. 1656 */ 1657 int 1658 ng_path2noderef(node_p here, const char *address, 1659 node_p *destp, hook_p *lasthook) 1660 { 1661 char fullpath[NG_PATHSIZ]; 1662 char *nodename, *path, pbuf[2]; 1663 node_p node, oldnode; 1664 char *cp; 1665 hook_p hook = NULL; 1666 1667 /* Initialize */ 1668 if (destp == NULL) { 1669 TRAP_ERROR(); 1670 return EINVAL; 1671 } 1672 *destp = NULL; 1673 1674 /* Make a writable copy of address for ng_path_parse() */ 1675 strncpy(fullpath, address, sizeof(fullpath) - 1); 1676 fullpath[sizeof(fullpath) - 1] = '\0'; 1677 1678 /* Parse out node and sequence of hooks */ 1679 if (ng_path_parse(fullpath, &nodename, &path, NULL) < 0) { 1680 TRAP_ERROR(); 1681 return EINVAL; 1682 } 1683 if (path == NULL) { 1684 pbuf[0] = '.'; /* Needs to be writable */ 1685 pbuf[1] = '\0'; 1686 path = pbuf; 1687 } 1688 1689 /* 1690 * For an absolute address, jump to the starting node. 1691 * Note that this holds a reference on the node for us. 1692 * Don't forget to drop the reference if we don't need it. 1693 */ 1694 if (nodename) { 1695 node = ng_name2noderef(here, nodename); 1696 if (node == NULL) { 1697 TRAP_ERROR(); 1698 return (ENOENT); 1699 } 1700 } else { 1701 if (here == NULL) { 1702 TRAP_ERROR(); 1703 return (EINVAL); 1704 } 1705 node = here; 1706 NG_NODE_REF(node); 1707 } 1708 1709 /* 1710 * Now follow the sequence of hooks 1711 * XXX 1712 * We actually cannot guarantee that the sequence 1713 * is not being demolished as we crawl along it 1714 * without extra-ordinary locking etc. 1715 * So this is a bit dodgy to say the least. 1716 * We can probably hold up some things by holding 1717 * the nodelist mutex for the time of this 1718 * crawl if we wanted.. At least that way we wouldn't have to 1719 * worry about the nodes disappearing, but the hooks would still 1720 * be a problem. 1721 */ 1722 for (cp = path; node != NULL && *cp != '\0'; ) { 1723 char *segment; 1724 1725 /* 1726 * Break out the next path segment. Replace the dot we just 1727 * found with a NUL; "cp" points to the next segment (or the 1728 * NUL at the end). 1729 */ 1730 for (segment = cp; *cp != '\0'; cp++) { 1731 if (*cp == '.') { 1732 *cp++ = '\0'; 1733 break; 1734 } 1735 } 1736 1737 /* Empty segment */ 1738 if (*segment == '\0') 1739 continue; 1740 1741 /* We have a segment, so look for a hook by that name */ 1742 hook = ng_findhook(node, segment); 1743 1744 /* Can't get there from here... */ 1745 if (hook == NULL 1746 || NG_HOOK_PEER(hook) == NULL 1747 || NG_HOOK_NOT_VALID(hook) 1748 || NG_HOOK_NOT_VALID(NG_HOOK_PEER(hook))) { 1749 TRAP_ERROR(); 1750 NG_NODE_UNREF(node); 1751 #if 0 1752 printf("hooknotvalid %s %s %d %d %d %d ", 1753 path, 1754 segment, 1755 hook == NULL, 1756 NG_HOOK_PEER(hook) == NULL, 1757 NG_HOOK_NOT_VALID(hook), 1758 NG_HOOK_NOT_VALID(NG_HOOK_PEER(hook))); 1759 #endif 1760 return (ENOENT); 1761 } 1762 1763 /* 1764 * Hop on over to the next node 1765 * XXX 1766 * Big race conditions here as hooks and nodes go away 1767 * *** Idea.. store an ng_ID_t in each hook and use that 1768 * instead of the direct hook in this crawl? 1769 */ 1770 oldnode = node; 1771 if ((node = NG_PEER_NODE(hook))) 1772 NG_NODE_REF(node); /* XXX RACE */ 1773 NG_NODE_UNREF(oldnode); /* XXX another race */ 1774 if (NG_NODE_NOT_VALID(node)) { 1775 NG_NODE_UNREF(node); /* XXX more races */ 1776 node = NULL; 1777 } 1778 } 1779 1780 /* If node somehow missing, fail here (probably this is not needed) */ 1781 if (node == NULL) { 1782 TRAP_ERROR(); 1783 return (ENXIO); 1784 } 1785 1786 /* Done */ 1787 *destp = node; 1788 if (lasthook != NULL) 1789 *lasthook = (hook ? NG_HOOK_PEER(hook) : NULL); 1790 return (0); 1791 } 1792 1793 /***************************************************************\ 1794 * Input queue handling. 1795 * All activities are submitted to the node via the input queue 1796 * which implements a multiple-reader/single-writer gate. 1797 * Items which cannot be handled immediately are queued. 1798 * 1799 * read-write queue locking inline functions * 1800 \***************************************************************/ 1801 1802 static __inline void ng_queue_rw(node_p node, item_p item, int rw); 1803 static __inline item_p ng_dequeue(node_p node, int *rw); 1804 static __inline item_p ng_acquire_read(node_p node, item_p item); 1805 static __inline item_p ng_acquire_write(node_p node, item_p item); 1806 static __inline void ng_leave_read(node_p node); 1807 static __inline void ng_leave_write(node_p node); 1808 1809 /* 1810 * Definition of the bits fields in the ng_queue flag word. 1811 * Defined here rather than in netgraph.h because no-one should fiddle 1812 * with them. 1813 * 1814 * The ordering here may be important! don't shuffle these. 1815 */ 1816 /*- 1817 Safety Barrier--------+ (adjustable to suit taste) (not used yet) 1818 | 1819 V 1820 +-------+-------+-------+-------+-------+-------+-------+-------+ 1821 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | 1822 | |A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |P|A| 1823 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |O|W| 1824 +-------+-------+-------+-------+-------+-------+-------+-------+ 1825 \___________________________ ____________________________/ | | 1826 V | | 1827 [active reader count] | | 1828 | | 1829 Operation Pending -------------------------------+ | 1830 | 1831 Active Writer ---------------------------------------+ 1832 1833 Node queue has such semantics: 1834 - All flags modifications are atomic. 1835 - Reader count can be incremented only if there is no writer or pending flags. 1836 As soon as this can't be done with single operation, it is implemented with 1837 spin loop and atomic_cmpset(). 1838 - Writer flag can be set only if there is no any bits set. 1839 It is implemented with atomic_cmpset(). 1840 - Pending flag can be set any time, but to avoid collision on queue processing 1841 all queue fields are protected by the mutex. 1842 - Queue processing thread reads queue holding the mutex, but releases it while 1843 processing. When queue is empty pending flag is removed. 1844 */ 1845 1846 #define WRITER_ACTIVE 0x00000001 1847 #define OP_PENDING 0x00000002 1848 #define READER_INCREMENT 0x00000004 1849 #define READER_MASK 0xfffffffc /* Not valid if WRITER_ACTIVE is set */ 1850 #define SAFETY_BARRIER 0x00100000 /* 128K items queued should be enough */ 1851 1852 /* Defines of more elaborate states on the queue */ 1853 /* Mask of bits a new read cares about */ 1854 #define NGQ_RMASK (WRITER_ACTIVE|OP_PENDING) 1855 1856 /* Mask of bits a new write cares about */ 1857 #define NGQ_WMASK (NGQ_RMASK|READER_MASK) 1858 1859 /* Test to decide if there is something on the queue. */ 1860 #define QUEUE_ACTIVE(QP) ((QP)->q_flags & OP_PENDING) 1861 1862 /* How to decide what the next queued item is. */ 1863 #define HEAD_IS_READER(QP) NGI_QUEUED_READER(STAILQ_FIRST(&(QP)->queue)) 1864 #define HEAD_IS_WRITER(QP) NGI_QUEUED_WRITER(STAILQ_FIRST(&(QP)->queue)) /* notused */ 1865 1866 /* Read the status to decide if the next item on the queue can now run. */ 1867 #define QUEUED_READER_CAN_PROCEED(QP) \ 1868 (((QP)->q_flags & (NGQ_RMASK & ~OP_PENDING)) == 0) 1869 #define QUEUED_WRITER_CAN_PROCEED(QP) \ 1870 (((QP)->q_flags & (NGQ_WMASK & ~OP_PENDING)) == 0) 1871 1872 /* Is there a chance of getting ANY work off the queue? */ 1873 #define NEXT_QUEUED_ITEM_CAN_PROCEED(QP) \ 1874 ((HEAD_IS_READER(QP)) ? QUEUED_READER_CAN_PROCEED(QP) : \ 1875 QUEUED_WRITER_CAN_PROCEED(QP)) 1876 1877 #define NGQRW_R 0 1878 #define NGQRW_W 1 1879 1880 #define NGQ2_WORKQ 0x00000001 1881 1882 /* 1883 * Taking into account the current state of the queue and node, possibly take 1884 * the next entry off the queue and return it. Return NULL if there was 1885 * nothing we could return, either because there really was nothing there, or 1886 * because the node was in a state where it cannot yet process the next item 1887 * on the queue. 1888 */ 1889 static __inline item_p 1890 ng_dequeue(node_p node, int *rw) 1891 { 1892 item_p item; 1893 struct ng_queue *ngq = &node->nd_input_queue; 1894 1895 /* This MUST be called with the mutex held. */ 1896 mtx_assert(&ngq->q_mtx, MA_OWNED); 1897 1898 /* If there is nothing queued, then just return. */ 1899 if (!QUEUE_ACTIVE(ngq)) { 1900 CTR4(KTR_NET, "%20s: node [%x] (%p) queue empty; " 1901 "queue flags 0x%lx", __func__, 1902 node->nd_ID, node, ngq->q_flags); 1903 return (NULL); 1904 } 1905 1906 /* 1907 * From here, we can assume there is a head item. 1908 * We need to find out what it is and if it can be dequeued, given 1909 * the current state of the node. 1910 */ 1911 if (HEAD_IS_READER(ngq)) { 1912 while (1) { 1913 long t = ngq->q_flags; 1914 if (t & WRITER_ACTIVE) { 1915 /* There is writer, reader can't proceed. */ 1916 CTR4(KTR_NET, "%20s: node [%x] (%p) queued reader " 1917 "can't proceed; queue flags 0x%lx", __func__, 1918 node->nd_ID, node, t); 1919 return (NULL); 1920 } 1921 if (atomic_cmpset_acq_int(&ngq->q_flags, t, 1922 t + READER_INCREMENT)) 1923 break; 1924 cpu_spinwait(); 1925 } 1926 /* We have got reader lock for the node. */ 1927 *rw = NGQRW_R; 1928 } else if (atomic_cmpset_acq_int(&ngq->q_flags, OP_PENDING, 1929 OP_PENDING + WRITER_ACTIVE)) { 1930 /* We have got writer lock for the node. */ 1931 *rw = NGQRW_W; 1932 } else { 1933 /* There is somebody other, writer can't proceed. */ 1934 CTR4(KTR_NET, "%20s: node [%x] (%p) queued writer " 1935 "can't proceed; queue flags 0x%lx", __func__, 1936 node->nd_ID, node, ngq->q_flags); 1937 return (NULL); 1938 } 1939 1940 /* 1941 * Now we dequeue the request (whatever it may be) and correct the 1942 * pending flags and the next and last pointers. 1943 */ 1944 item = STAILQ_FIRST(&ngq->queue); 1945 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 1946 if (STAILQ_EMPTY(&ngq->queue)) 1947 atomic_clear_int(&ngq->q_flags, OP_PENDING); 1948 CTR6(KTR_NET, "%20s: node [%x] (%p) returning item %p as %s; " 1949 "queue flags 0x%lx", __func__, 1950 node->nd_ID, node, item, *rw ? "WRITER" : "READER" , 1951 ngq->q_flags); 1952 return (item); 1953 } 1954 1955 /* 1956 * Queue a packet to be picked up later by someone else. 1957 * If the queue could be run now, add node to the queue handler's worklist. 1958 */ 1959 static __inline void 1960 ng_queue_rw(node_p node, item_p item, int rw) 1961 { 1962 struct ng_queue *ngq = &node->nd_input_queue; 1963 if (rw == NGQRW_W) 1964 NGI_SET_WRITER(item); 1965 else 1966 NGI_SET_READER(item); 1967 1968 NG_QUEUE_LOCK(ngq); 1969 /* Set OP_PENDING flag and enqueue the item. */ 1970 atomic_set_int(&ngq->q_flags, OP_PENDING); 1971 STAILQ_INSERT_TAIL(&ngq->queue, item, el_next); 1972 1973 CTR5(KTR_NET, "%20s: node [%x] (%p) queued item %p as %s", __func__, 1974 node->nd_ID, node, item, rw ? "WRITER" : "READER" ); 1975 1976 /* 1977 * We can take the worklist lock with the node locked 1978 * BUT NOT THE REVERSE! 1979 */ 1980 if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 1981 ng_worklist_add(node); 1982 NG_QUEUE_UNLOCK(ngq); 1983 } 1984 1985 /* Acquire reader lock on node. If node is busy, queue the packet. */ 1986 static __inline item_p 1987 ng_acquire_read(node_p node, item_p item) 1988 { 1989 KASSERT(node != &ng_deadnode, 1990 ("%s: working on deadnode", __func__)); 1991 1992 /* Reader needs node without writer and pending items. */ 1993 while (1) { 1994 long t = node->nd_input_queue.q_flags; 1995 if (t & NGQ_RMASK) 1996 break; /* Node is not ready for reader. */ 1997 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, 1998 t, t + READER_INCREMENT)) { 1999 /* Successfully grabbed node */ 2000 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2001 __func__, node->nd_ID, node, item); 2002 return (item); 2003 } 2004 cpu_spinwait(); 2005 }; 2006 2007 /* Queue the request for later. */ 2008 ng_queue_rw(node, item, NGQRW_R); 2009 2010 return (NULL); 2011 } 2012 2013 /* Acquire writer lock on node. If node is busy, queue the packet. */ 2014 static __inline item_p 2015 ng_acquire_write(node_p node, item_p item) 2016 { 2017 KASSERT(node != &ng_deadnode, 2018 ("%s: working on deadnode", __func__)); 2019 2020 /* Writer needs completely idle node. */ 2021 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, 2022 0, WRITER_ACTIVE)) { 2023 /* Successfully grabbed node */ 2024 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 2025 __func__, node->nd_ID, node, item); 2026 return (item); 2027 } 2028 2029 /* Queue the request for later. */ 2030 ng_queue_rw(node, item, NGQRW_W); 2031 2032 return (NULL); 2033 } 2034 2035 #if 0 2036 static __inline item_p 2037 ng_upgrade_write(node_p node, item_p item) 2038 { 2039 struct ng_queue *ngq = &node->nd_input_queue; 2040 KASSERT(node != &ng_deadnode, 2041 ("%s: working on deadnode", __func__)); 2042 2043 NGI_SET_WRITER(item); 2044 2045 NG_QUEUE_LOCK(ngq); 2046 2047 /* 2048 * There will never be no readers as we are there ourselves. 2049 * Set the WRITER_ACTIVE flags ASAP to block out fast track readers. 2050 * The caller we are running from will call ng_leave_read() 2051 * soon, so we must account for that. We must leave again with the 2052 * READER lock. If we find other readers, then 2053 * queue the request for later. However "later" may be rignt now 2054 * if there are no readers. We don't really care if there are queued 2055 * items as we will bypass them anyhow. 2056 */ 2057 atomic_add_int(&ngq->q_flags, WRITER_ACTIVE - READER_INCREMENT); 2058 if ((ngq->q_flags & (NGQ_WMASK & ~OP_PENDING)) == WRITER_ACTIVE) { 2059 NG_QUEUE_UNLOCK(ngq); 2060 2061 /* It's just us, act on the item. */ 2062 /* will NOT drop writer lock when done */ 2063 ng_apply_item(node, item, 0); 2064 2065 /* 2066 * Having acted on the item, atomically 2067 * down grade back to READER and finish up 2068 */ 2069 atomic_add_int(&ngq->q_flags, 2070 READER_INCREMENT - WRITER_ACTIVE); 2071 2072 /* Our caller will call ng_leave_read() */ 2073 return; 2074 } 2075 /* 2076 * It's not just us active, so queue us AT THE HEAD. 2077 * "Why?" I hear you ask. 2078 * Put us at the head of the queue as we've already been 2079 * through it once. If there is nothing else waiting, 2080 * set the correct flags. 2081 */ 2082 if (STAILQ_EMPTY(&ngq->queue)) { 2083 /* We've gone from, 0 to 1 item in the queue */ 2084 atomic_set_int(&ngq->q_flags, OP_PENDING); 2085 2086 CTR3(KTR_NET, "%20s: node [%x] (%p) set OP_PENDING", __func__, 2087 node->nd_ID, node); 2088 }; 2089 STAILQ_INSERT_HEAD(&ngq->queue, item, el_next); 2090 CTR4(KTR_NET, "%20s: node [%x] (%p) requeued item %p as WRITER", 2091 __func__, node->nd_ID, node, item ); 2092 2093 /* Reverse what we did above. That downgrades us back to reader */ 2094 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2095 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2096 ng_worklist_add(node); 2097 NG_QUEUE_UNLOCK(ngq); 2098 2099 return; 2100 } 2101 #endif 2102 2103 /* Release reader lock. */ 2104 static __inline void 2105 ng_leave_read(node_p node) 2106 { 2107 atomic_subtract_rel_int(&node->nd_input_queue.q_flags, READER_INCREMENT); 2108 } 2109 2110 /* Release writer lock. */ 2111 static __inline void 2112 ng_leave_write(node_p node) 2113 { 2114 atomic_clear_rel_int(&node->nd_input_queue.q_flags, WRITER_ACTIVE); 2115 } 2116 2117 /* Purge node queue. Called on node shutdown. */ 2118 static void 2119 ng_flush_input_queue(node_p node) 2120 { 2121 struct ng_queue *ngq = &node->nd_input_queue; 2122 item_p item; 2123 2124 NG_QUEUE_LOCK(ngq); 2125 while ((item = STAILQ_FIRST(&ngq->queue)) != NULL) { 2126 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2127 if (STAILQ_EMPTY(&ngq->queue)) 2128 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2129 NG_QUEUE_UNLOCK(ngq); 2130 2131 /* If the item is supplying a callback, call it with an error */ 2132 if (item->apply != NULL) { 2133 if (item->depth == 1) 2134 item->apply->error = ENOENT; 2135 if (refcount_release(&item->apply->refs)) { 2136 (*item->apply->apply)(item->apply->context, 2137 item->apply->error); 2138 } 2139 } 2140 NG_FREE_ITEM(item); 2141 NG_QUEUE_LOCK(ngq); 2142 } 2143 NG_QUEUE_UNLOCK(ngq); 2144 } 2145 2146 /*********************************************************************** 2147 * Externally visible method for sending or queueing messages or data. 2148 ***********************************************************************/ 2149 2150 /* 2151 * The module code should have filled out the item correctly by this stage: 2152 * Common: 2153 * reference to destination node. 2154 * Reference to destination rcv hook if relevant. 2155 * apply pointer must be or NULL or reference valid struct ng_apply_info. 2156 * Data: 2157 * pointer to mbuf 2158 * Control_Message: 2159 * pointer to msg. 2160 * ID of original sender node. (return address) 2161 * Function: 2162 * Function pointer 2163 * void * argument 2164 * integer argument 2165 * 2166 * The nodes have several routines and macros to help with this task: 2167 */ 2168 2169 int 2170 ng_snd_item(item_p item, int flags) 2171 { 2172 hook_p hook; 2173 node_p node; 2174 int queue, rw; 2175 struct ng_queue *ngq; 2176 int error = 0; 2177 2178 /* We are sending item, so it must be present! */ 2179 KASSERT(item != NULL, ("ng_snd_item: item is NULL")); 2180 2181 #ifdef NETGRAPH_DEBUG 2182 _ngi_check(item, __FILE__, __LINE__); 2183 #endif 2184 2185 /* Item was sent once more, postpone apply() call. */ 2186 if (item->apply) 2187 refcount_acquire(&item->apply->refs); 2188 2189 node = NGI_NODE(item); 2190 /* Node is never optional. */ 2191 KASSERT(node != NULL, ("ng_snd_item: node is NULL")); 2192 2193 hook = NGI_HOOK(item); 2194 /* Valid hook and mbuf are mandatory for data. */ 2195 if ((item->el_flags & NGQF_TYPE) == NGQF_DATA) { 2196 KASSERT(hook != NULL, ("ng_snd_item: hook for data is NULL")); 2197 if (NGI_M(item) == NULL) 2198 ERROUT(EINVAL); 2199 CHECK_DATA_MBUF(NGI_M(item)); 2200 } 2201 2202 /* 2203 * If the item or the node specifies single threading, force 2204 * writer semantics. Similarly, the node may say one hook always 2205 * produces writers. These are overrides. 2206 */ 2207 if (((item->el_flags & NGQF_RW) == NGQF_WRITER) || 2208 (node->nd_flags & NGF_FORCE_WRITER) || 2209 (hook && (hook->hk_flags & HK_FORCE_WRITER))) { 2210 rw = NGQRW_W; 2211 } else { 2212 rw = NGQRW_R; 2213 } 2214 2215 /* 2216 * If sender or receiver requests queued delivery, or call graph 2217 * loops back from outbound to inbound path, or stack usage 2218 * level is dangerous - enqueue message. 2219 */ 2220 if ((flags & NG_QUEUE) || (hook && (hook->hk_flags & HK_QUEUE))) { 2221 queue = 1; 2222 } else if (hook && (hook->hk_flags & HK_TO_INBOUND) && 2223 curthread->td_ng_outbound) { 2224 queue = 1; 2225 } else { 2226 queue = 0; 2227 #ifdef GET_STACK_USAGE 2228 /* 2229 * Most of netgraph nodes have small stack consumption and 2230 * for them 25% of free stack space is more than enough. 2231 * Nodes/hooks with higher stack usage should be marked as 2232 * HI_STACK. For them 50% of stack will be guaranteed then. 2233 * XXX: Values 25% and 50% are completely empirical. 2234 */ 2235 size_t st, su, sl; 2236 GET_STACK_USAGE(st, su); 2237 sl = st - su; 2238 if ((sl * 4 < st) || 2239 ((sl * 2 < st) && ((node->nd_flags & NGF_HI_STACK) || 2240 (hook && (hook->hk_flags & HK_HI_STACK))))) { 2241 queue = 1; 2242 } 2243 #endif 2244 } 2245 2246 if (queue) { 2247 item->depth = 1; 2248 /* Put it on the queue for that node*/ 2249 ng_queue_rw(node, item, rw); 2250 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2251 } 2252 2253 /* 2254 * We already decided how we will be queueud or treated. 2255 * Try get the appropriate operating permission. 2256 */ 2257 if (rw == NGQRW_R) 2258 item = ng_acquire_read(node, item); 2259 else 2260 item = ng_acquire_write(node, item); 2261 2262 /* Item was queued while trying to get permission. */ 2263 if (item == NULL) 2264 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2265 2266 NGI_GET_NODE(item, node); /* zaps stored node */ 2267 2268 item->depth++; 2269 error = ng_apply_item(node, item, rw); /* drops r/w lock when done */ 2270 2271 /* If something is waiting on queue and ready, schedule it. */ 2272 ngq = &node->nd_input_queue; 2273 if (QUEUE_ACTIVE(ngq)) { 2274 NG_QUEUE_LOCK(ngq); 2275 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2276 ng_worklist_add(node); 2277 NG_QUEUE_UNLOCK(ngq); 2278 } 2279 2280 /* 2281 * Node may go away as soon as we remove the reference. 2282 * Whatever we do, DO NOT access the node again! 2283 */ 2284 NG_NODE_UNREF(node); 2285 2286 return (error); 2287 2288 done: 2289 /* If was not sent, apply callback here. */ 2290 if (item->apply != NULL) { 2291 if (item->depth == 0 && error != 0) 2292 item->apply->error = error; 2293 if (refcount_release(&item->apply->refs)) { 2294 (*item->apply->apply)(item->apply->context, 2295 item->apply->error); 2296 } 2297 } 2298 2299 NG_FREE_ITEM(item); 2300 return (error); 2301 } 2302 2303 /* 2304 * We have an item that was possibly queued somewhere. 2305 * It should contain all the information needed 2306 * to run it on the appropriate node/hook. 2307 * If there is apply pointer and we own the last reference, call apply(). 2308 */ 2309 static int 2310 ng_apply_item(node_p node, item_p item, int rw) 2311 { 2312 hook_p hook; 2313 ng_rcvdata_t *rcvdata; 2314 ng_rcvmsg_t *rcvmsg; 2315 struct ng_apply_info *apply; 2316 int error = 0, depth; 2317 2318 /* Node and item are never optional. */ 2319 KASSERT(node != NULL, ("ng_apply_item: node is NULL")); 2320 KASSERT(item != NULL, ("ng_apply_item: item is NULL")); 2321 2322 NGI_GET_HOOK(item, hook); /* clears stored hook */ 2323 #ifdef NETGRAPH_DEBUG 2324 _ngi_check(item, __FILE__, __LINE__); 2325 #endif 2326 2327 apply = item->apply; 2328 depth = item->depth; 2329 2330 switch (item->el_flags & NGQF_TYPE) { 2331 case NGQF_DATA: 2332 /* 2333 * Check things are still ok as when we were queued. 2334 */ 2335 KASSERT(hook != NULL, ("ng_apply_item: hook for data is NULL")); 2336 if (NG_HOOK_NOT_VALID(hook) || 2337 NG_NODE_NOT_VALID(node)) { 2338 error = EIO; 2339 NG_FREE_ITEM(item); 2340 break; 2341 } 2342 /* 2343 * If no receive method, just silently drop it. 2344 * Give preference to the hook over-ride method 2345 */ 2346 if ((!(rcvdata = hook->hk_rcvdata)) 2347 && (!(rcvdata = NG_HOOK_NODE(hook)->nd_type->rcvdata))) { 2348 error = 0; 2349 NG_FREE_ITEM(item); 2350 break; 2351 } 2352 error = (*rcvdata)(hook, item); 2353 break; 2354 case NGQF_MESG: 2355 if (hook && NG_HOOK_NOT_VALID(hook)) { 2356 /* 2357 * The hook has been zapped then we can't use it. 2358 * Immediately drop its reference. 2359 * The message may not need it. 2360 */ 2361 NG_HOOK_UNREF(hook); 2362 hook = NULL; 2363 } 2364 /* 2365 * Similarly, if the node is a zombie there is 2366 * nothing we can do with it, drop everything. 2367 */ 2368 if (NG_NODE_NOT_VALID(node)) { 2369 TRAP_ERROR(); 2370 error = EINVAL; 2371 NG_FREE_ITEM(item); 2372 break; 2373 } 2374 /* 2375 * Call the appropriate message handler for the object. 2376 * It is up to the message handler to free the message. 2377 * If it's a generic message, handle it generically, 2378 * otherwise call the type's message handler (if it exists). 2379 * XXX (race). Remember that a queued message may 2380 * reference a node or hook that has just been 2381 * invalidated. It will exist as the queue code 2382 * is holding a reference, but.. 2383 */ 2384 if ((NGI_MSG(item)->header.typecookie == NGM_GENERIC_COOKIE) && 2385 ((NGI_MSG(item)->header.flags & NGF_RESP) == 0)) { 2386 error = ng_generic_msg(node, item, hook); 2387 break; 2388 } 2389 if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg))) && 2390 (!(rcvmsg = node->nd_type->rcvmsg))) { 2391 TRAP_ERROR(); 2392 error = 0; 2393 NG_FREE_ITEM(item); 2394 break; 2395 } 2396 error = (*rcvmsg)(node, item, hook); 2397 break; 2398 case NGQF_FN: 2399 case NGQF_FN2: 2400 /* 2401 * In the case of the shutdown message we allow it to hit 2402 * even if the node is invalid. 2403 */ 2404 if (NG_NODE_NOT_VALID(node) && 2405 NGI_FN(item) != &ng_rmnode) { 2406 TRAP_ERROR(); 2407 error = EINVAL; 2408 NG_FREE_ITEM(item); 2409 break; 2410 } 2411 /* Same is about some internal functions and invalid hook. */ 2412 if (hook && NG_HOOK_NOT_VALID(hook) && 2413 NGI_FN2(item) != &ng_con_part2 && 2414 NGI_FN2(item) != &ng_con_part3 && 2415 NGI_FN(item) != &ng_rmhook_part2) { 2416 TRAP_ERROR(); 2417 error = EINVAL; 2418 NG_FREE_ITEM(item); 2419 break; 2420 } 2421 2422 if ((item->el_flags & NGQF_TYPE) == NGQF_FN) { 2423 (*NGI_FN(item))(node, hook, NGI_ARG1(item), 2424 NGI_ARG2(item)); 2425 NG_FREE_ITEM(item); 2426 } else /* it is NGQF_FN2 */ 2427 error = (*NGI_FN2(item))(node, item, hook); 2428 break; 2429 } 2430 /* 2431 * We held references on some of the resources 2432 * that we took from the item. Now that we have 2433 * finished doing everything, drop those references. 2434 */ 2435 if (hook) 2436 NG_HOOK_UNREF(hook); 2437 2438 if (rw == NGQRW_R) 2439 ng_leave_read(node); 2440 else 2441 ng_leave_write(node); 2442 2443 /* Apply callback. */ 2444 if (apply != NULL) { 2445 if (depth == 1 && error != 0) 2446 apply->error = error; 2447 if (refcount_release(&apply->refs)) 2448 (*apply->apply)(apply->context, apply->error); 2449 } 2450 2451 return (error); 2452 } 2453 2454 /*********************************************************************** 2455 * Implement the 'generic' control messages 2456 ***********************************************************************/ 2457 static int 2458 ng_generic_msg(node_p here, item_p item, hook_p lasthook) 2459 { 2460 INIT_VNET_NETGRAPH(curvnet); 2461 int error = 0; 2462 struct ng_mesg *msg; 2463 struct ng_mesg *resp = NULL; 2464 2465 NGI_GET_MSG(item, msg); 2466 if (msg->header.typecookie != NGM_GENERIC_COOKIE) { 2467 TRAP_ERROR(); 2468 error = EINVAL; 2469 goto out; 2470 } 2471 switch (msg->header.cmd) { 2472 case NGM_SHUTDOWN: 2473 ng_rmnode(here, NULL, NULL, 0); 2474 break; 2475 case NGM_MKPEER: 2476 { 2477 struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data; 2478 2479 if (msg->header.arglen != sizeof(*mkp)) { 2480 TRAP_ERROR(); 2481 error = EINVAL; 2482 break; 2483 } 2484 mkp->type[sizeof(mkp->type) - 1] = '\0'; 2485 mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0'; 2486 mkp->peerhook[sizeof(mkp->peerhook) - 1] = '\0'; 2487 error = ng_mkpeer(here, mkp->ourhook, mkp->peerhook, mkp->type); 2488 break; 2489 } 2490 case NGM_CONNECT: 2491 { 2492 struct ngm_connect *const con = 2493 (struct ngm_connect *) msg->data; 2494 node_p node2; 2495 2496 if (msg->header.arglen != sizeof(*con)) { 2497 TRAP_ERROR(); 2498 error = EINVAL; 2499 break; 2500 } 2501 con->path[sizeof(con->path) - 1] = '\0'; 2502 con->ourhook[sizeof(con->ourhook) - 1] = '\0'; 2503 con->peerhook[sizeof(con->peerhook) - 1] = '\0'; 2504 /* Don't forget we get a reference.. */ 2505 error = ng_path2noderef(here, con->path, &node2, NULL); 2506 if (error) 2507 break; 2508 error = ng_con_nodes(item, here, con->ourhook, 2509 node2, con->peerhook); 2510 NG_NODE_UNREF(node2); 2511 break; 2512 } 2513 case NGM_NAME: 2514 { 2515 struct ngm_name *const nam = (struct ngm_name *) msg->data; 2516 2517 if (msg->header.arglen != sizeof(*nam)) { 2518 TRAP_ERROR(); 2519 error = EINVAL; 2520 break; 2521 } 2522 nam->name[sizeof(nam->name) - 1] = '\0'; 2523 error = ng_name_node(here, nam->name); 2524 break; 2525 } 2526 case NGM_RMHOOK: 2527 { 2528 struct ngm_rmhook *const rmh = (struct ngm_rmhook *) msg->data; 2529 hook_p hook; 2530 2531 if (msg->header.arglen != sizeof(*rmh)) { 2532 TRAP_ERROR(); 2533 error = EINVAL; 2534 break; 2535 } 2536 rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0'; 2537 if ((hook = ng_findhook(here, rmh->ourhook)) != NULL) 2538 ng_destroy_hook(hook); 2539 break; 2540 } 2541 case NGM_NODEINFO: 2542 { 2543 struct nodeinfo *ni; 2544 2545 NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT); 2546 if (resp == NULL) { 2547 error = ENOMEM; 2548 break; 2549 } 2550 2551 /* Fill in node info */ 2552 ni = (struct nodeinfo *) resp->data; 2553 if (NG_NODE_HAS_NAME(here)) 2554 strcpy(ni->name, NG_NODE_NAME(here)); 2555 strcpy(ni->type, here->nd_type->name); 2556 ni->id = ng_node2ID(here); 2557 ni->hooks = here->nd_numhooks; 2558 break; 2559 } 2560 case NGM_LISTHOOKS: 2561 { 2562 const int nhooks = here->nd_numhooks; 2563 struct hooklist *hl; 2564 struct nodeinfo *ni; 2565 hook_p hook; 2566 2567 /* Get response struct */ 2568 NG_MKRESPONSE(resp, msg, sizeof(*hl) 2569 + (nhooks * sizeof(struct linkinfo)), M_NOWAIT); 2570 if (resp == NULL) { 2571 error = ENOMEM; 2572 break; 2573 } 2574 hl = (struct hooklist *) resp->data; 2575 ni = &hl->nodeinfo; 2576 2577 /* Fill in node info */ 2578 if (NG_NODE_HAS_NAME(here)) 2579 strcpy(ni->name, NG_NODE_NAME(here)); 2580 strcpy(ni->type, here->nd_type->name); 2581 ni->id = ng_node2ID(here); 2582 2583 /* Cycle through the linked list of hooks */ 2584 ni->hooks = 0; 2585 LIST_FOREACH(hook, &here->nd_hooks, hk_hooks) { 2586 struct linkinfo *const link = &hl->link[ni->hooks]; 2587 2588 if (ni->hooks >= nhooks) { 2589 log(LOG_ERR, "%s: number of %s changed\n", 2590 __func__, "hooks"); 2591 break; 2592 } 2593 if (NG_HOOK_NOT_VALID(hook)) 2594 continue; 2595 strcpy(link->ourhook, NG_HOOK_NAME(hook)); 2596 strcpy(link->peerhook, NG_PEER_HOOK_NAME(hook)); 2597 if (NG_PEER_NODE_NAME(hook)[0] != '\0') 2598 strcpy(link->nodeinfo.name, 2599 NG_PEER_NODE_NAME(hook)); 2600 strcpy(link->nodeinfo.type, 2601 NG_PEER_NODE(hook)->nd_type->name); 2602 link->nodeinfo.id = ng_node2ID(NG_PEER_NODE(hook)); 2603 link->nodeinfo.hooks = NG_PEER_NODE(hook)->nd_numhooks; 2604 ni->hooks++; 2605 } 2606 break; 2607 } 2608 2609 case NGM_LISTNAMES: 2610 case NGM_LISTNODES: 2611 { 2612 const int unnamed = (msg->header.cmd == NGM_LISTNODES); 2613 struct namelist *nl; 2614 node_p node; 2615 int num = 0, i; 2616 2617 mtx_lock(&ng_namehash_mtx); 2618 /* Count number of nodes */ 2619 for (i = 0; i < NG_NAME_HASH_SIZE; i++) { 2620 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2621 if (NG_NODE_IS_VALID(node) && 2622 (unnamed || NG_NODE_HAS_NAME(node))) { 2623 num++; 2624 } 2625 } 2626 } 2627 mtx_unlock(&ng_namehash_mtx); 2628 2629 /* Get response struct */ 2630 NG_MKRESPONSE(resp, msg, sizeof(*nl) 2631 + (num * sizeof(struct nodeinfo)), M_NOWAIT); 2632 if (resp == NULL) { 2633 error = ENOMEM; 2634 break; 2635 } 2636 nl = (struct namelist *) resp->data; 2637 2638 /* Cycle through the linked list of nodes */ 2639 nl->numnames = 0; 2640 mtx_lock(&ng_namehash_mtx); 2641 for (i = 0; i < NG_NAME_HASH_SIZE; i++) { 2642 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2643 struct nodeinfo *const np = 2644 &nl->nodeinfo[nl->numnames]; 2645 2646 if (NG_NODE_NOT_VALID(node)) 2647 continue; 2648 if (!unnamed && (! NG_NODE_HAS_NAME(node))) 2649 continue; 2650 if (nl->numnames >= num) { 2651 log(LOG_ERR, "%s: number of nodes changed\n", 2652 __func__); 2653 break; 2654 } 2655 if (NG_NODE_HAS_NAME(node)) 2656 strcpy(np->name, NG_NODE_NAME(node)); 2657 strcpy(np->type, node->nd_type->name); 2658 np->id = ng_node2ID(node); 2659 np->hooks = node->nd_numhooks; 2660 nl->numnames++; 2661 } 2662 } 2663 mtx_unlock(&ng_namehash_mtx); 2664 break; 2665 } 2666 2667 case NGM_LISTTYPES: 2668 { 2669 struct typelist *tl; 2670 struct ng_type *type; 2671 int num = 0; 2672 2673 mtx_lock(&ng_typelist_mtx); 2674 /* Count number of types */ 2675 LIST_FOREACH(type, &ng_typelist, types) { 2676 num++; 2677 } 2678 mtx_unlock(&ng_typelist_mtx); 2679 2680 /* Get response struct */ 2681 NG_MKRESPONSE(resp, msg, sizeof(*tl) 2682 + (num * sizeof(struct typeinfo)), M_NOWAIT); 2683 if (resp == NULL) { 2684 error = ENOMEM; 2685 break; 2686 } 2687 tl = (struct typelist *) resp->data; 2688 2689 /* Cycle through the linked list of types */ 2690 tl->numtypes = 0; 2691 mtx_lock(&ng_typelist_mtx); 2692 LIST_FOREACH(type, &ng_typelist, types) { 2693 struct typeinfo *const tp = &tl->typeinfo[tl->numtypes]; 2694 2695 if (tl->numtypes >= num) { 2696 log(LOG_ERR, "%s: number of %s changed\n", 2697 __func__, "types"); 2698 break; 2699 } 2700 strcpy(tp->type_name, type->name); 2701 tp->numnodes = type->refs - 1; /* don't count list */ 2702 tl->numtypes++; 2703 } 2704 mtx_unlock(&ng_typelist_mtx); 2705 break; 2706 } 2707 2708 case NGM_BINARY2ASCII: 2709 { 2710 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2711 const struct ng_parse_type *argstype; 2712 const struct ng_cmdlist *c; 2713 struct ng_mesg *binary, *ascii; 2714 2715 /* Data area must contain a valid netgraph message */ 2716 binary = (struct ng_mesg *)msg->data; 2717 if (msg->header.arglen < sizeof(struct ng_mesg) || 2718 (msg->header.arglen - sizeof(struct ng_mesg) < 2719 binary->header.arglen)) { 2720 TRAP_ERROR(); 2721 error = EINVAL; 2722 break; 2723 } 2724 2725 /* Get a response message with lots of room */ 2726 NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT); 2727 if (resp == NULL) { 2728 error = ENOMEM; 2729 break; 2730 } 2731 ascii = (struct ng_mesg *)resp->data; 2732 2733 /* Copy binary message header to response message payload */ 2734 bcopy(binary, ascii, sizeof(*binary)); 2735 2736 /* Find command by matching typecookie and command number */ 2737 for (c = here->nd_type->cmdlist; 2738 c != NULL && c->name != NULL; c++) { 2739 if (binary->header.typecookie == c->cookie 2740 && binary->header.cmd == c->cmd) 2741 break; 2742 } 2743 if (c == NULL || c->name == NULL) { 2744 for (c = ng_generic_cmds; c->name != NULL; c++) { 2745 if (binary->header.typecookie == c->cookie 2746 && binary->header.cmd == c->cmd) 2747 break; 2748 } 2749 if (c->name == NULL) { 2750 NG_FREE_MSG(resp); 2751 error = ENOSYS; 2752 break; 2753 } 2754 } 2755 2756 /* Convert command name to ASCII */ 2757 snprintf(ascii->header.cmdstr, sizeof(ascii->header.cmdstr), 2758 "%s", c->name); 2759 2760 /* Convert command arguments to ASCII */ 2761 argstype = (binary->header.flags & NGF_RESP) ? 2762 c->respType : c->mesgType; 2763 if (argstype == NULL) { 2764 *ascii->data = '\0'; 2765 } else { 2766 if ((error = ng_unparse(argstype, 2767 (u_char *)binary->data, 2768 ascii->data, bufSize)) != 0) { 2769 NG_FREE_MSG(resp); 2770 break; 2771 } 2772 } 2773 2774 /* Return the result as struct ng_mesg plus ASCII string */ 2775 bufSize = strlen(ascii->data) + 1; 2776 ascii->header.arglen = bufSize; 2777 resp->header.arglen = sizeof(*ascii) + bufSize; 2778 break; 2779 } 2780 2781 case NGM_ASCII2BINARY: 2782 { 2783 int bufSize = 2000; /* XXX hard coded constant */ 2784 const struct ng_cmdlist *c; 2785 const struct ng_parse_type *argstype; 2786 struct ng_mesg *ascii, *binary; 2787 int off = 0; 2788 2789 /* Data area must contain at least a struct ng_mesg + '\0' */ 2790 ascii = (struct ng_mesg *)msg->data; 2791 if ((msg->header.arglen < sizeof(*ascii) + 1) || 2792 (ascii->header.arglen < 1) || 2793 (msg->header.arglen < sizeof(*ascii) + 2794 ascii->header.arglen)) { 2795 TRAP_ERROR(); 2796 error = EINVAL; 2797 break; 2798 } 2799 ascii->data[ascii->header.arglen - 1] = '\0'; 2800 2801 /* Get a response message with lots of room */ 2802 NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT); 2803 if (resp == NULL) { 2804 error = ENOMEM; 2805 break; 2806 } 2807 binary = (struct ng_mesg *)resp->data; 2808 2809 /* Copy ASCII message header to response message payload */ 2810 bcopy(ascii, binary, sizeof(*ascii)); 2811 2812 /* Find command by matching ASCII command string */ 2813 for (c = here->nd_type->cmdlist; 2814 c != NULL && c->name != NULL; c++) { 2815 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2816 break; 2817 } 2818 if (c == NULL || c->name == NULL) { 2819 for (c = ng_generic_cmds; c->name != NULL; c++) { 2820 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2821 break; 2822 } 2823 if (c->name == NULL) { 2824 NG_FREE_MSG(resp); 2825 error = ENOSYS; 2826 break; 2827 } 2828 } 2829 2830 /* Convert command name to binary */ 2831 binary->header.cmd = c->cmd; 2832 binary->header.typecookie = c->cookie; 2833 2834 /* Convert command arguments to binary */ 2835 argstype = (binary->header.flags & NGF_RESP) ? 2836 c->respType : c->mesgType; 2837 if (argstype == NULL) { 2838 bufSize = 0; 2839 } else { 2840 if ((error = ng_parse(argstype, ascii->data, 2841 &off, (u_char *)binary->data, &bufSize)) != 0) { 2842 NG_FREE_MSG(resp); 2843 break; 2844 } 2845 } 2846 2847 /* Return the result */ 2848 binary->header.arglen = bufSize; 2849 resp->header.arglen = sizeof(*binary) + bufSize; 2850 break; 2851 } 2852 2853 case NGM_TEXT_CONFIG: 2854 case NGM_TEXT_STATUS: 2855 /* 2856 * This one is tricky as it passes the command down to the 2857 * actual node, even though it is a generic type command. 2858 * This means we must assume that the item/msg is already freed 2859 * when control passes back to us. 2860 */ 2861 if (here->nd_type->rcvmsg != NULL) { 2862 NGI_MSG(item) = msg; /* put it back as we found it */ 2863 return((*here->nd_type->rcvmsg)(here, item, lasthook)); 2864 } 2865 /* Fall through if rcvmsg not supported */ 2866 default: 2867 TRAP_ERROR(); 2868 error = EINVAL; 2869 } 2870 /* 2871 * Sometimes a generic message may be statically allocated 2872 * to avoid problems with allocating when in tight memeory situations. 2873 * Don't free it if it is so. 2874 * I break them appart here, because erros may cause a free if the item 2875 * in which case we'd be doing it twice. 2876 * they are kept together above, to simplify freeing. 2877 */ 2878 out: 2879 NG_RESPOND_MSG(error, here, item, resp); 2880 NG_FREE_MSG(msg); 2881 return (error); 2882 } 2883 2884 /************************************************************************ 2885 Queue element get/free routines 2886 ************************************************************************/ 2887 2888 uma_zone_t ng_qzone; 2889 uma_zone_t ng_qdzone; 2890 static int numthreads = 0; /* number of queue threads */ 2891 static int maxalloc = 4096;/* limit the damage of a leak */ 2892 static int maxdata = 512; /* limit the damage of a DoS */ 2893 2894 TUNABLE_INT("net.graph.threads", &numthreads); 2895 SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads, 2896 0, "Number of queue processing threads"); 2897 TUNABLE_INT("net.graph.maxalloc", &maxalloc); 2898 SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc, 2899 0, "Maximum number of non-data queue items to allocate"); 2900 TUNABLE_INT("net.graph.maxdata", &maxdata); 2901 SYSCTL_INT(_net_graph, OID_AUTO, maxdata, CTLFLAG_RDTUN, &maxdata, 2902 0, "Maximum number of data queue items to allocate"); 2903 2904 #ifdef NETGRAPH_DEBUG 2905 static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist); 2906 static int allocated; /* number of items malloc'd */ 2907 #endif 2908 2909 /* 2910 * Get a queue entry. 2911 * This is usually called when a packet first enters netgraph. 2912 * By definition, this is usually from an interrupt, or from a user. 2913 * Users are not so important, but try be quick for the times that it's 2914 * an interrupt. 2915 */ 2916 static __inline item_p 2917 ng_alloc_item(int type, int flags) 2918 { 2919 item_p item; 2920 2921 KASSERT(((type & ~NGQF_TYPE) == 0), 2922 ("%s: incorrect item type: %d", __func__, type)); 2923 2924 item = uma_zalloc((type == NGQF_DATA)?ng_qdzone:ng_qzone, 2925 ((flags & NG_WAITOK) ? M_WAITOK : M_NOWAIT) | M_ZERO); 2926 2927 if (item) { 2928 item->el_flags = type; 2929 #ifdef NETGRAPH_DEBUG 2930 mtx_lock(&ngq_mtx); 2931 TAILQ_INSERT_TAIL(&ng_itemlist, item, all); 2932 allocated++; 2933 mtx_unlock(&ngq_mtx); 2934 #endif 2935 } 2936 2937 return (item); 2938 } 2939 2940 /* 2941 * Release a queue entry 2942 */ 2943 void 2944 ng_free_item(item_p item) 2945 { 2946 /* 2947 * The item may hold resources on it's own. We need to free 2948 * these before we can free the item. What they are depends upon 2949 * what kind of item it is. it is important that nodes zero 2950 * out pointers to resources that they remove from the item 2951 * or we release them again here. 2952 */ 2953 switch (item->el_flags & NGQF_TYPE) { 2954 case NGQF_DATA: 2955 /* If we have an mbuf still attached.. */ 2956 NG_FREE_M(_NGI_M(item)); 2957 break; 2958 case NGQF_MESG: 2959 _NGI_RETADDR(item) = 0; 2960 NG_FREE_MSG(_NGI_MSG(item)); 2961 break; 2962 case NGQF_FN: 2963 case NGQF_FN2: 2964 /* nothing to free really, */ 2965 _NGI_FN(item) = NULL; 2966 _NGI_ARG1(item) = NULL; 2967 _NGI_ARG2(item) = 0; 2968 break; 2969 } 2970 /* If we still have a node or hook referenced... */ 2971 _NGI_CLR_NODE(item); 2972 _NGI_CLR_HOOK(item); 2973 2974 #ifdef NETGRAPH_DEBUG 2975 mtx_lock(&ngq_mtx); 2976 TAILQ_REMOVE(&ng_itemlist, item, all); 2977 allocated--; 2978 mtx_unlock(&ngq_mtx); 2979 #endif 2980 uma_zfree(((item->el_flags & NGQF_TYPE) == NGQF_DATA)? 2981 ng_qdzone:ng_qzone, item); 2982 } 2983 2984 /* 2985 * Change type of the queue entry. 2986 * Possibly reallocates it from another UMA zone. 2987 */ 2988 static __inline item_p 2989 ng_realloc_item(item_p pitem, int type, int flags) 2990 { 2991 item_p item; 2992 int from, to; 2993 2994 KASSERT((pitem != NULL), ("%s: can't reallocate NULL", __func__)); 2995 KASSERT(((type & ~NGQF_TYPE) == 0), 2996 ("%s: incorrect item type: %d", __func__, type)); 2997 2998 from = ((pitem->el_flags & NGQF_TYPE) == NGQF_DATA); 2999 to = (type == NGQF_DATA); 3000 if (from != to) { 3001 /* If reallocation is required do it and copy item. */ 3002 if ((item = ng_alloc_item(type, flags)) == NULL) { 3003 ng_free_item(pitem); 3004 return (NULL); 3005 } 3006 *item = *pitem; 3007 ng_free_item(pitem); 3008 } else 3009 item = pitem; 3010 item->el_flags = (item->el_flags & ~NGQF_TYPE) | type; 3011 3012 return (item); 3013 } 3014 3015 /************************************************************************ 3016 Module routines 3017 ************************************************************************/ 3018 3019 /* 3020 * Handle the loading/unloading of a netgraph node type module 3021 */ 3022 int 3023 ng_mod_event(module_t mod, int event, void *data) 3024 { 3025 struct ng_type *const type = data; 3026 int s, error = 0; 3027 3028 switch (event) { 3029 case MOD_LOAD: 3030 3031 /* Register new netgraph node type */ 3032 s = splnet(); 3033 if ((error = ng_newtype(type)) != 0) { 3034 splx(s); 3035 break; 3036 } 3037 3038 /* Call type specific code */ 3039 if (type->mod_event != NULL) 3040 if ((error = (*type->mod_event)(mod, event, data))) { 3041 mtx_lock(&ng_typelist_mtx); 3042 type->refs--; /* undo it */ 3043 LIST_REMOVE(type, types); 3044 mtx_unlock(&ng_typelist_mtx); 3045 } 3046 splx(s); 3047 break; 3048 3049 case MOD_UNLOAD: 3050 s = splnet(); 3051 if (type->refs > 1) { /* make sure no nodes exist! */ 3052 error = EBUSY; 3053 } else { 3054 if (type->refs == 0) { 3055 /* failed load, nothing to undo */ 3056 splx(s); 3057 break; 3058 } 3059 if (type->mod_event != NULL) { /* check with type */ 3060 error = (*type->mod_event)(mod, event, data); 3061 if (error != 0) { /* type refuses.. */ 3062 splx(s); 3063 break; 3064 } 3065 } 3066 mtx_lock(&ng_typelist_mtx); 3067 LIST_REMOVE(type, types); 3068 mtx_unlock(&ng_typelist_mtx); 3069 } 3070 splx(s); 3071 break; 3072 3073 default: 3074 if (type->mod_event != NULL) 3075 error = (*type->mod_event)(mod, event, data); 3076 else 3077 error = EOPNOTSUPP; /* XXX ? */ 3078 break; 3079 } 3080 return (error); 3081 } 3082 3083 #ifndef VIMAGE_GLOBALS 3084 static const vnet_modinfo_t vnet_netgraph_modinfo = { 3085 .vmi_id = VNET_MOD_NETGRAPH, 3086 .vmi_name = "netgraph", 3087 .vmi_size = sizeof(struct vnet_netgraph), 3088 .vmi_dependson = VNET_MOD_LOIF, 3089 .vmi_iattach = vnet_netgraph_iattach, 3090 #ifdef VIMAGE 3091 .vmi_idetach = vnet_netgraph_idetach 3092 #endif 3093 }; 3094 #endif 3095 3096 static int 3097 vnet_netgraph_iattach(const void *unused __unused) 3098 { 3099 INIT_VNET_NETGRAPH(curvnet); 3100 3101 V_nextID = 1; 3102 3103 return (0); 3104 } 3105 3106 #ifdef VIMAGE 3107 static int 3108 vnet_netgraph_idetach(const void *unused __unused) 3109 { 3110 INIT_VNET_NETGRAPH(curvnet); 3111 node_p node, last_killed = NULL; 3112 3113 while ((node = LIST_FIRST(&V_ng_nodelist)) != NULL) { 3114 if (node == last_killed) { 3115 /* This should never happen */ 3116 node->nd_flags |= NGF_REALLY_DIE; 3117 printf("netgraph node %s needs NGF_REALLY_DIE\n", 3118 node->nd_name); 3119 ng_rmnode(node, NULL, NULL, 0); 3120 /* This must never happen */ 3121 if (node == LIST_FIRST(&V_ng_nodelist)) 3122 panic("netgraph node %s won't die", 3123 node->nd_name); 3124 } 3125 ng_rmnode(node, NULL, NULL, 0); 3126 last_killed = node; 3127 } 3128 3129 return (0); 3130 } 3131 #endif /* VIMAGE */ 3132 3133 /* 3134 * Handle loading and unloading for this code. 3135 * The only thing we need to link into is the NETISR strucure. 3136 */ 3137 static int 3138 ngb_mod_event(module_t mod, int event, void *data) 3139 { 3140 struct proc *p; 3141 struct thread *td; 3142 int i, error = 0; 3143 3144 switch (event) { 3145 case MOD_LOAD: 3146 /* Initialize everything. */ 3147 #ifndef VIMAGE_GLOBALS 3148 vnet_mod_register(&vnet_netgraph_modinfo); 3149 #else 3150 vnet_netgraph_iattach(NULL); 3151 #endif 3152 NG_WORKLIST_LOCK_INIT(); 3153 mtx_init(&ng_typelist_mtx, "netgraph types mutex", NULL, 3154 MTX_DEF); 3155 mtx_init(&ng_idhash_mtx, "netgraph idhash mutex", NULL, 3156 MTX_DEF); 3157 mtx_init(&ng_namehash_mtx, "netgraph namehash mutex", NULL, 3158 MTX_DEF); 3159 mtx_init(&ng_topo_mtx, "netgraph topology mutex", NULL, 3160 MTX_DEF); 3161 #ifdef NETGRAPH_DEBUG 3162 mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", NULL, 3163 MTX_DEF); 3164 mtx_init(&ngq_mtx, "netgraph item list mutex", NULL, 3165 MTX_DEF); 3166 #endif 3167 ng_qzone = uma_zcreate("NetGraph items", sizeof(struct ng_item), 3168 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 3169 uma_zone_set_max(ng_qzone, maxalloc); 3170 ng_qdzone = uma_zcreate("NetGraph data items", sizeof(struct ng_item), 3171 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 3172 uma_zone_set_max(ng_qdzone, maxdata); 3173 /* Autoconfigure number of threads. */ 3174 if (numthreads <= 0) 3175 numthreads = mp_ncpus; 3176 /* Create threads. */ 3177 p = NULL; /* start with no process */ 3178 for (i = 0; i < numthreads; i++) { 3179 if (kproc_kthread_add(ngthread, NULL, &p, &td, 3180 RFHIGHPID, 0, "ng_queue", "ng_queue%d", i)) { 3181 numthreads = i; 3182 break; 3183 } 3184 } 3185 break; 3186 case MOD_UNLOAD: 3187 /* You can't unload it because an interface may be using it. */ 3188 error = EBUSY; 3189 break; 3190 default: 3191 error = EOPNOTSUPP; 3192 break; 3193 } 3194 return (error); 3195 } 3196 3197 static moduledata_t netgraph_mod = { 3198 "netgraph", 3199 ngb_mod_event, 3200 (NULL) 3201 }; 3202 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_NETGRAPH, SI_ORDER_MIDDLE); 3203 SYSCTL_NODE(_net, OID_AUTO, graph, CTLFLAG_RW, 0, "netgraph Family"); 3204 SYSCTL_INT(_net_graph, OID_AUTO, abi_version, CTLFLAG_RD, 0, NG_ABI_VERSION,""); 3205 SYSCTL_INT(_net_graph, OID_AUTO, msg_version, CTLFLAG_RD, 0, NG_VERSION, ""); 3206 3207 #ifdef NETGRAPH_DEBUG 3208 void 3209 dumphook (hook_p hook, char *file, int line) 3210 { 3211 printf("hook: name %s, %d refs, Last touched:\n", 3212 _NG_HOOK_NAME(hook), hook->hk_refs); 3213 printf(" Last active @ %s, line %d\n", 3214 hook->lastfile, hook->lastline); 3215 if (line) { 3216 printf(" problem discovered at file %s, line %d\n", file, line); 3217 } 3218 } 3219 3220 void 3221 dumpnode(node_p node, char *file, int line) 3222 { 3223 printf("node: ID [%x]: type '%s', %d hooks, flags 0x%x, %d refs, %s:\n", 3224 _NG_NODE_ID(node), node->nd_type->name, 3225 node->nd_numhooks, node->nd_flags, 3226 node->nd_refs, node->nd_name); 3227 printf(" Last active @ %s, line %d\n", 3228 node->lastfile, node->lastline); 3229 if (line) { 3230 printf(" problem discovered at file %s, line %d\n", file, line); 3231 } 3232 } 3233 3234 void 3235 dumpitem(item_p item, char *file, int line) 3236 { 3237 printf(" ACTIVE item, last used at %s, line %d", 3238 item->lastfile, item->lastline); 3239 switch(item->el_flags & NGQF_TYPE) { 3240 case NGQF_DATA: 3241 printf(" - [data]\n"); 3242 break; 3243 case NGQF_MESG: 3244 printf(" - retaddr[%d]:\n", _NGI_RETADDR(item)); 3245 break; 3246 case NGQF_FN: 3247 printf(" - fn@%p (%p, %p, %p, %d (%x))\n", 3248 _NGI_FN(item), 3249 _NGI_NODE(item), 3250 _NGI_HOOK(item), 3251 item->body.fn.fn_arg1, 3252 item->body.fn.fn_arg2, 3253 item->body.fn.fn_arg2); 3254 break; 3255 case NGQF_FN2: 3256 printf(" - fn2@%p (%p, %p, %p, %d (%x))\n", 3257 _NGI_FN2(item), 3258 _NGI_NODE(item), 3259 _NGI_HOOK(item), 3260 item->body.fn.fn_arg1, 3261 item->body.fn.fn_arg2, 3262 item->body.fn.fn_arg2); 3263 break; 3264 } 3265 if (line) { 3266 printf(" problem discovered at file %s, line %d\n", file, line); 3267 if (_NGI_NODE(item)) { 3268 printf("node %p ([%x])\n", 3269 _NGI_NODE(item), ng_node2ID(_NGI_NODE(item))); 3270 } 3271 } 3272 } 3273 3274 static void 3275 ng_dumpitems(void) 3276 { 3277 item_p item; 3278 int i = 1; 3279 TAILQ_FOREACH(item, &ng_itemlist, all) { 3280 printf("[%d] ", i++); 3281 dumpitem(item, NULL, 0); 3282 } 3283 } 3284 3285 static void 3286 ng_dumpnodes(void) 3287 { 3288 node_p node; 3289 int i = 1; 3290 mtx_lock(&ng_nodelist_mtx); 3291 SLIST_FOREACH(node, &ng_allnodes, nd_all) { 3292 printf("[%d] ", i++); 3293 dumpnode(node, NULL, 0); 3294 } 3295 mtx_unlock(&ng_nodelist_mtx); 3296 } 3297 3298 static void 3299 ng_dumphooks(void) 3300 { 3301 hook_p hook; 3302 int i = 1; 3303 mtx_lock(&ng_nodelist_mtx); 3304 SLIST_FOREACH(hook, &ng_allhooks, hk_all) { 3305 printf("[%d] ", i++); 3306 dumphook(hook, NULL, 0); 3307 } 3308 mtx_unlock(&ng_nodelist_mtx); 3309 } 3310 3311 static int 3312 sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS) 3313 { 3314 int error; 3315 int val; 3316 int i; 3317 3318 val = allocated; 3319 i = 1; 3320 error = sysctl_handle_int(oidp, &val, 0, req); 3321 if (error != 0 || req->newptr == NULL) 3322 return (error); 3323 if (val == 42) { 3324 ng_dumpitems(); 3325 ng_dumpnodes(); 3326 ng_dumphooks(); 3327 } 3328 return (0); 3329 } 3330 3331 SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, CTLTYPE_INT | CTLFLAG_RW, 3332 0, sizeof(int), sysctl_debug_ng_dump_items, "I", "Number of allocated items"); 3333 #endif /* NETGRAPH_DEBUG */ 3334 3335 3336 /*********************************************************************** 3337 * Worklist routines 3338 **********************************************************************/ 3339 /* 3340 * Pick a node off the list of nodes with work, 3341 * try get an item to process off it. Remove the node from the list. 3342 */ 3343 static void 3344 ngthread(void *arg) 3345 { 3346 for (;;) { 3347 node_p node; 3348 3349 /* Get node from the worklist. */ 3350 NG_WORKLIST_LOCK(); 3351 while ((node = STAILQ_FIRST(&ng_worklist)) == NULL) 3352 NG_WORKLIST_SLEEP(); 3353 STAILQ_REMOVE_HEAD(&ng_worklist, nd_input_queue.q_work); 3354 NG_WORKLIST_UNLOCK(); 3355 CURVNET_SET(node->nd_vnet); 3356 CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist", 3357 __func__, node->nd_ID, node); 3358 /* 3359 * We have the node. We also take over the reference 3360 * that the list had on it. 3361 * Now process as much as you can, until it won't 3362 * let you have another item off the queue. 3363 * All this time, keep the reference 3364 * that lets us be sure that the node still exists. 3365 * Let the reference go at the last minute. 3366 */ 3367 for (;;) { 3368 item_p item; 3369 int rw; 3370 3371 NG_QUEUE_LOCK(&node->nd_input_queue); 3372 item = ng_dequeue(node, &rw); 3373 if (item == NULL) { 3374 node->nd_input_queue.q_flags2 &= ~NGQ2_WORKQ; 3375 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3376 break; /* go look for another node */ 3377 } else { 3378 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3379 NGI_GET_NODE(item, node); /* zaps stored node */ 3380 ng_apply_item(node, item, rw); 3381 NG_NODE_UNREF(node); 3382 } 3383 } 3384 NG_NODE_UNREF(node); 3385 CURVNET_RESTORE(); 3386 } 3387 } 3388 3389 /* 3390 * XXX 3391 * It's posible that a debugging NG_NODE_REF may need 3392 * to be outside the mutex zone 3393 */ 3394 static void 3395 ng_worklist_add(node_p node) 3396 { 3397 3398 mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); 3399 3400 if ((node->nd_input_queue.q_flags2 & NGQ2_WORKQ) == 0) { 3401 /* 3402 * If we are not already on the work queue, 3403 * then put us on. 3404 */ 3405 node->nd_input_queue.q_flags2 |= NGQ2_WORKQ; 3406 NG_NODE_REF(node); /* XXX fafe in mutex? */ 3407 NG_WORKLIST_LOCK(); 3408 STAILQ_INSERT_TAIL(&ng_worklist, node, nd_input_queue.q_work); 3409 NG_WORKLIST_UNLOCK(); 3410 CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__, 3411 node->nd_ID, node); 3412 NG_WORKLIST_WAKEUP(); 3413 } else { 3414 CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist", 3415 __func__, node->nd_ID, node); 3416 } 3417 } 3418 3419 3420 /*********************************************************************** 3421 * Externally useable functions to set up a queue item ready for sending 3422 ***********************************************************************/ 3423 3424 #ifdef NETGRAPH_DEBUG 3425 #define ITEM_DEBUG_CHECKS \ 3426 do { \ 3427 if (NGI_NODE(item) ) { \ 3428 printf("item already has node"); \ 3429 kdb_enter(KDB_WHY_NETGRAPH, "has node"); \ 3430 NGI_CLR_NODE(item); \ 3431 } \ 3432 if (NGI_HOOK(item) ) { \ 3433 printf("item already has hook"); \ 3434 kdb_enter(KDB_WHY_NETGRAPH, "has hook"); \ 3435 NGI_CLR_HOOK(item); \ 3436 } \ 3437 } while (0) 3438 #else 3439 #define ITEM_DEBUG_CHECKS 3440 #endif 3441 3442 /* 3443 * Put mbuf into the item. 3444 * Hook and node references will be removed when the item is dequeued. 3445 * (or equivalent) 3446 * (XXX) Unsafe because no reference held by peer on remote node. 3447 * remote node might go away in this timescale. 3448 * We know the hooks can't go away because that would require getting 3449 * a writer item on both nodes and we must have at least a reader 3450 * here to be able to do this. 3451 * Note that the hook loaded is the REMOTE hook. 3452 * 3453 * This is possibly in the critical path for new data. 3454 */ 3455 item_p 3456 ng_package_data(struct mbuf *m, int flags) 3457 { 3458 item_p item; 3459 3460 if ((item = ng_alloc_item(NGQF_DATA, flags)) == NULL) { 3461 NG_FREE_M(m); 3462 return (NULL); 3463 } 3464 ITEM_DEBUG_CHECKS; 3465 item->el_flags |= NGQF_READER; 3466 NGI_M(item) = m; 3467 return (item); 3468 } 3469 3470 /* 3471 * Allocate a queue item and put items into it.. 3472 * Evaluate the address as this will be needed to queue it and 3473 * to work out what some of the fields should be. 3474 * Hook and node references will be removed when the item is dequeued. 3475 * (or equivalent) 3476 */ 3477 item_p 3478 ng_package_msg(struct ng_mesg *msg, int flags) 3479 { 3480 item_p item; 3481 3482 if ((item = ng_alloc_item(NGQF_MESG, flags)) == NULL) { 3483 NG_FREE_MSG(msg); 3484 return (NULL); 3485 } 3486 ITEM_DEBUG_CHECKS; 3487 /* Messages items count as writers unless explicitly exempted. */ 3488 if (msg->header.cmd & NGM_READONLY) 3489 item->el_flags |= NGQF_READER; 3490 else 3491 item->el_flags |= NGQF_WRITER; 3492 /* 3493 * Set the current lasthook into the queue item 3494 */ 3495 NGI_MSG(item) = msg; 3496 NGI_RETADDR(item) = 0; 3497 return (item); 3498 } 3499 3500 3501 3502 #define SET_RETADDR(item, here, retaddr) \ 3503 do { /* Data or fn items don't have retaddrs */ \ 3504 if ((item->el_flags & NGQF_TYPE) == NGQF_MESG) { \ 3505 if (retaddr) { \ 3506 NGI_RETADDR(item) = retaddr; \ 3507 } else { \ 3508 /* \ 3509 * The old return address should be ok. \ 3510 * If there isn't one, use the address \ 3511 * here. \ 3512 */ \ 3513 if (NGI_RETADDR(item) == 0) { \ 3514 NGI_RETADDR(item) \ 3515 = ng_node2ID(here); \ 3516 } \ 3517 } \ 3518 } \ 3519 } while (0) 3520 3521 int 3522 ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr) 3523 { 3524 hook_p peer; 3525 node_p peernode; 3526 ITEM_DEBUG_CHECKS; 3527 /* 3528 * Quick sanity check.. 3529 * Since a hook holds a reference on it's node, once we know 3530 * that the peer is still connected (even if invalid,) we know 3531 * that the peer node is present, though maybe invalid. 3532 */ 3533 if ((hook == NULL) || 3534 NG_HOOK_NOT_VALID(hook) || 3535 NG_HOOK_NOT_VALID(peer = NG_HOOK_PEER(hook)) || 3536 NG_NODE_NOT_VALID(peernode = NG_PEER_NODE(hook))) { 3537 NG_FREE_ITEM(item); 3538 TRAP_ERROR(); 3539 return (ENETDOWN); 3540 } 3541 3542 /* 3543 * Transfer our interest to the other (peer) end. 3544 */ 3545 NG_HOOK_REF(peer); 3546 NG_NODE_REF(peernode); 3547 NGI_SET_HOOK(item, peer); 3548 NGI_SET_NODE(item, peernode); 3549 SET_RETADDR(item, here, retaddr); 3550 return (0); 3551 } 3552 3553 int 3554 ng_address_path(node_p here, item_p item, char *address, ng_ID_t retaddr) 3555 { 3556 node_p dest = NULL; 3557 hook_p hook = NULL; 3558 int error; 3559 3560 ITEM_DEBUG_CHECKS; 3561 /* 3562 * Note that ng_path2noderef increments the reference count 3563 * on the node for us if it finds one. So we don't have to. 3564 */ 3565 error = ng_path2noderef(here, address, &dest, &hook); 3566 if (error) { 3567 NG_FREE_ITEM(item); 3568 return (error); 3569 } 3570 NGI_SET_NODE(item, dest); 3571 if ( hook) { 3572 NG_HOOK_REF(hook); /* don't let it go while on the queue */ 3573 NGI_SET_HOOK(item, hook); 3574 } 3575 SET_RETADDR(item, here, retaddr); 3576 return (0); 3577 } 3578 3579 int 3580 ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr) 3581 { 3582 node_p dest; 3583 3584 ITEM_DEBUG_CHECKS; 3585 /* 3586 * Find the target node. 3587 */ 3588 dest = ng_ID2noderef(ID); /* GETS REFERENCE! */ 3589 if (dest == NULL) { 3590 NG_FREE_ITEM(item); 3591 TRAP_ERROR(); 3592 return(EINVAL); 3593 } 3594 /* Fill out the contents */ 3595 NGI_SET_NODE(item, dest); 3596 NGI_CLR_HOOK(item); 3597 SET_RETADDR(item, here, retaddr); 3598 return (0); 3599 } 3600 3601 /* 3602 * special case to send a message to self (e.g. destroy node) 3603 * Possibly indicate an arrival hook too. 3604 * Useful for removing that hook :-) 3605 */ 3606 item_p 3607 ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg) 3608 { 3609 item_p item; 3610 3611 /* 3612 * Find the target node. 3613 * If there is a HOOK argument, then use that in preference 3614 * to the address. 3615 */ 3616 if ((item = ng_alloc_item(NGQF_MESG, NG_NOFLAGS)) == NULL) { 3617 NG_FREE_MSG(msg); 3618 return (NULL); 3619 } 3620 3621 /* Fill out the contents */ 3622 item->el_flags |= NGQF_WRITER; 3623 NG_NODE_REF(here); 3624 NGI_SET_NODE(item, here); 3625 if (hook) { 3626 NG_HOOK_REF(hook); 3627 NGI_SET_HOOK(item, hook); 3628 } 3629 NGI_MSG(item) = msg; 3630 NGI_RETADDR(item) = ng_node2ID(here); 3631 return (item); 3632 } 3633 3634 /* 3635 * Send ng_item_fn function call to the specified node. 3636 */ 3637 3638 int 3639 ng_send_fn(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2) 3640 { 3641 3642 return ng_send_fn1(node, hook, fn, arg1, arg2, NG_NOFLAGS); 3643 } 3644 3645 int 3646 ng_send_fn1(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2, 3647 int flags) 3648 { 3649 item_p item; 3650 3651 if ((item = ng_alloc_item(NGQF_FN, flags)) == NULL) { 3652 return (ENOMEM); 3653 } 3654 item->el_flags |= NGQF_WRITER; 3655 NG_NODE_REF(node); /* and one for the item */ 3656 NGI_SET_NODE(item, node); 3657 if (hook) { 3658 NG_HOOK_REF(hook); 3659 NGI_SET_HOOK(item, hook); 3660 } 3661 NGI_FN(item) = fn; 3662 NGI_ARG1(item) = arg1; 3663 NGI_ARG2(item) = arg2; 3664 return(ng_snd_item(item, flags)); 3665 } 3666 3667 /* 3668 * Send ng_item_fn2 function call to the specified node. 3669 * 3670 * If an optional pitem parameter is supplied, its apply 3671 * callback will be copied to the new item. If also NG_REUSE_ITEM 3672 * flag is set, no new item will be allocated, but pitem will 3673 * be used. 3674 */ 3675 int 3676 ng_send_fn2(node_p node, hook_p hook, item_p pitem, ng_item_fn2 *fn, void *arg1, 3677 int arg2, int flags) 3678 { 3679 item_p item; 3680 3681 KASSERT((pitem != NULL || (flags & NG_REUSE_ITEM) == 0), 3682 ("%s: NG_REUSE_ITEM but no pitem", __func__)); 3683 3684 /* 3685 * Allocate a new item if no supplied or 3686 * if we can't use supplied one. 3687 */ 3688 if (pitem == NULL || (flags & NG_REUSE_ITEM) == 0) { 3689 if ((item = ng_alloc_item(NGQF_FN2, flags)) == NULL) 3690 return (ENOMEM); 3691 if (pitem != NULL) 3692 item->apply = pitem->apply; 3693 } else { 3694 if ((item = ng_realloc_item(pitem, NGQF_FN2, flags)) == NULL) 3695 return (ENOMEM); 3696 } 3697 3698 item->el_flags = (item->el_flags & ~NGQF_RW) | NGQF_WRITER; 3699 NG_NODE_REF(node); /* and one for the item */ 3700 NGI_SET_NODE(item, node); 3701 if (hook) { 3702 NG_HOOK_REF(hook); 3703 NGI_SET_HOOK(item, hook); 3704 } 3705 NGI_FN2(item) = fn; 3706 NGI_ARG1(item) = arg1; 3707 NGI_ARG2(item) = arg2; 3708 return(ng_snd_item(item, flags)); 3709 } 3710 3711 /* 3712 * Official timeout routines for Netgraph nodes. 3713 */ 3714 static void 3715 ng_callout_trampoline(void *arg) 3716 { 3717 item_p item = arg; 3718 3719 CURVNET_SET(NGI_NODE(item)->nd_vnet); 3720 ng_snd_item(item, 0); 3721 CURVNET_RESTORE(); 3722 } 3723 3724 3725 int 3726 ng_callout(struct callout *c, node_p node, hook_p hook, int ticks, 3727 ng_item_fn *fn, void * arg1, int arg2) 3728 { 3729 item_p item, oitem; 3730 3731 if ((item = ng_alloc_item(NGQF_FN, NG_NOFLAGS)) == NULL) 3732 return (ENOMEM); 3733 3734 item->el_flags |= NGQF_WRITER; 3735 NG_NODE_REF(node); /* and one for the item */ 3736 NGI_SET_NODE(item, node); 3737 if (hook) { 3738 NG_HOOK_REF(hook); 3739 NGI_SET_HOOK(item, hook); 3740 } 3741 NGI_FN(item) = fn; 3742 NGI_ARG1(item) = arg1; 3743 NGI_ARG2(item) = arg2; 3744 oitem = c->c_arg; 3745 if (callout_reset(c, ticks, &ng_callout_trampoline, item) == 1 && 3746 oitem != NULL) 3747 NG_FREE_ITEM(oitem); 3748 return (0); 3749 } 3750 3751 /* A special modified version of untimeout() */ 3752 int 3753 ng_uncallout(struct callout *c, node_p node) 3754 { 3755 item_p item; 3756 int rval; 3757 3758 KASSERT(c != NULL, ("ng_uncallout: NULL callout")); 3759 KASSERT(node != NULL, ("ng_uncallout: NULL node")); 3760 3761 rval = callout_stop(c); 3762 item = c->c_arg; 3763 /* Do an extra check */ 3764 if ((rval > 0) && (c->c_func == &ng_callout_trampoline) && 3765 (NGI_NODE(item) == node)) { 3766 /* 3767 * We successfully removed it from the queue before it ran 3768 * So now we need to unreference everything that was 3769 * given extra references. (NG_FREE_ITEM does this). 3770 */ 3771 NG_FREE_ITEM(item); 3772 } 3773 c->c_arg = NULL; 3774 3775 return (rval); 3776 } 3777 3778 /* 3779 * Set the address, if none given, give the node here. 3780 */ 3781 void 3782 ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr) 3783 { 3784 if (retaddr) { 3785 NGI_RETADDR(item) = retaddr; 3786 } else { 3787 /* 3788 * The old return address should be ok. 3789 * If there isn't one, use the address here. 3790 */ 3791 NGI_RETADDR(item) = ng_node2ID(here); 3792 } 3793 } 3794 3795 #define TESTING 3796 #ifdef TESTING 3797 /* just test all the macros */ 3798 void 3799 ng_macro_test(item_p item); 3800 void 3801 ng_macro_test(item_p item) 3802 { 3803 node_p node = NULL; 3804 hook_p hook = NULL; 3805 struct mbuf *m; 3806 struct ng_mesg *msg; 3807 ng_ID_t retaddr; 3808 int error; 3809 3810 NGI_GET_M(item, m); 3811 NGI_GET_MSG(item, msg); 3812 retaddr = NGI_RETADDR(item); 3813 NG_SEND_DATA(error, hook, m, NULL); 3814 NG_SEND_DATA_ONLY(error, hook, m); 3815 NG_FWD_NEW_DATA(error, item, hook, m); 3816 NG_FWD_ITEM_HOOK(error, item, hook); 3817 NG_SEND_MSG_HOOK(error, node, msg, hook, retaddr); 3818 NG_SEND_MSG_ID(error, node, msg, retaddr, retaddr); 3819 NG_SEND_MSG_PATH(error, node, msg, ".:", retaddr); 3820 NG_FWD_MSG_HOOK(error, node, item, hook, retaddr); 3821 } 3822 #endif /* TESTING */ 3823 3824