1 /*- 2 * Copyright (c) 1996-1999 Whistle Communications, Inc. 3 * All rights reserved. 4 * 5 * Subject to the following obligations and disclaimer of warranty, use and 6 * redistribution of this software, in source or object code forms, with or 7 * without modifications are expressly permitted by Whistle Communications; 8 * provided, however, that: 9 * 1. Any and all reproductions of the source or object code must include the 10 * copyright notice above and the following disclaimer of warranties; and 11 * 2. No rights are granted, in any manner or form, to use Whistle 12 * Communications, Inc. trademarks, including the mark "WHISTLE 13 * COMMUNICATIONS" on advertising, endorsements, or otherwise except as 14 * such appears in the above copyright notice or in the software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND 17 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO 18 * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, 19 * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 21 * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY 22 * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS 23 * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE. 24 * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES 25 * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING 26 * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 27 * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR 28 * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY 32 * OF SUCH DAMAGE. 33 * 34 * Authors: Julian Elischer <julian@freebsd.org> 35 * Archie Cobbs <archie@freebsd.org> 36 * 37 * $FreeBSD$ 38 * $Whistle: ng_base.c,v 1.39 1999/01/28 23:54:53 julian Exp $ 39 */ 40 41 /* 42 * This file implements the base netgraph code. 43 */ 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/ctype.h> 48 #include <sys/errno.h> 49 #include <sys/kdb.h> 50 #include <sys/kernel.h> 51 #include <sys/ktr.h> 52 #include <sys/limits.h> 53 #include <sys/malloc.h> 54 #include <sys/mbuf.h> 55 #include <sys/queue.h> 56 #include <sys/sysctl.h> 57 #include <sys/syslog.h> 58 #include <sys/refcount.h> 59 #include <sys/proc.h> 60 #include <sys/unistd.h> 61 #include <sys/kthread.h> 62 #include <sys/smp.h> 63 #include <machine/cpu.h> 64 65 #include <net/netisr.h> 66 #include <net/vnet.h> 67 68 #include <netgraph/ng_message.h> 69 #include <netgraph/netgraph.h> 70 #include <netgraph/ng_parse.h> 71 72 MODULE_VERSION(netgraph, NG_ABI_VERSION); 73 74 /* Mutex to protect topology events. */ 75 static struct mtx ng_topo_mtx; 76 77 #ifdef NETGRAPH_DEBUG 78 static struct mtx ng_nodelist_mtx; /* protects global node/hook lists */ 79 static struct mtx ngq_mtx; /* protects the queue item list */ 80 81 static SLIST_HEAD(, ng_node) ng_allnodes; 82 static LIST_HEAD(, ng_node) ng_freenodes; /* in debug, we never free() them */ 83 static SLIST_HEAD(, ng_hook) ng_allhooks; 84 static LIST_HEAD(, ng_hook) ng_freehooks; /* in debug, we never free() them */ 85 86 static void ng_dumpitems(void); 87 static void ng_dumpnodes(void); 88 static void ng_dumphooks(void); 89 90 #endif /* NETGRAPH_DEBUG */ 91 /* 92 * DEAD versions of the structures. 93 * In order to avoid races, it is sometimes neccesary to point 94 * at SOMETHING even though theoretically, the current entity is 95 * INVALID. Use these to avoid these races. 96 */ 97 struct ng_type ng_deadtype = { 98 NG_ABI_VERSION, 99 "dead", 100 NULL, /* modevent */ 101 NULL, /* constructor */ 102 NULL, /* rcvmsg */ 103 NULL, /* shutdown */ 104 NULL, /* newhook */ 105 NULL, /* findhook */ 106 NULL, /* connect */ 107 NULL, /* rcvdata */ 108 NULL, /* disconnect */ 109 NULL, /* cmdlist */ 110 }; 111 112 struct ng_node ng_deadnode = { 113 "dead", 114 &ng_deadtype, 115 NGF_INVALID, 116 0, /* numhooks */ 117 NULL, /* private */ 118 0, /* ID */ 119 LIST_HEAD_INITIALIZER(ng_deadnode.nd_hooks), 120 {}, /* all_nodes list entry */ 121 {}, /* id hashtable list entry */ 122 { 0, 123 0, 124 {}, /* should never use! (should hang) */ 125 {}, /* workqueue entry */ 126 STAILQ_HEAD_INITIALIZER(ng_deadnode.nd_input_queue.queue), 127 }, 128 1, /* refs */ 129 NULL, /* vnet */ 130 #ifdef NETGRAPH_DEBUG 131 ND_MAGIC, 132 __FILE__, 133 __LINE__, 134 {NULL} 135 #endif /* NETGRAPH_DEBUG */ 136 }; 137 138 struct ng_hook ng_deadhook = { 139 "dead", 140 NULL, /* private */ 141 HK_INVALID | HK_DEAD, 142 0, /* undefined data link type */ 143 &ng_deadhook, /* Peer is self */ 144 &ng_deadnode, /* attached to deadnode */ 145 {}, /* hooks list */ 146 NULL, /* override rcvmsg() */ 147 NULL, /* override rcvdata() */ 148 1, /* refs always >= 1 */ 149 #ifdef NETGRAPH_DEBUG 150 HK_MAGIC, 151 __FILE__, 152 __LINE__, 153 {NULL} 154 #endif /* NETGRAPH_DEBUG */ 155 }; 156 157 /* 158 * END DEAD STRUCTURES 159 */ 160 /* List nodes with unallocated work */ 161 static STAILQ_HEAD(, ng_node) ng_worklist = STAILQ_HEAD_INITIALIZER(ng_worklist); 162 static struct mtx ng_worklist_mtx; /* MUST LOCK NODE FIRST */ 163 164 /* List of installed types */ 165 static LIST_HEAD(, ng_type) ng_typelist; 166 static struct mtx ng_typelist_mtx; 167 168 /* Hash related definitions */ 169 /* XXX Don't need to initialise them because it's a LIST */ 170 static VNET_DEFINE(LIST_HEAD(, ng_node), ng_ID_hash[NG_ID_HASH_SIZE]); 171 #define V_ng_ID_hash VNET(ng_ID_hash) 172 173 static struct mtx ng_idhash_mtx; 174 /* Method to find a node.. used twice so do it here */ 175 #define NG_IDHASH_FN(ID) ((ID) % (NG_ID_HASH_SIZE)) 176 #define NG_IDHASH_FIND(ID, node) \ 177 do { \ 178 mtx_assert(&ng_idhash_mtx, MA_OWNED); \ 179 LIST_FOREACH(node, &V_ng_ID_hash[NG_IDHASH_FN(ID)], \ 180 nd_idnodes) { \ 181 if (NG_NODE_IS_VALID(node) \ 182 && (NG_NODE_ID(node) == ID)) { \ 183 break; \ 184 } \ 185 } \ 186 } while (0) 187 188 static VNET_DEFINE(LIST_HEAD(, ng_node), ng_name_hash[NG_NAME_HASH_SIZE]); 189 #define V_ng_name_hash VNET(ng_name_hash) 190 191 static struct mtx ng_namehash_mtx; 192 #define NG_NAMEHASH(NAME, HASH) \ 193 do { \ 194 u_char h = 0; \ 195 const u_char *c; \ 196 for (c = (const u_char*)(NAME); *c; c++)\ 197 h += *c; \ 198 (HASH) = h % (NG_NAME_HASH_SIZE); \ 199 } while (0) 200 201 202 /* Internal functions */ 203 static int ng_add_hook(node_p node, const char *name, hook_p * hookp); 204 static int ng_generic_msg(node_p here, item_p item, hook_p lasthook); 205 static ng_ID_t ng_decodeidname(const char *name); 206 static int ngb_mod_event(module_t mod, int event, void *data); 207 static void ng_worklist_add(node_p node); 208 static void ngthread(void *); 209 static int ng_apply_item(node_p node, item_p item, int rw); 210 static void ng_flush_input_queue(node_p node); 211 static node_p ng_ID2noderef(ng_ID_t ID); 212 static int ng_con_nodes(item_p item, node_p node, const char *name, 213 node_p node2, const char *name2); 214 static int ng_con_part2(node_p node, item_p item, hook_p hook); 215 static int ng_con_part3(node_p node, item_p item, hook_p hook); 216 static int ng_mkpeer(node_p node, const char *name, 217 const char *name2, char *type); 218 219 /* Imported, these used to be externally visible, some may go back. */ 220 void ng_destroy_hook(hook_p hook); 221 int ng_path2noderef(node_p here, const char *path, 222 node_p *dest, hook_p *lasthook); 223 int ng_make_node(const char *type, node_p *nodepp); 224 int ng_path_parse(char *addr, char **node, char **path, char **hook); 225 void ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3); 226 void ng_unname(node_p node); 227 228 229 /* Our own netgraph malloc type */ 230 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages"); 231 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage"); 232 static MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", 233 "netgraph hook structures"); 234 static MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", 235 "netgraph node structures"); 236 static MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", 237 "netgraph item structures"); 238 239 /* Should not be visible outside this file */ 240 241 #define _NG_ALLOC_HOOK(hook) \ 242 hook = malloc(sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO) 243 #define _NG_ALLOC_NODE(node) \ 244 node = malloc(sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO) 245 246 #define NG_QUEUE_LOCK_INIT(n) \ 247 mtx_init(&(n)->q_mtx, "ng_node", NULL, MTX_DEF) 248 #define NG_QUEUE_LOCK(n) \ 249 mtx_lock(&(n)->q_mtx) 250 #define NG_QUEUE_UNLOCK(n) \ 251 mtx_unlock(&(n)->q_mtx) 252 #define NG_WORKLIST_LOCK_INIT() \ 253 mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_DEF) 254 #define NG_WORKLIST_LOCK() \ 255 mtx_lock(&ng_worklist_mtx) 256 #define NG_WORKLIST_UNLOCK() \ 257 mtx_unlock(&ng_worklist_mtx) 258 #define NG_WORKLIST_SLEEP() \ 259 mtx_sleep(&ng_worklist, &ng_worklist_mtx, PI_NET, "sleep", 0) 260 #define NG_WORKLIST_WAKEUP() \ 261 wakeup_one(&ng_worklist) 262 263 #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ 264 /* 265 * In debug mode: 266 * In an attempt to help track reference count screwups 267 * we do not free objects back to the malloc system, but keep them 268 * in a local cache where we can examine them and keep information safely 269 * after they have been freed. 270 * We use this scheme for nodes and hooks, and to some extent for items. 271 */ 272 static __inline hook_p 273 ng_alloc_hook(void) 274 { 275 hook_p hook; 276 SLIST_ENTRY(ng_hook) temp; 277 mtx_lock(&ng_nodelist_mtx); 278 hook = LIST_FIRST(&ng_freehooks); 279 if (hook) { 280 LIST_REMOVE(hook, hk_hooks); 281 bcopy(&hook->hk_all, &temp, sizeof(temp)); 282 bzero(hook, sizeof(struct ng_hook)); 283 bcopy(&temp, &hook->hk_all, sizeof(temp)); 284 mtx_unlock(&ng_nodelist_mtx); 285 hook->hk_magic = HK_MAGIC; 286 } else { 287 mtx_unlock(&ng_nodelist_mtx); 288 _NG_ALLOC_HOOK(hook); 289 if (hook) { 290 hook->hk_magic = HK_MAGIC; 291 mtx_lock(&ng_nodelist_mtx); 292 SLIST_INSERT_HEAD(&ng_allhooks, hook, hk_all); 293 mtx_unlock(&ng_nodelist_mtx); 294 } 295 } 296 return (hook); 297 } 298 299 static __inline node_p 300 ng_alloc_node(void) 301 { 302 node_p node; 303 SLIST_ENTRY(ng_node) temp; 304 mtx_lock(&ng_nodelist_mtx); 305 node = LIST_FIRST(&ng_freenodes); 306 if (node) { 307 LIST_REMOVE(node, nd_nodes); 308 bcopy(&node->nd_all, &temp, sizeof(temp)); 309 bzero(node, sizeof(struct ng_node)); 310 bcopy(&temp, &node->nd_all, sizeof(temp)); 311 mtx_unlock(&ng_nodelist_mtx); 312 node->nd_magic = ND_MAGIC; 313 } else { 314 mtx_unlock(&ng_nodelist_mtx); 315 _NG_ALLOC_NODE(node); 316 if (node) { 317 node->nd_magic = ND_MAGIC; 318 mtx_lock(&ng_nodelist_mtx); 319 SLIST_INSERT_HEAD(&ng_allnodes, node, nd_all); 320 mtx_unlock(&ng_nodelist_mtx); 321 } 322 } 323 return (node); 324 } 325 326 #define NG_ALLOC_HOOK(hook) do { (hook) = ng_alloc_hook(); } while (0) 327 #define NG_ALLOC_NODE(node) do { (node) = ng_alloc_node(); } while (0) 328 329 330 #define NG_FREE_HOOK(hook) \ 331 do { \ 332 mtx_lock(&ng_nodelist_mtx); \ 333 LIST_INSERT_HEAD(&ng_freehooks, hook, hk_hooks); \ 334 hook->hk_magic = 0; \ 335 mtx_unlock(&ng_nodelist_mtx); \ 336 } while (0) 337 338 #define NG_FREE_NODE(node) \ 339 do { \ 340 mtx_lock(&ng_nodelist_mtx); \ 341 LIST_INSERT_HEAD(&ng_freenodes, node, nd_nodes); \ 342 node->nd_magic = 0; \ 343 mtx_unlock(&ng_nodelist_mtx); \ 344 } while (0) 345 346 #else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 347 348 #define NG_ALLOC_HOOK(hook) _NG_ALLOC_HOOK(hook) 349 #define NG_ALLOC_NODE(node) _NG_ALLOC_NODE(node) 350 351 #define NG_FREE_HOOK(hook) do { free((hook), M_NETGRAPH_HOOK); } while (0) 352 #define NG_FREE_NODE(node) do { free((node), M_NETGRAPH_NODE); } while (0) 353 354 #endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/ 355 356 /* Set this to kdb_enter("X") to catch all errors as they occur */ 357 #ifndef TRAP_ERROR 358 #define TRAP_ERROR() 359 #endif 360 361 static VNET_DEFINE(ng_ID_t, nextID) = 1; 362 #define V_nextID VNET(nextID) 363 364 #ifdef INVARIANTS 365 #define CHECK_DATA_MBUF(m) do { \ 366 struct mbuf *n; \ 367 int total; \ 368 \ 369 M_ASSERTPKTHDR(m); \ 370 for (total = 0, n = (m); n != NULL; n = n->m_next) { \ 371 total += n->m_len; \ 372 if (n->m_nextpkt != NULL) \ 373 panic("%s: m_nextpkt", __func__); \ 374 } \ 375 \ 376 if ((m)->m_pkthdr.len != total) { \ 377 panic("%s: %d != %d", \ 378 __func__, (m)->m_pkthdr.len, total); \ 379 } \ 380 } while (0) 381 #else 382 #define CHECK_DATA_MBUF(m) 383 #endif 384 385 #define ERROUT(x) do { error = (x); goto done; } while (0) 386 387 /************************************************************************ 388 Parse type definitions for generic messages 389 ************************************************************************/ 390 391 /* Handy structure parse type defining macro */ 392 #define DEFINE_PARSE_STRUCT_TYPE(lo, up, args) \ 393 static const struct ng_parse_struct_field \ 394 ng_ ## lo ## _type_fields[] = NG_GENERIC_ ## up ## _INFO args; \ 395 static const struct ng_parse_type ng_generic_ ## lo ## _type = { \ 396 &ng_parse_struct_type, \ 397 &ng_ ## lo ## _type_fields \ 398 } 399 400 DEFINE_PARSE_STRUCT_TYPE(mkpeer, MKPEER, ()); 401 DEFINE_PARSE_STRUCT_TYPE(connect, CONNECT, ()); 402 DEFINE_PARSE_STRUCT_TYPE(name, NAME, ()); 403 DEFINE_PARSE_STRUCT_TYPE(rmhook, RMHOOK, ()); 404 DEFINE_PARSE_STRUCT_TYPE(nodeinfo, NODEINFO, ()); 405 DEFINE_PARSE_STRUCT_TYPE(typeinfo, TYPEINFO, ()); 406 DEFINE_PARSE_STRUCT_TYPE(linkinfo, LINKINFO, (&ng_generic_nodeinfo_type)); 407 408 /* Get length of an array when the length is stored as a 32 bit 409 value immediately preceding the array -- as with struct namelist 410 and struct typelist. */ 411 static int 412 ng_generic_list_getLength(const struct ng_parse_type *type, 413 const u_char *start, const u_char *buf) 414 { 415 return *((const u_int32_t *)(buf - 4)); 416 } 417 418 /* Get length of the array of struct linkinfo inside a struct hooklist */ 419 static int 420 ng_generic_linkinfo_getLength(const struct ng_parse_type *type, 421 const u_char *start, const u_char *buf) 422 { 423 const struct hooklist *hl = (const struct hooklist *)start; 424 425 return hl->nodeinfo.hooks; 426 } 427 428 /* Array type for a variable length array of struct namelist */ 429 static const struct ng_parse_array_info ng_nodeinfoarray_type_info = { 430 &ng_generic_nodeinfo_type, 431 &ng_generic_list_getLength 432 }; 433 static const struct ng_parse_type ng_generic_nodeinfoarray_type = { 434 &ng_parse_array_type, 435 &ng_nodeinfoarray_type_info 436 }; 437 438 /* Array type for a variable length array of struct typelist */ 439 static const struct ng_parse_array_info ng_typeinfoarray_type_info = { 440 &ng_generic_typeinfo_type, 441 &ng_generic_list_getLength 442 }; 443 static const struct ng_parse_type ng_generic_typeinfoarray_type = { 444 &ng_parse_array_type, 445 &ng_typeinfoarray_type_info 446 }; 447 448 /* Array type for array of struct linkinfo in struct hooklist */ 449 static const struct ng_parse_array_info ng_generic_linkinfo_array_type_info = { 450 &ng_generic_linkinfo_type, 451 &ng_generic_linkinfo_getLength 452 }; 453 static const struct ng_parse_type ng_generic_linkinfo_array_type = { 454 &ng_parse_array_type, 455 &ng_generic_linkinfo_array_type_info 456 }; 457 458 DEFINE_PARSE_STRUCT_TYPE(typelist, TYPELIST, (&ng_generic_nodeinfoarray_type)); 459 DEFINE_PARSE_STRUCT_TYPE(hooklist, HOOKLIST, 460 (&ng_generic_nodeinfo_type, &ng_generic_linkinfo_array_type)); 461 DEFINE_PARSE_STRUCT_TYPE(listnodes, LISTNODES, 462 (&ng_generic_nodeinfoarray_type)); 463 464 /* List of commands and how to convert arguments to/from ASCII */ 465 static const struct ng_cmdlist ng_generic_cmds[] = { 466 { 467 NGM_GENERIC_COOKIE, 468 NGM_SHUTDOWN, 469 "shutdown", 470 NULL, 471 NULL 472 }, 473 { 474 NGM_GENERIC_COOKIE, 475 NGM_MKPEER, 476 "mkpeer", 477 &ng_generic_mkpeer_type, 478 NULL 479 }, 480 { 481 NGM_GENERIC_COOKIE, 482 NGM_CONNECT, 483 "connect", 484 &ng_generic_connect_type, 485 NULL 486 }, 487 { 488 NGM_GENERIC_COOKIE, 489 NGM_NAME, 490 "name", 491 &ng_generic_name_type, 492 NULL 493 }, 494 { 495 NGM_GENERIC_COOKIE, 496 NGM_RMHOOK, 497 "rmhook", 498 &ng_generic_rmhook_type, 499 NULL 500 }, 501 { 502 NGM_GENERIC_COOKIE, 503 NGM_NODEINFO, 504 "nodeinfo", 505 NULL, 506 &ng_generic_nodeinfo_type 507 }, 508 { 509 NGM_GENERIC_COOKIE, 510 NGM_LISTHOOKS, 511 "listhooks", 512 NULL, 513 &ng_generic_hooklist_type 514 }, 515 { 516 NGM_GENERIC_COOKIE, 517 NGM_LISTNAMES, 518 "listnames", 519 NULL, 520 &ng_generic_listnodes_type /* same as NGM_LISTNODES */ 521 }, 522 { 523 NGM_GENERIC_COOKIE, 524 NGM_LISTNODES, 525 "listnodes", 526 NULL, 527 &ng_generic_listnodes_type 528 }, 529 { 530 NGM_GENERIC_COOKIE, 531 NGM_LISTTYPES, 532 "listtypes", 533 NULL, 534 &ng_generic_typeinfo_type 535 }, 536 { 537 NGM_GENERIC_COOKIE, 538 NGM_TEXT_CONFIG, 539 "textconfig", 540 NULL, 541 &ng_parse_string_type 542 }, 543 { 544 NGM_GENERIC_COOKIE, 545 NGM_TEXT_STATUS, 546 "textstatus", 547 NULL, 548 &ng_parse_string_type 549 }, 550 { 551 NGM_GENERIC_COOKIE, 552 NGM_ASCII2BINARY, 553 "ascii2binary", 554 &ng_parse_ng_mesg_type, 555 &ng_parse_ng_mesg_type 556 }, 557 { 558 NGM_GENERIC_COOKIE, 559 NGM_BINARY2ASCII, 560 "binary2ascii", 561 &ng_parse_ng_mesg_type, 562 &ng_parse_ng_mesg_type 563 }, 564 { 0 } 565 }; 566 567 /************************************************************************ 568 Node routines 569 ************************************************************************/ 570 571 /* 572 * Instantiate a node of the requested type 573 */ 574 int 575 ng_make_node(const char *typename, node_p *nodepp) 576 { 577 struct ng_type *type; 578 int error; 579 580 /* Check that the type makes sense */ 581 if (typename == NULL) { 582 TRAP_ERROR(); 583 return (EINVAL); 584 } 585 586 /* Locate the node type. If we fail we return. Do not try to load 587 * module. 588 */ 589 if ((type = ng_findtype(typename)) == NULL) 590 return (ENXIO); 591 592 /* 593 * If we have a constructor, then make the node and 594 * call the constructor to do type specific initialisation. 595 */ 596 if (type->constructor != NULL) { 597 if ((error = ng_make_node_common(type, nodepp)) == 0) { 598 if ((error = ((*type->constructor)(*nodepp))) != 0) { 599 NG_NODE_UNREF(*nodepp); 600 } 601 } 602 } else { 603 /* 604 * Node has no constructor. We cannot ask for one 605 * to be made. It must be brought into existence by 606 * some external agency. The external agency should 607 * call ng_make_node_common() directly to get the 608 * netgraph part initialised. 609 */ 610 TRAP_ERROR(); 611 error = EINVAL; 612 } 613 return (error); 614 } 615 616 /* 617 * Generic node creation. Called by node initialisation for externally 618 * instantiated nodes (e.g. hardware, sockets, etc ). 619 * The returned node has a reference count of 1. 620 */ 621 int 622 ng_make_node_common(struct ng_type *type, node_p *nodepp) 623 { 624 node_p node; 625 626 /* Require the node type to have been already installed */ 627 if (ng_findtype(type->name) == NULL) { 628 TRAP_ERROR(); 629 return (EINVAL); 630 } 631 632 /* Make a node and try attach it to the type */ 633 NG_ALLOC_NODE(node); 634 if (node == NULL) { 635 TRAP_ERROR(); 636 return (ENOMEM); 637 } 638 node->nd_type = type; 639 #ifdef VIMAGE 640 node->nd_vnet = curvnet; 641 #endif 642 NG_NODE_REF(node); /* note reference */ 643 type->refs++; 644 645 NG_QUEUE_LOCK_INIT(&node->nd_input_queue); 646 STAILQ_INIT(&node->nd_input_queue.queue); 647 node->nd_input_queue.q_flags = 0; 648 649 /* Initialize hook list for new node */ 650 LIST_INIT(&node->nd_hooks); 651 652 /* Link us into the name hash. */ 653 mtx_lock(&ng_namehash_mtx); 654 LIST_INSERT_HEAD(&V_ng_name_hash[0], node, nd_nodes); 655 mtx_unlock(&ng_namehash_mtx); 656 657 /* get an ID and put us in the hash chain */ 658 mtx_lock(&ng_idhash_mtx); 659 for (;;) { /* wrap protection, even if silly */ 660 node_p node2 = NULL; 661 node->nd_ID = V_nextID++; /* 137/sec for 1 year before wrap */ 662 663 /* Is there a problem with the new number? */ 664 NG_IDHASH_FIND(node->nd_ID, node2); /* already taken? */ 665 if ((node->nd_ID != 0) && (node2 == NULL)) { 666 break; 667 } 668 } 669 LIST_INSERT_HEAD(&V_ng_ID_hash[NG_IDHASH_FN(node->nd_ID)], node, 670 nd_idnodes); 671 mtx_unlock(&ng_idhash_mtx); 672 673 /* Done */ 674 *nodepp = node; 675 return (0); 676 } 677 678 /* 679 * Forceably start the shutdown process on a node. Either call 680 * its shutdown method, or do the default shutdown if there is 681 * no type-specific method. 682 * 683 * We can only be called from a shutdown message, so we know we have 684 * a writer lock, and therefore exclusive access. It also means 685 * that we should not be on the work queue, but we check anyhow. 686 * 687 * Persistent node types must have a type-specific method which 688 * allocates a new node in which case, this one is irretrievably going away, 689 * or cleans up anything it needs, and just makes the node valid again, 690 * in which case we allow the node to survive. 691 * 692 * XXX We need to think of how to tell a persistent node that we 693 * REALLY need to go away because the hardware has gone or we 694 * are rebooting.... etc. 695 */ 696 void 697 ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3) 698 { 699 hook_p hook; 700 701 /* Check if it's already shutting down */ 702 if ((node->nd_flags & NGF_CLOSING) != 0) 703 return; 704 705 if (node == &ng_deadnode) { 706 printf ("shutdown called on deadnode\n"); 707 return; 708 } 709 710 /* Add an extra reference so it doesn't go away during this */ 711 NG_NODE_REF(node); 712 713 /* 714 * Mark it invalid so any newcomers know not to try use it 715 * Also add our own mark so we can't recurse 716 * note that NGF_INVALID does not do this as it's also set during 717 * creation 718 */ 719 node->nd_flags |= NGF_INVALID|NGF_CLOSING; 720 721 /* If node has its pre-shutdown method, then call it first*/ 722 if (node->nd_type && node->nd_type->close) 723 (*node->nd_type->close)(node); 724 725 /* Notify all remaining connected nodes to disconnect */ 726 while ((hook = LIST_FIRST(&node->nd_hooks)) != NULL) 727 ng_destroy_hook(hook); 728 729 /* 730 * Drain the input queue forceably. 731 * it has no hooks so what's it going to do, bleed on someone? 732 * Theoretically we came here from a queue entry that was added 733 * Just before the queue was closed, so it should be empty anyway. 734 * Also removes us from worklist if needed. 735 */ 736 ng_flush_input_queue(node); 737 738 /* Ask the type if it has anything to do in this case */ 739 if (node->nd_type && node->nd_type->shutdown) { 740 (*node->nd_type->shutdown)(node); 741 if (NG_NODE_IS_VALID(node)) { 742 /* 743 * Well, blow me down if the node code hasn't declared 744 * that it doesn't want to die. 745 * Presumably it is a persistant node. 746 * If we REALLY want it to go away, 747 * e.g. hardware going away, 748 * Our caller should set NGF_REALLY_DIE in nd_flags. 749 */ 750 node->nd_flags &= ~(NGF_INVALID|NGF_CLOSING); 751 NG_NODE_UNREF(node); /* Assume they still have theirs */ 752 return; 753 } 754 } else { /* do the default thing */ 755 NG_NODE_UNREF(node); 756 } 757 758 ng_unname(node); /* basically a NOP these days */ 759 760 /* 761 * Remove extra reference, possibly the last 762 * Possible other holders of references may include 763 * timeout callouts, but theoretically the node's supposed to 764 * have cancelled them. Possibly hardware dependencies may 765 * force a driver to 'linger' with a reference. 766 */ 767 NG_NODE_UNREF(node); 768 } 769 770 /* 771 * Remove a reference to the node, possibly the last. 772 * deadnode always acts as it it were the last. 773 */ 774 void 775 ng_unref_node(node_p node) 776 { 777 778 if (node == &ng_deadnode) 779 return; 780 781 if (refcount_release(&node->nd_refs)) { /* we were the last */ 782 783 mtx_lock(&ng_namehash_mtx); 784 node->nd_type->refs--; /* XXX maybe should get types lock? */ 785 LIST_REMOVE(node, nd_nodes); 786 mtx_unlock(&ng_namehash_mtx); 787 788 mtx_lock(&ng_idhash_mtx); 789 LIST_REMOVE(node, nd_idnodes); 790 mtx_unlock(&ng_idhash_mtx); 791 792 mtx_destroy(&node->nd_input_queue.q_mtx); 793 NG_FREE_NODE(node); 794 } 795 } 796 797 /************************************************************************ 798 Node ID handling 799 ************************************************************************/ 800 static node_p 801 ng_ID2noderef(ng_ID_t ID) 802 { 803 node_p node; 804 mtx_lock(&ng_idhash_mtx); 805 NG_IDHASH_FIND(ID, node); 806 if(node) 807 NG_NODE_REF(node); 808 mtx_unlock(&ng_idhash_mtx); 809 return(node); 810 } 811 812 ng_ID_t 813 ng_node2ID(node_p node) 814 { 815 return (node ? NG_NODE_ID(node) : 0); 816 } 817 818 /************************************************************************ 819 Node name handling 820 ************************************************************************/ 821 822 /* 823 * Assign a node a name. 824 */ 825 int 826 ng_name_node(node_p node, const char *name) 827 { 828 int i, hash; 829 node_p node2; 830 831 /* Check the name is valid */ 832 for (i = 0; i < NG_NODESIZ; i++) { 833 if (name[i] == '\0' || name[i] == '.' || name[i] == ':') 834 break; 835 } 836 if (i == 0 || name[i] != '\0') { 837 TRAP_ERROR(); 838 return (EINVAL); 839 } 840 if (ng_decodeidname(name) != 0) { /* valid IDs not allowed here */ 841 TRAP_ERROR(); 842 return (EINVAL); 843 } 844 845 /* Check the name isn't already being used */ 846 if ((node2 = ng_name2noderef(node, name)) != NULL) { 847 NG_NODE_UNREF(node2); 848 TRAP_ERROR(); 849 return (EADDRINUSE); 850 } 851 852 /* copy it */ 853 strlcpy(NG_NODE_NAME(node), name, NG_NODESIZ); 854 855 /* Update name hash. */ 856 NG_NAMEHASH(name, hash); 857 mtx_lock(&ng_namehash_mtx); 858 LIST_REMOVE(node, nd_nodes); 859 LIST_INSERT_HEAD(&V_ng_name_hash[hash], node, nd_nodes); 860 mtx_unlock(&ng_namehash_mtx); 861 862 return (0); 863 } 864 865 /* 866 * Find a node by absolute name. The name should NOT end with ':' 867 * The name "." means "this node" and "[xxx]" means "the node 868 * with ID (ie, at address) xxx". 869 * 870 * Returns the node if found, else NULL. 871 * Eventually should add something faster than a sequential search. 872 * Note it acquires a reference on the node so you can be sure it's still 873 * there. 874 */ 875 node_p 876 ng_name2noderef(node_p here, const char *name) 877 { 878 node_p node; 879 ng_ID_t temp; 880 int hash; 881 882 /* "." means "this node" */ 883 if (strcmp(name, ".") == 0) { 884 NG_NODE_REF(here); 885 return(here); 886 } 887 888 /* Check for name-by-ID */ 889 if ((temp = ng_decodeidname(name)) != 0) { 890 return (ng_ID2noderef(temp)); 891 } 892 893 /* Find node by name */ 894 NG_NAMEHASH(name, hash); 895 mtx_lock(&ng_namehash_mtx); 896 LIST_FOREACH(node, &V_ng_name_hash[hash], nd_nodes) { 897 if (NG_NODE_IS_VALID(node) && 898 (strcmp(NG_NODE_NAME(node), name) == 0)) { 899 break; 900 } 901 } 902 if (node) 903 NG_NODE_REF(node); 904 mtx_unlock(&ng_namehash_mtx); 905 return (node); 906 } 907 908 /* 909 * Decode an ID name, eg. "[f03034de]". Returns 0 if the 910 * string is not valid, otherwise returns the value. 911 */ 912 static ng_ID_t 913 ng_decodeidname(const char *name) 914 { 915 const int len = strlen(name); 916 char *eptr; 917 u_long val; 918 919 /* Check for proper length, brackets, no leading junk */ 920 if ((len < 3) || (name[0] != '[') || (name[len - 1] != ']') || 921 (!isxdigit(name[1]))) 922 return ((ng_ID_t)0); 923 924 /* Decode number */ 925 val = strtoul(name + 1, &eptr, 16); 926 if ((eptr - name != len - 1) || (val == ULONG_MAX) || (val == 0)) 927 return ((ng_ID_t)0); 928 929 return ((ng_ID_t)val); 930 } 931 932 /* 933 * Remove a name from a node. This should only be called 934 * when shutting down and removing the node. 935 */ 936 void 937 ng_unname(node_p node) 938 { 939 } 940 941 /************************************************************************ 942 Hook routines 943 Names are not optional. Hooks are always connected, except for a 944 brief moment within these routines. On invalidation or during creation 945 they are connected to the 'dead' hook. 946 ************************************************************************/ 947 948 /* 949 * Remove a hook reference 950 */ 951 void 952 ng_unref_hook(hook_p hook) 953 { 954 955 if (hook == &ng_deadhook) 956 return; 957 958 if (refcount_release(&hook->hk_refs)) { /* we were the last */ 959 if (_NG_HOOK_NODE(hook)) /* it'll probably be ng_deadnode */ 960 _NG_NODE_UNREF((_NG_HOOK_NODE(hook))); 961 NG_FREE_HOOK(hook); 962 } 963 } 964 965 /* 966 * Add an unconnected hook to a node. Only used internally. 967 * Assumes node is locked. (XXX not yet true ) 968 */ 969 static int 970 ng_add_hook(node_p node, const char *name, hook_p *hookp) 971 { 972 hook_p hook; 973 int error = 0; 974 975 /* Check that the given name is good */ 976 if (name == NULL) { 977 TRAP_ERROR(); 978 return (EINVAL); 979 } 980 if (ng_findhook(node, name) != NULL) { 981 TRAP_ERROR(); 982 return (EEXIST); 983 } 984 985 /* Allocate the hook and link it up */ 986 NG_ALLOC_HOOK(hook); 987 if (hook == NULL) { 988 TRAP_ERROR(); 989 return (ENOMEM); 990 } 991 hook->hk_refs = 1; /* add a reference for us to return */ 992 hook->hk_flags = HK_INVALID; 993 hook->hk_peer = &ng_deadhook; /* start off this way */ 994 hook->hk_node = node; 995 NG_NODE_REF(node); /* each hook counts as a reference */ 996 997 /* Set hook name */ 998 strlcpy(NG_HOOK_NAME(hook), name, NG_HOOKSIZ); 999 1000 /* 1001 * Check if the node type code has something to say about it 1002 * If it fails, the unref of the hook will also unref the node. 1003 */ 1004 if (node->nd_type->newhook != NULL) { 1005 if ((error = (*node->nd_type->newhook)(node, hook, name))) { 1006 NG_HOOK_UNREF(hook); /* this frees the hook */ 1007 return (error); 1008 } 1009 } 1010 /* 1011 * The 'type' agrees so far, so go ahead and link it in. 1012 * We'll ask again later when we actually connect the hooks. 1013 */ 1014 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1015 node->nd_numhooks++; 1016 NG_HOOK_REF(hook); /* one for the node */ 1017 1018 if (hookp) 1019 *hookp = hook; 1020 return (0); 1021 } 1022 1023 /* 1024 * Find a hook 1025 * 1026 * Node types may supply their own optimized routines for finding 1027 * hooks. If none is supplied, we just do a linear search. 1028 * XXX Possibly we should add a reference to the hook? 1029 */ 1030 hook_p 1031 ng_findhook(node_p node, const char *name) 1032 { 1033 hook_p hook; 1034 1035 if (node->nd_type->findhook != NULL) 1036 return (*node->nd_type->findhook)(node, name); 1037 LIST_FOREACH(hook, &node->nd_hooks, hk_hooks) { 1038 if (NG_HOOK_IS_VALID(hook) && 1039 (strcmp(NG_HOOK_NAME(hook), name) == 0)) 1040 return (hook); 1041 } 1042 return (NULL); 1043 } 1044 1045 /* 1046 * Destroy a hook 1047 * 1048 * As hooks are always attached, this really destroys two hooks. 1049 * The one given, and the one attached to it. Disconnect the hooks 1050 * from each other first. We reconnect the peer hook to the 'dead' 1051 * hook so that it can still exist after we depart. We then 1052 * send the peer its own destroy message. This ensures that we only 1053 * interact with the peer's structures when it is locked processing that 1054 * message. We hold a reference to the peer hook so we are guaranteed that 1055 * the peer hook and node are still going to exist until 1056 * we are finished there as the hook holds a ref on the node. 1057 * We run this same code again on the peer hook, but that time it is already 1058 * attached to the 'dead' hook. 1059 * 1060 * This routine is called at all stages of hook creation 1061 * on error detection and must be able to handle any such stage. 1062 */ 1063 void 1064 ng_destroy_hook(hook_p hook) 1065 { 1066 hook_p peer; 1067 node_p node; 1068 1069 if (hook == &ng_deadhook) { /* better safe than sorry */ 1070 printf("ng_destroy_hook called on deadhook\n"); 1071 return; 1072 } 1073 1074 /* 1075 * Protect divorce process with mutex, to avoid races on 1076 * simultaneous disconnect. 1077 */ 1078 mtx_lock(&ng_topo_mtx); 1079 1080 hook->hk_flags |= HK_INVALID; 1081 1082 peer = NG_HOOK_PEER(hook); 1083 node = NG_HOOK_NODE(hook); 1084 1085 if (peer && (peer != &ng_deadhook)) { 1086 /* 1087 * Set the peer to point to ng_deadhook 1088 * from this moment on we are effectively independent it. 1089 * send it an rmhook message of it's own. 1090 */ 1091 peer->hk_peer = &ng_deadhook; /* They no longer know us */ 1092 hook->hk_peer = &ng_deadhook; /* Nor us, them */ 1093 if (NG_HOOK_NODE(peer) == &ng_deadnode) { 1094 /* 1095 * If it's already divorced from a node, 1096 * just free it. 1097 */ 1098 mtx_unlock(&ng_topo_mtx); 1099 } else { 1100 mtx_unlock(&ng_topo_mtx); 1101 ng_rmhook_self(peer); /* Send it a surprise */ 1102 } 1103 NG_HOOK_UNREF(peer); /* account for peer link */ 1104 NG_HOOK_UNREF(hook); /* account for peer link */ 1105 } else 1106 mtx_unlock(&ng_topo_mtx); 1107 1108 mtx_assert(&ng_topo_mtx, MA_NOTOWNED); 1109 1110 /* 1111 * Remove the hook from the node's list to avoid possible recursion 1112 * in case the disconnection results in node shutdown. 1113 */ 1114 if (node == &ng_deadnode) { /* happens if called from ng_con_nodes() */ 1115 return; 1116 } 1117 LIST_REMOVE(hook, hk_hooks); 1118 node->nd_numhooks--; 1119 if (node->nd_type->disconnect) { 1120 /* 1121 * The type handler may elect to destroy the node so don't 1122 * trust its existence after this point. (except 1123 * that we still hold a reference on it. (which we 1124 * inherrited from the hook we are destroying) 1125 */ 1126 (*node->nd_type->disconnect) (hook); 1127 } 1128 1129 /* 1130 * Note that because we will point to ng_deadnode, the original node 1131 * is not decremented automatically so we do that manually. 1132 */ 1133 _NG_HOOK_NODE(hook) = &ng_deadnode; 1134 NG_NODE_UNREF(node); /* We no longer point to it so adjust count */ 1135 NG_HOOK_UNREF(hook); /* Account for linkage (in list) to node */ 1136 } 1137 1138 /* 1139 * Take two hooks on a node and merge the connection so that the given node 1140 * is effectively bypassed. 1141 */ 1142 int 1143 ng_bypass(hook_p hook1, hook_p hook2) 1144 { 1145 if (hook1->hk_node != hook2->hk_node) { 1146 TRAP_ERROR(); 1147 return (EINVAL); 1148 } 1149 mtx_lock(&ng_topo_mtx); 1150 hook1->hk_peer->hk_peer = hook2->hk_peer; 1151 hook2->hk_peer->hk_peer = hook1->hk_peer; 1152 1153 hook1->hk_peer = &ng_deadhook; 1154 hook2->hk_peer = &ng_deadhook; 1155 mtx_unlock(&ng_topo_mtx); 1156 1157 NG_HOOK_UNREF(hook1); 1158 NG_HOOK_UNREF(hook2); 1159 1160 /* XXX If we ever cache methods on hooks update them as well */ 1161 ng_destroy_hook(hook1); 1162 ng_destroy_hook(hook2); 1163 return (0); 1164 } 1165 1166 /* 1167 * Install a new netgraph type 1168 */ 1169 int 1170 ng_newtype(struct ng_type *tp) 1171 { 1172 const size_t namelen = strlen(tp->name); 1173 1174 /* Check version and type name fields */ 1175 if ((tp->version != NG_ABI_VERSION) || (namelen == 0) || 1176 (namelen >= NG_TYPESIZ)) { 1177 TRAP_ERROR(); 1178 if (tp->version != NG_ABI_VERSION) { 1179 printf("Netgraph: Node type rejected. ABI mismatch. " 1180 "Suggest recompile\n"); 1181 } 1182 return (EINVAL); 1183 } 1184 1185 /* Check for name collision */ 1186 if (ng_findtype(tp->name) != NULL) { 1187 TRAP_ERROR(); 1188 return (EEXIST); 1189 } 1190 1191 1192 /* Link in new type */ 1193 mtx_lock(&ng_typelist_mtx); 1194 LIST_INSERT_HEAD(&ng_typelist, tp, types); 1195 tp->refs = 1; /* first ref is linked list */ 1196 mtx_unlock(&ng_typelist_mtx); 1197 return (0); 1198 } 1199 1200 /* 1201 * unlink a netgraph type 1202 * If no examples exist 1203 */ 1204 int 1205 ng_rmtype(struct ng_type *tp) 1206 { 1207 /* Check for name collision */ 1208 if (tp->refs != 1) { 1209 TRAP_ERROR(); 1210 return (EBUSY); 1211 } 1212 1213 /* Unlink type */ 1214 mtx_lock(&ng_typelist_mtx); 1215 LIST_REMOVE(tp, types); 1216 mtx_unlock(&ng_typelist_mtx); 1217 return (0); 1218 } 1219 1220 /* 1221 * Look for a type of the name given 1222 */ 1223 struct ng_type * 1224 ng_findtype(const char *typename) 1225 { 1226 struct ng_type *type; 1227 1228 mtx_lock(&ng_typelist_mtx); 1229 LIST_FOREACH(type, &ng_typelist, types) { 1230 if (strcmp(type->name, typename) == 0) 1231 break; 1232 } 1233 mtx_unlock(&ng_typelist_mtx); 1234 return (type); 1235 } 1236 1237 /************************************************************************ 1238 Composite routines 1239 ************************************************************************/ 1240 /* 1241 * Connect two nodes using the specified hooks, using queued functions. 1242 */ 1243 static int 1244 ng_con_part3(node_p node, item_p item, hook_p hook) 1245 { 1246 int error = 0; 1247 1248 /* 1249 * When we run, we know that the node 'node' is locked for us. 1250 * Our caller has a reference on the hook. 1251 * Our caller has a reference on the node. 1252 * (In this case our caller is ng_apply_item() ). 1253 * The peer hook has a reference on the hook. 1254 * We are all set up except for the final call to the node, and 1255 * the clearing of the INVALID flag. 1256 */ 1257 if (NG_HOOK_NODE(hook) == &ng_deadnode) { 1258 /* 1259 * The node must have been freed again since we last visited 1260 * here. ng_destry_hook() has this effect but nothing else does. 1261 * We should just release our references and 1262 * free anything we can think of. 1263 * Since we know it's been destroyed, and it's our caller 1264 * that holds the references, just return. 1265 */ 1266 ERROUT(ENOENT); 1267 } 1268 if (hook->hk_node->nd_type->connect) { 1269 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1270 ng_destroy_hook(hook); /* also zaps peer */ 1271 printf("failed in ng_con_part3()\n"); 1272 ERROUT(error); 1273 } 1274 } 1275 /* 1276 * XXX this is wrong for SMP. Possibly we need 1277 * to separate out 'create' and 'invalid' flags. 1278 * should only set flags on hooks we have locked under our node. 1279 */ 1280 hook->hk_flags &= ~HK_INVALID; 1281 done: 1282 NG_FREE_ITEM(item); 1283 return (error); 1284 } 1285 1286 static int 1287 ng_con_part2(node_p node, item_p item, hook_p hook) 1288 { 1289 hook_p peer; 1290 int error = 0; 1291 1292 /* 1293 * When we run, we know that the node 'node' is locked for us. 1294 * Our caller has a reference on the hook. 1295 * Our caller has a reference on the node. 1296 * (In this case our caller is ng_apply_item() ). 1297 * The peer hook has a reference on the hook. 1298 * our node pointer points to the 'dead' node. 1299 * First check the hook name is unique. 1300 * Should not happen because we checked before queueing this. 1301 */ 1302 if (ng_findhook(node, NG_HOOK_NAME(hook)) != NULL) { 1303 TRAP_ERROR(); 1304 ng_destroy_hook(hook); /* should destroy peer too */ 1305 printf("failed in ng_con_part2()\n"); 1306 ERROUT(EEXIST); 1307 } 1308 /* 1309 * Check if the node type code has something to say about it 1310 * If it fails, the unref of the hook will also unref the attached node, 1311 * however since that node is 'ng_deadnode' this will do nothing. 1312 * The peer hook will also be destroyed. 1313 */ 1314 if (node->nd_type->newhook != NULL) { 1315 if ((error = (*node->nd_type->newhook)(node, hook, 1316 hook->hk_name))) { 1317 ng_destroy_hook(hook); /* should destroy peer too */ 1318 printf("failed in ng_con_part2()\n"); 1319 ERROUT(error); 1320 } 1321 } 1322 1323 /* 1324 * The 'type' agrees so far, so go ahead and link it in. 1325 * We'll ask again later when we actually connect the hooks. 1326 */ 1327 hook->hk_node = node; /* just overwrite ng_deadnode */ 1328 NG_NODE_REF(node); /* each hook counts as a reference */ 1329 LIST_INSERT_HEAD(&node->nd_hooks, hook, hk_hooks); 1330 node->nd_numhooks++; 1331 NG_HOOK_REF(hook); /* one for the node */ 1332 1333 /* 1334 * We now have a symmetrical situation, where both hooks have been 1335 * linked to their nodes, the newhook methods have been called 1336 * And the references are all correct. The hooks are still marked 1337 * as invalid, as we have not called the 'connect' methods 1338 * yet. 1339 * We can call the local one immediately as we have the 1340 * node locked, but we need to queue the remote one. 1341 */ 1342 if (hook->hk_node->nd_type->connect) { 1343 if ((error = (*hook->hk_node->nd_type->connect) (hook))) { 1344 ng_destroy_hook(hook); /* also zaps peer */ 1345 printf("failed in ng_con_part2(A)\n"); 1346 ERROUT(error); 1347 } 1348 } 1349 1350 /* 1351 * Acquire topo mutex to avoid race with ng_destroy_hook(). 1352 */ 1353 mtx_lock(&ng_topo_mtx); 1354 peer = hook->hk_peer; 1355 if (peer == &ng_deadhook) { 1356 mtx_unlock(&ng_topo_mtx); 1357 printf("failed in ng_con_part2(B)\n"); 1358 ng_destroy_hook(hook); 1359 ERROUT(ENOENT); 1360 } 1361 mtx_unlock(&ng_topo_mtx); 1362 1363 if ((error = ng_send_fn2(peer->hk_node, peer, item, &ng_con_part3, 1364 NULL, 0, NG_REUSE_ITEM))) { 1365 printf("failed in ng_con_part2(C)\n"); 1366 ng_destroy_hook(hook); /* also zaps peer */ 1367 return (error); /* item was consumed. */ 1368 } 1369 hook->hk_flags &= ~HK_INVALID; /* need both to be able to work */ 1370 return (0); /* item was consumed. */ 1371 done: 1372 NG_FREE_ITEM(item); 1373 return (error); 1374 } 1375 1376 /* 1377 * Connect this node with another node. We assume that this node is 1378 * currently locked, as we are only called from an NGM_CONNECT message. 1379 */ 1380 static int 1381 ng_con_nodes(item_p item, node_p node, const char *name, 1382 node_p node2, const char *name2) 1383 { 1384 int error; 1385 hook_p hook; 1386 hook_p hook2; 1387 1388 if (ng_findhook(node2, name2) != NULL) { 1389 return(EEXIST); 1390 } 1391 if ((error = ng_add_hook(node, name, &hook))) /* gives us a ref */ 1392 return (error); 1393 /* Allocate the other hook and link it up */ 1394 NG_ALLOC_HOOK(hook2); 1395 if (hook2 == NULL) { 1396 TRAP_ERROR(); 1397 ng_destroy_hook(hook); /* XXX check ref counts so far */ 1398 NG_HOOK_UNREF(hook); /* including our ref */ 1399 return (ENOMEM); 1400 } 1401 hook2->hk_refs = 1; /* start with a reference for us. */ 1402 hook2->hk_flags = HK_INVALID; 1403 hook2->hk_peer = hook; /* Link the two together */ 1404 hook->hk_peer = hook2; 1405 NG_HOOK_REF(hook); /* Add a ref for the peer to each*/ 1406 NG_HOOK_REF(hook2); 1407 hook2->hk_node = &ng_deadnode; 1408 strlcpy(NG_HOOK_NAME(hook2), name2, NG_HOOKSIZ); 1409 1410 /* 1411 * Queue the function above. 1412 * Procesing continues in that function in the lock context of 1413 * the other node. 1414 */ 1415 if ((error = ng_send_fn2(node2, hook2, item, &ng_con_part2, NULL, 0, 1416 NG_NOFLAGS))) { 1417 printf("failed in ng_con_nodes(): %d\n", error); 1418 ng_destroy_hook(hook); /* also zaps peer */ 1419 } 1420 1421 NG_HOOK_UNREF(hook); /* Let each hook go if it wants to */ 1422 NG_HOOK_UNREF(hook2); 1423 return (error); 1424 } 1425 1426 /* 1427 * Make a peer and connect. 1428 * We assume that the local node is locked. 1429 * The new node probably doesn't need a lock until 1430 * it has a hook, because it cannot really have any work until then, 1431 * but we should think about it a bit more. 1432 * 1433 * The problem may come if the other node also fires up 1434 * some hardware or a timer or some other source of activation, 1435 * also it may already get a command msg via it's ID. 1436 * 1437 * We could use the same method as ng_con_nodes() but we'd have 1438 * to add ability to remove the node when failing. (Not hard, just 1439 * make arg1 point to the node to remove). 1440 * Unless of course we just ignore failure to connect and leave 1441 * an unconnected node? 1442 */ 1443 static int 1444 ng_mkpeer(node_p node, const char *name, const char *name2, char *type) 1445 { 1446 node_p node2; 1447 hook_p hook1, hook2; 1448 int error; 1449 1450 if ((error = ng_make_node(type, &node2))) { 1451 return (error); 1452 } 1453 1454 if ((error = ng_add_hook(node, name, &hook1))) { /* gives us a ref */ 1455 ng_rmnode(node2, NULL, NULL, 0); 1456 return (error); 1457 } 1458 1459 if ((error = ng_add_hook(node2, name2, &hook2))) { 1460 ng_rmnode(node2, NULL, NULL, 0); 1461 ng_destroy_hook(hook1); 1462 NG_HOOK_UNREF(hook1); 1463 return (error); 1464 } 1465 1466 /* 1467 * Actually link the two hooks together. 1468 */ 1469 hook1->hk_peer = hook2; 1470 hook2->hk_peer = hook1; 1471 1472 /* Each hook is referenced by the other */ 1473 NG_HOOK_REF(hook1); 1474 NG_HOOK_REF(hook2); 1475 1476 /* Give each node the opportunity to veto the pending connection */ 1477 if (hook1->hk_node->nd_type->connect) { 1478 error = (*hook1->hk_node->nd_type->connect) (hook1); 1479 } 1480 1481 if ((error == 0) && hook2->hk_node->nd_type->connect) { 1482 error = (*hook2->hk_node->nd_type->connect) (hook2); 1483 1484 } 1485 1486 /* 1487 * drop the references we were holding on the two hooks. 1488 */ 1489 if (error) { 1490 ng_destroy_hook(hook2); /* also zaps hook1 */ 1491 ng_rmnode(node2, NULL, NULL, 0); 1492 } else { 1493 /* As a last act, allow the hooks to be used */ 1494 hook1->hk_flags &= ~HK_INVALID; 1495 hook2->hk_flags &= ~HK_INVALID; 1496 } 1497 NG_HOOK_UNREF(hook1); 1498 NG_HOOK_UNREF(hook2); 1499 return (error); 1500 } 1501 1502 /************************************************************************ 1503 Utility routines to send self messages 1504 ************************************************************************/ 1505 1506 /* Shut this node down as soon as everyone is clear of it */ 1507 /* Should add arg "immediately" to jump the queue */ 1508 int 1509 ng_rmnode_self(node_p node) 1510 { 1511 int error; 1512 1513 if (node == &ng_deadnode) 1514 return (0); 1515 node->nd_flags |= NGF_INVALID; 1516 if (node->nd_flags & NGF_CLOSING) 1517 return (0); 1518 1519 error = ng_send_fn(node, NULL, &ng_rmnode, NULL, 0); 1520 return (error); 1521 } 1522 1523 static void 1524 ng_rmhook_part2(node_p node, hook_p hook, void *arg1, int arg2) 1525 { 1526 ng_destroy_hook(hook); 1527 return ; 1528 } 1529 1530 int 1531 ng_rmhook_self(hook_p hook) 1532 { 1533 int error; 1534 node_p node = NG_HOOK_NODE(hook); 1535 1536 if (node == &ng_deadnode) 1537 return (0); 1538 1539 error = ng_send_fn(node, hook, &ng_rmhook_part2, NULL, 0); 1540 return (error); 1541 } 1542 1543 /*********************************************************************** 1544 * Parse and verify a string of the form: <NODE:><PATH> 1545 * 1546 * Such a string can refer to a specific node or a specific hook 1547 * on a specific node, depending on how you look at it. In the 1548 * latter case, the PATH component must not end in a dot. 1549 * 1550 * Both <NODE:> and <PATH> are optional. The <PATH> is a string 1551 * of hook names separated by dots. This breaks out the original 1552 * string, setting *nodep to "NODE" (or NULL if none) and *pathp 1553 * to "PATH" (or NULL if degenerate). Also, *hookp will point to 1554 * the final hook component of <PATH>, if any, otherwise NULL. 1555 * 1556 * This returns -1 if the path is malformed. The char ** are optional. 1557 ***********************************************************************/ 1558 int 1559 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp) 1560 { 1561 char *node, *path, *hook; 1562 int k; 1563 1564 /* 1565 * Extract absolute NODE, if any 1566 */ 1567 for (path = addr; *path && *path != ':'; path++); 1568 if (*path) { 1569 node = addr; /* Here's the NODE */ 1570 *path++ = '\0'; /* Here's the PATH */ 1571 1572 /* Node name must not be empty */ 1573 if (!*node) 1574 return -1; 1575 1576 /* A name of "." is OK; otherwise '.' not allowed */ 1577 if (strcmp(node, ".") != 0) { 1578 for (k = 0; node[k]; k++) 1579 if (node[k] == '.') 1580 return -1; 1581 } 1582 } else { 1583 node = NULL; /* No absolute NODE */ 1584 path = addr; /* Here's the PATH */ 1585 } 1586 1587 /* Snoop for illegal characters in PATH */ 1588 for (k = 0; path[k]; k++) 1589 if (path[k] == ':') 1590 return -1; 1591 1592 /* Check for no repeated dots in PATH */ 1593 for (k = 0; path[k]; k++) 1594 if (path[k] == '.' && path[k + 1] == '.') 1595 return -1; 1596 1597 /* Remove extra (degenerate) dots from beginning or end of PATH */ 1598 if (path[0] == '.') 1599 path++; 1600 if (*path && path[strlen(path) - 1] == '.') 1601 path[strlen(path) - 1] = 0; 1602 1603 /* If PATH has a dot, then we're not talking about a hook */ 1604 if (*path) { 1605 for (hook = path, k = 0; path[k]; k++) 1606 if (path[k] == '.') { 1607 hook = NULL; 1608 break; 1609 } 1610 } else 1611 path = hook = NULL; 1612 1613 /* Done */ 1614 if (nodep) 1615 *nodep = node; 1616 if (pathp) 1617 *pathp = path; 1618 if (hookp) 1619 *hookp = hook; 1620 return (0); 1621 } 1622 1623 /* 1624 * Given a path, which may be absolute or relative, and a starting node, 1625 * return the destination node. 1626 */ 1627 int 1628 ng_path2noderef(node_p here, const char *address, node_p *destp, 1629 hook_p *lasthook) 1630 { 1631 char fullpath[NG_PATHSIZ]; 1632 char *nodename, *path; 1633 node_p node, oldnode; 1634 1635 /* Initialize */ 1636 if (destp == NULL) { 1637 TRAP_ERROR(); 1638 return EINVAL; 1639 } 1640 *destp = NULL; 1641 1642 /* Make a writable copy of address for ng_path_parse() */ 1643 strncpy(fullpath, address, sizeof(fullpath) - 1); 1644 fullpath[sizeof(fullpath) - 1] = '\0'; 1645 1646 /* Parse out node and sequence of hooks */ 1647 if (ng_path_parse(fullpath, &nodename, &path, NULL) < 0) { 1648 TRAP_ERROR(); 1649 return EINVAL; 1650 } 1651 1652 /* 1653 * For an absolute address, jump to the starting node. 1654 * Note that this holds a reference on the node for us. 1655 * Don't forget to drop the reference if we don't need it. 1656 */ 1657 if (nodename) { 1658 node = ng_name2noderef(here, nodename); 1659 if (node == NULL) { 1660 TRAP_ERROR(); 1661 return (ENOENT); 1662 } 1663 } else { 1664 if (here == NULL) { 1665 TRAP_ERROR(); 1666 return (EINVAL); 1667 } 1668 node = here; 1669 NG_NODE_REF(node); 1670 } 1671 1672 if (path == NULL) { 1673 if (lasthook != NULL) 1674 *lasthook = NULL; 1675 *destp = node; 1676 return (0); 1677 } 1678 1679 /* 1680 * Now follow the sequence of hooks 1681 * 1682 * XXXGL: The path may demolish as we go the sequence, but if 1683 * we hold the topology mutex at critical places, then, I hope, 1684 * we would always have valid pointers in hand, although the 1685 * path behind us may no longer exist. 1686 */ 1687 for (;;) { 1688 hook_p hook; 1689 char *segment; 1690 1691 /* 1692 * Break out the next path segment. Replace the dot we just 1693 * found with a NUL; "path" points to the next segment (or the 1694 * NUL at the end). 1695 */ 1696 for (segment = path; *path != '\0'; path++) { 1697 if (*path == '.') { 1698 *path++ = '\0'; 1699 break; 1700 } 1701 } 1702 1703 /* We have a segment, so look for a hook by that name */ 1704 hook = ng_findhook(node, segment); 1705 1706 mtx_lock(&ng_topo_mtx); 1707 /* Can't get there from here... */ 1708 if (hook == NULL || NG_HOOK_PEER(hook) == NULL || 1709 NG_HOOK_NOT_VALID(hook) || 1710 NG_HOOK_NOT_VALID(NG_HOOK_PEER(hook))) { 1711 TRAP_ERROR(); 1712 NG_NODE_UNREF(node); 1713 mtx_unlock(&ng_topo_mtx); 1714 return (ENOENT); 1715 } 1716 1717 /* 1718 * Hop on over to the next node 1719 * XXX 1720 * Big race conditions here as hooks and nodes go away 1721 * *** Idea.. store an ng_ID_t in each hook and use that 1722 * instead of the direct hook in this crawl? 1723 */ 1724 oldnode = node; 1725 if ((node = NG_PEER_NODE(hook))) 1726 NG_NODE_REF(node); /* XXX RACE */ 1727 NG_NODE_UNREF(oldnode); /* XXX another race */ 1728 if (NG_NODE_NOT_VALID(node)) { 1729 NG_NODE_UNREF(node); /* XXX more races */ 1730 mtx_unlock(&ng_topo_mtx); 1731 TRAP_ERROR(); 1732 return (ENXIO); 1733 } 1734 1735 if (*path == '\0') { 1736 if (lasthook != NULL) { 1737 if (hook != NULL) { 1738 *lasthook = NG_HOOK_PEER(hook); 1739 NG_HOOK_REF(*lasthook); 1740 } else 1741 *lasthook = NULL; 1742 } 1743 mtx_unlock(&ng_topo_mtx); 1744 *destp = node; 1745 return (0); 1746 } 1747 mtx_unlock(&ng_topo_mtx); 1748 } 1749 } 1750 1751 /***************************************************************\ 1752 * Input queue handling. 1753 * All activities are submitted to the node via the input queue 1754 * which implements a multiple-reader/single-writer gate. 1755 * Items which cannot be handled immediately are queued. 1756 * 1757 * read-write queue locking inline functions * 1758 \***************************************************************/ 1759 1760 static __inline void ng_queue_rw(node_p node, item_p item, int rw); 1761 static __inline item_p ng_dequeue(node_p node, int *rw); 1762 static __inline item_p ng_acquire_read(node_p node, item_p item); 1763 static __inline item_p ng_acquire_write(node_p node, item_p item); 1764 static __inline void ng_leave_read(node_p node); 1765 static __inline void ng_leave_write(node_p node); 1766 1767 /* 1768 * Definition of the bits fields in the ng_queue flag word. 1769 * Defined here rather than in netgraph.h because no-one should fiddle 1770 * with them. 1771 * 1772 * The ordering here may be important! don't shuffle these. 1773 */ 1774 /*- 1775 Safety Barrier--------+ (adjustable to suit taste) (not used yet) 1776 | 1777 V 1778 +-------+-------+-------+-------+-------+-------+-------+-------+ 1779 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | 1780 | |A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |P|A| 1781 | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |O|W| 1782 +-------+-------+-------+-------+-------+-------+-------+-------+ 1783 \___________________________ ____________________________/ | | 1784 V | | 1785 [active reader count] | | 1786 | | 1787 Operation Pending -------------------------------+ | 1788 | 1789 Active Writer ---------------------------------------+ 1790 1791 Node queue has such semantics: 1792 - All flags modifications are atomic. 1793 - Reader count can be incremented only if there is no writer or pending flags. 1794 As soon as this can't be done with single operation, it is implemented with 1795 spin loop and atomic_cmpset(). 1796 - Writer flag can be set only if there is no any bits set. 1797 It is implemented with atomic_cmpset(). 1798 - Pending flag can be set any time, but to avoid collision on queue processing 1799 all queue fields are protected by the mutex. 1800 - Queue processing thread reads queue holding the mutex, but releases it while 1801 processing. When queue is empty pending flag is removed. 1802 */ 1803 1804 #define WRITER_ACTIVE 0x00000001 1805 #define OP_PENDING 0x00000002 1806 #define READER_INCREMENT 0x00000004 1807 #define READER_MASK 0xfffffffc /* Not valid if WRITER_ACTIVE is set */ 1808 #define SAFETY_BARRIER 0x00100000 /* 128K items queued should be enough */ 1809 1810 /* Defines of more elaborate states on the queue */ 1811 /* Mask of bits a new read cares about */ 1812 #define NGQ_RMASK (WRITER_ACTIVE|OP_PENDING) 1813 1814 /* Mask of bits a new write cares about */ 1815 #define NGQ_WMASK (NGQ_RMASK|READER_MASK) 1816 1817 /* Test to decide if there is something on the queue. */ 1818 #define QUEUE_ACTIVE(QP) ((QP)->q_flags & OP_PENDING) 1819 1820 /* How to decide what the next queued item is. */ 1821 #define HEAD_IS_READER(QP) NGI_QUEUED_READER(STAILQ_FIRST(&(QP)->queue)) 1822 #define HEAD_IS_WRITER(QP) NGI_QUEUED_WRITER(STAILQ_FIRST(&(QP)->queue)) /* notused */ 1823 1824 /* Read the status to decide if the next item on the queue can now run. */ 1825 #define QUEUED_READER_CAN_PROCEED(QP) \ 1826 (((QP)->q_flags & (NGQ_RMASK & ~OP_PENDING)) == 0) 1827 #define QUEUED_WRITER_CAN_PROCEED(QP) \ 1828 (((QP)->q_flags & (NGQ_WMASK & ~OP_PENDING)) == 0) 1829 1830 /* Is there a chance of getting ANY work off the queue? */ 1831 #define NEXT_QUEUED_ITEM_CAN_PROCEED(QP) \ 1832 ((HEAD_IS_READER(QP)) ? QUEUED_READER_CAN_PROCEED(QP) : \ 1833 QUEUED_WRITER_CAN_PROCEED(QP)) 1834 1835 #define NGQRW_R 0 1836 #define NGQRW_W 1 1837 1838 #define NGQ2_WORKQ 0x00000001 1839 1840 /* 1841 * Taking into account the current state of the queue and node, possibly take 1842 * the next entry off the queue and return it. Return NULL if there was 1843 * nothing we could return, either because there really was nothing there, or 1844 * because the node was in a state where it cannot yet process the next item 1845 * on the queue. 1846 */ 1847 static __inline item_p 1848 ng_dequeue(node_p node, int *rw) 1849 { 1850 item_p item; 1851 struct ng_queue *ngq = &node->nd_input_queue; 1852 1853 /* This MUST be called with the mutex held. */ 1854 mtx_assert(&ngq->q_mtx, MA_OWNED); 1855 1856 /* If there is nothing queued, then just return. */ 1857 if (!QUEUE_ACTIVE(ngq)) { 1858 CTR4(KTR_NET, "%20s: node [%x] (%p) queue empty; " 1859 "queue flags 0x%lx", __func__, 1860 node->nd_ID, node, ngq->q_flags); 1861 return (NULL); 1862 } 1863 1864 /* 1865 * From here, we can assume there is a head item. 1866 * We need to find out what it is and if it can be dequeued, given 1867 * the current state of the node. 1868 */ 1869 if (HEAD_IS_READER(ngq)) { 1870 while (1) { 1871 long t = ngq->q_flags; 1872 if (t & WRITER_ACTIVE) { 1873 /* There is writer, reader can't proceed. */ 1874 CTR4(KTR_NET, "%20s: node [%x] (%p) queued " 1875 "reader can't proceed; queue flags 0x%lx", 1876 __func__, node->nd_ID, node, t); 1877 return (NULL); 1878 } 1879 if (atomic_cmpset_acq_int(&ngq->q_flags, t, 1880 t + READER_INCREMENT)) 1881 break; 1882 cpu_spinwait(); 1883 } 1884 /* We have got reader lock for the node. */ 1885 *rw = NGQRW_R; 1886 } else if (atomic_cmpset_acq_int(&ngq->q_flags, OP_PENDING, 1887 OP_PENDING + WRITER_ACTIVE)) { 1888 /* We have got writer lock for the node. */ 1889 *rw = NGQRW_W; 1890 } else { 1891 /* There is somebody other, writer can't proceed. */ 1892 CTR4(KTR_NET, "%20s: node [%x] (%p) queued writer can't " 1893 "proceed; queue flags 0x%lx", __func__, node->nd_ID, node, 1894 ngq->q_flags); 1895 return (NULL); 1896 } 1897 1898 /* 1899 * Now we dequeue the request (whatever it may be) and correct the 1900 * pending flags and the next and last pointers. 1901 */ 1902 item = STAILQ_FIRST(&ngq->queue); 1903 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 1904 if (STAILQ_EMPTY(&ngq->queue)) 1905 atomic_clear_int(&ngq->q_flags, OP_PENDING); 1906 CTR6(KTR_NET, "%20s: node [%x] (%p) returning item %p as %s; queue " 1907 "flags 0x%lx", __func__, node->nd_ID, node, item, *rw ? "WRITER" : 1908 "READER", ngq->q_flags); 1909 return (item); 1910 } 1911 1912 /* 1913 * Queue a packet to be picked up later by someone else. 1914 * If the queue could be run now, add node to the queue handler's worklist. 1915 */ 1916 static __inline void 1917 ng_queue_rw(node_p node, item_p item, int rw) 1918 { 1919 struct ng_queue *ngq = &node->nd_input_queue; 1920 if (rw == NGQRW_W) 1921 NGI_SET_WRITER(item); 1922 else 1923 NGI_SET_READER(item); 1924 1925 NG_QUEUE_LOCK(ngq); 1926 /* Set OP_PENDING flag and enqueue the item. */ 1927 atomic_set_int(&ngq->q_flags, OP_PENDING); 1928 STAILQ_INSERT_TAIL(&ngq->queue, item, el_next); 1929 1930 CTR5(KTR_NET, "%20s: node [%x] (%p) queued item %p as %s", __func__, 1931 node->nd_ID, node, item, rw ? "WRITER" : "READER" ); 1932 1933 /* 1934 * We can take the worklist lock with the node locked 1935 * BUT NOT THE REVERSE! 1936 */ 1937 if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 1938 ng_worklist_add(node); 1939 NG_QUEUE_UNLOCK(ngq); 1940 } 1941 1942 /* Acquire reader lock on node. If node is busy, queue the packet. */ 1943 static __inline item_p 1944 ng_acquire_read(node_p node, item_p item) 1945 { 1946 KASSERT(node != &ng_deadnode, 1947 ("%s: working on deadnode", __func__)); 1948 1949 /* Reader needs node without writer and pending items. */ 1950 for (;;) { 1951 long t = node->nd_input_queue.q_flags; 1952 if (t & NGQ_RMASK) 1953 break; /* Node is not ready for reader. */ 1954 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, t, 1955 t + READER_INCREMENT)) { 1956 /* Successfully grabbed node */ 1957 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 1958 __func__, node->nd_ID, node, item); 1959 return (item); 1960 } 1961 cpu_spinwait(); 1962 }; 1963 1964 /* Queue the request for later. */ 1965 ng_queue_rw(node, item, NGQRW_R); 1966 1967 return (NULL); 1968 } 1969 1970 /* Acquire writer lock on node. If node is busy, queue the packet. */ 1971 static __inline item_p 1972 ng_acquire_write(node_p node, item_p item) 1973 { 1974 KASSERT(node != &ng_deadnode, 1975 ("%s: working on deadnode", __func__)); 1976 1977 /* Writer needs completely idle node. */ 1978 if (atomic_cmpset_acq_int(&node->nd_input_queue.q_flags, 0, 1979 WRITER_ACTIVE)) { 1980 /* Successfully grabbed node */ 1981 CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p", 1982 __func__, node->nd_ID, node, item); 1983 return (item); 1984 } 1985 1986 /* Queue the request for later. */ 1987 ng_queue_rw(node, item, NGQRW_W); 1988 1989 return (NULL); 1990 } 1991 1992 #if 0 1993 static __inline item_p 1994 ng_upgrade_write(node_p node, item_p item) 1995 { 1996 struct ng_queue *ngq = &node->nd_input_queue; 1997 KASSERT(node != &ng_deadnode, 1998 ("%s: working on deadnode", __func__)); 1999 2000 NGI_SET_WRITER(item); 2001 2002 NG_QUEUE_LOCK(ngq); 2003 2004 /* 2005 * There will never be no readers as we are there ourselves. 2006 * Set the WRITER_ACTIVE flags ASAP to block out fast track readers. 2007 * The caller we are running from will call ng_leave_read() 2008 * soon, so we must account for that. We must leave again with the 2009 * READER lock. If we find other readers, then 2010 * queue the request for later. However "later" may be rignt now 2011 * if there are no readers. We don't really care if there are queued 2012 * items as we will bypass them anyhow. 2013 */ 2014 atomic_add_int(&ngq->q_flags, WRITER_ACTIVE - READER_INCREMENT); 2015 if ((ngq->q_flags & (NGQ_WMASK & ~OP_PENDING)) == WRITER_ACTIVE) { 2016 NG_QUEUE_UNLOCK(ngq); 2017 2018 /* It's just us, act on the item. */ 2019 /* will NOT drop writer lock when done */ 2020 ng_apply_item(node, item, 0); 2021 2022 /* 2023 * Having acted on the item, atomically 2024 * downgrade back to READER and finish up. 2025 */ 2026 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2027 2028 /* Our caller will call ng_leave_read() */ 2029 return; 2030 } 2031 /* 2032 * It's not just us active, so queue us AT THE HEAD. 2033 * "Why?" I hear you ask. 2034 * Put us at the head of the queue as we've already been 2035 * through it once. If there is nothing else waiting, 2036 * set the correct flags. 2037 */ 2038 if (STAILQ_EMPTY(&ngq->queue)) { 2039 /* We've gone from, 0 to 1 item in the queue */ 2040 atomic_set_int(&ngq->q_flags, OP_PENDING); 2041 2042 CTR3(KTR_NET, "%20s: node [%x] (%p) set OP_PENDING", __func__, 2043 node->nd_ID, node); 2044 }; 2045 STAILQ_INSERT_HEAD(&ngq->queue, item, el_next); 2046 CTR4(KTR_NET, "%20s: node [%x] (%p) requeued item %p as WRITER", 2047 __func__, node->nd_ID, node, item ); 2048 2049 /* Reverse what we did above. That downgrades us back to reader */ 2050 atomic_add_int(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); 2051 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2052 ng_worklist_add(node); 2053 NG_QUEUE_UNLOCK(ngq); 2054 2055 return; 2056 } 2057 #endif 2058 2059 /* Release reader lock. */ 2060 static __inline void 2061 ng_leave_read(node_p node) 2062 { 2063 atomic_subtract_rel_int(&node->nd_input_queue.q_flags, READER_INCREMENT); 2064 } 2065 2066 /* Release writer lock. */ 2067 static __inline void 2068 ng_leave_write(node_p node) 2069 { 2070 atomic_clear_rel_int(&node->nd_input_queue.q_flags, WRITER_ACTIVE); 2071 } 2072 2073 /* Purge node queue. Called on node shutdown. */ 2074 static void 2075 ng_flush_input_queue(node_p node) 2076 { 2077 struct ng_queue *ngq = &node->nd_input_queue; 2078 item_p item; 2079 2080 NG_QUEUE_LOCK(ngq); 2081 while ((item = STAILQ_FIRST(&ngq->queue)) != NULL) { 2082 STAILQ_REMOVE_HEAD(&ngq->queue, el_next); 2083 if (STAILQ_EMPTY(&ngq->queue)) 2084 atomic_clear_int(&ngq->q_flags, OP_PENDING); 2085 NG_QUEUE_UNLOCK(ngq); 2086 2087 /* If the item is supplying a callback, call it with an error */ 2088 if (item->apply != NULL) { 2089 if (item->depth == 1) 2090 item->apply->error = ENOENT; 2091 if (refcount_release(&item->apply->refs)) { 2092 (*item->apply->apply)(item->apply->context, 2093 item->apply->error); 2094 } 2095 } 2096 NG_FREE_ITEM(item); 2097 NG_QUEUE_LOCK(ngq); 2098 } 2099 NG_QUEUE_UNLOCK(ngq); 2100 } 2101 2102 /*********************************************************************** 2103 * Externally visible method for sending or queueing messages or data. 2104 ***********************************************************************/ 2105 2106 /* 2107 * The module code should have filled out the item correctly by this stage: 2108 * Common: 2109 * reference to destination node. 2110 * Reference to destination rcv hook if relevant. 2111 * apply pointer must be or NULL or reference valid struct ng_apply_info. 2112 * Data: 2113 * pointer to mbuf 2114 * Control_Message: 2115 * pointer to msg. 2116 * ID of original sender node. (return address) 2117 * Function: 2118 * Function pointer 2119 * void * argument 2120 * integer argument 2121 * 2122 * The nodes have several routines and macros to help with this task: 2123 */ 2124 2125 int 2126 ng_snd_item(item_p item, int flags) 2127 { 2128 hook_p hook; 2129 node_p node; 2130 int queue, rw; 2131 struct ng_queue *ngq; 2132 int error = 0; 2133 2134 /* We are sending item, so it must be present! */ 2135 KASSERT(item != NULL, ("ng_snd_item: item is NULL")); 2136 2137 #ifdef NETGRAPH_DEBUG 2138 _ngi_check(item, __FILE__, __LINE__); 2139 #endif 2140 2141 /* Item was sent once more, postpone apply() call. */ 2142 if (item->apply) 2143 refcount_acquire(&item->apply->refs); 2144 2145 node = NGI_NODE(item); 2146 /* Node is never optional. */ 2147 KASSERT(node != NULL, ("ng_snd_item: node is NULL")); 2148 2149 hook = NGI_HOOK(item); 2150 /* Valid hook and mbuf are mandatory for data. */ 2151 if ((item->el_flags & NGQF_TYPE) == NGQF_DATA) { 2152 KASSERT(hook != NULL, ("ng_snd_item: hook for data is NULL")); 2153 if (NGI_M(item) == NULL) 2154 ERROUT(EINVAL); 2155 CHECK_DATA_MBUF(NGI_M(item)); 2156 } 2157 2158 /* 2159 * If the item or the node specifies single threading, force 2160 * writer semantics. Similarly, the node may say one hook always 2161 * produces writers. These are overrides. 2162 */ 2163 if (((item->el_flags & NGQF_RW) == NGQF_WRITER) || 2164 (node->nd_flags & NGF_FORCE_WRITER) || 2165 (hook && (hook->hk_flags & HK_FORCE_WRITER))) { 2166 rw = NGQRW_W; 2167 } else { 2168 rw = NGQRW_R; 2169 } 2170 2171 /* 2172 * If sender or receiver requests queued delivery, or call graph 2173 * loops back from outbound to inbound path, or stack usage 2174 * level is dangerous - enqueue message. 2175 */ 2176 if ((flags & NG_QUEUE) || (hook && (hook->hk_flags & HK_QUEUE))) { 2177 queue = 1; 2178 } else if (hook && (hook->hk_flags & HK_TO_INBOUND) && 2179 curthread->td_ng_outbound) { 2180 queue = 1; 2181 } else { 2182 queue = 0; 2183 #ifdef GET_STACK_USAGE 2184 /* 2185 * Most of netgraph nodes have small stack consumption and 2186 * for them 25% of free stack space is more than enough. 2187 * Nodes/hooks with higher stack usage should be marked as 2188 * HI_STACK. For them 50% of stack will be guaranteed then. 2189 * XXX: Values 25% and 50% are completely empirical. 2190 */ 2191 size_t st, su, sl; 2192 GET_STACK_USAGE(st, su); 2193 sl = st - su; 2194 if ((sl * 4 < st) || ((sl * 2 < st) && 2195 ((node->nd_flags & NGF_HI_STACK) || (hook && 2196 (hook->hk_flags & HK_HI_STACK))))) 2197 queue = 1; 2198 #endif 2199 } 2200 2201 if (queue) { 2202 item->depth = 1; 2203 /* Put it on the queue for that node*/ 2204 ng_queue_rw(node, item, rw); 2205 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2206 } 2207 2208 /* 2209 * We already decided how we will be queueud or treated. 2210 * Try get the appropriate operating permission. 2211 */ 2212 if (rw == NGQRW_R) 2213 item = ng_acquire_read(node, item); 2214 else 2215 item = ng_acquire_write(node, item); 2216 2217 /* Item was queued while trying to get permission. */ 2218 if (item == NULL) 2219 return ((flags & NG_PROGRESS) ? EINPROGRESS : 0); 2220 2221 NGI_GET_NODE(item, node); /* zaps stored node */ 2222 2223 item->depth++; 2224 error = ng_apply_item(node, item, rw); /* drops r/w lock when done */ 2225 2226 /* If something is waiting on queue and ready, schedule it. */ 2227 ngq = &node->nd_input_queue; 2228 if (QUEUE_ACTIVE(ngq)) { 2229 NG_QUEUE_LOCK(ngq); 2230 if (QUEUE_ACTIVE(ngq) && NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) 2231 ng_worklist_add(node); 2232 NG_QUEUE_UNLOCK(ngq); 2233 } 2234 2235 /* 2236 * Node may go away as soon as we remove the reference. 2237 * Whatever we do, DO NOT access the node again! 2238 */ 2239 NG_NODE_UNREF(node); 2240 2241 return (error); 2242 2243 done: 2244 /* If was not sent, apply callback here. */ 2245 if (item->apply != NULL) { 2246 if (item->depth == 0 && error != 0) 2247 item->apply->error = error; 2248 if (refcount_release(&item->apply->refs)) { 2249 (*item->apply->apply)(item->apply->context, 2250 item->apply->error); 2251 } 2252 } 2253 2254 NG_FREE_ITEM(item); 2255 return (error); 2256 } 2257 2258 /* 2259 * We have an item that was possibly queued somewhere. 2260 * It should contain all the information needed 2261 * to run it on the appropriate node/hook. 2262 * If there is apply pointer and we own the last reference, call apply(). 2263 */ 2264 static int 2265 ng_apply_item(node_p node, item_p item, int rw) 2266 { 2267 hook_p hook; 2268 ng_rcvdata_t *rcvdata; 2269 ng_rcvmsg_t *rcvmsg; 2270 struct ng_apply_info *apply; 2271 int error = 0, depth; 2272 2273 /* Node and item are never optional. */ 2274 KASSERT(node != NULL, ("ng_apply_item: node is NULL")); 2275 KASSERT(item != NULL, ("ng_apply_item: item is NULL")); 2276 2277 NGI_GET_HOOK(item, hook); /* clears stored hook */ 2278 #ifdef NETGRAPH_DEBUG 2279 _ngi_check(item, __FILE__, __LINE__); 2280 #endif 2281 2282 apply = item->apply; 2283 depth = item->depth; 2284 2285 switch (item->el_flags & NGQF_TYPE) { 2286 case NGQF_DATA: 2287 /* 2288 * Check things are still ok as when we were queued. 2289 */ 2290 KASSERT(hook != NULL, ("ng_apply_item: hook for data is NULL")); 2291 if (NG_HOOK_NOT_VALID(hook) || 2292 NG_NODE_NOT_VALID(node)) { 2293 error = EIO; 2294 NG_FREE_ITEM(item); 2295 break; 2296 } 2297 /* 2298 * If no receive method, just silently drop it. 2299 * Give preference to the hook over-ride method. 2300 */ 2301 if ((!(rcvdata = hook->hk_rcvdata)) && 2302 (!(rcvdata = NG_HOOK_NODE(hook)->nd_type->rcvdata))) { 2303 error = 0; 2304 NG_FREE_ITEM(item); 2305 break; 2306 } 2307 error = (*rcvdata)(hook, item); 2308 break; 2309 case NGQF_MESG: 2310 if (hook && NG_HOOK_NOT_VALID(hook)) { 2311 /* 2312 * The hook has been zapped then we can't use it. 2313 * Immediately drop its reference. 2314 * The message may not need it. 2315 */ 2316 NG_HOOK_UNREF(hook); 2317 hook = NULL; 2318 } 2319 /* 2320 * Similarly, if the node is a zombie there is 2321 * nothing we can do with it, drop everything. 2322 */ 2323 if (NG_NODE_NOT_VALID(node)) { 2324 TRAP_ERROR(); 2325 error = EINVAL; 2326 NG_FREE_ITEM(item); 2327 break; 2328 } 2329 /* 2330 * Call the appropriate message handler for the object. 2331 * It is up to the message handler to free the message. 2332 * If it's a generic message, handle it generically, 2333 * otherwise call the type's message handler (if it exists). 2334 * XXX (race). Remember that a queued message may 2335 * reference a node or hook that has just been 2336 * invalidated. It will exist as the queue code 2337 * is holding a reference, but.. 2338 */ 2339 if ((NGI_MSG(item)->header.typecookie == NGM_GENERIC_COOKIE) && 2340 ((NGI_MSG(item)->header.flags & NGF_RESP) == 0)) { 2341 error = ng_generic_msg(node, item, hook); 2342 break; 2343 } 2344 if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg))) && 2345 (!(rcvmsg = node->nd_type->rcvmsg))) { 2346 TRAP_ERROR(); 2347 error = 0; 2348 NG_FREE_ITEM(item); 2349 break; 2350 } 2351 error = (*rcvmsg)(node, item, hook); 2352 break; 2353 case NGQF_FN: 2354 case NGQF_FN2: 2355 /* 2356 * In the case of the shutdown message we allow it to hit 2357 * even if the node is invalid. 2358 */ 2359 if (NG_NODE_NOT_VALID(node) && 2360 NGI_FN(item) != &ng_rmnode) { 2361 TRAP_ERROR(); 2362 error = EINVAL; 2363 NG_FREE_ITEM(item); 2364 break; 2365 } 2366 /* Same is about some internal functions and invalid hook. */ 2367 if (hook && NG_HOOK_NOT_VALID(hook) && 2368 NGI_FN2(item) != &ng_con_part2 && 2369 NGI_FN2(item) != &ng_con_part3 && 2370 NGI_FN(item) != &ng_rmhook_part2) { 2371 TRAP_ERROR(); 2372 error = EINVAL; 2373 NG_FREE_ITEM(item); 2374 break; 2375 } 2376 2377 if ((item->el_flags & NGQF_TYPE) == NGQF_FN) { 2378 (*NGI_FN(item))(node, hook, NGI_ARG1(item), 2379 NGI_ARG2(item)); 2380 NG_FREE_ITEM(item); 2381 } else /* it is NGQF_FN2 */ 2382 error = (*NGI_FN2(item))(node, item, hook); 2383 break; 2384 } 2385 /* 2386 * We held references on some of the resources 2387 * that we took from the item. Now that we have 2388 * finished doing everything, drop those references. 2389 */ 2390 if (hook) 2391 NG_HOOK_UNREF(hook); 2392 2393 if (rw == NGQRW_R) 2394 ng_leave_read(node); 2395 else 2396 ng_leave_write(node); 2397 2398 /* Apply callback. */ 2399 if (apply != NULL) { 2400 if (depth == 1 && error != 0) 2401 apply->error = error; 2402 if (refcount_release(&apply->refs)) 2403 (*apply->apply)(apply->context, apply->error); 2404 } 2405 2406 return (error); 2407 } 2408 2409 /*********************************************************************** 2410 * Implement the 'generic' control messages 2411 ***********************************************************************/ 2412 static int 2413 ng_generic_msg(node_p here, item_p item, hook_p lasthook) 2414 { 2415 int error = 0; 2416 struct ng_mesg *msg; 2417 struct ng_mesg *resp = NULL; 2418 2419 NGI_GET_MSG(item, msg); 2420 if (msg->header.typecookie != NGM_GENERIC_COOKIE) { 2421 TRAP_ERROR(); 2422 error = EINVAL; 2423 goto out; 2424 } 2425 switch (msg->header.cmd) { 2426 case NGM_SHUTDOWN: 2427 ng_rmnode(here, NULL, NULL, 0); 2428 break; 2429 case NGM_MKPEER: 2430 { 2431 struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data; 2432 2433 if (msg->header.arglen != sizeof(*mkp)) { 2434 TRAP_ERROR(); 2435 error = EINVAL; 2436 break; 2437 } 2438 mkp->type[sizeof(mkp->type) - 1] = '\0'; 2439 mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0'; 2440 mkp->peerhook[sizeof(mkp->peerhook) - 1] = '\0'; 2441 error = ng_mkpeer(here, mkp->ourhook, mkp->peerhook, mkp->type); 2442 break; 2443 } 2444 case NGM_CONNECT: 2445 { 2446 struct ngm_connect *const con = 2447 (struct ngm_connect *) msg->data; 2448 node_p node2; 2449 2450 if (msg->header.arglen != sizeof(*con)) { 2451 TRAP_ERROR(); 2452 error = EINVAL; 2453 break; 2454 } 2455 con->path[sizeof(con->path) - 1] = '\0'; 2456 con->ourhook[sizeof(con->ourhook) - 1] = '\0'; 2457 con->peerhook[sizeof(con->peerhook) - 1] = '\0'; 2458 /* Don't forget we get a reference.. */ 2459 error = ng_path2noderef(here, con->path, &node2, NULL); 2460 if (error) 2461 break; 2462 error = ng_con_nodes(item, here, con->ourhook, 2463 node2, con->peerhook); 2464 NG_NODE_UNREF(node2); 2465 break; 2466 } 2467 case NGM_NAME: 2468 { 2469 struct ngm_name *const nam = (struct ngm_name *) msg->data; 2470 2471 if (msg->header.arglen != sizeof(*nam)) { 2472 TRAP_ERROR(); 2473 error = EINVAL; 2474 break; 2475 } 2476 nam->name[sizeof(nam->name) - 1] = '\0'; 2477 error = ng_name_node(here, nam->name); 2478 break; 2479 } 2480 case NGM_RMHOOK: 2481 { 2482 struct ngm_rmhook *const rmh = (struct ngm_rmhook *) msg->data; 2483 hook_p hook; 2484 2485 if (msg->header.arglen != sizeof(*rmh)) { 2486 TRAP_ERROR(); 2487 error = EINVAL; 2488 break; 2489 } 2490 rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0'; 2491 if ((hook = ng_findhook(here, rmh->ourhook)) != NULL) 2492 ng_destroy_hook(hook); 2493 break; 2494 } 2495 case NGM_NODEINFO: 2496 { 2497 struct nodeinfo *ni; 2498 2499 NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT); 2500 if (resp == NULL) { 2501 error = ENOMEM; 2502 break; 2503 } 2504 2505 /* Fill in node info */ 2506 ni = (struct nodeinfo *) resp->data; 2507 if (NG_NODE_HAS_NAME(here)) 2508 strcpy(ni->name, NG_NODE_NAME(here)); 2509 strcpy(ni->type, here->nd_type->name); 2510 ni->id = ng_node2ID(here); 2511 ni->hooks = here->nd_numhooks; 2512 break; 2513 } 2514 case NGM_LISTHOOKS: 2515 { 2516 const int nhooks = here->nd_numhooks; 2517 struct hooklist *hl; 2518 struct nodeinfo *ni; 2519 hook_p hook; 2520 2521 /* Get response struct */ 2522 NG_MKRESPONSE(resp, msg, sizeof(*hl) + 2523 (nhooks * sizeof(struct linkinfo)), M_NOWAIT); 2524 if (resp == NULL) { 2525 error = ENOMEM; 2526 break; 2527 } 2528 hl = (struct hooklist *) resp->data; 2529 ni = &hl->nodeinfo; 2530 2531 /* Fill in node info */ 2532 if (NG_NODE_HAS_NAME(here)) 2533 strcpy(ni->name, NG_NODE_NAME(here)); 2534 strcpy(ni->type, here->nd_type->name); 2535 ni->id = ng_node2ID(here); 2536 2537 /* Cycle through the linked list of hooks */ 2538 ni->hooks = 0; 2539 LIST_FOREACH(hook, &here->nd_hooks, hk_hooks) { 2540 struct linkinfo *const link = &hl->link[ni->hooks]; 2541 2542 if (ni->hooks >= nhooks) { 2543 log(LOG_ERR, "%s: number of %s changed\n", 2544 __func__, "hooks"); 2545 break; 2546 } 2547 if (NG_HOOK_NOT_VALID(hook)) 2548 continue; 2549 strcpy(link->ourhook, NG_HOOK_NAME(hook)); 2550 strcpy(link->peerhook, NG_PEER_HOOK_NAME(hook)); 2551 if (NG_PEER_NODE_NAME(hook)[0] != '\0') 2552 strcpy(link->nodeinfo.name, 2553 NG_PEER_NODE_NAME(hook)); 2554 strcpy(link->nodeinfo.type, 2555 NG_PEER_NODE(hook)->nd_type->name); 2556 link->nodeinfo.id = ng_node2ID(NG_PEER_NODE(hook)); 2557 link->nodeinfo.hooks = NG_PEER_NODE(hook)->nd_numhooks; 2558 ni->hooks++; 2559 } 2560 break; 2561 } 2562 2563 case NGM_LISTNAMES: 2564 case NGM_LISTNODES: 2565 { 2566 const int unnamed = (msg->header.cmd == NGM_LISTNODES); 2567 struct namelist *nl; 2568 node_p node; 2569 int num = 0, i; 2570 2571 mtx_lock(&ng_namehash_mtx); 2572 /* Count number of nodes */ 2573 for (i = 0; i < NG_NAME_HASH_SIZE; i++) { 2574 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2575 if (NG_NODE_IS_VALID(node) && 2576 (unnamed || NG_NODE_HAS_NAME(node))) { 2577 num++; 2578 } 2579 } 2580 } 2581 mtx_unlock(&ng_namehash_mtx); 2582 2583 /* Get response struct */ 2584 NG_MKRESPONSE(resp, msg, sizeof(*nl) + 2585 (num * sizeof(struct nodeinfo)), M_NOWAIT); 2586 if (resp == NULL) { 2587 error = ENOMEM; 2588 break; 2589 } 2590 nl = (struct namelist *) resp->data; 2591 2592 /* Cycle through the linked list of nodes */ 2593 nl->numnames = 0; 2594 mtx_lock(&ng_namehash_mtx); 2595 for (i = 0; i < NG_NAME_HASH_SIZE; i++) { 2596 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 2597 struct nodeinfo *const np = 2598 &nl->nodeinfo[nl->numnames]; 2599 2600 if (NG_NODE_NOT_VALID(node)) 2601 continue; 2602 if (!unnamed && (! NG_NODE_HAS_NAME(node))) 2603 continue; 2604 if (nl->numnames >= num) { 2605 log(LOG_ERR, "%s: number of nodes changed\n", 2606 __func__); 2607 break; 2608 } 2609 if (NG_NODE_HAS_NAME(node)) 2610 strcpy(np->name, NG_NODE_NAME(node)); 2611 strcpy(np->type, node->nd_type->name); 2612 np->id = ng_node2ID(node); 2613 np->hooks = node->nd_numhooks; 2614 nl->numnames++; 2615 } 2616 } 2617 mtx_unlock(&ng_namehash_mtx); 2618 break; 2619 } 2620 2621 case NGM_LISTTYPES: 2622 { 2623 struct typelist *tl; 2624 struct ng_type *type; 2625 int num = 0; 2626 2627 mtx_lock(&ng_typelist_mtx); 2628 /* Count number of types */ 2629 LIST_FOREACH(type, &ng_typelist, types) { 2630 num++; 2631 } 2632 mtx_unlock(&ng_typelist_mtx); 2633 2634 /* Get response struct */ 2635 NG_MKRESPONSE(resp, msg, sizeof(*tl) + 2636 (num * sizeof(struct typeinfo)), M_NOWAIT); 2637 if (resp == NULL) { 2638 error = ENOMEM; 2639 break; 2640 } 2641 tl = (struct typelist *) resp->data; 2642 2643 /* Cycle through the linked list of types */ 2644 tl->numtypes = 0; 2645 mtx_lock(&ng_typelist_mtx); 2646 LIST_FOREACH(type, &ng_typelist, types) { 2647 struct typeinfo *const tp = &tl->typeinfo[tl->numtypes]; 2648 2649 if (tl->numtypes >= num) { 2650 log(LOG_ERR, "%s: number of %s changed\n", 2651 __func__, "types"); 2652 break; 2653 } 2654 strcpy(tp->type_name, type->name); 2655 tp->numnodes = type->refs - 1; /* don't count list */ 2656 tl->numtypes++; 2657 } 2658 mtx_unlock(&ng_typelist_mtx); 2659 break; 2660 } 2661 2662 case NGM_BINARY2ASCII: 2663 { 2664 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2665 const struct ng_parse_type *argstype; 2666 const struct ng_cmdlist *c; 2667 struct ng_mesg *binary, *ascii; 2668 2669 /* Data area must contain a valid netgraph message */ 2670 binary = (struct ng_mesg *)msg->data; 2671 if (msg->header.arglen < sizeof(struct ng_mesg) || 2672 (msg->header.arglen - sizeof(struct ng_mesg) < 2673 binary->header.arglen)) { 2674 TRAP_ERROR(); 2675 error = EINVAL; 2676 break; 2677 } 2678 2679 /* Get a response message with lots of room */ 2680 NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT); 2681 if (resp == NULL) { 2682 error = ENOMEM; 2683 break; 2684 } 2685 ascii = (struct ng_mesg *)resp->data; 2686 2687 /* Copy binary message header to response message payload */ 2688 bcopy(binary, ascii, sizeof(*binary)); 2689 2690 /* Find command by matching typecookie and command number */ 2691 for (c = here->nd_type->cmdlist; c != NULL && c->name != NULL; 2692 c++) { 2693 if (binary->header.typecookie == c->cookie && 2694 binary->header.cmd == c->cmd) 2695 break; 2696 } 2697 if (c == NULL || c->name == NULL) { 2698 for (c = ng_generic_cmds; c->name != NULL; c++) { 2699 if (binary->header.typecookie == c->cookie && 2700 binary->header.cmd == c->cmd) 2701 break; 2702 } 2703 if (c->name == NULL) { 2704 NG_FREE_MSG(resp); 2705 error = ENOSYS; 2706 break; 2707 } 2708 } 2709 2710 /* Convert command name to ASCII */ 2711 snprintf(ascii->header.cmdstr, sizeof(ascii->header.cmdstr), 2712 "%s", c->name); 2713 2714 /* Convert command arguments to ASCII */ 2715 argstype = (binary->header.flags & NGF_RESP) ? 2716 c->respType : c->mesgType; 2717 if (argstype == NULL) { 2718 *ascii->data = '\0'; 2719 } else { 2720 if ((error = ng_unparse(argstype, 2721 (u_char *)binary->data, 2722 ascii->data, bufSize)) != 0) { 2723 NG_FREE_MSG(resp); 2724 break; 2725 } 2726 } 2727 2728 /* Return the result as struct ng_mesg plus ASCII string */ 2729 bufSize = strlen(ascii->data) + 1; 2730 ascii->header.arglen = bufSize; 2731 resp->header.arglen = sizeof(*ascii) + bufSize; 2732 break; 2733 } 2734 2735 case NGM_ASCII2BINARY: 2736 { 2737 int bufSize = 20 * 1024; /* XXX hard coded constant */ 2738 const struct ng_cmdlist *c; 2739 const struct ng_parse_type *argstype; 2740 struct ng_mesg *ascii, *binary; 2741 int off = 0; 2742 2743 /* Data area must contain at least a struct ng_mesg + '\0' */ 2744 ascii = (struct ng_mesg *)msg->data; 2745 if ((msg->header.arglen < sizeof(*ascii) + 1) || 2746 (ascii->header.arglen < 1) || 2747 (msg->header.arglen < sizeof(*ascii) + 2748 ascii->header.arglen)) { 2749 TRAP_ERROR(); 2750 error = EINVAL; 2751 break; 2752 } 2753 ascii->data[ascii->header.arglen - 1] = '\0'; 2754 2755 /* Get a response message with lots of room */ 2756 NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT); 2757 if (resp == NULL) { 2758 error = ENOMEM; 2759 break; 2760 } 2761 binary = (struct ng_mesg *)resp->data; 2762 2763 /* Copy ASCII message header to response message payload */ 2764 bcopy(ascii, binary, sizeof(*ascii)); 2765 2766 /* Find command by matching ASCII command string */ 2767 for (c = here->nd_type->cmdlist; 2768 c != NULL && c->name != NULL; c++) { 2769 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2770 break; 2771 } 2772 if (c == NULL || c->name == NULL) { 2773 for (c = ng_generic_cmds; c->name != NULL; c++) { 2774 if (strcmp(ascii->header.cmdstr, c->name) == 0) 2775 break; 2776 } 2777 if (c->name == NULL) { 2778 NG_FREE_MSG(resp); 2779 error = ENOSYS; 2780 break; 2781 } 2782 } 2783 2784 /* Convert command name to binary */ 2785 binary->header.cmd = c->cmd; 2786 binary->header.typecookie = c->cookie; 2787 2788 /* Convert command arguments to binary */ 2789 argstype = (binary->header.flags & NGF_RESP) ? 2790 c->respType : c->mesgType; 2791 if (argstype == NULL) { 2792 bufSize = 0; 2793 } else { 2794 if ((error = ng_parse(argstype, ascii->data, &off, 2795 (u_char *)binary->data, &bufSize)) != 0) { 2796 NG_FREE_MSG(resp); 2797 break; 2798 } 2799 } 2800 2801 /* Return the result */ 2802 binary->header.arglen = bufSize; 2803 resp->header.arglen = sizeof(*binary) + bufSize; 2804 break; 2805 } 2806 2807 case NGM_TEXT_CONFIG: 2808 case NGM_TEXT_STATUS: 2809 /* 2810 * This one is tricky as it passes the command down to the 2811 * actual node, even though it is a generic type command. 2812 * This means we must assume that the item/msg is already freed 2813 * when control passes back to us. 2814 */ 2815 if (here->nd_type->rcvmsg != NULL) { 2816 NGI_MSG(item) = msg; /* put it back as we found it */ 2817 return((*here->nd_type->rcvmsg)(here, item, lasthook)); 2818 } 2819 /* Fall through if rcvmsg not supported */ 2820 default: 2821 TRAP_ERROR(); 2822 error = EINVAL; 2823 } 2824 /* 2825 * Sometimes a generic message may be statically allocated 2826 * to avoid problems with allocating when in tight memory situations. 2827 * Don't free it if it is so. 2828 * I break them appart here, because erros may cause a free if the item 2829 * in which case we'd be doing it twice. 2830 * they are kept together above, to simplify freeing. 2831 */ 2832 out: 2833 NG_RESPOND_MSG(error, here, item, resp); 2834 NG_FREE_MSG(msg); 2835 return (error); 2836 } 2837 2838 /************************************************************************ 2839 Queue element get/free routines 2840 ************************************************************************/ 2841 2842 uma_zone_t ng_qzone; 2843 uma_zone_t ng_qdzone; 2844 static int numthreads = 0; /* number of queue threads */ 2845 static int maxalloc = 4096;/* limit the damage of a leak */ 2846 static int maxdata = 512; /* limit the damage of a DoS */ 2847 2848 TUNABLE_INT("net.graph.threads", &numthreads); 2849 SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads, 2850 0, "Number of queue processing threads"); 2851 TUNABLE_INT("net.graph.maxalloc", &maxalloc); 2852 SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc, 2853 0, "Maximum number of non-data queue items to allocate"); 2854 TUNABLE_INT("net.graph.maxdata", &maxdata); 2855 SYSCTL_INT(_net_graph, OID_AUTO, maxdata, CTLFLAG_RDTUN, &maxdata, 2856 0, "Maximum number of data queue items to allocate"); 2857 2858 #ifdef NETGRAPH_DEBUG 2859 static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist); 2860 static int allocated; /* number of items malloc'd */ 2861 #endif 2862 2863 /* 2864 * Get a queue entry. 2865 * This is usually called when a packet first enters netgraph. 2866 * By definition, this is usually from an interrupt, or from a user. 2867 * Users are not so important, but try be quick for the times that it's 2868 * an interrupt. 2869 */ 2870 static __inline item_p 2871 ng_alloc_item(int type, int flags) 2872 { 2873 item_p item; 2874 2875 KASSERT(((type & ~NGQF_TYPE) == 0), 2876 ("%s: incorrect item type: %d", __func__, type)); 2877 2878 item = uma_zalloc((type == NGQF_DATA) ? ng_qdzone : ng_qzone, 2879 ((flags & NG_WAITOK) ? M_WAITOK : M_NOWAIT) | M_ZERO); 2880 2881 if (item) { 2882 item->el_flags = type; 2883 #ifdef NETGRAPH_DEBUG 2884 mtx_lock(&ngq_mtx); 2885 TAILQ_INSERT_TAIL(&ng_itemlist, item, all); 2886 allocated++; 2887 mtx_unlock(&ngq_mtx); 2888 #endif 2889 } 2890 2891 return (item); 2892 } 2893 2894 /* 2895 * Release a queue entry 2896 */ 2897 void 2898 ng_free_item(item_p item) 2899 { 2900 /* 2901 * The item may hold resources on it's own. We need to free 2902 * these before we can free the item. What they are depends upon 2903 * what kind of item it is. it is important that nodes zero 2904 * out pointers to resources that they remove from the item 2905 * or we release them again here. 2906 */ 2907 switch (item->el_flags & NGQF_TYPE) { 2908 case NGQF_DATA: 2909 /* If we have an mbuf still attached.. */ 2910 NG_FREE_M(_NGI_M(item)); 2911 break; 2912 case NGQF_MESG: 2913 _NGI_RETADDR(item) = 0; 2914 NG_FREE_MSG(_NGI_MSG(item)); 2915 break; 2916 case NGQF_FN: 2917 case NGQF_FN2: 2918 /* nothing to free really, */ 2919 _NGI_FN(item) = NULL; 2920 _NGI_ARG1(item) = NULL; 2921 _NGI_ARG2(item) = 0; 2922 break; 2923 } 2924 /* If we still have a node or hook referenced... */ 2925 _NGI_CLR_NODE(item); 2926 _NGI_CLR_HOOK(item); 2927 2928 #ifdef NETGRAPH_DEBUG 2929 mtx_lock(&ngq_mtx); 2930 TAILQ_REMOVE(&ng_itemlist, item, all); 2931 allocated--; 2932 mtx_unlock(&ngq_mtx); 2933 #endif 2934 uma_zfree(((item->el_flags & NGQF_TYPE) == NGQF_DATA) ? 2935 ng_qdzone : ng_qzone, item); 2936 } 2937 2938 /* 2939 * Change type of the queue entry. 2940 * Possibly reallocates it from another UMA zone. 2941 */ 2942 static __inline item_p 2943 ng_realloc_item(item_p pitem, int type, int flags) 2944 { 2945 item_p item; 2946 int from, to; 2947 2948 KASSERT((pitem != NULL), ("%s: can't reallocate NULL", __func__)); 2949 KASSERT(((type & ~NGQF_TYPE) == 0), 2950 ("%s: incorrect item type: %d", __func__, type)); 2951 2952 from = ((pitem->el_flags & NGQF_TYPE) == NGQF_DATA); 2953 to = (type == NGQF_DATA); 2954 if (from != to) { 2955 /* If reallocation is required do it and copy item. */ 2956 if ((item = ng_alloc_item(type, flags)) == NULL) { 2957 ng_free_item(pitem); 2958 return (NULL); 2959 } 2960 *item = *pitem; 2961 ng_free_item(pitem); 2962 } else 2963 item = pitem; 2964 item->el_flags = (item->el_flags & ~NGQF_TYPE) | type; 2965 2966 return (item); 2967 } 2968 2969 /************************************************************************ 2970 Module routines 2971 ************************************************************************/ 2972 2973 /* 2974 * Handle the loading/unloading of a netgraph node type module 2975 */ 2976 int 2977 ng_mod_event(module_t mod, int event, void *data) 2978 { 2979 struct ng_type *const type = data; 2980 int error = 0; 2981 2982 switch (event) { 2983 case MOD_LOAD: 2984 2985 /* Register new netgraph node type */ 2986 if ((error = ng_newtype(type)) != 0) 2987 break; 2988 2989 /* Call type specific code */ 2990 if (type->mod_event != NULL) 2991 if ((error = (*type->mod_event)(mod, event, data))) { 2992 mtx_lock(&ng_typelist_mtx); 2993 type->refs--; /* undo it */ 2994 LIST_REMOVE(type, types); 2995 mtx_unlock(&ng_typelist_mtx); 2996 } 2997 break; 2998 2999 case MOD_UNLOAD: 3000 if (type->refs > 1) { /* make sure no nodes exist! */ 3001 error = EBUSY; 3002 } else { 3003 if (type->refs == 0) /* failed load, nothing to undo */ 3004 break; 3005 if (type->mod_event != NULL) { /* check with type */ 3006 error = (*type->mod_event)(mod, event, data); 3007 if (error != 0) /* type refuses.. */ 3008 break; 3009 } 3010 mtx_lock(&ng_typelist_mtx); 3011 LIST_REMOVE(type, types); 3012 mtx_unlock(&ng_typelist_mtx); 3013 } 3014 break; 3015 3016 default: 3017 if (type->mod_event != NULL) 3018 error = (*type->mod_event)(mod, event, data); 3019 else 3020 error = EOPNOTSUPP; /* XXX ? */ 3021 break; 3022 } 3023 return (error); 3024 } 3025 3026 #ifdef VIMAGE 3027 static void 3028 vnet_netgraph_uninit(const void *unused __unused) 3029 { 3030 node_p node = NULL, last_killed = NULL; 3031 int i; 3032 3033 do { 3034 /* Find a node to kill */ 3035 mtx_lock(&ng_namehash_mtx); 3036 for (i = 0; i < NG_NAME_HASH_SIZE; i++) { 3037 LIST_FOREACH(node, &V_ng_name_hash[i], nd_nodes) { 3038 if (node != &ng_deadnode) { 3039 NG_NODE_REF(node); 3040 break; 3041 } 3042 } 3043 if (node != NULL) 3044 break; 3045 } 3046 mtx_unlock(&ng_namehash_mtx); 3047 3048 /* Attempt to kill it only if it is a regular node */ 3049 if (node != NULL) { 3050 if (node == last_killed) { 3051 /* This should never happen */ 3052 printf("ng node %s needs NGF_REALLY_DIE\n", 3053 node->nd_name); 3054 if (node->nd_flags & NGF_REALLY_DIE) 3055 panic("ng node %s won't die", 3056 node->nd_name); 3057 node->nd_flags |= NGF_REALLY_DIE; 3058 } 3059 ng_rmnode(node, NULL, NULL, 0); 3060 NG_NODE_UNREF(node); 3061 last_killed = node; 3062 } 3063 } while (node != NULL); 3064 } 3065 VNET_SYSUNINIT(vnet_netgraph_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, 3066 vnet_netgraph_uninit, NULL); 3067 #endif /* VIMAGE */ 3068 3069 /* 3070 * Handle loading and unloading for this code. 3071 * The only thing we need to link into is the NETISR strucure. 3072 */ 3073 static int 3074 ngb_mod_event(module_t mod, int event, void *data) 3075 { 3076 struct proc *p; 3077 struct thread *td; 3078 int i, error = 0; 3079 3080 switch (event) { 3081 case MOD_LOAD: 3082 /* Initialize everything. */ 3083 NG_WORKLIST_LOCK_INIT(); 3084 mtx_init(&ng_typelist_mtx, "netgraph types mutex", NULL, 3085 MTX_DEF); 3086 mtx_init(&ng_idhash_mtx, "netgraph idhash mutex", NULL, 3087 MTX_DEF); 3088 mtx_init(&ng_namehash_mtx, "netgraph namehash mutex", NULL, 3089 MTX_DEF); 3090 mtx_init(&ng_topo_mtx, "netgraph topology mutex", NULL, 3091 MTX_DEF); 3092 #ifdef NETGRAPH_DEBUG 3093 mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", NULL, 3094 MTX_DEF); 3095 mtx_init(&ngq_mtx, "netgraph item list mutex", NULL, 3096 MTX_DEF); 3097 #endif 3098 ng_qzone = uma_zcreate("NetGraph items", sizeof(struct ng_item), 3099 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 3100 uma_zone_set_max(ng_qzone, maxalloc); 3101 ng_qdzone = uma_zcreate("NetGraph data items", 3102 sizeof(struct ng_item), NULL, NULL, NULL, NULL, 3103 UMA_ALIGN_CACHE, 0); 3104 uma_zone_set_max(ng_qdzone, maxdata); 3105 /* Autoconfigure number of threads. */ 3106 if (numthreads <= 0) 3107 numthreads = mp_ncpus; 3108 /* Create threads. */ 3109 p = NULL; /* start with no process */ 3110 for (i = 0; i < numthreads; i++) { 3111 if (kproc_kthread_add(ngthread, NULL, &p, &td, 3112 RFHIGHPID, 0, "ng_queue", "ng_queue%d", i)) { 3113 numthreads = i; 3114 break; 3115 } 3116 } 3117 break; 3118 case MOD_UNLOAD: 3119 /* You can't unload it because an interface may be using it. */ 3120 error = EBUSY; 3121 break; 3122 default: 3123 error = EOPNOTSUPP; 3124 break; 3125 } 3126 return (error); 3127 } 3128 3129 static moduledata_t netgraph_mod = { 3130 "netgraph", 3131 ngb_mod_event, 3132 (NULL) 3133 }; 3134 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_NETGRAPH, SI_ORDER_MIDDLE); 3135 SYSCTL_NODE(_net, OID_AUTO, graph, CTLFLAG_RW, 0, "netgraph Family"); 3136 SYSCTL_INT(_net_graph, OID_AUTO, abi_version, CTLFLAG_RD, 0, NG_ABI_VERSION,""); 3137 SYSCTL_INT(_net_graph, OID_AUTO, msg_version, CTLFLAG_RD, 0, NG_VERSION, ""); 3138 3139 #ifdef NETGRAPH_DEBUG 3140 void 3141 dumphook (hook_p hook, char *file, int line) 3142 { 3143 printf("hook: name %s, %d refs, Last touched:\n", 3144 _NG_HOOK_NAME(hook), hook->hk_refs); 3145 printf(" Last active @ %s, line %d\n", 3146 hook->lastfile, hook->lastline); 3147 if (line) { 3148 printf(" problem discovered at file %s, line %d\n", file, line); 3149 #ifdef KDB 3150 kdb_backtrace(); 3151 #endif 3152 } 3153 } 3154 3155 void 3156 dumpnode(node_p node, char *file, int line) 3157 { 3158 printf("node: ID [%x]: type '%s', %d hooks, flags 0x%x, %d refs, %s:\n", 3159 _NG_NODE_ID(node), node->nd_type->name, 3160 node->nd_numhooks, node->nd_flags, 3161 node->nd_refs, node->nd_name); 3162 printf(" Last active @ %s, line %d\n", 3163 node->lastfile, node->lastline); 3164 if (line) { 3165 printf(" problem discovered at file %s, line %d\n", file, line); 3166 #ifdef KDB 3167 kdb_backtrace(); 3168 #endif 3169 } 3170 } 3171 3172 void 3173 dumpitem(item_p item, char *file, int line) 3174 { 3175 printf(" ACTIVE item, last used at %s, line %d", 3176 item->lastfile, item->lastline); 3177 switch(item->el_flags & NGQF_TYPE) { 3178 case NGQF_DATA: 3179 printf(" - [data]\n"); 3180 break; 3181 case NGQF_MESG: 3182 printf(" - retaddr[%d]:\n", _NGI_RETADDR(item)); 3183 break; 3184 case NGQF_FN: 3185 printf(" - fn@%p (%p, %p, %p, %d (%x))\n", 3186 _NGI_FN(item), 3187 _NGI_NODE(item), 3188 _NGI_HOOK(item), 3189 item->body.fn.fn_arg1, 3190 item->body.fn.fn_arg2, 3191 item->body.fn.fn_arg2); 3192 break; 3193 case NGQF_FN2: 3194 printf(" - fn2@%p (%p, %p, %p, %d (%x))\n", 3195 _NGI_FN2(item), 3196 _NGI_NODE(item), 3197 _NGI_HOOK(item), 3198 item->body.fn.fn_arg1, 3199 item->body.fn.fn_arg2, 3200 item->body.fn.fn_arg2); 3201 break; 3202 } 3203 if (line) { 3204 printf(" problem discovered at file %s, line %d\n", file, line); 3205 if (_NGI_NODE(item)) { 3206 printf("node %p ([%x])\n", 3207 _NGI_NODE(item), ng_node2ID(_NGI_NODE(item))); 3208 } 3209 } 3210 } 3211 3212 static void 3213 ng_dumpitems(void) 3214 { 3215 item_p item; 3216 int i = 1; 3217 TAILQ_FOREACH(item, &ng_itemlist, all) { 3218 printf("[%d] ", i++); 3219 dumpitem(item, NULL, 0); 3220 } 3221 } 3222 3223 static void 3224 ng_dumpnodes(void) 3225 { 3226 node_p node; 3227 int i = 1; 3228 mtx_lock(&ng_nodelist_mtx); 3229 SLIST_FOREACH(node, &ng_allnodes, nd_all) { 3230 printf("[%d] ", i++); 3231 dumpnode(node, NULL, 0); 3232 } 3233 mtx_unlock(&ng_nodelist_mtx); 3234 } 3235 3236 static void 3237 ng_dumphooks(void) 3238 { 3239 hook_p hook; 3240 int i = 1; 3241 mtx_lock(&ng_nodelist_mtx); 3242 SLIST_FOREACH(hook, &ng_allhooks, hk_all) { 3243 printf("[%d] ", i++); 3244 dumphook(hook, NULL, 0); 3245 } 3246 mtx_unlock(&ng_nodelist_mtx); 3247 } 3248 3249 static int 3250 sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS) 3251 { 3252 int error; 3253 int val; 3254 int i; 3255 3256 val = allocated; 3257 i = 1; 3258 error = sysctl_handle_int(oidp, &val, 0, req); 3259 if (error != 0 || req->newptr == NULL) 3260 return (error); 3261 if (val == 42) { 3262 ng_dumpitems(); 3263 ng_dumpnodes(); 3264 ng_dumphooks(); 3265 } 3266 return (0); 3267 } 3268 3269 SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, CTLTYPE_INT | CTLFLAG_RW, 3270 0, sizeof(int), sysctl_debug_ng_dump_items, "I", "Number of allocated items"); 3271 #endif /* NETGRAPH_DEBUG */ 3272 3273 3274 /*********************************************************************** 3275 * Worklist routines 3276 **********************************************************************/ 3277 /* 3278 * Pick a node off the list of nodes with work, 3279 * try get an item to process off it. Remove the node from the list. 3280 */ 3281 static void 3282 ngthread(void *arg) 3283 { 3284 for (;;) { 3285 node_p node; 3286 3287 /* Get node from the worklist. */ 3288 NG_WORKLIST_LOCK(); 3289 while ((node = STAILQ_FIRST(&ng_worklist)) == NULL) 3290 NG_WORKLIST_SLEEP(); 3291 STAILQ_REMOVE_HEAD(&ng_worklist, nd_input_queue.q_work); 3292 NG_WORKLIST_UNLOCK(); 3293 CURVNET_SET(node->nd_vnet); 3294 CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist", 3295 __func__, node->nd_ID, node); 3296 /* 3297 * We have the node. We also take over the reference 3298 * that the list had on it. 3299 * Now process as much as you can, until it won't 3300 * let you have another item off the queue. 3301 * All this time, keep the reference 3302 * that lets us be sure that the node still exists. 3303 * Let the reference go at the last minute. 3304 */ 3305 for (;;) { 3306 item_p item; 3307 int rw; 3308 3309 NG_QUEUE_LOCK(&node->nd_input_queue); 3310 item = ng_dequeue(node, &rw); 3311 if (item == NULL) { 3312 node->nd_input_queue.q_flags2 &= ~NGQ2_WORKQ; 3313 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3314 break; /* go look for another node */ 3315 } else { 3316 NG_QUEUE_UNLOCK(&node->nd_input_queue); 3317 NGI_GET_NODE(item, node); /* zaps stored node */ 3318 ng_apply_item(node, item, rw); 3319 NG_NODE_UNREF(node); 3320 } 3321 } 3322 NG_NODE_UNREF(node); 3323 CURVNET_RESTORE(); 3324 } 3325 } 3326 3327 /* 3328 * XXX 3329 * It's posible that a debugging NG_NODE_REF may need 3330 * to be outside the mutex zone 3331 */ 3332 static void 3333 ng_worklist_add(node_p node) 3334 { 3335 3336 mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); 3337 3338 if ((node->nd_input_queue.q_flags2 & NGQ2_WORKQ) == 0) { 3339 /* 3340 * If we are not already on the work queue, 3341 * then put us on. 3342 */ 3343 node->nd_input_queue.q_flags2 |= NGQ2_WORKQ; 3344 NG_NODE_REF(node); /* XXX safe in mutex? */ 3345 NG_WORKLIST_LOCK(); 3346 STAILQ_INSERT_TAIL(&ng_worklist, node, nd_input_queue.q_work); 3347 NG_WORKLIST_UNLOCK(); 3348 CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__, 3349 node->nd_ID, node); 3350 NG_WORKLIST_WAKEUP(); 3351 } else { 3352 CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist", 3353 __func__, node->nd_ID, node); 3354 } 3355 } 3356 3357 3358 /*********************************************************************** 3359 * Externally useable functions to set up a queue item ready for sending 3360 ***********************************************************************/ 3361 3362 #ifdef NETGRAPH_DEBUG 3363 #define ITEM_DEBUG_CHECKS \ 3364 do { \ 3365 if (NGI_NODE(item) ) { \ 3366 printf("item already has node"); \ 3367 kdb_enter(KDB_WHY_NETGRAPH, "has node"); \ 3368 NGI_CLR_NODE(item); \ 3369 } \ 3370 if (NGI_HOOK(item) ) { \ 3371 printf("item already has hook"); \ 3372 kdb_enter(KDB_WHY_NETGRAPH, "has hook"); \ 3373 NGI_CLR_HOOK(item); \ 3374 } \ 3375 } while (0) 3376 #else 3377 #define ITEM_DEBUG_CHECKS 3378 #endif 3379 3380 /* 3381 * Put mbuf into the item. 3382 * Hook and node references will be removed when the item is dequeued. 3383 * (or equivalent) 3384 * (XXX) Unsafe because no reference held by peer on remote node. 3385 * remote node might go away in this timescale. 3386 * We know the hooks can't go away because that would require getting 3387 * a writer item on both nodes and we must have at least a reader 3388 * here to be able to do this. 3389 * Note that the hook loaded is the REMOTE hook. 3390 * 3391 * This is possibly in the critical path for new data. 3392 */ 3393 item_p 3394 ng_package_data(struct mbuf *m, int flags) 3395 { 3396 item_p item; 3397 3398 if ((item = ng_alloc_item(NGQF_DATA, flags)) == NULL) { 3399 NG_FREE_M(m); 3400 return (NULL); 3401 } 3402 ITEM_DEBUG_CHECKS; 3403 item->el_flags |= NGQF_READER; 3404 NGI_M(item) = m; 3405 return (item); 3406 } 3407 3408 /* 3409 * Allocate a queue item and put items into it.. 3410 * Evaluate the address as this will be needed to queue it and 3411 * to work out what some of the fields should be. 3412 * Hook and node references will be removed when the item is dequeued. 3413 * (or equivalent) 3414 */ 3415 item_p 3416 ng_package_msg(struct ng_mesg *msg, int flags) 3417 { 3418 item_p item; 3419 3420 if ((item = ng_alloc_item(NGQF_MESG, flags)) == NULL) { 3421 NG_FREE_MSG(msg); 3422 return (NULL); 3423 } 3424 ITEM_DEBUG_CHECKS; 3425 /* Messages items count as writers unless explicitly exempted. */ 3426 if (msg->header.cmd & NGM_READONLY) 3427 item->el_flags |= NGQF_READER; 3428 else 3429 item->el_flags |= NGQF_WRITER; 3430 /* 3431 * Set the current lasthook into the queue item 3432 */ 3433 NGI_MSG(item) = msg; 3434 NGI_RETADDR(item) = 0; 3435 return (item); 3436 } 3437 3438 3439 3440 #define SET_RETADDR(item, here, retaddr) \ 3441 do { /* Data or fn items don't have retaddrs */ \ 3442 if ((item->el_flags & NGQF_TYPE) == NGQF_MESG) { \ 3443 if (retaddr) { \ 3444 NGI_RETADDR(item) = retaddr; \ 3445 } else { \ 3446 /* \ 3447 * The old return address should be ok. \ 3448 * If there isn't one, use the address \ 3449 * here. \ 3450 */ \ 3451 if (NGI_RETADDR(item) == 0) { \ 3452 NGI_RETADDR(item) \ 3453 = ng_node2ID(here); \ 3454 } \ 3455 } \ 3456 } \ 3457 } while (0) 3458 3459 int 3460 ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr) 3461 { 3462 hook_p peer; 3463 node_p peernode; 3464 ITEM_DEBUG_CHECKS; 3465 /* 3466 * Quick sanity check.. 3467 * Since a hook holds a reference on it's node, once we know 3468 * that the peer is still connected (even if invalid,) we know 3469 * that the peer node is present, though maybe invalid. 3470 */ 3471 mtx_lock(&ng_topo_mtx); 3472 if ((hook == NULL) || NG_HOOK_NOT_VALID(hook) || 3473 NG_HOOK_NOT_VALID(peer = NG_HOOK_PEER(hook)) || 3474 NG_NODE_NOT_VALID(peernode = NG_PEER_NODE(hook))) { 3475 NG_FREE_ITEM(item); 3476 TRAP_ERROR(); 3477 mtx_unlock(&ng_topo_mtx); 3478 return (ENETDOWN); 3479 } 3480 3481 /* 3482 * Transfer our interest to the other (peer) end. 3483 */ 3484 NG_HOOK_REF(peer); 3485 NG_NODE_REF(peernode); 3486 NGI_SET_HOOK(item, peer); 3487 NGI_SET_NODE(item, peernode); 3488 SET_RETADDR(item, here, retaddr); 3489 3490 mtx_unlock(&ng_topo_mtx); 3491 3492 return (0); 3493 } 3494 3495 int 3496 ng_address_path(node_p here, item_p item, const char *address, ng_ID_t retaddr) 3497 { 3498 node_p dest = NULL; 3499 hook_p hook = NULL; 3500 int error; 3501 3502 ITEM_DEBUG_CHECKS; 3503 /* 3504 * Note that ng_path2noderef increments the reference count 3505 * on the node for us if it finds one. So we don't have to. 3506 */ 3507 error = ng_path2noderef(here, address, &dest, &hook); 3508 if (error) { 3509 NG_FREE_ITEM(item); 3510 return (error); 3511 } 3512 NGI_SET_NODE(item, dest); 3513 if (hook) 3514 NGI_SET_HOOK(item, hook); 3515 3516 SET_RETADDR(item, here, retaddr); 3517 return (0); 3518 } 3519 3520 int 3521 ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr) 3522 { 3523 node_p dest; 3524 3525 ITEM_DEBUG_CHECKS; 3526 /* 3527 * Find the target node. 3528 */ 3529 dest = ng_ID2noderef(ID); /* GETS REFERENCE! */ 3530 if (dest == NULL) { 3531 NG_FREE_ITEM(item); 3532 TRAP_ERROR(); 3533 return(EINVAL); 3534 } 3535 /* Fill out the contents */ 3536 NGI_SET_NODE(item, dest); 3537 NGI_CLR_HOOK(item); 3538 SET_RETADDR(item, here, retaddr); 3539 return (0); 3540 } 3541 3542 /* 3543 * special case to send a message to self (e.g. destroy node) 3544 * Possibly indicate an arrival hook too. 3545 * Useful for removing that hook :-) 3546 */ 3547 item_p 3548 ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg) 3549 { 3550 item_p item; 3551 3552 /* 3553 * Find the target node. 3554 * If there is a HOOK argument, then use that in preference 3555 * to the address. 3556 */ 3557 if ((item = ng_alloc_item(NGQF_MESG, NG_NOFLAGS)) == NULL) { 3558 NG_FREE_MSG(msg); 3559 return (NULL); 3560 } 3561 3562 /* Fill out the contents */ 3563 item->el_flags |= NGQF_WRITER; 3564 NG_NODE_REF(here); 3565 NGI_SET_NODE(item, here); 3566 if (hook) { 3567 NG_HOOK_REF(hook); 3568 NGI_SET_HOOK(item, hook); 3569 } 3570 NGI_MSG(item) = msg; 3571 NGI_RETADDR(item) = ng_node2ID(here); 3572 return (item); 3573 } 3574 3575 /* 3576 * Send ng_item_fn function call to the specified node. 3577 */ 3578 3579 int 3580 ng_send_fn(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2) 3581 { 3582 3583 return ng_send_fn1(node, hook, fn, arg1, arg2, NG_NOFLAGS); 3584 } 3585 3586 int 3587 ng_send_fn1(node_p node, hook_p hook, ng_item_fn *fn, void * arg1, int arg2, 3588 int flags) 3589 { 3590 item_p item; 3591 3592 if ((item = ng_alloc_item(NGQF_FN, flags)) == NULL) { 3593 return (ENOMEM); 3594 } 3595 item->el_flags |= NGQF_WRITER; 3596 NG_NODE_REF(node); /* and one for the item */ 3597 NGI_SET_NODE(item, node); 3598 if (hook) { 3599 NG_HOOK_REF(hook); 3600 NGI_SET_HOOK(item, hook); 3601 } 3602 NGI_FN(item) = fn; 3603 NGI_ARG1(item) = arg1; 3604 NGI_ARG2(item) = arg2; 3605 return(ng_snd_item(item, flags)); 3606 } 3607 3608 /* 3609 * Send ng_item_fn2 function call to the specified node. 3610 * 3611 * If an optional pitem parameter is supplied, its apply 3612 * callback will be copied to the new item. If also NG_REUSE_ITEM 3613 * flag is set, no new item will be allocated, but pitem will 3614 * be used. 3615 */ 3616 int 3617 ng_send_fn2(node_p node, hook_p hook, item_p pitem, ng_item_fn2 *fn, void *arg1, 3618 int arg2, int flags) 3619 { 3620 item_p item; 3621 3622 KASSERT((pitem != NULL || (flags & NG_REUSE_ITEM) == 0), 3623 ("%s: NG_REUSE_ITEM but no pitem", __func__)); 3624 3625 /* 3626 * Allocate a new item if no supplied or 3627 * if we can't use supplied one. 3628 */ 3629 if (pitem == NULL || (flags & NG_REUSE_ITEM) == 0) { 3630 if ((item = ng_alloc_item(NGQF_FN2, flags)) == NULL) 3631 return (ENOMEM); 3632 if (pitem != NULL) 3633 item->apply = pitem->apply; 3634 } else { 3635 if ((item = ng_realloc_item(pitem, NGQF_FN2, flags)) == NULL) 3636 return (ENOMEM); 3637 } 3638 3639 item->el_flags = (item->el_flags & ~NGQF_RW) | NGQF_WRITER; 3640 NG_NODE_REF(node); /* and one for the item */ 3641 NGI_SET_NODE(item, node); 3642 if (hook) { 3643 NG_HOOK_REF(hook); 3644 NGI_SET_HOOK(item, hook); 3645 } 3646 NGI_FN2(item) = fn; 3647 NGI_ARG1(item) = arg1; 3648 NGI_ARG2(item) = arg2; 3649 return(ng_snd_item(item, flags)); 3650 } 3651 3652 /* 3653 * Official timeout routines for Netgraph nodes. 3654 */ 3655 static void 3656 ng_callout_trampoline(void *arg) 3657 { 3658 item_p item = arg; 3659 3660 CURVNET_SET(NGI_NODE(item)->nd_vnet); 3661 ng_snd_item(item, 0); 3662 CURVNET_RESTORE(); 3663 } 3664 3665 3666 int 3667 ng_callout(struct callout *c, node_p node, hook_p hook, int ticks, 3668 ng_item_fn *fn, void * arg1, int arg2) 3669 { 3670 item_p item, oitem; 3671 3672 if ((item = ng_alloc_item(NGQF_FN, NG_NOFLAGS)) == NULL) 3673 return (ENOMEM); 3674 3675 item->el_flags |= NGQF_WRITER; 3676 NG_NODE_REF(node); /* and one for the item */ 3677 NGI_SET_NODE(item, node); 3678 if (hook) { 3679 NG_HOOK_REF(hook); 3680 NGI_SET_HOOK(item, hook); 3681 } 3682 NGI_FN(item) = fn; 3683 NGI_ARG1(item) = arg1; 3684 NGI_ARG2(item) = arg2; 3685 oitem = c->c_arg; 3686 if (callout_reset(c, ticks, &ng_callout_trampoline, item) == 1 && 3687 oitem != NULL) 3688 NG_FREE_ITEM(oitem); 3689 return (0); 3690 } 3691 3692 /* A special modified version of untimeout() */ 3693 int 3694 ng_uncallout(struct callout *c, node_p node) 3695 { 3696 item_p item; 3697 int rval; 3698 3699 KASSERT(c != NULL, ("ng_uncallout: NULL callout")); 3700 KASSERT(node != NULL, ("ng_uncallout: NULL node")); 3701 3702 rval = callout_stop(c); 3703 item = c->c_arg; 3704 /* Do an extra check */ 3705 if ((rval > 0) && (c->c_func == &ng_callout_trampoline) && 3706 (NGI_NODE(item) == node)) { 3707 /* 3708 * We successfully removed it from the queue before it ran 3709 * So now we need to unreference everything that was 3710 * given extra references. (NG_FREE_ITEM does this). 3711 */ 3712 NG_FREE_ITEM(item); 3713 } 3714 c->c_arg = NULL; 3715 3716 return (rval); 3717 } 3718 3719 /* 3720 * Set the address, if none given, give the node here. 3721 */ 3722 void 3723 ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr) 3724 { 3725 if (retaddr) { 3726 NGI_RETADDR(item) = retaddr; 3727 } else { 3728 /* 3729 * The old return address should be ok. 3730 * If there isn't one, use the address here. 3731 */ 3732 NGI_RETADDR(item) = ng_node2ID(here); 3733 } 3734 } 3735 3736 #define TESTING 3737 #ifdef TESTING 3738 /* just test all the macros */ 3739 void 3740 ng_macro_test(item_p item); 3741 void 3742 ng_macro_test(item_p item) 3743 { 3744 node_p node = NULL; 3745 hook_p hook = NULL; 3746 struct mbuf *m; 3747 struct ng_mesg *msg; 3748 ng_ID_t retaddr; 3749 int error; 3750 3751 NGI_GET_M(item, m); 3752 NGI_GET_MSG(item, msg); 3753 retaddr = NGI_RETADDR(item); 3754 NG_SEND_DATA(error, hook, m, NULL); 3755 NG_SEND_DATA_ONLY(error, hook, m); 3756 NG_FWD_NEW_DATA(error, item, hook, m); 3757 NG_FWD_ITEM_HOOK(error, item, hook); 3758 NG_SEND_MSG_HOOK(error, node, msg, hook, retaddr); 3759 NG_SEND_MSG_ID(error, node, msg, retaddr, retaddr); 3760 NG_SEND_MSG_PATH(error, node, msg, ".:", retaddr); 3761 NG_FWD_MSG_HOOK(error, node, item, hook, retaddr); 3762 } 3763 #endif /* TESTING */ 3764