1 /*- 2 * Copyright (c) 2007-2009 Robert N. M. Watson 3 * Copyright (c) 2010-2011 Juniper Networks, Inc. 4 * All rights reserved. 5 * 6 * This software was developed by Robert N. M. Watson under contract 7 * to Juniper Networks, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 /* 35 * netisr is a packet dispatch service, allowing synchronous (directly 36 * dispatched) and asynchronous (deferred dispatch) processing of packets by 37 * registered protocol handlers. Callers pass a protocol identifier and 38 * packet to netisr, along with a direct dispatch hint, and work will either 39 * be immediately processed by the registered handler, or passed to a 40 * software interrupt (SWI) thread for deferred dispatch. Callers will 41 * generally select one or the other based on: 42 * 43 * - Whether directly dispatching a netisr handler lead to code reentrance or 44 * lock recursion, such as entering the socket code from the socket code. 45 * - Whether directly dispatching a netisr handler lead to recursive 46 * processing, such as when decapsulating several wrapped layers of tunnel 47 * information (IPSEC within IPSEC within ...). 48 * 49 * Maintaining ordering for protocol streams is a critical design concern. 50 * Enforcing ordering limits the opportunity for concurrency, but maintains 51 * the strong ordering requirements found in some protocols, such as TCP. Of 52 * related concern is CPU affinity--it is desirable to process all data 53 * associated with a particular stream on the same CPU over time in order to 54 * avoid acquiring locks associated with the connection on different CPUs, 55 * keep connection data in one cache, and to generally encourage associated 56 * user threads to live on the same CPU as the stream. It's also desirable 57 * to avoid lock migration and contention where locks are associated with 58 * more than one flow. 59 * 60 * netisr supports several policy variations, represented by the 61 * NETISR_POLICY_* constants, allowing protocols to play various roles in 62 * identifying flows, assigning work to CPUs, etc. These are described in 63 * netisr.h. 64 */ 65 66 #include "opt_ddb.h" 67 #include "opt_device_polling.h" 68 69 #include <sys/param.h> 70 #include <sys/bus.h> 71 #include <sys/kernel.h> 72 #include <sys/kthread.h> 73 #include <sys/interrupt.h> 74 #include <sys/lock.h> 75 #include <sys/mbuf.h> 76 #include <sys/mutex.h> 77 #include <sys/pcpu.h> 78 #include <sys/proc.h> 79 #include <sys/rmlock.h> 80 #include <sys/sched.h> 81 #include <sys/smp.h> 82 #include <sys/socket.h> 83 #include <sys/sysctl.h> 84 #include <sys/systm.h> 85 86 #ifdef DDB 87 #include <ddb/ddb.h> 88 #endif 89 90 #define _WANT_NETISR_INTERNAL /* Enable definitions from netisr_internal.h */ 91 #include <net/if.h> 92 #include <net/if_var.h> 93 #include <net/netisr.h> 94 #include <net/netisr_internal.h> 95 #include <net/vnet.h> 96 97 /*- 98 * Synchronize use and modification of the registered netisr data structures; 99 * acquire a read lock while modifying the set of registered protocols to 100 * prevent partially registered or unregistered protocols from being run. 101 * 102 * The following data structures and fields are protected by this lock: 103 * 104 * - The netisr_proto array, including all fields of struct netisr_proto. 105 * - The nws array, including all fields of struct netisr_worker. 106 * - The nws_array array. 107 * 108 * Note: the NETISR_LOCKING define controls whether read locks are acquired 109 * in packet processing paths requiring netisr registration stability. This 110 * is disabled by default as it can lead to measurable performance 111 * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and 112 * because netisr registration and unregistration is extremely rare at 113 * runtime. If it becomes more common, this decision should be revisited. 114 * 115 * XXXRW: rmlocks don't support assertions. 116 */ 117 static struct rmlock netisr_rmlock; 118 #define NETISR_LOCK_INIT() rm_init_flags(&netisr_rmlock, "netisr", \ 119 RM_NOWITNESS) 120 #define NETISR_LOCK_ASSERT() 121 #define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) 122 #define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) 123 #define NETISR_WLOCK() rm_wlock(&netisr_rmlock) 124 #define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) 125 /* #define NETISR_LOCKING */ 126 127 static SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr"); 128 129 /*- 130 * Three global direct dispatch policies are supported: 131 * 132 * NETISR_DISPATCH_DEFERRED: All work is deferred for a netisr, regardless of 133 * context (may be overriden by protocols). 134 * 135 * NETISR_DISPATCH_HYBRID: If the executing context allows direct dispatch, 136 * and we're running on the CPU the work would be performed on, then direct 137 * dispatch it if it wouldn't violate ordering constraints on the workstream. 138 * 139 * NETISR_DISPATCH_DIRECT: If the executing context allows direct dispatch, 140 * always direct dispatch. (The default.) 141 * 142 * Notice that changing the global policy could lead to short periods of 143 * misordered processing, but this is considered acceptable as compared to 144 * the complexity of enforcing ordering during policy changes. Protocols can 145 * override the global policy (when they're not doing that, they select 146 * NETISR_DISPATCH_DEFAULT). 147 */ 148 #define NETISR_DISPATCH_POLICY_DEFAULT NETISR_DISPATCH_DIRECT 149 #define NETISR_DISPATCH_POLICY_MAXSTR 20 /* Used for temporary buffers. */ 150 static u_int netisr_dispatch_policy = NETISR_DISPATCH_POLICY_DEFAULT; 151 static int sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS); 152 SYSCTL_PROC(_net_isr, OID_AUTO, dispatch, CTLTYPE_STRING | CTLFLAG_RW | 153 CTLFLAG_TUN, 0, 0, sysctl_netisr_dispatch_policy, "A", 154 "netisr dispatch policy"); 155 156 /* 157 * Allow the administrator to limit the number of threads (CPUs) to use for 158 * netisr. We don't check netisr_maxthreads before creating the thread for 159 * CPU 0, so in practice we ignore values <= 1. This must be set at boot. 160 * We will create at most one thread per CPU. 161 */ 162 static int netisr_maxthreads = -1; /* Max number of threads. */ 163 TUNABLE_INT("net.isr.maxthreads", &netisr_maxthreads); 164 SYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RDTUN, 165 &netisr_maxthreads, 0, 166 "Use at most this many CPUs for netisr processing"); 167 168 static int netisr_bindthreads = 0; /* Bind threads to CPUs. */ 169 TUNABLE_INT("net.isr.bindthreads", &netisr_bindthreads); 170 SYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RDTUN, 171 &netisr_bindthreads, 0, "Bind netisr threads to CPUs."); 172 173 /* 174 * Limit per-workstream mbuf queue limits s to at most net.isr.maxqlimit, 175 * both for initial configuration and later modification using 176 * netisr_setqlimit(). 177 */ 178 #define NETISR_DEFAULT_MAXQLIMIT 10240 179 static u_int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; 180 TUNABLE_INT("net.isr.maxqlimit", &netisr_maxqlimit); 181 SYSCTL_UINT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RDTUN, 182 &netisr_maxqlimit, 0, 183 "Maximum netisr per-protocol, per-CPU queue depth."); 184 185 /* 186 * The default per-workstream mbuf queue limit for protocols that don't 187 * initialize the nh_qlimit field of their struct netisr_handler. If this is 188 * set above netisr_maxqlimit, we truncate it to the maximum during boot. 189 */ 190 #define NETISR_DEFAULT_DEFAULTQLIMIT 256 191 static u_int netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT; 192 TUNABLE_INT("net.isr.defaultqlimit", &netisr_defaultqlimit); 193 SYSCTL_UINT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RDTUN, 194 &netisr_defaultqlimit, 0, 195 "Default netisr per-protocol, per-CPU queue limit if not set by protocol"); 196 197 /* 198 * Store and export the compile-time constant NETISR_MAXPROT limit on the 199 * number of protocols that can register with netisr at a time. This is 200 * required for crashdump analysis, as it sizes netisr_proto[]. 201 */ 202 static u_int netisr_maxprot = NETISR_MAXPROT; 203 SYSCTL_UINT(_net_isr, OID_AUTO, maxprot, CTLFLAG_RD, 204 &netisr_maxprot, 0, 205 "Compile-time limit on the number of protocols supported by netisr."); 206 207 /* 208 * The netisr_proto array describes all registered protocols, indexed by 209 * protocol number. See netisr_internal.h for more details. 210 */ 211 static struct netisr_proto netisr_proto[NETISR_MAXPROT]; 212 213 /* 214 * Per-CPU workstream data. See netisr_internal.h for more details. 215 */ 216 DPCPU_DEFINE(struct netisr_workstream, nws); 217 218 /* 219 * Map contiguous values between 0 and nws_count into CPU IDs appropriate for 220 * accessing workstreams. This allows constructions of the form 221 * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws). 222 */ 223 static u_int nws_array[MAXCPU]; 224 225 /* 226 * Number of registered workstreams. Will be at most the number of running 227 * CPUs once fully started. 228 */ 229 static u_int nws_count; 230 SYSCTL_UINT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD, 231 &nws_count, 0, "Number of extant netisr threads."); 232 233 /* 234 * Synchronization for each workstream: a mutex protects all mutable fields 235 * in each stream, including per-protocol state (mbuf queues). The SWI is 236 * woken up if asynchronous dispatch is required. 237 */ 238 #define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) 239 #define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) 240 #define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) 241 #define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) 242 243 /* 244 * Utility routines for protocols that implement their own mapping of flows 245 * to CPUs. 246 */ 247 u_int 248 netisr_get_cpucount(void) 249 { 250 251 return (nws_count); 252 } 253 254 u_int 255 netisr_get_cpuid(u_int cpunumber) 256 { 257 258 KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber, 259 nws_count)); 260 261 return (nws_array[cpunumber]); 262 } 263 264 /* 265 * The default implementation of flow -> CPU ID mapping. 266 * 267 * Non-static so that protocols can use it to map their own work to specific 268 * CPUs in a manner consistent to netisr for affinity purposes. 269 */ 270 u_int 271 netisr_default_flow2cpu(u_int flowid) 272 { 273 274 return (nws_array[flowid % nws_count]); 275 } 276 277 /* 278 * Dispatch tunable and sysctl configuration. 279 */ 280 struct netisr_dispatch_table_entry { 281 u_int ndte_policy; 282 const char *ndte_policy_str; 283 }; 284 static const struct netisr_dispatch_table_entry netisr_dispatch_table[] = { 285 { NETISR_DISPATCH_DEFAULT, "default" }, 286 { NETISR_DISPATCH_DEFERRED, "deferred" }, 287 { NETISR_DISPATCH_HYBRID, "hybrid" }, 288 { NETISR_DISPATCH_DIRECT, "direct" }, 289 }; 290 static const u_int netisr_dispatch_table_len = 291 (sizeof(netisr_dispatch_table) / sizeof(netisr_dispatch_table[0])); 292 293 static void 294 netisr_dispatch_policy_to_str(u_int dispatch_policy, char *buffer, 295 u_int buflen) 296 { 297 const struct netisr_dispatch_table_entry *ndtep; 298 const char *str; 299 u_int i; 300 301 str = "unknown"; 302 for (i = 0; i < netisr_dispatch_table_len; i++) { 303 ndtep = &netisr_dispatch_table[i]; 304 if (ndtep->ndte_policy == dispatch_policy) { 305 str = ndtep->ndte_policy_str; 306 break; 307 } 308 } 309 snprintf(buffer, buflen, "%s", str); 310 } 311 312 static int 313 netisr_dispatch_policy_from_str(const char *str, u_int *dispatch_policyp) 314 { 315 const struct netisr_dispatch_table_entry *ndtep; 316 u_int i; 317 318 for (i = 0; i < netisr_dispatch_table_len; i++) { 319 ndtep = &netisr_dispatch_table[i]; 320 if (strcmp(ndtep->ndte_policy_str, str) == 0) { 321 *dispatch_policyp = ndtep->ndte_policy; 322 return (0); 323 } 324 } 325 return (EINVAL); 326 } 327 328 static int 329 sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS) 330 { 331 char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 332 u_int dispatch_policy; 333 int error; 334 335 netisr_dispatch_policy_to_str(netisr_dispatch_policy, tmp, 336 sizeof(tmp)); 337 error = sysctl_handle_string(oidp, tmp, sizeof(tmp), req); 338 if (error == 0 && req->newptr != NULL) { 339 error = netisr_dispatch_policy_from_str(tmp, 340 &dispatch_policy); 341 if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 342 error = EINVAL; 343 if (error == 0) 344 netisr_dispatch_policy = dispatch_policy; 345 } 346 return (error); 347 } 348 349 /* 350 * Register a new netisr handler, which requires initializing per-protocol 351 * fields for each workstream. All netisr work is briefly suspended while 352 * the protocol is installed. 353 */ 354 void 355 netisr_register(const struct netisr_handler *nhp) 356 { 357 struct netisr_work *npwp; 358 const char *name; 359 u_int i, proto; 360 361 proto = nhp->nh_proto; 362 name = nhp->nh_name; 363 364 /* 365 * Test that the requested registration is valid. 366 */ 367 KASSERT(nhp->nh_name != NULL, 368 ("%s: nh_name NULL for %u", __func__, proto)); 369 KASSERT(nhp->nh_handler != NULL, 370 ("%s: nh_handler NULL for %s", __func__, name)); 371 KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || 372 nhp->nh_policy == NETISR_POLICY_FLOW || 373 nhp->nh_policy == NETISR_POLICY_CPU, 374 ("%s: unsupported nh_policy %u for %s", __func__, 375 nhp->nh_policy, name)); 376 KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || 377 nhp->nh_m2flow == NULL, 378 ("%s: nh_policy != FLOW but m2flow defined for %s", __func__, 379 name)); 380 KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL, 381 ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__, 382 name)); 383 KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL, 384 ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__, 385 name)); 386 KASSERT(nhp->nh_dispatch == NETISR_DISPATCH_DEFAULT || 387 nhp->nh_dispatch == NETISR_DISPATCH_DEFERRED || 388 nhp->nh_dispatch == NETISR_DISPATCH_HYBRID || 389 nhp->nh_dispatch == NETISR_DISPATCH_DIRECT, 390 ("%s: invalid nh_dispatch (%u)", __func__, nhp->nh_dispatch)); 391 392 KASSERT(proto < NETISR_MAXPROT, 393 ("%s(%u, %s): protocol too big", __func__, proto, name)); 394 395 /* 396 * Test that no existing registration exists for this protocol. 397 */ 398 NETISR_WLOCK(); 399 KASSERT(netisr_proto[proto].np_name == NULL, 400 ("%s(%u, %s): name present", __func__, proto, name)); 401 KASSERT(netisr_proto[proto].np_handler == NULL, 402 ("%s(%u, %s): handler present", __func__, proto, name)); 403 404 netisr_proto[proto].np_name = name; 405 netisr_proto[proto].np_handler = nhp->nh_handler; 406 netisr_proto[proto].np_m2flow = nhp->nh_m2flow; 407 netisr_proto[proto].np_m2cpuid = nhp->nh_m2cpuid; 408 netisr_proto[proto].np_drainedcpu = nhp->nh_drainedcpu; 409 if (nhp->nh_qlimit == 0) 410 netisr_proto[proto].np_qlimit = netisr_defaultqlimit; 411 else if (nhp->nh_qlimit > netisr_maxqlimit) { 412 printf("%s: %s requested queue limit %u capped to " 413 "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit, 414 netisr_maxqlimit); 415 netisr_proto[proto].np_qlimit = netisr_maxqlimit; 416 } else 417 netisr_proto[proto].np_qlimit = nhp->nh_qlimit; 418 netisr_proto[proto].np_policy = nhp->nh_policy; 419 netisr_proto[proto].np_dispatch = nhp->nh_dispatch; 420 CPU_FOREACH(i) { 421 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 422 bzero(npwp, sizeof(*npwp)); 423 npwp->nw_qlimit = netisr_proto[proto].np_qlimit; 424 } 425 NETISR_WUNLOCK(); 426 } 427 428 /* 429 * Clear drop counters across all workstreams for a protocol. 430 */ 431 void 432 netisr_clearqdrops(const struct netisr_handler *nhp) 433 { 434 struct netisr_work *npwp; 435 #ifdef INVARIANTS 436 const char *name; 437 #endif 438 u_int i, proto; 439 440 proto = nhp->nh_proto; 441 #ifdef INVARIANTS 442 name = nhp->nh_name; 443 #endif 444 KASSERT(proto < NETISR_MAXPROT, 445 ("%s(%u): protocol too big for %s", __func__, proto, name)); 446 447 NETISR_WLOCK(); 448 KASSERT(netisr_proto[proto].np_handler != NULL, 449 ("%s(%u): protocol not registered for %s", __func__, proto, 450 name)); 451 452 CPU_FOREACH(i) { 453 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 454 npwp->nw_qdrops = 0; 455 } 456 NETISR_WUNLOCK(); 457 } 458 459 /* 460 * Query current drop counters across all workstreams for a protocol. 461 */ 462 void 463 netisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) 464 { 465 struct netisr_work *npwp; 466 struct rm_priotracker tracker; 467 #ifdef INVARIANTS 468 const char *name; 469 #endif 470 u_int i, proto; 471 472 *qdropp = 0; 473 proto = nhp->nh_proto; 474 #ifdef INVARIANTS 475 name = nhp->nh_name; 476 #endif 477 KASSERT(proto < NETISR_MAXPROT, 478 ("%s(%u): protocol too big for %s", __func__, proto, name)); 479 480 NETISR_RLOCK(&tracker); 481 KASSERT(netisr_proto[proto].np_handler != NULL, 482 ("%s(%u): protocol not registered for %s", __func__, proto, 483 name)); 484 485 CPU_FOREACH(i) { 486 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 487 *qdropp += npwp->nw_qdrops; 488 } 489 NETISR_RUNLOCK(&tracker); 490 } 491 492 /* 493 * Query current per-workstream queue limit for a protocol. 494 */ 495 void 496 netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) 497 { 498 struct rm_priotracker tracker; 499 #ifdef INVARIANTS 500 const char *name; 501 #endif 502 u_int proto; 503 504 proto = nhp->nh_proto; 505 #ifdef INVARIANTS 506 name = nhp->nh_name; 507 #endif 508 KASSERT(proto < NETISR_MAXPROT, 509 ("%s(%u): protocol too big for %s", __func__, proto, name)); 510 511 NETISR_RLOCK(&tracker); 512 KASSERT(netisr_proto[proto].np_handler != NULL, 513 ("%s(%u): protocol not registered for %s", __func__, proto, 514 name)); 515 *qlimitp = netisr_proto[proto].np_qlimit; 516 NETISR_RUNLOCK(&tracker); 517 } 518 519 /* 520 * Update the queue limit across per-workstream queues for a protocol. We 521 * simply change the limits, and don't drain overflowed packets as they will 522 * (hopefully) take care of themselves shortly. 523 */ 524 int 525 netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit) 526 { 527 struct netisr_work *npwp; 528 #ifdef INVARIANTS 529 const char *name; 530 #endif 531 u_int i, proto; 532 533 if (qlimit > netisr_maxqlimit) 534 return (EINVAL); 535 536 proto = nhp->nh_proto; 537 #ifdef INVARIANTS 538 name = nhp->nh_name; 539 #endif 540 KASSERT(proto < NETISR_MAXPROT, 541 ("%s(%u): protocol too big for %s", __func__, proto, name)); 542 543 NETISR_WLOCK(); 544 KASSERT(netisr_proto[proto].np_handler != NULL, 545 ("%s(%u): protocol not registered for %s", __func__, proto, 546 name)); 547 548 netisr_proto[proto].np_qlimit = qlimit; 549 CPU_FOREACH(i) { 550 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 551 npwp->nw_qlimit = qlimit; 552 } 553 NETISR_WUNLOCK(); 554 return (0); 555 } 556 557 /* 558 * Drain all packets currently held in a particular protocol work queue. 559 */ 560 static void 561 netisr_drain_proto(struct netisr_work *npwp) 562 { 563 struct mbuf *m; 564 565 /* 566 * We would assert the lock on the workstream but it's not passed in. 567 */ 568 while ((m = npwp->nw_head) != NULL) { 569 npwp->nw_head = m->m_nextpkt; 570 m->m_nextpkt = NULL; 571 if (npwp->nw_head == NULL) 572 npwp->nw_tail = NULL; 573 npwp->nw_len--; 574 m_freem(m); 575 } 576 KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__)); 577 KASSERT(npwp->nw_len == 0, ("%s: len", __func__)); 578 } 579 580 /* 581 * Remove the registration of a network protocol, which requires clearing 582 * per-protocol fields across all workstreams, including freeing all mbufs in 583 * the queues at time of unregister. All work in netisr is briefly suspended 584 * while this takes place. 585 */ 586 void 587 netisr_unregister(const struct netisr_handler *nhp) 588 { 589 struct netisr_work *npwp; 590 #ifdef INVARIANTS 591 const char *name; 592 #endif 593 u_int i, proto; 594 595 proto = nhp->nh_proto; 596 #ifdef INVARIANTS 597 name = nhp->nh_name; 598 #endif 599 KASSERT(proto < NETISR_MAXPROT, 600 ("%s(%u): protocol too big for %s", __func__, proto, name)); 601 602 NETISR_WLOCK(); 603 KASSERT(netisr_proto[proto].np_handler != NULL, 604 ("%s(%u): protocol not registered for %s", __func__, proto, 605 name)); 606 607 netisr_proto[proto].np_name = NULL; 608 netisr_proto[proto].np_handler = NULL; 609 netisr_proto[proto].np_m2flow = NULL; 610 netisr_proto[proto].np_m2cpuid = NULL; 611 netisr_proto[proto].np_qlimit = 0; 612 netisr_proto[proto].np_policy = 0; 613 CPU_FOREACH(i) { 614 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 615 netisr_drain_proto(npwp); 616 bzero(npwp, sizeof(*npwp)); 617 } 618 NETISR_WUNLOCK(); 619 } 620 621 /* 622 * Compose the global and per-protocol policies on dispatch, and return the 623 * dispatch policy to use. 624 */ 625 static u_int 626 netisr_get_dispatch(struct netisr_proto *npp) 627 { 628 629 /* 630 * Protocol-specific configuration overrides the global default. 631 */ 632 if (npp->np_dispatch != NETISR_DISPATCH_DEFAULT) 633 return (npp->np_dispatch); 634 return (netisr_dispatch_policy); 635 } 636 637 /* 638 * Look up the workstream given a packet and source identifier. Do this by 639 * checking the protocol's policy, and optionally call out to the protocol 640 * for assistance if required. 641 */ 642 static struct mbuf * 643 netisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy, 644 uintptr_t source, struct mbuf *m, u_int *cpuidp) 645 { 646 struct ifnet *ifp; 647 u_int policy; 648 649 NETISR_LOCK_ASSERT(); 650 651 /* 652 * In the event we have only one worker, shortcut and deliver to it 653 * without further ado. 654 */ 655 if (nws_count == 1) { 656 *cpuidp = nws_array[0]; 657 return (m); 658 } 659 660 /* 661 * What happens next depends on the policy selected by the protocol. 662 * If we want to support per-interface policies, we should do that 663 * here first. 664 */ 665 policy = npp->np_policy; 666 if (policy == NETISR_POLICY_CPU) { 667 m = npp->np_m2cpuid(m, source, cpuidp); 668 if (m == NULL) 669 return (NULL); 670 671 /* 672 * It's possible for a protocol not to have a good idea about 673 * where to process a packet, in which case we fall back on 674 * the netisr code to decide. In the hybrid case, return the 675 * current CPU ID, which will force an immediate direct 676 * dispatch. In the queued case, fall back on the SOURCE 677 * policy. 678 */ 679 if (*cpuidp != NETISR_CPUID_NONE) 680 return (m); 681 if (dispatch_policy == NETISR_DISPATCH_HYBRID) { 682 *cpuidp = curcpu; 683 return (m); 684 } 685 policy = NETISR_POLICY_SOURCE; 686 } 687 688 if (policy == NETISR_POLICY_FLOW) { 689 if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { 690 m = npp->np_m2flow(m, source); 691 if (m == NULL) 692 return (NULL); 693 } 694 if (m->m_flags & M_FLOWID) { 695 *cpuidp = 696 netisr_default_flow2cpu(m->m_pkthdr.flowid); 697 return (m); 698 } 699 policy = NETISR_POLICY_SOURCE; 700 } 701 702 KASSERT(policy == NETISR_POLICY_SOURCE, 703 ("%s: invalid policy %u for %s", __func__, npp->np_policy, 704 npp->np_name)); 705 706 ifp = m->m_pkthdr.rcvif; 707 if (ifp != NULL) 708 *cpuidp = nws_array[(ifp->if_index + source) % nws_count]; 709 else 710 *cpuidp = nws_array[source % nws_count]; 711 return (m); 712 } 713 714 /* 715 * Process packets associated with a workstream and protocol. For reasons of 716 * fairness, we process up to one complete netisr queue at a time, moving the 717 * queue to a stack-local queue for processing, but do not loop refreshing 718 * from the global queue. The caller is responsible for deciding whether to 719 * loop, and for setting the NWS_RUNNING flag. The passed workstream will be 720 * locked on entry and relocked before return, but will be released while 721 * processing. The number of packets processed is returned. 722 */ 723 static u_int 724 netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) 725 { 726 struct netisr_work local_npw, *npwp; 727 u_int handled; 728 struct mbuf *m; 729 730 NETISR_LOCK_ASSERT(); 731 NWS_LOCK_ASSERT(nwsp); 732 733 KASSERT(nwsp->nws_flags & NWS_RUNNING, 734 ("%s(%u): not running", __func__, proto)); 735 KASSERT(proto >= 0 && proto < NETISR_MAXPROT, 736 ("%s(%u): invalid proto\n", __func__, proto)); 737 738 npwp = &nwsp->nws_work[proto]; 739 if (npwp->nw_len == 0) 740 return (0); 741 742 /* 743 * Move the global work queue to a thread-local work queue. 744 * 745 * Notice that this means the effective maximum length of the queue 746 * is actually twice that of the maximum queue length specified in 747 * the protocol registration call. 748 */ 749 handled = npwp->nw_len; 750 local_npw = *npwp; 751 npwp->nw_head = NULL; 752 npwp->nw_tail = NULL; 753 npwp->nw_len = 0; 754 nwsp->nws_pendingbits &= ~(1 << proto); 755 NWS_UNLOCK(nwsp); 756 while ((m = local_npw.nw_head) != NULL) { 757 local_npw.nw_head = m->m_nextpkt; 758 m->m_nextpkt = NULL; 759 if (local_npw.nw_head == NULL) 760 local_npw.nw_tail = NULL; 761 local_npw.nw_len--; 762 VNET_ASSERT(m->m_pkthdr.rcvif != NULL, 763 ("%s:%d rcvif == NULL: m=%p", __func__, __LINE__, m)); 764 CURVNET_SET(m->m_pkthdr.rcvif->if_vnet); 765 netisr_proto[proto].np_handler(m); 766 CURVNET_RESTORE(); 767 } 768 KASSERT(local_npw.nw_len == 0, 769 ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); 770 if (netisr_proto[proto].np_drainedcpu) 771 netisr_proto[proto].np_drainedcpu(nwsp->nws_cpu); 772 NWS_LOCK(nwsp); 773 npwp->nw_handled += handled; 774 return (handled); 775 } 776 777 /* 778 * SWI handler for netisr -- processes packets in a set of workstreams that 779 * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already 780 * being direct dispatched, go back to sleep and wait for the dispatching 781 * thread to wake us up again. 782 */ 783 static void 784 swi_net(void *arg) 785 { 786 #ifdef NETISR_LOCKING 787 struct rm_priotracker tracker; 788 #endif 789 struct netisr_workstream *nwsp; 790 u_int bits, prot; 791 792 nwsp = arg; 793 794 #ifdef DEVICE_POLLING 795 KASSERT(nws_count == 1, 796 ("%s: device_polling but nws_count != 1", __func__)); 797 netisr_poll(); 798 #endif 799 #ifdef NETISR_LOCKING 800 NETISR_RLOCK(&tracker); 801 #endif 802 NWS_LOCK(nwsp); 803 KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); 804 if (nwsp->nws_flags & NWS_DISPATCHING) 805 goto out; 806 nwsp->nws_flags |= NWS_RUNNING; 807 nwsp->nws_flags &= ~NWS_SCHEDULED; 808 while ((bits = nwsp->nws_pendingbits) != 0) { 809 while ((prot = ffs(bits)) != 0) { 810 prot--; 811 bits &= ~(1 << prot); 812 (void)netisr_process_workstream_proto(nwsp, prot); 813 } 814 } 815 nwsp->nws_flags &= ~NWS_RUNNING; 816 out: 817 NWS_UNLOCK(nwsp); 818 #ifdef NETISR_LOCKING 819 NETISR_RUNLOCK(&tracker); 820 #endif 821 #ifdef DEVICE_POLLING 822 netisr_pollmore(); 823 #endif 824 } 825 826 static int 827 netisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto, 828 struct netisr_work *npwp, struct mbuf *m, int *dosignalp) 829 { 830 831 NWS_LOCK_ASSERT(nwsp); 832 833 *dosignalp = 0; 834 if (npwp->nw_len < npwp->nw_qlimit) { 835 m->m_nextpkt = NULL; 836 if (npwp->nw_head == NULL) { 837 npwp->nw_head = m; 838 npwp->nw_tail = m; 839 } else { 840 npwp->nw_tail->m_nextpkt = m; 841 npwp->nw_tail = m; 842 } 843 npwp->nw_len++; 844 if (npwp->nw_len > npwp->nw_watermark) 845 npwp->nw_watermark = npwp->nw_len; 846 847 /* 848 * We must set the bit regardless of NWS_RUNNING, so that 849 * swi_net() keeps calling netisr_process_workstream_proto(). 850 */ 851 nwsp->nws_pendingbits |= (1 << proto); 852 if (!(nwsp->nws_flags & 853 (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) { 854 nwsp->nws_flags |= NWS_SCHEDULED; 855 *dosignalp = 1; /* Defer until unlocked. */ 856 } 857 npwp->nw_queued++; 858 return (0); 859 } else { 860 m_freem(m); 861 npwp->nw_qdrops++; 862 return (ENOBUFS); 863 } 864 } 865 866 static int 867 netisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) 868 { 869 struct netisr_workstream *nwsp; 870 struct netisr_work *npwp; 871 int dosignal, error; 872 873 #ifdef NETISR_LOCKING 874 NETISR_LOCK_ASSERT(); 875 #endif 876 KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__, 877 cpuid, mp_maxid)); 878 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 879 880 dosignal = 0; 881 error = 0; 882 nwsp = DPCPU_ID_PTR(cpuid, nws); 883 npwp = &nwsp->nws_work[proto]; 884 NWS_LOCK(nwsp); 885 error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); 886 NWS_UNLOCK(nwsp); 887 if (dosignal) 888 NWS_SIGNAL(nwsp); 889 return (error); 890 } 891 892 int 893 netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m) 894 { 895 #ifdef NETISR_LOCKING 896 struct rm_priotracker tracker; 897 #endif 898 u_int cpuid; 899 int error; 900 901 KASSERT(proto < NETISR_MAXPROT, 902 ("%s: invalid proto %u", __func__, proto)); 903 904 #ifdef NETISR_LOCKING 905 NETISR_RLOCK(&tracker); 906 #endif 907 KASSERT(netisr_proto[proto].np_handler != NULL, 908 ("%s: invalid proto %u", __func__, proto)); 909 910 m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_DEFERRED, 911 source, m, &cpuid); 912 if (m != NULL) { 913 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, 914 cpuid)); 915 error = netisr_queue_internal(proto, m, cpuid); 916 } else 917 error = ENOBUFS; 918 #ifdef NETISR_LOCKING 919 NETISR_RUNLOCK(&tracker); 920 #endif 921 return (error); 922 } 923 924 int 925 netisr_queue(u_int proto, struct mbuf *m) 926 { 927 928 return (netisr_queue_src(proto, 0, m)); 929 } 930 931 /* 932 * Dispatch a packet for netisr processing; direct dispatch is permitted by 933 * calling context. 934 */ 935 int 936 netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) 937 { 938 #ifdef NETISR_LOCKING 939 struct rm_priotracker tracker; 940 #endif 941 struct netisr_workstream *nwsp; 942 struct netisr_proto *npp; 943 struct netisr_work *npwp; 944 int dosignal, error; 945 u_int cpuid, dispatch_policy; 946 947 KASSERT(proto < NETISR_MAXPROT, 948 ("%s: invalid proto %u", __func__, proto)); 949 #ifdef NETISR_LOCKING 950 NETISR_RLOCK(&tracker); 951 #endif 952 npp = &netisr_proto[proto]; 953 KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__, 954 proto)); 955 956 dispatch_policy = netisr_get_dispatch(npp); 957 if (dispatch_policy == NETISR_DISPATCH_DEFERRED) 958 return (netisr_queue_src(proto, source, m)); 959 960 /* 961 * If direct dispatch is forced, then unconditionally dispatch 962 * without a formal CPU selection. Borrow the current CPU's stats, 963 * even if there's no worker on it. In this case we don't update 964 * nws_flags because all netisr processing will be source ordered due 965 * to always being forced to directly dispatch. 966 */ 967 if (dispatch_policy == NETISR_DISPATCH_DIRECT) { 968 nwsp = DPCPU_PTR(nws); 969 npwp = &nwsp->nws_work[proto]; 970 npwp->nw_dispatched++; 971 npwp->nw_handled++; 972 netisr_proto[proto].np_handler(m); 973 error = 0; 974 goto out_unlock; 975 } 976 977 KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID, 978 ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy)); 979 980 /* 981 * Otherwise, we execute in a hybrid mode where we will try to direct 982 * dispatch if we're on the right CPU and the netisr worker isn't 983 * already running. 984 */ 985 sched_pin(); 986 m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID, 987 source, m, &cpuid); 988 if (m == NULL) { 989 error = ENOBUFS; 990 goto out_unpin; 991 } 992 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 993 if (cpuid != curcpu) 994 goto queue_fallback; 995 nwsp = DPCPU_PTR(nws); 996 npwp = &nwsp->nws_work[proto]; 997 998 /*- 999 * We are willing to direct dispatch only if three conditions hold: 1000 * 1001 * (1) The netisr worker isn't already running, 1002 * (2) Another thread isn't already directly dispatching, and 1003 * (3) The netisr hasn't already been woken up. 1004 */ 1005 NWS_LOCK(nwsp); 1006 if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { 1007 error = netisr_queue_workstream(nwsp, proto, npwp, m, 1008 &dosignal); 1009 NWS_UNLOCK(nwsp); 1010 if (dosignal) 1011 NWS_SIGNAL(nwsp); 1012 goto out_unpin; 1013 } 1014 1015 /* 1016 * The current thread is now effectively the netisr worker, so set 1017 * the dispatching flag to prevent concurrent processing of the 1018 * stream from another thread (even the netisr worker), which could 1019 * otherwise lead to effective misordering of the stream. 1020 */ 1021 nwsp->nws_flags |= NWS_DISPATCHING; 1022 NWS_UNLOCK(nwsp); 1023 netisr_proto[proto].np_handler(m); 1024 NWS_LOCK(nwsp); 1025 nwsp->nws_flags &= ~NWS_DISPATCHING; 1026 npwp->nw_handled++; 1027 npwp->nw_hybrid_dispatched++; 1028 1029 /* 1030 * If other work was enqueued by another thread while we were direct 1031 * dispatching, we need to signal the netisr worker to do that work. 1032 * In the future, we might want to do some of that work in the 1033 * current thread, rather than trigger further context switches. If 1034 * so, we'll want to establish a reasonable bound on the work done in 1035 * the "borrowed" context. 1036 */ 1037 if (nwsp->nws_pendingbits != 0) { 1038 nwsp->nws_flags |= NWS_SCHEDULED; 1039 dosignal = 1; 1040 } else 1041 dosignal = 0; 1042 NWS_UNLOCK(nwsp); 1043 if (dosignal) 1044 NWS_SIGNAL(nwsp); 1045 error = 0; 1046 goto out_unpin; 1047 1048 queue_fallback: 1049 error = netisr_queue_internal(proto, m, cpuid); 1050 out_unpin: 1051 sched_unpin(); 1052 out_unlock: 1053 #ifdef NETISR_LOCKING 1054 NETISR_RUNLOCK(&tracker); 1055 #endif 1056 return (error); 1057 } 1058 1059 int 1060 netisr_dispatch(u_int proto, struct mbuf *m) 1061 { 1062 1063 return (netisr_dispatch_src(proto, 0, m)); 1064 } 1065 1066 #ifdef DEVICE_POLLING 1067 /* 1068 * Kernel polling borrows a netisr thread to run interface polling in; this 1069 * function allows kernel polling to request that the netisr thread be 1070 * scheduled even if no packets are pending for protocols. 1071 */ 1072 void 1073 netisr_sched_poll(void) 1074 { 1075 struct netisr_workstream *nwsp; 1076 1077 nwsp = DPCPU_ID_PTR(nws_array[0], nws); 1078 NWS_SIGNAL(nwsp); 1079 } 1080 #endif 1081 1082 static void 1083 netisr_start_swi(u_int cpuid, struct pcpu *pc) 1084 { 1085 char swiname[12]; 1086 struct netisr_workstream *nwsp; 1087 int error; 1088 1089 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 1090 1091 nwsp = DPCPU_ID_PTR(cpuid, nws); 1092 mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF); 1093 nwsp->nws_cpu = cpuid; 1094 snprintf(swiname, sizeof(swiname), "netisr %u", cpuid); 1095 error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, 1096 SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); 1097 if (error) 1098 panic("%s: swi_add %d", __func__, error); 1099 pc->pc_netisr = nwsp->nws_intr_event; 1100 if (netisr_bindthreads) { 1101 error = intr_event_bind(nwsp->nws_intr_event, cpuid); 1102 if (error != 0) 1103 printf("%s: cpu %u: intr_event_bind: %d", __func__, 1104 cpuid, error); 1105 } 1106 NETISR_WLOCK(); 1107 nws_array[nws_count] = nwsp->nws_cpu; 1108 nws_count++; 1109 NETISR_WUNLOCK(); 1110 } 1111 1112 /* 1113 * Initialize the netisr subsystem. We rely on BSS and static initialization 1114 * of most fields in global data structures. 1115 * 1116 * Start a worker thread for the boot CPU so that we can support network 1117 * traffic immediately in case the network stack is used before additional 1118 * CPUs are started (for example, diskless boot). 1119 */ 1120 static void 1121 netisr_init(void *arg) 1122 { 1123 char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 1124 u_int dispatch_policy; 1125 int error; 1126 1127 KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__)); 1128 1129 NETISR_LOCK_INIT(); 1130 if (netisr_maxthreads < 1) 1131 netisr_maxthreads = 1; 1132 if (netisr_maxthreads > mp_ncpus) { 1133 printf("netisr_init: forcing maxthreads from %d to %d\n", 1134 netisr_maxthreads, mp_ncpus); 1135 netisr_maxthreads = mp_ncpus; 1136 } 1137 if (netisr_defaultqlimit > netisr_maxqlimit) { 1138 printf("netisr_init: forcing defaultqlimit from %d to %d\n", 1139 netisr_defaultqlimit, netisr_maxqlimit); 1140 netisr_defaultqlimit = netisr_maxqlimit; 1141 } 1142 #ifdef DEVICE_POLLING 1143 /* 1144 * The device polling code is not yet aware of how to deal with 1145 * multiple netisr threads, so for the time being compiling in device 1146 * polling disables parallel netisr workers. 1147 */ 1148 if (netisr_maxthreads != 1 || netisr_bindthreads != 0) { 1149 printf("netisr_init: forcing maxthreads to 1 and " 1150 "bindthreads to 0 for device polling\n"); 1151 netisr_maxthreads = 1; 1152 netisr_bindthreads = 0; 1153 } 1154 #endif 1155 1156 if (TUNABLE_STR_FETCH("net.isr.dispatch", tmp, sizeof(tmp))) { 1157 error = netisr_dispatch_policy_from_str(tmp, 1158 &dispatch_policy); 1159 if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 1160 error = EINVAL; 1161 if (error == 0) 1162 netisr_dispatch_policy = dispatch_policy; 1163 else 1164 printf( 1165 "%s: invalid dispatch policy %s, using default\n", 1166 __func__, tmp); 1167 } 1168 1169 netisr_start_swi(curcpu, pcpu_find(curcpu)); 1170 } 1171 SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); 1172 1173 /* 1174 * Start worker threads for additional CPUs. No attempt to gracefully handle 1175 * work reassignment, we don't yet support dynamic reconfiguration. 1176 */ 1177 static void 1178 netisr_start(void *arg) 1179 { 1180 struct pcpu *pc; 1181 1182 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 1183 if (nws_count >= netisr_maxthreads) 1184 break; 1185 /* XXXRW: Is skipping absent CPUs still required here? */ 1186 if (CPU_ABSENT(pc->pc_cpuid)) 1187 continue; 1188 /* Worker will already be present for boot CPU. */ 1189 if (pc->pc_netisr != NULL) 1190 continue; 1191 netisr_start_swi(pc->pc_cpuid, pc); 1192 } 1193 } 1194 SYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL); 1195 1196 /* 1197 * Sysctl monitoring for netisr: query a list of registered protocols. 1198 */ 1199 static int 1200 sysctl_netisr_proto(SYSCTL_HANDLER_ARGS) 1201 { 1202 struct rm_priotracker tracker; 1203 struct sysctl_netisr_proto *snpp, *snp_array; 1204 struct netisr_proto *npp; 1205 u_int counter, proto; 1206 int error; 1207 1208 if (req->newptr != NULL) 1209 return (EINVAL); 1210 snp_array = malloc(sizeof(*snp_array) * NETISR_MAXPROT, M_TEMP, 1211 M_ZERO | M_WAITOK); 1212 counter = 0; 1213 NETISR_RLOCK(&tracker); 1214 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1215 npp = &netisr_proto[proto]; 1216 if (npp->np_name == NULL) 1217 continue; 1218 snpp = &snp_array[counter]; 1219 snpp->snp_version = sizeof(*snpp); 1220 strlcpy(snpp->snp_name, npp->np_name, NETISR_NAMEMAXLEN); 1221 snpp->snp_proto = proto; 1222 snpp->snp_qlimit = npp->np_qlimit; 1223 snpp->snp_policy = npp->np_policy; 1224 snpp->snp_dispatch = npp->np_dispatch; 1225 if (npp->np_m2flow != NULL) 1226 snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW; 1227 if (npp->np_m2cpuid != NULL) 1228 snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID; 1229 if (npp->np_drainedcpu != NULL) 1230 snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU; 1231 counter++; 1232 } 1233 NETISR_RUNLOCK(&tracker); 1234 KASSERT(counter <= NETISR_MAXPROT, 1235 ("sysctl_netisr_proto: counter too big (%d)", counter)); 1236 error = SYSCTL_OUT(req, snp_array, sizeof(*snp_array) * counter); 1237 free(snp_array, M_TEMP); 1238 return (error); 1239 } 1240 1241 SYSCTL_PROC(_net_isr, OID_AUTO, proto, 1242 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_proto, 1243 "S,sysctl_netisr_proto", 1244 "Return list of protocols registered with netisr"); 1245 1246 /* 1247 * Sysctl monitoring for netisr: query a list of workstreams. 1248 */ 1249 static int 1250 sysctl_netisr_workstream(SYSCTL_HANDLER_ARGS) 1251 { 1252 struct rm_priotracker tracker; 1253 struct sysctl_netisr_workstream *snwsp, *snws_array; 1254 struct netisr_workstream *nwsp; 1255 u_int counter, cpuid; 1256 int error; 1257 1258 if (req->newptr != NULL) 1259 return (EINVAL); 1260 snws_array = malloc(sizeof(*snws_array) * MAXCPU, M_TEMP, 1261 M_ZERO | M_WAITOK); 1262 counter = 0; 1263 NETISR_RLOCK(&tracker); 1264 CPU_FOREACH(cpuid) { 1265 nwsp = DPCPU_ID_PTR(cpuid, nws); 1266 if (nwsp->nws_intr_event == NULL) 1267 continue; 1268 NWS_LOCK(nwsp); 1269 snwsp = &snws_array[counter]; 1270 snwsp->snws_version = sizeof(*snwsp); 1271 1272 /* 1273 * For now, we equate workstream IDs and CPU IDs in the 1274 * kernel, but expose them independently to userspace in case 1275 * that assumption changes in the future. 1276 */ 1277 snwsp->snws_wsid = cpuid; 1278 snwsp->snws_cpu = cpuid; 1279 if (nwsp->nws_intr_event != NULL) 1280 snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR; 1281 NWS_UNLOCK(nwsp); 1282 counter++; 1283 } 1284 NETISR_RUNLOCK(&tracker); 1285 KASSERT(counter <= MAXCPU, 1286 ("sysctl_netisr_workstream: counter too big (%d)", counter)); 1287 error = SYSCTL_OUT(req, snws_array, sizeof(*snws_array) * counter); 1288 free(snws_array, M_TEMP); 1289 return (error); 1290 } 1291 1292 SYSCTL_PROC(_net_isr, OID_AUTO, workstream, 1293 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_workstream, 1294 "S,sysctl_netisr_workstream", 1295 "Return list of workstreams implemented by netisr"); 1296 1297 /* 1298 * Sysctl monitoring for netisr: query per-protocol data across all 1299 * workstreams. 1300 */ 1301 static int 1302 sysctl_netisr_work(SYSCTL_HANDLER_ARGS) 1303 { 1304 struct rm_priotracker tracker; 1305 struct sysctl_netisr_work *snwp, *snw_array; 1306 struct netisr_workstream *nwsp; 1307 struct netisr_proto *npp; 1308 struct netisr_work *nwp; 1309 u_int counter, cpuid, proto; 1310 int error; 1311 1312 if (req->newptr != NULL) 1313 return (EINVAL); 1314 snw_array = malloc(sizeof(*snw_array) * MAXCPU * NETISR_MAXPROT, 1315 M_TEMP, M_ZERO | M_WAITOK); 1316 counter = 0; 1317 NETISR_RLOCK(&tracker); 1318 CPU_FOREACH(cpuid) { 1319 nwsp = DPCPU_ID_PTR(cpuid, nws); 1320 if (nwsp->nws_intr_event == NULL) 1321 continue; 1322 NWS_LOCK(nwsp); 1323 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1324 npp = &netisr_proto[proto]; 1325 if (npp->np_name == NULL) 1326 continue; 1327 nwp = &nwsp->nws_work[proto]; 1328 snwp = &snw_array[counter]; 1329 snwp->snw_version = sizeof(*snwp); 1330 snwp->snw_wsid = cpuid; /* See comment above. */ 1331 snwp->snw_proto = proto; 1332 snwp->snw_len = nwp->nw_len; 1333 snwp->snw_watermark = nwp->nw_watermark; 1334 snwp->snw_dispatched = nwp->nw_dispatched; 1335 snwp->snw_hybrid_dispatched = 1336 nwp->nw_hybrid_dispatched; 1337 snwp->snw_qdrops = nwp->nw_qdrops; 1338 snwp->snw_queued = nwp->nw_queued; 1339 snwp->snw_handled = nwp->nw_handled; 1340 counter++; 1341 } 1342 NWS_UNLOCK(nwsp); 1343 } 1344 KASSERT(counter <= MAXCPU * NETISR_MAXPROT, 1345 ("sysctl_netisr_work: counter too big (%d)", counter)); 1346 NETISR_RUNLOCK(&tracker); 1347 error = SYSCTL_OUT(req, snw_array, sizeof(*snw_array) * counter); 1348 free(snw_array, M_TEMP); 1349 return (error); 1350 } 1351 1352 SYSCTL_PROC(_net_isr, OID_AUTO, work, 1353 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_work, 1354 "S,sysctl_netisr_work", 1355 "Return list of per-workstream, per-protocol work in netisr"); 1356 1357 #ifdef DDB 1358 DB_SHOW_COMMAND(netisr, db_show_netisr) 1359 { 1360 struct netisr_workstream *nwsp; 1361 struct netisr_work *nwp; 1362 int first, proto; 1363 u_int cpuid; 1364 1365 db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto", 1366 "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue"); 1367 CPU_FOREACH(cpuid) { 1368 nwsp = DPCPU_ID_PTR(cpuid, nws); 1369 if (nwsp->nws_intr_event == NULL) 1370 continue; 1371 first = 1; 1372 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1373 if (netisr_proto[proto].np_handler == NULL) 1374 continue; 1375 nwp = &nwsp->nws_work[proto]; 1376 if (first) { 1377 db_printf("%3d ", cpuid); 1378 first = 0; 1379 } else 1380 db_printf("%3s ", ""); 1381 db_printf( 1382 "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n", 1383 netisr_proto[proto].np_name, nwp->nw_len, 1384 nwp->nw_watermark, nwp->nw_qlimit, 1385 nwp->nw_dispatched, nwp->nw_hybrid_dispatched, 1386 nwp->nw_qdrops, nwp->nw_queued); 1387 } 1388 } 1389 } 1390 #endif 1391