1 /*- 2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org> 3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/rtprio.h> 33 #include <sys/systm.h> 34 #include <sys/interrupt.h> 35 #include <sys/kernel.h> 36 #include <sys/kthread.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/proc.h> 40 #include <sys/random.h> 41 #include <sys/resourcevar.h> 42 #include <sys/sysctl.h> 43 #include <sys/unistd.h> 44 #include <machine/atomic.h> 45 #include <machine/cpu.h> 46 #include <machine/stdarg.h> 47 48 #include <sys/mbuf.h> 49 #include <sys/socket.h> 50 51 #include <net/if.h> 52 #include <net/if_types.h> 53 #include <net/if_var.h> 54 #include <net/netisr.h> 55 56 /* 57 * debug_mpsafenet controls network subsystem-wide use of the Giant lock, 58 * from system calls down to interrupt handlers. It can be changed only 59 * via a tunable at boot, not at run-time, due to the complexity of 60 * unwinding. 61 */ 62 int debug_mpsafenet = 0; 63 TUNABLE_INT("debug.mpsafenet", &debug_mpsafenet); 64 SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0, 65 "Enable/disable MPSAFE network support"); 66 67 volatile unsigned int netisr; /* scheduling bits for network */ 68 69 struct netisr { 70 netisr_t *ni_handler; 71 struct ifqueue *ni_queue; 72 int ni_flags; 73 } netisrs[32]; 74 75 static void *net_ih; 76 77 void 78 legacy_setsoftnet(void) 79 { 80 swi_sched(net_ih, 0); 81 } 82 83 void 84 netisr_register(int num, netisr_t *handler, struct ifqueue *inq, int flags) 85 { 86 87 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 88 ("bad isr %d", num)); 89 netisrs[num].ni_handler = handler; 90 netisrs[num].ni_queue = inq; 91 if ((flags & NETISR_MPSAFE) && !debug_mpsafenet) 92 flags &= ~NETISR_MPSAFE; 93 netisrs[num].ni_flags = flags; 94 } 95 96 void 97 netisr_unregister(int num) 98 { 99 struct netisr *ni; 100 101 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 102 ("bad isr %d", num)); 103 ni = &netisrs[num]; 104 ni->ni_handler = NULL; 105 if (ni->ni_queue != NULL) 106 IF_DRAIN(ni->ni_queue); 107 } 108 109 struct isrstat { 110 int isrs_count; /* dispatch count */ 111 int isrs_directed; /* ...directly dispatched */ 112 int isrs_deferred; /* ...queued instead */ 113 int isrs_queued; /* intentionally queueued */ 114 int isrs_drop; /* dropped 'cuz no handler */ 115 int isrs_swi_count; /* swi_net handlers called */ 116 }; 117 static struct isrstat isrstat; 118 119 SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters"); 120 121 static int netisr_enable = 0; 122 SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW, 123 &netisr_enable, 0, "enable direct dispatch"); 124 TUNABLE_INT("net.isr.enable", &netisr_enable); 125 126 SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD, 127 &isrstat.isrs_count, 0, ""); 128 SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD, 129 &isrstat.isrs_directed, 0, ""); 130 SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD, 131 &isrstat.isrs_deferred, 0, ""); 132 SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD, 133 &isrstat.isrs_queued, 0, ""); 134 SYSCTL_INT(_net_isr, OID_AUTO, drop, CTLFLAG_RD, 135 &isrstat.isrs_drop, 0, ""); 136 SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD, 137 &isrstat.isrs_swi_count, 0, ""); 138 139 /* 140 * Process all packets currently present in a netisr queue. Used to 141 * drain an existing set of packets waiting for processing when we 142 * begin direct dispatch, to avoid processing packets out of order. 143 */ 144 static void 145 netisr_processqueue(struct netisr *ni) 146 { 147 struct mbuf *m; 148 149 for (;;) { 150 IF_DEQUEUE(ni->ni_queue, m); 151 if (m == NULL) 152 break; 153 ni->ni_handler(m); 154 } 155 } 156 157 /* 158 * Call the netisr directly instead of queueing the packet, if possible. 159 */ 160 void 161 netisr_dispatch(int num, struct mbuf *m) 162 { 163 struct netisr *ni; 164 165 isrstat.isrs_count++; /* XXX redundant */ 166 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 167 ("bad isr %d", num)); 168 ni = &netisrs[num]; 169 if (ni->ni_queue == NULL) { 170 isrstat.isrs_drop++; 171 m_freem(m); 172 return; 173 } 174 /* 175 * Do direct dispatch only for MPSAFE netisrs (and 176 * only when enabled). Note that when a netisr is 177 * marked MPSAFE we permit multiple concurrent instances 178 * to run. We guarantee only the order in which 179 * packets are processed for each "dispatch point" in 180 * the system (i.e. call to netisr_dispatch or 181 * netisr_queue). This insures ordering of packets 182 * from an interface but does not guarantee ordering 183 * between multiple places in the system (e.g. IP 184 * dispatched from interfaces vs. IP queued from IPSec). 185 */ 186 if (netisr_enable && (ni->ni_flags & NETISR_MPSAFE)) { 187 isrstat.isrs_directed++; 188 /* 189 * NB: We used to drain the queue before handling 190 * the packet but now do not. Doing so here will 191 * not preserve ordering so instead we fallback to 192 * guaranteeing order only from dispatch points 193 * in the system (see above). 194 */ 195 ni->ni_handler(m); 196 } else { 197 isrstat.isrs_deferred++; 198 if (IF_HANDOFF(ni->ni_queue, m, NULL)) 199 schednetisr(num); 200 } 201 } 202 203 /* 204 * Same as above, but always queue. 205 * This is either used in places where we are not confident that 206 * direct dispatch is possible, or where queueing is required. 207 */ 208 int 209 netisr_queue(int num, struct mbuf *m) 210 { 211 struct netisr *ni; 212 213 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 214 ("bad isr %d", num)); 215 ni = &netisrs[num]; 216 if (ni->ni_queue == NULL) { 217 isrstat.isrs_drop++; 218 m_freem(m); 219 return (1); 220 } 221 isrstat.isrs_queued++; 222 if (!IF_HANDOFF(ni->ni_queue, m, NULL)) 223 return (0); 224 schednetisr(num); 225 return (1); 226 } 227 228 static void 229 swi_net(void *dummy) 230 { 231 struct netisr *ni; 232 u_int bits; 233 int i; 234 #ifdef DEVICE_POLLING 235 const int polling = 1; 236 #else 237 const int polling = 0; 238 #endif 239 240 do { 241 bits = atomic_readandclear_int(&netisr); 242 if (bits == 0) 243 break; 244 while ((i = ffs(bits)) != 0) { 245 isrstat.isrs_swi_count++; 246 i--; 247 bits &= ~(1 << i); 248 ni = &netisrs[i]; 249 if (ni->ni_handler == NULL) { 250 printf("swi_net: unregistered isr %d.\n", i); 251 continue; 252 } 253 if ((ni->ni_flags & NETISR_MPSAFE) == 0) { 254 mtx_lock(&Giant); 255 if (ni->ni_queue == NULL) 256 ni->ni_handler(NULL); 257 else 258 netisr_processqueue(ni); 259 mtx_unlock(&Giant); 260 } else { 261 if (ni->ni_queue == NULL) 262 ni->ni_handler(NULL); 263 else 264 netisr_processqueue(ni); 265 } 266 } 267 } while (polling); 268 } 269 270 static void 271 start_netisr(void *dummy) 272 { 273 274 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, INTR_MPSAFE, &net_ih)) 275 panic("start_netisr"); 276 } 277 SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL) 278