1 /*- 2 * Copyright (c) 2001 Luigi Rizzo 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/socket.h> /* needed by net/if.h */ 32 #include <sys/sysctl.h> 33 34 #include <net/if.h> /* for IFF_* flags */ 35 #include <net/netisr.h> /* for NETISR_POLL */ 36 37 #include <sys/proc.h> 38 #include <sys/resourcevar.h> 39 #include <sys/kthread.h> 40 41 #ifdef SMP 42 #error DEVICE_POLLING is not compatible with SMP 43 #endif 44 45 void ether_poll1(void); 46 void ether_poll(int); /* polling while in trap */ 47 void ether_pollmore(void); 48 void hardclock_device_poll(void); 49 50 /* 51 * Polling support for [network] device drivers. 52 * 53 * Drivers which support this feature try to register with the 54 * polling code. 55 * 56 * If registration is successful, the driver must disable interrupts, 57 * and further I/O is performed through the handler, which is invoked 58 * (at least once per clock tick) with 3 arguments: the "arg" passed at 59 * register time (a struct ifnet pointer), a command, and a "count" limit. 60 * 61 * The command can be one of the following: 62 * POLL_ONLY: quick move of "count" packets from input/output queues. 63 * POLL_AND_CHECK_STATUS: as above, plus check status registers or do 64 * other more expensive operations. This command is issued periodically 65 * but less frequently than POLL_ONLY. 66 * POLL_DEREGISTER: deregister and return to interrupt mode. 67 * 68 * The first two commands are only issued if the interface is marked as 69 * 'IFF_UP and IFF_RUNNING', the last one only if IFF_RUNNING is set. 70 * 71 * The count limit specifies how much work the handler can do during the 72 * call -- typically this is the number of packets to be received, or 73 * transmitted, etc. (drivers are free to interpret this number, as long 74 * as the max time spent in the function grows roughly linearly with the 75 * count). 76 * 77 * Deregistration can be requested by the driver itself (typically in the 78 * *_stop() routine), or by the polling code, by invoking the handler. 79 * 80 * Polling can be globally enabled or disabled with the sysctl variable 81 * kern.polling.enable (default is 0, disabled) 82 * 83 * A second variable controls the sharing of CPU between polling/kernel 84 * network processing, and other activities (typically userlevel tasks): 85 * kern.polling.user_frac (between 0 and 100, default 50) sets the share 86 * of CPU allocated to user tasks. CPU is allocated proportionally to the 87 * shares, by dynamically adjusting the "count" (poll_burst). 88 * 89 * Other parameters can should be left to their default values. 90 * The following constraints hold 91 * 92 * 1 <= poll_each_burst <= poll_burst <= poll_burst_max 93 * 0 <= poll_in_trap <= poll_each_burst 94 * MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX 95 */ 96 97 #define MIN_POLL_BURST_MAX 10 98 #define MAX_POLL_BURST_MAX 1000 99 100 SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0, 101 "Device polling parameters"); 102 103 static u_int32_t poll_burst = 5; 104 SYSCTL_ULONG(_kern_polling, OID_AUTO, burst, CTLFLAG_RW, 105 &poll_burst, 0, "Current polling burst size"); 106 107 static u_int32_t poll_each_burst = 5; 108 SYSCTL_ULONG(_kern_polling, OID_AUTO, each_burst, CTLFLAG_RW, 109 &poll_each_burst, 0, "Max size of each burst"); 110 111 static u_int32_t poll_burst_max = 150; /* good for 100Mbit net and HZ=1000 */ 112 SYSCTL_ULONG(_kern_polling, OID_AUTO, burst_max, CTLFLAG_RW, 113 &poll_burst_max, 0, "Max Polling burst size"); 114 115 u_int32_t poll_in_trap; /* used in trap.c */ 116 SYSCTL_ULONG(_kern_polling, OID_AUTO, poll_in_trap, CTLFLAG_RW, 117 &poll_in_trap, 0, "Poll burst size during a trap"); 118 119 static u_int32_t user_frac = 50; 120 SYSCTL_ULONG(_kern_polling, OID_AUTO, user_frac, CTLFLAG_RW, 121 &user_frac, 0, "Desired user fraction of cpu time"); 122 123 static u_int32_t reg_frac = 20 ; 124 SYSCTL_ULONG(_kern_polling, OID_AUTO, reg_frac, CTLFLAG_RW, 125 ®_frac, 0, "Every this many cycles poll register"); 126 127 static u_int32_t short_ticks; 128 SYSCTL_ULONG(_kern_polling, OID_AUTO, short_ticks, CTLFLAG_RW, 129 &short_ticks, 0, "Hardclock ticks shorter than they should be"); 130 131 static u_int32_t lost_polls; 132 SYSCTL_ULONG(_kern_polling, OID_AUTO, lost_polls, CTLFLAG_RW, 133 &lost_polls, 0, "How many times we would have lost a poll tick"); 134 135 static u_int32_t poll_handlers; /* next free entry in pr[]. */ 136 SYSCTL_ULONG(_kern_polling, OID_AUTO, handlers, CTLFLAG_RD, 137 &poll_handlers, 0, "Number of registered poll handlers"); 138 139 static u_int32_t poll_in_idle=1; /* boolean */ 140 SYSCTL_ULONG(_kern_polling, OID_AUTO, poll_in_idle, CTLFLAG_RW, 141 &poll_in_idle, 0, "Poll during idle loop"); 142 143 static u_int32_t idlepoll_sleeping; /* idlepoll is sleeping */ 144 SYSCTL_ULONG(_kern_polling, OID_AUTO, idlepoll_sleeping, CTLFLAG_RD, 145 &idlepoll_sleeping, 0, "idlepoll is sleeping"); 146 147 static int polling = 0; /* global polling enable */ 148 SYSCTL_ULONG(_kern_polling, OID_AUTO, enable, CTLFLAG_RW, 149 &polling, 0, "Polling enabled"); 150 151 152 static u_int32_t poll1_active; 153 static u_int32_t need_poll_again; 154 155 #define POLL_LIST_LEN 128 156 struct pollrec { 157 poll_handler_t *handler; 158 struct ifnet *ifp; 159 }; 160 161 static struct pollrec pr[POLL_LIST_LEN]; 162 163 /* 164 * Hook from hardclock. Tries to schedule a netisr, but keeps track 165 * of lost ticks due to the previous handler taking too long. 166 * The first part of the code is just for debugging purposes, and tries 167 * to count how often hardclock ticks are shorter than they should, 168 * meaning either stray interrupts or delayed events. 169 */ 170 void 171 hardclock_device_poll(void) 172 { 173 static struct timeval prev_t, t; 174 int delta; 175 176 microuptime(&t); 177 delta = (t.tv_usec - prev_t.tv_usec) + 178 (t.tv_sec - prev_t.tv_sec)*1000000; 179 if (delta * hz < 500000) 180 short_ticks++; 181 else 182 prev_t = t; 183 184 if (poll_handlers > 0) { 185 if (poll1_active) { 186 lost_polls++; 187 need_poll_again++; 188 } else { 189 poll1_active = 1; 190 schednetisr(NETISR_POLL); 191 } 192 } 193 } 194 195 /* 196 * ether_poll is called from the idle loop or from the trap handler. 197 */ 198 void 199 ether_poll(int count) 200 { 201 int i; 202 int s = splimp(); 203 204 mtx_lock(&Giant); 205 206 if (count > poll_each_burst) 207 count = poll_each_burst; 208 for (i = 0 ; i < poll_handlers ; i++) 209 if (pr[i].handler && (IFF_UP|IFF_RUNNING) == 210 (pr[i].ifp->if_flags & (IFF_UP|IFF_RUNNING)) ) 211 pr[i].handler(pr[i].ifp, 0, count); /* quick check */ 212 mtx_unlock(&Giant); 213 splx(s); 214 } 215 216 /* 217 * ether_pollmore is called after other netisr's, possibly scheduling 218 * another NETISR_POLL call, or adapting the burst size for the next cycle. 219 * 220 * It is very bad to fetch large bursts of packets from a single card at once, 221 * because the burst could take a long time to be completely processed, or 222 * could saturate the intermediate queue (ipintrq or similar) leading to 223 * losses or unfairness. To reduce the problem, and also to account better for 224 * time spent in network-related processnig, we split the burst in smaller 225 * chunks of fixed size, giving control to the other netisr's between chunks. 226 * This helps in improving the fairness, reducing livelock (because we 227 * emulate more closely the "process to completion" that we have with 228 * fastforwarding) and accounting for the work performed in low level 229 * handling and forwarding. 230 */ 231 232 static int residual_burst = 0; 233 234 static struct timeval poll_start_t; 235 236 void 237 ether_pollmore() 238 { 239 struct timeval t; 240 int kern_load; 241 int s = splhigh(); 242 243 if (residual_burst > 0) { 244 schednetisr(NETISR_POLL); 245 /* will run immediately on return, followed by netisrs */ 246 splx(s); 247 return ; 248 } 249 /* here we can account time spent in netisr's in this tick */ 250 microuptime(&t); 251 kern_load = (t.tv_usec - poll_start_t.tv_usec) + 252 (t.tv_sec - poll_start_t.tv_sec)*1000000; /* us */ 253 kern_load = (kern_load * hz) / 10000; /* 0..100 */ 254 if (kern_load > (100 - user_frac)) { /* try decrease ticks */ 255 if (poll_burst > 1) 256 poll_burst--; 257 } else { 258 if (poll_burst < poll_burst_max) 259 poll_burst++; 260 } 261 262 if (need_poll_again) { 263 /* 264 * Last cycle was long and caused us to miss one or more 265 * hardclock ticks. Restart processnig again, but slightly 266 * reduce the burst size to prevent that this happens again. 267 */ 268 need_poll_again--; 269 poll_burst -= (poll_burst / 8); 270 if (poll_burst < 1) 271 poll_burst = 1; 272 schednetisr(NETISR_POLL); 273 } else 274 poll1_active = 0; 275 splx(s); 276 } 277 278 /* 279 * ether_poll1 is called by schednetisr when appropriate, typically once 280 * per tick. It is called at splnet() so first thing to do is to upgrade to 281 * splimp(), and call all registered handlers. 282 */ 283 void 284 ether_poll1(void) 285 { 286 static int reg_frac_count; 287 int i, cycles; 288 enum poll_cmd arg = POLL_ONLY; 289 int s=splimp(); 290 mtx_lock(&Giant); 291 292 if (residual_burst == 0) { /* first call in this tick */ 293 microuptime(&poll_start_t); 294 /* 295 * Check that paremeters are consistent with runtime 296 * variables. Some of these tests could be done at sysctl 297 * time, but the savings would be very limited because we 298 * still have to check against reg_frac_count and 299 * poll_each_burst. So, instead of writing separate sysctl 300 * handlers, we do all here. 301 */ 302 303 if (reg_frac > hz) 304 reg_frac = hz; 305 else if (reg_frac < 1) 306 reg_frac = 1; 307 if (reg_frac_count > reg_frac) 308 reg_frac_count = reg_frac - 1; 309 if (reg_frac_count-- == 0) { 310 arg = POLL_AND_CHECK_STATUS; 311 reg_frac_count = reg_frac - 1; 312 } 313 if (poll_burst_max < MIN_POLL_BURST_MAX) 314 poll_burst_max = MIN_POLL_BURST_MAX; 315 else if (poll_burst_max > MAX_POLL_BURST_MAX) 316 poll_burst_max = MAX_POLL_BURST_MAX; 317 318 if (poll_each_burst < 1) 319 poll_each_burst = 1; 320 else if (poll_each_burst > poll_burst_max) 321 poll_each_burst = poll_burst_max; 322 323 residual_burst = poll_burst; 324 } 325 cycles = (residual_burst < poll_each_burst) ? 326 residual_burst : poll_each_burst; 327 residual_burst -= cycles; 328 329 if (polling) { 330 for (i = 0 ; i < poll_handlers ; i++) 331 if (pr[i].handler && (IFF_UP|IFF_RUNNING) == 332 (pr[i].ifp->if_flags & (IFF_UP|IFF_RUNNING)) ) 333 pr[i].handler(pr[i].ifp, arg, cycles); 334 } else { /* unregister */ 335 for (i = 0 ; i < poll_handlers ; i++) { 336 if (pr[i].handler && 337 pr[i].ifp->if_flags & IFF_RUNNING) { 338 pr[i].ifp->if_ipending &= ~IFF_POLLING; 339 pr[i].handler(pr[i].ifp, POLL_DEREGISTER, 1); 340 } 341 pr[i].handler=NULL; 342 } 343 residual_burst = 0; 344 poll_handlers = 0; 345 } 346 /* on -stable, schednetisr(NETISR_POLLMORE); */ 347 mtx_unlock(&Giant); 348 splx(s); 349 } 350 351 /* 352 * Try to register routine for polling. Returns 1 if successful 353 * (and polling should be enabled), 0 otherwise. 354 * A device is not supposed to register itself multiple times. 355 * 356 * This is called from within the *_intr() function, so we should 357 * probably not need further locking. XXX 358 */ 359 int 360 ether_poll_register(poll_handler_t *h, struct ifnet *ifp) 361 { 362 int s; 363 364 if (polling == 0) /* polling disabled, cannot register */ 365 return 0; 366 if (h == NULL || ifp == NULL) /* bad arguments */ 367 return 0; 368 if ( !(ifp->if_flags & IFF_UP) ) /* must be up */ 369 return 0; 370 if (ifp->if_ipending & IFF_POLLING) /* already polling */ 371 return 0; 372 373 s = splhigh(); 374 if (poll_handlers >= POLL_LIST_LEN) { 375 /* 376 * List full, cannot register more entries. 377 * This should never happen; if it does, it is probably a 378 * broken driver trying to register multiple times. Checking 379 * this at runtime is expensive, and won't solve the problem 380 * anyways, so just report a few times and then give up. 381 */ 382 static int verbose = 10 ; 383 splx(s); 384 if (verbose >0) { 385 printf("poll handlers list full, " 386 "maybe a broken driver ?\n"); 387 verbose--; 388 } 389 return 0; /* no polling for you */ 390 } 391 392 pr[poll_handlers].handler = h; 393 pr[poll_handlers].ifp = ifp; 394 poll_handlers++; 395 ifp->if_ipending |= IFF_POLLING; 396 splx(s); 397 if (idlepoll_sleeping) 398 wakeup(&idlepoll_sleeping); 399 return 1; /* polling enabled in next call */ 400 } 401 402 /* 403 * Remove the interface from the list of polling ones. 404 * Normally run by *_stop(). 405 * We allow it being called with IFF_POLLING clear, the 406 * call is sufficiently rare so it is preferable to save the 407 * space for the extra test in each device in exchange of one 408 * additional function call. 409 */ 410 int 411 ether_poll_deregister(struct ifnet *ifp) 412 { 413 int i; 414 415 mtx_lock(&Giant); 416 if ( !ifp || !(ifp->if_ipending & IFF_POLLING) ) { 417 mtx_unlock(&Giant); 418 return 0; 419 } 420 for (i = 0 ; i < poll_handlers ; i++) 421 if (pr[i].ifp == ifp) /* found it */ 422 break; 423 ifp->if_ipending &= ~IFF_POLLING; /* found or not... */ 424 if (i == poll_handlers) { 425 mtx_unlock(&Giant); 426 printf("ether_poll_deregister: ifp not found!!!\n"); 427 return 0; 428 } 429 poll_handlers--; 430 if (i < poll_handlers) { /* Last entry replaces this one. */ 431 pr[i].handler = pr[poll_handlers].handler; 432 pr[i].ifp = pr[poll_handlers].ifp; 433 } 434 mtx_unlock(&Giant); 435 return 1; 436 } 437 438 static void 439 poll_idle(void) 440 { 441 struct thread *td = curthread; 442 struct rtprio rtp; 443 int pri; 444 445 rtp.prio = RTP_PRIO_MAX; /* lowest priority */ 446 rtp.type = RTP_PRIO_IDLE; 447 mtx_lock_spin(&sched_lock); 448 rtp_to_pri(&rtp, &td->td_ksegrp->kg_pri); 449 pri = td->td_ksegrp->kg_pri.pri_level; 450 mtx_unlock_spin(&sched_lock); 451 452 for (;;) { 453 if (poll_in_idle && poll_handlers > 0) { 454 idlepoll_sleeping = 0; 455 mtx_lock(&Giant); 456 ether_poll(poll_each_burst); 457 mtx_unlock(&Giant); 458 mtx_assert(&Giant, MA_NOTOWNED); 459 mtx_lock_spin(&sched_lock); 460 setrunqueue(td); 461 td->td_proc->p_stats->p_ru.ru_nvcsw++; 462 mi_switch(); 463 mtx_unlock_spin(&sched_lock); 464 } else { 465 idlepoll_sleeping = 1; 466 tsleep(&idlepoll_sleeping, pri, "pollid", hz * 3); 467 } 468 } 469 } 470 471 static struct proc *idlepoll; 472 static struct kproc_desc idlepoll_kp = { 473 "idlepoll", 474 poll_idle, 475 &idlepoll 476 }; 477 SYSINIT(idlepoll, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, kproc_start, &idlepoll_kp) 478