1e4fc250cSLuigi Rizzo /*- 2daccb638SLuigi Rizzo * Copyright (c) 2001-2002 Luigi Rizzo 3daccb638SLuigi Rizzo * 4daccb638SLuigi Rizzo * Supported by: the Xorp Project (www.xorp.org) 5e4fc250cSLuigi Rizzo * 6e4fc250cSLuigi Rizzo * Redistribution and use in source and binary forms, with or without 7e4fc250cSLuigi Rizzo * modification, are permitted provided that the following conditions 8e4fc250cSLuigi Rizzo * are met: 9e4fc250cSLuigi Rizzo * 1. Redistributions of source code must retain the above copyright 10e4fc250cSLuigi Rizzo * notice, this list of conditions and the following disclaimer. 11e4fc250cSLuigi Rizzo * 2. Redistributions in binary form must reproduce the above copyright 12e4fc250cSLuigi Rizzo * notice, this list of conditions and the following disclaimer in the 13e4fc250cSLuigi Rizzo * documentation and/or other materials provided with the distribution. 14e4fc250cSLuigi Rizzo * 15e4fc250cSLuigi Rizzo * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16e4fc250cSLuigi Rizzo * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17e4fc250cSLuigi Rizzo * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18e4fc250cSLuigi Rizzo * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19e4fc250cSLuigi Rizzo * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20e4fc250cSLuigi Rizzo * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21e4fc250cSLuigi Rizzo * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22e4fc250cSLuigi Rizzo * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23e4fc250cSLuigi Rizzo * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24e4fc250cSLuigi Rizzo * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25e4fc250cSLuigi Rizzo * SUCH DAMAGE. 26e4fc250cSLuigi Rizzo */ 27e4fc250cSLuigi Rizzo 28677b542eSDavid E. O'Brien #include <sys/cdefs.h> 29677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 30677b542eSDavid E. O'Brien 31e4fc250cSLuigi Rizzo #include <sys/param.h> 32e4fc250cSLuigi Rizzo #include <sys/systm.h> 33e4fc250cSLuigi Rizzo #include <sys/kernel.h> 34e4fc250cSLuigi Rizzo #include <sys/socket.h> /* needed by net/if.h */ 35e4fc250cSLuigi Rizzo #include <sys/sysctl.h> 3616901c01SGleb Smirnoff #include <sys/syslog.h> 37e4fc250cSLuigi Rizzo 38e4fc250cSLuigi Rizzo #include <net/if.h> /* for IFF_* flags */ 39e4fc250cSLuigi Rizzo #include <net/netisr.h> /* for NETISR_POLL */ 40e4fc250cSLuigi Rizzo 41d105c784SLuigi Rizzo #include <sys/proc.h> 42d105c784SLuigi Rizzo #include <sys/resourcevar.h> 43d105c784SLuigi Rizzo #include <sys/kthread.h> 44d105c784SLuigi Rizzo 45daccb638SLuigi Rizzo static void netisr_poll(void); /* the two netisr handlers */ 461cafed39SJonathan Lemon static void netisr_pollmore(void); 47daccb638SLuigi Rizzo 48daccb638SLuigi Rizzo void hardclock_device_poll(void); /* hook from hardclock */ 49e4fc250cSLuigi Rizzo void ether_poll(int); /* polling while in trap */ 50e4fc250cSLuigi Rizzo 51e4fc250cSLuigi Rizzo /* 52e4fc250cSLuigi Rizzo * Polling support for [network] device drivers. 53e4fc250cSLuigi Rizzo * 54e4fc250cSLuigi Rizzo * Drivers which support this feature try to register with the 55e4fc250cSLuigi Rizzo * polling code. 56e4fc250cSLuigi Rizzo * 57e4fc250cSLuigi Rizzo * If registration is successful, the driver must disable interrupts, 58e4fc250cSLuigi Rizzo * and further I/O is performed through the handler, which is invoked 59e4fc250cSLuigi Rizzo * (at least once per clock tick) with 3 arguments: the "arg" passed at 60e4fc250cSLuigi Rizzo * register time (a struct ifnet pointer), a command, and a "count" limit. 61e4fc250cSLuigi Rizzo * 62e4fc250cSLuigi Rizzo * The command can be one of the following: 63e4fc250cSLuigi Rizzo * POLL_ONLY: quick move of "count" packets from input/output queues. 64e4fc250cSLuigi Rizzo * POLL_AND_CHECK_STATUS: as above, plus check status registers or do 65e4fc250cSLuigi Rizzo * other more expensive operations. This command is issued periodically 66e4fc250cSLuigi Rizzo * but less frequently than POLL_ONLY. 67e4fc250cSLuigi Rizzo * POLL_DEREGISTER: deregister and return to interrupt mode. 68e4fc250cSLuigi Rizzo * 69e4fc250cSLuigi Rizzo * The first two commands are only issued if the interface is marked as 7013f4c340SRobert Watson * 'IFF_UP and IFF_DRV_RUNNING', the last one only if IFF_DRV_RUNNING is set. 71e4fc250cSLuigi Rizzo * 72e4fc250cSLuigi Rizzo * The count limit specifies how much work the handler can do during the 73e4fc250cSLuigi Rizzo * call -- typically this is the number of packets to be received, or 74e4fc250cSLuigi Rizzo * transmitted, etc. (drivers are free to interpret this number, as long 75e4fc250cSLuigi Rizzo * as the max time spent in the function grows roughly linearly with the 76e4fc250cSLuigi Rizzo * count). 77e4fc250cSLuigi Rizzo * 78e4fc250cSLuigi Rizzo * Deregistration can be requested by the driver itself (typically in the 79e4fc250cSLuigi Rizzo * *_stop() routine), or by the polling code, by invoking the handler. 80e4fc250cSLuigi Rizzo * 81e4fc250cSLuigi Rizzo * Polling can be globally enabled or disabled with the sysctl variable 82e4fc250cSLuigi Rizzo * kern.polling.enable (default is 0, disabled) 83e4fc250cSLuigi Rizzo * 84e4fc250cSLuigi Rizzo * A second variable controls the sharing of CPU between polling/kernel 85e4fc250cSLuigi Rizzo * network processing, and other activities (typically userlevel tasks): 86e4fc250cSLuigi Rizzo * kern.polling.user_frac (between 0 and 100, default 50) sets the share 87e4fc250cSLuigi Rizzo * of CPU allocated to user tasks. CPU is allocated proportionally to the 88e4fc250cSLuigi Rizzo * shares, by dynamically adjusting the "count" (poll_burst). 89e4fc250cSLuigi Rizzo * 90e4fc250cSLuigi Rizzo * Other parameters can should be left to their default values. 91e4fc250cSLuigi Rizzo * The following constraints hold 92e4fc250cSLuigi Rizzo * 93e4fc250cSLuigi Rizzo * 1 <= poll_each_burst <= poll_burst <= poll_burst_max 94e4fc250cSLuigi Rizzo * 0 <= poll_in_trap <= poll_each_burst 95e4fc250cSLuigi Rizzo * MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX 96e4fc250cSLuigi Rizzo */ 97e4fc250cSLuigi Rizzo 98e4fc250cSLuigi Rizzo #define MIN_POLL_BURST_MAX 10 99e4fc250cSLuigi Rizzo #define MAX_POLL_BURST_MAX 1000 100e4fc250cSLuigi Rizzo 101e4fc250cSLuigi Rizzo SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0, 102e4fc250cSLuigi Rizzo "Device polling parameters"); 103e4fc250cSLuigi Rizzo 104e4fc250cSLuigi Rizzo static u_int32_t poll_burst = 5; 105daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, burst, CTLFLAG_RW, 106e4fc250cSLuigi Rizzo &poll_burst, 0, "Current polling burst size"); 107e4fc250cSLuigi Rizzo 108e4fc250cSLuigi Rizzo static u_int32_t poll_each_burst = 5; 109daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, each_burst, CTLFLAG_RW, 110e4fc250cSLuigi Rizzo &poll_each_burst, 0, "Max size of each burst"); 111e4fc250cSLuigi Rizzo 112e4fc250cSLuigi Rizzo static u_int32_t poll_burst_max = 150; /* good for 100Mbit net and HZ=1000 */ 113daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, burst_max, CTLFLAG_RW, 114e4fc250cSLuigi Rizzo &poll_burst_max, 0, "Max Polling burst size"); 115e4fc250cSLuigi Rizzo 1166c240564SSam Leffler static u_int32_t poll_in_idle_loop=0; /* do we poll in idle loop ? */ 117daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, idle_poll, CTLFLAG_RW, 118daccb638SLuigi Rizzo &poll_in_idle_loop, 0, "Enable device polling in idle loop"); 119daccb638SLuigi Rizzo 120e4fc250cSLuigi Rizzo u_int32_t poll_in_trap; /* used in trap.c */ 121daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, poll_in_trap, CTLFLAG_RW, 122e4fc250cSLuigi Rizzo &poll_in_trap, 0, "Poll burst size during a trap"); 123e4fc250cSLuigi Rizzo 124e4fc250cSLuigi Rizzo static u_int32_t user_frac = 50; 125daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, user_frac, CTLFLAG_RW, 126e4fc250cSLuigi Rizzo &user_frac, 0, "Desired user fraction of cpu time"); 127e4fc250cSLuigi Rizzo 128e4fc250cSLuigi Rizzo static u_int32_t reg_frac = 20 ; 129daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, reg_frac, CTLFLAG_RW, 130e4fc250cSLuigi Rizzo ®_frac, 0, "Every this many cycles poll register"); 131e4fc250cSLuigi Rizzo 132e4fc250cSLuigi Rizzo static u_int32_t short_ticks; 133daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, short_ticks, CTLFLAG_RW, 134e4fc250cSLuigi Rizzo &short_ticks, 0, "Hardclock ticks shorter than they should be"); 135e4fc250cSLuigi Rizzo 136e4fc250cSLuigi Rizzo static u_int32_t lost_polls; 137daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, lost_polls, CTLFLAG_RW, 138e4fc250cSLuigi Rizzo &lost_polls, 0, "How many times we would have lost a poll tick"); 139e4fc250cSLuigi Rizzo 140daccb638SLuigi Rizzo static u_int32_t pending_polls; 141daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, pending_polls, CTLFLAG_RW, 142daccb638SLuigi Rizzo &pending_polls, 0, "Do we need to poll again"); 143daccb638SLuigi Rizzo 144daccb638SLuigi Rizzo static int residual_burst = 0; 145daccb638SLuigi Rizzo SYSCTL_INT(_kern_polling, OID_AUTO, residual_burst, CTLFLAG_RW, 146daccb638SLuigi Rizzo &residual_burst, 0, "# of residual cycles in burst"); 147daccb638SLuigi Rizzo 148e4fc250cSLuigi Rizzo static u_int32_t poll_handlers; /* next free entry in pr[]. */ 149daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, handlers, CTLFLAG_RD, 150e4fc250cSLuigi Rizzo &poll_handlers, 0, "Number of registered poll handlers"); 151e4fc250cSLuigi Rizzo 152e4fc250cSLuigi Rizzo static int polling = 0; /* global polling enable */ 153daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, enable, CTLFLAG_RW, 154e4fc250cSLuigi Rizzo &polling, 0, "Polling enabled"); 155e4fc250cSLuigi Rizzo 1562dbd9d5bSLuigi Rizzo static u_int32_t phase; 157daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, phase, CTLFLAG_RW, 158daccb638SLuigi Rizzo &phase, 0, "Polling phase"); 159e4fc250cSLuigi Rizzo 160daccb638SLuigi Rizzo static u_int32_t suspect; 161daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, suspect, CTLFLAG_RW, 162daccb638SLuigi Rizzo &suspect, 0, "suspect event"); 163daccb638SLuigi Rizzo 1642dbd9d5bSLuigi Rizzo static u_int32_t stalled; 165daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, stalled, CTLFLAG_RW, 166daccb638SLuigi Rizzo &stalled, 0, "potential stalls"); 167daccb638SLuigi Rizzo 168daccb638SLuigi Rizzo static u_int32_t idlepoll_sleeping; /* idlepoll is sleeping */ 169daccb638SLuigi Rizzo SYSCTL_UINT(_kern_polling, OID_AUTO, idlepoll_sleeping, CTLFLAG_RD, 170daccb638SLuigi Rizzo &idlepoll_sleeping, 0, "idlepoll is sleeping"); 171daccb638SLuigi Rizzo 172e4fc250cSLuigi Rizzo 173e4fc250cSLuigi Rizzo #define POLL_LIST_LEN 128 174e4fc250cSLuigi Rizzo struct pollrec { 175e4fc250cSLuigi Rizzo poll_handler_t *handler; 176e4fc250cSLuigi Rizzo struct ifnet *ifp; 17716901c01SGleb Smirnoff #define PRF_RUNNING 0x1 17816901c01SGleb Smirnoff #define PRF_LEAVING 0x2 17916901c01SGleb Smirnoff uint32_t flags; 180e4fc250cSLuigi Rizzo }; 181e4fc250cSLuigi Rizzo 182e4fc250cSLuigi Rizzo static struct pollrec pr[POLL_LIST_LEN]; 183e4fc250cSLuigi Rizzo 18416901c01SGleb Smirnoff #define PR_VALID(i) (pr[(i)].handler != NULL && \ 18516901c01SGleb Smirnoff !(pr[(i)].flags & (PRF_RUNNING|PRF_LEAVING)) && \ 18616901c01SGleb Smirnoff (pr[(i)].ifp->if_drv_flags & IFF_DRV_RUNNING) &&\ 18716901c01SGleb Smirnoff (pr[(i)].ifp->if_flags & IFF_UP)) 18816901c01SGleb Smirnoff 18916901c01SGleb Smirnoff static struct mtx poll_mtx; 19016901c01SGleb Smirnoff 1911cafed39SJonathan Lemon static void 192daccb638SLuigi Rizzo init_device_poll(void) 193daccb638SLuigi Rizzo { 1941cafed39SJonathan Lemon 19516901c01SGleb Smirnoff mtx_init(&poll_mtx, "polling", NULL, MTX_DEF); 19616901c01SGleb Smirnoff netisr_register(NETISR_POLL, (netisr_t *)netisr_poll, NULL, 19716901c01SGleb Smirnoff NETISR_MPSAFE); 19816901c01SGleb Smirnoff netisr_register(NETISR_POLLMORE, (netisr_t *)netisr_pollmore, NULL, 19916901c01SGleb Smirnoff NETISR_MPSAFE); 200daccb638SLuigi Rizzo } 2011cafed39SJonathan Lemon SYSINIT(device_poll, SI_SUB_CLOCKS, SI_ORDER_MIDDLE, init_device_poll, NULL) 2021cafed39SJonathan Lemon 203daccb638SLuigi Rizzo 204daccb638SLuigi Rizzo /* 205e4fc250cSLuigi Rizzo * Hook from hardclock. Tries to schedule a netisr, but keeps track 206e4fc250cSLuigi Rizzo * of lost ticks due to the previous handler taking too long. 207d26d355fSLuigi Rizzo * Normally, this should not happen, because polling handler should 208d26d355fSLuigi Rizzo * run for a short time. However, in some cases (e.g. when there are 209d26d355fSLuigi Rizzo * changes in link status etc.) the drivers take a very long time 210d26d355fSLuigi Rizzo * (even in the order of milliseconds) to reset and reconfigure the 211d26d355fSLuigi Rizzo * device, causing apparent lost polls. 212d26d355fSLuigi Rizzo * 213e4fc250cSLuigi Rizzo * The first part of the code is just for debugging purposes, and tries 214e4fc250cSLuigi Rizzo * to count how often hardclock ticks are shorter than they should, 215e4fc250cSLuigi Rizzo * meaning either stray interrupts or delayed events. 216e4fc250cSLuigi Rizzo */ 217e4fc250cSLuigi Rizzo void 218e4fc250cSLuigi Rizzo hardclock_device_poll(void) 219e4fc250cSLuigi Rizzo { 220e4fc250cSLuigi Rizzo static struct timeval prev_t, t; 221e4fc250cSLuigi Rizzo int delta; 222e4fc250cSLuigi Rizzo 223daccb638SLuigi Rizzo if (poll_handlers == 0) 224daccb638SLuigi Rizzo return; 225daccb638SLuigi Rizzo 226e4fc250cSLuigi Rizzo microuptime(&t); 227e4fc250cSLuigi Rizzo delta = (t.tv_usec - prev_t.tv_usec) + 228e4fc250cSLuigi Rizzo (t.tv_sec - prev_t.tv_sec)*1000000; 229e4fc250cSLuigi Rizzo if (delta * hz < 500000) 230e4fc250cSLuigi Rizzo short_ticks++; 231e4fc250cSLuigi Rizzo else 232e4fc250cSLuigi Rizzo prev_t = t; 233e4fc250cSLuigi Rizzo 234daccb638SLuigi Rizzo if (pending_polls > 100) { 235d26d355fSLuigi Rizzo /* 236d26d355fSLuigi Rizzo * Too much, assume it has stalled (not always true 237d26d355fSLuigi Rizzo * see comment above). 238d26d355fSLuigi Rizzo */ 239daccb638SLuigi Rizzo stalled++; 240daccb638SLuigi Rizzo pending_polls = 0; 241daccb638SLuigi Rizzo phase = 0; 242daccb638SLuigi Rizzo } 243daccb638SLuigi Rizzo 244daccb638SLuigi Rizzo if (phase <= 2) { 245daccb638SLuigi Rizzo if (phase != 0) 246daccb638SLuigi Rizzo suspect++; 247daccb638SLuigi Rizzo phase = 1; 2481cafed39SJonathan Lemon schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); 249daccb638SLuigi Rizzo phase = 2; 250e4fc250cSLuigi Rizzo } 251daccb638SLuigi Rizzo if (pending_polls++ > 0) 252daccb638SLuigi Rizzo lost_polls++; 253e4fc250cSLuigi Rizzo } 254e4fc250cSLuigi Rizzo 255e4fc250cSLuigi Rizzo /* 256e4fc250cSLuigi Rizzo * ether_poll is called from the idle loop or from the trap handler. 257e4fc250cSLuigi Rizzo */ 258e4fc250cSLuigi Rizzo void 259e4fc250cSLuigi Rizzo ether_poll(int count) 260e4fc250cSLuigi Rizzo { 261e4fc250cSLuigi Rizzo int i; 262e4fc250cSLuigi Rizzo 26316901c01SGleb Smirnoff mtx_lock(&poll_mtx); 264e4fc250cSLuigi Rizzo 265e4fc250cSLuigi Rizzo if (count > poll_each_burst) 266e4fc250cSLuigi Rizzo count = poll_each_burst; 26716901c01SGleb Smirnoff 26816901c01SGleb Smirnoff for (i = 0 ; i < poll_handlers ; i++) { 26916901c01SGleb Smirnoff if (PR_VALID(i)) { 27016901c01SGleb Smirnoff pr[i].flags |= PRF_RUNNING; 27116901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 27216901c01SGleb Smirnoff NET_LOCK_GIANT(); 27316901c01SGleb Smirnoff pr[i].handler(pr[i].ifp, POLL_ONLY, count); 27416901c01SGleb Smirnoff NET_UNLOCK_GIANT(); 27516901c01SGleb Smirnoff mtx_lock(&poll_mtx); 27616901c01SGleb Smirnoff pr[i].flags &= ~PRF_RUNNING; 27716901c01SGleb Smirnoff } 27816901c01SGleb Smirnoff } 27916901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 280e4fc250cSLuigi Rizzo } 281e4fc250cSLuigi Rizzo 282e4fc250cSLuigi Rizzo /* 283daccb638SLuigi Rizzo * netisr_pollmore is called after other netisr's, possibly scheduling 284e4fc250cSLuigi Rizzo * another NETISR_POLL call, or adapting the burst size for the next cycle. 285e4fc250cSLuigi Rizzo * 286e4fc250cSLuigi Rizzo * It is very bad to fetch large bursts of packets from a single card at once, 287e4fc250cSLuigi Rizzo * because the burst could take a long time to be completely processed, or 288e4fc250cSLuigi Rizzo * could saturate the intermediate queue (ipintrq or similar) leading to 289e4fc250cSLuigi Rizzo * losses or unfairness. To reduce the problem, and also to account better for 290daccb638SLuigi Rizzo * time spent in network-related processing, we split the burst in smaller 291e4fc250cSLuigi Rizzo * chunks of fixed size, giving control to the other netisr's between chunks. 292e4fc250cSLuigi Rizzo * This helps in improving the fairness, reducing livelock (because we 293e4fc250cSLuigi Rizzo * emulate more closely the "process to completion" that we have with 294e4fc250cSLuigi Rizzo * fastforwarding) and accounting for the work performed in low level 295e4fc250cSLuigi Rizzo * handling and forwarding. 296e4fc250cSLuigi Rizzo */ 297e4fc250cSLuigi Rizzo 298e4fc250cSLuigi Rizzo static struct timeval poll_start_t; 299e4fc250cSLuigi Rizzo 300e4fc250cSLuigi Rizzo void 301daccb638SLuigi Rizzo netisr_pollmore() 302e4fc250cSLuigi Rizzo { 303e4fc250cSLuigi Rizzo struct timeval t; 304e4fc250cSLuigi Rizzo int kern_load; 305e4fc250cSLuigi Rizzo 30616901c01SGleb Smirnoff NET_ASSERT_GIANT(); 30716901c01SGleb Smirnoff 30816901c01SGleb Smirnoff mtx_lock(&poll_mtx); 309daccb638SLuigi Rizzo phase = 5; 310e4fc250cSLuigi Rizzo if (residual_burst > 0) { 3111cafed39SJonathan Lemon schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); 31216901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 313e4fc250cSLuigi Rizzo /* will run immediately on return, followed by netisrs */ 314e4fc250cSLuigi Rizzo return; 315e4fc250cSLuigi Rizzo } 316e4fc250cSLuigi Rizzo /* here we can account time spent in netisr's in this tick */ 317e4fc250cSLuigi Rizzo microuptime(&t); 318e4fc250cSLuigi Rizzo kern_load = (t.tv_usec - poll_start_t.tv_usec) + 319e4fc250cSLuigi Rizzo (t.tv_sec - poll_start_t.tv_sec)*1000000; /* us */ 320e4fc250cSLuigi Rizzo kern_load = (kern_load * hz) / 10000; /* 0..100 */ 321e4fc250cSLuigi Rizzo if (kern_load > (100 - user_frac)) { /* try decrease ticks */ 322e4fc250cSLuigi Rizzo if (poll_burst > 1) 323e4fc250cSLuigi Rizzo poll_burst--; 324e4fc250cSLuigi Rizzo } else { 325e4fc250cSLuigi Rizzo if (poll_burst < poll_burst_max) 326e4fc250cSLuigi Rizzo poll_burst++; 327e4fc250cSLuigi Rizzo } 328e4fc250cSLuigi Rizzo 329daccb638SLuigi Rizzo pending_polls--; 330daccb638SLuigi Rizzo if (pending_polls == 0) /* we are done */ 331daccb638SLuigi Rizzo phase = 0; 332daccb638SLuigi Rizzo else { 333e4fc250cSLuigi Rizzo /* 334e4fc250cSLuigi Rizzo * Last cycle was long and caused us to miss one or more 335daccb638SLuigi Rizzo * hardclock ticks. Restart processing again, but slightly 336e4fc250cSLuigi Rizzo * reduce the burst size to prevent that this happens again. 337e4fc250cSLuigi Rizzo */ 338e4fc250cSLuigi Rizzo poll_burst -= (poll_burst / 8); 339e4fc250cSLuigi Rizzo if (poll_burst < 1) 340e4fc250cSLuigi Rizzo poll_burst = 1; 3411cafed39SJonathan Lemon schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); 342daccb638SLuigi Rizzo phase = 6; 343daccb638SLuigi Rizzo } 34416901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 345e4fc250cSLuigi Rizzo } 346e4fc250cSLuigi Rizzo 347e4fc250cSLuigi Rizzo /* 348daccb638SLuigi Rizzo * netisr_poll is scheduled by schednetisr when appropriate, typically once 34916901c01SGleb Smirnoff * per tick. 350e4fc250cSLuigi Rizzo */ 351daccb638SLuigi Rizzo static void 352daccb638SLuigi Rizzo netisr_poll(void) 353e4fc250cSLuigi Rizzo { 354e4fc250cSLuigi Rizzo static int reg_frac_count; 355e4fc250cSLuigi Rizzo int i, cycles; 356e4fc250cSLuigi Rizzo enum poll_cmd arg = POLL_ONLY; 357e4fc250cSLuigi Rizzo 35816901c01SGleb Smirnoff NET_ASSERT_GIANT(); 35916901c01SGleb Smirnoff 36016901c01SGleb Smirnoff mtx_lock(&poll_mtx); 361daccb638SLuigi Rizzo phase = 3; 362e4fc250cSLuigi Rizzo if (residual_burst == 0) { /* first call in this tick */ 363e4fc250cSLuigi Rizzo microuptime(&poll_start_t); 364e4fc250cSLuigi Rizzo /* 365e4fc250cSLuigi Rizzo * Check that paremeters are consistent with runtime 366e4fc250cSLuigi Rizzo * variables. Some of these tests could be done at sysctl 367e4fc250cSLuigi Rizzo * time, but the savings would be very limited because we 368e4fc250cSLuigi Rizzo * still have to check against reg_frac_count and 369e4fc250cSLuigi Rizzo * poll_each_burst. So, instead of writing separate sysctl 370e4fc250cSLuigi Rizzo * handlers, we do all here. 371e4fc250cSLuigi Rizzo */ 372e4fc250cSLuigi Rizzo 373e4fc250cSLuigi Rizzo if (reg_frac > hz) 374e4fc250cSLuigi Rizzo reg_frac = hz; 375e4fc250cSLuigi Rizzo else if (reg_frac < 1) 376e4fc250cSLuigi Rizzo reg_frac = 1; 377e4fc250cSLuigi Rizzo if (reg_frac_count > reg_frac) 378e4fc250cSLuigi Rizzo reg_frac_count = reg_frac - 1; 379e4fc250cSLuigi Rizzo if (reg_frac_count-- == 0) { 380e4fc250cSLuigi Rizzo arg = POLL_AND_CHECK_STATUS; 381e4fc250cSLuigi Rizzo reg_frac_count = reg_frac - 1; 382e4fc250cSLuigi Rizzo } 383e4fc250cSLuigi Rizzo if (poll_burst_max < MIN_POLL_BURST_MAX) 384e4fc250cSLuigi Rizzo poll_burst_max = MIN_POLL_BURST_MAX; 385e4fc250cSLuigi Rizzo else if (poll_burst_max > MAX_POLL_BURST_MAX) 386e4fc250cSLuigi Rizzo poll_burst_max = MAX_POLL_BURST_MAX; 387e4fc250cSLuigi Rizzo 388e4fc250cSLuigi Rizzo if (poll_each_burst < 1) 389e4fc250cSLuigi Rizzo poll_each_burst = 1; 390e4fc250cSLuigi Rizzo else if (poll_each_burst > poll_burst_max) 391e4fc250cSLuigi Rizzo poll_each_burst = poll_burst_max; 392e4fc250cSLuigi Rizzo 39361f7581dSRuslan Ermilov if (poll_burst > poll_burst_max) 39461f7581dSRuslan Ermilov poll_burst = poll_burst_max; 395e4fc250cSLuigi Rizzo residual_burst = poll_burst; 396e4fc250cSLuigi Rizzo } 397e4fc250cSLuigi Rizzo cycles = (residual_burst < poll_each_burst) ? 398e4fc250cSLuigi Rizzo residual_burst : poll_each_burst; 399e4fc250cSLuigi Rizzo residual_burst -= cycles; 400e4fc250cSLuigi Rizzo 401e4fc250cSLuigi Rizzo if (polling) { 40216901c01SGleb Smirnoff for (i = 0 ; i < poll_handlers ; i++) { 40316901c01SGleb Smirnoff if (PR_VALID(i)) { 40416901c01SGleb Smirnoff pr[i].flags |= PRF_RUNNING; 40516901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 406e4fc250cSLuigi Rizzo pr[i].handler(pr[i].ifp, arg, cycles); 40716901c01SGleb Smirnoff mtx_lock(&poll_mtx); 40816901c01SGleb Smirnoff pr[i].flags &= ~PRF_RUNNING; 40916901c01SGleb Smirnoff } 41016901c01SGleb Smirnoff } 411e4fc250cSLuigi Rizzo } else { /* unregister */ 412e4fc250cSLuigi Rizzo for (i = 0 ; i < poll_handlers ; i++) { 41316901c01SGleb Smirnoff if (pr[i].handler != NULL && 41413f4c340SRobert Watson pr[i].ifp->if_drv_flags & IFF_DRV_RUNNING) { 41562f76486SMaxim Sobolev pr[i].ifp->if_flags &= ~IFF_POLLING; 41616901c01SGleb Smirnoff pr[i].flags |= PRF_LEAVING; 41716901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 418e4fc250cSLuigi Rizzo pr[i].handler(pr[i].ifp, POLL_DEREGISTER, 1); 41916901c01SGleb Smirnoff mtx_lock(&poll_mtx); 42016901c01SGleb Smirnoff pr[i].flags &= ~PRF_LEAVING; 421e4fc250cSLuigi Rizzo } 422e4fc250cSLuigi Rizzo pr[i].handler = NULL; 423e4fc250cSLuigi Rizzo } 424e4fc250cSLuigi Rizzo residual_burst = 0; 425e4fc250cSLuigi Rizzo poll_handlers = 0; 426e4fc250cSLuigi Rizzo } 42716901c01SGleb Smirnoff 428daccb638SLuigi Rizzo phase = 4; 42916901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 430e4fc250cSLuigi Rizzo } 431e4fc250cSLuigi Rizzo 432e4fc250cSLuigi Rizzo /* 433e4fc250cSLuigi Rizzo * Try to register routine for polling. Returns 1 if successful 434e4fc250cSLuigi Rizzo * (and polling should be enabled), 0 otherwise. 435e4fc250cSLuigi Rizzo * A device is not supposed to register itself multiple times. 436e4fc250cSLuigi Rizzo * 437daccb638SLuigi Rizzo * This is called from within the *_intr() functions, so we do not need 43816901c01SGleb Smirnoff * further ifnet locking. 439e4fc250cSLuigi Rizzo */ 440e4fc250cSLuigi Rizzo int 441e4fc250cSLuigi Rizzo ether_poll_register(poll_handler_t *h, struct ifnet *ifp) 442e4fc250cSLuigi Rizzo { 44316901c01SGleb Smirnoff int i; 44416901c01SGleb Smirnoff 44516901c01SGleb Smirnoff NET_ASSERT_GIANT(); 446e4fc250cSLuigi Rizzo 447e4fc250cSLuigi Rizzo if (polling == 0) /* polling disabled, cannot register */ 448e4fc250cSLuigi Rizzo return 0; 449e4fc250cSLuigi Rizzo if (h == NULL || ifp == NULL) /* bad arguments */ 450e4fc250cSLuigi Rizzo return 0; 451e4fc250cSLuigi Rizzo if ( !(ifp->if_flags & IFF_UP) ) /* must be up */ 452e4fc250cSLuigi Rizzo return 0; 45362f76486SMaxim Sobolev if (ifp->if_flags & IFF_POLLING) /* already polling */ 454e4fc250cSLuigi Rizzo return 0; 455e4fc250cSLuigi Rizzo 45616901c01SGleb Smirnoff mtx_lock(&poll_mtx); 457e4fc250cSLuigi Rizzo if (poll_handlers >= POLL_LIST_LEN) { 458e4fc250cSLuigi Rizzo /* 459e4fc250cSLuigi Rizzo * List full, cannot register more entries. 460e4fc250cSLuigi Rizzo * This should never happen; if it does, it is probably a 461e4fc250cSLuigi Rizzo * broken driver trying to register multiple times. Checking 462e4fc250cSLuigi Rizzo * this at runtime is expensive, and won't solve the problem 463e4fc250cSLuigi Rizzo * anyways, so just report a few times and then give up. 464e4fc250cSLuigi Rizzo */ 465e4fc250cSLuigi Rizzo static int verbose = 10 ; 466e4fc250cSLuigi Rizzo if (verbose >0) { 46716901c01SGleb Smirnoff log(LOG_ERR, "poll handlers list full, " 468e4fc250cSLuigi Rizzo "maybe a broken driver ?\n"); 469e4fc250cSLuigi Rizzo verbose--; 470e4fc250cSLuigi Rizzo } 47116901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 472e4fc250cSLuigi Rizzo return 0; /* no polling for you */ 473e4fc250cSLuigi Rizzo } 474e4fc250cSLuigi Rizzo 47516901c01SGleb Smirnoff for (i = 0 ; i < poll_handlers ; i++) 47616901c01SGleb Smirnoff if (pr[i].ifp == ifp && pr[i].handler != NULL) { 47716901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 47816901c01SGleb Smirnoff log(LOG_DEBUG, "ether_poll_register: %s: handler" 47916901c01SGleb Smirnoff " already registered\n", ifp->if_xname); 48016901c01SGleb Smirnoff return (0); 48116901c01SGleb Smirnoff } 48216901c01SGleb Smirnoff 483e4fc250cSLuigi Rizzo pr[poll_handlers].handler = h; 484e4fc250cSLuigi Rizzo pr[poll_handlers].ifp = ifp; 485e4fc250cSLuigi Rizzo poll_handlers++; 48662f76486SMaxim Sobolev ifp->if_flags |= IFF_POLLING; 48716901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 488d105c784SLuigi Rizzo if (idlepoll_sleeping) 489d105c784SLuigi Rizzo wakeup(&idlepoll_sleeping); 490e4fc250cSLuigi Rizzo return 1; /* polling enabled in next call */ 491e4fc250cSLuigi Rizzo } 492e4fc250cSLuigi Rizzo 493e4fc250cSLuigi Rizzo /* 494daccb638SLuigi Rizzo * Remove interface from the polling list. Normally called by *_stop(). 495daccb638SLuigi Rizzo * It is not an error to call it with IFF_POLLING clear, the call is 496daccb638SLuigi Rizzo * sufficiently rare to be preferable to save the space for the extra 497daccb638SLuigi Rizzo * test in each driver in exchange of one additional function call. 498e4fc250cSLuigi Rizzo */ 499e4fc250cSLuigi Rizzo int 500e4fc250cSLuigi Rizzo ether_poll_deregister(struct ifnet *ifp) 501e4fc250cSLuigi Rizzo { 502e4fc250cSLuigi Rizzo int i; 503e4fc250cSLuigi Rizzo 50416901c01SGleb Smirnoff NET_ASSERT_GIANT(); 50516901c01SGleb Smirnoff 50662f76486SMaxim Sobolev if ( !ifp || !(ifp->if_flags & IFF_POLLING) ) { 507e4fc250cSLuigi Rizzo return 0; 508e4fc250cSLuigi Rizzo } 50916901c01SGleb Smirnoff mtx_lock(&poll_mtx); 510e4fc250cSLuigi Rizzo for (i = 0 ; i < poll_handlers ; i++) 511e4fc250cSLuigi Rizzo if (pr[i].ifp == ifp) /* found it */ 512e4fc250cSLuigi Rizzo break; 51362f76486SMaxim Sobolev ifp->if_flags &= ~IFF_POLLING; /* found or not... */ 514e4fc250cSLuigi Rizzo if (i == poll_handlers) { 51516901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 51616901c01SGleb Smirnoff log(LOG_DEBUG, "ether_poll_deregister: %s: not found!\n", 51716901c01SGleb Smirnoff ifp->if_xname); 51816901c01SGleb Smirnoff return (0); 519e4fc250cSLuigi Rizzo } 520e4fc250cSLuigi Rizzo poll_handlers--; 521e4fc250cSLuigi Rizzo if (i < poll_handlers) { /* Last entry replaces this one. */ 522e4fc250cSLuigi Rizzo pr[i].handler = pr[poll_handlers].handler; 523e4fc250cSLuigi Rizzo pr[i].ifp = pr[poll_handlers].ifp; 524e4fc250cSLuigi Rizzo } 52516901c01SGleb Smirnoff mtx_unlock(&poll_mtx); 52616901c01SGleb Smirnoff return (1); 527e4fc250cSLuigi Rizzo } 528d105c784SLuigi Rizzo 529d105c784SLuigi Rizzo static void 530d105c784SLuigi Rizzo poll_idle(void) 531d105c784SLuigi Rizzo { 532d105c784SLuigi Rizzo struct thread *td = curthread; 533d105c784SLuigi Rizzo struct rtprio rtp; 534d105c784SLuigi Rizzo int pri; 535d105c784SLuigi Rizzo 536d105c784SLuigi Rizzo rtp.prio = RTP_PRIO_MAX; /* lowest priority */ 537d105c784SLuigi Rizzo rtp.type = RTP_PRIO_IDLE; 538d105c784SLuigi Rizzo mtx_lock_spin(&sched_lock); 539e5223044SLuigi Rizzo rtp_to_pri(&rtp, td->td_ksegrp); 5402c100766SJulian Elischer pri = td->td_priority; 541d105c784SLuigi Rizzo mtx_unlock_spin(&sched_lock); 542d105c784SLuigi Rizzo 543d105c784SLuigi Rizzo for (;;) { 544daccb638SLuigi Rizzo if (poll_in_idle_loop && poll_handlers > 0) { 545d105c784SLuigi Rizzo idlepoll_sleeping = 0; 546d105c784SLuigi Rizzo ether_poll(poll_each_burst); 547d105c784SLuigi Rizzo mtx_lock_spin(&sched_lock); 548b5cbda50SJohn Baldwin mi_switch(SW_VOL, NULL); 549d105c784SLuigi Rizzo mtx_unlock_spin(&sched_lock); 550d105c784SLuigi Rizzo } else { 551d105c784SLuigi Rizzo idlepoll_sleeping = 1; 552d105c784SLuigi Rizzo tsleep(&idlepoll_sleeping, pri, "pollid", hz * 3); 553d105c784SLuigi Rizzo } 554d105c784SLuigi Rizzo } 555d105c784SLuigi Rizzo } 556d105c784SLuigi Rizzo 557d105c784SLuigi Rizzo static struct proc *idlepoll; 558d105c784SLuigi Rizzo static struct kproc_desc idlepoll_kp = { 559d105c784SLuigi Rizzo "idlepoll", 560d105c784SLuigi Rizzo poll_idle, 561d105c784SLuigi Rizzo &idlepoll 562d105c784SLuigi Rizzo }; 563d105c784SLuigi Rizzo SYSINIT(idlepoll, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, kproc_start, &idlepoll_kp) 564