1 /*- 2 * Copyright (c) 1982, 1986, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * (c) UNIX System Laboratories, Inc. 5 * All or some portions of this file are derived from material licensed 6 * to the University of California by American Telephone and Telegraph 7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 8 * the permission of UNIX System Laboratories, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the University of 21 * California, Berkeley and its contributors. 22 * 4. Neither the name of the University nor the names of its contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 39 * $FreeBSD$ 40 */ 41 42 #include "opt_ntp.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/dkstat.h> 47 #include <sys/callout.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/ktr.h> 51 #include <sys/mutex.h> 52 #include <sys/proc.h> 53 #include <sys/resourcevar.h> 54 #include <sys/signalvar.h> 55 #include <sys/smp.h> 56 #include <sys/timetc.h> 57 #include <sys/timepps.h> 58 #include <vm/vm.h> 59 #include <vm/pmap.h> 60 #include <vm/vm_map.h> 61 #include <sys/sysctl.h> 62 #include <sys/bus.h> 63 #include <sys/interrupt.h> 64 65 #include <machine/cpu.h> 66 #include <machine/limits.h> 67 68 #ifdef GPROF 69 #include <sys/gmon.h> 70 #endif 71 72 #ifdef DEVICE_POLLING 73 #include <net/netisr.h> /* for NETISR_POLL */ 74 75 extern void ether_poll1(void); 76 extern void hardclock_device_poll(void); 77 #endif /* DEVICE_POLLING */ 78 79 static void initclocks __P((void *dummy)); 80 SYSINIT(clocks, SI_SUB_CLOCKS, SI_ORDER_FIRST, initclocks, NULL) 81 82 /* Some of these don't belong here, but it's easiest to concentrate them. */ 83 long cp_time[CPUSTATES]; 84 85 SYSCTL_OPAQUE(_kern, OID_AUTO, cp_time, CTLFLAG_RD, &cp_time, sizeof(cp_time), 86 "LU", "CPU time statistics"); 87 88 long tk_cancc; 89 long tk_nin; 90 long tk_nout; 91 long tk_rawcc; 92 93 /* 94 * Clock handling routines. 95 * 96 * This code is written to operate with two timers that run independently of 97 * each other. 98 * 99 * The main timer, running hz times per second, is used to trigger interval 100 * timers, timeouts and rescheduling as needed. 101 * 102 * The second timer handles kernel and user profiling, 103 * and does resource use estimation. If the second timer is programmable, 104 * it is randomized to avoid aliasing between the two clocks. For example, 105 * the randomization prevents an adversary from always giving up the cpu 106 * just before its quantum expires. Otherwise, it would never accumulate 107 * cpu ticks. The mean frequency of the second timer is stathz. 108 * 109 * If no second timer exists, stathz will be zero; in this case we drive 110 * profiling and statistics off the main clock. This WILL NOT be accurate; 111 * do not do it unless absolutely necessary. 112 * 113 * The statistics clock may (or may not) be run at a higher rate while 114 * profiling. This profile clock runs at profhz. We require that profhz 115 * be an integral multiple of stathz. 116 * 117 * If the statistics clock is running fast, it must be divided by the ratio 118 * profhz/stathz for statistics. (For profiling, every tick counts.) 119 * 120 * Time-of-day is maintained using a "timecounter", which may or may 121 * not be related to the hardware generating the above mentioned 122 * interrupts. 123 */ 124 125 int stathz; 126 int profhz; 127 static int profprocs; 128 int ticks; 129 static int psdiv, pscnt; /* prof => stat divider */ 130 int psratio; /* ratio: prof / stat */ 131 132 /* 133 * Initialize clock frequencies and start both clocks running. 134 */ 135 /* ARGSUSED*/ 136 static void 137 initclocks(dummy) 138 void *dummy; 139 { 140 register int i; 141 142 /* 143 * Set divisors to 1 (normal case) and let the machine-specific 144 * code do its bit. 145 */ 146 psdiv = pscnt = 1; 147 cpu_initclocks(); 148 149 #ifdef DEVICE_POLLING 150 register_netisr(NETISR_POLL, ether_poll1); 151 #endif 152 /* 153 * Compute profhz/stathz, and fix profhz if needed. 154 */ 155 i = stathz ? stathz : hz; 156 if (profhz == 0) 157 profhz = i; 158 psratio = profhz / i; 159 } 160 161 /* 162 * Each time the real-time timer fires, this function is called on all CPUs 163 * with each CPU passing in its curthread as the first argument. If possible 164 * a nice optimization in the future would be to allow the CPU receiving the 165 * actual real-time timer interrupt to call this function on behalf of the 166 * other CPUs rather than sending an IPI to all other CPUs so that they 167 * can call this function. Note that hardclock() calls hardclock_process() 168 * for the CPU receiving the timer interrupt, so only the other CPUs in the 169 * system need to call this function (or have it called on their behalf. 170 */ 171 void 172 hardclock_process(td, user) 173 struct thread *td; 174 int user; 175 { 176 struct pstats *pstats; 177 struct proc *p = td->td_proc; 178 179 /* 180 * Run current process's virtual and profile time, as needed. 181 */ 182 mtx_assert(&sched_lock, MA_OWNED); 183 if (p->p_flag & P_KSES) { 184 /* XXXKSE What to do? */ 185 } else { 186 pstats = p->p_stats; 187 if (user && 188 timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && 189 itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { 190 p->p_sflag |= PS_ALRMPEND; 191 td->td_kse->ke_flags |= KEF_ASTPENDING; 192 } 193 if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) && 194 itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { 195 p->p_sflag |= PS_PROFPEND; 196 td->td_kse->ke_flags |= KEF_ASTPENDING; 197 } 198 } 199 } 200 201 /* 202 * The real-time timer, interrupting hz times per second. 203 */ 204 void 205 hardclock(frame) 206 register struct clockframe *frame; 207 { 208 int need_softclock = 0; 209 210 CTR0(KTR_INTR, "hardclock fired"); 211 mtx_lock_spin_flags(&sched_lock, MTX_QUIET); 212 hardclock_process(curthread, CLKF_USERMODE(frame)); 213 mtx_unlock_spin_flags(&sched_lock, MTX_QUIET); 214 215 /* 216 * If no separate statistics clock is available, run it from here. 217 * 218 * XXX: this only works for UP 219 */ 220 if (stathz == 0) 221 statclock(frame); 222 223 tc_windup(); 224 #ifdef DEVICE_POLLING 225 hardclock_device_poll(); 226 #endif /* DEVICE_POLLING */ 227 228 /* 229 * Process callouts at a very low cpu priority, so we don't keep the 230 * relatively high clock interrupt priority any longer than necessary. 231 */ 232 mtx_lock_spin_flags(&callout_lock, MTX_QUIET); 233 ticks++; 234 if (TAILQ_FIRST(&callwheel[ticks & callwheelmask]) != NULL) { 235 need_softclock = 1; 236 } else if (softticks + 1 == ticks) 237 ++softticks; 238 mtx_unlock_spin_flags(&callout_lock, MTX_QUIET); 239 240 /* 241 * swi_sched acquires sched_lock, so we don't want to call it with 242 * callout_lock held; incorrect locking order. 243 */ 244 if (need_softclock) 245 swi_sched(softclock_ih, SWI_NOSWITCH); 246 } 247 248 /* 249 * Compute number of ticks in the specified amount of time. 250 */ 251 int 252 tvtohz(tv) 253 struct timeval *tv; 254 { 255 register unsigned long ticks; 256 register long sec, usec; 257 258 /* 259 * If the number of usecs in the whole seconds part of the time 260 * difference fits in a long, then the total number of usecs will 261 * fit in an unsigned long. Compute the total and convert it to 262 * ticks, rounding up and adding 1 to allow for the current tick 263 * to expire. Rounding also depends on unsigned long arithmetic 264 * to avoid overflow. 265 * 266 * Otherwise, if the number of ticks in the whole seconds part of 267 * the time difference fits in a long, then convert the parts to 268 * ticks separately and add, using similar rounding methods and 269 * overflow avoidance. This method would work in the previous 270 * case but it is slightly slower and assumes that hz is integral. 271 * 272 * Otherwise, round the time difference down to the maximum 273 * representable value. 274 * 275 * If ints have 32 bits, then the maximum value for any timeout in 276 * 10ms ticks is 248 days. 277 */ 278 sec = tv->tv_sec; 279 usec = tv->tv_usec; 280 if (usec < 0) { 281 sec--; 282 usec += 1000000; 283 } 284 if (sec < 0) { 285 #ifdef DIAGNOSTIC 286 if (usec > 0) { 287 sec++; 288 usec -= 1000000; 289 } 290 printf("tvotohz: negative time difference %ld sec %ld usec\n", 291 sec, usec); 292 #endif 293 ticks = 1; 294 } else if (sec <= LONG_MAX / 1000000) 295 ticks = (sec * 1000000 + (unsigned long)usec + (tick - 1)) 296 / tick + 1; 297 else if (sec <= LONG_MAX / hz) 298 ticks = sec * hz 299 + ((unsigned long)usec + (tick - 1)) / tick + 1; 300 else 301 ticks = LONG_MAX; 302 if (ticks > INT_MAX) 303 ticks = INT_MAX; 304 return ((int)ticks); 305 } 306 307 /* 308 * Start profiling on a process. 309 * 310 * Kernel profiling passes proc0 which never exits and hence 311 * keeps the profile clock running constantly. 312 */ 313 void 314 startprofclock(p) 315 register struct proc *p; 316 { 317 int s; 318 319 /* 320 * XXX; Right now sched_lock protects statclock(), but perhaps 321 * it should be protected later on by a time_lock, which would 322 * cover psdiv, etc. as well. 323 */ 324 mtx_lock_spin(&sched_lock); 325 if ((p->p_sflag & PS_PROFIL) == 0) { 326 p->p_sflag |= PS_PROFIL; 327 if (++profprocs == 1 && stathz != 0) { 328 s = splstatclock(); 329 psdiv = pscnt = psratio; 330 setstatclockrate(profhz); 331 splx(s); 332 } 333 } 334 mtx_unlock_spin(&sched_lock); 335 } 336 337 /* 338 * Stop profiling on a process. 339 */ 340 void 341 stopprofclock(p) 342 register struct proc *p; 343 { 344 int s; 345 346 mtx_lock_spin(&sched_lock); 347 if (p->p_sflag & PS_PROFIL) { 348 p->p_sflag &= ~PS_PROFIL; 349 if (--profprocs == 0 && stathz != 0) { 350 s = splstatclock(); 351 psdiv = pscnt = 1; 352 setstatclockrate(stathz); 353 splx(s); 354 } 355 } 356 mtx_unlock_spin(&sched_lock); 357 } 358 359 /* 360 * Do process and kernel statistics. Most of the statistics are only 361 * used by user-level statistics programs. The main exceptions are 362 * ke->ke_uticks, p->p_sticks, p->p_iticks, and p->p_estcpu. This function 363 * should be called by all CPUs in the system for each statistics clock 364 * interrupt. See the description of hardclock_process for more detail on 365 * this function's relationship to statclock. 366 */ 367 void 368 statclock_process(ke, pc, user) 369 struct kse *ke; 370 register_t pc; 371 int user; 372 { 373 #ifdef GPROF 374 struct gmonparam *g; 375 int i; 376 #endif 377 struct pstats *pstats; 378 long rss; 379 struct rusage *ru; 380 struct vmspace *vm; 381 struct proc *p = ke->ke_proc; 382 struct thread *td = ke->ke_thread; /* current thread */ 383 384 KASSERT(ke == curthread->td_kse, ("statclock_process: td != curthread")); 385 mtx_assert(&sched_lock, MA_OWNED); 386 if (user) { 387 /* 388 * Came from user mode; CPU was in user state. 389 * If this process is being profiled, record the tick. 390 */ 391 if (p->p_sflag & PS_PROFIL) 392 addupc_intr(ke, pc, 1); 393 if (pscnt < psdiv) 394 return; 395 /* 396 * Charge the time as appropriate. 397 */ 398 ke->ke_uticks++; 399 if (ke->ke_ksegrp->kg_nice > NZERO) 400 cp_time[CP_NICE]++; 401 else 402 cp_time[CP_USER]++; 403 } else { 404 #ifdef GPROF 405 /* 406 * Kernel statistics are just like addupc_intr, only easier. 407 */ 408 g = &_gmonparam; 409 if (g->state == GMON_PROF_ON) { 410 i = pc - g->lowpc; 411 if (i < g->textsize) { 412 i /= HISTFRACTION * sizeof(*g->kcount); 413 g->kcount[i]++; 414 } 415 } 416 #endif 417 if (pscnt < psdiv) 418 return; 419 /* 420 * Came from kernel mode, so we were: 421 * - handling an interrupt, 422 * - doing syscall or trap work on behalf of the current 423 * user process, or 424 * - spinning in the idle loop. 425 * Whichever it is, charge the time as appropriate. 426 * Note that we charge interrupts to the current process, 427 * regardless of whether they are ``for'' that process, 428 * so that we know how much of its real time was spent 429 * in ``non-process'' (i.e., interrupt) work. 430 */ 431 if ((td->td_ithd != NULL) || td->td_intr_nesting_level >= 2) { 432 ke->ke_iticks++; 433 cp_time[CP_INTR]++; 434 } else { 435 ke->ke_sticks++; 436 if (p != PCPU_GET(idlethread)->td_proc) 437 cp_time[CP_SYS]++; 438 else 439 cp_time[CP_IDLE]++; 440 } 441 } 442 443 schedclock(ke->ke_thread); 444 445 /* Update resource usage integrals and maximums. */ 446 if ((pstats = p->p_stats) != NULL && 447 (ru = &pstats->p_ru) != NULL && 448 (vm = p->p_vmspace) != NULL) { 449 ru->ru_ixrss += pgtok(vm->vm_tsize); 450 ru->ru_idrss += pgtok(vm->vm_dsize); 451 ru->ru_isrss += pgtok(vm->vm_ssize); 452 rss = pgtok(vmspace_resident_count(vm)); 453 if (ru->ru_maxrss < rss) 454 ru->ru_maxrss = rss; 455 } 456 } 457 458 /* 459 * Statistics clock. Grab profile sample, and if divider reaches 0, 460 * do process and kernel statistics. Most of the statistics are only 461 * used by user-level statistics programs. The main exceptions are 462 * ke->ke_uticks, p->p_sticks, p->p_iticks, and p->p_estcpu. 463 */ 464 void 465 statclock(frame) 466 register struct clockframe *frame; 467 { 468 469 CTR0(KTR_INTR, "statclock fired"); 470 mtx_lock_spin_flags(&sched_lock, MTX_QUIET); 471 if (--pscnt == 0) 472 pscnt = psdiv; 473 statclock_process(curthread->td_kse, CLKF_PC(frame), CLKF_USERMODE(frame)); 474 mtx_unlock_spin_flags(&sched_lock, MTX_QUIET); 475 } 476 477 /* 478 * Return information about system clocks. 479 */ 480 static int 481 sysctl_kern_clockrate(SYSCTL_HANDLER_ARGS) 482 { 483 struct clockinfo clkinfo; 484 /* 485 * Construct clockinfo structure. 486 */ 487 clkinfo.hz = hz; 488 clkinfo.tick = tick; 489 clkinfo.tickadj = tickadj; 490 clkinfo.profhz = profhz; 491 clkinfo.stathz = stathz ? stathz : hz; 492 return (sysctl_handle_opaque(oidp, &clkinfo, sizeof clkinfo, req)); 493 } 494 495 SYSCTL_PROC(_kern, KERN_CLOCKRATE, clockrate, CTLTYPE_STRUCT|CTLFLAG_RD, 496 0, 0, sysctl_kern_clockrate, "S,clockinfo", 497 "Rate and period of various kernel clocks"); 498