1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 24 */ 25 26 /* 27 * The System Duty Cycle (SDC) scheduling class 28 * -------------------------------------------- 29 * 30 * Background 31 * 32 * Kernel threads in Solaris have traditionally not been large consumers 33 * of CPU time. They typically wake up, perform a small amount of 34 * work, then go back to sleep waiting for either a timeout or another 35 * signal. On the assumption that the small amount of work that they do 36 * is important for the behavior of the whole system, these threads are 37 * treated kindly by the dispatcher and the SYS scheduling class: they run 38 * without preemption from anything other than real-time and interrupt 39 * threads; when preempted, they are put at the front of the queue, so they 40 * generally do not migrate between CPUs; and they are allowed to stay 41 * running until they voluntarily give up the CPU. 42 * 43 * As Solaris has evolved, new workloads have emerged which require the 44 * kernel to perform significant amounts of CPU-intensive work. One 45 * example of such a workload is ZFS's transaction group sync processing. 46 * Each sync operation generates a large batch of I/Os, and each I/O 47 * may need to be compressed and/or checksummed before it is written to 48 * storage. The taskq threads which perform the compression and checksums 49 * will run nonstop as long as they have work to do; a large sync operation 50 * on a compression-heavy dataset can keep them busy for seconds on end. 51 * This causes human-time-scale dispatch latency bubbles for any other 52 * threads which have the misfortune to share a CPU with the taskq threads. 53 * 54 * The SDC scheduling class is a solution to this problem. 55 * 56 * 57 * Overview 58 * 59 * SDC is centered around the concept of a thread's duty cycle (DC): 60 * 61 * ONPROC time 62 * Duty Cycle = ---------------------- 63 * ONPROC + Runnable time 64 * 65 * This is the ratio of the time that the thread spent running on a CPU 66 * divided by the time it spent running or trying to run. It is unaffected 67 * by any time the thread spent sleeping, stopped, etc. 68 * 69 * A thread joining the SDC class specifies a "target" DC that it wants 70 * to run at. To implement this policy, the routine sysdc_update() scans 71 * the list of active SDC threads every few ticks and uses each thread's 72 * microstate data to compute the actual duty cycle that that thread 73 * has experienced recently. If the thread is under its target DC, its 74 * priority is increased to the maximum available (sysdc_maxpri, which is 75 * 99 by default). If the thread is over its target DC, its priority is 76 * reduced to the minimum available (sysdc_minpri, 0 by default). This 77 * is a fairly primitive approach, in that it doesn't use any of the 78 * intermediate priorities, but it's not completely inappropriate. Even 79 * though threads in the SDC class might take a while to do their job, they 80 * are by some definition important if they're running inside the kernel, 81 * so it is reasonable that they should get to run at priority 99. 82 * 83 * If a thread is running when sysdc_update() calculates its actual duty 84 * cycle, and there are other threads of equal or greater priority on its 85 * CPU's dispatch queue, sysdc_update() preempts that thread. The thread 86 * acknowledges the preemption by calling sysdc_preempt(), which calls 87 * setbackdq(), which gives other threads with the same priority a chance 88 * to run. This creates a de facto time quantum for threads in the SDC 89 * scheduling class. 90 * 91 * An SDC thread which is assigned priority 0 can continue to run if 92 * nothing else needs to use the CPU that it's running on. Similarly, an 93 * SDC thread at priority 99 might not get to run as much as it wants to 94 * if there are other priority-99 or higher threads on its CPU. These 95 * situations would cause the thread to get ahead of or behind its target 96 * DC; the longer the situations lasted, the further ahead or behind the 97 * thread would get. Rather than condemning a thread to a lifetime of 98 * paying for its youthful indiscretions, SDC keeps "base" values for 99 * ONPROC and Runnable times in each thread's sysdc data, and updates these 100 * values periodically. The duty cycle is then computed using the elapsed 101 * amount of ONPROC and Runnable times since those base times. 102 * 103 * Since sysdc_update() scans SDC threads fairly frequently, it tries to 104 * keep the list of "active" threads small by pruning out threads which 105 * have been asleep for a brief time. They are not pruned immediately upon 106 * going to sleep, since some threads may bounce back and forth between 107 * sleeping and being runnable. 108 * 109 * 110 * Interfaces 111 * 112 * void sysdc_thread_enter(t, dc, flags) 113 * 114 * Moves a kernel thread from the SYS scheduling class to the 115 * SDC class. t must have an associated LWP (created by calling 116 * lwp_kernel_create()). The thread will have a target DC of dc. 117 * Flags should be either 0 or SYSDC_THREAD_BATCH. If 118 * SYSDC_THREAD_BATCH is specified, the thread is expected to be 119 * doing large amounts of processing. 120 * 121 * 122 * Complications 123 * 124 * - Run queue balancing 125 * 126 * The Solaris dispatcher is biased towards letting a thread run 127 * on the same CPU which it last ran on, if no more than 3 ticks 128 * (i.e. rechoose_interval) have passed since the thread last ran. 129 * This helps to preserve cache warmth. On the other hand, it also 130 * tries to keep the per-CPU run queues fairly balanced; if the CPU 131 * chosen for a runnable thread has a run queue which is three or 132 * more threads longer than a neighboring CPU's queue, the runnable 133 * thread is dispatched onto the neighboring CPU instead. 134 * 135 * These policies work well for some workloads, but not for many SDC 136 * threads. The taskq client of SDC, for example, has many discrete 137 * units of work to do. The work units are largely independent, so 138 * cache warmth is not an important consideration. It is important 139 * that the threads fan out quickly to different CPUs, since the 140 * amount of work these threads have to do (a few seconds worth at a 141 * time) doesn't leave much time to correct thread placement errors 142 * (i.e. two SDC threads being dispatched to the same CPU). 143 * 144 * To fix this, SDC uses the TS_RUNQMATCH flag introduced for FSS. 145 * This tells the dispatcher to keep neighboring run queues' lengths 146 * more evenly matched, which allows SDC threads to migrate more 147 * easily. 148 * 149 * - LWPs and system processes 150 * 151 * SDC can only be used for kernel threads. Since SDC uses microstate 152 * accounting data to compute each thread's actual duty cycle, all 153 * threads entering the SDC class must have associated LWPs (which 154 * store the microstate data). This means that the threads have to 155 * be associated with an SSYS process, i.e. one created by newproc(). 156 * If the microstate accounting information is ever moved into the 157 * kthread_t, this restriction could be lifted. 158 * 159 * - Dealing with oversubscription 160 * 161 * Since SDC duty cycles are per-thread, it is possible that the 162 * aggregate requested duty cycle of all SDC threads in a processor 163 * set could be greater than the total CPU time available in that set. 164 * The FSS scheduling class has an analogous situation, which it deals 165 * with by reducing each thread's allotted CPU time proportionally. 166 * Since SDC doesn't need to be as precise as FSS, it uses a simpler 167 * solution to the oversubscription problem. 168 * 169 * sysdc_update() accumulates the amount of time that max-priority SDC 170 * threads have spent on-CPU in each processor set, and uses that sum 171 * to create an implied duty cycle for that processor set: 172 * 173 * accumulated CPU time 174 * pset DC = ----------------------------------- 175 * (# CPUs) * time since last update 176 * 177 * If this implied duty cycle is above a maximum pset duty cycle (90% 178 * by default), sysdc_update() sets the priority of all SDC threads 179 * in that processor set to sysdc_minpri for a "break" period. After 180 * the break period, it waits for a "nobreak" period before trying to 181 * enforce the pset duty cycle limit again. 182 * 183 * - Processor sets 184 * 185 * As the above implies, SDC is processor set aware, but it does not 186 * currently allow threads to change processor sets while in the SDC 187 * class. Instead, those threads must join the desired processor set 188 * before entering SDC. [1] 189 * 190 * - Batch threads 191 * 192 * A thread joining the SDC class can specify the SDC_THREAD_BATCH 193 * flag. This flag currently has no effect, but marks threads which 194 * do bulk processing. 195 * 196 * - Why not FSS? 197 * 198 * It might seem that the existing FSS scheduling class could solve 199 * the problems that SDC is attempting to solve. FSS's more precise 200 * solution to the oversubscription problem would hardly cause 201 * trouble, as long as it performed well. SDC is implemented as 202 * a separate scheduling class for two main reasons: the initial 203 * consumer of SDC does not map well onto the "project" abstraction 204 * that is central to FSS, and FSS does not expect to run at kernel 205 * priorities. 206 * 207 * 208 * Tunables 209 * 210 * - sysdc_update_interval_msec: Number of milliseconds between 211 * consecutive thread priority updates. 212 * 213 * - sysdc_reset_interval_msec: Number of milliseconds between 214 * consecutive resets of a thread's base ONPROC and Runnable 215 * times. 216 * 217 * - sysdc_prune_interval_msec: Number of milliseconds of sleeping 218 * before a thread is pruned from the active list. 219 * 220 * - sysdc_max_pset_DC: Allowable percentage of a processor set's 221 * CPU time which SDC can give to its high-priority threads. 222 * 223 * - sysdc_break_msec: Number of milliseconds of "break" taken when 224 * sysdc_max_pset_DC is exceeded. 225 * 226 * 227 * Future work (in SDC and related subsystems) 228 * 229 * - Per-thread rechoose interval (0 for SDC) 230 * 231 * Allow each thread to specify its own rechoose interval. SDC 232 * threads would specify an interval of zero, which would rechoose 233 * the CPU with the lowest priority once per update. 234 * 235 * - Allow threads to change processor sets after joining the SDC class 236 * 237 * - Thread groups and per-group DC 238 * 239 * It might be nice to be able to specify a duty cycle which applies 240 * to a group of threads in aggregate. 241 * 242 * - Per-group DC callback to allow dynamic DC tuning 243 * 244 * Currently, DCs are assigned when the thread joins SDC. Some 245 * workloads could benefit from being able to tune their DC using 246 * subsystem-specific knowledge about the workload. 247 * 248 * - Finer-grained priority updates 249 * 250 * - More nuanced management of oversubscription 251 * 252 * - Moving other CPU-intensive threads into SDC 253 * 254 * - Move msacct data into kthread_t 255 * 256 * This would allow kernel threads without LWPs to join SDC. 257 * 258 * 259 * Footnotes 260 * 261 * [1] The details of doing so are left as an exercise for the reader. 262 */ 263 264 #include <sys/types.h> 265 #include <sys/sysdc.h> 266 #include <sys/sysdc_impl.h> 267 268 #include <sys/class.h> 269 #include <sys/cmn_err.h> 270 #include <sys/cpuvar.h> 271 #include <sys/cpupart.h> 272 #include <sys/debug.h> 273 #include <sys/disp.h> 274 #include <sys/errno.h> 275 #include <sys/inline.h> 276 #include <sys/kmem.h> 277 #include <sys/modctl.h> 278 #include <sys/schedctl.h> 279 #include <sys/sdt.h> 280 #include <sys/sunddi.h> 281 #include <sys/sysmacros.h> 282 #include <sys/systm.h> 283 #include <sys/var.h> 284 285 /* 286 * Tunables - loaded into the internal state at module load time 287 */ 288 uint_t sysdc_update_interval_msec = 20; 289 uint_t sysdc_reset_interval_msec = 400; 290 uint_t sysdc_prune_interval_msec = 100; 291 uint_t sysdc_max_pset_DC = 90; 292 uint_t sysdc_break_msec = 80; 293 294 /* 295 * Internal state - constants set up by sysdc_initparam() 296 */ 297 static clock_t sysdc_update_ticks; /* ticks between updates */ 298 static uint_t sysdc_prune_updates; /* updates asleep before pruning */ 299 static uint_t sysdc_reset_updates; /* # of updates before reset */ 300 static uint_t sysdc_break_updates; /* updates to break */ 301 static uint_t sysdc_nobreak_updates; /* updates to not check */ 302 static uint_t sysdc_minDC; /* minimum allowed DC */ 303 static uint_t sysdc_maxDC; /* maximum allowed DC */ 304 static pri_t sysdc_minpri; /* minimum allowed priority */ 305 static pri_t sysdc_maxpri; /* maximum allowed priority */ 306 307 /* 308 * Internal state 309 */ 310 static kmutex_t sysdc_pset_lock; /* lock protecting pset data */ 311 static list_t sysdc_psets; /* list of psets with SDC threads */ 312 static uint_t sysdc_param_init; /* sysdc_initparam() has been called */ 313 static uint_t sysdc_update_timeout_started; /* update timeout is active */ 314 static hrtime_t sysdc_last_update; /* time of last sysdc_update() */ 315 static sysdc_t sysdc_dummy; /* used to terminate active lists */ 316 317 /* 318 * Internal state - active hash table 319 */ 320 #define SYSDC_NLISTS 8 321 #define SYSDC_HASH(sdc) (((uintptr_t)(sdc) >> 6) & (SYSDC_NLISTS - 1)) 322 static sysdc_list_t sysdc_active[SYSDC_NLISTS]; 323 #define SYSDC_LIST(sdc) (&sysdc_active[SYSDC_HASH(sdc)]) 324 325 #ifdef DEBUG 326 static struct { 327 uint64_t sysdc_update_times_asleep; 328 uint64_t sysdc_update_times_base_ran_backwards; 329 uint64_t sysdc_update_times_already_done; 330 uint64_t sysdc_update_times_cur_ran_backwards; 331 uint64_t sysdc_compute_pri_breaking; 332 uint64_t sysdc_activate_enter; 333 uint64_t sysdc_update_enter; 334 uint64_t sysdc_update_exited; 335 uint64_t sysdc_update_not_sdc; 336 uint64_t sysdc_update_idle; 337 uint64_t sysdc_update_take_break; 338 uint64_t sysdc_update_no_psets; 339 uint64_t sysdc_tick_not_sdc; 340 uint64_t sysdc_tick_quantum_expired; 341 uint64_t sysdc_thread_enter_enter; 342 } sysdc_stats; 343 344 #define SYSDC_INC_STAT(x) (sysdc_stats.x++) 345 #else 346 #define SYSDC_INC_STAT(x) ((void)0) 347 #endif 348 349 /* macros are UPPER CASE */ 350 #define HOWMANY(a, b) howmany((a), (b)) 351 #define MSECTOTICKS(a) HOWMANY((a) * 1000, usec_per_tick) 352 353 static void 354 sysdc_initparam(void) 355 { 356 uint_t sysdc_break_ticks; 357 358 /* update / prune intervals */ 359 sysdc_update_ticks = MSECTOTICKS(sysdc_update_interval_msec); 360 361 sysdc_prune_updates = HOWMANY(sysdc_prune_interval_msec, 362 sysdc_update_interval_msec); 363 sysdc_reset_updates = HOWMANY(sysdc_reset_interval_msec, 364 sysdc_update_interval_msec); 365 366 /* We must get at least a little time on CPU. */ 367 sysdc_minDC = 1; 368 sysdc_maxDC = SYSDC_DC_MAX; 369 sysdc_minpri = 0; 370 sysdc_maxpri = maxclsyspri - 1; 371 372 /* break parameters */ 373 if (sysdc_max_pset_DC > SYSDC_DC_MAX) { 374 sysdc_max_pset_DC = SYSDC_DC_MAX; 375 } 376 sysdc_break_ticks = MSECTOTICKS(sysdc_break_msec); 377 sysdc_break_updates = HOWMANY(sysdc_break_ticks, sysdc_update_ticks); 378 379 /* 380 * We want: 381 * 382 * sysdc_max_pset_DC = (nobreak / (break + nobreak)) 383 * 384 * ==> nobreak = sysdc_max_pset_DC * (break + nobreak) 385 * 386 * sysdc_max_pset_DC * break 387 * ==> nobreak = ------------------------- 388 * 1 - sysdc_max_pset_DC 389 */ 390 sysdc_nobreak_updates = 391 HOWMANY((uint64_t)sysdc_break_updates * sysdc_max_pset_DC, 392 (SYSDC_DC_MAX - sysdc_max_pset_DC)); 393 394 sysdc_param_init = 1; 395 } 396 397 #undef HOWMANY 398 #undef MSECTOTICKS 399 400 #define SDC_UPDATE_INITIAL 0x1 /* for the initial update */ 401 #define SDC_UPDATE_TIMEOUT 0x2 /* from sysdc_update() */ 402 #define SDC_UPDATE_TICK 0x4 /* from sysdc_tick(), on expiry */ 403 404 /* 405 * Updates the recorded times in the sdc, and returns the elapsed ONPROC 406 * and Runnable times since the last reset. 407 * 408 * newO is the thread's actual ONPROC time; it's used during sysdc_update() 409 * to track processor set usage. 410 */ 411 static void 412 sysdc_update_times(sysdc_t *sdc, uint_t flags, 413 hrtime_t *O, hrtime_t *R, hrtime_t *newO) 414 { 415 kthread_t *const t = sdc->sdc_thread; 416 const uint_t initial = (flags & SDC_UPDATE_INITIAL); 417 const uint_t update = (flags & SDC_UPDATE_TIMEOUT); 418 const clock_t now = ddi_get_lbolt(); 419 uint_t do_reset; 420 421 ASSERT(THREAD_LOCK_HELD(t)); 422 423 *O = *R = 0; 424 425 /* If we've been sleeping, we know we haven't had any ONPROC time. */ 426 if (sdc->sdc_sleep_updates != 0 && 427 sdc->sdc_sleep_updates != sdc->sdc_nupdates) { 428 *newO = sdc->sdc_last_base_O; 429 SYSDC_INC_STAT(sysdc_update_times_asleep); 430 return; 431 } 432 433 /* 434 * If this is our first update, or we've hit the reset point, 435 * we need to reset our base_{O,R}. Once we've updated them, we 436 * report O and R for the entire prior interval. 437 */ 438 do_reset = initial; 439 if (update) { 440 ++sdc->sdc_nupdates; 441 if ((sdc->sdc_nupdates % sysdc_reset_updates) == 0) 442 do_reset = 1; 443 } 444 if (do_reset) { 445 hrtime_t baseO, baseR; 446 if (initial) { 447 /* 448 * Start off our cycle count somewhere in the middle, 449 * to keep the resets from all happening at once. 450 * 451 * 4999 is a handy prime much larger than 452 * sysdc_reset_updates, so that we don't run into 453 * trouble if the resolution is a multiple of 454 * sysdc_reset_updates. 455 */ 456 sdc->sdc_nupdates = (uint_t)((gethrtime() % 4999) % 457 sysdc_reset_updates); 458 baseO = baseR = 0; 459 } else { 460 baseO = sdc->sdc_base_O; 461 baseR = sdc->sdc_base_R; 462 } 463 464 mstate_systhread_times(t, &sdc->sdc_base_O, &sdc->sdc_base_R); 465 *newO = sdc->sdc_base_O; 466 467 sdc->sdc_reset = now; 468 sdc->sdc_pri_check = -1; /* force mismatch below */ 469 470 /* 471 * See below for rationale. 472 */ 473 if (baseO > sdc->sdc_base_O || baseR > sdc->sdc_base_R) { 474 SYSDC_INC_STAT(sysdc_update_times_base_ran_backwards); 475 baseO = sdc->sdc_base_O; 476 baseR = sdc->sdc_base_R; 477 } 478 479 /* compute based on the entire interval */ 480 *O = (sdc->sdc_base_O - baseO); 481 *R = (sdc->sdc_base_R - baseR); 482 return; 483 } 484 485 /* 486 * If we're called from sysdc_update(), we *must* return a value 487 * for newO, so we always call mstate_systhread_times(). 488 * 489 * Otherwise, if we've already done a pri check this tick, 490 * we can skip it. 491 */ 492 if (!update && sdc->sdc_pri_check == now) { 493 SYSDC_INC_STAT(sysdc_update_times_already_done); 494 return; 495 } 496 497 /* Get the current times from the thread */ 498 sdc->sdc_pri_check = now; 499 mstate_systhread_times(t, &sdc->sdc_cur_O, &sdc->sdc_cur_R); 500 *newO = sdc->sdc_cur_O; 501 502 /* 503 * The updating of microstate accounting is not done under a 504 * consistent set of locks, particularly the t_waitrq field. This 505 * can lead to narrow windows in which we account for time in the 506 * wrong bucket, which on the next read will be accounted for 507 * correctly. 508 * 509 * If our sdc_base_* fields were affected by one of these blips, we 510 * throw away the old data, and pretend this tick didn't happen. 511 */ 512 if (sdc->sdc_cur_O < sdc->sdc_base_O || 513 sdc->sdc_cur_R < sdc->sdc_base_R) { 514 515 sdc->sdc_base_O = sdc->sdc_cur_O; 516 sdc->sdc_base_R = sdc->sdc_cur_R; 517 518 SYSDC_INC_STAT(sysdc_update_times_cur_ran_backwards); 519 return; 520 } 521 522 *O = sdc->sdc_cur_O - sdc->sdc_base_O; 523 *R = sdc->sdc_cur_R - sdc->sdc_base_R; 524 } 525 526 /* 527 * sysdc_compute_pri() 528 * 529 * Recomputes the priority of the thread, leaving the result in 530 * sdc->sdc_epri. Returns 1 if a priority update should occur 531 * (which will also trigger a cpu_surrender()), otherwise 532 * returns 0. 533 */ 534 static uint_t 535 sysdc_compute_pri(sysdc_t *sdc, uint_t flags) 536 { 537 kthread_t *const t = sdc->sdc_thread; 538 const uint_t update = (flags & SDC_UPDATE_TIMEOUT); 539 const uint_t tick = (flags & SDC_UPDATE_TICK); 540 541 hrtime_t O, R; 542 hrtime_t newO = -1; 543 544 ASSERT(THREAD_LOCK_HELD(t)); 545 546 sysdc_update_times(sdc, flags, &O, &R, &newO); 547 ASSERT(!update || newO != -1); 548 549 /* If we have new data, recompute our priority. */ 550 if ((O + R) != 0) { 551 sdc->sdc_cur_DC = (O * SYSDC_DC_MAX) / (O + R); 552 553 /* Adjust our priority to move our DC closer to the target. */ 554 if (sdc->sdc_cur_DC < sdc->sdc_target_DC) 555 sdc->sdc_pri = sdc->sdc_maxpri; 556 else 557 sdc->sdc_pri = sdc->sdc_minpri; 558 } 559 560 /* 561 * If our per-pset duty cycle goes over the max, we will take a break. 562 * This forces all sysdc threads in the pset to minimum priority, in 563 * order to let everyone else have a chance at the CPU. 564 */ 565 if (sdc->sdc_pset->sdp_need_break) { 566 SYSDC_INC_STAT(sysdc_compute_pri_breaking); 567 sdc->sdc_epri = sdc->sdc_minpri; 568 } else { 569 sdc->sdc_epri = sdc->sdc_pri; 570 } 571 572 DTRACE_PROBE4(sysdc__compute__pri, 573 kthread_t *, t, pri_t, sdc->sdc_epri, uint_t, sdc->sdc_cur_DC, 574 uint_t, sdc->sdc_target_DC); 575 576 /* 577 * For sysdc_update(), we compute the ONPROC time for high-priority 578 * threads, which is used to calculate the per-pset duty cycle. We 579 * will always tell our callers to update the thread's priority, 580 * since we want to force a cpu_surrender(). 581 * 582 * We reset sdc_update_ticks so that sysdc_tick() will only update 583 * the thread's priority if our timeout is delayed by a tick or 584 * more. 585 */ 586 if (update) { 587 /* SDC threads are not allowed to change cpupart bindings. */ 588 ASSERT(t->t_cpupart == sdc->sdc_pset->sdp_cpupart); 589 590 /* If we were at MAXPRI, account for our onproc time. */ 591 if (t->t_pri == sdc->sdc_maxpri && 592 sdc->sdc_last_base_O != 0 && 593 sdc->sdc_last_base_O < newO) { 594 sdc->sdc_last_O = newO - sdc->sdc_last_base_O; 595 sdc->sdc_pset->sdp_onproc_time += 596 (uint64_t)sdc->sdc_last_O; 597 sdc->sdc_pset->sdp_onproc_threads++; 598 } else { 599 sdc->sdc_last_O = 0; 600 } 601 sdc->sdc_last_base_O = newO; 602 603 sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks + 1; 604 return (1); 605 } 606 607 /* 608 * Like sysdc_update(), sysdc_tick() always wants to update the 609 * thread's priority, so that the CPU is surrendered if necessary. 610 * We reset sdc_update_ticks so that if the timeout continues to be 611 * delayed, we'll update at the regular interval. 612 */ 613 if (tick) { 614 ASSERT(sdc->sdc_ticks == sdc->sdc_update_ticks); 615 sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks; 616 return (1); 617 } 618 619 /* 620 * Otherwise, only tell our callers to update the priority if it has 621 * changed. 622 */ 623 return (sdc->sdc_epri != t->t_pri); 624 } 625 626 static void 627 sysdc_update_pri(sysdc_t *sdc, uint_t flags) 628 { 629 kthread_t *t = sdc->sdc_thread; 630 631 ASSERT(THREAD_LOCK_HELD(t)); 632 633 if (sysdc_compute_pri(sdc, flags)) { 634 if (!thread_change_pri(t, sdc->sdc_epri, 0)) { 635 cpu_surrender(t); 636 } 637 } 638 } 639 640 /* 641 * Add a thread onto the active list. It will only be removed by 642 * sysdc_update(). 643 */ 644 static void 645 sysdc_activate(sysdc_t *sdc) 646 { 647 sysdc_t *volatile *headp = &SYSDC_LIST(sdc)->sdl_list; 648 sysdc_t *head; 649 kthread_t *t = sdc->sdc_thread; 650 651 SYSDC_INC_STAT(sysdc_activate_enter); 652 653 ASSERT(sdc->sdc_next == NULL); 654 ASSERT(THREAD_LOCK_HELD(t)); 655 656 do { 657 head = *headp; 658 sdc->sdc_next = head; 659 } while (atomic_cas_ptr(headp, head, sdc) != head); 660 } 661 662 /* 663 * sysdc_update() has two jobs: 664 * 665 * 1. It updates the priorities of all active SDC threads on the system. 666 * 2. It measures pset CPU usage and enforces sysdc_max_pset_DC. 667 */ 668 static void 669 sysdc_update(void *arg) 670 { 671 int idx; 672 sysdc_t *freelist = NULL; 673 sysdc_pset_t *cur; 674 hrtime_t now, diff; 675 uint_t redeploy = 1; 676 677 SYSDC_INC_STAT(sysdc_update_enter); 678 679 ASSERT(sysdc_update_timeout_started); 680 681 /* 682 * If this is our first time through, diff will be gigantic, and 683 * no breaks will be necessary. 684 */ 685 now = gethrtime(); 686 diff = now - sysdc_last_update; 687 sysdc_last_update = now; 688 689 mutex_enter(&sysdc_pset_lock); 690 for (cur = list_head(&sysdc_psets); cur != NULL; 691 cur = list_next(&sysdc_psets, cur)) { 692 boolean_t breaking = (cur->sdp_should_break != 0); 693 694 if (cur->sdp_need_break != breaking) { 695 DTRACE_PROBE2(sdc__pset__break, sysdc_pset_t *, cur, 696 boolean_t, breaking); 697 } 698 cur->sdp_onproc_time = 0; 699 cur->sdp_onproc_threads = 0; 700 cur->sdp_need_break = breaking; 701 } 702 mutex_exit(&sysdc_pset_lock); 703 704 for (idx = 0; idx < SYSDC_NLISTS; idx++) { 705 sysdc_list_t *sdl = &sysdc_active[idx]; 706 sysdc_t *volatile *headp = &sdl->sdl_list; 707 sysdc_t *head, *tail; 708 sysdc_t **prevptr; 709 710 if (*headp == &sysdc_dummy) 711 continue; 712 713 /* Prevent any threads from exiting while we're poking them. */ 714 mutex_enter(&sdl->sdl_lock); 715 716 /* 717 * Each sdl_list contains a singly-linked list of active 718 * threads. Threads which become active while we are 719 * processing the list will be added to sdl_list. Since we 720 * don't want that to interfere with our own processing, we 721 * swap in an empty list. Any newly active threads will 722 * go on to this empty list. When finished, we'll put any 723 * such threads at the end of the processed list. 724 */ 725 head = atomic_swap_ptr(headp, &sysdc_dummy); 726 prevptr = &head; 727 while (*prevptr != &sysdc_dummy) { 728 sysdc_t *const sdc = *prevptr; 729 kthread_t *const t = sdc->sdc_thread; 730 731 /* 732 * If the thread has exited, move its sysdc_t onto 733 * freelist, to be freed later. 734 */ 735 if (t == NULL) { 736 *prevptr = sdc->sdc_next; 737 SYSDC_INC_STAT(sysdc_update_exited); 738 sdc->sdc_next = freelist; 739 freelist = sdc; 740 continue; 741 } 742 743 thread_lock(t); 744 if (t->t_cid != sysdccid) { 745 thread_unlock(t); 746 prevptr = &sdc->sdc_next; 747 SYSDC_INC_STAT(sysdc_update_not_sdc); 748 continue; 749 } 750 ASSERT(t->t_cldata == sdc); 751 752 /* 753 * If the thread has been sleeping for longer 754 * than sysdc_prune_interval, make it inactive by 755 * removing it from the list. 756 */ 757 if (!(t->t_state & (TS_RUN | TS_ONPROC)) && 758 sdc->sdc_sleep_updates != 0 && 759 (sdc->sdc_sleep_updates - sdc->sdc_nupdates) > 760 sysdc_prune_updates) { 761 *prevptr = sdc->sdc_next; 762 SYSDC_INC_STAT(sysdc_update_idle); 763 sdc->sdc_next = NULL; 764 thread_unlock(t); 765 continue; 766 } 767 sysdc_update_pri(sdc, SDC_UPDATE_TIMEOUT); 768 thread_unlock(t); 769 770 prevptr = &sdc->sdc_next; 771 } 772 773 /* 774 * Add our list to the bucket, putting any new entries 775 * added while we were working at the tail of the list. 776 */ 777 do { 778 tail = *headp; 779 *prevptr = tail; 780 } while (atomic_cas_ptr(headp, tail, head) != tail); 781 782 mutex_exit(&sdl->sdl_lock); 783 } 784 785 mutex_enter(&sysdc_pset_lock); 786 for (cur = list_head(&sysdc_psets); cur != NULL; 787 cur = list_next(&sysdc_psets, cur)) { 788 789 cur->sdp_vtime_last_interval = 790 diff * cur->sdp_cpupart->cp_ncpus; 791 cur->sdp_DC_last_interval = 792 (cur->sdp_onproc_time * SYSDC_DC_MAX) / 793 cur->sdp_vtime_last_interval; 794 795 if (cur->sdp_should_break > 0) { 796 cur->sdp_should_break--; /* breaking */ 797 continue; 798 } 799 if (cur->sdp_dont_break > 0) { 800 cur->sdp_dont_break--; /* waiting before checking */ 801 continue; 802 } 803 if (cur->sdp_DC_last_interval > sysdc_max_pset_DC) { 804 cur->sdp_should_break = sysdc_break_updates; 805 cur->sdp_dont_break = sysdc_nobreak_updates; 806 SYSDC_INC_STAT(sysdc_update_take_break); 807 } 808 } 809 810 /* 811 * If there are no sysdc_psets, there can be no threads, so 812 * we can stop doing our timeout. Since we're holding the 813 * sysdc_pset_lock, no new sysdc_psets can come in, which will 814 * prevent anyone from racing with this and dropping our timeout 815 * on the floor. 816 */ 817 if (list_is_empty(&sysdc_psets)) { 818 SYSDC_INC_STAT(sysdc_update_no_psets); 819 ASSERT(sysdc_update_timeout_started); 820 sysdc_update_timeout_started = 0; 821 822 redeploy = 0; 823 } 824 mutex_exit(&sysdc_pset_lock); 825 826 while (freelist != NULL) { 827 sysdc_t *cur = freelist; 828 freelist = cur->sdc_next; 829 kmem_free(cur, sizeof (*cur)); 830 } 831 832 if (redeploy) { 833 (void) timeout(sysdc_update, arg, sysdc_update_ticks); 834 } 835 } 836 837 static void 838 sysdc_preempt(kthread_t *t) 839 { 840 ASSERT(t == curthread); 841 ASSERT(THREAD_LOCK_HELD(t)); 842 843 setbackdq(t); /* give others a chance to run */ 844 } 845 846 static void 847 sysdc_tick(kthread_t *t) 848 { 849 sysdc_t *sdc; 850 851 thread_lock(t); 852 if (t->t_cid != sysdccid) { 853 SYSDC_INC_STAT(sysdc_tick_not_sdc); 854 thread_unlock(t); 855 return; 856 } 857 sdc = t->t_cldata; 858 if (t->t_state == TS_ONPROC && 859 t->t_pri < t->t_disp_queue->disp_maxrunpri) { 860 cpu_surrender(t); 861 } 862 863 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) { 864 ASSERT(sdc->sdc_sleep_updates == 0); 865 } 866 867 ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks); 868 sdc->sdc_ticks++; 869 if (sdc->sdc_ticks == sdc->sdc_update_ticks) { 870 SYSDC_INC_STAT(sysdc_tick_quantum_expired); 871 sysdc_update_pri(sdc, SDC_UPDATE_TICK); 872 ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks); 873 } 874 thread_unlock(t); 875 } 876 877 static void 878 sysdc_setrun(kthread_t *t) 879 { 880 sysdc_t *sdc = t->t_cldata; 881 882 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */ 883 884 sdc->sdc_sleep_updates = 0; 885 886 if (sdc->sdc_next == NULL) { 887 /* 888 * Since we're in transition, we don't want to use the 889 * full thread_update_pri(). 890 */ 891 if (sysdc_compute_pri(sdc, 0)) { 892 THREAD_CHANGE_PRI(t, sdc->sdc_epri); 893 } 894 sysdc_activate(sdc); 895 896 ASSERT(sdc->sdc_next != NULL); 897 } 898 899 setbackdq(t); 900 } 901 902 static void 903 sysdc_wakeup(kthread_t *t) 904 { 905 sysdc_setrun(t); 906 } 907 908 static void 909 sysdc_sleep(kthread_t *t) 910 { 911 sysdc_t *sdc = t->t_cldata; 912 913 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */ 914 915 sdc->sdc_sleep_updates = sdc->sdc_nupdates; 916 } 917 918 /*ARGSUSED*/ 919 static int 920 sysdc_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp, 921 void *bufp) 922 { 923 cpupart_t *const cpupart = t->t_cpupart; 924 sysdc_t *sdc = bufp; 925 sysdc_params_t *sdpp = parmsp; 926 sysdc_pset_t *newpset = sdc->sdc_pset; 927 sysdc_pset_t *pset; 928 int start_timeout; 929 930 if (t->t_cid != syscid) 931 return (EPERM); 932 933 ASSERT(ttolwp(t) != NULL); 934 ASSERT(sdpp != NULL); 935 ASSERT(newpset != NULL); 936 ASSERT(sysdc_param_init); 937 938 ASSERT(sdpp->sdp_minpri >= sysdc_minpri); 939 ASSERT(sdpp->sdp_maxpri <= sysdc_maxpri); 940 ASSERT(sdpp->sdp_DC >= sysdc_minDC); 941 ASSERT(sdpp->sdp_DC <= sysdc_maxDC); 942 943 sdc->sdc_thread = t; 944 sdc->sdc_pri = sdpp->sdp_maxpri; /* start off maximally */ 945 sdc->sdc_minpri = sdpp->sdp_minpri; 946 sdc->sdc_maxpri = sdpp->sdp_maxpri; 947 sdc->sdc_target_DC = sdpp->sdp_DC; 948 sdc->sdc_ticks = 0; 949 sdc->sdc_update_ticks = sysdc_update_ticks + 1; 950 951 /* Assign ourselves to the appropriate pset. */ 952 sdc->sdc_pset = NULL; 953 mutex_enter(&sysdc_pset_lock); 954 for (pset = list_head(&sysdc_psets); pset != NULL; 955 pset = list_next(&sysdc_psets, pset)) { 956 if (pset->sdp_cpupart == cpupart) { 957 break; 958 } 959 } 960 if (pset == NULL) { 961 pset = newpset; 962 newpset = NULL; 963 pset->sdp_cpupart = cpupart; 964 list_insert_tail(&sysdc_psets, pset); 965 } 966 pset->sdp_nthreads++; 967 ASSERT(pset->sdp_nthreads > 0); 968 969 sdc->sdc_pset = pset; 970 971 start_timeout = (sysdc_update_timeout_started == 0); 972 sysdc_update_timeout_started = 1; 973 mutex_exit(&sysdc_pset_lock); 974 975 if (newpset != NULL) 976 kmem_free(newpset, sizeof (*newpset)); 977 978 /* Update t's scheduling class and priority. */ 979 thread_lock(t); 980 t->t_clfuncs = &(sclass[cid].cl_funcs->thread); 981 t->t_cid = cid; 982 t->t_cldata = sdc; 983 t->t_schedflag |= TS_RUNQMATCH; 984 985 sysdc_update_pri(sdc, SDC_UPDATE_INITIAL); 986 thread_unlock(t); 987 988 /* Kick off the thread timeout if we're the first one in. */ 989 if (start_timeout) { 990 (void) timeout(sysdc_update, NULL, sysdc_update_ticks); 991 } 992 993 return (0); 994 } 995 996 static void 997 sysdc_leave(sysdc_t *sdc) 998 { 999 sysdc_pset_t *sdp = sdc->sdc_pset; 1000 sysdc_list_t *sdl = SYSDC_LIST(sdc); 1001 uint_t freedc; 1002 1003 mutex_enter(&sdl->sdl_lock); /* block sysdc_update() */ 1004 sdc->sdc_thread = NULL; 1005 freedc = (sdc->sdc_next == NULL); 1006 mutex_exit(&sdl->sdl_lock); 1007 1008 mutex_enter(&sysdc_pset_lock); 1009 ASSERT(sdp != NULL); 1010 ASSERT(sdp->sdp_nthreads > 0); 1011 --sdp->sdp_nthreads; 1012 if (sdp->sdp_nthreads == 0) { 1013 list_remove(&sysdc_psets, sdp); 1014 } else { 1015 sdp = NULL; 1016 } 1017 mutex_exit(&sysdc_pset_lock); 1018 1019 if (freedc) 1020 kmem_free(sdc, sizeof (*sdc)); 1021 if (sdp != NULL) 1022 kmem_free(sdp, sizeof (*sdp)); 1023 } 1024 1025 static void 1026 sysdc_exitclass(void *buf) 1027 { 1028 sysdc_leave((sysdc_t *)buf); 1029 } 1030 1031 /*ARGSUSED*/ 1032 static int 1033 sysdc_canexit(kthread_t *t, cred_t *reqpcredp) 1034 { 1035 /* Threads cannot exit SDC once joined, except in a body bag. */ 1036 return (EPERM); 1037 } 1038 1039 static void 1040 sysdc_exit(kthread_t *t) 1041 { 1042 sysdc_t *sdc; 1043 1044 /* We're exiting, so we just rejoin the SYS class. */ 1045 thread_lock(t); 1046 ASSERT(t->t_cid == sysdccid); 1047 sdc = t->t_cldata; 1048 t->t_cid = syscid; 1049 t->t_cldata = NULL; 1050 t->t_clfuncs = &(sclass[syscid].cl_funcs->thread); 1051 (void) thread_change_pri(t, maxclsyspri, 0); 1052 t->t_schedflag &= ~TS_RUNQMATCH; 1053 thread_unlock_nopreempt(t); 1054 1055 /* Unlink the sdc from everything. */ 1056 sysdc_leave(sdc); 1057 } 1058 1059 /*ARGSUSED*/ 1060 static int 1061 sysdc_fork(kthread_t *t, kthread_t *ct, void *bufp) 1062 { 1063 /* 1064 * Threads cannot be created with SDC as their class; they must 1065 * be created as SYS and then added with sysdc_thread_enter(). 1066 * Because of this restriction, sysdc_fork() should never be called. 1067 */ 1068 panic("sysdc cannot be forked"); 1069 1070 return (ENOSYS); 1071 } 1072 1073 /*ARGSUSED*/ 1074 static void 1075 sysdc_forkret(kthread_t *t, kthread_t *ct) 1076 { 1077 /* SDC threads are part of system processes, which never fork. */ 1078 panic("sysdc cannot be forked"); 1079 } 1080 1081 static pri_t 1082 sysdc_globpri(kthread_t *t) 1083 { 1084 return (t->t_epri); 1085 } 1086 1087 /*ARGSUSED*/ 1088 static pri_t 1089 sysdc_no_swap(kthread_t *t, int flags) 1090 { 1091 /* SDC threads cannot be swapped. */ 1092 return (-1); 1093 } 1094 1095 /* 1096 * Get maximum and minimum priorities enjoyed by SDC threads. 1097 */ 1098 static int 1099 sysdc_getclpri(pcpri_t *pcprip) 1100 { 1101 pcprip->pc_clpmax = sysdc_maxpri; 1102 pcprip->pc_clpmin = sysdc_minpri; 1103 return (0); 1104 } 1105 1106 /*ARGSUSED*/ 1107 static int 1108 sysdc_getclinfo(void *arg) 1109 { 1110 return (0); /* no class-specific info */ 1111 } 1112 1113 /*ARGSUSED*/ 1114 static int 1115 sysdc_alloc(void **p, int flag) 1116 { 1117 sysdc_t *new; 1118 1119 *p = NULL; 1120 if ((new = kmem_zalloc(sizeof (*new), flag)) == NULL) { 1121 return (ENOMEM); 1122 } 1123 if ((new->sdc_pset = kmem_zalloc(sizeof (*new->sdc_pset), flag)) == 1124 NULL) { 1125 kmem_free(new, sizeof (*new)); 1126 return (ENOMEM); 1127 } 1128 *p = new; 1129 return (0); 1130 } 1131 1132 static void 1133 sysdc_free(void *p) 1134 { 1135 sysdc_t *sdc = p; 1136 1137 if (sdc != NULL) { 1138 /* 1139 * We must have failed CL_ENTERCLASS(), so our pset should be 1140 * there and unused. 1141 */ 1142 ASSERT(sdc->sdc_pset != NULL); 1143 ASSERT(sdc->sdc_pset->sdp_cpupart == NULL); 1144 kmem_free(sdc->sdc_pset, sizeof (*sdc->sdc_pset)); 1145 kmem_free(sdc, sizeof (*sdc)); 1146 } 1147 } 1148 1149 static int sysdc_enosys(); /* Boy, ANSI-C's K&R compatibility is weird. */ 1150 static int sysdc_einval(); 1151 static void sysdc_nullsys(); 1152 1153 static struct classfuncs sysdc_classfuncs = { 1154 /* messages to class manager */ 1155 { 1156 sysdc_enosys, /* admin */ 1157 sysdc_getclinfo, 1158 sysdc_enosys, /* parmsin */ 1159 sysdc_enosys, /* parmsout */ 1160 sysdc_enosys, /* vaparmsin */ 1161 sysdc_enosys, /* vaparmsout */ 1162 sysdc_getclpri, 1163 sysdc_alloc, 1164 sysdc_free, 1165 }, 1166 /* operations on threads */ 1167 { 1168 sysdc_enterclass, 1169 sysdc_exitclass, 1170 sysdc_canexit, 1171 sysdc_fork, 1172 sysdc_forkret, 1173 sysdc_nullsys, /* parmsget */ 1174 sysdc_enosys, /* parmsset */ 1175 sysdc_nullsys, /* stop */ 1176 sysdc_exit, 1177 sysdc_nullsys, /* active */ 1178 sysdc_nullsys, /* inactive */ 1179 sysdc_no_swap, /* swapin */ 1180 sysdc_no_swap, /* swapout */ 1181 sysdc_nullsys, /* trapret */ 1182 sysdc_preempt, 1183 sysdc_setrun, 1184 sysdc_sleep, 1185 sysdc_tick, 1186 sysdc_wakeup, 1187 sysdc_einval, /* donice */ 1188 sysdc_globpri, 1189 sysdc_nullsys, /* set_process_group */ 1190 sysdc_nullsys, /* yield */ 1191 sysdc_einval, /* doprio */ 1192 } 1193 }; 1194 1195 static int 1196 sysdc_enosys() 1197 { 1198 return (ENOSYS); 1199 } 1200 1201 static int 1202 sysdc_einval() 1203 { 1204 return (EINVAL); 1205 } 1206 1207 static void 1208 sysdc_nullsys() 1209 { 1210 } 1211 1212 /*ARGSUSED*/ 1213 static pri_t 1214 sysdc_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp) 1215 { 1216 int idx; 1217 1218 list_create(&sysdc_psets, sizeof (sysdc_pset_t), 1219 offsetof(sysdc_pset_t, sdp_node)); 1220 1221 for (idx = 0; idx < SYSDC_NLISTS; idx++) { 1222 sysdc_active[idx].sdl_list = &sysdc_dummy; 1223 } 1224 1225 sysdc_initparam(); 1226 1227 sysdccid = cid; 1228 *clfuncspp = &sysdc_classfuncs; 1229 1230 return ((pri_t)v.v_maxsyspri); 1231 } 1232 1233 static struct sclass csw = { 1234 "SDC", 1235 sysdc_init, 1236 0 1237 }; 1238 1239 static struct modlsched modlsched = { 1240 &mod_schedops, "system duty cycle scheduling class", &csw 1241 }; 1242 1243 static struct modlinkage modlinkage = { 1244 MODREV_1, (void *)&modlsched, NULL 1245 }; 1246 1247 int 1248 _init() 1249 { 1250 return (mod_install(&modlinkage)); 1251 } 1252 1253 int 1254 _fini() 1255 { 1256 return (EBUSY); /* can't unload for now */ 1257 } 1258 1259 int 1260 _info(struct modinfo *modinfop) 1261 { 1262 return (mod_info(&modlinkage, modinfop)); 1263 } 1264 1265 /* --- consolidation-private interfaces --- */ 1266 void 1267 sysdc_thread_enter(kthread_t *t, uint_t dc, uint_t flags) 1268 { 1269 void *buf = NULL; 1270 sysdc_params_t sdp; 1271 1272 SYSDC_INC_STAT(sysdc_thread_enter_enter); 1273 1274 ASSERT(sysdc_param_init); 1275 ASSERT(sysdccid >= 0); 1276 1277 ASSERT((flags & ~SYSDC_THREAD_BATCH) == 0); 1278 1279 sdp.sdp_minpri = sysdc_minpri; 1280 sdp.sdp_maxpri = sysdc_maxpri; 1281 sdp.sdp_DC = MAX(MIN(dc, sysdc_maxDC), sysdc_minDC); 1282 1283 VERIFY0(CL_ALLOC(&buf, sysdccid, KM_SLEEP)); 1284 1285 ASSERT(t->t_lwp != NULL); 1286 ASSERT(t->t_cid == syscid); 1287 ASSERT(t->t_cldata == NULL); 1288 VERIFY0(CL_CANEXIT(t, NULL)); 1289 VERIFY0(CL_ENTERCLASS(t, sysdccid, &sdp, kcred, buf)); 1290 CL_EXITCLASS(syscid, NULL); 1291 } 1292