27  * The System Duty Cycle (SDC) scheduling class
28  * --------------------------------------------
38  * without preemption from anything other than real-time and interrupt
44  * kernel to perform significant amounts of CPU-intensive work.  One
50  * on a compression-heavy dataset can keep them busy for seconds on end.
51  * This causes human-time-scale dispatch latency bubbles for any other
59  * SDC is centered around the concept of a thread's duty cycle (DC):
62  *	Duty Cycle =	----------------------
72  * microstate data to compute the actual duty cycle that that thread
83  * If a thread is running when sysdc_update() calculates its actual duty
84  * cycle, and there are other threads of equal or greater priority on its
94  * if there are other priority-99 or higher threads on its CPU.  These
100  * values periodically.  The duty cycle is then computed using the elapsed
124  * - Run queue balancing
130  *	tries to keep the per-CPU run queues fairly balanced; if the CPU
149  * - LWPs and system processes
152  *	accounting data to compute each thread's actual duty cycle, all
159  * - Dealing with oversubscription
161  *	Since SDC duty cycles are per-thread, it is possible that the
162  *	aggregate requested duty cycle of all SDC threads in a processor
169  *	sysdc_update() accumulates the amount of time that max-priority SDC
170  *	threads have spent on-CPU in each processor set, and uses that sum
171  *	to create an implied duty cycle for that processor set:
174  *	   pset DC =	-----------------------------------
177  *	If this implied duty cycle is above a maximum pset duty cycle (90%
181  *	enforce the pset duty cycle limit again.
183  * - Processor sets
190  * - Batch threads
196  * - Why not FSS?
210  * - sysdc_update_interval_msec:  Number of milliseconds between
213  * - sysdc_reset_interval_msec:  Number of milliseconds between
217  * - sysdc_prune_interval_msec:  Number of milliseconds of sleeping
220  * - sysdc_max_pset_DC:  Allowable percentage of a processor set's
221  *	CPU time which SDC can give to its high-priority threads.
223  * - sysdc_break_msec:  Number of milliseconds of "break" taken when
229  * - Per-thread rechoose interval (0 for SDC)
235  * - Allow threads to change processor sets after joining the SDC class
237  * - Thread groups and per-group DC
239  *	It might be nice to be able to specify a duty cycle which applies
242  * - Per-group DC callback to allow dynamic DC tuning
246  *	subsystem-specific knowledge about the workload.
248  * - Finer-grained priority updates
250  * - More nuanced management of oversubscription
252  * - Moving other CPU-intensive threads into SDC
254  * - Move msacct data into kthread_t
286  * Tunables - loaded into the internal state at module load time
295  * Internal state - constants set up by sysdc_initparam()
318  * Internal state - active hash table
321 #define	SYSDC_HASH(sdc)	(((uintptr_t)(sdc) >> 6) & (SYSDC_NLISTS - 1))
370 	sysdc_maxpri = maxclsyspri - 1;  in sysdc_initparam()
387 	 *	==>	  nobreak = -------------------------  in sysdc_initparam()
388 	 *			    1 - sysdc_max_pset_DC  in sysdc_initparam()
392 	    (SYSDC_DC_MAX - sysdc_max_pset_DC));  in sysdc_initparam()
415 	kthread_t *const t = sdc->sdc_thread;  in sysdc_update_times()
426 	if (sdc->sdc_sleep_updates != 0 &&  in sysdc_update_times()
427 	    sdc->sdc_sleep_updates != sdc->sdc_nupdates) {  in sysdc_update_times()
428 		*newO = sdc->sdc_last_base_O;  in sysdc_update_times()
440 		++sdc->sdc_nupdates;  in sysdc_update_times()
441 		if ((sdc->sdc_nupdates % sysdc_reset_updates) == 0)  in sysdc_update_times()
448 			 * Start off our cycle count somewhere in the middle,  in sysdc_update_times()
456 			sdc->sdc_nupdates = (uint_t)((gethrtime() % 4999) %  in sysdc_update_times()
460 			baseO = sdc->sdc_base_O;  in sysdc_update_times()
461 			baseR = sdc->sdc_base_R;  in sysdc_update_times()
464 		mstate_systhread_times(t, &sdc->sdc_base_O, &sdc->sdc_base_R);  in sysdc_update_times()
465 		*newO = sdc->sdc_base_O;  in sysdc_update_times()
467 		sdc->sdc_reset = now;  in sysdc_update_times()
468 		sdc->sdc_pri_check = -1; /* force mismatch below */  in sysdc_update_times()
473 		if (baseO > sdc->sdc_base_O || baseR > sdc->sdc_base_R) {  in sysdc_update_times()
475 			baseO = sdc->sdc_base_O;  in sysdc_update_times()
476 			baseR = sdc->sdc_base_R;  in sysdc_update_times()
480 		*O = (sdc->sdc_base_O - baseO);  in sysdc_update_times()
481 		*R = (sdc->sdc_base_R - baseR);  in sysdc_update_times()
492 	if (!update && sdc->sdc_pri_check == now) {  in sysdc_update_times()
498 	sdc->sdc_pri_check = now;  in sysdc_update_times()
499 	mstate_systhread_times(t, &sdc->sdc_cur_O, &sdc->sdc_cur_R);  in sysdc_update_times()
500 	*newO = sdc->sdc_cur_O;  in sysdc_update_times()
512 	if (sdc->sdc_cur_O < sdc->sdc_base_O ||  in sysdc_update_times()
513 	    sdc->sdc_cur_R < sdc->sdc_base_R) {  in sysdc_update_times()
515 		sdc->sdc_base_O = sdc->sdc_cur_O;  in sysdc_update_times()
516 		sdc->sdc_base_R = sdc->sdc_cur_R;  in sysdc_update_times()
522 	*O = sdc->sdc_cur_O - sdc->sdc_base_O;  in sysdc_update_times()
523 	*R = sdc->sdc_cur_R - sdc->sdc_base_R;  in sysdc_update_times()
530  *	sdc->sdc_epri.  Returns 1 if a priority update should occur
537 	kthread_t *const t = sdc->sdc_thread;  in sysdc_compute_pri()
542 	hrtime_t	newO = -1;  in sysdc_compute_pri()
547 	ASSERT(!update || newO != -1);  in sysdc_compute_pri()
551 		sdc->sdc_cur_DC = (O * SYSDC_DC_MAX) / (O + R);  in sysdc_compute_pri()
554 		if (sdc->sdc_cur_DC < sdc->sdc_target_DC)  in sysdc_compute_pri()
555 			sdc->sdc_pri = sdc->sdc_maxpri;  in sysdc_compute_pri()
557 			sdc->sdc_pri = sdc->sdc_minpri;  in sysdc_compute_pri()
561 	 * If our per-pset duty cycle goes over the max, we will take a break.  in sysdc_compute_pri()
565 	if (sdc->sdc_pset->sdp_need_break) {  in sysdc_compute_pri()
567 		sdc->sdc_epri = sdc->sdc_minpri;  in sysdc_compute_pri()
569 		sdc->sdc_epri = sdc->sdc_pri;  in sysdc_compute_pri()
573 	    kthread_t *, t, pri_t, sdc->sdc_epri, uint_t, sdc->sdc_cur_DC,  in sysdc_compute_pri()
574 	    uint_t, sdc->sdc_target_DC);  in sysdc_compute_pri()
577 	 * For sysdc_update(), we compute the ONPROC time for high-priority  in sysdc_compute_pri()
578 	 * threads, which is used to calculate the per-pset duty cycle.  We  in sysdc_compute_pri()
588 		ASSERT(t->t_cpupart == sdc->sdc_pset->sdp_cpupart);  in sysdc_compute_pri()
591 		if (t->t_pri == sdc->sdc_maxpri &&  in sysdc_compute_pri()
592 		    sdc->sdc_last_base_O != 0 &&  in sysdc_compute_pri()
593 		    sdc->sdc_last_base_O < newO) {  in sysdc_compute_pri()
594 			sdc->sdc_last_O = newO - sdc->sdc_last_base_O;  in sysdc_compute_pri()
595 			sdc->sdc_pset->sdp_onproc_time +=  in sysdc_compute_pri()
596 			    (uint64_t)sdc->sdc_last_O;  in sysdc_compute_pri()
597 			sdc->sdc_pset->sdp_onproc_threads++;  in sysdc_compute_pri()
599 			sdc->sdc_last_O = 0;  in sysdc_compute_pri()
601 		sdc->sdc_last_base_O = newO;  in sysdc_compute_pri()
603 		sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks + 1;  in sysdc_compute_pri()
614 		ASSERT(sdc->sdc_ticks == sdc->sdc_update_ticks);  in sysdc_compute_pri()
615 		sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks;  in sysdc_compute_pri()
623 	return (sdc->sdc_epri != t->t_pri);  in sysdc_compute_pri()
629 	kthread_t *t = sdc->sdc_thread;  in sysdc_update_pri()
634 		if (!thread_change_pri(t, sdc->sdc_epri, 0)) {  in sysdc_update_pri()
647 	sysdc_t *volatile *headp = &SYSDC_LIST(sdc)->sdl_list;  in sysdc_activate()
649 	kthread_t	*t = sdc->sdc_thread;  in sysdc_activate()
653 	ASSERT(sdc->sdc_next == NULL);  in sysdc_activate()
658 		sdc->sdc_next = head;  in sysdc_activate()
686 	diff = now - sysdc_last_update;  in sysdc_update()
692 		boolean_t breaking = (cur->sdp_should_break != 0);  in sysdc_update()
694 		if (cur->sdp_need_break != breaking) {  in sysdc_update()
698 		cur->sdp_onproc_time = 0;  in sysdc_update()
699 		cur->sdp_onproc_threads = 0;  in sysdc_update()
700 		cur->sdp_need_break = breaking;  in sysdc_update()
706 		sysdc_t *volatile	*headp = &sdl->sdl_list;  in sysdc_update()
714 		mutex_enter(&sdl->sdl_lock);  in sysdc_update()
717 		 * Each sdl_list contains a singly-linked list of active  in sysdc_update()
729 			kthread_t	*const	t = sdc->sdc_thread;  in sysdc_update()
736 				*prevptr = sdc->sdc_next;  in sysdc_update()
738 				sdc->sdc_next = freelist;  in sysdc_update()
744 			if (t->t_cid != sysdccid) {  in sysdc_update()
746 				prevptr = &sdc->sdc_next;  in sysdc_update()
750 			ASSERT(t->t_cldata == sdc);  in sysdc_update()
757 			if (!(t->t_state & (TS_RUN | TS_ONPROC)) &&  in sysdc_update()
758 			    sdc->sdc_sleep_updates != 0 &&  in sysdc_update()
759 			    (sdc->sdc_sleep_updates - sdc->sdc_nupdates) >  in sysdc_update()
761 				*prevptr = sdc->sdc_next;  in sysdc_update()
763 				sdc->sdc_next = NULL;  in sysdc_update()
770 			prevptr = &sdc->sdc_next;  in sysdc_update()
782 		mutex_exit(&sdl->sdl_lock);  in sysdc_update()
789 		cur->sdp_vtime_last_interval =  in sysdc_update()
790 		    diff * cur->sdp_cpupart->cp_ncpus;  in sysdc_update()
791 		cur->sdp_DC_last_interval =  in sysdc_update()
792 		    (cur->sdp_onproc_time * SYSDC_DC_MAX) /  in sysdc_update()
793 		    cur->sdp_vtime_last_interval;  in sysdc_update()
795 		if (cur->sdp_should_break > 0) {  in sysdc_update()
796 			cur->sdp_should_break--;	/* breaking */  in sysdc_update()
799 		if (cur->sdp_dont_break > 0) {  in sysdc_update()
800 			cur->sdp_dont_break--;	/* waiting before checking */  in sysdc_update()
803 		if (cur->sdp_DC_last_interval > sysdc_max_pset_DC) {  in sysdc_update()
804 			cur->sdp_should_break = sysdc_break_updates;  in sysdc_update()
805 			cur->sdp_dont_break = sysdc_nobreak_updates;  in sysdc_update()
828 		freelist = cur->sdc_next;  in sysdc_update()
852 	if (t->t_cid != sysdccid) {  in sysdc_tick()
857 	sdc = t->t_cldata;  in sysdc_tick()
858 	if (t->t_state == TS_ONPROC &&  in sysdc_tick()
859 	    t->t_pri < t->t_disp_queue->disp_maxrunpri) {  in sysdc_tick()
863 	if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {  in sysdc_tick()
864 		ASSERT(sdc->sdc_sleep_updates == 0);  in sysdc_tick()
867 	ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);  in sysdc_tick()
868 	sdc->sdc_ticks++;  in sysdc_tick()
869 	if (sdc->sdc_ticks == sdc->sdc_update_ticks) {  in sysdc_tick()
872 		ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);  in sysdc_tick()
880 	sysdc_t *sdc = t->t_cldata;  in sysdc_setrun()
884 	sdc->sdc_sleep_updates = 0;  in sysdc_setrun()
886 	if (sdc->sdc_next == NULL) {  in sysdc_setrun()
892 			THREAD_CHANGE_PRI(t, sdc->sdc_epri);  in sysdc_setrun()
896 		ASSERT(sdc->sdc_next != NULL);  in sysdc_setrun()
911 	sysdc_t *sdc = t->t_cldata;  in sysdc_sleep()
915 	sdc->sdc_sleep_updates = sdc->sdc_nupdates;  in sysdc_sleep()
923 	cpupart_t *const cpupart = t->t_cpupart;  in sysdc_enterclass()
926 	sysdc_pset_t *newpset = sdc->sdc_pset;  in sysdc_enterclass()
930 	if (t->t_cid != syscid)  in sysdc_enterclass()
938 	ASSERT(sdpp->sdp_minpri >= sysdc_minpri);  in sysdc_enterclass()
939 	ASSERT(sdpp->sdp_maxpri <= sysdc_maxpri);  in sysdc_enterclass()
940 	ASSERT(sdpp->sdp_DC >= sysdc_minDC);  in sysdc_enterclass()
941 	ASSERT(sdpp->sdp_DC <= sysdc_maxDC);  in sysdc_enterclass()
943 	sdc->sdc_thread = t;  in sysdc_enterclass()
944 	sdc->sdc_pri = sdpp->sdp_maxpri;	/* start off maximally */  in sysdc_enterclass()
945 	sdc->sdc_minpri = sdpp->sdp_minpri;  in sysdc_enterclass()
946 	sdc->sdc_maxpri = sdpp->sdp_maxpri;  in sysdc_enterclass()
947 	sdc->sdc_target_DC = sdpp->sdp_DC;  in sysdc_enterclass()
948 	sdc->sdc_ticks = 0;  in sysdc_enterclass()
949 	sdc->sdc_update_ticks = sysdc_update_ticks + 1;  in sysdc_enterclass()
952 	sdc->sdc_pset = NULL;  in sysdc_enterclass()
956 		if (pset->sdp_cpupart == cpupart) {  in sysdc_enterclass()
963 		pset->sdp_cpupart = cpupart;  in sysdc_enterclass()
966 	pset->sdp_nthreads++;  in sysdc_enterclass()
967 	ASSERT(pset->sdp_nthreads > 0);  in sysdc_enterclass()
969 	sdc->sdc_pset = pset;  in sysdc_enterclass()
980 	t->t_clfuncs = &(sclass[cid].cl_funcs->thread);  in sysdc_enterclass()
981 	t->t_cid = cid;  in sysdc_enterclass()
982 	t->t_cldata = sdc;  in sysdc_enterclass()
983 	t->t_schedflag |= TS_RUNQMATCH;  in sysdc_enterclass()
999 	sysdc_pset_t *sdp = sdc->sdc_pset;  in sysdc_leave()
1003 	mutex_enter(&sdl->sdl_lock);		/* block sysdc_update() */  in sysdc_leave()
1004 	sdc->sdc_thread = NULL;  in sysdc_leave()
1005 	freedc = (sdc->sdc_next == NULL);  in sysdc_leave()
1006 	mutex_exit(&sdl->sdl_lock);  in sysdc_leave()
1010 	ASSERT(sdp->sdp_nthreads > 0);  in sysdc_leave()
1011 	--sdp->sdp_nthreads;  in sysdc_leave()
1012 	if (sdp->sdp_nthreads == 0) {  in sysdc_leave()
1046 	ASSERT(t->t_cid == sysdccid);  in sysdc_exit()
1047 	sdc = t->t_cldata;  in sysdc_exit()
1048 	t->t_cid = syscid;  in sysdc_exit()
1049 	t->t_cldata = NULL;  in sysdc_exit()
1050 	t->t_clfuncs = &(sclass[syscid].cl_funcs->thread);  in sysdc_exit()
1052 	t->t_schedflag &= ~TS_RUNQMATCH;  in sysdc_exit()
1084 	return (t->t_epri);  in sysdc_globpri()
1092 	return (-1);  in sysdc_no_swap()
1101 	pcprip->pc_clpmax = sysdc_maxpri;  in sysdc_getclpri()
1102 	pcprip->pc_clpmin = sysdc_minpri;  in sysdc_getclpri()
1110 	return (0);		/* no class-specific info */  in sysdc_getclinfo()
1123 	if ((new->sdc_pset = kmem_zalloc(sizeof (*new->sdc_pset), flag)) ==  in sysdc_alloc()
1142 		ASSERT(sdc->sdc_pset != NULL);  in sysdc_free()
1143 		ASSERT(sdc->sdc_pset->sdp_cpupart == NULL);  in sysdc_free()
1144 		kmem_free(sdc->sdc_pset, sizeof (*sdc->sdc_pset));  in sysdc_free()
1149 static int sysdc_enosys();	/* Boy, ANSI-C's K&R compatibility is weird. */
1240 	&mod_schedops, "system duty cycle scheduling class", &csw
1265 /* --- consolidation-private interfaces --- */
1285 	ASSERT(t->t_lwp != NULL);  in sysdc_thread_enter()
1286 	ASSERT(t->t_cid == syscid);  in sysdc_thread_enter()
1287 	ASSERT(t->t_cldata == NULL);  in sysdc_thread_enter()