321 #define	SYSDC_HASH(sdc)	(((uintptr_t)(sdc) >> 6) & (SYSDC_NLISTS - 1))  argument
323 #define	SYSDC_LIST(sdc)		(&sysdc_active[SYSDC_HASH(sdc)])  argument
412 sysdc_update_times(sysdc_t *sdc, uint_t flags,  in sysdc_update_times()  argument
415 	kthread_t *const t = sdc->sdc_thread;  in sysdc_update_times()
426 	if (sdc->sdc_sleep_updates != 0 &&  in sysdc_update_times()
427 	    sdc->sdc_sleep_updates != sdc->sdc_nupdates) {  in sysdc_update_times()
428 		*newO = sdc->sdc_last_base_O;  in sysdc_update_times()
440 		++sdc->sdc_nupdates;  in sysdc_update_times()
441 		if ((sdc->sdc_nupdates % sysdc_reset_updates) == 0)  in sysdc_update_times()
456 			sdc->sdc_nupdates = (uint_t)((gethrtime() % 4999) %  in sysdc_update_times()
460 			baseO = sdc->sdc_base_O;  in sysdc_update_times()
461 			baseR = sdc->sdc_base_R;  in sysdc_update_times()
464 		mstate_systhread_times(t, &sdc->sdc_base_O, &sdc->sdc_base_R);  in sysdc_update_times()
465 		*newO = sdc->sdc_base_O;  in sysdc_update_times()
467 		sdc->sdc_reset = now;  in sysdc_update_times()
468 		sdc->sdc_pri_check = -1; /* force mismatch below */  in sysdc_update_times()
473 		if (baseO > sdc->sdc_base_O || baseR > sdc->sdc_base_R) {  in sysdc_update_times()
475 			baseO = sdc->sdc_base_O;  in sysdc_update_times()
476 			baseR = sdc->sdc_base_R;  in sysdc_update_times()
480 		*O = (sdc->sdc_base_O - baseO);  in sysdc_update_times()
481 		*R = (sdc->sdc_base_R - baseR);  in sysdc_update_times()
492 	if (!update && sdc->sdc_pri_check == now) {  in sysdc_update_times()
498 	sdc->sdc_pri_check = now;  in sysdc_update_times()
499 	mstate_systhread_times(t, &sdc->sdc_cur_O, &sdc->sdc_cur_R);  in sysdc_update_times()
500 	*newO = sdc->sdc_cur_O;  in sysdc_update_times()
512 	if (sdc->sdc_cur_O < sdc->sdc_base_O ||  in sysdc_update_times()
513 	    sdc->sdc_cur_R < sdc->sdc_base_R) {  in sysdc_update_times()
515 		sdc->sdc_base_O = sdc->sdc_cur_O;  in sysdc_update_times()
516 		sdc->sdc_base_R = sdc->sdc_cur_R;  in sysdc_update_times()
522 	*O = sdc->sdc_cur_O - sdc->sdc_base_O;  in sysdc_update_times()
523 	*R = sdc->sdc_cur_R - sdc->sdc_base_R;  in sysdc_update_times()
535 sysdc_compute_pri(sysdc_t *sdc, uint_t flags)  in sysdc_compute_pri()  argument
537 	kthread_t *const t = sdc->sdc_thread;  in sysdc_compute_pri()
546 	sysdc_update_times(sdc, flags, &O, &R, &newO);  in sysdc_compute_pri()
551 		sdc->sdc_cur_DC = (O * SYSDC_DC_MAX) / (O + R);  in sysdc_compute_pri()
554 		if (sdc->sdc_cur_DC < sdc->sdc_target_DC)  in sysdc_compute_pri()
555 			sdc->sdc_pri = sdc->sdc_maxpri;  in sysdc_compute_pri()
557 			sdc->sdc_pri = sdc->sdc_minpri;  in sysdc_compute_pri()
565 	if (sdc->sdc_pset->sdp_need_break) {  in sysdc_compute_pri()
567 		sdc->sdc_epri = sdc->sdc_minpri;  in sysdc_compute_pri()
569 		sdc->sdc_epri = sdc->sdc_pri;  in sysdc_compute_pri()
573 	    kthread_t *, t, pri_t, sdc->sdc_epri, uint_t, sdc->sdc_cur_DC,  in sysdc_compute_pri()
574 	    uint_t, sdc->sdc_target_DC);  in sysdc_compute_pri()
588 		ASSERT(t->t_cpupart == sdc->sdc_pset->sdp_cpupart);  in sysdc_compute_pri()
591 		if (t->t_pri == sdc->sdc_maxpri &&  in sysdc_compute_pri()
592 		    sdc->sdc_last_base_O != 0 &&  in sysdc_compute_pri()
593 		    sdc->sdc_last_base_O < newO) {  in sysdc_compute_pri()
594 			sdc->sdc_last_O = newO - sdc->sdc_last_base_O;  in sysdc_compute_pri()
595 			sdc->sdc_pset->sdp_onproc_time +=  in sysdc_compute_pri()
596 			    (uint64_t)sdc->sdc_last_O;  in sysdc_compute_pri()
597 			sdc->sdc_pset->sdp_onproc_threads++;  in sysdc_compute_pri()
599 			sdc->sdc_last_O = 0;  in sysdc_compute_pri()
601 		sdc->sdc_last_base_O = newO;  in sysdc_compute_pri()
603 		sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks + 1;  in sysdc_compute_pri()
614 		ASSERT(sdc->sdc_ticks == sdc->sdc_update_ticks);  in sysdc_compute_pri()
615 		sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks;  in sysdc_compute_pri()
623 	return (sdc->sdc_epri != t->t_pri);  in sysdc_compute_pri()
627 sysdc_update_pri(sysdc_t *sdc, uint_t flags)  in sysdc_update_pri()  argument
629 	kthread_t *t = sdc->sdc_thread;  in sysdc_update_pri()
633 	if (sysdc_compute_pri(sdc, flags)) {  in sysdc_update_pri()
634 		if (!thread_change_pri(t, sdc->sdc_epri, 0)) {  in sysdc_update_pri()
645 sysdc_activate(sysdc_t *sdc)  in sysdc_activate()  argument
647 	sysdc_t *volatile *headp = &SYSDC_LIST(sdc)->sdl_list;  in sysdc_activate()
649 	kthread_t	*t = sdc->sdc_thread;  in sysdc_activate()
653 	ASSERT(sdc->sdc_next == NULL);  in sysdc_activate()
658 		sdc->sdc_next = head;  in sysdc_activate()
659 	} while (atomic_cas_ptr(headp, head, sdc) != head);  in sysdc_activate()
728 			sysdc_t		*const	sdc = *prevptr;  in sysdc_update()  local
729 			kthread_t	*const	t = sdc->sdc_thread;  in sysdc_update()
736 				*prevptr = sdc->sdc_next;  in sysdc_update()
738 				sdc->sdc_next = freelist;  in sysdc_update()
739 				freelist = sdc;  in sysdc_update()
746 				prevptr = &sdc->sdc_next;  in sysdc_update()
750 			ASSERT(t->t_cldata == sdc);  in sysdc_update()
758 			    sdc->sdc_sleep_updates != 0 &&  in sysdc_update()
759 			    (sdc->sdc_sleep_updates - sdc->sdc_nupdates) >  in sysdc_update()
761 				*prevptr = sdc->sdc_next;  in sysdc_update()
763 				sdc->sdc_next = NULL;  in sysdc_update()
767 			sysdc_update_pri(sdc, SDC_UPDATE_TIMEOUT);  in sysdc_update()
770 			prevptr = &sdc->sdc_next;  in sysdc_update()
849 	sysdc_t *sdc;  in sysdc_tick()  local
857 	sdc = t->t_cldata;  in sysdc_tick()
864 		ASSERT(sdc->sdc_sleep_updates == 0);  in sysdc_tick()
867 	ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);  in sysdc_tick()
868 	sdc->sdc_ticks++;  in sysdc_tick()
869 	if (sdc->sdc_ticks == sdc->sdc_update_ticks) {  in sysdc_tick()
871 		sysdc_update_pri(sdc, SDC_UPDATE_TICK);  in sysdc_tick()
872 		ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);  in sysdc_tick()
880 	sysdc_t *sdc = t->t_cldata;  in sysdc_setrun()  local
884 	sdc->sdc_sleep_updates = 0;  in sysdc_setrun()
886 	if (sdc->sdc_next == NULL) {  in sysdc_setrun()
891 		if (sysdc_compute_pri(sdc, 0)) {  in sysdc_setrun()
892 			THREAD_CHANGE_PRI(t, sdc->sdc_epri);  in sysdc_setrun()
894 		sysdc_activate(sdc);  in sysdc_setrun()
896 		ASSERT(sdc->sdc_next != NULL);  in sysdc_setrun()
911 	sysdc_t *sdc = t->t_cldata;  in sysdc_sleep()  local
915 	sdc->sdc_sleep_updates = sdc->sdc_nupdates;  in sysdc_sleep()
924 	sysdc_t *sdc = bufp;  in sysdc_enterclass()  local
926 	sysdc_pset_t *newpset = sdc->sdc_pset;  in sysdc_enterclass()
943 	sdc->sdc_thread = t;  in sysdc_enterclass()
944 	sdc->sdc_pri = sdpp->sdp_maxpri;	/* start off maximally */  in sysdc_enterclass()
945 	sdc->sdc_minpri = sdpp->sdp_minpri;  in sysdc_enterclass()
946 	sdc->sdc_maxpri = sdpp->sdp_maxpri;  in sysdc_enterclass()
947 	sdc->sdc_target_DC = sdpp->sdp_DC;  in sysdc_enterclass()
948 	sdc->sdc_ticks = 0;  in sysdc_enterclass()
949 	sdc->sdc_update_ticks = sysdc_update_ticks + 1;  in sysdc_enterclass()
952 	sdc->sdc_pset = NULL;  in sysdc_enterclass()
969 	sdc->sdc_pset = pset;  in sysdc_enterclass()
982 	t->t_cldata = sdc;  in sysdc_enterclass()
985 	sysdc_update_pri(sdc, SDC_UPDATE_INITIAL);  in sysdc_enterclass()
997 sysdc_leave(sysdc_t *sdc)  in sysdc_leave()  argument
999 	sysdc_pset_t *sdp = sdc->sdc_pset;  in sysdc_leave()
1000 	sysdc_list_t *sdl = SYSDC_LIST(sdc);  in sysdc_leave()
1004 	sdc->sdc_thread = NULL;  in sysdc_leave()
1005 	freedc = (sdc->sdc_next == NULL);  in sysdc_leave()
1020 		kmem_free(sdc, sizeof (*sdc));  in sysdc_leave()
1042 	sysdc_t *sdc;  in sysdc_exit()  local
1047 	sdc = t->t_cldata;  in sysdc_exit()
1056 	sysdc_leave(sdc);  in sysdc_exit()
1135 	sysdc_t *sdc = p;  in sysdc_free()  local
1137 	if (sdc != NULL) {  in sysdc_free()
1142 		ASSERT(sdc->sdc_pset != NULL);  in sysdc_free()
1143 		ASSERT(sdc->sdc_pset->sdp_cpupart == NULL);  in sysdc_free()
1144 		kmem_free(sdc->sdc_pset, sizeof (*sdc->sdc_pset));  in sysdc_free()
1145 		kmem_free(sdc, sizeof (*sdc));  in sysdc_free()