97 #define	FX_CB_LIST_INSERT(fxpp)						\  argument
99 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);			\
103 	fxpp->fx_cb_next = headp->fx_cb_next;				\
104 	fxpp->fx_cb_prev = headp;					\
105 	headp->fx_cb_next->fx_cb_prev = fxpp;				\
106 	headp->fx_cb_next = fxpp;					\
113 #define	FX_CB_LIST_DELETE(fxpp)						\  argument
115 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);			\
118 	fxpp->fx_cb_prev->fx_cb_next = fxpp->fx_cb_next;		\
119 	fxpp->fx_cb_next->fx_cb_prev = fxpp->fx_cb_prev;		\
123 #define	FX_HAS_CB(fxpp)	(fxpp->fx_callback != NULL)  argument
444 	fxproc_t	*fxpp;  in fx_enterclass()  local
448 	fxpp = (fxproc_t *)bufp;  in fx_enterclass()
449 	ASSERT(fxpp != NULL);  in fx_enterclass()
454 	fxpp->fx_flags = 0;  in fx_enterclass()
455 	fxpp->fx_callback = NULL;  in fx_enterclass()
456 	fxpp->fx_cookie = 0;  in fx_enterclass()
462 		fxpp->fx_pri = fxpp->fx_uprilim = 0;  in fx_enterclass()
463 		fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;  in fx_enterclass()
464 		fxpp->fx_nice =  NZERO;  in fx_enterclass()
498 		fxpp->fx_uprilim = reqfxuprilim;  in fx_enterclass()
499 		fxpp->fx_pri = reqfxupri;  in fx_enterclass()
501 		fxpp->fx_nice = NZERO - (NZERO * reqfxupri) / fx_maxupri;  in fx_enterclass()
505 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;  in fx_enterclass()
511 				fxpp->fx_pquantum = FX_TQINF;  in fx_enterclass()
513 				fxpp->fx_pquantum = fxkparmsp->fx_tqntm;  in fx_enterclass()
519 	fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_enterclass()
520 	cpucaps_sc_init(&fxpp->fx_caps);  in fx_enterclass()
521 	fxpp->fx_tp = t;  in fx_enterclass()
526 	t->t_cldata = (void *)fxpp;  in fx_enterclass()
528 	fx_change_priority(t, fxpp);  in fx_enterclass()
540 	fxproc_t *fxpp;  in fx_exit()  local
543 	fxpp = (fxproc_t *)(t->t_cldata);  in fx_exit()
552 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ONLY);  in fx_exit()
554 	if (FX_HAS_CB(fxpp)) {  in fx_exit()
555 		FX_CB_EXIT(FX_CALLB(fxpp), fxpp->fx_cookie);  in fx_exit()
556 		fxpp->fx_callback = NULL;  in fx_exit()
557 		fxpp->fx_cookie = 0;  in fx_exit()
559 		FX_CB_LIST_DELETE(fxpp);  in fx_exit()
572 	fxproc_t *fxpp = (fxproc_t *)procp;  in fx_exitclass()  local
574 	thread_lock(fxpp->fx_tp);  in fx_exitclass()
575 	if (FX_HAS_CB(fxpp)) {  in fx_exitclass()
577 		FX_CB_EXIT(FX_CALLB(fxpp), fxpp->fx_cookie);  in fx_exitclass()
579 		fxpp->fx_callback = NULL;  in fx_exitclass()
580 		fxpp->fx_cookie = 0;  in fx_exitclass()
581 		thread_unlock(fxpp->fx_tp);  in fx_exitclass()
582 		FX_CB_LIST_DELETE(fxpp);  in fx_exitclass()
584 		thread_unlock(fxpp->fx_tp);  in fx_exitclass()
586 	kmem_free(fxpp, sizeof (fxproc_t));  in fx_exitclass()
650 	fxproc_t *fxpp;  in fx_forkret()  local
668 	fxpp = (fxproc_t *)(t->t_cldata);  in fx_forkret()
669 	t->t_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;  in fx_forkret()
723 	fxproc_t *fxpp = (fxproc_t *)t->t_cldata;  in fx_parmsget()  local
726 	fxkparmsp->fx_upri = fxpp->fx_pri;  in fx_parmsget()
727 	fxkparmsp->fx_uprilim = fxpp->fx_uprilim;  in fx_parmsget()
728 	fxkparmsp->fx_tqntm = fxpp->fx_pquantum;  in fx_parmsget()
1018 	fxproc_t	*fxpp;  in fx_parmsset()  local
1024 	fxpp = (fxproc_t *)tx->t_cldata;  in fx_parmsset()
1027 		reqfxuprilim = fxpp->fx_uprilim;  in fx_parmsset()
1047 	    (reqfxuprilim > fxpp->fx_uprilim ||  in fx_parmsset()
1057 		reqfxupri = fxpp->fx_pri;  in fx_parmsset()
1079 	fxpp->fx_uprilim = reqfxuprilim;  in fx_parmsset()
1080 	fxpp->fx_pri = reqfxupri;  in fx_parmsset()
1083 		fxpp->fx_pquantum = FX_TQINF;  in fx_parmsset()
1085 		fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;  in fx_parmsset()
1087 		fxpp->fx_pquantum = fxkparmsp->fx_tqntm;  in fx_parmsset()
1089 	fxpp->fx_nice = nice;  in fx_parmsset()
1091 	fx_change_priority(tx, fxpp);  in fx_parmsset()
1104 	fxproc_t *fxpp;  in fx_globpri()  local
1108 	fxpp = (fxproc_t *)t->t_cldata;  in fx_globpri()
1109 	return (fx_dptbl[fxpp->fx_pri].fx_globpri);  in fx_globpri()
1122 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_preempt()  local
1127 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);  in fx_preempt()
1141 		if (fxpp->fx_pquantum == FX_TQINF ||  in fx_preempt()
1142 		    fxpp->fx_timeleft > -SC_MAX_TICKS) {  in fx_preempt()
1156 	if (FX_HAS_CB(fxpp)) {  in fx_preempt()
1157 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;  in fx_preempt()
1158 		pri_t	newpri = fxpp->fx_pri;  in fx_preempt()
1159 		FX_CB_PREEMPT(FX_CALLB(fxpp), fxpp->fx_cookie,  in fx_preempt()
1162 		if ((int)new_quantum != fxpp->fx_pquantum) {  in fx_preempt()
1163 			fxpp->fx_pquantum = (int)new_quantum;  in fx_preempt()
1164 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_preempt()
1167 		fxpp->fx_pri = newpri;  in fx_preempt()
1168 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);  in fx_preempt()
1179 	if ((fxpp->fx_flags & (FXBACKQ)) == FXBACKQ) {  in fx_preempt()
1180 		fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_preempt()
1181 		fxpp->fx_flags &= ~FXBACKQ;  in fx_preempt()
1191 	fxproc_t *fxpp = (fxproc_t *)(t->t_cldata);  in fx_setrun()  local
1194 	fxpp->fx_flags &= ~FXBACKQ;  in fx_setrun()
1210 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_sleep()  local
1218 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);  in fx_sleep()
1220 	if (FX_HAS_CB(fxpp)) {  in fx_sleep()
1221 		FX_CB_SLEEP(FX_CALLB(fxpp), fxpp->fx_cookie);  in fx_sleep()
1272 	fxproc_t *fxpp = (fxproc_t *)(t->t_cldata);  in fx_stop()  local
1276 	if (FX_HAS_CB(fxpp)) {  in fx_stop()
1277 		FX_CB_STOP(FX_CALLB(fxpp), fxpp->fx_cookie);  in fx_stop()
1289 	fxproc_t *fxpp;  in fx_tick()  local
1295 	fxpp = (fxproc_t *)(t->t_cldata);  in fx_tick()
1297 	if (FX_HAS_CB(fxpp)) {  in fx_tick()
1298 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;  in fx_tick()
1299 		pri_t	newpri = fxpp->fx_pri;  in fx_tick()
1300 		FX_CB_TICK(FX_CALLB(fxpp), fxpp->fx_cookie,  in fx_tick()
1303 		if ((int)new_quantum != fxpp->fx_pquantum) {  in fx_tick()
1304 			fxpp->fx_pquantum = (int)new_quantum;  in fx_tick()
1305 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_tick()
1308 		if (newpri != fxpp->fx_pri) {  in fx_tick()
1309 			fxpp->fx_pri = newpri;  in fx_tick()
1310 			fx_change_priority(t, fxpp);  in fx_tick()
1318 	call_cpu_surrender =  CPUCAPS_CHARGE(t, &fxpp->fx_caps,  in fx_tick()
1321 	if ((fxpp->fx_pquantum != FX_TQINF) &&  in fx_tick()
1322 	    (--fxpp->fx_timeleft <= 0)) {  in fx_tick()
1332 			if (fxpp->fx_timeleft > -SC_MAX_TICKS) {  in fx_tick()
1342 		new_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;  in fx_tick()
1354 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_tick()
1364 		fxpp->fx_flags |= FXBACKQ;  in fx_tick()
1389 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_wakeup()  local
1394 	if (FX_HAS_CB(fxpp)) {  in fx_wakeup()
1395 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;  in fx_wakeup()
1396 		pri_t	newpri = fxpp->fx_pri;  in fx_wakeup()
1397 		FX_CB_WAKEUP(FX_CALLB(fxpp), fxpp->fx_cookie,  in fx_wakeup()
1400 		if ((int)new_quantum != fxpp->fx_pquantum) {  in fx_wakeup()
1401 			fxpp->fx_pquantum = (int)new_quantum;  in fx_wakeup()
1402 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_wakeup()
1406 		if (newpri != fxpp->fx_pri) {  in fx_wakeup()
1407 			fxpp->fx_pri = newpri;  in fx_wakeup()
1408 			THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);  in fx_wakeup()
1412 	fxpp->fx_flags &= ~FXBACKQ;  in fx_wakeup()
1427 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_yield()  local
1435 	(void) CPUCAPS_CHARGE(t, &fxpp->fx_caps, CPUCAPS_CHARGE_ENFORCE);  in fx_yield()
1437 	if (FX_HAS_CB(fxpp))  {  in fx_yield()
1438 		clock_t new_quantum =  (clock_t)fxpp->fx_pquantum;  in fx_yield()
1439 		pri_t	newpri = fxpp->fx_pri;  in fx_yield()
1440 		FX_CB_PREEMPT(FX_CALLB(fxpp), fxpp->fx_cookie,  in fx_yield()
1443 		if ((int)new_quantum != fxpp->fx_pquantum) {  in fx_yield()
1444 			fxpp->fx_pquantum = (int)new_quantum;  in fx_yield()
1445 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_yield()
1448 		fxpp->fx_pri = newpri;  in fx_yield()
1449 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);  in fx_yield()
1459 	if (fxpp->fx_timeleft <= 0) {  in fx_yield()
1464 		DTRACE_SCHED1(schedctl__yield, int, -fxpp->fx_timeleft);  in fx_yield()
1465 		fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_yield()
1466 		THREAD_CHANGE_PRI(t, fx_dptbl[fxpp->fx_pri].fx_globpri);  in fx_yield()
1470 	fxpp->fx_flags &= ~FXBACKQ;  in fx_yield()
1482 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_donice()  local
1490 			*retvalp = fxpp->fx_nice - NZERO;  in fx_donice()
1509 	newnice = fxpp->fx_nice + incr;  in fx_donice()
1537 	fxpp->fx_nice = (char)newnice;  in fx_donice()
1553 	fxproc_t	*fxpp = (fxproc_t *)(t->t_cldata);  in fx_doprio()  local
1560 		*retvalp = fxpp->fx_pri;  in fx_doprio()
1564 	newpri = fxpp->fx_pri + incr;  in fx_doprio()
1580 fx_change_priority(kthread_t *t, fxproc_t *fxpp)  in fx_change_priority()  argument
1585 	new_pri = fx_dptbl[fxpp->fx_pri].fx_globpri;  in fx_change_priority()
1587 	t->t_cpri = fxpp->fx_pri;  in fx_change_priority()
1595 			fxpp->fx_flags |= FXBACKQ;  in fx_change_priority()
1598 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_change_priority()
1614 			fxpp->fx_timeleft = fxpp->fx_pquantum;  in fx_change_priority()
1616 			fxpp->fx_flags |= FXBACKQ;  in fx_change_priority()
1646 fx_list_release(fxproc_t *fxpp)  in fx_list_release()  argument
1648 	int index = FX_CB_LIST_HASH(fxpp->fx_ktid);  in fx_list_release()
1658 	fxproc_t *fxpp;  in fx_list_lookup()  local
1662 	for (fxpp = fx_cb_plisthead[index].fx_cb_next;  in fx_list_lookup()
1663 	    fxpp != &fx_cb_plisthead[index]; fxpp = fxpp->fx_cb_next) {  in fx_list_lookup()
1664 		if (fxpp->fx_tp->t_cid == fx_cid && fxpp->fx_ktid == ktid &&  in fx_list_lookup()
1665 		    fxpp->fx_callback != NULL) {  in fx_list_lookup()
1671 			return (fxpp);  in fx_list_lookup()
1688 	fxproc_t	*fxpp;  in fx_register_callbacks()  local
1709 	fxpp = (fxproc_t *)(curthread->t_cldata);  in fx_register_callbacks()
1710 	ASSERT(fxpp != NULL);  in fx_register_callbacks()
1711 	if (FX_HAS_CB(fxpp)) {  in fx_register_callbacks()
1716 	fxpp->fx_callback = fx_callback;  in fx_register_callbacks()
1717 	fxpp->fx_cookie = cookie;  in fx_register_callbacks()
1720 		fxpp->fx_pri = pri;  in fx_register_callbacks()
1721 		FX_ADJUST_PRI(fxpp->fx_pri);  in fx_register_callbacks()
1723 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;  in fx_register_callbacks()
1725 			fxpp->fx_pquantum = FX_TQINF;  in fx_register_callbacks()
1728 			fxpp->fx_pquantum = quantum;  in fx_register_callbacks()
1732 			fxpp->fx_pquantum = FX_TQINF;  in fx_register_callbacks()
1735 			fxpp->fx_pquantum = quantum;  in fx_register_callbacks()
1739 	fxpp->fx_ktid = ddi_get_kt_did();  in fx_register_callbacks()
1741 	fx_change_priority(curthread, fxpp);  in fx_register_callbacks()
1748 	FX_CB_LIST_INSERT(fxpp);  in fx_register_callbacks()
1756 	fxproc_t	*fxpp;  in fx_unregister_callbacks()  local
1758 	if ((fxpp = fx_list_lookup(ddi_get_kt_did())) == NULL) {  in fx_unregister_callbacks()
1765 	thread_lock(fxpp->fx_tp);  in fx_unregister_callbacks()
1766 	fxpp->fx_callback = NULL;  in fx_unregister_callbacks()
1767 	fxpp->fx_cookie = 0;  in fx_unregister_callbacks()
1768 	thread_unlock(fxpp->fx_tp);  in fx_unregister_callbacks()
1769 	fx_list_release(fxpp);  in fx_unregister_callbacks()
1771 	FX_CB_LIST_DELETE(fxpp);  in fx_unregister_callbacks()
1781 	fxproc_t	*fxpp;  in fx_modify_priority()  local
1786 	if ((fxpp = fx_list_lookup(ktid)) == NULL) {  in fx_modify_priority()
1794 	thread_lock(fxpp->fx_tp);  in fx_modify_priority()
1797 		fxpp->fx_pri = pri;  in fx_modify_priority()
1798 		FX_ADJUST_PRI(fxpp->fx_pri);  in fx_modify_priority()
1800 			fxpp->fx_pquantum = fx_dptbl[fxpp->fx_pri].fx_quantum;  in fx_modify_priority()
1802 			fxpp->fx_pquantum = FX_TQINF;  in fx_modify_priority()
1805 			fxpp->fx_pquantum = quantum;  in fx_modify_priority()
1809 			fxpp->fx_pquantum = FX_TQINF;  in fx_modify_priority()
1812 			fxpp->fx_pquantum = quantum;  in fx_modify_priority()
1816 	fx_change_priority(fxpp->fx_tp, fxpp);  in fx_modify_priority()
1818 	thread_unlock(fxpp->fx_tp);  in fx_modify_priority()
1819 	fx_list_release(fxpp);  in fx_modify_priority()