Lines Matching refs:tp
101 void disp_swapped_enq(kthread_t *tp);
102 static void disp_swapped_setrun(kthread_t *tp);
114 static void setkpdq(kthread_t *tp, int borf);
156 static kthread_t *disp_ratify(kthread_t *tp, disp_t *kpq);
723 kthread_t *tp;
746 (tp = disp_getbest(kpq)) != NULL) {
747 if (disp_ratify(tp, kpq) != NULL) {
749 "disp_end:tid %p", tp);
750 return (tp);
767 if ((tp = disp_getwork(cpup)) == NULL ||
768 tp == T_DONTSTEAL) {
769 tp = cpup->cpu_idle_thread;
771 THREAD_ONPROC(tp, cpup);
772 cpup->cpu_dispthread = tp;
779 tp = cpup->cpu_idle_thread;
780 THREAD_ONPROC(tp, cpup);
781 cpup->cpu_dispthread = tp;
787 "disp_end:tid %p", tp);
788 return (tp);
792 tp = dq->dq_first;
794 ASSERT(tp != NULL);
795 ASSERT(tp->t_schedflag & TS_LOAD); /* thread must be swapped in */
797 DTRACE_SCHED2(dequeue, kthread_t *, tp, disp_t *, dp);
804 if ((dq->dq_first = tp->t_link) == NULL) {
831 tp->t_link = NULL;
839 tp->t_schedflag |= TS_DONT_SWAP;
840 cpup->cpu_dispthread = tp; /* protected by spl only */
842 ASSERT(pri == DISP_PRIO(tp));
843 thread_onproc(tp, cpup); /* set t_state to TS_ONPROC */
846 ASSERT(tp != NULL);
848 "disp_end:tid %p", tp);
850 if (disp_ratify(tp, kpq) == NULL)
853 return (tp);
1015 * Search the given dispatch queues for thread tp.
1016 * Return 1 if tp is found, otherwise return 0.
1019 search_disp_queues(disp_t *dp, kthread_t *tp)
1032 if (tp == rp) {
1045 * queues for thread tp. Return 1 if tp is found, otherwise return 0.
1048 thread_on_queue(kthread_t *tp)
1056 * Search the per-CPU dispatch queues for tp.
1060 if (search_disp_queues(cp->cpu_disp, tp))
1065 * Search the partition-wide kpreempt queues for tp.
1069 if (search_disp_queues(&part->cp_kp_queue, tp))
1078 #define thread_on_queue(tp) 0 /* ASSERT must be !thread_on_queue */
1197 setbackdq(kthread_t *tp)
1206 ASSERT(THREAD_LOCK_HELD(tp));
1207 ASSERT((tp->t_schedflag & TS_ALLSTART) == 0);
1208 ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */
1214 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) {
1215 disp_swapped_setrun(tp);
1219 self = (tp == curthread);
1221 if (tp->t_bound_cpu || tp->t_weakbound_cpu)
1226 tpri = DISP_PRIO(tp);
1228 cp = tp->t_cpu;
1231 setkpdq(tp, SETKP_BACK);
1243 if ((!THREAD_HAS_CACHE_WARMTH(tp)) ||
1244 (tp->t_cpu == cpu_inmotion)) {
1245 cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri, NULL);
1246 } else if (!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, tp->t_cpu)) {
1247 cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri,
1248 self ? tp->t_cpu : NULL);
1250 cp = tp->t_cpu;
1253 if (tp->t_cpupart == cp->cpu_part) {
1259 cp = cmt_balance(tp, cp);
1266 !(tp->t_schedflag & TS_RUNQMATCH))
1271 if (tp->t_lpl->lpl_lgrpid == LGRP_ROOTID) {
1279 kthread_t *, tp,
1288 cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist,
1289 tp->t_lpl, tp->t_pri, NULL);
1299 cp = tp->t_weakbound_cpu ?
1300 tp->t_weakbound_cpu : tp->t_bound_cpu;
1310 if ((!self) && (tp->t_waitrq == 0)) {
1314 (void) cpu_update_pct(tp, curtime);
1315 tp->t_waitrq = curtime;
1317 (void) cpu_update_pct(tp, gethrtime_unscaled());
1323 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 0);
1325 tpri, cp, tp);
1330 tnf_thread_queue(tp, cp, tpri);
1335 THREAD_RUN(tp, &dp->disp_lock); /* set t_state to TS_RUN */
1336 tp->t_disp_queue = dp;
1337 tp->t_link = NULL;
1347 dq->dq_last->t_link = tp;
1348 dq->dq_last = tp;
1352 dq->dq_first = dq->dq_last = tp;
1387 setfrontdq(kthread_t *tp)
1395 ASSERT(THREAD_LOCK_HELD(tp));
1396 ASSERT((tp->t_schedflag & TS_ALLSTART) == 0);
1397 ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */
1403 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) {
1404 disp_swapped_setrun(tp);
1408 if (tp->t_bound_cpu || tp->t_weakbound_cpu)
1413 tpri = DISP_PRIO(tp);
1415 cp = tp->t_cpu;
1418 setkpdq(tp, SETKP_FRONT);
1421 cp = tp->t_cpu;
1422 if (tp->t_cpupart == cp->cpu_part) {
1434 if ((!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, cp)) ||
1436 cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri,
1437 (tp == curthread) ? cp : NULL);
1439 (!THREAD_HAS_CACHE_WARMTH(tp))) {
1440 cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri,
1447 cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist,
1448 tp->t_lpl, tp->t_pri, NULL);
1458 cp = tp->t_weakbound_cpu ?
1459 tp->t_weakbound_cpu : tp->t_bound_cpu;
1470 if ((tp != curthread) && (tp->t_waitrq == 0)) {
1474 (void) cpu_update_pct(tp, curtime);
1475 tp->t_waitrq = curtime;
1477 (void) cpu_update_pct(tp, gethrtime_unscaled());
1483 TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, tp);
1484 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 1);
1489 tnf_thread_queue(tp, cp, tpri);
1494 THREAD_RUN(tp, &dp->disp_lock); /* set TS_RUN state and lock */
1495 tp->t_disp_queue = dp;
1505 tp->t_link = dq->dq_first;
1506 dq->dq_first = tp;
1510 tp->t_link = NULL;
1511 dq->dq_first = dq->dq_last = tp;
1521 if (tp == curthread && dp->disp_max_unbound_pri == -1 &&
1542 setkpdq(kthread_t *tp, int borf)
1549 tpri = DISP_PRIO(tp);
1551 dp = &tp->t_cpupart->cp_kp_queue;
1554 TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, tp);
1557 DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, borf);
1558 THREAD_RUN(tp, &dp->disp_lock); /* set t_state to TS_RUN */
1559 tp->t_disp_queue = dp;
1566 tp->t_link = NULL;
1567 dq->dq_last->t_link = tp;
1568 dq->dq_last = tp;
1571 tp->t_link = dq->dq_first;
1572 dq->dq_first = tp;
1578 dq->dq_first = dq->dq_last = tp;
1582 tp->t_link = NULL;
1583 dq->dq_first = dq->dq_last = tp;
1594 cp = tp->t_cpu;
1595 if (tp->t_cpupart != cp->cpu_part) {
1597 cp = tp->t_cpupart->cp_cpulist;
1599 cp = disp_lowpri_cpu(cp, tp->t_lpl, tp->t_pri, NULL);
1606 tnf_thread_queue(tp, cp, tpri);
1622 dispdeq(kthread_t *tp)
1631 ASSERT(THREAD_LOCK_HELD(tp));
1633 if (tp->t_state != TS_RUN)
1640 if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD)
1643 tpri = DISP_PRIO(tp);
1644 dp = tp->t_disp_queue;
1657 while (rp != tp && rp != NULL) {
1667 DTRACE_SCHED2(dequeue, kthread_t *, tp, disp_t *, dp);
1691 tp->t_link = NULL;
1692 THREAD_TRANSITION(tp); /* put in intermediate state */
1743 disp_swapped_enq(kthread_t *tp)
1745 ASSERT(THREAD_LOCK_HELD(tp));
1746 ASSERT(tp->t_schedflag & TS_LOAD);
1748 switch (tp->t_state) {
1751 THREAD_SWAP(tp, &swapped_lock); /* set TS_RUN state and lock */
1755 THREAD_TRANSITION(tp);
1757 THREAD_SWAP(tp, &swapped_lock); /* set TS_RUN state and lock */
1760 panic("disp_swapped: tp: %p bad t_state", (void *)tp);
1775 disp_swapped_setrun(kthread_t *tp)
1777 ASSERT(THREAD_LOCK_HELD(tp));
1778 ASSERT((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD);
1780 switch (tp->t_state) {
1787 if (DISP_PRIO(tp) > maxclsyspri)
1791 THREAD_RUN(tp, &swapped_lock); /* set TS_RUN state and lock */
1796 panic("disp_swapped_setrun: tp: %p bad t_state", (void *)tp);
1812 cpu_surrender(kthread_t *tp)
1819 ASSERT(THREAD_LOCK_HELD(tp));
1821 if (tp->t_state != TS_ONPROC)
1823 cpup = tp->t_disp_queue->disp_cpu; /* CPU thread dispatched to */
1829 if (tp->t_cid == sysdccid) {
1830 uint_t t_pri = DISP_PRIO(tp);
1846 DTRACE_SCHED1(surrender, kthread_t *, tp);
1853 if (tp != curthread || (lwp = tp->t_lwp) == NULL ||
1855 aston(tp);
1860 "cpu_surrender:tid %p cpu %p", tp, cpup);
1868 disp_ratify(kthread_t *tp, disp_t *kpq)
1874 ASSERT(tp != NULL);
1886 tpri = DISP_PRIO(tp);
1898 thread_lock_high(tp);
1899 THREAD_TRANSITION(tp);
1900 setfrontdq(tp);
1901 thread_unlock_nopreempt(tp);
1903 tp = NULL;
1905 return (tp);
1918 kthread_t *tp;
1935 tp = (disp_getbest(kpq));
1936 if (tp)
1937 return (disp_ratify(tp, kpq));
2076 tp = disp_getbest(tcp->cpu_disp);
2077 if (tp == NULL || tp == T_DONTSTEAL)
2078 return (tp);
2079 return (disp_ratify(tp, kpq));
2100 kthread_t *tp;
2140 tp = dq->dq_first;
2142 while (tp && (tp->t_bound_cpu || tp->t_weakbound_cpu)) {
2143 tp = tp->t_link;
2149 } while (tp == NULL);
2164 disp_adjust_unbound_pri(kthread_t *tp)
2169 ASSERT(THREAD_LOCK_HELD(tp));
2175 if (tp->t_bound_cpu == NULL ||
2176 tp->t_state != TS_RUN ||
2177 tp->t_schedflag & TS_ON_SWAPQ)
2180 tpri = DISP_PRIO(tp);
2181 dp = tp->t_bound_cpu->cpu_disp;
2201 kthread_t *tp;
2231 for (tp = dq->dq_first; tp != NULL; tp = tp->t_link) {
2238 if (tp->t_bound_cpu || tp->t_weakbound_cpu)
2278 tp->t_cpu != tcp)
2297 rqtime = now - tp->t_waitrq;
2311 DTRACE_PROBE4(nosteal, kthread_t *, tp,
2341 if (tp == NULL) {
2358 thread_was_on_queue = dispdeq(tp); /* drops disp_lock */
2363 (void) dispdeq(tp); /* drops disp_lock */
2372 tp->t_schedflag |= TS_DONT_SWAP;
2377 tp->t_disp_queue = cp->cpu_disp;
2379 cp->cpu_dispthread = tp; /* protected by spl only */
2404 ASSERT(pri == DISP_PRIO(tp));
2406 DTRACE_PROBE3(steal, kthread_t *, tp, cpu_t *, tcp, cpu_t *, cp);
2408 thread_onproc(tp, cp); /* set t_state to TS_ONPROC */
2415 return (tp);
2429 kthread_t *tp;
2435 tp = curthread; /* faster than allthreads */
2437 if (tp->t_state != TS_FREE) {
2443 if ((tp->t_flag & T_INTR_THREAD) &&
2451 if (tp == cp->cpu_idle_thread)
2458 if (tp == cp->cpu_pause_thread)
2462 (tp->t_bound_cpu == cp ||
2463 tp->t_bind_cpu == cp->cpu_id ||
2464 tp->t_weakbound_cpu == cp)) {
2470 (tp->t_cpupart == cp->cpu_part)) {
2475 } while ((tp = tp->t_next) != curthread && found == 0);
2521 kthread_t *tp;
2530 tp = dq->dq_first;
2535 while (tp != NULL && tp->t_bound_cpu != NULL) {
2536 tp = tp->t_link;
2539 if (tp == NULL) {
2545 wasonq = dispdeq(tp); /* drops disp_lock */
2547 ASSERT(tp->t_weakbound_cpu == NULL);
2549 setbackdq(tp);
2554 * and tp->t_cpu has been changed so there is no risk of
2555 * tp ending up back on cp.
2563 ASSERT(tp->t_cpu != cp);
2564 thread_unlock(tp);