Lines Matching refs:td
82 #define TS_NAME_LEN (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))
109 #define THREAD_CAN_MIGRATE(td) ((td)->td_pinned == 0)
110 #define THREAD_CAN_SCHED(td, cpu) \
111 CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)
390 struct thread *td;
402 TAILQ_FOREACH(td, rqh, td_runq) {
404 td, td->td_name, td->td_priority,
405 td->td_rqindex, pri);
478 tdq_runq_add(struct tdq *tdq, struct thread *td, int flags)
484 THREAD_LOCK_BLOCKED_ASSERT(td, MA_OWNED);
486 pri = td->td_priority;
487 ts = td_get_sched(td);
488 TD_SET_RUNQ(td);
489 if (THREAD_CAN_MIGRATE(td)) {
516 runq_add_pri(ts->ts_runq, td, pri, flags);
520 runq_add(ts->ts_runq, td, flags);
529 tdq_runq_rem(struct tdq *tdq, struct thread *td)
533 ts = td_get_sched(td);
535 THREAD_LOCK_BLOCKED_ASSERT(td, MA_OWNED);
537 ("tdq_runq_remove: thread %p null ts_runq", td));
544 runq_remove_idx(ts->ts_runq, td, &tdq->tdq_ridx);
546 runq_remove_idx(ts->ts_runq, td, NULL);
548 runq_remove(ts->ts_runq, td);
556 tdq_load_add(struct tdq *tdq, struct thread *td)
560 THREAD_LOCK_BLOCKED_ASSERT(td, MA_OWNED);
563 if ((td->td_flags & TDF_NOLOAD) == 0)
574 tdq_load_rem(struct tdq *tdq, struct thread *td)
578 THREAD_LOCK_BLOCKED_ASSERT(td, MA_OWNED);
583 if ((td->td_flags & TDF_NOLOAD) == 0)
620 struct thread *td;
625 td = tdq_choose(tdq);
626 if (td == NULL || td->td_priority > ctd->td_priority)
629 tdq->tdq_lowpri = td->td_priority;
849 struct thread *td;
871 td = tdq->tdq_curthread;
872 if (td->td_lock == TDQ_LOCKPTR(tdq) &&
873 (td->td_flags & TDF_IDLETD) == 0 &&
874 THREAD_CAN_MIGRATE(td)) {
875 td->td_flags |= TDF_PICKCPU;
876 ast_sched_locked(td, TDA_SCHED);
995 struct thread *td;
1002 td = tdq_steal(from, cpu);
1003 if (td == NULL)
1010 thread_lock_block_wait(td);
1011 sched_rem(td);
1012 THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(from));
1013 td->td_lock = TDQ_LOCKPTR(to);
1014 td_get_sched(td)->ts_cpu = cpu;
1015 return (tdq_add(to, td, SRQ_YIELDING));
1190 struct thread *td, *first;
1207 TAILQ_FOREACH(td, rqh, td_runq) {
1209 if (THREAD_CAN_MIGRATE(td) &&
1210 THREAD_CAN_SCHED(td, cpu))
1211 return (td);
1213 first = td;
1236 struct thread *td;
1248 TAILQ_FOREACH(td, rqh, td_runq)
1249 if (THREAD_CAN_MIGRATE(td) &&
1250 THREAD_CAN_SCHED(td, cpu))
1251 return (td);
1263 struct thread *td;
1266 if ((td = runq_steal(&tdq->tdq_realtime, cpu)) != NULL)
1267 return (td);
1268 if ((td = runq_steal_from(&tdq->tdq_timeshare,
1270 return (td);
1279 sched_setcpu(struct thread *td, int cpu, int flags)
1285 THREAD_LOCK_ASSERT(td, MA_OWNED);
1287 td_get_sched(td)->ts_cpu = cpu;
1291 if (td->td_lock == TDQ_LOCKPTR(tdq)) {
1302 mtx = thread_lock_block(td);
1306 thread_lock_unblock(td, TDQ_LOCKPTR(tdq));
1319 sched_pickcpu(struct thread *td, int flags)
1328 ts = td_get_sched(td);
1330 "absent CPU %d for thread %s.", ts->ts_cpu, td->td_name));
1336 if ((flags & SRQ_OURSELF) || !THREAD_CAN_MIGRATE(td))
1342 if (td->td_priority <= PRI_MAX_ITHD && THREAD_CAN_SCHED(td, self) &&
1362 if (THREAD_CAN_SCHED(td, ts->ts_cpu) &&
1403 mask = &td->td_cpuset->cs_mask;
1404 pri = td->td_priority;
1405 r = TD_IS_RUNNING(td);
1441 if (THREAD_CAN_SCHED(td, self) && TDQ_SELF()->tdq_lowpri > pri &&
1459 struct thread *td;
1462 td = runq_choose(&tdq->tdq_realtime);
1463 if (td != NULL)
1464 return (td);
1465 td = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
1466 if (td != NULL) {
1467 KASSERT(td->td_priority >= PRI_MIN_BATCH,
1469 td->td_priority));
1470 return (td);
1472 td = runq_choose(&tdq->tdq_idle);
1473 if (td != NULL) {
1474 KASSERT(td->td_priority >= PRI_MIN_IDLE,
1476 td->td_priority));
1477 return (td);
1614 sched_interact_score(struct thread *td)
1619 ts = td_get_sched(td);
1654 sched_priority(struct thread *td)
1658 if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)
1673 score = imax(0, sched_interact_score(td) + td->td_proc->p_nice);
1683 if (td_get_sched(td)->ts_ticks)
1684 pri += min(SCHED_PRI_TICKS(td_get_sched(td)),
1686 pri += SCHED_PRI_NICE(td->td_proc->p_nice);
1690 pri, td->td_proc->p_nice, td_get_sched(td)->ts_ticks,
1691 td_get_sched(td)->ts_ftick, td_get_sched(td)->ts_ltick,
1692 SCHED_PRI_TICKS(td_get_sched(td))));
1694 sched_user_prio(td, pri);
1705 sched_interact_update(struct thread *td)
1710 ts = td_get_sched(td);
1750 sched_interact_fork(struct thread *td)
1756 ts = td_get_sched(td);
1850 sched_thread_priority(struct thread *td, u_char prio)
1855 KTR_POINT3(KTR_SCHED, "thread", sched_tdname(td), "prio",
1856 "prio:%d", td->td_priority, "new prio:%d", prio,
1858 SDT_PROBE3(sched, , , change__pri, td, td->td_proc, prio);
1859 if (td != curthread && prio < td->td_priority) {
1861 "lend prio", "prio:%d", td->td_priority, "new prio:%d",
1862 prio, KTR_ATTR_LINKED, sched_tdname(td));
1863 SDT_PROBE4(sched, , , lend__pri, td, td->td_proc, prio,
1866 THREAD_LOCK_ASSERT(td, MA_OWNED);
1867 if (td->td_priority == prio)
1875 if (TD_ON_RUNQ(td) && prio < td->td_priority) {
1876 sched_rem(td);
1877 td->td_priority = prio;
1878 sched_add(td, SRQ_BORROWING | SRQ_HOLDTD);
1885 if (TD_IS_RUNNING(td)) {
1886 tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);
1887 oldpri = td->td_priority;
1888 td->td_priority = prio;
1892 tdq_setlowpri(tdq, td);
1895 td->td_priority = prio;
1903 sched_lend_prio(struct thread *td, u_char prio)
1906 td->td_flags |= TDF_BORROWING;
1907 sched_thread_priority(td, prio);
1919 sched_unlend_prio(struct thread *td, u_char prio)
1923 if (td->td_base_pri >= PRI_MIN_TIMESHARE &&
1924 td->td_base_pri <= PRI_MAX_TIMESHARE)
1925 base_pri = td->td_user_pri;
1927 base_pri = td->td_base_pri;
1929 td->td_flags &= ~TDF_BORROWING;
1930 sched_thread_priority(td, base_pri);
1932 sched_lend_prio(td, prio);
1939 sched_prio(struct thread *td, u_char prio)
1944 td->td_base_pri = prio;
1950 if (td->td_flags & TDF_BORROWING && td->td_priority < prio)
1954 oldprio = td->td_priority;
1955 sched_thread_priority(td, prio);
1961 if (TD_ON_LOCK(td) && oldprio != prio)
1962 turnstile_adjust(td, oldprio);
1969 sched_ithread_prio(struct thread *td, u_char prio)
1971 THREAD_LOCK_ASSERT(td, MA_OWNED);
1972 MPASS(td->td_pri_class == PRI_ITHD);
1973 td->td_base_ithread_pri = prio;
1974 sched_prio(td, prio);
1981 sched_user_prio(struct thread *td, u_char prio)
1984 td->td_base_user_pri = prio;
1985 if (td->td_lend_user_pri <= prio)
1987 td->td_user_pri = prio;
1991 sched_lend_user_prio(struct thread *td, u_char prio)
1994 THREAD_LOCK_ASSERT(td, MA_OWNED);
1995 td->td_lend_user_pri = prio;
1996 td->td_user_pri = min(prio, td->td_base_user_pri);
1997 if (td->td_priority > td->td_user_pri)
1998 sched_prio(td, td->td_user_pri);
1999 else if (td->td_priority != td->td_user_pri)
2000 ast_sched_locked(td, TDA_SCHED);
2007 sched_lend_user_prio_cond(struct thread *td, u_char prio)
2010 if (td->td_lend_user_pri == prio)
2013 thread_lock(td);
2014 sched_lend_user_prio(td, prio);
2015 thread_unlock(td);
2131 sched_switch_migrate(struct tdq *tdq, struct thread *td, int flags)
2138 KASSERT(THREAD_CAN_MIGRATE(td) ||
2139 (td_get_sched(td)->ts_flags & TSF_BOUND) != 0,
2140 ("Thread %p shouldn't migrate", td));
2141 KASSERT(!CPU_ABSENT(td_get_sched(td)->ts_cpu), ("sched_switch_migrate: "
2142 "thread %s queued on absent CPU %d.", td->td_name,
2143 td_get_sched(td)->ts_cpu));
2144 tdn = TDQ_CPU(td_get_sched(td)->ts_cpu);
2146 tdq_load_rem(tdq, td);
2154 lowpri = tdq_add(tdn, td, flags);
2166 thread_unblock_switch(struct thread *td, struct mtx *mtx)
2168 atomic_store_rel_ptr((volatile uintptr_t *)&td->td_lock,
2179 sched_switch(struct thread *td, int flags)
2191 THREAD_LOCK_ASSERT(td, MA_OWNED);
2195 ts = td_get_sched(td);
2198 pickcpu = (td->td_flags & TDF_PICKCPU) != 0;
2204 td->td_lastcpu = td->td_oncpu;
2205 preempted = (td->td_flags & TDF_SLICEEND) == 0 &&
2207 td->td_flags &= ~(TDF_PICKCPU | TDF_SLICEEND);
2208 ast_unsched_locked(td, TDA_SCHED);
2209 td->td_owepreempt = 0;
2211 if (!TD_IS_IDLETHREAD(td))
2217 mtx = thread_lock_block(td);
2219 if (TD_IS_IDLETHREAD(td)) {
2221 TD_SET_CAN_RUN(td);
2222 } else if (TD_IS_RUNNING(td)) {
2227 if (THREAD_CAN_MIGRATE(td) && (!THREAD_CAN_SCHED(td, ts->ts_cpu)
2229 ts->ts_cpu = sched_pickcpu(td, 0);
2232 tdq_runq_add(tdq, td, srqflag);
2234 mtx = sched_switch_migrate(tdq, td, srqflag);
2241 tdq_load_rem(tdq, td);
2249 if (TD_IS_IDLETHREAD(td))
2250 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "idle",
2251 "prio:%d", td->td_priority);
2253 KTR_STATE3(KTR_SCHED, "thread", sched_tdname(td), KTDSTATE(td),
2254 "prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg,
2255 "lockname:\"%s\"", td->td_lockname);
2264 MPASS(td == tdq->tdq_curthread);
2272 if (td != newtd) {
2274 if (PMC_PROC_IS_USING_PMCS(td->td_proc))
2275 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
2288 td->td_oncpu = NOCPU;
2289 cpu_switch(td, newtd, mtx);
2290 cpuid = td->td_oncpu = PCPU_GET(cpuid);
2294 if (PMC_PROC_IS_USING_PMCS(td->td_proc))
2295 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
2298 thread_unblock_switch(td, mtx);
2304 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
2305 "prio:%d", td->td_priority);
2314 struct thread *td;
2319 FOREACH_THREAD_IN_PROC(p, td) {
2320 thread_lock(td);
2321 sched_priority(td);
2322 sched_prio(td, td->td_base_user_pri);
2323 thread_unlock(td);
2331 sched_sleep(struct thread *td, int prio)
2334 THREAD_LOCK_ASSERT(td, MA_OWNED);
2336 td->td_slptick = ticks;
2337 if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)
2340 sched_prio(td, prio);
2341 else if (static_boost && td->td_priority > static_boost)
2342 sched_prio(td, static_boost);
2352 sched_wakeup(struct thread *td, int srqflags)
2357 THREAD_LOCK_ASSERT(td, MA_OWNED);
2358 ts = td_get_sched(td);
2364 slptick = td->td_slptick;
2365 td->td_slptick = 0;
2368 sched_interact_update(td);
2376 if (PRI_BASE(td->td_pri_class) == PRI_ITHD &&
2377 td->td_priority != td->td_base_ithread_pri)
2378 sched_prio(td, td->td_base_ithread_pri);
2384 sched_add(td, SRQ_BORING | srqflags);
2392 sched_fork(struct thread *td, struct thread *child)
2394 THREAD_LOCK_ASSERT(td, MA_OWNED);
2395 sched_pctcpu_update(td_get_sched(td), 1);
2396 sched_fork_thread(td, child);
2402 td_get_sched(td)->ts_runtime += tickincr;
2403 sched_interact_update(td);
2404 sched_priority(td);
2411 sched_fork_thread(struct thread *td, struct thread *child)
2418 THREAD_LOCK_ASSERT(td, MA_OWNED);
2422 ts = td_get_sched(td);
2427 child->td_cpuset = cpuset_ref(td->td_cpuset);
2428 child->td_domain.dr_policy = td->td_cpuset->cs_domain;
2457 sched_class(struct thread *td, int class)
2460 THREAD_LOCK_ASSERT(td, MA_OWNED);
2461 if (td->td_pri_class == class)
2463 td->td_pri_class = class;
2472 struct thread *td;
2477 td = FIRST_THREAD_IN_PROC(p);
2478 sched_exit_thread(td, child);
2488 sched_exit_thread(struct thread *td, struct thread *child)
2498 thread_lock(td);
2499 td_get_sched(td)->ts_runtime += td_get_sched(child)->ts_runtime;
2500 sched_interact_update(td);
2501 sched_priority(td);
2502 thread_unlock(td);
2506 sched_preempt(struct thread *td)
2511 SDT_PROBE2(sched, , , surrender, td, td->td_proc);
2513 thread_lock(td);
2516 if (td->td_priority > tdq->tdq_lowpri) {
2517 if (td->td_critnest == 1) {
2519 flags |= TD_IS_IDLETHREAD(td) ? SWT_REMOTEWAKEIDLE :
2525 td->td_owepreempt = 1;
2529 thread_unlock(td);
2537 sched_userret_slowpath(struct thread *td)
2540 thread_lock(td);
2541 td->td_priority = td->td_user_pri;
2542 td->td_base_pri = td->td_user_pri;
2543 tdq_setlowpri(TDQ_SELF(), td);
2544 thread_unlock(td);
2556 td_slice(struct thread *td, struct tdq *tdq)
2558 if (PRI_BASE(td->td_pri_class) == PRI_ITHD)
2568 sched_clock(struct thread *td, int cnt)
2573 THREAD_LOCK_ASSERT(td, MA_OWNED);
2603 ts = td_get_sched(td);
2605 if ((td->td_pri_class & PRI_FIFO_BIT) || TD_IS_IDLETHREAD(td))
2608 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) {
2613 td_get_sched(td)->ts_runtime += tickincr * cnt;
2614 sched_interact_update(td);
2615 sched_priority(td);
2623 if (ts->ts_slice >= td_slice(td, tdq)) {
2630 if (PRI_BASE(td->td_pri_class) == PRI_ITHD) {
2632 td->td_owepreempt = 1;
2633 if (td->td_base_pri + RQ_PPQ < PRI_MAX_ITHD) {
2635 sched_prio(td, td->td_base_pri + RQ_PPQ);
2638 ast_sched_locked(td, TDA_SCHED);
2639 td->td_flags |= TDF_SLICEEND;
2645 sched_estcpu(struct thread *td __unused)
2682 struct thread *td;
2687 td = tdq_choose(tdq);
2688 if (td != NULL) {
2689 tdq_runq_rem(tdq, td);
2690 tdq->tdq_lowpri = td->td_priority;
2693 td = PCPU_GET(idlethread);
2695 tdq->tdq_curthread = td;
2696 return (td);
2729 tdq_add(struct tdq *tdq, struct thread *td, int flags)
2734 THREAD_LOCK_BLOCKED_ASSERT(td, MA_OWNED);
2735 KASSERT((td->td_inhibitors == 0),
2737 KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
2739 KASSERT(td->td_flags & TDF_INMEM,
2743 if (td->td_priority < lowpri)
2744 tdq->tdq_lowpri = td->td_priority;
2745 tdq_runq_add(tdq, td, flags);
2746 tdq_load_add(tdq, td);
2757 sched_add(struct thread *td, int flags)
2764 KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",
2765 "prio:%d", td->td_priority, KTR_ATTR_LINKED,
2768 KTR_ATTR_LINKED, sched_tdname(td));
2769 SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
2771 THREAD_LOCK_ASSERT(td, MA_OWNED);
2776 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
2777 sched_priority(td);
2783 cpu = sched_pickcpu(td, flags);
2784 tdq = sched_setcpu(td, cpu, flags);
2785 lowpri = tdq_add(tdq, td, flags);
2789 sched_setpreempt(td->td_priority);
2796 if (td->td_lock != TDQ_LOCKPTR(tdq)) {
2799 td->td_lock = TDQ_LOCKPTR(tdq);
2801 thread_lock_set(td, TDQ_LOCKPTR(tdq));
2803 (void)tdq_add(tdq, td, flags);
2805 sched_setpreempt(td->td_priority);
2808 thread_unlock(td);
2817 sched_rem(struct thread *td)
2821 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "runq rem",
2822 "prio:%d", td->td_priority);
2823 SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);
2824 tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);
2826 MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
2827 KASSERT(TD_ON_RUNQ(td),
2829 tdq_runq_rem(tdq, td);
2830 tdq_load_rem(tdq, td);
2831 TD_SET_CAN_RUN(td);
2832 if (td->td_priority == tdq->tdq_lowpri)
2840 sched_pctcpu(struct thread *td)
2846 ts = td_get_sched(td);
2848 THREAD_LOCK_ASSERT(td, MA_OWNED);
2849 sched_pctcpu_update(ts, TD_IS_RUNNING(td));
2866 sched_affinity(struct thread *td)
2871 THREAD_LOCK_ASSERT(td, MA_OWNED);
2872 ts = td_get_sched(td);
2873 if (THREAD_CAN_SCHED(td, ts->ts_cpu))
2875 if (TD_ON_RUNQ(td)) {
2876 sched_rem(td);
2877 sched_add(td, SRQ_BORING | SRQ_HOLDTD);
2880 if (!TD_IS_RUNNING(td))
2887 ast_sched_locked(td, TDA_SCHED);
2888 if (td != curthread)
2897 sched_bind(struct thread *td, int cpu)
2901 THREAD_LOCK_ASSERT(td, MA_OWNED|MA_NOTRECURSED);
2902 KASSERT(td == curthread, ("sched_bind: can only bind curthread"));
2903 ts = td_get_sched(td);
2905 sched_unbind(td);
2906 KASSERT(THREAD_CAN_MIGRATE(td), ("%p must be migratable", td));
2914 thread_lock(td);
2921 sched_unbind(struct thread *td)
2925 THREAD_LOCK_ASSERT(td, MA_OWNED);
2926 KASSERT(td == curthread, ("sched_unbind: can only bind curthread"));
2927 ts = td_get_sched(td);
2935 sched_is_bound(struct thread *td)
2937 THREAD_LOCK_ASSERT(td, MA_OWNED);
2938 return (td_get_sched(td)->ts_flags & TSF_BOUND);
2945 sched_relinquish(struct thread *td)
2947 thread_lock(td);
2995 struct thread *td;
3001 td = curthread;
3007 thread_lock(td);
3122 sched_throw(struct thread *td)
3129 MPASS(td != NULL);
3130 THREAD_LOCK_ASSERT(td, MA_OWNED);
3131 THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq));
3133 tdq_load_rem(tdq, td);
3134 td->td_lastcpu = td->td_oncpu;
3135 td->td_oncpu = NOCPU;
3136 thread_lock_block(td);
3141 cpu_switch(td, newtd, TDQ_LOCKPTR(tdq));
3149 sched_fork_exit(struct thread *td)
3164 MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
3165 td->td_oncpu = cpuid;
3166 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
3167 "prio:%d", td->td_priority);
3175 sched_tdname(struct thread *td)
3180 ts = td_get_sched(td);
3183 "%s tid %d", td->td_name, td->td_tid);
3186 return (td->td_name);
3192 sched_clear_tdname(struct thread *td)
3196 ts = td_get_sched(td);