Lines Matching +full:charge +full:- +full:vol
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2002-2007, Jeffrey Roberson <jeff@freebsd.org>
32 * performance under load even on uni-processor systems.
84 #define TDQ_LOADNAME_LEN (sizeof("CPU ") + sizeof(__XSTRING(MAXCPU)) - 1 + sizeof(" load"))
91 struct runq *ts_runq; /* Run-queue we're queued on. */
96 u_int ts_slptime; /* Number of ticks we vol. slept */
109 #define THREAD_CAN_MIGRATE(td) ((td)->td_pinned == 0)
111 CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)
118 * Priority ranges used for interactive and non-interactive timeshare
121 * (NHALF, x, and NHALF) handle non-interactive threads with the outer
124 #define PRI_TIMESHARE_RANGE (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1)
125 #define PRI_INTERACT_RANGE ((PRI_TIMESHARE_RANGE - SCHED_PRI_NRESV) / 2)
126 #define PRI_BATCH_RANGE (PRI_TIMESHARE_RANGE - PRI_INTERACT_RANGE)
129 #define PRI_MAX_INTERACT (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE - 1)
147 #define SCHED_TICK_HZ(ts) ((ts)->ts_ticks >> SCHED_TICK_SHIFT)
148 #define SCHED_TICK_TOTAL(ts) (max((ts)->ts_ltick - (ts)->ts_ftick, hz))
151 * These macros determine priorities for non-interactive threads. They are
162 #define SCHED_PRI_NRESV (PRIO_MAX - PRIO_MIN)
165 #define SCHED_PRI_MAX (PRI_MAX_BATCH - SCHED_PRI_NHALF)
166 #define SCHED_PRI_RANGE (SCHED_PRI_MAX - SCHED_PRI_MIN + 1)
224 static int __read_mostly sched_idlespinthresh = -1;
227 * tdq - per processor runqs and statistics. A mutex synchronizes access to
233 * (l) all accesses are CPU-local
258 struct runq tdq_realtime; /* (t) real-time run queue. */
272 #define TDQ_LOAD(tdq) atomic_load_int(&(tdq)->tdq_load)
273 #define TDQ_TRANSFERABLE(tdq) atomic_load_int(&(tdq)->tdq_transferable)
274 #define TDQ_SWITCHCNT(tdq) (atomic_load_short(&(tdq)->tdq_switchcnt) + \
275 atomic_load_short(&(tdq)->tdq_oldswitchcnt))
276 #define TDQ_SWITCHCNT_INC(tdq) (atomic_store_short(&(tdq)->tdq_switchcnt, \
277 atomic_load_short(&(tdq)->tdq_switchcnt) + 1))
283 #define SCHED_AFFINITY(ts, t) ((ts)->ts_rltick > ticks - ((t) * affinity))
286 * Run-time tunables.
306 #define TDQ_ID(x) ((x)->tdq_id)
321 #define TDQ_LOCKPTR(t) ((struct mtx *)(&(t)->tdq_lock))
384 * Print the threads waiting on a run-queue.
397 i, rq->rq_status.rqb_bits[i]);
399 if (rq->rq_status.rqb_bits[i] & (1ul << j)) {
401 rqh = &rq->rq_queues[pri];
404 td, td->td_name, td->td_priority,
405 td->td_rqindex, pri);
412 * Print the status of a per-cpu thread queue. Should be a ddb show cmd.
423 printf("\tLock name: %s\n", tdq->tdq_name);
424 printf("\tload: %d\n", tdq->tdq_load);
425 printf("\tswitch cnt: %d\n", tdq->tdq_switchcnt);
426 printf("\told switch cnt: %d\n", tdq->tdq_oldswitchcnt);
427 printf("\ttimeshare idx: %d\n", tdq->tdq_idx);
428 printf("\ttimeshare ridx: %d\n", tdq->tdq_ridx);
429 printf("\tload transferable: %d\n", tdq->tdq_transferable);
430 printf("\tlowest priority: %d\n", tdq->tdq_lowpri);
432 runq_print(&tdq->tdq_realtime);
434 runq_print(&tdq->tdq_timeshare);
436 runq_print(&tdq->tdq_idle);
464 * If we're interactive or better and there is non-interactive
473 * Add a thread to the actual run-queue. Keeps transferable counts up to
474 * date with what is actually on the run-queue. Selects the correct
486 pri = td->td_priority;
490 tdq->tdq_transferable++;
491 ts->ts_flags |= TSF_XFERABLE;
494 ts->ts_runq = &tdq->tdq_realtime;
496 ts->ts_runq = &tdq->tdq_timeshare;
504 pri = RQ_NQS * (pri - PRI_MIN_BATCH) / PRI_BATCH_RANGE;
505 pri = (pri + tdq->tdq_idx) % RQ_NQS;
511 if (tdq->tdq_ridx != tdq->tdq_idx &&
512 pri == tdq->tdq_ridx)
513 pri = (unsigned char)(pri - 1) % RQ_NQS;
515 pri = tdq->tdq_ridx;
516 runq_add_pri(ts->ts_runq, td, pri, flags);
519 ts->ts_runq = &tdq->tdq_idle;
520 runq_add(ts->ts_runq, td, flags);
524 * Remove a thread from a run-queue. This typically happens when a thread
536 KASSERT(ts->ts_runq != NULL,
538 if (ts->ts_flags & TSF_XFERABLE) {
539 tdq->tdq_transferable--;
540 ts->ts_flags &= ~TSF_XFERABLE;
542 if (ts->ts_runq == &tdq->tdq_timeshare) {
543 if (tdq->tdq_idx != tdq->tdq_ridx)
544 runq_remove_idx(ts->ts_runq, td, &tdq->tdq_ridx);
546 runq_remove_idx(ts->ts_runq, td, NULL);
548 runq_remove(ts->ts_runq, td);
562 tdq->tdq_load++;
563 if ((td->td_flags & TDF_NOLOAD) == 0)
564 tdq->tdq_sysload++;
565 KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);
566 SDT_PROBE2(sched, , , load__change, (int)TDQ_ID(tdq), tdq->tdq_load);
579 KASSERT(tdq->tdq_load != 0,
582 tdq->tdq_load--;
583 if ((td->td_flags & TDF_NOLOAD) == 0)
584 tdq->tdq_sysload--;
585 KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);
586 SDT_PROBE2(sched, , , load__change, (int)TDQ_ID(tdq), tdq->tdq_load);
605 load = tdq->tdq_sysload - 1;
614 * Set lowpri to its exact value by searching the run-queue and
624 ctd = tdq->tdq_curthread;
626 if (td == NULL || td->td_priority > ctd->td_priority)
627 tdq->tdq_lowpri = ctd->td_priority;
629 tdq->tdq_lowpri = td->td_priority;
682 r->csr_cpu = -1;
685 if (cg->cg_children > 0) {
686 for (c = cg->cg_children - 1; c >= 0; c--) {
687 load = cpu_search_lowest(&cg->cg_child[c], s, &lr);
692 * It allows round-robin between SMT groups with equal
695 if (__predict_false(s->cs_running) &&
696 (cg->cg_child[c].cg_flags & CG_FLAG_THREAD) &&
701 (load == bload && lr.csr_load < r->csr_load))) {
703 r->csr_cpu = lr.csr_cpu;
704 r->csr_load = lr.csr_load;
711 for (c = cg->cg_last; c >= cg->cg_first; c--) {
712 if (!CPU_ISSET(c, &cg->cg_mask))
716 if (c == s->cs_prefer) {
717 if (__predict_false(s->cs_running))
718 l--;
723 total += load - p;
730 if (l > s->cs_load ||
731 (atomic_load_char(&tdq->tdq_lowpri) <= s->cs_pri &&
732 (!s->cs_running || c != s->cs_prefer)) ||
733 !CPU_ISSET(c, s->cs_mask))
738 * It allows round-robin between CPUs with equal load
741 if (__predict_false(s->cs_running) && l > 0)
744 load -= sched_random() % 128;
745 if (bload > load - p) {
746 bload = load - p;
747 r->csr_cpu = c;
748 r->csr_load = load;
764 r->csr_cpu = -1;
767 if (cg->cg_children > 0) {
768 for (c = cg->cg_children - 1; c >= 0; c--) {
769 load = cpu_search_highest(&cg->cg_child[c], s, &lr);
772 (load == bload && lr.csr_load > r->csr_load))) {
774 r->csr_cpu = lr.csr_cpu;
775 r->csr_load = lr.csr_load;
782 for (c = cg->cg_last; c >= cg->cg_first; c--) {
783 if (!CPU_ISSET(c, &cg->cg_mask))
793 if (l < s->cs_load || TDQ_TRANSFERABLE(tdq) < s->cs_trans ||
794 !CPU_ISSET(c, s->cs_mask))
797 load -= sched_random() % 256;
800 r->csr_cpu = c;
803 r->csr_load = bload;
809 * lowpri greater than pri pri. A pri of -1 indicates any priority is
857 if (high == -1)
871 td = tdq->tdq_curthread;
872 if (td->td_lock == TDQ_LOCKPTR(tdq) &&
873 (td->td_flags & TDF_IDLETD) == 0 &&
875 td->td_flags |= TDF_PICKCPU;
887 low = sched_lowest(cg, &lmask, -1, TDQ_LOAD(tdq) - 1, high, 1);
889 if (anylow && low == -1)
892 if (low == -1)
965 if (high->tdq_transferable != 0 && high->tdq_load > low->tdq_load) {
967 if (lowpri != -1) {
977 sched_setpreempt(low->tdq_lowpri);
986 * Move a thread from one thread queue to another. Returns -1 if the source
1004 return (-1);
1013 td->td_lock = TDQ_LOCKPTR(to);
1014 td_get_sched(td)->ts_cpu = cpu;
1030 if (smp_started == 0 || steal_idle == 0 || tdq->tdq_cg == NULL)
1036 for (cg = tdq->tdq_cg, goup = 0; ; ) {
1051 if (cpu == -1) {
1053 cg = cg->cg_parent;
1056 parent = cg->cg_parent;
1059 if (parent->cg_children == 2) {
1060 if (cg == &parent->cg_child[0])
1061 cg = &parent->cg_child[1];
1063 cg = &parent->cg_child[0];
1088 if (tdq->tdq_load > 0) {
1113 if (tdq_move(steal, tdq) != -1)
1143 KASSERT(tdq->tdq_lowpri <= lowpri,
1144 ("tdq_notify: lowpri %d > tdq_lowpri %d", lowpri, tdq->tdq_lowpri));
1146 if (tdq->tdq_owepreempt)
1153 if (!sched_shouldpreempt(tdq->tdq_lowpri, lowpri, 1))
1169 if (TD_IS_IDLETHREAD(tdq->tdq_curthread) &&
1170 (atomic_load_int(&tdq->tdq_cpu_idle) == 0 || cpu_idle_wakeup(cpu)))
1177 tdq->tdq_owepreempt = 1;
1194 rqb = &rq->rq_status;
1195 bit = start & (RQB_BPW -1);
1199 if (rqb->rqb_bits[i] == 0)
1202 bit = RQB_FFS(rqb->rqb_bits[i]);
1204 if ((rqb->rqb_bits[i] & (1ul << bit)) == 0)
1206 rqh = &rq->rq_queues[bit + (i << RQB_L2BPW)];
1240 rqb = &rq->rq_status;
1242 if (rqb->rqb_bits[word] == 0)
1245 if ((rqb->rqb_bits[word] & (1ul << bit)) == 0)
1247 rqh = &rq->rq_queues[bit + (word << RQB_L2BPW)];
1266 if ((td = runq_steal(&tdq->tdq_realtime, cpu)) != NULL)
1268 if ((td = runq_steal_from(&tdq->tdq_timeshare,
1269 cpu, tdq->tdq_ridx)) != NULL)
1271 return (runq_steal(&tdq->tdq_idle, cpu));
1287 td_get_sched(td)->ts_cpu = cpu;
1291 if (td->td_lock == TDQ_LOCKPTR(tdq)) {
1329 KASSERT(!CPU_ABSENT(ts->ts_cpu), ("sched_pickcpu: Start scheduler on "
1330 "absent CPU %d for thread %s.", ts->ts_cpu, td->td_name));
1337 return (ts->ts_cpu);
1342 if (td->td_priority <= PRI_MAX_ITHD && THREAD_CAN_SCHED(td, self) &&
1343 curthread->td_intr_nesting_level) {
1345 if (tdq->tdq_lowpri >= PRI_MIN_IDLE) {
1349 ts->ts_cpu = self;
1351 cg = tdq->tdq_cg;
1355 tdq = TDQ_CPU(ts->ts_cpu);
1356 cg = tdq->tdq_cg;
1362 if (THREAD_CAN_SCHED(td, ts->ts_cpu) &&
1363 atomic_load_char(&tdq->tdq_lowpri) >= PRI_MIN_IDLE &&
1365 if (cg->cg_flags & CG_FLAG_THREAD) {
1367 for (cpu = cg->cg_first; cpu <= cg->cg_last; cpu++) {
1369 atomic_load_char(&TDQ_CPU(cpu)->tdq_lowpri);
1370 if (CPU_ISSET(cpu, &cg->cg_mask) &&
1374 if (cpu > cg->cg_last) {
1376 return (ts->ts_cpu);
1380 return (ts->ts_cpu);
1389 for (ccg = NULL; cg != NULL; cg = cg->cg_parent) {
1390 if (cg->cg_flags & CG_FLAG_THREAD)
1392 if (cg->cg_children == 1 || cg->cg_count == 1)
1394 if (cg->cg_level == CG_SHARE_NONE ||
1395 (!intr && !SCHED_AFFINITY(ts, cg->cg_level)))
1402 cpu = -1;
1403 mask = &td->td_cpuset->cs_mask;
1404 pri = td->td_priority;
1412 cpu = sched_lowest(ccg, mask, pri, INT_MAX, ts->ts_cpu, r);
1419 INT_MAX, ts->ts_cpu, r);
1425 cpu = sched_lowest(cpu_top, mask, pri, INT_MAX, ts->ts_cpu, r);
1431 cpu = sched_lowest(cpu_top, mask, -1, INT_MAX, ts->ts_cpu, r);
1441 if (THREAD_CAN_SCHED(td, self) && TDQ_SELF()->tdq_lowpri > pri &&
1442 atomic_load_char(&tdq->tdq_lowpri) < PRI_MIN_IDLE &&
1447 if (cpu != ts->ts_cpu)
1462 td = runq_choose(&tdq->tdq_realtime);
1465 td = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
1467 KASSERT(td->td_priority >= PRI_MIN_BATCH,
1469 td->td_priority));
1472 td = runq_choose(&tdq->tdq_idle);
1474 KASSERT(td->td_priority >= PRI_MIN_IDLE,
1476 td->td_priority));
1492 runq_init(&tdq->tdq_realtime);
1493 runq_init(&tdq->tdq_timeshare);
1494 runq_init(&tdq->tdq_idle);
1495 tdq->tdq_id = id;
1496 snprintf(tdq->tdq_name, sizeof(tdq->tdq_name),
1498 mtx_init(&tdq->tdq_lock, tdq->tdq_name, "sched lock", MTX_SPIN);
1500 snprintf(tdq->tdq_loadname, sizeof(tdq->tdq_loadname),
1516 tdq->tdq_cg = smp_topo_find(cpu_top, i);
1517 if (tdq->tdq_cg == NULL)
1546 tdq->tdq_curthread = &thread0;
1547 tdq->tdq_lowpri = thread0.td_priority;
1596 * waiting on a run-queue. Would be prettier if we had floating point.
1602 * interactivity score = ---------------------
1610 * interactivity score = 2 * scaling factor - ---------------------
1626 ts->ts_runtime >= ts->ts_slptime)
1629 if (ts->ts_runtime > ts->ts_slptime) {
1630 div = max(1, ts->ts_runtime / SCHED_INTERACT_HALF);
1632 (SCHED_INTERACT_HALF - (ts->ts_slptime / div)));
1634 if (ts->ts_slptime > ts->ts_runtime) {
1635 div = max(1, ts->ts_slptime / SCHED_INTERACT_HALF);
1636 return (ts->ts_runtime / div);
1639 if (ts->ts_runtime)
1658 if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)
1673 score = imax(0, sched_interact_score(td) + td->td_proc->p_nice);
1676 pri += (PRI_MAX_INTERACT - PRI_MIN_INTERACT + 1) * score /
1683 if (td_get_sched(td)->ts_ticks)
1685 SCHED_PRI_RANGE - 1);
1686 pri += SCHED_PRI_NICE(td->td_proc->p_nice);
1690 pri, td->td_proc->p_nice, td_get_sched(td)->ts_ticks,
1691 td_get_sched(td)->ts_ftick, td_get_sched(td)->ts_ltick,
1711 sum = ts->ts_runtime + ts->ts_slptime;
1720 if (ts->ts_runtime > ts->ts_slptime) {
1721 ts->ts_runtime = SCHED_SLP_RUN_MAX;
1722 ts->ts_slptime = 1;
1724 ts->ts_slptime = SCHED_SLP_RUN_MAX;
1725 ts->ts_runtime = 1;
1735 ts->ts_runtime /= 2;
1736 ts->ts_slptime /= 2;
1739 ts->ts_runtime = (ts->ts_runtime / 5) * 4;
1740 ts->ts_slptime = (ts->ts_slptime / 5) * 4;
1757 sum = ts->ts_runtime + ts->ts_slptime;
1760 ts->ts_runtime /= ratio;
1761 ts->ts_slptime /= ratio;
1777 ts0->ts_ltick = ticks;
1778 ts0->ts_ftick = ticks;
1779 ts0->ts_slice = 0;
1780 ts0->ts_cpu = curcpu; /* set valid CPU number */
1800 PCPU_GET(idlethread)->td_lock = TDQ_LOCKPTR(TDQ_SELF());
1831 if ((u_int)(t - ts->ts_ltick) >= SCHED_TICK_TARG) {
1832 ts->ts_ticks = 0;
1833 ts->ts_ftick = t - SCHED_TICK_TARG;
1834 } else if (t - ts->ts_ftick >= SCHED_TICK_MAX) {
1835 ts->ts_ticks = (ts->ts_ticks / (ts->ts_ltick - ts->ts_ftick)) *
1836 (ts->ts_ltick - (t - SCHED_TICK_TARG));
1837 ts->ts_ftick = t - SCHED_TICK_TARG;
1840 ts->ts_ticks += (t - ts->ts_ltick) << SCHED_TICK_SHIFT;
1841 ts->ts_ltick = t;
1845 * Adjust the priority of a thread. Move it to the appropriate run-queue
1846 * if necessary. This is the back-end for several priority related
1856 "prio:%d", td->td_priority, "new prio:%d", prio,
1858 SDT_PROBE3(sched, , , change__pri, td, td->td_proc, prio);
1859 if (td != curthread && prio < td->td_priority) {
1861 "lend prio", "prio:%d", td->td_priority, "new prio:%d",
1863 SDT_PROBE4(sched, , , lend__pri, td, td->td_proc, prio,
1867 if (td->td_priority == prio)
1872 * queue. This could be optimized to not re-add in some
1875 if (TD_ON_RUNQ(td) && prio < td->td_priority) {
1877 td->td_priority = prio;
1886 tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);
1887 oldpri = td->td_priority;
1888 td->td_priority = prio;
1889 if (prio < tdq->tdq_lowpri)
1890 tdq->tdq_lowpri = prio;
1891 else if (tdq->tdq_lowpri == oldpri)
1895 td->td_priority = prio;
1906 td->td_flags |= TDF_BORROWING;
1923 if (td->td_base_pri >= PRI_MIN_TIMESHARE &&
1924 td->td_base_pri <= PRI_MAX_TIMESHARE)
1925 base_pri = td->td_user_pri;
1927 base_pri = td->td_base_pri;
1929 td->td_flags &= ~TDF_BORROWING;
1944 td->td_base_pri = prio;
1950 if (td->td_flags & TDF_BORROWING && td->td_priority < prio)
1954 oldprio = td->td_priority;
1972 MPASS(td->td_pri_class == PRI_ITHD);
1973 td->td_base_ithread_pri = prio;
1984 td->td_base_user_pri = prio;
1985 if (td->td_lend_user_pri <= prio)
1987 td->td_user_pri = prio;
1995 td->td_lend_user_pri = prio;
1996 td->td_user_pri = min(prio, td->td_base_user_pri);
1997 if (td->td_priority > td->td_user_pri)
1998 sched_prio(td, td->td_user_pri);
1999 else if (td->td_priority != td->td_user_pri)
2010 if (td->td_lend_user_pri == prio)
2032 tdq->tdq_cg == NULL)
2039 for (i = 1, cg = tdq->tdq_cg, goup = 0; ; ) {
2056 if (cpu == -1) {
2058 cg = cg->cg_parent;
2065 parent = cg->cg_parent;
2070 if (parent->cg_children == 2) {
2071 if (cg == &parent->cg_child[0])
2072 cg = &parent->cg_child[1];
2074 cg = &parent->cg_child[0];
2097 if (tdq->tdq_load > 0)
2115 if (tdq_move(steal, tdq) == -1) {
2139 (td_get_sched(td)->ts_flags & TSF_BOUND) != 0,
2141 KASSERT(!CPU_ABSENT(td_get_sched(td)->ts_cpu), ("sched_switch_migrate: "
2142 "thread %s queued on absent CPU %d.", td->td_name,
2143 td_get_sched(td)->ts_cpu));
2144 tdn = TDQ_CPU(td_get_sched(td)->ts_cpu);
2150 * prevent preemption while we're holding neither run-queue lock.
2168 atomic_store_rel_ptr((volatile uintptr_t *)&td->td_lock,
2198 pickcpu = (td->td_flags & TDF_PICKCPU) != 0;
2200 ts->ts_rltick = ticks - affinity * MAX_CACHE_LEVELS;
2202 ts->ts_rltick = ticks;
2204 td->td_lastcpu = td->td_oncpu;
2205 preempted = (td->td_flags & TDF_SLICEEND) == 0 &&
2207 td->td_flags &= ~(TDF_PICKCPU | TDF_SLICEEND);
2209 td->td_owepreempt = 0;
2210 atomic_store_char(&tdq->tdq_owepreempt, 0);
2227 if (THREAD_CAN_MIGRATE(td) && (!THREAD_CAN_SCHED(td, ts->ts_cpu)
2229 ts->ts_cpu = sched_pickcpu(td, 0);
2231 if (ts->ts_cpu == cpuid)
2243 if (tdq->tdq_load == 0)
2251 "prio:%d", td->td_priority);
2254 "prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg,
2255 "lockname:\"%s\"", td->td_lockname);
2260 * appropriate cpu run-queue or sleep-queue and with the current
2261 * thread-queue locked.
2264 MPASS(td == tdq->tdq_curthread);
2274 if (PMC_PROC_IS_USING_PMCS(td->td_proc))
2277 SDT_PROBE2(sched, , , off__cpu, newtd, newtd->td_proc);
2288 td->td_oncpu = NOCPU;
2290 cpuid = td->td_oncpu = PCPU_GET(cpuid);
2294 if (PMC_PROC_IS_USING_PMCS(td->td_proc))
2301 KASSERT(curthread->td_md.md_spinlock_count == 1,
2302 ("invalid count %d", curthread->td_md.md_spinlock_count));
2305 "prio:%d", td->td_priority);
2318 p->p_nice = nice;
2322 sched_prio(td, td->td_base_user_pri);
2336 td->td_slptick = ticks;
2337 if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)
2341 else if (static_boost && td->td_priority > static_boost)
2364 slptick = td->td_slptick;
2365 td->td_slptick = 0;
2367 ts->ts_slptime += (ticks - slptick) << SCHED_TICK_SHIFT;
2376 if (PRI_BASE(td->td_pri_class) == PRI_ITHD &&
2377 td->td_priority != td->td_base_ithread_pri)
2378 sched_prio(td, td->td_base_ithread_pri);
2381 * Reset the slice value since we slept and advanced the round-robin.
2383 ts->ts_slice = 0;
2402 td_get_sched(td)->ts_runtime += tickincr;
2424 child->td_oncpu = NOCPU;
2425 child->td_lastcpu = NOCPU;
2426 child->td_lock = TDQ_LOCKPTR(tdq);
2427 child->td_cpuset = cpuset_ref(td->td_cpuset);
2428 child->td_domain.dr_policy = td->td_cpuset->cs_domain;
2429 ts2->ts_cpu = ts->ts_cpu;
2430 ts2->ts_flags = 0;
2434 ts2->ts_ticks = ts->ts_ticks;
2435 ts2->ts_ltick = ts->ts_ltick;
2436 ts2->ts_ftick = ts->ts_ftick;
2440 child->td_priority = child->td_base_pri;
2444 ts2->ts_slptime = ts->ts_slptime;
2445 ts2->ts_runtime = ts->ts_runtime;
2447 ts2->ts_slice = tdq_slice(tdq) - sched_slice_min;
2449 bzero(ts2->ts_name, sizeof(ts2->ts_name));
2461 if (td->td_pri_class == class)
2463 td->td_pri_class = class;
2475 "prio:%d", child->td_priority);
2492 "prio:%d", child->td_priority);
2499 td_get_sched(td)->ts_runtime += td_get_sched(child)->ts_runtime;
2511 SDT_PROBE2(sched, , , surrender, td, td->td_proc);
2516 if (td->td_priority > tdq->tdq_lowpri) {
2517 if (td->td_critnest == 1) {
2525 td->td_owepreempt = 1;
2527 tdq->tdq_owepreempt = 0;
2533 * Fix priorities on return to user-space. Priorities may be elevated due
2541 td->td_priority = td->td_user_pri;
2542 td->td_base_pri = td->td_user_pri;
2549 "Interrupt thread preemptions due to time-sharing");
2558 if (PRI_BASE(td->td_pri_class) == PRI_ITHD)
2581 balance_ticks -= cnt;
2591 tdq->tdq_oldswitchcnt = tdq->tdq_switchcnt;
2592 tdq->tdq_switchcnt = tdq->tdq_load;
2598 if (tdq->tdq_idx == tdq->tdq_ridx) {
2599 tdq->tdq_idx = (tdq->tdq_idx + 1) % RQ_NQS;
2600 if (TAILQ_EMPTY(&tdq->tdq_timeshare.rq_queues[tdq->tdq_ridx]))
2601 tdq->tdq_ridx = tdq->tdq_idx;
2605 if ((td->td_pri_class & PRI_FIFO_BIT) || TD_IS_IDLETHREAD(td))
2608 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) {
2610 * We used a tick; charge it to the thread so
2613 td_get_sched(td)->ts_runtime += tickincr * cnt;
2622 ts->ts_slice += cnt;
2623 if (ts->ts_slice >= td_slice(td, tdq)) {
2624 ts->ts_slice = 0;
2630 if (PRI_BASE(td->td_pri_class) == PRI_ITHD) {
2632 td->td_owepreempt = 1;
2633 if (td->td_base_pri + RQ_PPQ < PRI_MAX_ITHD) {
2635 sched_prio(td, td->td_base_pri + RQ_PPQ);
2639 td->td_flags |= TDF_SLICEEND;
2652 * Return whether the current CPU has runnable tasks. Used for in-kernel
2664 if ((curthread->td_flags & TDF_IDLETD) != 0) {
2668 if (TDQ_LOAD(tdq) - 1 > 0)
2677 * the run-queue while running however the load remains.
2690 tdq->tdq_lowpri = td->td_priority;
2692 tdq->tdq_lowpri = PRI_MAX_IDLE;
2695 tdq->tdq_curthread = td;
2713 cpri = ctd->td_priority;
2720 ctd->td_owepreempt = 1;
2735 KASSERT((td->td_inhibitors == 0),
2739 KASSERT(td->td_flags & TDF_INMEM,
2742 lowpri = tdq->tdq_lowpri;
2743 if (td->td_priority < lowpri)
2744 tdq->tdq_lowpri = td->td_priority;
2765 "prio:%d", td->td_priority, KTR_ATTR_LINKED,
2769 SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
2774 * run-queue.
2776 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
2789 sched_setpreempt(td->td_priority);
2793 * Now that the thread is moving to the run-queue, set the lock
2796 if (td->td_lock != TDQ_LOCKPTR(tdq)) {
2799 td->td_lock = TDQ_LOCKPTR(tdq);
2805 sched_setpreempt(td->td_priority);
2812 * Remove a thread from a run-queue without running it. This is used
2822 "prio:%d", td->td_priority);
2823 SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);
2824 tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);
2826 MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
2832 if (td->td_priority == tdq->tdq_lowpri)
2850 if (ts->ts_ticks) {
2873 if (THREAD_CAN_SCHED(td, ts->ts_cpu))
2889 ipi_cpu(ts->ts_cpu, IPI_PREEMPT);
2904 if (ts->ts_flags & TSF_BOUND)
2907 ts->ts_flags |= TSF_BOUND;
2911 ts->ts_cpu = cpu;
2928 if ((ts->ts_flags & TSF_BOUND) == 0)
2930 ts->ts_flags &= ~TSF_BOUND;
2938 return (td_get_sched(td)->ts_flags & TSF_BOUND);
2963 total += atomic_load_int(&TDQ_CPU(i)->tdq_sysload);
2966 return (atomic_load_int(&TDQ_SELF()->tdq_sysload));
2984 ((tdq)->tdq_cg != NULL && ((tdq)->tdq_cg->cg_flags & CG_FLAG_THREAD) == 0)
3004 oldswitchcnt = -1;
3042 atomic_store_int(&tdq->tdq_cpu_idle, 1);
3055 atomic_store_int(&tdq->tdq_cpu_idle, 0);
3059 atomic_store_int(&tdq->tdq_cpu_idle, 0);
3062 * Account thread-less hardware interrupts and
3087 KASSERT(curthread->td_md.md_spinlock_count == 1,
3088 ("invalid count %d", curthread->td_md.md_spinlock_count));
3134 td->td_lastcpu = td->td_oncpu;
3135 td->td_oncpu = NOCPU;
3156 * non-nested critical section with the scheduler lock held.
3158 KASSERT(curthread->td_md.md_spinlock_count == 1,
3159 ("invalid count %d", curthread->td_md.md_spinlock_count));
3164 MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
3165 td->td_oncpu = cpuid;
3167 "prio:%d", td->td_priority);
3181 if (ts->ts_name[0] == '\0')
3182 snprintf(ts->ts_name, sizeof(ts->ts_name),
3183 "%s tid %d", td->td_name, td->td_tid);
3184 return (ts->ts_name);
3186 return (td->td_name);
3197 ts->ts_name[0] = '\0';
3214 sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent,
3215 "", 1 + indent / 2, cg->cg_level);
3217 cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask));
3219 for (i = cg->cg_first; i <= cg->cg_last; i++) {
3220 if (CPU_ISSET(i, &cg->cg_mask)) {
3230 if (cg->cg_flags != 0) {
3232 if ((cg->cg_flags & CG_FLAG_HTT) != 0)
3234 if ((cg->cg_flags & CG_FLAG_THREAD) != 0)
3236 if ((cg->cg_flags & CG_FLAG_SMT) != 0)
3238 if ((cg->cg_flags & CG_FLAG_NODE) != 0)
3243 if (cg->cg_children > 0) {
3245 for (i = 0; i < cg->cg_children; i++)
3247 &cg->cg_child[i], indent+2);
3291 if (error != 0 || req->newptr == NULL)
3328 "Enables the long-term load balancer");
3331 "Average period in stathz ticks to run the long-term balancer");