1 /*
2 * Copyright 2026 The FreeBSD Foundation
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 */
9
10 #include "opt_sched.h"
11
12 #include <sys/systm.h>
13 #include <sys/kernel.h>
14 #include <sys/lock.h>
15 #include <sys/proc.h>
16 #include <sys/runq.h>
17 #include <sys/sbuf.h>
18 #include <sys/sched.h>
19 #include <sys/smp.h>
20 #include <sys/sysctl.h>
21 #include <machine/ifunc.h>
22
23 const struct sched_instance *active_sched;
24
25 #define __DEFINE_SHIM(__m, __r, __n, __p, __a) \
26 DEFINE_IFUNC(, __r, __n, __p) \
27 { \
28 return (active_sched->__m); \
29 }
30 #define DEFINE_SHIM0(__m, __r, __n) \
31 __DEFINE_SHIM(__m, __r, __n, (void), ())
32 #define DEFINE_SHIM1(__m, __r, __n, __t1, __a1) \
33 __DEFINE_SHIM(__m, __r, __n, (__t1 __a1), (__a1))
34 #define DEFINE_SHIM2(__m, __r, __n, __t1, __a1, __t2, __a2) \
35 __DEFINE_SHIM(__m, __r, __n, (__t1 __a1, __t2 __a2), (__a1, __a2))
36
37 DEFINE_SHIM0(load, int, sched_load)
38 DEFINE_SHIM0(rr_interval, int, sched_rr_interval)
39 DEFINE_SHIM0(runnable, bool, sched_runnable)
40 DEFINE_SHIM2(exit, void, sched_exit, struct proc *, p,
41 struct thread *, childtd)
42 DEFINE_SHIM2(fork, void, sched_fork, struct thread *, td,
43 struct thread *, childtd)
44 DEFINE_SHIM1(fork_exit, void, sched_fork_exit, struct thread *, td)
45 DEFINE_SHIM2(class, void, sched_class, struct thread *, td, int, class)
46 DEFINE_SHIM2(nice, void, sched_nice, struct proc *, p, int, nice)
47 DEFINE_SHIM0(ap_entry, void, sched_ap_entry)
48 DEFINE_SHIM2(exit_thread, void, sched_exit_thread, struct thread *, td,
49 struct thread *, child)
50 DEFINE_SHIM1(estcpu, u_int, sched_estcpu, struct thread *, td)
51 DEFINE_SHIM2(fork_thread, void, sched_fork_thread, struct thread *, td,
52 struct thread *, child)
53 DEFINE_SHIM2(ithread_prio, void, sched_ithread_prio, struct thread *, td,
54 u_char, prio)
55 DEFINE_SHIM2(lend_prio, void, sched_lend_prio, struct thread *, td,
56 u_char, prio)
57 DEFINE_SHIM2(lend_user_prio, void, sched_lend_user_prio, struct thread *, td,
58 u_char, pri)
59 DEFINE_SHIM2(lend_user_prio_cond, void, sched_lend_user_prio_cond,
60 struct thread *, td, u_char, pri)
61 DEFINE_SHIM1(pctcpu, fixpt_t, sched_pctcpu, struct thread *, td)
62 DEFINE_SHIM2(prio, void, sched_prio, struct thread *, td, u_char, prio)
63 DEFINE_SHIM2(sleep, void, sched_sleep, struct thread *, td, int, prio)
64 DEFINE_SHIM2(sswitch, void, sched_switch, struct thread *, td, int, flags)
65 DEFINE_SHIM1(throw, void, sched_throw, struct thread *, td)
66 DEFINE_SHIM2(unlend_prio, void, sched_unlend_prio, struct thread *, td,
67 u_char, prio)
68 DEFINE_SHIM2(user_prio, void, sched_user_prio, struct thread *, td,
69 u_char, prio)
70 DEFINE_SHIM1(userret_slowpath, void, sched_userret_slowpath,
71 struct thread *, td)
72 DEFINE_SHIM2(add, void, sched_add, struct thread *, td, int, flags)
73 DEFINE_SHIM0(choose, struct thread *, sched_choose)
74 DEFINE_SHIM2(clock, void, sched_clock, struct thread *, td, int, cnt)
75 DEFINE_SHIM1(idletd, void, sched_idletd, void *, dummy)
76 DEFINE_SHIM1(preempt, void, sched_preempt, struct thread *, td)
77 DEFINE_SHIM1(relinquish, void, sched_relinquish, struct thread *, td)
78 DEFINE_SHIM1(rem, void, sched_rem, struct thread *, td)
79 DEFINE_SHIM2(wakeup, void, sched_wakeup, struct thread *, td, int, srqflags)
80 DEFINE_SHIM2(bind, void, sched_bind, struct thread *, td, int, cpu)
81 DEFINE_SHIM1(unbind, void, sched_unbind, struct thread *, td)
82 DEFINE_SHIM1(is_bound, int, sched_is_bound, struct thread *, td)
83 DEFINE_SHIM1(affinity, void, sched_affinity, struct thread *, td)
84 DEFINE_SHIM0(sizeof_proc, int, sched_sizeof_proc)
85 DEFINE_SHIM0(sizeof_thread, int, sched_sizeof_thread)
86 DEFINE_SHIM1(tdname, char *, sched_tdname, struct thread *, td)
87 DEFINE_SHIM1(clear_tdname, void, sched_clear_tdname, struct thread *, td)
88 DEFINE_SHIM1(find_l2_neighbor, int, sched_find_l2_neighbor, int, cpu)
89 DEFINE_SHIM0(init_ap, void, schedinit_ap)
90
91
92 SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions");
93 SCHED_STAT_DEFINE(ithread_preemptions,
94 "Interrupt thread preemptions due to time-sharing");
95
96 SDT_PROVIDER_DEFINE(sched);
97
98 SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *",
99 "struct proc *", "uint8_t");
100 SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *",
101 "struct proc *", "void *");
102 SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *",
103 "struct proc *", "void *", "int");
104 SDT_PROBE_DEFINE4(sched, , , lend__pri, "struct thread *",
105 "struct proc *", "uint8_t", "struct thread *");
106 SDT_PROBE_DEFINE2(sched, , , load__change, "int", "int");
107 SDT_PROBE_DEFINE2(sched, , , off__cpu, "struct thread *",
108 "struct proc *");
109 SDT_PROBE_DEFINE(sched, , , on__cpu);
110 SDT_PROBE_DEFINE(sched, , , remain__cpu);
111 SDT_PROBE_DEFINE2(sched, , , surrender, "struct thread *",
112 "struct proc *");
113
114 #ifdef KDTRACE_HOOKS
115 #include <sys/dtrace_bsd.h>
116 int __read_mostly dtrace_vtime_active;
117 dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
118 #endif
119
120 static char sched_name[32] = "ULE";
121
122 SET_DECLARE(sched_instance_set, struct sched_selection);
123
124 void
sched_instance_select(void)125 sched_instance_select(void)
126 {
127 struct sched_selection *s, **ss;
128 int i;
129
130 TUNABLE_STR_FETCH("kern.sched.name", sched_name, sizeof(sched_name));
131 SET_FOREACH(ss, sched_instance_set) {
132 s = *ss;
133 for (i = 0; s->name[i] == sched_name[i]; i++) {
134 if (s->name[i] == '\0') {
135 active_sched = s->instance;
136 return;
137 }
138 }
139 }
140
141 /*
142 * No scheduler matching the configuration was found. If
143 * there is any scheduler compiled in, at all, use the first
144 * scheduler from the linker set.
145 */
146 if (SET_BEGIN(sched_instance_set) < SET_LIMIT(sched_instance_set)) {
147 s = *SET_BEGIN(sched_instance_set);
148 active_sched = s->instance;
149 for (i = 0;; i++) {
150 sched_name[i] = s->name[i];
151 if (s->name[i] == '\0')
152 break;
153 }
154 }
155 }
156
157 void
schedinit(void)158 schedinit(void)
159 {
160 if (active_sched == NULL)
161 panic("Cannot find scheduler %s", sched_name);
162 active_sched->init();
163 }
164
165 struct cpu_group __read_mostly *cpu_top; /* CPU topology */
166
167 static void
sched_setup(void * dummy)168 sched_setup(void *dummy)
169 {
170 cpu_top = smp_topo();
171 active_sched->setup();
172 }
173 SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);
174
175 static void
sched_initticks(void * dummy)176 sched_initticks(void *dummy)
177 {
178 active_sched->initticks();
179 }
180 SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
181 NULL);
182
183 static void
sched_sysinit(void)184 sched_sysinit(void)
185 {
186 active_sched->sysinit();
187 }
188 SYSINIT(sched_sysinit, SI_SUB_LAST, SI_ORDER_FIRST, sched_sysinit, NULL);
189
190 SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
191 "Scheduler");
192
193 SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, sched_name, 0,
194 "Scheduler name");
195
196 static int
sysctl_kern_sched_available(SYSCTL_HANDLER_ARGS)197 sysctl_kern_sched_available(SYSCTL_HANDLER_ARGS)
198 {
199 struct sched_selection *s, **ss;
200 struct sbuf *sb, sm;
201 int error;
202 bool first;
203
204 sb = sbuf_new_for_sysctl(&sm, NULL, 0, req);
205 if (sb == NULL)
206 return (ENOMEM);
207 first = true;
208 SET_FOREACH(ss, sched_instance_set) {
209 s = *ss;
210 if (first)
211 first = false;
212 else
213 sbuf_cat(sb, ",");
214 sbuf_cat(sb, s->name);
215 }
216 error = sbuf_finish(sb);
217 sbuf_delete(sb);
218 return (error);
219 }
220
221 SYSCTL_PROC(_kern_sched, OID_AUTO, available,
222 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
223 NULL, 0, sysctl_kern_sched_available, "A",
224 "List of available schedulers");
225
226 fixpt_t ccpu;
227 SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0,
228 "Decay factor used for updating %CPU");
229
230 /*
231 * Build the CPU topology dump string. Is recursively called to collect
232 * the topology tree.
233 */
234 static int
sysctl_kern_sched_topology_spec_internal(struct sbuf * sb,struct cpu_group * cg,int indent)235 sysctl_kern_sched_topology_spec_internal(struct sbuf *sb,
236 struct cpu_group *cg, int indent)
237 {
238 char cpusetbuf[CPUSETBUFSIZ];
239 int i, first;
240
241 if (cpu_top == NULL) {
242 sbuf_printf(sb, "%*s<group level=\"1\" cache-level=\"1\">\n",
243 indent, "");
244 sbuf_printf(sb, "%*s</group>\n", indent, "");
245 return (0);
246 }
247
248 sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent,
249 "", 1 + indent / 2, cg->cg_level);
250 sbuf_printf(sb, "%*s <cpu count=\"%d\" mask=\"%s\">", indent, "",
251 cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask));
252 first = TRUE;
253 for (i = cg->cg_first; i <= cg->cg_last; i++) {
254 if (CPU_ISSET(i, &cg->cg_mask)) {
255 if (!first)
256 sbuf_cat(sb, ", ");
257 else
258 first = FALSE;
259 sbuf_printf(sb, "%d", i);
260 }
261 }
262 sbuf_cat(sb, "</cpu>\n");
263
264 if (cg->cg_flags != 0) {
265 sbuf_printf(sb, "%*s <flags>", indent, "");
266 if ((cg->cg_flags & CG_FLAG_HTT) != 0)
267 sbuf_cat(sb, "<flag name=\"HTT\">HTT group</flag>");
268 if ((cg->cg_flags & CG_FLAG_THREAD) != 0)
269 sbuf_cat(sb, "<flag name=\"THREAD\">THREAD group</flag>");
270 if ((cg->cg_flags & CG_FLAG_SMT) != 0)
271 sbuf_cat(sb, "<flag name=\"SMT\">SMT group</flag>");
272 if ((cg->cg_flags & CG_FLAG_NODE) != 0)
273 sbuf_cat(sb, "<flag name=\"NODE\">NUMA node</flag>");
274 sbuf_cat(sb, "</flags>\n");
275 }
276
277 if (cg->cg_children > 0) {
278 sbuf_printf(sb, "%*s <children>\n", indent, "");
279 for (i = 0; i < cg->cg_children; i++)
280 sysctl_kern_sched_topology_spec_internal(sb,
281 &cg->cg_child[i], indent + 2);
282 sbuf_printf(sb, "%*s </children>\n", indent, "");
283 }
284 sbuf_printf(sb, "%*s</group>\n", indent, "");
285 return (0);
286 }
287
288 /*
289 * Sysctl handler for retrieving topology dump. It's a wrapper for
290 * the recursive sysctl_kern_smp_topology_spec_internal().
291 */
292 static int
sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS)293 sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS)
294 {
295 struct sbuf *topo;
296 int err;
297
298 topo = sbuf_new_for_sysctl(NULL, NULL, 512, req);
299 if (topo == NULL)
300 return (ENOMEM);
301
302 sbuf_cat(topo, "<groups>\n");
303 err = sysctl_kern_sched_topology_spec_internal(topo, cpu_top, 1);
304 sbuf_cat(topo, "</groups>\n");
305
306 if (err == 0)
307 err = sbuf_finish(topo);
308 sbuf_delete(topo);
309 return (err);
310 }
311
312 SYSCTL_PROC(_kern_sched, OID_AUTO, topology_spec, CTLTYPE_STRING |
313 CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0,
314 sysctl_kern_sched_topology_spec, "A",
315 "XML dump of detected CPU topology");
316