1 /*-
2 * SPDX-License-Identifier: (BSD-4-Clause AND BSD-2-Clause)
3 *
4 * Copyright (c) 1996, 1997
5 * HD Associates, Inc. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by HD Associates, Inc
18 * and Jukka Antero Ukkonen.
19 * 4. Neither the name of the author nor the names of any co-contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY HD ASSOCIATES AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL HD ASSOCIATES OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36 /*-
37 * Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org>
38 * All rights reserved.
39 *
40 * Redistribution and use in source and binary forms, with or without
41 * modification, are permitted provided that the following conditions
42 * are met:
43 * 1. Redistributions of source code must retain the above copyright
44 * notice unmodified, this list of conditions, and the following
45 * disclaimer.
46 * 2. Redistributions in binary form must reproduce the above copyright
47 * notice, this list of conditions and the following disclaimer in the
48 * documentation and/or other materials provided with the distribution.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
54 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
55 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
56 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
57 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
58 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
59 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 */
61
62 #ifndef _SCHED_H_
63 #define _SCHED_H_
64
65 #ifdef _KERNEL
66
67 #include <sys/types.h>
68 #ifdef SCHED_STATS
69 #include <sys/pcpu.h>
70 #endif
71 #include <sys/linker_set.h>
72 #include <sys/sdt.h>
73
74 struct proc;
75 struct thread;
76
77 /*
78 * General scheduling info.
79 *
80 * sched_load:
81 * Total runnable non-ithread threads in the system.
82 *
83 * sched_runnable:
84 * Runnable threads for this processor.
85 */
86 int sched_load(void);
87 int sched_rr_interval(void);
88 bool sched_runnable(void);
89
90 /*
91 * Proc related scheduling hooks.
92 */
93 void sched_exit(struct proc *p, struct thread *childtd);
94 void sched_fork(struct thread *td, struct thread *childtd);
95 void sched_fork_exit(struct thread *td);
96 void sched_class(struct thread *td, int class);
97 void sched_nice(struct proc *p, int nice);
98
99 /*
100 * Threads are switched in and out, block on resources, have temporary
101 * priorities inherited from their procs, and use up cpu time.
102 */
103 void sched_ap_entry(void);
104 void sched_exit_thread(struct thread *td, struct thread *child);
105 u_int sched_estcpu(struct thread *td);
106 void sched_fork_thread(struct thread *td, struct thread *child);
107 void sched_ithread_prio(struct thread *td, u_char prio);
108 void sched_lend_prio(struct thread *td, u_char prio);
109 void sched_lend_user_prio(struct thread *td, u_char pri);
110 void sched_lend_user_prio_cond(struct thread *td, u_char pri);
111 fixpt_t sched_pctcpu(struct thread *td);
112 void sched_prio(struct thread *td, u_char prio);
113 void sched_sleep(struct thread *td, int prio);
114 void sched_switch(struct thread *td, int flags);
115 void sched_throw(struct thread *td);
116 void sched_unlend_prio(struct thread *td, u_char prio);
117 void sched_user_prio(struct thread *td, u_char prio);
118 void sched_userret_slowpath(struct thread *td);
119
120 static inline void
sched_userret(struct thread * td)121 sched_userret(struct thread *td)
122 {
123
124 /*
125 * XXX we cheat slightly on the locking here to avoid locking in
126 * the usual case. Setting td_priority here is essentially an
127 * incomplete workaround for not setting it properly elsewhere.
128 * Now that some interrupt handlers are threads, not setting it
129 * properly elsewhere can clobber it in the window between setting
130 * it here and returning to user mode, so don't waste time setting
131 * it perfectly here.
132 */
133 KASSERT((td->td_flags & TDF_BORROWING) == 0,
134 ("thread with borrowed priority returning to userland"));
135 if (__predict_false(td->td_priority != td->td_user_pri))
136 sched_userret_slowpath(td);
137 }
138
139 /*
140 * Threads are moved on and off of run queues
141 */
142 void sched_add(struct thread *td, int flags);
143 struct thread *sched_choose(void);
144 void sched_clock(struct thread *td, int cnt);
145 void sched_idletd(void *);
146 void sched_preempt(struct thread *td);
147 void sched_relinquish(struct thread *td);
148 void sched_rem(struct thread *td);
149 void sched_wakeup(struct thread *td, int srqflags);
150
151 /*
152 * Binding makes cpu affinity permanent while pinning is used to temporarily
153 * hold a thread on a particular CPU.
154 */
155 void sched_bind(struct thread *td, int cpu);
156 static __inline void sched_pin(void);
157 void sched_unbind(struct thread *td);
158 static __inline void sched_unpin(void);
159 int sched_is_bound(struct thread *td);
160 void sched_affinity(struct thread *td);
161
162 /*
163 * These procedures tell the process data structure allocation code how
164 * many bytes to actually allocate.
165 */
166 int sched_sizeof_proc(void);
167 int sched_sizeof_thread(void);
168
169 /*
170 * This routine provides a consistent thread name for use with KTR graphing
171 * functions.
172 */
173 char *sched_tdname(struct thread *td);
174 void sched_clear_tdname(struct thread *td);
175
176 static __inline void
sched_pin(void)177 sched_pin(void)
178 {
179 curthread->td_pinned++;
180 atomic_interrupt_fence();
181 }
182
183 static __inline void
sched_unpin(void)184 sched_unpin(void)
185 {
186 atomic_interrupt_fence();
187 MPASS(curthread->td_pinned > 0);
188 curthread->td_pinned--;
189 }
190
191 void ast_scheduler(struct thread *td, int tda);
192
193 /* sched_add arguments (formerly setrunqueue) */
194 #define SRQ_BORING 0x0000 /* No special circumstances. */
195 #define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */
196 #define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */
197 #define SRQ_INTR 0x0004 /* It is probably urgent. */
198 #define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */
199 #define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */
200 #define SRQ_HOLD 0x0020 /* Return holding original td lock */
201 #define SRQ_HOLDTD 0x0040 /* Return holding td lock */
202
203 /* Scheduler stats. */
204 #ifdef SCHED_STATS
205 DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);
206
207 #define SCHED_STAT_DEFINE_VAR(name, ptr, descr) \
208 static void name ## _add_proc(void *dummy __unused) \
209 { \
210 \
211 SYSCTL_ADD_PROC(NULL, \
212 SYSCTL_STATIC_CHILDREN(_kern_sched_stats), OID_AUTO, \
213 #name, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE, \
214 ptr, 0, sysctl_dpcpu_long, "LU", descr); \
215 } \
216 SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL);
217
218 #define SCHED_STAT_DEFINE(name, descr) \
219 DPCPU_DEFINE(unsigned long, name); \
220 SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr)
221
222 #define SCHED_STAT_DECLARE(name) \
223 DPCPU_DECLARE(unsigned long, name);
224
225 /*
226 * Sched stats are always incremented in critical sections so no atomic
227 * is necessary to increment them.
228 */
229 #define SCHED_STAT_INC(var) DPCPU_GET(var)++;
230 #else
231 #define SCHED_STAT_DEFINE_VAR(name, descr, ptr)
232 #define SCHED_STAT_DEFINE(name, descr)
233 #define SCHED_STAT_DECLARE(name)
234 #define SCHED_STAT_INC(var) (void)0
235 #endif
236
237 SCHED_STAT_DECLARE(ithread_demotions);
238 SCHED_STAT_DECLARE(ithread_preemptions);
239
240 SDT_PROBE_DECLARE(sched, , , change__pri);
241 SDT_PROBE_DECLARE(sched, , , dequeue);
242 SDT_PROBE_DECLARE(sched, , , enqueue);
243 SDT_PROBE_DECLARE(sched, , , lend__pri);
244 SDT_PROBE_DECLARE(sched, , , load__change);
245 SDT_PROBE_DECLARE(sched, , , off__cpu);
246 SDT_PROBE_DECLARE(sched, , , on__cpu);
247 SDT_PROBE_DECLARE(sched, , , remain__cpu);
248 SDT_PROBE_DECLARE(sched, , , surrender);
249
250 #ifdef KDTRACE_HOOKS
251 #include <sys/dtrace_bsd.h>
252 extern int dtrace_vtime_active;
253 extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
254 #endif
255
256 /*
257 * Fixup scheduler state for proc0 and thread0
258 */
259 void schedinit(void);
260
261 /*
262 * Fixup scheduler state for secondary APs
263 */
264 void schedinit_ap(void);
265
266 /*
267 * Find an L2 neighbor of the given CPU or return -1 if none found. This
268 * does not distinguish among multiple L2 neighbors if the given CPU has
269 * more than one (it will always return the same result in that case).
270 */
271 int sched_find_l2_neighbor(int cpu);
272
273 /*
274 * The scheduler selection interface uses names that are reserved words in
275 * C++, causing problems for downstream projects that use C++ in the
276 * kernel.
277 */
278 #ifndef __cplusplus
279
280 struct sched_instance {
281 int (*load)(void);
282 int (*rr_interval)(void);
283 bool (*runnable)(void);
284 void (*exit)(struct proc *p, struct thread *childtd);
285 void (*fork)(struct thread *td, struct thread *childtd);
286 void (*fork_exit)(struct thread *td);
287 void (*class)(struct thread *td, int class);
288 void (*nice)(struct proc *p, int nice);
289 void (*ap_entry)(void);
290 void (*exit_thread)(struct thread *td, struct thread *child);
291 u_int (*estcpu)(struct thread *td);
292 void (*fork_thread)(struct thread *td, struct thread *child);
293 void (*ithread_prio)(struct thread *td, u_char prio);
294 void (*lend_prio)(struct thread *td, u_char prio);
295 void (*lend_user_prio)(struct thread *td, u_char pri);
296 void (*lend_user_prio_cond)(struct thread *td, u_char pri);
297 fixpt_t (*pctcpu)(struct thread *td);
298 void (*prio)(struct thread *td, u_char prio);
299 void (*sleep)(struct thread *td, int prio);
300 void (*sswitch)(struct thread *td, int flags);
301 void (*throw)(struct thread *td);
302 void (*unlend_prio)(struct thread *td, u_char prio);
303 void (*user_prio)(struct thread *td, u_char prio);
304 void (*userret_slowpath)(struct thread *td);
305 void (*add)(struct thread *td, int flags);
306 struct thread *(*choose)(void);
307 void (*clock)(struct thread *td, int cnt);
308 void (*idletd)(void *);
309 void (*preempt)(struct thread *td);
310 void (*relinquish)(struct thread *td);
311 void (*rem)(struct thread *td);
312 void (*wakeup)(struct thread *td, int srqflags);
313 void (*bind)(struct thread *td, int cpu);
314 void (*unbind)(struct thread *td);
315 int (*is_bound)(struct thread *td);
316 void (*affinity)(struct thread *td);
317 int (*sizeof_proc)(void);
318 int (*sizeof_thread)(void);
319 char *(*tdname)(struct thread *td);
320 void (*clear_tdname)(struct thread *td);
321 int (*find_l2_neighbor)(int cpuid);
322 void (*init)(void);
323 void (*init_ap)(void);
324 void (*setup)(void);
325 void (*initticks)(void);
326 void (*schedcpu)(void);
327 };
328
329 extern const struct sched_instance *active_sched;
330
331 struct sched_selection {
332 const char *name;
333 const struct sched_instance *instance;
334 };
335 #define DECLARE_SCHEDULER(xsel_name, xsched_name, xsched_instance) \
336 static struct sched_selection xsel_name = { \
337 .name = xsched_name, \
338 .instance = xsched_instance, \
339 }; \
340 DATA_SET(sched_instance_set, xsel_name);
341
342 void sched_instance_select(void);
343
344 #endif /* !__cplusplus */
345
346 #endif /* _KERNEL */
347
348 /* POSIX 1003.1b Process Scheduling */
349
350 /*
351 * POSIX scheduling policies
352 */
353 #define SCHED_FIFO 1
354 #define SCHED_OTHER 2
355 #define SCHED_RR 3
356
357 struct sched_param {
358 int sched_priority;
359 };
360
361 /*
362 * POSIX scheduling declarations for userland.
363 */
364 #ifndef _KERNEL
365 #include <sys/cdefs.h>
366 #include <sys/_timespec.h>
367 #include <sys/_types.h>
368
369 #ifndef _PID_T_DECLARED
370 typedef __pid_t pid_t;
371 #define _PID_T_DECLARED
372 #endif /* !_PID_T_DECLARED */
373
374 __BEGIN_DECLS
375 int sched_get_priority_max(int);
376 int sched_get_priority_min(int);
377 int sched_getparam(pid_t, struct sched_param *);
378 int sched_getscheduler(pid_t);
379 int sched_rr_get_interval(pid_t, struct timespec *);
380 int sched_setparam(pid_t, const struct sched_param *);
381 int sched_setscheduler(pid_t, int, const struct sched_param *);
382 int sched_yield(void);
383 __END_DECLS
384
385 #endif /* !_KERNEL */
386
387 #endif /* !_SCHED_H_ */
388