1 /*-
2 * SPDX-License-Identifier: (BSD-4-Clause AND BSD-2-Clause)
3 *
4 * Copyright (c) 1996, 1997
5 * HD Associates, Inc. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by HD Associates, Inc
18 * and Jukka Antero Ukkonen.
19 * 4. Neither the name of the author nor the names of any co-contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY HD ASSOCIATES AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL HD ASSOCIATES OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36 /*-
37 * Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org>
38 * All rights reserved.
39 *
40 * Redistribution and use in source and binary forms, with or without
41 * modification, are permitted provided that the following conditions
42 * are met:
43 * 1. Redistributions of source code must retain the above copyright
44 * notice unmodified, this list of conditions, and the following
45 * disclaimer.
46 * 2. Redistributions in binary form must reproduce the above copyright
47 * notice, this list of conditions and the following disclaimer in the
48 * documentation and/or other materials provided with the distribution.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
54 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
55 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
56 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
57 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
58 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
59 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 */
61
62 #ifndef _SCHED_H_
63 #define _SCHED_H_
64
65 #ifdef _KERNEL
66
67 #include <sys/types.h>
68 #ifdef SCHED_STATS
69 #include <sys/pcpu.h>
70 #endif
71 #include <sys/linker_set.h>
72 #include <sys/sdt.h>
73
74 struct proc;
75 struct thread;
76
77 /*
78 * General scheduling info.
79 *
80 * sched_load:
81 * Total runnable non-ithread threads in the system.
82 *
83 * sched_runnable:
84 * Runnable threads for this processor.
85 */
86 int sched_load(void);
87 int sched_rr_interval(void);
88 bool sched_runnable(void);
89
90 /*
91 * Proc related scheduling hooks.
92 */
93 void sched_exit(struct proc *p, struct thread *childtd);
94 void sched_fork(struct thread *td, struct thread *childtd);
95 void sched_fork_exit(struct thread *td);
96 void sched_class(struct thread *td, int class);
97 void sched_nice(struct proc *p, int nice);
98
99 /*
100 * Threads are switched in and out, block on resources, have temporary
101 * priorities inherited from their procs, and use up cpu time.
102 */
103 void sched_ap_entry(void);
104 void sched_exit_thread(struct thread *td, struct thread *child);
105 u_int sched_estcpu(struct thread *td);
106 void sched_fork_thread(struct thread *td, struct thread *child);
107 void sched_ithread_prio(struct thread *td, u_char prio);
108 void sched_lend_prio(struct thread *td, u_char prio);
109 void sched_lend_user_prio(struct thread *td, u_char pri);
110 void sched_lend_user_prio_cond(struct thread *td, u_char pri);
111 fixpt_t sched_pctcpu(struct thread *td);
112 void sched_prio(struct thread *td, u_char prio);
113 void sched_sleep(struct thread *td, int prio);
114 void sched_switch(struct thread *td, int flags);
115 void sched_throw(struct thread *td);
116 void sched_unlend_prio(struct thread *td, u_char prio);
117 void sched_user_prio(struct thread *td, u_char prio);
118 void sched_userret_slowpath(struct thread *td);
119
120 static inline void
sched_userret(struct thread * td)121 sched_userret(struct thread *td)
122 {
123
124 /*
125 * XXX we cheat slightly on the locking here to avoid locking in
126 * the usual case. Setting td_priority here is essentially an
127 * incomplete workaround for not setting it properly elsewhere.
128 * Now that some interrupt handlers are threads, not setting it
129 * properly elsewhere can clobber it in the window between setting
130 * it here and returning to user mode, so don't waste time setting
131 * it perfectly here.
132 */
133 KASSERT((td->td_flags & TDF_BORROWING) == 0,
134 ("thread with borrowed priority returning to userland"));
135 if (__predict_false(td->td_priority != td->td_user_pri))
136 sched_userret_slowpath(td);
137 }
138
139 /*
140 * Threads are moved on and off of run queues
141 */
142 void sched_add(struct thread *td, int flags);
143 struct thread *sched_choose(void);
144 void sched_clock(struct thread *td, int cnt);
145 void sched_idletd(void *);
146 void sched_preempt(struct thread *td);
147 void sched_relinquish(struct thread *td);
148 void sched_rem(struct thread *td);
149 void sched_wakeup(struct thread *td, int srqflags);
150
151 /*
152 * Binding makes cpu affinity permanent while pinning is used to temporarily
153 * hold a thread on a particular CPU.
154 */
155 void sched_bind(struct thread *td, int cpu);
156 static __inline void sched_pin(void);
157 void sched_unbind(struct thread *td);
158 static __inline void sched_unpin(void);
159 int sched_is_bound(struct thread *td);
160 void sched_affinity(struct thread *td);
161
162 /*
163 * These procedures tell the process data structure allocation code how
164 * many bytes to actually allocate.
165 */
166 int sched_sizeof_proc(void);
167 int sched_sizeof_thread(void);
168
169 /*
170 * This routine provides a consistent thread name for use with KTR graphing
171 * functions.
172 */
173 char *sched_tdname(struct thread *td);
174 void sched_clear_tdname(struct thread *td);
175
176 static __inline void
sched_pin(void)177 sched_pin(void)
178 {
179 curthread->td_pinned++;
180 atomic_interrupt_fence();
181 }
182
183 static __inline void
sched_unpin(void)184 sched_unpin(void)
185 {
186 atomic_interrupt_fence();
187 MPASS(curthread->td_pinned > 0);
188 curthread->td_pinned--;
189 }
190
191 void ast_scheduler(struct thread *td, int tda);
192
193 /* sched_add arguments (formerly setrunqueue) */
194 #define SRQ_BORING 0x0000 /* No special circumstances. */
195 #define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */
196 #define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */
197 #define SRQ_INTR 0x0004 /* It is probably urgent. */
198 #define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */
199 #define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */
200 #define SRQ_HOLD 0x0020 /* Return holding original td lock */
201 #define SRQ_HOLDTD 0x0040 /* Return holding td lock */
202
203 /* Scheduler stats. */
204 #ifdef SCHED_STATS
205 DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);
206
207 #define SCHED_STAT_DEFINE_VAR(name, ptr, descr) \
208 static void name ## _add_proc(void *dummy __unused) \
209 { \
210 \
211 SYSCTL_ADD_PROC(NULL, \
212 SYSCTL_STATIC_CHILDREN(_kern_sched_stats), OID_AUTO, \
213 #name, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE, \
214 ptr, 0, sysctl_dpcpu_long, "LU", descr); \
215 } \
216 SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL);
217
218 #define SCHED_STAT_DEFINE(name, descr) \
219 DPCPU_DEFINE(unsigned long, name); \
220 SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr)
221
222 #define SCHED_STAT_DECLARE(name) \
223 DPCPU_DECLARE(unsigned long, name);
224
225 /*
226 * Sched stats are always incremented in critical sections so no atomic
227 * is necessary to increment them.
228 */
229 #define SCHED_STAT_INC(var) DPCPU_GET(var)++;
230 #else
231 #define SCHED_STAT_DEFINE_VAR(name, descr, ptr)
232 #define SCHED_STAT_DEFINE(name, descr)
233 #define SCHED_STAT_DECLARE(name)
234 #define SCHED_STAT_INC(var) (void)0
235 #endif
236
237 SCHED_STAT_DECLARE(ithread_demotions);
238 SCHED_STAT_DECLARE(ithread_preemptions);
239
240 SDT_PROBE_DECLARE(sched, , , change__pri);
241 SDT_PROBE_DECLARE(sched, , , dequeue);
242 SDT_PROBE_DECLARE(sched, , , enqueue);
243 SDT_PROBE_DECLARE(sched, , , lend__pri);
244 SDT_PROBE_DECLARE(sched, , , load__change);
245 SDT_PROBE_DECLARE(sched, , , off__cpu);
246 SDT_PROBE_DECLARE(sched, , , on__cpu);
247 SDT_PROBE_DECLARE(sched, , , remain__cpu);
248 SDT_PROBE_DECLARE(sched, , , surrender);
249
250 #ifdef KDTRACE_HOOKS
251 #include <sys/dtrace_bsd.h>
252 extern int dtrace_vtime_active;
253 extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
254 #endif
255
256 /*
257 * Fixup scheduler state for proc0 and thread0
258 */
259 void schedinit(void);
260
261 /*
262 * Fixup scheduler state for secondary APs
263 */
264 void schedinit_ap(void);
265
266 bool sched_do_timer_accounting(void);
267
268 /*
269 * Find an L2 neighbor of the given CPU or return -1 if none found. This
270 * does not distinguish among multiple L2 neighbors if the given CPU has
271 * more than one (it will always return the same result in that case).
272 */
273 int sched_find_l2_neighbor(int cpu);
274
275 struct sched_instance {
276 int (*load)(void);
277 int (*rr_interval)(void);
278 bool (*runnable)(void);
279 void (*exit)(struct proc *p, struct thread *childtd);
280 void (*fork)(struct thread *td, struct thread *childtd);
281 void (*fork_exit)(struct thread *td);
282 void (*class)(struct thread *td, int class);
283 void (*nice)(struct proc *p, int nice);
284 void (*ap_entry)(void);
285 void (*exit_thread)(struct thread *td, struct thread *child);
286 u_int (*estcpu)(struct thread *td);
287 void (*fork_thread)(struct thread *td, struct thread *child);
288 void (*ithread_prio)(struct thread *td, u_char prio);
289 void (*lend_prio)(struct thread *td, u_char prio);
290 void (*lend_user_prio)(struct thread *td, u_char pri);
291 void (*lend_user_prio_cond)(struct thread *td, u_char pri);
292 fixpt_t (*pctcpu)(struct thread *td);
293 void (*prio)(struct thread *td, u_char prio);
294 void (*sleep)(struct thread *td, int prio);
295 void (*sswitch)(struct thread *td, int flags);
296 void (*throw)(struct thread *td);
297 void (*unlend_prio)(struct thread *td, u_char prio);
298 void (*user_prio)(struct thread *td, u_char prio);
299 void (*userret_slowpath)(struct thread *td);
300 void (*add)(struct thread *td, int flags);
301 struct thread *(*choose)(void);
302 void (*clock)(struct thread *td, int cnt);
303 void (*idletd)(void *);
304 void (*preempt)(struct thread *td);
305 void (*relinquish)(struct thread *td);
306 void (*rem)(struct thread *td);
307 void (*wakeup)(struct thread *td, int srqflags);
308 void (*bind)(struct thread *td, int cpu);
309 void (*unbind)(struct thread *td);
310 int (*is_bound)(struct thread *td);
311 void (*affinity)(struct thread *td);
312 int (*sizeof_proc)(void);
313 int (*sizeof_thread)(void);
314 char *(*tdname)(struct thread *td);
315 void (*clear_tdname)(struct thread *td);
316 bool (*do_timer_accounting)(void);
317 int (*find_l2_neighbor)(int cpuid);
318 void (*init)(void);
319 void (*init_ap)(void);
320 void (*setup)(void);
321 void (*initticks)(void);
322 void (*schedcpu)(void);
323 };
324
325 extern const struct sched_instance *active_sched;
326
327 struct sched_selection {
328 const char *name;
329 const struct sched_instance *instance;
330 };
331 #define DECLARE_SCHEDULER(xsel_name, xsched_name, xsched_instance) \
332 static struct sched_selection xsel_name = { \
333 .name = xsched_name, \
334 .instance = xsched_instance, \
335 }; \
336 DATA_SET(sched_instance_set, xsel_name);
337
338 void sched_instance_select(void);
339
340 #endif /* _KERNEL */
341
342 /* POSIX 1003.1b Process Scheduling */
343
344 /*
345 * POSIX scheduling policies
346 */
347 #define SCHED_FIFO 1
348 #define SCHED_OTHER 2
349 #define SCHED_RR 3
350
351 struct sched_param {
352 int sched_priority;
353 };
354
355 /*
356 * POSIX scheduling declarations for userland.
357 */
358 #ifndef _KERNEL
359 #include <sys/cdefs.h>
360 #include <sys/_timespec.h>
361 #include <sys/_types.h>
362
363 #ifndef _PID_T_DECLARED
364 typedef __pid_t pid_t;
365 #define _PID_T_DECLARED
366 #endif
367
368 __BEGIN_DECLS
369 int sched_get_priority_max(int);
370 int sched_get_priority_min(int);
371 int sched_getparam(pid_t, struct sched_param *);
372 int sched_getscheduler(pid_t);
373 int sched_rr_get_interval(pid_t, struct timespec *);
374 int sched_setparam(pid_t, const struct sched_param *);
375 int sched_setscheduler(pid_t, int, const struct sched_param *);
376 int sched_yield(void);
377 __END_DECLS
378
379 #endif
380 #endif /* !_SCHED_H_ */
381