1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 6 */ 7 #ifdef CONFIG_SCHED_CLASS_EXT 8 9 struct sched_enq_and_set_ctx { 10 struct task_struct *p; 11 int queue_flags; 12 bool queued; 13 bool running; 14 }; 15 16 void sched_deq_and_put_task(struct task_struct *p, int queue_flags, 17 struct sched_enq_and_set_ctx *ctx); 18 void sched_enq_and_set_task(struct sched_enq_and_set_ctx *ctx); 19 20 extern const struct sched_class ext_sched_class; 21 22 DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); 23 DECLARE_STATIC_KEY_FALSE(__scx_switched_all); 24 #define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) 25 #define scx_switched_all() static_branch_unlikely(&__scx_switched_all) 26 27 DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); 28 29 static inline bool task_on_scx(const struct task_struct *p) 30 { 31 return scx_enabled() && p->sched_class == &ext_sched_class; 32 } 33 34 void scx_next_task_picked(struct rq *rq, struct task_struct *p, 35 const struct sched_class *active); 36 void scx_tick(struct rq *rq); 37 void init_scx_entity(struct sched_ext_entity *scx); 38 void scx_pre_fork(struct task_struct *p); 39 int scx_fork(struct task_struct *p); 40 void scx_post_fork(struct task_struct *p); 41 void scx_cancel_fork(struct task_struct *p); 42 bool scx_can_stop_tick(struct rq *rq); 43 void scx_rq_activate(struct rq *rq); 44 void scx_rq_deactivate(struct rq *rq); 45 int scx_check_setscheduler(struct task_struct *p, int policy); 46 bool task_should_scx(struct task_struct *p); 47 void init_sched_ext_class(void); 48 49 static inline const struct sched_class *next_active_class(const struct sched_class *class) 50 { 51 class++; 52 if (scx_switched_all() && class == &fair_sched_class) 53 class++; 54 if (!scx_enabled() && class == &ext_sched_class) 55 class++; 56 return class; 57 } 58 59 #define for_active_class_range(class, _from, _to) \ 60 for (class = (_from); class != (_to); class = next_active_class(class)) 61 62 #define for_each_active_class(class) \ 63 for_active_class_range(class, __sched_class_highest, __sched_class_lowest) 64 65 /* 66 * SCX requires a balance() call before every pick_next_task() call including 67 * when waking up from idle. 68 */ 69 #define for_balance_class_range(class, prev_class, end_class) \ 70 for_active_class_range(class, (prev_class) > &ext_sched_class ? \ 71 &ext_sched_class : (prev_class), (end_class)) 72 73 #ifdef CONFIG_SCHED_CORE 74 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, 75 bool in_fi); 76 #endif 77 78 #else /* CONFIG_SCHED_CLASS_EXT */ 79 80 #define scx_enabled() false 81 #define scx_switched_all() false 82 83 static inline void scx_next_task_picked(struct rq *rq, struct task_struct *p, 84 const struct sched_class *active) {} 85 static inline void scx_tick(struct rq *rq) {} 86 static inline void scx_pre_fork(struct task_struct *p) {} 87 static inline int scx_fork(struct task_struct *p) { return 0; } 88 static inline void scx_post_fork(struct task_struct *p) {} 89 static inline void scx_cancel_fork(struct task_struct *p) {} 90 static inline bool scx_can_stop_tick(struct rq *rq) { return true; } 91 static inline void scx_rq_activate(struct rq *rq) {} 92 static inline void scx_rq_deactivate(struct rq *rq) {} 93 static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } 94 static inline bool task_on_scx(const struct task_struct *p) { return false; } 95 static inline void init_sched_ext_class(void) {} 96 97 #define for_each_active_class for_each_class 98 #define for_balance_class_range for_class_range 99 100 #endif /* CONFIG_SCHED_CLASS_EXT */ 101 102 #if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) 103 void __scx_update_idle(struct rq *rq, bool idle); 104 105 static inline void scx_update_idle(struct rq *rq, bool idle) 106 { 107 if (scx_enabled()) 108 __scx_update_idle(rq, idle); 109 } 110 #else 111 static inline void scx_update_idle(struct rq *rq, bool idle) {} 112 #endif 113