1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 7 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 8 */ 9 #ifdef CONFIG_SCHED_CLASS_EXT 10 11 struct sched_enq_and_set_ctx { 12 struct task_struct *p; 13 int queue_flags; 14 bool queued; 15 bool running; 16 }; 17 18 void sched_deq_and_put_task(struct task_struct *p, int queue_flags, 19 struct sched_enq_and_set_ctx *ctx); 20 void sched_enq_and_set_task(struct sched_enq_and_set_ctx *ctx); 21 22 extern const struct sched_class ext_sched_class; 23 24 DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); 25 DECLARE_STATIC_KEY_FALSE(__scx_switched_all); 26 #define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) 27 #define scx_switched_all() static_branch_unlikely(&__scx_switched_all) 28 29 DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); 30 31 static inline bool task_on_scx(const struct task_struct *p) 32 { 33 return scx_enabled() && p->sched_class == &ext_sched_class; 34 } 35 36 void scx_tick(struct rq *rq); 37 void init_scx_entity(struct sched_ext_entity *scx); 38 void scx_pre_fork(struct task_struct *p); 39 int scx_fork(struct task_struct *p); 40 void scx_post_fork(struct task_struct *p); 41 void scx_cancel_fork(struct task_struct *p); 42 bool scx_can_stop_tick(struct rq *rq); 43 void scx_rq_activate(struct rq *rq); 44 void scx_rq_deactivate(struct rq *rq); 45 int scx_check_setscheduler(struct task_struct *p, int policy); 46 bool task_should_scx(struct task_struct *p); 47 void init_sched_ext_class(void); 48 49 static inline u32 scx_cpuperf_target(s32 cpu) 50 { 51 if (scx_enabled()) 52 return cpu_rq(cpu)->scx.cpuperf_target; 53 else 54 return 0; 55 } 56 57 static inline const struct sched_class *next_active_class(const struct sched_class *class) 58 { 59 class++; 60 if (scx_switched_all() && class == &fair_sched_class) 61 class++; 62 if (!scx_enabled() && class == &ext_sched_class) 63 class++; 64 return class; 65 } 66 67 #define for_active_class_range(class, _from, _to) \ 68 for (class = (_from); class != (_to); class = next_active_class(class)) 69 70 #define for_each_active_class(class) \ 71 for_active_class_range(class, __sched_class_highest, __sched_class_lowest) 72 73 /* 74 * SCX requires a balance() call before every pick_next_task() call including 75 * when waking up from idle. 76 */ 77 #define for_balance_class_range(class, prev_class, end_class) \ 78 for_active_class_range(class, (prev_class) > &ext_sched_class ? \ 79 &ext_sched_class : (prev_class), (end_class)) 80 81 #ifdef CONFIG_SCHED_CORE 82 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, 83 bool in_fi); 84 #endif 85 86 #else /* CONFIG_SCHED_CLASS_EXT */ 87 88 #define scx_enabled() false 89 #define scx_switched_all() false 90 91 static inline void scx_tick(struct rq *rq) {} 92 static inline void scx_pre_fork(struct task_struct *p) {} 93 static inline int scx_fork(struct task_struct *p) { return 0; } 94 static inline void scx_post_fork(struct task_struct *p) {} 95 static inline void scx_cancel_fork(struct task_struct *p) {} 96 static inline u32 scx_cpuperf_target(s32 cpu) { return 0; } 97 static inline bool scx_can_stop_tick(struct rq *rq) { return true; } 98 static inline void scx_rq_activate(struct rq *rq) {} 99 static inline void scx_rq_deactivate(struct rq *rq) {} 100 static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } 101 static inline bool task_on_scx(const struct task_struct *p) { return false; } 102 static inline void init_sched_ext_class(void) {} 103 104 #define for_each_active_class for_each_class 105 #define for_balance_class_range for_class_range 106 107 #endif /* CONFIG_SCHED_CLASS_EXT */ 108 109 #if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP) 110 void __scx_update_idle(struct rq *rq, bool idle); 111 112 static inline void scx_update_idle(struct rq *rq, bool idle) 113 { 114 if (scx_enabled()) 115 __scx_update_idle(rq, idle); 116 } 117 #else 118 static inline void scx_update_idle(struct rq *rq, bool idle) {} 119 #endif 120