1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Sub-scheduler hierarchy support. 6 * 7 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 8 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 9 */ 10 #ifndef _KERNEL_SCHED_EXT_SUB_H 11 #define _KERNEL_SCHED_EXT_SUB_H 12 13 #include "internal.h" 14 15 #ifdef CONFIG_EXT_SUB_SCHED 16 17 struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); 18 struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); 19 void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); 20 void scx_ops_cid_sched_updated_enable(struct scx_sched *sch); 21 void scx_ops_cid_sched_updated_disable(struct scx_sched *sch); 22 void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); 23 struct cgroup *sch_cgroup(struct scx_sched *sch); 24 void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); 25 void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog); 26 void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog); 27 void drain_descendants(struct scx_sched *sch); 28 void scx_sub_disable(struct scx_sched *sch); 29 void scx_sub_enable_workfn(struct kthread_work *work); 30 bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); 31 void scx_free_pshards(struct scx_sched *sch); 32 s32 scx_alloc_pshards(struct scx_sched *sch); 33 void scx_init_root_caps(struct scx_sched *sch); 34 void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); 35 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); 36 void scx_online_ecaps(struct rq *rq); 37 void scx_offline_ecaps(struct rq *rq); 38 void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); 39 void scx_discard_stale_ecaps_syncs(void); 40 struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, 41 struct task_struct *p, u64 *enq_flags); 42 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); 43 void scx_rescue_charge(struct rq *rq, s64 delta_exec); 44 void scx_rescue_end(struct rq *rq); 45 bool scx_rescue_keep(struct rq *rq, struct task_struct *p); 46 void scx_rescue_flush(struct rq *rq); 47 void scx_rescue_dump(struct seq_buf *s, struct rq *rq); 48 void scx_rescue_set_knobs(struct scx_sched *sch); 49 void scx_rescue_init(struct rq *rq); 50 51 /* 52 * cgrp->scx_sched is written by root/sub enable/disable under all of 53 * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the 54 * parent's sched under just cgroup_mutex but is not yet reachable by the other 55 * two lock holders. Any one of the three locks stabilizes the association. 56 */ 57 static inline struct scx_sched *scx_cgroup_sched(struct cgroup *cgrp) 58 { 59 return rcu_dereference_check(cgrp->scx_sched, 60 lockdep_is_held(&cgroup_mutex) || 61 percpu_rwsem_is_held(&scx_fork_rwsem) || 62 lockdep_is_held(&scx_enable_mutex)); 63 } 64 65 static inline const char *sch_cgrp_path(struct scx_sched *sch) 66 { 67 return sch->cgrp_path; 68 } 69 70 /* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */ 71 static inline void scx_dec_has_subs(struct scx_sched *sch) 72 { 73 if (sch->level) 74 static_branch_dec(&__scx_has_subs); 75 } 76 77 /* hot-path hooks, gated inline so that a root-only system skips the calls */ 78 static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) 79 { 80 lockdep_assert_rq_held(rq); 81 82 if (scx_has_subs()) 83 __scx_process_sync_ecaps(rq, prev); 84 } 85 86 static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, 87 struct rq *rq, 88 struct task_struct *p, 89 u64 *enq_flags) 90 { 91 if (!scx_has_subs()) 92 return &rq->scx.local_dsq; 93 return __scx_resolve_local_dsq(sch, rq, p, enq_flags); 94 } 95 96 #else /* CONFIG_EXT_SUB_SCHED */ 97 98 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } 99 static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } 100 static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {} 101 static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {} 102 static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {} 103 static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} 104 static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } 105 static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } 106 static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} 107 static inline void drain_descendants(struct scx_sched *sch) { } 108 static inline void scx_sub_disable(struct scx_sched *sch) { } 109 static inline void scx_free_pshards(struct scx_sched *sch) {} 110 static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } 111 static inline void scx_init_root_caps(struct scx_sched *sch) {} 112 static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} 113 static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} 114 static inline void scx_online_ecaps(struct rq *rq) {} 115 static inline void scx_offline_ecaps(struct rq *rq) {} 116 static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} 117 static inline void scx_discard_stale_ecaps_syncs(void) {} 118 static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } 119 static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } 120 static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} 121 static inline void scx_rescue_end(struct rq *rq) {} 122 static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } 123 static inline void scx_rescue_flush(struct rq *rq) {} 124 static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} 125 static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} 126 static inline void scx_rescue_init(struct rq *rq) {} 127 static inline void scx_dec_has_subs(struct scx_sched *sch) {} 128 129 #endif /* CONFIG_EXT_SUB_SCHED */ 130 131 /** 132 * scx_for_each_descendant_pre - pre-order walk of a sched's descendants 133 * @pos: iteration cursor 134 * @root: sched to walk the descendants of 135 * 136 * Walk @root's descendants. @root is included in the iteration and the first 137 * node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or 138 * RCU read lock. 139 */ 140 #define scx_for_each_descendant_pre(pos, root) \ 141 for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ 142 (pos) = scx_next_descendant_pre((pos), (root))) 143 144 #ifdef CONFIG_EXT_SUB_SCHED 145 146 /** 147 * scx_missing_caps - The caps in @needed that @sch lacks on @cpu 148 * @sch: sched to test 149 * @cpu: cpu to test on 150 * @needed: bitmask of SCX_CAP_* values 151 * 152 * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all. 153 */ 154 static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) 155 { 156 u64 ecaps; 157 158 /* no sub-scheds, no missing caps */ 159 if (!scx_has_subs()) 160 return 0; 161 162 /* root holds every cap on every cpu */ 163 if (!sch->level) 164 return 0; 165 166 ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps); 167 168 return needed & ~ecaps; 169 } 170 171 /* 172 * Cap semantics: which caps an action requires, and which caps a cap implies. 173 * Keep all such mappings collected here. 174 */ 175 176 /* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */ 177 static inline u64 scx_caps_for_enq(u64 enq_flags) 178 { 179 /* a restored task must be put into the local DSQ regardless of caps */ 180 if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) 181 return 0; 182 if (enq_flags & SCX_ENQ_IMMED) 183 return SCX_CAP_ENQ_IMMED; 184 return SCX_CAP_ENQ; 185 } 186 187 /* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ 188 static inline u64 scx_caps_for_task(struct task_struct *p) 189 { 190 if (p->scx.flags & SCX_TASK_IMMED) 191 return SCX_CAP_ENQ_IMMED; 192 return SCX_CAP_ENQ; 193 } 194 195 /* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */ 196 static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, 197 u64 enq_flags) 198 { 199 struct task_struct *donor = rq->donor; 200 201 /* a kernel-forced placement preempts regardless of caps */ 202 if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) 203 return 0; 204 /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ 205 if (donor->sched_class != &ext_sched_class || 206 scx_is_descendant(scx_task_sched(donor), sch)) 207 return 0; 208 return SCX_CAP_PREEMPT; 209 } 210 211 /* caps implied by holding @cap */ 212 static inline u64 scx_caps_implied(u64 cap) 213 { 214 switch (cap) { 215 case SCX_CAP_PREEMPT: 216 return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED; 217 case SCX_CAP_ENQ: 218 return SCX_CAP_ENQ_IMMED; 219 } 220 return 0; 221 } 222 223 /* may @p keep running on @rq's cpu? requires baseline cpu access */ 224 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) 225 { 226 s32 cpu = cpu_of(rq); 227 228 if (!scx_has_subs()) 229 return true; 230 231 /* a migration-disabled task is let in without caps, keep it likewise */ 232 if (unlikely(is_migration_disabled(p))) 233 return true; 234 235 /* check caps against the sched running a bypassed task */ 236 return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, 237 SCX_CAP_BASE)); 238 } 239 240 /* the task admitted for rescue on @rq, NULL if none */ 241 static inline struct task_struct *scx_rescuee(struct rq *rq) 242 { 243 lockdep_assert_rq_held(rq); 244 245 if (!scx_has_subs()) 246 return NULL; 247 248 return rq->scx.rescue.curr; 249 } 250 251 #else /* CONFIG_EXT_SUB_SCHED */ 252 253 static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } 254 static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } 255 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } 256 static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } 257 258 #endif /* CONFIG_EXT_SUB_SCHED */ 259 260 #endif /* _KERNEL_SCHED_EXT_SUB_H */ 261