1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Inline definitions layered on top of internal.h and cid.h. 6 * 7 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 8 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 9 */ 10 #ifndef _KERNEL_SCHED_EXT_INLINES_H 11 #define _KERNEL_SCHED_EXT_INLINES_H 12 13 #include "internal.h" 14 #include "cid.h" 15 16 /* 17 * One user of this function is scx_bpf_dispatch() which can be called 18 * recursively as sub-sched dispatches nest. Always inline to reduce stack usage 19 * from the call frame. 20 */ 21 static __always_inline bool 22 scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, 23 struct task_struct *prev, bool nested) 24 { 25 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 26 int nr_loops = SCX_DSP_MAX_LOOPS; 27 s32 cpu = cpu_of(rq); 28 bool prev_on_sch = (prev->sched_class == &ext_sched_class) && 29 scx_task_on_sched(sch, prev); 30 31 if (scx_consume_global_dsq(sch, rq)) 32 return true; 33 34 if (scx_bypass_dsp_enabled(sch)) { 35 /* if @sch is bypassing, only the bypass DSQs are active */ 36 if (scx_bypassing(sch, cpu)) 37 return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); 38 39 #ifdef CONFIG_EXT_SUB_SCHED 40 /* 41 * If @sch isn't bypassing but its children are, @sch is 42 * responsible for making forward progress for both its own 43 * tasks that aren't bypassing and the bypassing descendants' 44 * tasks. The following implements a simple built-in behavior - 45 * let each CPU try to run the bypass DSQ every Nth time. 46 * 47 * Later, if necessary, we can add an ops flag to suppress the 48 * auto-consumption and a kfunc to consume the bypass DSQ and, 49 * so that the BPF scheduler can fully control scheduling of 50 * bypassed tasks. 51 */ 52 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 53 54 if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && 55 scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { 56 __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); 57 return true; 58 } 59 #endif /* CONFIG_EXT_SUB_SCHED */ 60 } 61 62 if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) 63 return false; 64 65 dspc->rq = rq; 66 67 /* 68 * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq 69 * lock, the local DSQ might still end up empty after a successful 70 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() 71 * produced some tasks, retry. The BPF scheduler may depend on this 72 * looping behavior to simplify its implementation. 73 */ 74 do { 75 dspc->nr_tasks = 0; 76 77 #ifdef CONFIG_EXT_SUB_SCHED 78 /* stash @prev so that nested invocations can access it */ 79 if (!nested) 80 rq->scx.sub_dispatch_prev = prev; 81 #endif 82 83 SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), 84 prev_on_sch ? prev : NULL); 85 86 #ifdef CONFIG_EXT_SUB_SCHED 87 if (!nested) 88 rq->scx.sub_dispatch_prev = NULL; 89 #endif 90 91 scx_flush_dispatch_buf(sch, rq); 92 93 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { 94 rq->scx.flags |= SCX_RQ_BAL_KEEP; 95 return true; 96 } 97 if (rq->scx.local_dsq.nr) 98 return true; 99 if (scx_consume_global_dsq(sch, rq)) 100 return true; 101 102 /* 103 * ops.dispatch() can trap us in this loop by repeatedly 104 * dispatching ineligible tasks. Break out once in a while to 105 * allow the watchdog to run. As IRQ can't be enabled in 106 * balance(), we want to complete this scheduling cycle and then 107 * start a new one. IOW, we want to call resched_curr() on the 108 * next, most likely idle, task, not the current one. Use 109 * __scx_bpf_kick_cpu() for deferred kicking. 110 */ 111 if (unlikely(!--nr_loops)) { 112 scx_kick_cpu(sch, cpu, 0); 113 break; 114 } 115 } while (dspc->nr_tasks); 116 117 /* 118 * Prevent the CPU from going idle while bypassed descendants have tasks 119 * queued. Without this fallback, bypassed tasks could stall if the host 120 * scheduler's ops.dispatch() doesn't yield any tasks. 121 */ 122 if (scx_bypass_dsp_enabled(sch)) 123 return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); 124 125 return false; 126 } 127 128 #endif /* _KERNEL_SCHED_EXT_INLINES_H */ 129