Lines Matching full:sch
144 * interval is half of the shortest sch->watchdog_timeout.
262 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
264 __printf(5, 6) bool __scx_exit(struct scx_sched *sch, in __scx_exit() argument
272 ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); in __scx_exit()
293 * @sch: sched to test
296 * Test whether @sch is a descendant of @ancestor.
298 bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) in scx_is_descendant() argument
300 if (sch->level < ancestor->level) in scx_is_descendant()
302 return sch->ancestors[ancestor->level] == ancestor; in scx_is_descendant()
305 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) in find_global_dsq() argument
307 return &sch->pnode[cpu_to_node(cpu)]->global_dsq; in find_global_dsq()
310 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) in find_user_dsq() argument
312 return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); in find_user_dsq()
323 static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) in bypass_enq_target_dsq() argument
327 * If @sch is a sub-sched which is bypassing, its tasks should go into in bypass_enq_target_dsq()
337 while (scx_parent(sch) && scx_bypassing(sch, cpu)) in bypass_enq_target_dsq()
338 sch = scx_parent(sch); in bypass_enq_target_dsq()
341 return scx_bypass_dsq(sch, cpu); in bypass_enq_target_dsq()
444 * Flipped on enable per sch->is_cid_type. Declared in internal.h so
451 * @sch: scx_sched being invoked
460 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, in scx_call_op_set_cpumask() argument
465 struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); in scx_call_op_set_cpumask()
475 scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); in scx_call_op_set_cpumask()
477 SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); in scx_call_op_set_cpumask()
479 SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); in scx_call_op_set_cpumask()
932 static void scx_read_events(struct scx_sched *sch,
977 * @sch: scx_sched to abort on error
985 bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) in scx_cpu_valid() argument
990 scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); in scx_cpu_valid()
1082 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, in schedule_dsq_reenq() argument
1091 if (unlikely(READ_ONCE(sch->bypass_depth))) in schedule_dsq_reenq()
1104 if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) { in schedule_dsq_reenq()
1105 __scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1); in schedule_dsq_reenq()
1109 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); in schedule_dsq_reenq()
1149 scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); in schedule_dsq_reenq()
1171 * Only the low 20 bits of sch->id are packed, which is enough to make
1313 static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) in set_task_slice_oob() argument
1321 scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1); in set_task_slice_oob()
1327 ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur); in set_task_slice_oob()
1477 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) in refill_task_slice_dfl() argument
1483 set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); in refill_task_slice_dfl()
1484 __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); in refill_task_slice_dfl()
1506 static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, in call_task_dequeue() argument
1512 if (SCX_HAS_OP(sch, dequeue)) in call_task_dequeue()
1513 SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); in call_task_dequeue()
1518 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, in rq_owned_post_enq() argument
1522 call_task_dequeue(sch, rq, p, 0); in rq_owned_post_enq()
1585 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); in rq_owned_post_enq()
1589 static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, in scx_dispatch_enqueue() argument
1596 dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); in scx_dispatch_enqueue()
1609 scx_error(sch, "attempting to dispatch to a destroyed dsq"); in scx_dispatch_enqueue()
1612 dsq = find_global_dsq(sch, task_cpu(p)); in scx_dispatch_enqueue()
1626 scx_error(sch, "cannot use vtime ordering for built-in DSQs"); in scx_dispatch_enqueue()
1647 scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", in scx_dispatch_enqueue()
1672 scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", in scx_dispatch_enqueue()
1706 rq_owned_post_enq(sch, rq, dsq, p, enq_flags); in scx_dispatch_enqueue()
1717 call_task_dequeue(sch, rq, p, 0); in scx_dispatch_enqueue()
1820 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, in find_dsq_for_dispatch() argument
1830 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); in find_dsq_for_dispatch()
1832 if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) in find_dsq_for_dispatch()
1833 return find_global_dsq(sch, tcpu); in find_dsq_for_dispatch()
1839 dsq = find_global_dsq(sch, tcpu); in find_dsq_for_dispatch()
1841 dsq = find_user_dsq(sch, dsq_id); in find_dsq_for_dispatch()
1848 scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); in find_dsq_for_dispatch()
1849 return find_global_dsq(sch, tcpu); in find_dsq_for_dispatch()
1855 static void mark_direct_dispatch(struct scx_sched *sch, in mark_direct_dispatch() argument
1870 scx_error(sch, "%s[%d] already direct-dispatched", in mark_direct_dispatch()
1873 scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", in mark_direct_dispatch()
1909 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, in direct_dispatch() argument
1914 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); in direct_dispatch()
1959 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, in direct_dispatch()
1978 struct scx_sched *sch = scx_task_sched(p); in scx_do_enqueue_task() local
2005 __scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1); in scx_do_enqueue_task()
2008 __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), in scx_do_enqueue_task()
2023 if (scx_bypassing(sch, cpu_of(rq))) { in scx_do_enqueue_task()
2024 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); in scx_do_enqueue_task()
2032 if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && in scx_do_enqueue_task()
2034 __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); in scx_do_enqueue_task()
2040 if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && in scx_do_enqueue_task()
2042 __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); in scx_do_enqueue_task()
2046 if (unlikely(!SCX_HAS_OP(sch, enqueue))) in scx_do_enqueue_task()
2059 SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); in scx_do_enqueue_task()
2079 direct_dispatch(sch, p, enq_flags); in scx_do_enqueue_task()
2082 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); in scx_do_enqueue_task()
2088 dsq = find_global_dsq(sch, task_cpu(p)); in scx_do_enqueue_task()
2091 dsq = bypass_enq_target_dsq(sch, task_cpu(p)); in scx_do_enqueue_task()
2095 refill_task_slice_dfl(sch, p); in scx_do_enqueue_task()
2097 scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); in scx_do_enqueue_task()
2140 struct scx_sched *sch = scx_task_sched(p); in enqueue_task_scx() local
2169 if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) in enqueue_task_scx()
2170 SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); in enqueue_task_scx()
2185 __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); in enqueue_task_scx()
2190 struct scx_sched *sch = scx_task_sched(p); in ops_dequeue() local
2261 call_task_dequeue(sch, rq, p, deq_flags); in ops_dequeue()
2266 struct scx_sched *sch = scx_task_sched(p); in dequeue_task_scx() local
2296 (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { in dequeue_task_scx()
2298 if (SCX_HAS_OP(sch, stopping)) in dequeue_task_scx()
2299 SCX_CALL_OP_TASK(sch, stopping, rq, p, false); in dequeue_task_scx()
2302 if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) in dequeue_task_scx()
2303 SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); in dequeue_task_scx()
2327 struct scx_sched *sch = scx_task_sched(p); in yield_task_scx() local
2332 if (SCX_HAS_OP(sch, yield)) in yield_task_scx()
2333 SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); in yield_task_scx()
2341 struct scx_sched *sch = scx_task_sched(from); in yield_to_task_scx() local
2346 if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) in yield_to_task_scx()
2347 return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); in yield_to_task_scx()
2376 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, in scx_move_local_task_to_local_dsq() argument
2380 struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); in scx_move_local_task_to_local_dsq()
2397 rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags); in scx_move_local_task_to_local_dsq()
2402 * @sch: scheduler placing @p
2410 static void move_remote_task_to_local_dsq(struct scx_sched *sch, in move_remote_task_to_local_dsq() argument
2428 * @p's owner, dropping our scx flags and the placing @sch. We own @rq, in move_remote_task_to_local_dsq()
2436 dst_rq->scx.remote_activate_sch = sch; in move_remote_task_to_local_dsq()
2462 static bool task_can_run_on_remote_rq(struct scx_sched *sch, in task_can_run_on_remote_rq() argument
2492 scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", in task_can_run_on_remote_rq()
2505 scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", in task_can_run_on_remote_rq()
2512 __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); in task_can_run_on_remote_rq()
2571 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, in consume_remote_task() argument
2576 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); in consume_remote_task()
2586 * @sch: scx_sched being operated on
2600 static struct rq *move_task_between_dsqs(struct scx_sched *sch, in move_task_between_dsqs() argument
2614 unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { in move_task_between_dsqs()
2615 dst_dsq = find_global_dsq(sch, task_cpu(p)); in move_task_between_dsqs()
2632 scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); in move_task_between_dsqs()
2636 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); in move_task_between_dsqs()
2646 scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags); in move_task_between_dsqs()
2652 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, in scx_consume_dispatch_q() argument
2678 if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) in scx_consume_dispatch_q()
2683 scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); in scx_consume_dispatch_q()
2688 if (task_can_run_on_remote_rq(sch, p, rq, false)) { in scx_consume_dispatch_q()
2689 if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) in scx_consume_dispatch_q()
2699 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) in scx_consume_global_dsq() argument
2703 return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); in scx_consume_global_dsq()
2708 * @sch: scx_sched being operated on
2723 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, in dispatch_to_local_dsq() argument
2738 scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime, in dispatch_to_local_dsq()
2776 scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, in dispatch_to_local_dsq()
2778 } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { in dispatch_to_local_dsq()
2781 scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), in dispatch_to_local_dsq()
2787 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); in dispatch_to_local_dsq()
2804 * @sch: the scheduler
2822 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, in finish_dispatch() argument
2851 if (unlikely(!scx_task_on_sched(sch, p))) { in finish_dispatch()
2852 __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); in finish_dispatch()
2880 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); in finish_dispatch()
2883 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); in finish_dispatch()
2885 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, in finish_dispatch()
2889 void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) in scx_flush_dispatch_buf() argument
2891 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_flush_dispatch_buf()
2897 finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, in scx_flush_dispatch_buf()
2920 struct scx_sched *sch = scx_root_protected_live(); in dispatch_one() local
2929 if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && in dispatch_one()
2937 if (sch->ops.cpu_acquire) in dispatch_one()
2938 SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); in dispatch_one()
2956 !scx_bypassing(sch, cpu)) { in dispatch_one()
2968 verdict = scx_dispatch_sched(sch, rq, prev, false); in dispatch_one()
2977 (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && in dispatch_one()
2979 __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); in dispatch_one()
3006 struct scx_sched *sch = scx_task_sched(p); in set_next_task_scx() local
3020 if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) in set_next_task_scx()
3021 SCX_CALL_OP_TASK(sch, running, rq, p); in set_next_task_scx()
3079 struct scx_sched *sch = scx_root_protected_live(); in switch_class() local
3082 if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) in switch_class()
3104 if (sch->ops.cpu_release) { in switch_class()
3110 SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); in switch_class()
3119 struct scx_sched *sch = scx_task_sched(p); in put_prev_task_scx() local
3139 if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) in put_prev_task_scx()
3140 SCX_CALL_OP_TASK(sch, stopping, rq, p, true); in put_prev_task_scx()
3156 !scx_bypassing(sch, cpu_of(rq))) { in put_prev_task_scx()
3177 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, in put_prev_task_scx()
3197 !(sch->ops.flags & SCX_OPS_ENQ_LAST)); in put_prev_task_scx()
3405 struct scx_sched *sch = scx_task_sched(p); in do_pick_task_scx() local
3407 if (!scx_bypassing(sch, cpu_of(rq)) && in do_pick_task_scx()
3408 !sch->warned_zero_slice) { in do_pick_task_scx()
3411 sch->warned_zero_slice = true; in do_pick_task_scx()
3413 refill_task_slice_dfl(sch, p); in do_pick_task_scx()
3481 struct scx_sched *sch = NULL; in scx_prio_less() local
3486 sch = sch_a; in scx_prio_less()
3495 sch = anc; in scx_prio_less()
3510 if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b))) in scx_prio_less()
3511 return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a), in scx_prio_less()
3532 struct scx_sched *sch = scx_task_sched(p); in select_task_rq_scx() local
3548 bypassing = scx_bypassing(sch, task_cpu(p)); in select_task_rq_scx()
3549 if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { in select_task_rq_scx()
3558 cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, in select_task_rq_scx()
3560 cpu = scx_cpu_ret(sch, cpu); in select_task_rq_scx()
3564 if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) in select_task_rq_scx()
3579 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); in select_task_rq_scx()
3590 __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); in select_task_rq_scx()
3591 p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); in select_task_rq_scx()
3611 struct scx_sched *sch = scx_task_sched(p); in set_cpus_allowed_scx() local
3626 if (SCX_HAS_OP(sch, set_cpumask)) in set_cpus_allowed_scx()
3627 scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); in set_cpus_allowed_scx()
3632 struct scx_sched *sch = scx_root_protected(); in handle_hotplug() local
3643 if (unlikely(!sch)) in handle_hotplug()
3647 scx_idle_update_selcpu_topology(&sch->ops); in handle_hotplug()
3670 if (online && SCX_HAS_OP(sch, cpu_online)) in handle_hotplug()
3671 SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); in handle_hotplug()
3672 else if (!online && SCX_HAS_OP(sch, cpu_offline)) in handle_hotplug()
3673 SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid); in handle_hotplug()
3675 scx_exit(sch, SCX_EXIT_UNREG_KERN, in handle_hotplug()
3704 struct scx_sched *sch; in check_rq_for_timeouts() local
3710 sch = rcu_dereference_bh(scx_root); in check_rq_for_timeouts()
3711 if (unlikely(!sch)) in check_rq_for_timeouts()
3715 struct scx_sched *sch = scx_task_sched(p); in check_rq_for_timeouts() local
3719 last_runnable + READ_ONCE(sch->watchdog_timeout)))) { in check_rq_for_timeouts()
3731 sch = dsq->sched; in check_rq_for_timeouts()
3733 __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), in check_rq_for_timeouts()
3792 struct scx_sched *sch = scx_task_sched(curr); in task_tick_scx() local
3800 if (scx_bypassing(sch, cpu_of(rq))) in task_tick_scx()
3802 else if (SCX_HAS_OP(sch, tick)) in task_tick_scx()
3803 SCX_CALL_OP_TASK(sch, tick, rq, curr); in task_tick_scx()
3833 * @sch: sched to initialize @p for
3843 int __scx_init_task(struct scx_sched *sch, struct task_struct *p, in __scx_init_task() argument
3850 if (SCX_HAS_OP(sch, init_task)) { in __scx_init_task()
3856 ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); in __scx_init_task()
3858 ret = scx_ops_sanitize_err(sch, "init_task", ret); in __scx_init_task()
3864 if (unlikely(scx_parent(sch))) { in __scx_init_task()
3865 scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", in __scx_init_task()
3868 scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", in __scx_init_task()
3871 scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path", in __scx_init_task()
3898 static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) in __scx_enable_task() argument
3923 if (SCX_HAS_OP(sch, enable)) in __scx_enable_task()
3924 SCX_CALL_OP_TASK(sch, enable, rq, p); in __scx_enable_task()
3926 if (SCX_HAS_OP(sch, set_weight)) in __scx_enable_task()
3927 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); in __scx_enable_task()
3930 void scx_enable_task(struct scx_sched *sch, struct task_struct *p) in scx_enable_task() argument
3932 __scx_enable_task(sch, p); in scx_enable_task()
3936 static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) in scx_disable_task() argument
3945 if (SCX_HAS_OP(sch, disable)) in scx_disable_task()
3946 SCX_CALL_OP_TASK(sch, disable, rq, p); in scx_disable_task()
3966 void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) in __scx_disable_and_exit_task() argument
3984 scx_disable_task(sch, p); in __scx_disable_and_exit_task()
3991 if (SCX_HAS_OP(sch, exit_task)) in __scx_disable_and_exit_task()
3992 SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); in __scx_disable_and_exit_task()
3996 * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
4000 void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) in scx_sub_init_cancel_task() argument
4007 /* @p was never associated with @sch, dispatch on the explicit @sch */ in scx_sub_init_cancel_task()
4008 if (SCX_HAS_OP(sch, exit_task)) in scx_sub_init_cancel_task()
4009 __SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args); in scx_sub_init_cancel_task()
4012 void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) in scx_disable_and_exit_task() argument
4014 __scx_disable_and_exit_task(sch, p); in scx_disable_and_exit_task()
4095 struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp); in scx_fork() local
4097 struct scx_sched *sch = scx_root_protected_live(); in scx_fork()
4100 ret = __scx_init_task(sch, p, NULL, true); in scx_fork()
4106 scx_set_task_sched(p, sch); in scx_fork()
4229 struct scx_sched *sch = scx_task_sched(p); in reweight_task_scx() local
4248 if (SCX_HAS_OP(sch, set_weight)) in reweight_task_scx()
4249 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); in reweight_task_scx()
4258 struct scx_sched *sch = scx_task_sched(p); in switching_to_scx() local
4263 scx_enable_task(sch, p); in switching_to_scx()
4269 if (SCX_HAS_OP(sch, set_cpumask)) in switching_to_scx()
4270 scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); in switching_to_scx()
4320 struct scx_sched *sch = scx_task_sched(p); in process_ddsp_deferred_locals() local
4330 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); in process_ddsp_deferred_locals()
4332 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); in process_ddsp_deferred_locals()
4388 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) in reenq_local() argument
4425 if (!scx_is_descendant(task_sch, sch)) in reenq_local()
4476 struct scx_sched *sch; in process_deferred_reenq_locals() local
4491 sch = sch_pcpu->sch; in process_deferred_reenq_locals()
4501 reenq_local(sch, rq, reenq_flags); in process_deferred_reenq_locals()
4514 struct scx_sched *sch = dsq->sched; in reenq_user() local
4523 while (likely(!READ_ONCE(sch->bypass_depth))) { in reenq_user()
4643 struct scx_sched *sch = scx_task_sched(p); in scx_can_stop_tick() local
4657 if (scx_bypassing(sch, cpu_of(rq))) in scx_can_stop_tick()
4749 struct scx_sched *sch; in scx_tg_online() local
4758 sch = scx_cgroup_sched(tg->css.cgroup); in scx_tg_online()
4760 sch = scx_tg_sched(&root_task_group); in scx_tg_online()
4762 if (SCX_HAS_OP(sch, cgroup_init)) { in scx_tg_online()
4769 ret = SCX_CALL_OP_RET(sch, cgroup_init, in scx_tg_online()
4772 ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); in scx_tg_online()
4775 tg->scx.sched = sch; in scx_tg_online()
4787 struct scx_sched *sch = tg->scx.sched; in scx_tg_offline() local
4791 /* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */ in scx_tg_offline()
4793 SCX_HAS_OP(sch, cgroup_exit)) in scx_tg_offline()
4794 SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); in scx_tg_offline()
4819 struct scx_sched *sch = scx_cgroup_task_sched(p); in scx_cgroup_can_attach() local
4839 if (!sch || sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp)) in scx_cgroup_can_attach()
4842 if (SCX_HAS_OP(sch, cgroup_prep_move)) { in scx_cgroup_can_attach()
4843 ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, in scx_cgroup_can_attach()
4846 ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); in scx_cgroup_can_attach()
4858 struct scx_sched *sch = scx_cgroup_task_sched(p); in scx_cgroup_can_attach() local
4860 /* cgrp_moving_from implies non-NULL @sch, test it first */ in scx_cgroup_can_attach()
4861 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) in scx_cgroup_can_attach()
4862 SCX_CALL_OP(sch, cgroup_cancel_move, NULL, in scx_cgroup_can_attach()
4872 struct scx_sched *sch; in scx_cgroup_move_task() local
4883 sch = scx_cgroup_task_sched(p); in scx_cgroup_move_task()
4884 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) in scx_cgroup_move_task()
4885 SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), in scx_cgroup_move_task()
4900 struct scx_sched *sch = scx_cgroup_task_sched(p); in scx_cgroup_cancel_attach() local
4902 /* cgrp_moving_from implies non-NULL @sch, test it first */ in scx_cgroup_cancel_attach()
4903 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) in scx_cgroup_cancel_attach()
4904 SCX_CALL_OP(sch, cgroup_cancel_move, NULL, in scx_cgroup_cancel_attach()
4912 struct scx_sched *sch; in scx_group_set_weight() local
4915 sch = scx_tg_knob_sched(tg); in scx_group_set_weight()
4917 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) && in scx_group_set_weight()
4919 SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); in scx_group_set_weight()
4928 struct scx_sched *sch; in scx_group_set_idle() local
4931 sch = scx_tg_knob_sched(tg); in scx_group_set_idle()
4933 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) in scx_group_set_idle()
4934 SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); in scx_group_set_idle()
4945 struct scx_sched *sch; in scx_group_set_bandwidth() local
4948 sch = scx_tg_knob_sched(tg); in scx_group_set_bandwidth()
4950 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) && in scx_group_set_bandwidth()
4954 SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, in scx_group_set_bandwidth()
5040 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) in scx_init_dsq() argument
5049 dsq->sched = sch; in scx_init_dsq()
5106 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) in destroy_dsq() argument
5113 dsq = find_user_dsq(sch, dsq_id); in destroy_dsq()
5120 scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", in destroy_dsq()
5125 if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, in destroy_dsq()
5146 static void scx_cgroup_exit(struct scx_sched *sch) in scx_cgroup_exit() argument
5161 if (sch->ops.cgroup_exit) in scx_cgroup_exit()
5162 SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); in scx_cgroup_exit()
5167 static int scx_cgroup_init(struct scx_sched *sch) in scx_cgroup_init() argument
5182 if (sch->ops.cgroup_init) { in scx_cgroup_init()
5190 ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); in scx_cgroup_init()
5192 scx_error(sch, "ops.cgroup_init() failed (%d)", ret); in scx_cgroup_init()
5197 tg->scx.sched = sch; in scx_cgroup_init()
5205 static void scx_cgroup_exit(struct scx_sched *sch) {} in scx_cgroup_exit() argument
5206 static int scx_cgroup_init(struct scx_sched *sch) { return 0; } in scx_cgroup_init() argument
5271 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
5273 s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) in scx_set_cmask_scratch_alloc() argument
5279 if (!sch->is_cid_type || !sch->arena_pool) in scx_set_cmask_scratch_alloc()
5282 sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); in scx_set_cmask_scratch_alloc()
5283 if (!sch->set_cmask_scratch) in scx_set_cmask_scratch_alloc()
5287 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); in scx_set_cmask_scratch_alloc()
5289 *slot = scx_arena_alloc(sch, size); in scx_set_cmask_scratch_alloc()
5297 static void scx_set_cmask_scratch_free(struct scx_sched *sch) in scx_set_cmask_scratch_free() argument
5303 if (!sch->set_cmask_scratch) in scx_set_cmask_scratch_free()
5307 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); in scx_set_cmask_scratch_free()
5309 scx_arena_free(sch, *slot, size); in scx_set_cmask_scratch_free()
5311 free_percpu(sch->set_cmask_scratch); in scx_set_cmask_scratch_free()
5312 sch->set_cmask_scratch = NULL; in scx_set_cmask_scratch_free()
5318 struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); in scx_sched_free_rcu_work() local
5323 irq_work_sync(&sch->propagate_exit_irq_work); in scx_sched_free_rcu_work()
5324 irq_work_sync(&sch->disable_irq_work); in scx_sched_free_rcu_work()
5325 kthread_destroy_worker(sch->helper); in scx_sched_free_rcu_work()
5326 timer_shutdown_sync(&sch->bypass_lb_timer); in scx_sched_free_rcu_work()
5327 free_cpumask_var(sch->bypass_lb_donee_cpumask); in scx_sched_free_rcu_work()
5328 free_cpumask_var(sch->bypass_lb_resched_cpumask); in scx_sched_free_rcu_work()
5329 free_cpumask_var(sch->stall_cpus); in scx_sched_free_rcu_work()
5332 kfree(sch->cgrp_path); in scx_sched_free_rcu_work()
5333 if (sch_cgroup(sch)) in scx_sched_free_rcu_work()
5334 cgroup_put(sch_cgroup(sch)); in scx_sched_free_rcu_work()
5335 if (sch->sub_kset) in scx_sched_free_rcu_work()
5336 kobject_put(&sch->sub_kset->kobj); in scx_sched_free_rcu_work()
5337 if (scx_parent(sch)) in scx_sched_free_rcu_work()
5338 kobject_put(&scx_parent(sch)->kobj); in scx_sched_free_rcu_work()
5342 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_sched_free_rcu_work()
5345 * $sch would have entered bypass mode before the RCU grace in scx_sched_free_rcu_work()
5365 exit_dsq(scx_bypass_dsq(sch, cpu)); in scx_sched_free_rcu_work()
5368 free_percpu(sch->pcpu); in scx_sched_free_rcu_work()
5371 free_pnode(sch->pnode[node]); in scx_sched_free_rcu_work()
5372 kfree(sch->pnode); in scx_sched_free_rcu_work()
5374 scx_free_pshards(sch); in scx_sched_free_rcu_work()
5376 rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); in scx_sched_free_rcu_work()
5381 destroy_dsq(sch, dsq->id); in scx_sched_free_rcu_work()
5387 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); in scx_sched_free_rcu_work()
5388 free_exit_info(sch->exit_info); in scx_sched_free_rcu_work()
5389 scx_set_cmask_scratch_free(sch); in scx_sched_free_rcu_work()
5390 scx_arena_pool_destroy(sch); in scx_sched_free_rcu_work()
5391 if (sch->arena_map) in scx_sched_free_rcu_work()
5392 bpf_map_put(sch->arena_map); in scx_sched_free_rcu_work()
5394 /* @sch is completely inactive by now */ in scx_sched_free_rcu_work()
5395 scx_dec_has_subs(sch); in scx_sched_free_rcu_work()
5397 kfree(sch); in scx_sched_free_rcu_work()
5402 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); in scx_kobj_release() local
5404 INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); in scx_kobj_release()
5405 queue_rcu_work(system_dfl_wq, &sch->rcu_work); in scx_kobj_release()
5411 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); in scx_attr_ops_show() local
5413 return sysfs_emit(buf, "%s\n", sch->ops.name); in scx_attr_ops_show()
5424 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); in scx_attr_events_show() local
5428 scx_read_events(sch, &events); in scx_attr_events_show()
5447 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); in scx_attr_caps_show() local
5462 for (si = 0; si < sch->nr_pshards; si++) { in scx_attr_caps_show()
5463 struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask; in scx_attr_caps_show()
5497 const struct scx_sched *sch; in scx_uevent() local
5507 sch = container_of(kobj, struct scx_sched, kobj); in scx_uevent()
5509 return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); in scx_uevent()
5551 struct scx_sched *sch; in scx_allow_ttwu_queue() local
5556 sch = scx_task_sched(p); in scx_allow_ttwu_queue()
5557 if (unlikely(!sch)) in scx_allow_ttwu_queue()
5560 if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) in scx_allow_ttwu_queue()
5583 struct scx_sched *sch; in handle_lockup() local
5589 sch = rcu_dereference(scx_root); in handle_lockup()
5590 if (unlikely(!sch)) in handle_lockup()
5597 ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); in handle_lockup()
5620 struct scx_sched *sch; in scx_rcu_cpu_stall() local
5626 sch = rcu_dereference(scx_root); in scx_rcu_cpu_stall()
5627 if (unlikely(!sch)) in scx_rcu_cpu_stall()
5639 ei = sch->exit_info; in scx_rcu_cpu_stall()
5643 if (!scx_claim_exit(sch, SCX_EXIT_ERROR)) in scx_rcu_cpu_stall()
5654 cpumask_copy(sch->stall_cpus, stalled_mask); in scx_rcu_cpu_stall()
5656 irq_work_queue(&sch->disable_irq_work); in scx_rcu_cpu_stall()
5708 static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, in bypass_lb_cpu() argument
5713 struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor); in bypass_lb_cpu()
5761 donee_dsq = scx_bypass_dsq(sch, donee); in bypass_lb_cpu()
5767 if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) in bypass_lb_cpu()
5782 scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); in bypass_lb_cpu()
5814 static void bypass_lb_node(struct scx_sched *sch, int node) in bypass_lb_node() argument
5817 struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; in bypass_lb_node()
5818 struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; in bypass_lb_node()
5827 u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); in bypass_lb_node()
5849 if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) in bypass_lb_node()
5860 if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) in bypass_lb_node()
5863 nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, in bypass_lb_node()
5871 u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); in bypass_lb_node()
5893 struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); in scx_bypass_lb_timerfn() local
5897 if (!scx_bypass_dsp_enabled(sch)) in scx_bypass_lb_timerfn()
5901 bypass_lb_node(sch, node); in scx_bypass_lb_timerfn()
5908 static bool inc_bypass_depth(struct scx_sched *sch) in inc_bypass_depth() argument
5912 WARN_ON_ONCE(sch->bypass_depth < 0); in inc_bypass_depth()
5913 WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); in inc_bypass_depth()
5914 if (sch->bypass_depth != 1) in inc_bypass_depth()
5917 WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); in inc_bypass_depth()
5918 sch->bypass_timestamp = ktime_get_ns(); in inc_bypass_depth()
5919 scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); in inc_bypass_depth()
5923 static bool dec_bypass_depth(struct scx_sched *sch) in dec_bypass_depth() argument
5927 WARN_ON_ONCE(sch->bypass_depth < 1); in dec_bypass_depth()
5928 WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); in dec_bypass_depth()
5929 if (sch->bypass_depth != 0) in dec_bypass_depth()
5932 WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); in dec_bypass_depth()
5933 scx_add_event(sch, SCX_EV_BYPASS_DURATION, in dec_bypass_depth()
5934 ktime_get_ns() - sch->bypass_timestamp); in dec_bypass_depth()
5938 static void enable_bypass_dsp(struct scx_sched *sch) in enable_bypass_dsp() argument
5940 struct scx_sched *host = scx_parent(sch) ?: sch; in enable_bypass_dsp()
5945 * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. in enable_bypass_dsp()
5948 if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) in enable_bypass_dsp()
5954 * called iff @sch is not already bypassed due to an ancestor bypassing, in enable_bypass_dsp()
5965 * scheds. Bump enable depth on both @sch and bypass dispatch host. in enable_bypass_dsp()
5967 ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); in enable_bypass_dsp()
5970 if (host != sch) { in enable_bypass_dsp()
5985 void scx_disable_bypass_dsp(struct scx_sched *sch) in scx_disable_bypass_dsp() argument
5989 if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) in scx_disable_bypass_dsp()
5992 ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); in scx_disable_bypass_dsp()
5995 if (scx_parent(sch)) { in scx_disable_bypass_dsp()
5996 ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); in scx_disable_bypass_dsp()
6035 * @sch: sched to bypass
6065 void scx_bypass(struct scx_sched *sch, bool bypass) in scx_bypass() argument
6074 if (!inc_bypass_depth(sch)) in scx_bypass()
6077 enable_bypass_dsp(sch); in scx_bypass()
6079 if (!dec_bypass_depth(sch)) in scx_bypass()
6088 scx_for_each_descendant_pre(pos, sch) { in scx_bypass()
6089 if (pos == sch) in scx_bypass()
6114 scx_for_each_descendant_pre(pos, sch) { in scx_bypass()
6151 if (!scx_is_descendant(scx_task_sched(p), sch)) in scx_bypass()
6177 scx_disable_bypass_dsp(sch); in scx_bypass()
6259 struct scx_sched *sch; in refresh_watchdog() local
6264 list_for_each_entry_rcu(sch, &scx_sched_all, all) in refresh_watchdog()
6265 intv = max(min(intv, sch->watchdog_timeout / 2), 1); in refresh_watchdog()
6277 s32 scx_link_sched(struct scx_sched *sch) in scx_link_sched() argument
6282 struct scx_sched *parent = scx_parent(sch); in scx_link_sched()
6293 scx_error(sch, "parent bypassing (%d)", -EBUSY); in scx_link_sched()
6298 &sch->hash_node, scx_sched_hash_params); in scx_link_sched()
6300 scx_error(sch, "failed to insert into scx_sched_hash (%d)", in scx_link_sched()
6305 list_add_tail_rcu(&sch->sibling, &parent->children); in scx_link_sched()
6314 rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, in scx_link_sched()
6316 list_del_rcu(&sch->sibling); in scx_link_sched()
6317 scx_error(sch, "parent disabled (%d)", -ENOENT); in scx_link_sched()
6321 sch->linked = true; in scx_link_sched()
6325 list_add_tail_rcu(&sch->all, &scx_sched_all); in scx_link_sched()
6332 void scx_unlink_sched(struct scx_sched *sch) in scx_unlink_sched() argument
6336 if (sch->linked) { in scx_unlink_sched()
6337 rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, in scx_unlink_sched()
6339 list_del_rcu(&sch->sibling); in scx_unlink_sched()
6340 sch->linked = false; in scx_unlink_sched()
6343 list_del_rcu(&sch->all); in scx_unlink_sched()
6351 * @sch. Once @sch becomes empty during disable, there's no point in dumping it.
6352 * This prevents calling dump ops on a dead sch.
6354 void scx_disable_dump(struct scx_sched *sch) in scx_disable_dump() argument
6357 sch->dump_disabled = true; in scx_disable_dump()
6360 void scx_log_sched_disable(struct scx_sched *sch) in scx_log_sched_disable() argument
6362 struct scx_exit_info *ei = sch->exit_info; in scx_log_sched_disable()
6363 const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; in scx_log_sched_disable()
6367 sch->ops.name, ei->reason); in scx_log_sched_disable()
6370 pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); in scx_log_sched_disable()
6376 sch->ops.name, ei->reason); in scx_log_sched_disable()
6380 static void scx_root_disable(struct scx_sched *sch) in scx_root_disable() argument
6388 scx_bypass(sch, true); in scx_root_disable()
6389 drain_descendants(sch); in scx_root_disable()
6397 sch->exit_info->msg); in scx_root_disable()
6422 scx_cgroup_exit(sch); in scx_root_disable()
6452 scx_disable_dump(sch); in scx_root_disable()
6455 set_cgroup_sched(sch_cgroup(sch), NULL); in scx_root_disable()
6491 if (sch->ops.flags & SCX_OPS_TID_TO_TASK) in scx_root_disable()
6493 bitmap_zero(sch->has_op, SCX_OPI_END); in scx_root_disable()
6496 if (sch->ops.flags & SCX_OPS_TID_TO_TASK) in scx_root_disable()
6499 scx_log_sched_disable(sch); in scx_root_disable()
6501 if (sch->ops.exit) in scx_root_disable()
6502 SCX_CALL_OP(sch, exit, NULL, sch->exit_info); in scx_root_disable()
6505 * @sch's non-ops programs such as timers and tracers can fire after in scx_root_disable()
6507 * resolving to @sch and drain in-flight resolvers. in scx_root_disable()
6509 WRITE_ONCE(sch->dead, true); in scx_root_disable()
6512 scx_unlink_sched(sch); in scx_root_disable()
6529 if (sch->sub_kset) in scx_root_disable()
6530 kobject_del(&sch->sub_kset->kobj); in scx_root_disable()
6533 if (sch->kobj.state_in_sysfs) in scx_root_disable()
6534 kobject_del(&sch->kobj); in scx_root_disable()
6542 scx_bypass(sch, false); in scx_root_disable()
6557 struct scx_sched *sch = container_of(irq_work, struct scx_sched, in scx_propagate_exit_irq_workfn() local
6562 scx_for_each_descendant_pre(pos, sch) in scx_propagate_exit_irq_workfn()
6568 * Claim the exit on @sch. The caller must ensure that the helper kthread work
6576 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) in scx_claim_exit() argument
6585 if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) in scx_claim_exit()
6590 WRITE_ONCE(sch->aborting, true); in scx_claim_exit()
6595 * CPUs may be live-locked in the dispatch paths of @sch or its in scx_claim_exit()
6603 scx_for_each_descendant_pre(pos, sch) in scx_claim_exit()
6607 irq_work_queue(&sch->propagate_exit_irq_work); in scx_claim_exit()
6611 trace_sched_ext_exit(sch, kind); in scx_claim_exit()
6618 struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); in scx_disable_workfn() local
6619 struct scx_exit_info *ei = sch->exit_info; in scx_disable_workfn()
6622 kind = atomic_read(&sch->exit_kind); in scx_disable_workfn()
6627 if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) in scx_disable_workfn()
6633 if (scx_parent(sch)) in scx_disable_workfn()
6634 scx_sub_disable(sch); in scx_disable_workfn()
6636 scx_root_disable(sch); in scx_disable_workfn()
6639 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) in scx_disable() argument
6642 if (scx_claim_exit(sch, kind)) in scx_disable()
6643 irq_work_queue(&sch->disable_irq_work); in scx_disable()
6648 * @sch: the scheduler
6650 * sch->disable_work might still not queued, causing kthread_flush_work()
6654 void scx_flush_disable_work(struct scx_sched *sch) in scx_flush_disable_work() argument
6659 irq_work_sync(&sch->disable_irq_work); in scx_flush_disable_work()
6660 kthread_flush_work(&sch->disable_work); in scx_flush_disable_work()
6661 kind = atomic_read(&sch->exit_kind); in scx_flush_disable_work()
6776 static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, in scx_dump_task() argument
6787 own_marker = task_sch == sch ? "*" : ""; in scx_dump_task()
6814 if (SCX_HAS_OP(sch, dump_task)) { in scx_dump_task()
6816 SCX_CALL_OP(sch, dump_task, rq, dctx, p); in scx_dump_task()
6829 static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, in scx_dump_cpu() argument
6834 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_dump_cpu()
6847 if (idle && !SCX_HAS_OP(sch, dump_cpu)) in scx_dump_cpu()
6883 if (SCX_HAS_OP(sch, dump_cpu)) { in scx_dump_cpu()
6885 SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); in scx_dump_cpu()
6907 (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) in scx_dump_cpu()
6908 scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); in scx_dump_cpu()
6911 if (dump_all_tasks || scx_task_on_sched(sch, p)) in scx_dump_cpu()
6912 scx_dump_task(sch, s, dctx, rq, p, ' '); in scx_dump_cpu()
6919 * of which scheduler they belong to. If false, only dump tasks owned by @sch.
6924 static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, in scx_dump_state() argument
6941 if (sch->dump_disabled) in scx_dump_state()
6947 if (sch->level == 0) in scx_dump_state()
6948 scx_dump_line(&s, "%s: root", sch->ops.name); in scx_dump_state()
6951 sch->ops.name, sch->level, sch->ops.sub_cgroup_id, in scx_dump_state()
6952 sch->cgrp_path); in scx_dump_state()
6970 if (SCX_HAS_OP(sch, dump)) { in scx_dump_state()
6972 SCX_CALL_OP(sch, dump, NULL, &dctx); in scx_dump_state()
6984 if (!cpumask_empty(sch->stall_cpus)) { in scx_dump_state()
6985 for_each_cpu(cpu, sch->stall_cpus) in scx_dump_state()
6986 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); in scx_dump_state()
6988 if (!cpumask_test_cpu(cpu, sch->stall_cpus)) in scx_dump_state()
6989 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); in scx_dump_state()
6993 scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); in scx_dump_state()
6996 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); in scx_dump_state()
7004 scx_read_events(sch, &events); in scx_dump_state()
7016 struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); in scx_disable_irq_workfn() local
7017 struct scx_exit_info *ei = sch->exit_info; in scx_disable_irq_workfn()
7020 scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); in scx_disable_irq_workfn()
7022 kthread_queue_work(sch->helper, &sch->disable_work); in scx_disable_irq_workfn()
7026 static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind, in scx_finish_exit() argument
7029 struct scx_exit_info *ei = sch->exit_info; in scx_finish_exit()
7048 irq_work_queue(&sch->disable_irq_work); in scx_finish_exit()
7051 bool scx_vexit(struct scx_sched *sch, in scx_vexit() argument
7055 struct scx_exit_info *ei = sch->exit_info; in scx_vexit()
7059 if (!scx_claim_exit(sch, kind)) in scx_vexit()
7064 scx_finish_exit(sch, kind, exit_code, exit_cpu); in scx_vexit()
7103 static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) in alloc_pnode() argument
7111 if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { in alloc_pnode()
7128 struct scx_sched *sch; in scx_alloc_and_add_sched() local
7132 sch = kzalloc_flex(*sch, ancestors, level + 1); in scx_alloc_and_add_sched()
7133 if (!sch) { in scx_alloc_and_add_sched()
7138 sch->exit_info = alloc_exit_info(ops->exit_dump_len); in scx_alloc_and_add_sched()
7139 if (!sch->exit_info) { in scx_alloc_and_add_sched()
7144 ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); in scx_alloc_and_add_sched()
7148 sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); in scx_alloc_and_add_sched()
7149 if (!sch->pnode) { in scx_alloc_and_add_sched()
7155 sch->pnode[node] = alloc_pnode(sch, node); in scx_alloc_and_add_sched()
7156 if (!sch->pnode[node]) { in scx_alloc_and_add_sched()
7162 sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; in scx_alloc_and_add_sched()
7163 sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, in scx_alloc_and_add_sched()
7164 dsp_ctx.buf, sch->dsp_max_batch), in scx_alloc_and_add_sched()
7166 if (!sch->pcpu) { in scx_alloc_and_add_sched()
7172 ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); in scx_alloc_and_add_sched()
7180 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_alloc_and_add_sched()
7183 pcpu->sch = sch; in scx_alloc_and_add_sched()
7198 sch->helper = kthread_run_worker(0, "sched_ext_helper"); in scx_alloc_and_add_sched()
7199 if (IS_ERR(sch->helper)) { in scx_alloc_and_add_sched()
7200 ret = PTR_ERR(sch->helper); in scx_alloc_and_add_sched()
7204 sched_set_fifo(sch->helper->task); in scx_alloc_and_add_sched()
7207 memcpy(sch->ancestors, parent->ancestors, in scx_alloc_and_add_sched()
7209 sch->ancestors[level] = sch; in scx_alloc_and_add_sched()
7210 sch->level = level; in scx_alloc_and_add_sched()
7211 sch->id = atomic64_inc_return(&scx_sched_id_cursor); in scx_alloc_and_add_sched()
7214 sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); in scx_alloc_and_add_sched()
7216 sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; in scx_alloc_and_add_sched()
7218 sch->slice_dfl = SCX_SLICE_DFL; in scx_alloc_and_add_sched()
7219 atomic_set(&sch->exit_kind, SCX_EXIT_NONE); in scx_alloc_and_add_sched()
7220 sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); in scx_alloc_and_add_sched()
7221 sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn); in scx_alloc_and_add_sched()
7222 kthread_init_work(&sch->disable_work, scx_disable_workfn); in scx_alloc_and_add_sched()
7223 timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); in scx_alloc_and_add_sched()
7225 if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { in scx_alloc_and_add_sched()
7229 if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { in scx_alloc_and_add_sched()
7233 if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) { in scx_alloc_and_add_sched()
7243 sch->ops_cid = *cmd->ops_cid; in scx_alloc_and_add_sched()
7244 sch->is_cid_type = true; in scx_alloc_and_add_sched()
7246 sch->ops = *cmd->ops; in scx_alloc_and_add_sched()
7256 sch->cgrp_path = kstrdup(buf, GFP_KERNEL); in scx_alloc_and_add_sched()
7258 if (!sch->cgrp_path) { in scx_alloc_and_add_sched()
7263 sch->cgrp = cgrp; in scx_alloc_and_add_sched()
7264 INIT_LIST_HEAD(&sch->children); in scx_alloc_and_add_sched()
7265 INIT_LIST_HEAD(&sch->sibling); in scx_alloc_and_add_sched()
7269 * Publishing makes @sch visible to scx_prog_sched() readers. Failure in scx_alloc_and_add_sched()
7270 * paths after this point must free @sch through kobject_put() whose in scx_alloc_and_add_sched()
7273 rcu_assign_pointer(ops->priv, sch); in scx_alloc_and_add_sched()
7275 sch->kobj.kset = scx_kset; in scx_alloc_and_add_sched()
7276 INIT_LIST_HEAD(&sch->all); in scx_alloc_and_add_sched()
7281 * Pin @parent for @sch's lifetime. The kobject hierarchy pins in scx_alloc_and_add_sched()
7291 * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized. in scx_alloc_and_add_sched()
7293 kobject_init(&sch->kobj, &scx_ktype); in scx_alloc_and_add_sched()
7298 * drops the ref. After this point, sch owns the ref and any cleanup in scx_alloc_and_add_sched()
7301 sch->arena_map = cmd->arena_map; in scx_alloc_and_add_sched()
7304 if (sch->arena_map) in scx_alloc_and_add_sched()
7305 sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); in scx_alloc_and_add_sched()
7308 return sch; in scx_alloc_and_add_sched()
7312 free_cpumask_var(sch->stall_cpus); in scx_alloc_and_add_sched()
7315 free_cpumask_var(sch->bypass_lb_resched_cpumask); in scx_alloc_and_add_sched()
7317 free_cpumask_var(sch->bypass_lb_donee_cpumask); in scx_alloc_and_add_sched()
7319 kthread_destroy_worker(sch->helper); in scx_alloc_and_add_sched()
7322 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_alloc_and_add_sched()
7332 exit_dsq(scx_bypass_dsq(sch, cpu)); in scx_alloc_and_add_sched()
7334 free_percpu(sch->pcpu); in scx_alloc_and_add_sched()
7337 free_pnode(sch->pnode[node]); in scx_alloc_and_add_sched()
7338 kfree(sch->pnode); in scx_alloc_and_add_sched()
7340 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); in scx_alloc_and_add_sched()
7342 free_exit_info(sch->exit_info); in scx_alloc_and_add_sched()
7344 kfree(sch); in scx_alloc_and_add_sched()
7353 * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler
7354 * implements ops.sub_attach. Called by the enable workfns once @sch's
7357 int scx_sched_sysfs_add(struct scx_sched *sch) in scx_sched_sysfs_add() argument
7360 struct scx_sched *parent = scx_parent(sch); in scx_sched_sysfs_add()
7364 ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj, in scx_sched_sysfs_add()
7365 "sub-%llu", cgroup_id(sch_cgroup(sch))); in scx_sched_sysfs_add()
7367 ret = kobject_add(&sch->kobj, NULL, "root"); in scx_sched_sysfs_add()
7371 if (sch->ops.sub_attach) { in scx_sched_sysfs_add()
7372 sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); in scx_sched_sysfs_add()
7373 if (!sch->sub_kset) in scx_sched_sysfs_add()
7378 return kobject_add(&sch->kobj, NULL, "root"); in scx_sched_sysfs_add()
7382 static int check_hotplug_seq(struct scx_sched *sch, in check_hotplug_seq() argument
7395 scx_exit(sch, SCX_EXIT_UNREG_KERN, in check_hotplug_seq()
7406 int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) in scx_validate_ops() argument
7413 scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); in scx_validate_ops()
7422 if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && in scx_validate_ops()
7423 !(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) { in scx_validate_ops()
7424 scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); in scx_validate_ops()
7434 scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); in scx_validate_ops()
7443 if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) in scx_validate_ops()
7452 if (!sch->is_cid_type) { in scx_validate_ops()
7453 if (scx_parent(sch)) { in scx_validate_ops()
7454 scx_error(sch, "sub-sched requires cid-form struct_ops"); in scx_validate_ops()
7458 scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); in scx_validate_ops()
7471 struct scx_sched *sch; in scx_root_enable_workfn() local
7509 sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); in scx_root_enable_workfn()
7510 if (IS_ERR(sch)) { in scx_root_enable_workfn()
7511 ret = PTR_ERR(sch); in scx_root_enable_workfn()
7515 if (sch->is_cid_type) in scx_root_enable_workfn()
7530 rq->scx.local_dsq.sched = sch; in scx_root_enable_workfn()
7535 scx_rescue_set_knobs(sch); in scx_root_enable_workfn()
7548 ret = scx_cid_init(sch); in scx_root_enable_workfn()
7558 rcu_assign_pointer(scx_root, sch); in scx_root_enable_workfn()
7560 ret = scx_link_sched(sch); in scx_root_enable_workfn()
7573 if (sch->is_cid_type && sch->ops_cid.init_cids) { in scx_root_enable_workfn()
7574 ret = SCX_CALL_OP_RET(sch, init_cids, NULL); in scx_root_enable_workfn()
7576 ret = scx_ops_sanitize_err(sch, "init_cids", ret); in scx_root_enable_workfn()
7578 scx_error(sch, "ops.init_cids() failed (%d)", ret); in scx_root_enable_workfn()
7586 ret = scx_arena_pool_init(sch); in scx_root_enable_workfn()
7592 ret = scx_set_cmask_scratch_alloc(sch); in scx_root_enable_workfn()
7598 ret = scx_alloc_pshards(sch); in scx_root_enable_workfn()
7604 scx_init_root_caps(sch); in scx_root_enable_workfn()
7607 if (sch->ops.init) { in scx_root_enable_workfn()
7608 ret = SCX_CALL_OP_RET(sch, init, NULL); in scx_root_enable_workfn()
7610 ret = scx_ops_sanitize_err(sch, "init", ret); in scx_root_enable_workfn()
7612 scx_error(sch, "ops.init() failed (%d)", ret); in scx_root_enable_workfn()
7615 sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; in scx_root_enable_workfn()
7618 ret = scx_sched_sysfs_add(sch); in scx_root_enable_workfn()
7626 set_bit(i, sch->has_op); in scx_root_enable_workfn()
7628 ret = check_hotplug_seq(sch, ops); in scx_root_enable_workfn()
7637 ret = scx_validate_ops(sch, ops); in scx_root_enable_workfn()
7672 scx_bypass(sch, true); in scx_root_enable_workfn()
7676 set_bit(i, sch->has_op); in scx_root_enable_workfn()
7678 if (sch->ops.cpu_acquire || sch->ops.cpu_release) in scx_root_enable_workfn()
7679 sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; in scx_root_enable_workfn()
7708 set_cgroup_sched(sch_cgroup(sch), sch); in scx_root_enable_workfn()
7709 ret = scx_cgroup_init(sch); in scx_root_enable_workfn()
7739 ret = __scx_init_task(sch, p, NULL, false); in scx_root_enable_workfn()
7747 scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", in scx_root_enable_workfn()
7759 scx_sub_init_cancel_task(sch, p); in scx_root_enable_workfn()
7762 scx_set_task_sched(p, sch); in scx_root_enable_workfn()
7806 scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); in scx_root_enable_workfn()
7813 scx_bypass(sch, false); in scx_root_enable_workfn()
7816 WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); in scx_root_enable_workfn()
7844 sch->ops.name, scx_switched_all() ? "" : " (partial)"); in scx_root_enable_workfn()
7845 kobject_uevent(&sch->kobj, KOBJ_ADD); in scx_root_enable_workfn()
7876 * completion. sch's base reference will be put by bpf_scx_unreg(). in scx_root_enable_workfn()
7878 scx_error(sch, "scx_root_enable() failed (%d)", ret); in scx_root_enable_workfn()
7879 scx_flush_disable_work(sch); in scx_root_enable_workfn()
8183 struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); in bpf_scx_unreg() local
8185 scx_disable(sch, SCX_EXIT_UNREG); in bpf_scx_unreg()
8186 scx_flush_disable_work(sch); in bpf_scx_unreg()
8188 kobject_put(&sch->kobj); in bpf_scx_unreg()
8392 struct scx_sched *sch; in sysrq_handle_sched_ext_reset() local
8394 sch = rcu_dereference(scx_root); in sysrq_handle_sched_ext_reset()
8395 if (likely(sch)) in sysrq_handle_sched_ext_reset()
8396 scx_disable(sch, SCX_EXIT_SYSRQ); in sysrq_handle_sched_ext_reset()
8415 struct scx_sched *sch; in sysrq_handle_sched_ext_dump() local
8417 list_for_each_entry_rcu(sch, &scx_sched_all, all) in sysrq_handle_sched_ext_dump()
8418 scx_dump_state(sch, &ei, 0, false); in sysrq_handle_sched_ext_dump()
8469 if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { in kick_one_cpu()
8472 u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); in kick_one_cpu()
8474 if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) in kick_one_cpu()
8475 __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); in kick_one_cpu()
8477 __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); in kick_one_cpu()
8495 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); in kick_one_cpu()
8517 if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE))) in kick_one_cpu_if_idle()
8520 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); in kick_one_cpu_if_idle()
8590 struct scx_sched *sch; in print_scx_info() local
8599 sch = scx_task_sched_rcu(p); in print_scx_info()
8601 if (!sch) in print_scx_info()
8610 printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, in print_scx_info()
8622 log_lvl, sch->ops.name, scx_enable_state_str[state], all, in print_scx_info()
8628 struct scx_sched *sch; in scx_pm_handler() local
8632 sch = rcu_dereference(scx_root); in scx_pm_handler()
8633 if (!sch) in scx_pm_handler()
8646 scx_bypass(sch, true); in scx_pm_handler()
8651 scx_bypass(sch, false); in scx_pm_handler()
8680 /* local_dsq's sch will be set during scx_root_enable() */ in init_sched_ext_class()
8715 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) in scx_vet_enq_flags() argument
8721 scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); in scx_vet_enq_flags()
8727 scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); in scx_vet_enq_flags()
8730 } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { in scx_vet_enq_flags()
8735 scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id); in scx_vet_enq_flags()
8742 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, in scx_dsq_insert_preamble() argument
8748 scx_error(sch, "called with NULL task"); in scx_dsq_insert_preamble()
8753 if (unlikely(!scx_task_on_sched(sch, p))) { in scx_dsq_insert_preamble()
8754 __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); in scx_dsq_insert_preamble()
8758 if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) in scx_dsq_insert_preamble()
8764 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, in scx_dsq_insert_commit() argument
8767 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_dsq_insert_commit()
8772 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags); in scx_dsq_insert_commit()
8776 if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { in scx_dsq_insert_commit()
8777 scx_error(sch, "dispatch buffer overflow"); in scx_dsq_insert_commit()
8837 struct scx_sched *sch; in scx_bpf_dsq_insert___v2() local
8840 sch = scx_prog_sched(aux); in scx_bpf_dsq_insert___v2()
8841 if (unlikely(!sch)) in scx_bpf_dsq_insert___v2()
8844 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) in scx_bpf_dsq_insert___v2()
8847 scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags); in scx_bpf_dsq_insert___v2()
8862 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, in scx_dsq_insert_vtime() argument
8865 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) in scx_dsq_insert_vtime()
8868 scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ); in scx_dsq_insert_vtime()
8917 struct scx_sched *sch; in __scx_bpf_dsq_insert_vtime() local
8921 sch = scx_prog_sched(aux); in __scx_bpf_dsq_insert_vtime()
8922 if (unlikely(!sch)) in __scx_bpf_dsq_insert_vtime()
8925 return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, in __scx_bpf_dsq_insert_vtime()
8935 struct scx_sched *sch; in scx_bpf_dsq_insert_vtime() local
8939 sch = rcu_dereference(scx_root); in scx_bpf_dsq_insert_vtime()
8940 if (unlikely(!sch)) in scx_bpf_dsq_insert_vtime()
8948 if (unlikely(!list_empty(&sch->children))) { in scx_bpf_dsq_insert_vtime()
8954 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); in scx_bpf_dsq_insert_vtime()
8977 struct scx_sched *sch; in scx_dsq_move() local
8990 sch = src_dsq->sched; in scx_dsq_move()
8992 if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) in scx_dsq_move()
9003 if (unlikely(READ_ONCE(sch->aborting))) in scx_dsq_move()
9006 if (unlikely(!scx_task_on_sched(sch, p))) { in scx_dsq_move()
9007 …scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler… in scx_dsq_move()
9044 dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); in scx_dsq_move()
9057 p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); in scx_dsq_move()
9083 struct scx_sched *sch; in scx_bpf_dispatch_nr_slots() local
9087 sch = scx_prog_sched(aux); in scx_bpf_dispatch_nr_slots()
9088 if (unlikely(!sch)) in scx_bpf_dispatch_nr_slots()
9091 return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); in scx_bpf_dispatch_nr_slots()
9103 struct scx_sched *sch; in scx_bpf_dispatch_cancel() local
9108 sch = scx_prog_sched(aux); in scx_bpf_dispatch_cancel()
9109 if (unlikely(!sch)) in scx_bpf_dispatch_cancel()
9112 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_bpf_dispatch_cancel()
9117 scx_error(sch, "dispatch buffer underflow"); in scx_bpf_dispatch_cancel()
9149 struct scx_sched *sch; in scx_bpf_dsq_move_to_local___v2() local
9154 sch = scx_prog_sched(aux); in scx_bpf_dsq_move_to_local___v2()
9155 if (unlikely(!sch)) in scx_bpf_dsq_move_to_local___v2()
9158 if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) in scx_bpf_dsq_move_to_local___v2()
9161 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_bpf_dsq_move_to_local___v2()
9163 scx_flush_dispatch_buf(sch, dspc->rq); in scx_bpf_dsq_move_to_local___v2()
9165 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_move_to_local___v2()
9167 scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); in scx_bpf_dsq_move_to_local___v2()
9171 if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { in scx_bpf_dsq_move_to_local___v2()
9323 struct scx_sched *sch; in scx_bpf_reenqueue_local() local
9327 sch = scx_prog_sched(aux); in scx_bpf_reenqueue_local()
9328 if (unlikely(!sch)) in scx_bpf_reenqueue_local()
9334 return reenq_local(sch, rq, SCX_REENQ_ANY); in scx_bpf_reenqueue_local()
9363 struct scx_sched *sch; in scx_bpf_create_dsq() local
9379 * NULL @sch and update afterwards. in scx_bpf_create_dsq()
9389 sch = scx_prog_sched(aux); in scx_bpf_create_dsq()
9390 if (sch) { in scx_bpf_create_dsq()
9391 dsq->sched = sch; in scx_bpf_create_dsq()
9392 ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, in scx_bpf_create_dsq()
9450 struct scx_sched *sch; in scx_bpf_task_set_slice() local
9454 sch = scx_prog_sched(aux); in scx_bpf_task_set_slice()
9455 if (unlikely(!sch || !scx_task_on_sched(sch, p))) in scx_bpf_task_set_slice()
9475 set_task_slice_oob(sch, p, slice); in scx_bpf_task_set_slice()
9481 unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { in scx_bpf_task_set_slice()
9482 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); in scx_bpf_task_set_slice()
9487 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); in scx_bpf_task_set_slice()
9504 struct scx_sched *sch; in scx_bpf_task_set_dsq_vtime() local
9507 sch = scx_prog_sched(aux); in scx_bpf_task_set_dsq_vtime()
9508 if (unlikely(!sch || !scx_task_on_sched(sch, p))) in scx_bpf_task_set_dsq_vtime()
9515 void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) in scx_kick_cpu() argument
9526 scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); in scx_kick_cpu()
9533 pcpu = this_cpu_ptr(sch->pcpu); in scx_kick_cpu()
9540 if (scx_bypassing(sch, cpu_of(this_rq))) in scx_kick_cpu()
9548 * The kick masks are owned by @sch->pcpu, so that a preempt kick can be in scx_kick_cpu()
9549 * attributed to @sch. in scx_kick_cpu()
9555 scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); in scx_kick_cpu()
9596 struct scx_sched *sch; in scx_bpf_kick_cpu() local
9599 sch = scx_prog_sched(aux); in scx_bpf_kick_cpu()
9600 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_kick_cpu()
9601 scx_kick_cpu(sch, cpu, flags); in scx_bpf_kick_cpu()
9618 struct scx_sched *sch; in scx_bpf_kick_cid() local
9622 sch = scx_prog_sched(aux); in scx_bpf_kick_cid()
9623 if (unlikely(!sch)) in scx_bpf_kick_cid()
9625 cpu = scx_cid_to_cpu(sch, cid); in scx_bpf_kick_cid()
9628 scx_kick_cpu(sch, cpu, flags); in scx_bpf_kick_cid()
9645 struct scx_sched *sch; in scx_bpf_dsq_nr_queued() local
9651 sch = scx_prog_sched(aux); in scx_bpf_dsq_nr_queued()
9652 if (unlikely(!sch)) { in scx_bpf_dsq_nr_queued()
9661 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); in scx_bpf_dsq_nr_queued()
9663 if (scx_cpu_valid(sch, cpu, NULL)) { in scx_bpf_dsq_nr_queued()
9668 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_nr_queued()
9692 struct scx_sched *sch; in scx_bpf_destroy_dsq() local
9695 sch = scx_prog_sched(aux); in scx_bpf_destroy_dsq()
9696 if (sch) in scx_bpf_destroy_dsq()
9697 destroy_dsq(sch, dsq_id); in scx_bpf_destroy_dsq()
9715 struct scx_sched *sch; in bpf_iter_scx_dsq_new() local
9730 sch = scx_prog_sched(aux); in bpf_iter_scx_dsq_new()
9731 if (unlikely(!sch)) in bpf_iter_scx_dsq_new()
9737 kit->dsq = find_user_dsq(sch, dsq_id); in bpf_iter_scx_dsq_new()
9802 struct scx_sched *sch; in scx_bpf_dsq_peek() local
9805 sch = scx_prog_sched(aux); in scx_bpf_dsq_peek()
9806 if (unlikely(!sch)) in scx_bpf_dsq_peek()
9810 scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); in scx_bpf_dsq_peek()
9814 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_peek()
9816 scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); in scx_bpf_dsq_peek()
9846 struct scx_sched *sch; in scx_bpf_dsq_reenq() local
9851 sch = scx_prog_sched(aux); in scx_bpf_dsq_reenq()
9852 if (unlikely(!sch)) in scx_bpf_dsq_reenq()
9856 scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); in scx_bpf_dsq_reenq()
9864 dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); in scx_bpf_dsq_reenq()
9865 schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); in scx_bpf_dsq_reenq()
9887 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, in __bstr_format() argument
9896 scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); in __bstr_format()
9902 scx_error(sch, "failed to read data fields (%d)", ret); in __bstr_format()
9909 scx_error(sch, "format preparation failed (%d)", ret); in __bstr_format()
9917 scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); in __bstr_format()
9925 * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit
9930 * scx_error() calls in __bstr_format() and differs from @sch when a parent
9932 * claim - @sch still exits with the claimed kind and a fallback message.
9935 bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, in scx_exit_bstr() argument
9939 struct scx_exit_info *ei = sch->exit_info; in scx_exit_bstr()
9945 if (!scx_claim_exit(sch, kind)) in scx_exit_bstr()
9954 scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id()); in scx_exit_bstr()
9976 struct scx_sched *sch; in scx_bpf_exit_bstr() local
9980 sch = scx_prog_sched(aux); in scx_bpf_exit_bstr()
9981 if (likely(sch)) in scx_bpf_exit_bstr()
9982 scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt, in scx_bpf_exit_bstr()
10000 struct scx_sched *sch; in scx_bpf_error_bstr() local
10004 sch = scx_prog_sched(aux); in scx_bpf_error_bstr()
10005 if (likely(sch)) in scx_bpf_error_bstr()
10006 scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data, in scx_bpf_error_bstr()
10027 struct scx_sched *sch; in scx_bpf_dump_bstr() local
10034 sch = scx_prog_sched(aux); in scx_bpf_dump_bstr()
10035 if (unlikely(!sch)) in scx_bpf_dump_bstr()
10039 scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); in scx_bpf_dump_bstr()
10044 ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, in scx_bpf_dump_bstr()
10081 struct scx_sched *sch; in scx_bpf_cpuperf_cap() local
10085 sch = scx_prog_sched(aux); in scx_bpf_cpuperf_cap()
10086 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_cpuperf_cap()
10101 struct scx_sched *sch; in scx_bpf_cidperf_cap() local
10106 sch = scx_prog_sched(aux); in scx_bpf_cidperf_cap()
10107 if (unlikely(!sch)) in scx_bpf_cidperf_cap()
10109 cpu = scx_cid_to_cpu(sch, cid); in scx_bpf_cidperf_cap()
10132 struct scx_sched *sch; in scx_bpf_cpuperf_cur() local
10136 sch = scx_prog_sched(aux); in scx_bpf_cpuperf_cur()
10137 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_cpuperf_cur()
10152 struct scx_sched *sch; in scx_bpf_cidperf_cur() local
10157 sch = scx_prog_sched(aux); in scx_bpf_cidperf_cur()
10158 if (unlikely(!sch)) in scx_bpf_cidperf_cur()
10160 cpu = scx_cid_to_cpu(sch, cid); in scx_bpf_cidperf_cur()
10167 static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) in scx_cpuperf_set() argument
10174 scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); in scx_cpuperf_set()
10178 if (!scx_cpu_valid(sch, cpu, NULL)) in scx_cpuperf_set()
10189 scx_error(sch, "Invalid target CPU %d", cpu); in scx_cpuperf_set()
10207 if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) { in scx_cpuperf_set()
10212 __scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1); in scx_cpuperf_set()
10239 struct scx_sched *sch; in scx_bpf_cpuperf_set() local
10243 sch = scx_prog_sched(aux); in scx_bpf_cpuperf_set()
10244 if (unlikely(!sch)) in scx_bpf_cpuperf_set()
10247 scx_cpuperf_set(sch, cpu, perf); in scx_bpf_cpuperf_set()
10264 struct scx_sched *sch; in scx_bpf_cidperf_set() local
10269 sch = scx_prog_sched(aux); in scx_bpf_cidperf_set()
10270 if (unlikely(!sch)) in scx_bpf_cidperf_set()
10272 cpu = scx_cid_to_cpu(sch, cid); in scx_bpf_cidperf_set()
10276 return scx_cpuperf_set(sch, cpu, perf); in scx_bpf_cidperf_set()
10420 struct scx_sched *sch; in scx_bpf_locked_rq() local
10425 sch = scx_prog_sched(aux); in scx_bpf_locked_rq()
10426 if (unlikely(!sch)) in scx_bpf_locked_rq()
10431 scx_error(sch, "accessing rq without holding rq lock"); in scx_bpf_locked_rq()
10447 struct scx_sched *sch; in scx_bpf_cpu_curr() local
10451 sch = scx_prog_sched(aux); in scx_bpf_cpu_curr()
10452 if (unlikely(!sch)) in scx_bpf_cpu_curr()
10455 if (!scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_cpu_curr()
10471 struct scx_sched *sch; in scx_bpf_cid_curr() local
10476 sch = scx_prog_sched(aux); in scx_bpf_cid_curr()
10477 if (unlikely(!sch)) in scx_bpf_cid_curr()
10479 cpu = scx_cid_to_cpu(sch, cid); in scx_bpf_cid_curr()
10499 struct scx_sched *sch = rcu_dereference(scx_root); in scx_bpf_tid_to_task() local
10501 if (sch) in scx_bpf_tid_to_task()
10502 scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); in scx_bpf_tid_to_task()
10574 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) in scx_read_events() argument
10581 struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; in scx_read_events()
10600 struct scx_sched *sch; in scx_bpf_events() local
10604 sch = scx_prog_sched(aux); in scx_bpf_events()
10605 if (sch) in scx_bpf_events()
10606 scx_read_events(sch, &e_sys); in scx_bpf_events()
10639 struct scx_sched *sch; in scx_bpf_task_cgroup() local
10643 sch = scx_prog_sched(aux); in scx_bpf_task_cgroup()
10644 if (unlikely(!sch)) in scx_bpf_task_cgroup()
10647 if (!scx_kf_arg_task_ok(sch, p)) in scx_bpf_task_cgroup()