35 struct scx_sched __rcu *scx_root;
45 	.key_len		= sizeof_field(struct scx_sched, ops.sub_cgroup_id),
46 	.key_offset		= offsetof(struct scx_sched, ops.sub_cgroup_id),
47 	.head_offset		= offsetof(struct scx_sched, hash_node),
130 struct scx_sched *scx_enabling_sub_sched;
132 #define scx_enabling_sub_sched	(struct scx_sched *)NULL
262 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
264 __printf(5, 6) bool __scx_exit(struct scx_sched *sch,  in __scx_exit()
298 bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor)  in scx_is_descendant()
305 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu)  in find_global_dsq()
310 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)  in find_user_dsq()
323 static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu)  in bypass_enq_target_dsq()
460 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,  in scx_call_op_set_cpumask()
932 static void scx_read_events(struct scx_sched *sch,
985 bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)  in scx_cpu_valid()
1082 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,  in schedule_dsq_reenq()
1313 static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice)  in set_task_slice_oob()
1477 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)  in refill_task_slice_dfl()
1506 static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,  in call_task_dequeue()
1518 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,  in rq_owned_post_enq()
1589 static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,  in scx_dispatch_enqueue()
1820 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,  in find_dsq_for_dispatch()
1855 static void mark_direct_dispatch(struct scx_sched *sch,  in mark_direct_dispatch()
1909 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,  in direct_dispatch()
1978 	struct scx_sched *sch = scx_task_sched(p);  in scx_do_enqueue_task()
2140 	struct scx_sched *sch = scx_task_sched(p);  in enqueue_task_scx()
2190 	struct scx_sched *sch = scx_task_sched(p);  in ops_dequeue()
2266 	struct scx_sched *sch = scx_task_sched(p);  in dequeue_task_scx()
2327 	struct scx_sched *sch = scx_task_sched(p);  in yield_task_scx()
2341 	struct scx_sched *sch = scx_task_sched(from);  in yield_to_task_scx()
2376 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,  in scx_move_local_task_to_local_dsq()
2410 static void move_remote_task_to_local_dsq(struct scx_sched *sch,  in move_remote_task_to_local_dsq()
2462 static bool task_can_run_on_remote_rq(struct scx_sched *sch,  in task_can_run_on_remote_rq()
2571 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,  in consume_remote_task()
2600 static struct rq *move_task_between_dsqs(struct scx_sched *sch,  in move_task_between_dsqs()
2652 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,  in scx_consume_dispatch_q()
2699 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)  in scx_consume_global_dsq()
2723 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,  in dispatch_to_local_dsq()
2824 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p,  in finish_dispatch()
2891 void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)  in scx_flush_dispatch_buf()
2922 	struct scx_sched *sch = scx_root_protected_live();  in dispatch_one()
3008 	struct scx_sched *sch = scx_task_sched(p);  in set_next_task_scx()
3081 	struct scx_sched *sch = scx_root_protected_live();  in switch_class()
3121 	struct scx_sched *sch = scx_task_sched(p);  in put_prev_task_scx()
3407 			struct scx_sched *sch = scx_task_sched(p);  in do_pick_task_scx()
3481 	struct scx_sched *sch_a = scx_task_sched(a);  in scx_prio_less()
3482 	struct scx_sched *sch_b = scx_task_sched(b);  in scx_prio_less()
3483 	struct scx_sched *sch = NULL;  in scx_prio_less()
3493 			struct scx_sched *anc = sch_a->ancestors[level];  in scx_prio_less()
3534 	struct scx_sched *sch = scx_task_sched(p);  in select_task_rq_scx()
3613 	struct scx_sched *sch = scx_task_sched(p);  in set_cpus_allowed_scx()
3634 	struct scx_sched *sch = scx_root_protected();  in handle_hotplug()
3706 	struct scx_sched *sch;  in check_rq_for_timeouts()
3717 		struct scx_sched *sch = scx_task_sched(p);  in check_rq_for_timeouts()
3769 	struct scx_sched *root;  in scx_tick()
3794 	struct scx_sched *sch = scx_task_sched(curr);  in task_tick_scx()
3845 int __scx_init_task(struct scx_sched *sch, struct task_struct *p,  in __scx_init_task()
3900 static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)  in __scx_enable_task()
3932 void scx_enable_task(struct scx_sched *sch, struct task_struct *p)  in scx_enable_task()
3938 static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)  in scx_disable_task()
3968 void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)  in __scx_disable_and_exit_task()
4002 void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p)  in scx_sub_init_cancel_task()
4014 void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)  in scx_disable_and_exit_task()
4097 		struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp);  in scx_fork()
4099 		struct scx_sched *sch = scx_root_protected_live();  in scx_fork()
4231 	struct scx_sched *sch = scx_task_sched(p);  in reweight_task_scx()
4260 	struct scx_sched *sch = scx_task_sched(p);  in switching_to_scx()
4322 		struct scx_sched *sch = scx_task_sched(p);  in process_ddsp_deferred_locals()
4390 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)  in reenq_local()
4410 		struct scx_sched *task_sch = scx_task_sched(p);  in reenq_local()
4478 		struct scx_sched *sch;  in process_deferred_reenq_locals()
4516 	struct scx_sched *sch = dsq->sched;  in reenq_user()
4645 	struct scx_sched *sch = scx_task_sched(p);  in scx_can_stop_tick()
4707 static struct scx_sched *scx_tg_sched(struct task_group *tg)  in scx_tg_sched()
4734 static struct scx_sched *scx_tg_knob_sched(struct task_group *tg)  in scx_tg_knob_sched()
4751 		struct scx_sched *sch;  in scx_tg_online()
4789 	struct scx_sched *sch = tg->scx.sched;  in scx_tg_offline()
4806 static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p)  in scx_cgroup_task_sched()
4821 		struct scx_sched *sch = scx_cgroup_task_sched(p);  in scx_cgroup_can_attach()
4860 		struct scx_sched *sch = scx_cgroup_task_sched(p);  in scx_cgroup_can_attach()
4874 	struct scx_sched *sch;  in scx_cgroup_move_task()
4902 		struct scx_sched *sch = scx_cgroup_task_sched(p);  in scx_cgroup_cancel_attach()
4914 	struct scx_sched *sch;  in scx_group_set_weight()
4930 	struct scx_sched *sch;  in scx_group_set_idle()
4947 	struct scx_sched *sch;  in scx_group_set_bandwidth()
5042 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)  in scx_init_dsq()
5108 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)  in destroy_dsq()
5148 static void scx_cgroup_exit(struct scx_sched *sch)  in scx_cgroup_exit()
5169 static int scx_cgroup_init(struct scx_sched *sch)  in scx_cgroup_init()
5207 static void scx_cgroup_exit(struct scx_sched *sch) {}  in scx_cgroup_exit()
5208 static int scx_cgroup_init(struct scx_sched *sch) { return 0; }  in scx_cgroup_init()
5273 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
5275 s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)  in scx_set_cmask_scratch_alloc()
5299 static void scx_set_cmask_scratch_free(struct scx_sched *sch)  in scx_set_cmask_scratch_free()
5320 	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);  in scx_sched_free_rcu_work()
5404 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);  in scx_kobj_release()
5413 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);  in scx_attr_ops_show()
5426 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);  in scx_attr_events_show()
5449 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);  in scx_attr_caps_show()
5489 ATTRIBUTE_GROUPS(scx_sched);
5499 	const struct scx_sched *sch;  in scx_uevent()
5509 	sch = container_of(kobj, struct scx_sched, kobj);  in scx_uevent()
5553 	struct scx_sched *sch;  in scx_allow_ttwu_queue()
5585 	struct scx_sched *sch;  in handle_lockup()
5622 	struct scx_sched *sch;  in scx_rcu_cpu_stall()
5710 static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,  in bypass_lb_cpu()
5816 static void bypass_lb_node(struct scx_sched *sch, int node)  in bypass_lb_node()
5895 	struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer);  in scx_bypass_lb_timerfn()
5910 static bool inc_bypass_depth(struct scx_sched *sch)  in inc_bypass_depth()
5925 static bool dec_bypass_depth(struct scx_sched *sch)  in dec_bypass_depth()
5940 static void enable_bypass_dsp(struct scx_sched *sch)  in enable_bypass_dsp()
5942 	struct scx_sched *host = scx_parent(sch) ?: sch;  in enable_bypass_dsp()
5987 void scx_disable_bypass_dsp(struct scx_sched *sch)  in scx_disable_bypass_dsp()
6022 static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos,  in unbypass_renotify_idle()
6067 void scx_bypass(struct scx_sched *sch, bool bypass)  in scx_bypass()
6069 	struct scx_sched *pos;  in scx_bypass()
6261 	struct scx_sched *sch;  in refresh_watchdog()
6279 s32 scx_link_sched(struct scx_sched *sch)  in scx_link_sched()
6284 		struct scx_sched *parent = scx_parent(sch);  in scx_link_sched()
6334 void scx_unlink_sched(struct scx_sched *sch)  in scx_unlink_sched()
6356 void scx_disable_dump(struct scx_sched *sch)  in scx_disable_dump()
6362 void scx_log_sched_disable(struct scx_sched *sch)  in scx_log_sched_disable()
6382 static void scx_root_disable(struct scx_sched *sch)  in scx_root_disable()
6559 	struct scx_sched *sch = container_of(irq_work, struct scx_sched,  in scx_propagate_exit_irq_workfn()
6561 	struct scx_sched *pos;  in scx_propagate_exit_irq_workfn()
6578 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)  in scx_claim_exit()
6594 		struct scx_sched *pos;  in scx_claim_exit()
6620 	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);  in scx_disable_workfn()
6641 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)  in scx_disable()
6656 void scx_flush_disable_work(struct scx_sched *sch)  in scx_flush_disable_work()
6778 static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx,  in scx_dump_task()
6782 	struct scx_sched *task_sch = scx_task_sched(p);  in scx_dump_task()
6831 static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,  in scx_dump_cpu()
6926 static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,  in scx_dump_state()
7018 	struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work);  in scx_disable_irq_workfn()
7028 static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind,  in scx_finish_exit()
7053 bool scx_vexit(struct scx_sched *sch,  in scx_vexit()
7105 static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)  in alloc_pnode()
7125 struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,  in scx_alloc_and_add_sched()
7127 					  struct scx_sched *parent)  in scx_alloc_and_add_sched()
7130 	struct scx_sched *sch;  in scx_alloc_and_add_sched()
7359 int scx_sched_sysfs_add(struct scx_sched *sch)  in scx_sched_sysfs_add()
7362 	struct scx_sched *parent = scx_parent(sch);  in scx_sched_sysfs_add()
7384 static int check_hotplug_seq(struct scx_sched *sch,  in check_hotplug_seq()
7408 int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)  in scx_validate_ops()
7473 	struct scx_sched *sch;  in scx_root_enable_workfn()
8186 	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);  in bpf_scx_unreg()
8395 	struct scx_sched *sch;  in sysrq_handle_sched_ext_reset()
8418 	struct scx_sched *sch;  in sysrq_handle_sched_ext_dump()
8593 	struct scx_sched *sch;  in print_scx_info()
8631 	struct scx_sched *sch;  in scx_pm_handler()
8718 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)  in scx_vet_enq_flags()
8745 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p,  in scx_dsq_insert_preamble()
8767 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,  in scx_dsq_insert_commit()
8840 	struct scx_sched *sch;  in scx_bpf_dsq_insert___v2()
8865 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,  in scx_dsq_insert_vtime()
8920 	struct scx_sched *sch;  in __scx_bpf_dsq_insert_vtime()
8938 	struct scx_sched *sch;  in scx_bpf_dsq_insert_vtime()
8980 	struct scx_sched *sch;  in scx_dsq_move()
9091 	struct scx_sched *sch;  in scx_bpf_dispatch_nr_slots()
9111 	struct scx_sched *sch;  in scx_bpf_dispatch_cancel()
9157 	struct scx_sched *sch;  in scx_bpf_dsq_move_to_local___v2()
9331 	struct scx_sched *sch;  in scx_bpf_reenqueue_local()
9371 	struct scx_sched *sch;  in scx_bpf_create_dsq()
9458 	struct scx_sched *sch;  in scx_bpf_task_set_slice()
9512 	struct scx_sched *sch;  in scx_bpf_task_set_dsq_vtime()
9523 void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)  in scx_kick_cpu()
9604 	struct scx_sched *sch;  in scx_bpf_kick_cpu()
9626 	struct scx_sched *sch;  in scx_bpf_kick_cid()
9653 	struct scx_sched *sch;  in scx_bpf_dsq_nr_queued()
9700 	struct scx_sched *sch;  in scx_bpf_destroy_dsq()
9723 	struct scx_sched *sch;  in bpf_iter_scx_dsq_new()
9810 	struct scx_sched *sch;  in scx_bpf_dsq_peek()
9854 	struct scx_sched *sch;  in scx_bpf_dsq_reenq()
9895 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,  in __bstr_format()
9943 bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind,  in scx_exit_bstr()
9944 		   s64 exit_code, struct scx_sched *fmt_blame, char *fmt,  in scx_exit_bstr()
9984 	struct scx_sched *sch;  in scx_bpf_exit_bstr()
10008 	struct scx_sched *sch;  in scx_bpf_error_bstr()
10035 	struct scx_sched *sch;  in scx_bpf_dump_bstr()
10089 	struct scx_sched *sch;  in scx_bpf_cpuperf_cap()
10109 	struct scx_sched *sch;  in scx_bpf_cidperf_cap()
10140 	struct scx_sched *sch;  in scx_bpf_cpuperf_cur()
10160 	struct scx_sched *sch;  in scx_bpf_cidperf_cur()
10175 static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf)  in scx_cpuperf_set()
10247 	struct scx_sched *sch;  in scx_bpf_cpuperf_set()
10272 	struct scx_sched *sch;  in scx_bpf_cidperf_set()
10428 	struct scx_sched *sch;  in scx_bpf_locked_rq()
10455 	struct scx_sched *sch;  in scx_bpf_cpu_curr()
10479 	struct scx_sched *sch;  in scx_bpf_cid_curr()
10507 		struct scx_sched *sch = rcu_dereference(scx_root);  in scx_bpf_tid_to_task()
10582 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events)  in scx_read_events()
10608 	struct scx_sched *sch;  in scx_bpf_events()
10647 	struct scx_sched *sch;  in scx_bpf_task_cgroup()