101 void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)  in scx_set_task_sched()  argument
103 	rcu_assign_pointer(p->scx.sched, sch);  in scx_set_task_sched()
106 struct cgroup *sch_cgroup(struct scx_sched *sch)  in sch_cgroup()  argument
108 	return sch->cgrp;  in sch_cgroup()
111 /* for each descendant of @cgrp including self, set ->scx_sched to @sch */
112 void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)  in set_cgroup_sched()  argument
118 		rcu_assign_pointer(pos->scx_sched, sch);  in set_cgroup_sched()
129 		scx_arena_free(pshard->sch, cu->cmask_arena_out,  in free_pshard()
135 void scx_free_pshards(struct scx_sched *sch)  in scx_free_pshards()  argument
139 	if (!sch->pshard)  in scx_free_pshards()
141 	for (si = 0; si < sch->nr_pshards; si++)  in scx_free_pshards()
142 		free_pshard(sch->pshard[si]);  in scx_free_pshards()
143 	kfree(sch->pshard);  in scx_free_pshards()
146 static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node)  in alloc_pshard()  argument
162 	pshard->sch = sch;  in alloc_pshard()
174 	cu->cmask_arena_out = scx_arena_alloc(sch, cmask_size);  in alloc_pshard()
185 s32 scx_alloc_pshards(struct scx_sched *sch)  in scx_alloc_pshards()  argument
191 	if (!sch->is_cid_type || !sch->arena_pool)  in scx_alloc_pshards()
202 		pshard[si] = alloc_pshard(sch, si, shard_node[si]);  in scx_alloc_pshards()
211 	sch->nr_pshards = scx_nr_cid_shards;  in scx_alloc_pshards()
214 	 * @sch->pshard never sees a partially-filled array or unpublished cid  in scx_alloc_pshards()
219 	WRITE_ONCE(sch->pshard, pshard);  in scx_alloc_pshards()
227 void scx_init_root_caps(struct scx_sched *sch)  in scx_init_root_caps()  argument
231 	for (si = 0; si < sch->nr_pshards; si++) {  in scx_init_root_caps()
232 		struct scx_pshard *ps = sch->pshard[si];  in scx_init_root_caps()
622 static void scx_rescue_check_timeout(struct scx_sched *sch)  in scx_rescue_check_timeout()  argument
624 	if (!scx_rescue_bw_1024 || sch->watchdog_timeout > scx_rescue_overload_after)  in scx_rescue_check_timeout()
628 		sch->ops.name, jiffies_to_msecs(sch->watchdog_timeout),  in scx_rescue_check_timeout()
633 void scx_rescue_set_knobs(struct scx_sched *sch)  in scx_rescue_set_knobs()  argument
635 	s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT;  in scx_rescue_set_knobs()
636 	s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US;  in scx_rescue_set_knobs()
639 	if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) {  in scx_rescue_set_knobs()
664 			sch->ops.name, div_s64(period_ns, NSEC_PER_MSEC),  in scx_rescue_set_knobs()
667 	scx_rescue_check_timeout(sch);  in scx_rescue_set_knobs()
679  * @sch: enqueuing sub-sched
684  * Return @rq's local DSQ if @sch holds the required caps on @rq's cid.
697 struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,  in scx_resolve_local_dsq()  argument
704 	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;  in scx_resolve_local_dsq()
710 	 * @p's owner (@sch). Check caps against the scheduling sched.  in scx_resolve_local_dsq()
728 		__scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1);  in scx_resolve_local_dsq()
744 		__scx_add_event(sch, SCX_EV_SUB_RESCUE, 1);  in scx_resolve_local_dsq()
842 		struct scx_sched *sch = ps->sch;  in caps_updated_deliver()  local
855 				if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) &&  in caps_updated_deliver()
856 				    likely(!READ_ONCE(sch->aborting))) {  in caps_updated_deliver()
860 					scx_cmask_ref_init_kern(sch, cu->cmask_arena_out,  in caps_updated_deliver()
873 			SCX_CALL_OP(sch, sub_caps_updated, NULL, cu->cmask_arena_out, caps);  in caps_updated_deliver()
879  * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant
880  * taken during its sub_attach(), before has_op was set). Called once @sch is
883 static void scx_sub_seed_caps(struct scx_sched *sch)  in scx_sub_seed_caps()  argument
890 	for (si = 0; si < sch->nr_pshards; si++) {  in scx_sub_seed_caps()
891 		struct scx_pshard *ps = sch->pshard[si];  in scx_sub_seed_caps()
914  * queue_sync_ecaps - Queue ecaps update for a (sch, cid) pair
915  * @sch: sched to update
918  * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in
921 static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)  in queue_sync_ecaps()  argument
924 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);  in queue_sync_ecaps()
937 		scx_kick_cpu(sch->ancestors[0], cpu, 0);  in queue_sync_ecaps()
997 		struct scx_pshard *ps = pcpu->sch->pshard[shard];  in scx_process_sync_ecaps()
1024 		    SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&  in scx_process_sync_ecaps()
1025 		    !scx_bypassing(pcpu->sch, cpu)) {  in scx_process_sync_ecaps()
1026 			struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;  in scx_process_sync_ecaps()
1031 			SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),  in scx_process_sync_ecaps()
1034 			scx_flush_dispatch_buf(pcpu->sch, rq);  in scx_process_sync_ecaps()
1063  * @sch: scheduler that just left bypass on @rq's cpu
1072 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)  in scx_unbypass_replay_ecaps()  argument
1075 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);  in scx_unbypass_replay_ecaps()
1082 	if (!sch->level)  in scx_unbypass_replay_ecaps()
1089 	ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]];  in scx_unbypass_replay_ecaps()
1092 	queue_sync_ecaps(sch, cid);  in scx_unbypass_replay_ecaps()
1232 void drain_descendants(struct scx_sched *sch)  in drain_descendants()  argument
1236 	 * themselves off @sch->children. Wait for it to drain. As propagation  in drain_descendants()
1237 	 * is recursive, empty @sch->children means that all proper descendant  in drain_descendants()
1240 	wait_event(scx_unlink_waitq, list_empty(&sch->children));  in drain_descendants()
1294 static void scx_fail_parent(struct scx_sched *sch,  in scx_fail_parent()  argument
1297 	struct scx_sched *parent = scx_parent(sch);  in scx_fail_parent()
1311 	scx_task_iter_start(&sti, sch->cgrp);  in scx_fail_parent()
1324  * @sch: sub-scheduler being enabled
1326  * Called while enabling @sch, after the subtree's cgrp->scx_sched's are pointed
1327  * at @sch and before any task is claimed. This mirrors root enable's
1332  * First init each of the parent sched's subtree cgroups on @sch, and only then
1346  * Return 0 on success, -errno on failure. On failure, @sch has been
1349 static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)  in scx_cgroup_claim_subtree()  argument
1351 	struct cgroup *sub_cgrp = sch_cgroup(sch);  in scx_cgroup_claim_subtree()
1353 	struct scx_sched *parent = scx_parent(sch);  in scx_cgroup_claim_subtree()
1370 		if (SCX_HAS_OP(sch, cgroup_init)) {  in scx_cgroup_claim_subtree()
1371 			ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);  in scx_cgroup_claim_subtree()
1373 				scx_error(sch, "ops.cgroup_init() failed (%d)", ret);  in scx_cgroup_claim_subtree()
1393 		tg->scx.sched = sch;  in scx_cgroup_claim_subtree()
1407 		if (SCX_HAS_OP(sch, cgroup_exit))  in scx_cgroup_claim_subtree()
1408 			SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup);  in scx_cgroup_claim_subtree()
1416  * @sch: sub-scheduler being disabled
1418  * Called while disabling @sch, after the subtree's cgrp->scx_sched's are reset
1421  * scx_cgroup_claim_subtree(): exit @sch's cgroups from @sch, then init them on
1423  * while @sch had them.
1437 static void scx_cgroup_return_subtree(struct scx_sched *sch)  in scx_cgroup_return_subtree()  argument
1439 	struct cgroup *sub_cgrp = sch_cgroup(sch);  in scx_cgroup_return_subtree()
1441 	struct scx_sched *parent = scx_parent(sch);  in scx_cgroup_return_subtree()
1449 		if (tg->scx.sched != sch ||  in scx_cgroup_return_subtree()
1453 		/* skip the exit if @sch's ops.cgroup_init() failed for the tg */  in scx_cgroup_return_subtree()
1454 		if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(sch, cgroup_exit))  in scx_cgroup_return_subtree()
1455 			SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup);  in scx_cgroup_return_subtree()
1470 		WARN_ON_ONCE(tg->scx.sched == sch);  in scx_cgroup_return_subtree()
1502 static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; }  in scx_cgroup_claim_subtree()  argument
1503 static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {}  in scx_cgroup_return_subtree()  argument
1506 void scx_sub_disable(struct scx_sched *sch)  in scx_sub_disable()  argument
1508 	struct scx_sched *parent = scx_parent(sch);  in scx_sub_disable()
1518 	scx_bypass(sch, true);  in scx_sub_disable()
1519 	drain_descendants(sch);  in scx_sub_disable()
1538 	if (!sch->linked)  in scx_sub_disable()
1541 	set_cgroup_sched(sch_cgroup(sch), parent);  in scx_sub_disable()
1547 	scx_cgroup_return_subtree(sch);  in scx_sub_disable()
1549 	scx_task_iter_start(&sti, sch->cgrp);  in scx_sub_disable()
1562 		WARN_ON_ONCE(!scx_task_on_sched(sch, p));  in scx_sub_disable()
1574 		 * $p is READY or ENABLED on @sch. Initialize for $parent,  in scx_sub_disable()
1575 		 * disable and exit from @sch, and then switch over to $parent.  in scx_sub_disable()
1586 			scx_fail_parent(sch, p, ret);  in scx_sub_disable()
1596 			 * and this rq lock and ran exit_task() on @sch (the  in scx_sub_disable()
1615 	scx_disable_dump(sch);  in scx_sub_disable()
1621 	 * All tasks are moved off of @sch but there may still be on-going  in scx_sub_disable()
1628 	scx_disable_bypass_dsp(sch);  in scx_sub_disable()
1630 	scx_unlink_sched(sch);  in scx_sub_disable()
1635 	 * @sch is now unlinked from the parent's children list. Notify and call  in scx_sub_disable()
1641 	if (parent->ops.sub_detach && sch->sub_attached) {  in scx_sub_disable()
1643 			.ops = &sch->ops,  in scx_sub_disable()
1644 			.cgroup_path = sch->cgrp_path,  in scx_sub_disable()
1650 	scx_log_sched_disable(sch);  in scx_sub_disable()
1652 	if (sch->ops.exit)  in scx_sub_disable()
1653 		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);  in scx_sub_disable()
1656 	 * @sch's non-ops programs such as timers and tracers can fire after  in scx_sub_disable()
1658 	 * resolving to @sch and drain in-flight resolvers.  in scx_sub_disable()
1660 	WRITE_ONCE(sch->dead, true);  in scx_sub_disable()
1663 	if (sch->sub_kset)  in scx_sub_disable()
1664 		kobject_del(&sch->sub_kset->kobj);  in scx_sub_disable()
1666 	if (sch->kobj.state_in_sysfs)  in scx_sub_disable()
1667 		kobject_del(&sch->kobj);  in scx_sub_disable()
1714 	struct scx_sched *parent, *sch;  in scx_sub_enable_workfn()  local
1756 	sch = scx_alloc_and_add_sched(cmd, cgrp, parent);  in scx_sub_enable_workfn()
1758 	if (IS_ERR(sch)) {  in scx_sub_enable_workfn()
1760 		ret = PTR_ERR(sch);  in scx_sub_enable_workfn()
1765 	 * Validate before scx_link_sched() publishes @sch, so an invalid sub  in scx_sub_enable_workfn()
1768 	ret = scx_validate_ops(sch, ops);  in scx_sub_enable_workfn()
1772 	scx_rescue_check_timeout(sch);  in scx_sub_enable_workfn()
1775 	 * Allocate pshard[] before scx_link_sched() publishes @sch into the  in scx_sub_enable_workfn()
1777 	 * would otherwise dereference sch->pshard[si] while it's still NULL.  in scx_sub_enable_workfn()
1780 	 * scx_alloc_pshards() skips allocation when @sch's arena pool isn't  in scx_sub_enable_workfn()
1783 	ret = scx_arena_pool_init(sch);  in scx_sub_enable_workfn()
1787 	ret = scx_alloc_pshards(sch);  in scx_sub_enable_workfn()
1791 	ret = scx_link_sched(sch);  in scx_sub_enable_workfn()
1795 	ret = scx_sched_sysfs_add(sch);  in scx_sub_enable_workfn()
1799 	if (sch->level >= SCX_SUB_MAX_DEPTH) {  in scx_sub_enable_workfn()
1800 		scx_error(sch, "max nesting depth %d violated",  in scx_sub_enable_workfn()
1806 	if (sch->ops.init) {  in scx_sub_enable_workfn()
1807 		ret = SCX_CALL_OP_RET(sch, init, NULL);  in scx_sub_enable_workfn()
1809 			ret = scx_ops_sanitize_err(sch, "init", ret);  in scx_sub_enable_workfn()
1810 			scx_error(sch, "ops.init() failed (%d)", ret);  in scx_sub_enable_workfn()
1813 		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;  in scx_sub_enable_workfn()
1816 	ret = scx_set_cmask_scratch_alloc(sch);  in scx_sub_enable_workfn()
1821 		.ops = &sch->ops,  in scx_sub_enable_workfn()
1822 		.cgroup_path = sch->cgrp_path,  in scx_sub_enable_workfn()
1828 		ret = scx_ops_sanitize_err(sch, "sub_attach", ret);  in scx_sub_enable_workfn()
1829 		scx_error(sch, "parent rejected (%d)", ret);  in scx_sub_enable_workfn()
1832 	sch->sub_attached = true;  in scx_sub_enable_workfn()
1834 	scx_bypass(sch, true);  in scx_sub_enable_workfn()
1838 			set_bit(i, sch->has_op);  in scx_sub_enable_workfn()
1847 	set_cgroup_sched(sch_cgroup(sch), sch);  in scx_sub_enable_workfn()
1849 		scx_error(sch, "cgroup is not online");  in scx_sub_enable_workfn()
1858 	ret = scx_cgroup_claim_subtree(sch);  in scx_sub_enable_workfn()
1863 	 * Initialize tasks for the new child $sch without exiting them for  in scx_sub_enable_workfn()
1868 	scx_enabling_sub_sched = sch;  in scx_sub_enable_workfn()
1870 	scx_task_iter_start(&sti, sch->cgrp);  in scx_sub_enable_workfn()
1902 		ret = __scx_init_task(sch, p, NULL, false);  in scx_sub_enable_workfn()
1912 			 * sched @p was on at that point), not on @sch. @sch's  in scx_sub_enable_workfn()
1916 			scx_sub_init_cancel_task(sch, p);  in scx_sub_enable_workfn()
1931 	 * the new @sch.  in scx_sub_enable_workfn()
1933 	scx_task_iter_start(&sti, sch->cgrp);  in scx_sub_enable_workfn()
1953 			 * $p is now only initialized for @sch and READY, which  in scx_sub_enable_workfn()
1954 			 * is what we want. Assign it to @sch and, if it's on  in scx_sub_enable_workfn()
1959 			scx_set_task_sched(p, sch);  in scx_sub_enable_workfn()
1961 				scx_enable_task(sch, p);  in scx_sub_enable_workfn()
1973 	scx_bypass(sch, false);  in scx_sub_enable_workfn()
1975 	/* @sch is enabled; deliver any caps owed since its sub_attach() */  in scx_sub_enable_workfn()
1976 	scx_sub_seed_caps(sch);  in scx_sub_enable_workfn()
1978 	pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name);  in scx_sub_enable_workfn()
1979 	kobject_uevent(&sch->kobj, KOBJ_ADD);  in scx_sub_enable_workfn()
1996 	 * ran for @sch on them, so calling scx_disable_task() here would invoke  in scx_sub_enable_workfn()
2001 	scx_task_iter_start(&sti, sch->cgrp);  in scx_sub_enable_workfn()
2004 			scx_sub_init_cancel_task(sch, p);  in scx_sub_enable_workfn()
2021 	scx_error(sch, "scx_sub_enable() failed (%d)", ret);  in scx_sub_enable_workfn()
2022 	scx_flush_disable_work(sch);  in scx_sub_enable_workfn()
2124 	struct scx_sched *sch;  in scx_cgroup_lifetime_notify()  local
2137 		sch = scx_cgroup_sched(cgrp);  in scx_cgroup_lifetime_notify()
2138 		if (sch && sch->cgrp == cgrp)  in scx_cgroup_lifetime_notify()
2139 			scx_exit(sch, SCX_EXIT_UNREG_KERN,  in scx_cgroup_lifetime_notify()
2191 	struct scx_sched *sch;  in scx_pstack_recursion()  local
2194 	sch = scx_prog_sched(prog->aux);  in scx_pstack_recursion()
2195 	if (unlikely(!sch))  in scx_pstack_recursion()
2198 	scx_error(sch, "%s recursion detected", op);  in scx_pstack_recursion()
2529 	struct scx_sched *sch, *target;  in scx_bpf_sub_caps()  local
2535 	sch = scx_prog_sched(aux);  in scx_bpf_sub_caps()
2536 	if (unlikely(!sch))  in scx_bpf_sub_caps()
2540 		scx_error(sch, "sub-cap kfuncs require a cid-form scheduler");  in scx_bpf_sub_caps()
2545 		scx_error(sch, "invalid caps 0x%llx", caps);  in scx_bpf_sub_caps()
2554 		if (unlikely(scx_parent(target) != sch)) {  in scx_bpf_sub_caps()
2555 			scx_error(sch, "%s: sub-%llu is not a direct child",  in scx_bpf_sub_caps()
2556 				  sch->cgrp_path, cgroup_id);  in scx_bpf_sub_caps()
2560 		target = sch;  in scx_bpf_sub_caps()
2571 		scx_error(sch, "scx_bpf_sub_caps() called before caps storage is initialized");  in scx_bpf_sub_caps()
2575 	ret = scx_cmask_ref_init(sch, out__arena, &ref);  in scx_bpf_sub_caps()
2577 		scx_error(sch, "invalid out (%d)", ret);  in scx_bpf_sub_caps()