103 	rcu_assign_pointer(p->scx.sched, sch);  in scx_set_task_sched()
242 	s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap;  in scx_rescue_slice_remaining()
244 	return max(rq->scx.rescue.slice - served, 0);  in scx_rescue_slice_remaining()
285 	rq->scx.rescue.budget -= delta_exec;  in scx_rescue_charge()
292 		scx_task_slice_ended(rq, rq->scx.rescue.curr);  in scx_rescue_charge()
308 	rq->scx.rescue.curr = NULL;  in scx_rescue_end()
309 	if (list_empty(&rq->scx.rescue.dsq.list))  in scx_rescue_end()
310 		rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns);  in scx_rescue_end()
334 	if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) ||  in scx_rescue_keep()
354 	bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list);  in scx_rescue_accrue()
363 	delta = now - rq->scx.rescue.clock;  in scx_rescue_accrue()
364 	rq->scx.rescue.clock = now;  in scx_rescue_accrue()
371 		rq->scx.rescue.budget = cap;  in scx_rescue_accrue()
373 		rq->scx.rescue.budget =  in scx_rescue_accrue()
374 			min(cap, rq->scx.rescue.budget +  in scx_rescue_accrue()
385 	u32 depth = rq->scx.rescue.dsq.nr ?: 1;  in scx_rescue_next_slice()
392 	struct timer_list *timer = &rq->scx.rescue.timer;  in scx_rescue_timer_arm()
402 	if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) {  in scx_rescue_timer_arm()
403 		s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget;  in scx_rescue_timer_arm()
429 	WARN_ON_ONCE(rq->scx.rescue.curr);  in scx_rescue_admit()
431 	rq->scx.rescue.curr = p;  in scx_rescue_admit()
432 	rq->scx.rescue.slice = slice;  in scx_rescue_admit()
433 	rq->scx.rescue.exec_snap = p->se.sum_exec_runtime;  in scx_rescue_admit()
452 	if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) &&  in scx_rescue_try_admit()
453 	    rq->scx.rescue.budget >= scx_rescue_quantum_ns) {  in scx_rescue_try_admit()
481 	p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct,  in scx_rescue_check_overload()
482 				     scx.dsq_list.node);  in scx_rescue_check_overload()
487 	if (time_before(jiffies, p->scx.rescue_at + scx_rescue_overload_after))  in scx_rescue_check_overload()
495 	if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at +  in scx_rescue_check_overload()
512 	rq->scx.rescue.kill_at = get_jiffies_64();  in scx_rescue_check_overload()
513 	dur_ms = jiffies_to_msecs(jiffies - p->scx.rescue_at);  in scx_rescue_check_overload()
534 	struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer);  in scx_rescue_timerfn()
539 	p = rq->scx.rescue.curr;  in scx_rescue_timerfn()
540 	if (!p && list_empty(&rq->scx.rescue.dsq.list))  in scx_rescue_timerfn()
550 		if (rq->scx.rescue.budget < scx_rescue_quantum_ns)  in scx_rescue_timerfn()
554 		p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct,  in scx_rescue_timerfn()
555 				     scx.dsq_list.node);  in scx_rescue_timerfn()
556 		scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);  in scx_rescue_timerfn()
559 						 &rq->scx.rescue.dsq, rq);  in scx_rescue_timerfn()
562 	} else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {  in scx_rescue_timerfn()
571 		p->scx.flags |= SCX_TASK_PROTECTED;  in scx_rescue_timerfn()
572 		scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);  in scx_rescue_timerfn()
575 					&rq->scx.local_dsq, rq);  in scx_rescue_timerfn()
593 	if (rq->scx.rescue.curr)  in scx_rescue_flush()
594 		scx_task_slice_ended(rq, rq->scx.rescue.curr);  in scx_rescue_flush()
597 	list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {  in scx_rescue_flush()
598 		scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);  in scx_rescue_flush()
600 						 &rq->scx.rescue.dsq, rq);  in scx_rescue_flush()
603 	timer_delete(&rq->scx.rescue.timer);  in scx_rescue_flush()
608 	struct task_struct *p = rq->scx.rescue.curr;  in scx_rescue_dump()
611 		      rq->scx.rescue.dsq.nr,  in scx_rescue_dump()
612 		      div_s64(rq->scx.rescue.budget, NSEC_PER_USEC),  in scx_rescue_dump()
672 	BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL));  in scx_rescue_init()
673 	timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED);  in scx_rescue_init()
674 	rq->scx.rescue.kill_at = get_jiffies_64();  in scx_rescue_init()
701 		return &rq->scx.local_dsq;  in scx_resolve_local_dsq()
704 	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;  in scx_resolve_local_dsq()
718 		return &rq->scx.local_dsq;  in scx_resolve_local_dsq()
730 		return &rq->scx.local_dsq;  in scx_resolve_local_dsq()
740 	p->scx.flags &= ~SCX_TASK_IMMED;  in scx_resolve_local_dsq()
746 			return &rq->scx.local_dsq;  in scx_resolve_local_dsq()
749 		p->scx.rescue_at = jiffies;  in scx_resolve_local_dsq()
750 		return &rq->scx.rescue.dsq;  in scx_resolve_local_dsq()
753 	p->scx.reenq_reason_caps = missing;  in scx_resolve_local_dsq()
754 	p->scx.reenq_reason_cid = cid;  in scx_resolve_local_dsq()
756 	return &rq->scx.reject_dsq;  in scx_resolve_local_dsq()
772 	p->scx.reenq_reason_caps = missing;  in scx_task_reenq_on_cap_revoke()
773 	p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));  in scx_task_reenq_on_cap_revoke()
792 	if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))  in scx_reenq_reject()
799 	list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {  in scx_reenq_reject()
806 		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))  in scx_reenq_reject()
807 			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;  in scx_reenq_reject()
808 		p->scx.flags |= SCX_TASK_REENQ_CAP;  in scx_reenq_reject()
810 		list_add_tail(&p->scx.dsq_list.node, &tasks);  in scx_reenq_reject()
813 	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {  in scx_reenq_reject()
814 		list_del_init(&p->scx.dsq_list.node);  in scx_reenq_reject()
818 		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;  in scx_reenq_reject()
936 	if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync))  in queue_sync_ecaps()
947 	llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))  in discard_queued_syncs()
973 	if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))  in scx_process_sync_ecaps()
993 	batch = llist_del_all(&rq->scx.ecaps_to_sync);  in scx_process_sync_ecaps()
1030 			rq->scx.sub_dispatch_prev = prev;  in scx_process_sync_ecaps()
1033 			rq->scx.sub_dispatch_prev = NULL;  in scx_process_sync_ecaps()
1045 			rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;  in scx_process_sync_ecaps()
1176 			llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {  in scx_discard_ecaps_to_sync()
1187 				llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);  in scx_discard_ecaps_to_sync()
1360 			.weight = tg->scx.weight,  in scx_cgroup_claim_subtree()
1361 			.bw_period_us = tg->scx.bw_period_us,  in scx_cgroup_claim_subtree()
1362 			.bw_quota_us = tg->scx.bw_quota_us,  in scx_cgroup_claim_subtree()
1363 			.bw_burst_us = tg->scx.bw_burst_us,  in scx_cgroup_claim_subtree()
1366 		if (tg->scx.sched != parent ||  in scx_cgroup_claim_subtree()
1377 		tg->scx.flags |= SCX_TG_SUB_INIT;  in scx_cgroup_claim_subtree()
1387 		if (!(tg->scx.flags & SCX_TG_SUB_INIT))  in scx_cgroup_claim_subtree()
1391 		if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(parent, cgroup_exit))  in scx_cgroup_claim_subtree()
1393 		tg->scx.sched = sch;  in scx_cgroup_claim_subtree()
1394 		tg->scx.flags |= SCX_TG_INITED;  in scx_cgroup_claim_subtree()
1395 		tg->scx.flags &= ~SCX_TG_SUB_INIT;  in scx_cgroup_claim_subtree()
1404 		if (!(tg->scx.flags & SCX_TG_SUB_INIT))  in scx_cgroup_claim_subtree()
1409 		tg->scx.flags &= ~SCX_TG_SUB_INIT;  in scx_cgroup_claim_subtree()
1449 		if (tg->scx.sched != sch ||  in scx_cgroup_return_subtree()
1454 		if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(sch, cgroup_exit))  in scx_cgroup_return_subtree()
1456 		tg->scx.sched = parent;  in scx_cgroup_return_subtree()
1457 		tg->scx.flags |= SCX_TG_SUB_INIT;  in scx_cgroup_return_subtree()
1463 			.weight = tg->scx.weight,  in scx_cgroup_return_subtree()
1464 			.bw_period_us = tg->scx.bw_period_us,  in scx_cgroup_return_subtree()
1465 			.bw_quota_us = tg->scx.bw_quota_us,  in scx_cgroup_return_subtree()
1466 			.bw_burst_us = tg->scx.bw_burst_us,  in scx_cgroup_return_subtree()
1470 		WARN_ON_ONCE(tg->scx.sched == sch);  in scx_cgroup_return_subtree()
1477 		if (!(tg->scx.flags & SCX_TG_SUB_INIT))  in scx_cgroup_return_subtree()
1479 		tg->scx.flags &= ~(SCX_TG_SUB_INIT | SCX_TG_INITED);  in scx_cgroup_return_subtree()
1498 		tg->scx.flags |= SCX_TG_INITED;  in scx_cgroup_return_subtree()
1885 		if (p->scx.flags & SCX_TASK_SUB_INIT)  in scx_sub_enable_workfn()
1922 		p->scx.flags |= SCX_TASK_SUB_INIT;  in scx_sub_enable_workfn()
1939 		if (!(p->scx.flags & SCX_TASK_SUB_INIT))  in scx_sub_enable_workfn()
1963 			p->scx.flags &= ~SCX_TASK_SUB_INIT;  in scx_sub_enable_workfn()
2003 		if (p->scx.flags & SCX_TASK_SUB_INIT) {  in scx_sub_enable_workfn()
2005 			p->scx.flags &= ~SCX_TASK_SUB_INIT;  in scx_sub_enable_workfn()
2254 	return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) !=  in scx_bpf_sub_dispatch()