310 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)  in find_user_dsq()  argument
312 	return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);  in find_user_dsq()
1821 						    struct rq *rq, u64 dsq_id,  in find_dsq_for_dispatch()  argument
1826 	if (dsq_id == SCX_DSQ_LOCAL)  in find_dsq_for_dispatch()
1829 	if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {  in find_dsq_for_dispatch()
1830 		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);  in find_dsq_for_dispatch()
1838 	if (dsq_id == SCX_DSQ_GLOBAL)  in find_dsq_for_dispatch()
1841 		dsq = find_user_dsq(sch, dsq_id);  in find_dsq_for_dispatch()
1848 		scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);  in find_dsq_for_dispatch()
1857 				 struct task_struct *p, u64 dsq_id,  in mark_direct_dispatch()  argument
1884 	p->scx.ddsp_dsq_id = dsq_id;  in mark_direct_dispatch()
2825 			    unsigned long qseq_at_dispatch, u64 dsq_id,  in finish_dispatch()  argument
2882 	dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));  in finish_dispatch()
2899 		finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,  in scx_flush_dispatch_buf()
4324 		u64 dsq_id = p->scx.ddsp_dsq_id;  in process_ddsp_deferred_locals()  local
4332 		dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));  in process_ddsp_deferred_locals()
4595 		u64 dsq_id, reenq_flags;  in process_deferred_reenq_users()  local
4619 		dsq_id = READ_ONCE(dsq->id);  in process_deferred_reenq_users()
4620 		if (unlikely(dsq_id == SCX_DSQ_INVALID))  in process_deferred_reenq_users()
4623 		BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN);  in process_deferred_reenq_users()
5042 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)  in scx_init_dsq()  argument
5050 	dsq->id = dsq_id;  in scx_init_dsq()
5108 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)  in destroy_dsq()  argument
5115 	dsq = find_user_dsq(sch, dsq_id);  in destroy_dsq()
8718 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)  in scx_vet_enq_flags()  argument
8720 	bool is_local = dsq_id == SCX_DSQ_LOCAL ||  in scx_vet_enq_flags()
8721 		(dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;  in scx_vet_enq_flags()
8730 			scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);  in scx_vet_enq_flags()
8738 		scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id);  in scx_vet_enq_flags()
8746 				    u64 dsq_id, u64 *enq_flags)  in scx_dsq_insert_preamble()  argument
8761 	if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))  in scx_dsq_insert_preamble()
8768 				  u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)  in scx_dsq_insert_commit()  argument
8775 		mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);  in scx_dsq_insert_commit()
8787 		.dsq_id = dsq_id,  in scx_dsq_insert_commit()
8836 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,  in scx_bpf_dsq_insert___v2()  argument
8847 	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))  in scx_bpf_dsq_insert___v2()
8850 	scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);  in scx_bpf_dsq_insert___v2()
8858 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id,  in scx_bpf_dsq_insert()  argument
8862 	scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);  in scx_bpf_dsq_insert()
8866 				 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)  in scx_dsq_insert_vtime()  argument
8868 	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))  in scx_dsq_insert_vtime()
8871 	scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);  in scx_dsq_insert_vtime()
8878 	u64			dsq_id;  member
8928 	return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,  in __scx_bpf_dsq_insert_vtime()
8935 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,  in scx_bpf_dsq_insert_vtime()  argument
8957 	scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);  in scx_bpf_dsq_insert_vtime()
8976 			 struct task_struct *p, u64 dsq_id, u64 enq_flags,  in scx_dsq_move()  argument
8995 	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))  in scx_dsq_move()
9052 	dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p));  in scx_dsq_move()
9153 __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,  in scx_bpf_dsq_move_to_local___v2()  argument
9173 	dsq = find_user_dsq(sch, dsq_id);  in scx_bpf_dsq_move_to_local___v2()
9175 		scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id);  in scx_bpf_dsq_move_to_local___v2()
9196 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)  in scx_bpf_dsq_move_to_local()  argument
9198 	return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);  in scx_bpf_dsq_move_to_local()
9264 				  struct task_struct *p, u64 dsq_id,  in scx_bpf_dsq_move()  argument
9268 			    p, dsq_id, enq_flags, false);  in scx_bpf_dsq_move()
9289 					struct task_struct *p, u64 dsq_id,  in scx_bpf_dsq_move_vtime()  argument
9293 			    p, dsq_id, enq_flags, true);  in scx_bpf_dsq_move_vtime()
9368 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux)  in scx_bpf_create_dsq()  argument
9378 	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN))  in scx_bpf_create_dsq()
9389 	ret = scx_init_dsq(dsq, dsq_id, NULL);  in scx_bpf_create_dsq()
9651 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)  in scx_bpf_dsq_nr_queued()  argument
9665 	if (dsq_id == SCX_DSQ_LOCAL) {  in scx_bpf_dsq_nr_queued()
9668 	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {  in scx_bpf_dsq_nr_queued()
9669 		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);  in scx_bpf_dsq_nr_queued()
9676 		dsq = find_user_dsq(sch, dsq_id);  in scx_bpf_dsq_nr_queued()
9698 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)  in scx_bpf_destroy_dsq()  argument
9705 		destroy_dsq(sch, dsq_id);  in scx_bpf_destroy_dsq()
9719 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,  in bpf_iter_scx_dsq_new()  argument
9745 	kit->dsq = find_user_dsq(sch, dsq_id);  in bpf_iter_scx_dsq_new()
9807 __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,  in scx_bpf_dsq_peek()  argument
9817 	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) {  in scx_bpf_dsq_peek()
9818 		scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id);  in scx_bpf_dsq_peek()
9822 	dsq = find_user_dsq(sch, dsq_id);  in scx_bpf_dsq_peek()
9824 		scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id);  in scx_bpf_dsq_peek()
9850 __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,  in scx_bpf_dsq_reenq()  argument
9872 	dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id());  in scx_bpf_dsq_reenq()