Lines Matching defs:dsq_id
310 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)
312 return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);
1821 struct rq *rq, u64 dsq_id,
1826 if (dsq_id == SCX_DSQ_LOCAL)
1829 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
1830 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
1838 if (dsq_id == SCX_DSQ_GLOBAL)
1841 dsq = find_user_dsq(sch, dsq_id);
1848 scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
1857 struct task_struct *p, u64 dsq_id,
1884 p->scx.ddsp_dsq_id = dsq_id;
2808 * @dsq_id: destination DSQ ID
2825 unsigned long qseq_at_dispatch, u64 dsq_id,
2882 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
2899 finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,
4341 u64 dsq_id = p->scx.ddsp_dsq_id;
4349 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
4612 u64 dsq_id, reenq_flags;
4636 dsq_id = READ_ONCE(dsq->id);
4637 if (unlikely(dsq_id == SCX_DSQ_INVALID))
4640 BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN);
5061 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)
5069 dsq->id = dsq_id;
5127 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)
5134 dsq = find_user_dsq(sch, dsq_id);
6847 scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s",
8758 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
8760 bool is_local = dsq_id == SCX_DSQ_LOCAL ||
8761 (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
8770 scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);
8778 scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id);
8786 u64 dsq_id, u64 *enq_flags)
8801 if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))
8808 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8815 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
8827 .dsq_id = dsq_id,
8839 * @dsq_id: DSQ to insert into
8844 * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to
8851 * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p
8858 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
8876 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
8887 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
8890 scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
8898 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id,
8902 scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
8906 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8908 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
8911 scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);
8918 u64 dsq_id;
8928 * @args->dsq_id: DSQ to insert into
8939 * @args->dsq_id. Tasks queued into the priority queue are ordered by
8968 return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,
8975 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
9004 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
9023 struct task_struct *p, u64 dsq_id, u64 enq_flags,
9042 if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
9099 dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p));
9177 * @dsq_id: DSQ to move task from. Must be a user-created DSQ
9181 * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's
9200 __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
9220 dsq = find_user_dsq(sch, dsq_id);
9222 scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id);
9243 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)
9245 return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);
9289 * @dsq_id: DSQ to move @p to
9293 * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can
9307 * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local
9311 struct task_struct *p, u64 dsq_id,
9315 p, dsq_id, enq_flags, false);
9322 * @dsq_id: DSQ to move @p to
9326 * priority queue of the DSQ specified by @dsq_id. The destination must be a
9336 struct task_struct *p, u64 dsq_id,
9340 p, dsq_id, enq_flags, true);
9408 * @dsq_id: DSQ to create
9412 * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable
9415 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux)
9425 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN))
9436 ret = scx_init_dsq(dsq, dsq_id, NULL);
9688 * @dsq_id: id of the DSQ
9691 * Return the number of tasks in the DSQ matching @dsq_id. If not found,
9698 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
9712 if (dsq_id == SCX_DSQ_LOCAL) {
9715 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
9716 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
9723 dsq = find_user_dsq(sch, dsq_id);
9737 * @dsq_id: DSQ to destroy
9740 * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
9745 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
9752 destroy_dsq(sch, dsq_id);
9758 * @dsq_id: DSQ to iterate
9763 * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
9766 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
9792 kit->dsq = find_user_dsq(sch, dsq_id);
9844 * @dsq_id: DSQ to examine.
9854 __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
9864 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) {
9865 scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id);
9869 dsq = find_user_dsq(sch, dsq_id);
9871 scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id);
9880 * @dsq_id: DSQ to re-enqueue
9885 * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are
9897 __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
9919 dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id());