Lines Matching refs:enq_flags
347 * @enq_flags: optional %SCX_ENQ_* of the task being enqueued
354 static bool rq_is_open(struct rq *rq, u64 enq_flags)
390 if (enq_flags & SCX_ENQ_PREEMPT) {
1381 static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags)
1389 if (!(enq_flags & SCX_ENQ_SLICE_DFL))
1395 if (enq_flags & SCX_ENQ_DSQ_PRIQ)
1431 static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags)
1444 if (enq_flags & SCX_ENQ_IMMED) {
1446 WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
1464 if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags)))
1522 u64 enq_flags)
1583 if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1594 u64 slice, u64 vtime, u64 enq_flags)
1599 dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
1609 (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0);
1621 (enq_flags & SCX_ENQ_DSQ_PRIQ))) {
1630 enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
1634 * @dsq is locked and @enq_flags is sanitized. Commit the carried slice
1637 if (enq_flags & SCX_ENQ_APPLY_SLICE)
1638 apply_slice_vtime(p, slice, vtime, enq_flags);
1640 if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
1678 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) {
1698 dsq_inc_nr(dsq, p, enq_flags);
1709 rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
1739 if (enq_flags & SCX_ENQ_CLEAR_OPSS)
1869 u64 slice, u64 vtime, u64 enq_flags)
1896 p->scx.ddsp_enq_flags = enq_flags;
1921 u64 enq_flags)
1928 p->scx.ddsp_enq_flags |= enq_flags;
1986 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
2014 if (enq_flags & SCX_ENQ_REENQ) {
2046 enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */
2076 SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
2096 direct_dispatch(sch, p, enq_flags);
2099 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
2114 scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
2159 u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
2183 if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
2185 enq_flags |= SCX_ENQ_IGNORE_CAPS;
2199 SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
2205 scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
2209 if ((enq_flags & SCX_ENQ_CPU_SELECTED) &&
2404 u64 enq_flags, struct rq *dst_rq)
2406 struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
2412 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
2417 dsq_inc_nr(dst_dsq, p, enq_flags);
2420 rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags);
2427 * @enq_flags: %SCX_ENQ_*
2434 struct task_struct *p, u64 enq_flags,
2450 * activate_task() below truncates enq_flags to 32 bits and re-derives
2458 dst_rq->scx.remote_activate_enq_flags = enq_flags;
2595 struct task_struct *p, u64 enq_flags,
2599 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
2611 * @enq_flags: %SCX_ENQ_*
2624 struct task_struct *p, u64 enq_flags,
2640 enq_flags |= SCX_ENQ_GDSQ_FALLBACK;
2656 scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq);
2659 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
2669 scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags);
2676 struct scx_dispatch_q *dsq, u64 enq_flags)
2707 scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq);
2712 if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
2737 * @enq_flags: %SCX_ENQ_*
2748 u64 slice, u64 vtime, u64 enq_flags)
2762 enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
2800 slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
2806 enq_flags | SCX_ENQ_APPLY_SLICE |
2809 apply_slice_vtime(p, slice, vtime, enq_flags);
2810 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
2834 * @enq_flags: %SCX_ENQ_*
2849 u64 slice, u64 vtime, u64 enq_flags)
2908 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
2911 enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
2923 ent->slice, ent->vtime, ent->enq_flags);
3192 u64 enq_flags = 0;
3200 enq_flags |= SCX_ENQ_IGNORE_CAPS;
3202 enq_flags |= SCX_ENQ_HEAD;
3204 enq_flags |= SCX_ENQ_HEAD;
3208 enq_flags);
4407 u64 enq_flags = p->scx.ddsp_enq_flags;
4416 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
8337 static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {}
8338 static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {}
8341 static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {}
8837 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
8842 if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
8843 scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
8847 if (*enq_flags & SCX_ENQ_IMMED) {
8853 *enq_flags |= SCX_ENQ_IMMED;
8856 if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) {
8865 u64 dsq_id, u64 *enq_flags)
8880 if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))
8887 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8894 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
8909 .enq_flags = enq_flags,
8920 * @enq_flags: SCX_ENQ_*
8930 * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p
8934 * @enq_flags are OR'd with the enqueue flags on the enqueue path before the
8956 u64 slice, u64 enq_flags,
8966 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
8969 scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
8978 u64 slice, u64 enq_flags,
8981 scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
8985 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8987 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
8990 scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);
9000 u64 enq_flags;
9010 * @args->enq_flags: SCX_ENQ_*
9048 args->vtime, args->enq_flags);
9055 u64 slice, u64 vtime, u64 enq_flags)
9083 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
9102 struct task_struct *p, u64 dsq_id, u64 enq_flags,
9121 if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
9124 /* internal bit, can only go in after @enq_flags is vetted */
9126 enq_flags |= SCX_ENQ_DSQ_PRIQ;
9191 p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
9258 * @enq_flags: %SCX_ENQ_*
9261 * local DSQ for execution with @enq_flags applied. Can only be called from
9279 __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
9292 if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags))
9305 if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
9369 * @enq_flags: SCX_ENQ_*
9391 u64 enq_flags)
9394 p, dsq_id, enq_flags, false);
9402 * @enq_flags: SCX_ENQ_*
9416 u64 enq_flags)
9419 p, dsq_id, enq_flags, true);