Lines Matching refs:scx
382 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) in rq_is_open()
398 likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); in rq_is_open()
426 rq->scx.lock_drop_seq++; in scx_rq_lock_drop()
513 list_node = &cur->scx.dsq_list.node; in nldsq_next_task()
531 return container_of(dsq_lnode, struct task_struct, scx.dsq_list); in nldsq_next_task()
563 p = container_of(cursor, struct task_struct, scx.dsq_list); in nldsq_cursor_next_task()
568 } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); in nldsq_cursor_next_task()
572 list_move_tail(&cursor->node, &p->scx.dsq_list.node); in nldsq_cursor_next_task()
574 list_move(&cursor->node, &p->scx.dsq_list.node); in nldsq_cursor_next_task()
606 if (unlikely(p->scx.dsq != dsq || in nldsq_cursor_lost_task()
607 u32_before(cursor->priv, p->scx.dsq_seq) || in nldsq_cursor_lost_task()
608 p->scx.holding_cpu >= 0)) in nldsq_cursor_lost_task()
638 return p->scx.flags & SCX_TASK_STATE_MASK; in scx_get_task_state()
655 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; in scx_set_task_state()
677 p->scx.flags &= ~SCX_TASK_STATE_MASK; in scx_set_task_state()
678 p->scx.flags |= state; in scx_set_task_state()
850 return container_of(pos, struct task_struct, scx); in scx_task_iter_next()
967 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); in wait_ops_state()
1002 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); in deferred_irq_workfn()
1032 irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); in schedule_deferred()
1051 if (rq->scx.flags & SCX_RQ_IN_WAKEUP) in schedule_deferred_locked()
1055 if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) in schedule_deferred_locked()
1069 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) { in schedule_deferred_locked()
1070 rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; in schedule_deferred_locked()
1095 rq = container_of(dsq, struct rq, scx.local_dsq); in schedule_dsq_reenq()
1121 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); in schedule_dsq_reenq()
1124 list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); in schedule_dsq_reenq()
1142 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); in schedule_dsq_reenq()
1145 list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); in schedule_dsq_reenq()
1217 if (unlikely(atomic64_read(&p->scx.slice_oob))) in clear_task_slice_oob()
1218 atomic64_set(&p->scx.slice_oob, 0); in clear_task_slice_oob()
1239 list_add(&p->scx.dsq_list.node, &dsq->list); in dsq_insert_head()
1249 q = container_of(node, struct task_struct, scx.dsq_list); in dsq_insert_head()
1250 if (!(q->scx.flags & SCX_TASK_PROTECTED)) in dsq_insert_head()
1256 list_add(&p->scx.dsq_list.node, pos); in dsq_insert_head()
1273 if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) in set_task_slice_keep_oob()
1276 p->scx.slice = slice; in set_task_slice_keep_oob()
1307 p->scx.flags &= ~SCX_TASK_PROTECTED; in scx_task_slice_ended()
1326 atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING | in set_task_slice_oob()
1343 if (likely(!atomic64_read(&p->scx.slice_oob))) in apply_task_slice_oob()
1346 oob = atomic64_xchg(&p->scx.slice_oob, 0); in apply_task_slice_oob()
1358 if (slice > p->scx.slice && in apply_task_slice_oob()
1376 p->scx.slice = slice; in apply_slice_vtime()
1383 } else if (!p->scx.slice) { in apply_slice_vtime()
1384 p->scx.slice = 1; in apply_slice_vtime()
1388 p->scx.dsq_vtime = vtime; in apply_slice_vtime()
1403 if (curr->scx.slice != SCX_SLICE_INF) in update_curr_scx()
1404 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); in update_curr_scx()
1416 container_of(node_a, struct task_struct, scx.dsq_priq); in scx_dsq_priq_less()
1418 container_of(node_b, struct task_struct, scx.dsq_priq); in scx_dsq_priq_less()
1420 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); in scx_dsq_priq_less()
1441 p->scx.flags |= SCX_TASK_IMMED; in dsq_inc_nr()
1444 if (p->scx.flags & SCX_TASK_IMMED) { in dsq_inc_nr()
1445 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dsq_inc_nr()
1450 rq->scx.nr_immed++; in dsq_inc_nr()
1466 if (p->scx.flags & SCX_TASK_IMMED) { in dsq_dec_nr()
1467 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dsq_dec_nr()
1470 WARN_ON_ONCE(rq->scx.nr_immed <= 0)) in dsq_dec_nr()
1473 rq->scx.nr_immed--; in dsq_dec_nr()
1499 return p->scx.sticky_cpu >= 0; in task_scx_migrating()
1509 if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) in call_task_dequeue()
1515 p->scx.flags &= ~SCX_TASK_IN_CUSTODY; in call_task_dequeue()
1577 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) in rq_owned_post_enq()
1600 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in scx_dispatch_enqueue()
1601 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || in scx_dispatch_enqueue()
1602 !RB_EMPTY_NODE(&p->scx.dsq_priq)); in scx_dispatch_enqueue()
1650 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; in scx_dispatch_enqueue()
1651 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); in scx_dispatch_enqueue()
1657 rbp = rb_prev(&p->scx.dsq_priq); in scx_dispatch_enqueue()
1661 scx.dsq_priq); in scx_dispatch_enqueue()
1662 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); in scx_dispatch_enqueue()
1665 list_add(&p->scx.dsq_list.node, &dsq->list); in scx_dispatch_enqueue()
1685 list_add_tail(&p->scx.dsq_list.node, &dsq->list); in scx_dispatch_enqueue()
1693 p->scx.dsq_seq = dsq->seq; in scx_dispatch_enqueue()
1696 p->scx.dsq = dsq; in scx_dispatch_enqueue()
1719 p->scx.flags |= SCX_TASK_IN_CUSTODY; in scx_dispatch_enqueue()
1729 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in scx_dispatch_enqueue()
1734 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); in scx_task_unlink_from_dsq()
1736 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { in scx_task_unlink_from_dsq()
1737 rb_erase(&p->scx.dsq_priq, &dsq->priq); in scx_task_unlink_from_dsq()
1738 RB_CLEAR_NODE(&p->scx.dsq_priq); in scx_task_unlink_from_dsq()
1739 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; in scx_task_unlink_from_dsq()
1742 list_del_init(&p->scx.dsq_list.node); in scx_task_unlink_from_dsq()
1755 struct scx_dispatch_q *dsq = p->scx.dsq; in scx_dispatch_dequeue()
1765 if (unlikely(!list_empty(&p->scx.dsq_list.node))) in scx_dispatch_dequeue()
1766 list_del_init(&p->scx.dsq_list.node); in scx_dispatch_dequeue()
1774 if (p->scx.holding_cpu >= 0) in scx_dispatch_dequeue()
1775 p->scx.holding_cpu = -1; in scx_dispatch_dequeue()
1787 if (p->scx.holding_cpu < 0) { in scx_dispatch_dequeue()
1797 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); in scx_dispatch_dequeue()
1798 p->scx.holding_cpu = -1; in scx_dispatch_dequeue()
1800 p->scx.dsq = NULL; in scx_dispatch_dequeue()
1817 p->scx.dsq = NULL; in dispatch_dequeue_locked()
1827 return &rq->scx.local_dsq; in find_dsq_for_dispatch()
1835 return &cpu_rq(cpu)->scx.local_dsq; in find_dsq_for_dispatch()
1879 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); in mark_direct_dispatch()
1880 WARN_ON_ONCE(p->scx.ddsp_enq_flags); in mark_direct_dispatch()
1882 p->scx.ddsp_slice = slice; in mark_direct_dispatch()
1883 p->scx.ddsp_vtime = vtime; in mark_direct_dispatch()
1884 p->scx.ddsp_dsq_id = dsq_id; in mark_direct_dispatch()
1885 p->scx.ddsp_enq_flags = enq_flags; in mark_direct_dispatch()
1905 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; in clear_direct_dispatch()
1906 p->scx.ddsp_enq_flags = 0; in clear_direct_dispatch()
1914 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); in direct_dispatch()
1917 p->scx.ddsp_enq_flags |= enq_flags; in direct_dispatch()
1925 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { in direct_dispatch()
1928 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; in direct_dispatch()
1938 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in direct_dispatch()
1943 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in direct_dispatch()
1947 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in direct_dispatch()
1948 list_add_tail(&p->scx.dsq_list.node, in direct_dispatch()
1949 &rq->scx.ddsp_deferred_locals); in direct_dispatch()
1954 ddsp_enq_flags = p->scx.ddsp_enq_flags; in direct_dispatch()
1955 slice = p->scx.ddsp_slice; in direct_dispatch()
1956 vtime = p->scx.ddsp_vtime; in direct_dispatch()
1972 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); in scx_rq_online()
1983 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); in scx_do_enqueue_task()
1995 p->scx.flags &= ~SCX_TASK_IMMED; in scx_do_enqueue_task()
2004 if (++p->scx.reenq_cnt > 1) in scx_do_enqueue_task()
2007 if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) { in scx_do_enqueue_task()
2010 p->comm, p->pid, p->scx.reenq_cnt); in scx_do_enqueue_task()
2028 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) in scx_do_enqueue_task()
2050 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; in scx_do_enqueue_task()
2052 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); in scx_do_enqueue_task()
2053 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); in scx_do_enqueue_task()
2062 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) in scx_do_enqueue_task()
2069 p->scx.flags |= SCX_TASK_IN_CUSTODY; in scx_do_enqueue_task()
2075 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); in scx_do_enqueue_task()
2082 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); in scx_do_enqueue_task()
2085 dsq = &rq->scx.local_dsq; in scx_do_enqueue_task()
2102 return !list_empty(&p->scx.runnable_node); in task_runnable()
2109 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { in set_task_runnable()
2110 p->scx.runnable_at = jiffies; in set_task_runnable()
2111 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; in set_task_runnable()
2118 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); in set_task_runnable()
2125 WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq)); in set_task_runnable()
2130 list_del_init(&p->scx.runnable_node); in clr_task_runnable()
2131 WRITE_ONCE(p->scx.runnable_cpu, -1); in clr_task_runnable()
2133 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; in clr_task_runnable()
2134 p->scx.reenq_cnt = 0; in clr_task_runnable()
2141 int sticky_cpu = p->scx.sticky_cpu; in enqueue_task_scx()
2142 u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; in enqueue_task_scx()
2145 rq->scx.flags |= SCX_RQ_IN_WAKEUP; in enqueue_task_scx()
2159 if (p->scx.flags & SCX_TASK_QUEUED) { in enqueue_task_scx()
2165 p->scx.flags |= SCX_TASK_QUEUED; in enqueue_task_scx()
2166 rq->scx.nr_running++; in enqueue_task_scx()
2173 if (rq->scx.nr_running == 1) in enqueue_task_scx()
2179 p->scx.sticky_cpu = -1; in enqueue_task_scx()
2181 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; in enqueue_task_scx()
2184 unlikely(cpu_of(rq) != p->scx.selected_cpu)) in enqueue_task_scx()
2198 opss = atomic_long_read_acquire(&p->scx.ops_state); in ops_dequeue()
2219 if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { in ops_dequeue()
2224 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, in ops_dequeue()
2243 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); in ops_dequeue()
2276 if (!(p->scx.flags & SCX_TASK_QUEUED)) { in dequeue_task_scx()
2306 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; in dequeue_task_scx()
2308 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; in dequeue_task_scx()
2310 p->scx.flags &= ~SCX_TASK_QUEUED; in dequeue_task_scx()
2311 rq->scx.nr_running--; in dequeue_task_scx()
2372 if (rq->scx.nr_immed) in wakeup_preempt_scx()
2387 WARN_ON_ONCE(p->scx.holding_cpu >= 0); in scx_move_local_task_to_local_dsq()
2392 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); in scx_move_local_task_to_local_dsq()
2395 p->scx.dsq = dst_dsq; in scx_move_local_task_to_local_dsq()
2420 p->scx.sticky_cpu = cpu_of(dst_rq); in move_remote_task_to_local_dsq()
2433 WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || in move_remote_task_to_local_dsq()
2434 dst_rq->scx.remote_activate_sch); in move_remote_task_to_local_dsq()
2435 dst_rq->scx.remote_activate_enq_flags = enq_flags; in move_remote_task_to_local_dsq()
2436 dst_rq->scx.remote_activate_sch = sch; in move_remote_task_to_local_dsq()
2438 dst_rq->scx.remote_activate_enq_flags = 0; in move_remote_task_to_local_dsq()
2439 dst_rq->scx.remote_activate_sch = NULL; in move_remote_task_to_local_dsq()
2559 WARN_ON_ONCE(p->scx.holding_cpu >= 0); in unlink_dsq_and_switch_rq_lock()
2561 p->scx.holding_cpu = cpu; in unlink_dsq_and_switch_rq_lock()
2567 return likely(p->scx.holding_cpu == cpu) && in unlink_dsq_and_switch_rq_lock()
2612 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); in move_task_between_dsqs()
2728 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); in dispatch_to_local_dsq()
2754 p->scx.holding_cpu = raw_smp_processor_id(); in dispatch_to_local_dsq()
2757 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in dispatch_to_local_dsq()
2766 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && in dispatch_to_local_dsq()
2775 p->scx.holding_cpu = -1; in dispatch_to_local_dsq()
2776 scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, in dispatch_to_local_dsq()
2779 p->scx.holding_cpu = -1; in dispatch_to_local_dsq()
2834 opss = atomic_long_read(&p->scx.ops_state); in finish_dispatch()
2863 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, in finish_dispatch()
2878 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); in finish_dispatch()
2909 if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) in maybe_queue_balance_callback()
2912 queue_balance_callback(rq, &rq->scx.deferred_bal_cb, in maybe_queue_balance_callback()
2915 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; in maybe_queue_balance_callback()
2925 rq->scx.flags |= SCX_RQ_IN_DISPATCH; in dispatch_one()
2930 unlikely(rq->scx.cpu_released)) { in dispatch_one()
2939 rq->scx.cpu_released = false; in dispatch_one()
2955 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && in dispatch_one()
2963 if (rq->scx.local_dsq.nr) { in dispatch_one()
2976 if ((prev->scx.flags & SCX_TASK_QUEUED) && in dispatch_one()
2983 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; in dispatch_one()
2997 if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) in dispatch_one()
3000 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; in dispatch_one()
3008 if (p->scx.flags & SCX_TASK_QUEUED) { in set_next_task_scx()
3020 if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) in set_next_task_scx()
3037 if (p->scx.slice == SCX_SLICE_INF) { in set_next_task_scx()
3038 if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { in set_next_task_scx()
3045 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; in set_next_task_scx()
3051 if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { in set_next_task_scx()
3052 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; in set_next_task_scx()
3103 if (!rq->scx.cpu_released) { in switch_class()
3112 rq->scx.cpu_released = true; in switch_class()
3123 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); in put_prev_task_scx()
3131 if (!p->scx.slice) { in put_prev_task_scx()
3139 if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) in put_prev_task_scx()
3142 if (p->scx.flags & SCX_TASK_QUEUED) { in put_prev_task_scx()
3155 if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && in put_prev_task_scx()
3157 if (p->scx.flags & SCX_TASK_IMMED) { in put_prev_task_scx()
3158 p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; in put_prev_task_scx()
3160 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; in put_prev_task_scx()
3177 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, in put_prev_task_scx()
3242 for_each_cpu(cpu, rq->scx.cpus_to_sync) { in kick_sync_wait_bal_cb()
3248 smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { in kick_sync_wait_bal_cb()
3249 cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); in kick_sync_wait_bal_cb()
3255 while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { in kick_sync_wait_bal_cb()
3256 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); in kick_sync_wait_bal_cb()
3269 return list_first_entry_or_null(&rq->scx.local_dsq.list, in first_local_task()
3270 struct task_struct, scx.dsq_list.node); in first_local_task()
3291 if (unlikely(rq->scx.kick_sync_pending)) { in dispatch_pick()
3292 rq->scx.kick_sync_pending = false; in dispatch_pick()
3293 queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, in dispatch_pick()
3312 u32 seq = rq->scx.lock_drop_seq; in dispatch_core_pick()
3315 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) in dispatch_core_pick()
3326 if (unlikely(rq->scx.kick_sync_pending)) { in dispatch_core_pick()
3327 rq->scx.kick_sync_pending = false; in dispatch_core_pick()
3328 queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, in dispatch_core_pick()
3331 } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { in dispatch_core_pick()
3337 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; in dispatch_core_pick()
3344 if (rq->scx.lock_drop_seq != seq) in dispatch_core_pick()
3365 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); in do_pick_task_scx()
3394 if (!p->scx.slice) { in do_pick_task_scx()
3404 if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { in do_pick_task_scx()
3526 return time_after(a->scx.runnable_at, b->scx.runnable_at); in scx_prio_less()
3557 this_rq()->scx.in_select_cpu = true; in select_task_rq_scx()
3561 this_rq()->scx.in_select_cpu = false; in select_task_rq_scx()
3562 p->scx.selected_cpu = cpu; in select_task_rq_scx()
3580 p->scx.selected_cpu = prev_cpu; in select_task_rq_scx()
3591 p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); in select_task_rq_scx()
3592 p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL; in select_task_rq_scx()
3593 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; in select_task_rq_scx()
3597 p->scx.selected_cpu = cpu; in select_task_rq_scx()
3693 rq->scx.flags |= SCX_RQ_ONLINE; in rq_online_scx()
3698 rq->scx.flags &= ~SCX_RQ_ONLINE; in rq_offline_scx()
3714 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { in check_rq_for_timeouts()
3716 unsigned long last_runnable = p->scx.runnable_at; in check_rq_for_timeouts()
3720 struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); in check_rq_for_timeouts()
3805 if (!curr->scx.slice) in task_tick_scx()
3848 p->scx.disallow = false; in __scx_init_task()
3863 if (p->scx.disallow) { in __scx_init_task()
3910 WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); in __scx_enable_task()
3921 p->scx.weight = sched_weight_to_cgroup(weight); in __scx_enable_task()
3927 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); in __scx_enable_task()
3953 p->scx.dsq_vtime = 0; in scx_disable_task()
3956 p->scx.reenq_cnt = 0; in scx_disable_task()
3963 WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); in scx_disable_task()
4023 if (p->scx.flags & SCX_TASK_SUB_INIT) { in scx_disable_and_exit_task()
4026 p->scx.flags &= ~SCX_TASK_SUB_INIT; in scx_disable_and_exit_task()
4033 void init_scx_entity(struct sched_ext_entity *scx) in init_scx_entity() argument
4035 memset(scx, 0, sizeof(*scx)); in init_scx_entity()
4036 INIT_LIST_HEAD(&scx->dsq_list.node); in init_scx_entity()
4037 RB_CLEAR_NODE(&scx->dsq_priq); in init_scx_entity()
4038 scx->sticky_cpu = -1; in init_scx_entity()
4039 scx->holding_cpu = -1; in init_scx_entity()
4040 scx->runnable_cpu = -1; in init_scx_entity()
4041 INIT_LIST_HEAD(&scx->runnable_node); in init_scx_entity()
4042 scx->runnable_at = jiffies; in init_scx_entity()
4043 scx->ddsp_dsq_id = SCX_DSQ_INVALID; in init_scx_entity()
4044 scx->slice = SCX_SLICE_DFL; in init_scx_entity()
4069 &p->scx.tid_hash_node, in scx_tid_hash_insert()
4091 p->scx.tid = scx_alloc_tid(); in scx_fork()
4133 list_add_tail(&p->scx.tasks_node, &scx_tasks); in scx_post_fork()
4194 list_del_init(&p->scx.tasks_node); in sched_ext_dead()
4197 &p->scx.tid_hash_node, in sched_ext_dead()
4247 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); in reweight_task_scx()
4249 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); in reweight_task_scx()
4298 if (scx_enabled() && READ_ONCE(p->scx.disallow) && in scx_check_setscheduler()
4318 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, in process_ddsp_deferred_locals()
4319 struct task_struct, scx.dsq_list.node))) { in process_ddsp_deferred_locals()
4322 u64 dsq_id = p->scx.ddsp_dsq_id; in process_ddsp_deferred_locals()
4323 u64 enq_flags = p->scx.ddsp_enq_flags; in process_ddsp_deferred_locals()
4324 u64 slice = p->scx.ddsp_slice; in process_ddsp_deferred_locals()
4325 u64 vtime = p->scx.ddsp_vtime; in process_ddsp_deferred_locals()
4327 list_del_init(&p->scx.dsq_list.node); in process_ddsp_deferred_locals()
4367 if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) in local_task_should_reenq()
4372 if ((p->scx.flags & SCX_TASK_IMMED) && in local_task_should_reenq()
4406 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, in reenq_local()
4407 scx.dsq_list.node) { in reenq_local()
4433 if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) in reenq_local()
4434 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; in reenq_local()
4435 p->scx.flags |= reason; in reenq_local()
4437 list_add_tail(&p->scx.dsq_list.node, &tasks); in reenq_local()
4440 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { in reenq_local()
4441 list_del_init(&p->scx.dsq_list.node); in reenq_local()
4445 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; in reenq_local()
4479 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { in process_deferred_reenq_locals()
4481 list_first_entry_or_null(&rq->scx.deferred_reenq_locals, in process_deferred_reenq_locals()
4557 if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) in reenq_user()
4558 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; in reenq_user()
4559 p->scx.flags |= reason; in reenq_user()
4563 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; in reenq_user()
4595 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { in process_deferred_reenq_users()
4597 list_first_entry_or_null(&rq->scx.deferred_reenq_users, in process_deferred_reenq_users()
4630 if (!list_empty(&rq->scx.deferred_reenq_locals)) in run_deferred()
4633 if (!list_empty(&rq->scx.deferred_reenq_users)) in run_deferred()
4654 if (!rq->scx.nr_running) in scx_can_stop_tick()
4672 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; in scx_can_stop_tick()
4682 tg->scx.weight = CGROUP_WEIGHT_DFL; in scx_tg_init()
4683 tg->scx.bw_period_us = default_bw_period_us(); in scx_tg_init()
4684 tg->scx.bw_quota_us = RUNTIME_INF; in scx_tg_init()
4685 tg->scx.idle = false; in scx_tg_init()
4713 return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL; in scx_tg_sched()
4746 WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); in scx_tg_online()
4764 { .weight = tg->scx.weight, in scx_tg_online()
4765 .bw_period_us = tg->scx.bw_period_us, in scx_tg_online()
4766 .bw_quota_us = tg->scx.bw_quota_us, in scx_tg_online()
4767 .bw_burst_us = tg->scx.bw_burst_us }; in scx_tg_online()
4775 tg->scx.sched = sch; in scx_tg_online()
4776 tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; in scx_tg_online()
4779 tg->scx.flags |= SCX_TG_ONLINE; in scx_tg_online()
4787 struct scx_sched *sch = tg->scx.sched; in scx_tg_offline()
4789 WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); in scx_tg_offline()
4792 if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) && in scx_tg_offline()
4795 tg->scx.sched = NULL; in scx_tg_offline()
4796 tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); in scx_tg_offline()
4806 return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex)); in scx_cgroup_task_sched()
4823 WARN_ON_ONCE(p->scx.cgrp_moving_from); in scx_cgroup_can_attach()
4851 p->scx.cgrp_moving_from = from; in scx_cgroup_can_attach()
4861 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) in scx_cgroup_can_attach()
4863 p, p->scx.cgrp_moving_from, css->cgroup); in scx_cgroup_can_attach()
4864 p->scx.cgrp_moving_from = NULL; in scx_cgroup_can_attach()
4884 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) in scx_cgroup_move_task()
4886 p, p->scx.cgrp_moving_from, in scx_cgroup_move_task()
4888 p->scx.cgrp_moving_from = NULL; in scx_cgroup_move_task()
4903 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) in scx_cgroup_cancel_attach()
4905 p, p->scx.cgrp_moving_from, css->cgroup); in scx_cgroup_cancel_attach()
4906 p->scx.cgrp_moving_from = NULL; in scx_cgroup_cancel_attach()
4918 tg->scx.weight != weight) in scx_group_set_weight()
4921 tg->scx.weight = weight; in scx_group_set_weight()
4937 tg->scx.idle = idle; in scx_group_set_idle()
4951 (tg->scx.bw_period_us != period_us || in scx_group_set_bandwidth()
4952 tg->scx.bw_quota_us != quota_us || in scx_group_set_bandwidth()
4953 tg->scx.bw_burst_us != burst_us)) in scx_group_set_bandwidth()
4957 tg->scx.bw_period_us = period_us; in scx_group_set_bandwidth()
4958 tg->scx.bw_quota_us = quota_us; in scx_group_set_bandwidth()
4959 tg->scx.bw_burst_us = burst_us; in scx_group_set_bandwidth()
5079 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); in exit_dsq()
5158 tg->scx.sched = NULL; in scx_cgroup_exit()
5159 if (tg->scx.flags & SCX_TG_INITED) { in scx_cgroup_exit()
5160 tg->scx.flags &= ~SCX_TG_INITED; in scx_cgroup_exit()
5179 if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) in scx_cgroup_init()
5184 .weight = tg->scx.weight, in scx_cgroup_init()
5185 .bw_period_us = tg->scx.bw_period_us, in scx_cgroup_init()
5186 .bw_quota_us = tg->scx.bw_quota_us, in scx_cgroup_init()
5187 .bw_burst_us = tg->scx.bw_burst_us, in scx_cgroup_init()
5197 tg->scx.sched = sch; in scx_cgroup_init()
5198 tg->scx.flags |= SCX_TG_INITED; in scx_cgroup_init()
5358 irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); in scx_sched_free_rcu_work()
5733 n = container_of(&cursor, struct task_struct, scx.dsq_list); in bypass_lb_cpu()
5795 list_move_tail(&cursor.node, &n->scx.dsq_list.node); in bypass_lb_cpu()
6024 rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY; in unbypass_renotify_idle()
6029 rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; in unbypass_renotify_idle()
6149 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, in scx_bypass()
6150 scx.runnable_node) { in scx_bypass()
6250 irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); in free_kick_syncs()
6784 unsigned long ops_state = atomic_long_read(&p->scx.ops_state); in scx_dump_task()
6795 if (p->scx.dsq) in scx_dump_task()
6797 (unsigned long long)p->scx.dsq->id); in scx_dump_task()
6802 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); in scx_dump_task()
6805 p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, in scx_dump_task()
6808 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); in scx_dump_task()
6810 p->scx.dsq_vtime, p->scx.slice, p->scx.weight); in scx_dump_task()
6844 idle = list_empty(&rq->scx.runnable_list) && in scx_dump_cpu()
6861 cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, in scx_dump_cpu()
6862 rq->scx.ops_qseq, rq->scx.kick_sync); in scx_dump_cpu()
6878 if (!cpumask_empty(rq->scx.cpus_to_sync)) in scx_dump_cpu()
6880 cpumask_pr_args(rq->scx.cpus_to_sync)); in scx_dump_cpu()
6910 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) in scx_dump_cpu()
7530 rq->scx.local_dsq.sched = sch; in scx_root_enable_workfn()
7531 rq->scx.cpuperf_target = SCX_CPUPERF_ONE; in scx_root_enable_workfn()
7771 if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) in scx_root_enable_workfn()
7953 off >= offsetof(struct task_struct, scx.disallow) && in bpf_scx_btf_struct_access_common()
7954 off + size <= offsetofend(struct task_struct, scx.disallow)) in bpf_scx_btf_struct_access_common()
7968 if ((off >= offsetof(struct task_struct, scx.slice) && in bpf_scx_btf_struct_access()
7969 off + size <= offsetofend(struct task_struct, scx.slice)) || in bpf_scx_btf_struct_access()
7970 (off >= offsetof(struct task_struct, scx.dsq_vtime) && in bpf_scx_btf_struct_access()
7971 off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) in bpf_scx_btf_struct_access()
8442 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH); in can_skip_idle_kick()
8449 struct scx_rq *this_scx = &this_rq->scx; in kick_one_cpu()
8485 ksyncs[cpu] = rq->scx.kick_sync; in kick_one_cpu()
8530 struct scx_rq *this_scx = &this_rq->scx; in kick_cpus_irq_workfn()
8615 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, in print_scx_info()
8681 BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); in init_sched_ext_class()
8683 BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); in init_sched_ext_class()
8687 INIT_LIST_HEAD(&rq->scx.runnable_list); in init_sched_ext_class()
8688 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); in init_sched_ext_class()
8690 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); in init_sched_ext_class()
8691 INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick); in init_sched_ext_class()
8692 raw_spin_lock_init(&rq->scx.deferred_reenq_lock); in init_sched_ext_class()
8693 INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); in init_sched_ext_class()
8694 INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); in init_sched_ext_class()
8695 rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); in init_sched_ext_class()
8696 rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); in init_sched_ext_class()
8699 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; in init_sched_ext_class()
8783 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, in scx_dsq_insert_commit()
9052 p->scx.dsq_vtime = kit->vtime; in scx_dsq_move()
9473 (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && in scx_bpf_task_set_slice()
9480 if (slice > p->scx.slice && in scx_bpf_task_set_slice()
9511 p->scx.dsq_vtime = vtime; in scx_bpf_task_set_dsq_vtime()
9577 list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick); in scx_kick_cpu()
9578 irq_work_queue(&this_rq->scx.kick_cpus_irq_work); in scx_kick_cpu()
9658 ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr); in scx_bpf_dsq_nr_queued()
9664 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); in scx_bpf_dsq_nr_queued()
10208 rq->scx.cpuperf_target = perf; in scx_cpuperf_set()
10496 struct sched_ext_entity *scx; in scx_bpf_tid_to_task() local
10506 scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); in scx_bpf_tid_to_task()
10507 if (!scx) in scx_bpf_tid_to_task()
10510 return container_of(scx, struct task_struct, scx); in scx_bpf_tid_to_task()
10519 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { in __scx_bpf_now()
10521 return READ_ONCE(rq->scx.clock); in __scx_bpf_now()