Lines Matching refs:dsq
114 static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) in dsq_is_rq_owned() argument
116 switch (dsq->id) { in dsq_is_rq_owned()
517 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, in nldsq_next_task() argument
523 lockdep_assert_held(&dsq->lock); in nldsq_next_task()
528 list_node = &dsq->list; in nldsq_next_task()
537 if (list_node == &dsq->list) in nldsq_next_task()
547 #define nldsq_for_each_task(p, dsq) \ argument
548 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \
549 (p) = nldsq_next_task((dsq), (p), false))
565 struct scx_dispatch_q *dsq) in nldsq_cursor_next_task() argument
570 lockdep_assert_held(&dsq->lock); in nldsq_cursor_next_task()
580 p = nldsq_next_task(dsq, p, rev); in nldsq_cursor_next_task()
609 struct rq *rq, struct scx_dispatch_q *dsq, in nldsq_cursor_lost_task() argument
613 lockdep_assert_held(&dsq->lock); in nldsq_cursor_lost_task()
619 if (unlikely(p->scx.dsq != dsq || in nldsq_cursor_lost_task()
639 struct scx_dispatch_q *dsq; member
1099 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, in schedule_dsq_reenq() argument
1111 if (dsq->id == SCX_DSQ_LOCAL) { in schedule_dsq_reenq()
1112 rq = container_of(dsq, struct rq, scx.local_dsq); in schedule_dsq_reenq()
1144 } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { in schedule_dsq_reenq()
1147 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); in schedule_dsq_reenq()
1162 if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID)) in schedule_dsq_reenq()
1170 scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); in schedule_dsq_reenq()
1250 static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p) in dsq_insert_head() argument
1252 struct list_head *pos = &dsq->list; in dsq_insert_head()
1259 if (!dsq_is_rq_owned(dsq)) { in dsq_insert_head()
1260 list_add(&p->scx.dsq_list.node, &dsq->list); in dsq_insert_head()
1264 list_for_each_entry(node, &dsq->list, node) { in dsq_insert_head()
1279 return pos == &dsq->list; in dsq_insert_head()
1444 static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) in dsq_inc_nr() argument
1447 WRITE_ONCE(dsq->nr, dsq->nr + 1); in dsq_inc_nr()
1458 if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { in dsq_inc_nr()
1466 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dsq_inc_nr()
1468 if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) in dsq_inc_nr()
1477 if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) in dsq_inc_nr()
1482 static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) in dsq_dec_nr() argument
1485 WRITE_ONCE(dsq->nr, dsq->nr - 1); in dsq_dec_nr()
1488 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dsq_dec_nr()
1490 if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || in dsq_dec_nr()
1534 struct scx_dispatch_q *dsq, struct task_struct *p, in rq_owned_post_enq() argument
1544 if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { in rq_owned_post_enq()
1545 if (dsq->id == SCX_DSQ_REJECT) in rq_owned_post_enq()
1606 struct scx_dispatch_q *dsq, struct task_struct *p, in scx_dispatch_enqueue() argument
1611 if (dsq->id == SCX_DSQ_LOCAL) { in scx_dispatch_enqueue()
1612 dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); in scx_dispatch_enqueue()
1616 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in scx_dispatch_enqueue()
1621 raw_spin_lock_nested(&dsq->lock, in scx_dispatch_enqueue()
1624 if (unlikely(dsq->id == SCX_DSQ_INVALID)) { in scx_dispatch_enqueue()
1627 raw_spin_unlock(&dsq->lock); in scx_dispatch_enqueue()
1628 dsq = find_global_dsq(sch, task_cpu(p)); in scx_dispatch_enqueue()
1629 raw_spin_lock(&dsq->lock); in scx_dispatch_enqueue()
1633 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && in scx_dispatch_enqueue()
1661 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && in scx_dispatch_enqueue()
1662 nldsq_next_task(dsq, NULL, false))) in scx_dispatch_enqueue()
1664 dsq->id); in scx_dispatch_enqueue()
1667 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); in scx_dispatch_enqueue()
1681 list_add(&p->scx.dsq_list.node, &dsq->list); in scx_dispatch_enqueue()
1683 rcu_assign_pointer(dsq->first_task, p); in scx_dispatch_enqueue()
1687 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) in scx_dispatch_enqueue()
1689 dsq->id); in scx_dispatch_enqueue()
1693 if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) in scx_dispatch_enqueue()
1694 rcu_assign_pointer(dsq->first_task, p); in scx_dispatch_enqueue()
1701 list_add_tail(&p->scx.dsq_list.node, &dsq->list); in scx_dispatch_enqueue()
1702 if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) in scx_dispatch_enqueue()
1703 rcu_assign_pointer(dsq->first_task, p); in scx_dispatch_enqueue()
1708 WRITE_ONCE(dsq->seq, dsq->seq + 1); in scx_dispatch_enqueue()
1709 p->scx.dsq_seq = dsq->seq; in scx_dispatch_enqueue()
1711 dsq_inc_nr(dsq, p, enq_flags); in scx_dispatch_enqueue()
1712 p->scx.dsq = dsq; in scx_dispatch_enqueue()
1722 rq_owned_post_enq(sch, rq, dsq, p, enq_flags); in scx_dispatch_enqueue()
1737 if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) in scx_dispatch_enqueue()
1742 raw_spin_unlock(&dsq->lock); in scx_dispatch_enqueue()
1756 void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) in scx_task_unlink_from_dsq() argument
1761 rb_erase(&p->scx.dsq_priq, &dsq->priq); in scx_task_unlink_from_dsq()
1767 dsq_dec_nr(dsq, p); in scx_task_unlink_from_dsq()
1769 if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) { in scx_task_unlink_from_dsq()
1772 first_task = nldsq_next_task(dsq, NULL, false); in scx_task_unlink_from_dsq()
1773 rcu_assign_pointer(dsq->first_task, first_task); in scx_task_unlink_from_dsq()
1779 struct scx_dispatch_q *dsq = p->scx.dsq; in scx_dispatch_dequeue() local
1780 bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); in scx_dispatch_dequeue()
1784 if (!dsq) { in scx_dispatch_dequeue()
1805 raw_spin_lock(&dsq->lock); in scx_dispatch_dequeue()
1813 scx_task_unlink_from_dsq(p, dsq); in scx_dispatch_dequeue()
1824 p->scx.dsq = NULL; in scx_dispatch_dequeue()
1827 raw_spin_unlock(&dsq->lock); in scx_dispatch_dequeue()
1835 struct scx_dispatch_q *dsq) in dispatch_dequeue_locked() argument
1838 lockdep_assert_held(&dsq->lock); in dispatch_dequeue_locked()
1840 scx_task_unlink_from_dsq(p, dsq); in dispatch_dequeue_locked()
1841 p->scx.dsq = NULL; in dispatch_dequeue_locked()
1848 struct scx_dispatch_q *dsq; in find_dsq_for_dispatch() local
1863 dsq = find_global_dsq(sch, tcpu); in find_dsq_for_dispatch()
1865 dsq = find_user_dsq(sch, dsq_id); in find_dsq_for_dispatch()
1871 if (unlikely(!dsq)) { in find_dsq_for_dispatch()
1876 return dsq; in find_dsq_for_dispatch()
1937 struct scx_dispatch_q *dsq = in direct_dispatch() local
1949 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { in direct_dispatch()
1971 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in direct_dispatch()
1983 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, in direct_dispatch()
2004 struct scx_dispatch_q *dsq; in scx_do_enqueue_task() local
2115 dsq = &rq->scx.local_dsq; in scx_do_enqueue_task()
2118 dsq = find_global_dsq(sch, task_cpu(p)); in scx_do_enqueue_task()
2121 dsq = bypass_enq_target_dsq(sch, task_cpu(p)); in scx_do_enqueue_task()
2127 scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); in scx_do_enqueue_task()
2434 p->scx.dsq = dst_dsq; in scx_move_local_task_to_local_dsq()
2589 struct scx_dispatch_q *dsq, in unlink_dsq_and_switch_rq_lock() argument
2595 lockdep_assert_held(&dsq->lock); in unlink_dsq_and_switch_rq_lock()
2599 scx_task_unlink_from_dsq(p, dsq); in unlink_dsq_and_switch_rq_lock()
2602 raw_spin_unlock(&dsq->lock); in unlink_dsq_and_switch_rq_lock()
2616 struct scx_dispatch_q *dsq, struct rq *src_rq) in consume_remote_task() argument
2618 if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { in consume_remote_task()
2698 struct scx_dispatch_q *dsq, u64 enq_flags) in scx_consume_dispatch_q() argument
2707 if (list_empty(&dsq->list)) in scx_consume_dispatch_q()
2710 raw_spin_lock(&dsq->lock); in scx_consume_dispatch_q()
2712 nldsq_for_each_task(p, dsq) { in scx_consume_dispatch_q()
2723 if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) in scx_consume_dispatch_q()
2727 scx_task_unlink_from_dsq(p, dsq); in scx_consume_dispatch_q()
2728 raw_spin_unlock(&dsq->lock); in scx_consume_dispatch_q()
2734 if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) in scx_consume_dispatch_q()
2740 raw_spin_unlock(&dsq->lock); in scx_consume_dispatch_q()
2873 struct scx_dispatch_q *dsq; in finish_dispatch() local
2927 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); in finish_dispatch()
2929 if (dsq->id == SCX_DSQ_LOCAL) in finish_dispatch()
2930 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); in finish_dispatch()
2932 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, in finish_dispatch()
3265 if (p->scx.dsq == &rq->scx.local_dsq) in put_prev_task_scx()
3798 if (p->scx.dsq == &rq->scx.local_dsq) in rq_offline_scx()
3822 struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); in check_rq_for_timeouts() local
3832 if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL) in check_rq_for_timeouts()
3833 sch = dsq->sched; in check_rq_for_timeouts()
4443 struct scx_dispatch_q *dsq; in process_ddsp_deferred_locals() local
4452 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); in process_ddsp_deferred_locals()
4453 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) in process_ddsp_deferred_locals()
4454 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); in process_ddsp_deferred_locals()
4646 static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) in reenq_user() argument
4649 struct scx_sched *sch = dsq->sched; in reenq_user()
4650 struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); in reenq_user()
4656 raw_spin_lock(&dsq->lock); in reenq_user()
4662 p = nldsq_cursor_next_task(&cursor, dsq); in reenq_user()
4677 raw_spin_unlock(&dsq->lock); in reenq_user()
4679 raw_spin_lock(&dsq->lock); in reenq_user()
4684 if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) in reenq_user()
4690 dispatch_dequeue_locked(p, dsq); in reenq_user()
4691 raw_spin_unlock(&dsq->lock); in reenq_user()
4708 raw_spin_lock(&dsq->lock); in reenq_user()
4712 raw_spin_unlock(&dsq->lock); in reenq_user()
4728 struct scx_dispatch_q *dsq; in process_deferred_reenq_users() local
4743 dsq = dsq_pcpu->dsq; in process_deferred_reenq_users()
4753 dsq_id = READ_ONCE(dsq->id); in process_deferred_reenq_users()
4758 reenq_user(rq, dsq, reenq_flags); in process_deferred_reenq_users()
5178 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) in scx_init_dsq() argument
5182 memset(dsq, 0, sizeof(*dsq)); in scx_init_dsq()
5184 raw_spin_lock_init(&dsq->lock); in scx_init_dsq()
5185 INIT_LIST_HEAD(&dsq->list); in scx_init_dsq()
5186 dsq->id = dsq_id; in scx_init_dsq()
5187 dsq->sched = sch; in scx_init_dsq()
5189 dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); in scx_init_dsq()
5190 if (!dsq->pcpu) in scx_init_dsq()
5194 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); in scx_init_dsq()
5196 pcpu->dsq = dsq; in scx_init_dsq()
5203 static void exit_dsq(struct scx_dispatch_q *dsq) in exit_dsq() argument
5208 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); in exit_dsq()
5222 free_percpu(dsq->pcpu); in exit_dsq()
5227 struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); in free_dsq_rcufn() local
5229 exit_dsq(dsq); in free_dsq_rcufn()
5230 kfree(dsq); in free_dsq_rcufn()
5236 struct scx_dispatch_q *dsq, *tmp_dsq; in free_dsq_irq_workfn() local
5238 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) { in free_dsq_irq_workfn()
5249 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); in free_dsq_irq_workfn()
5256 call_rcu(&dsq->rcu, free_dsq_rcufn); in free_dsq_irq_workfn()
5264 struct scx_dispatch_q *dsq; in destroy_dsq() local
5269 dsq = find_user_dsq(sch, dsq_id); in destroy_dsq()
5270 if (!dsq) in destroy_dsq()
5273 raw_spin_lock_irqsave(&dsq->lock, flags); in destroy_dsq()
5275 if (dsq->nr) { in destroy_dsq()
5277 dsq->id, dsq->nr); in destroy_dsq()
5281 if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, in destroy_dsq()
5291 dsq->id = SCX_DSQ_INVALID; in destroy_dsq()
5292 if (llist_add(&dsq->free_node, &dsqs_to_free)) in destroy_dsq()
5296 raw_spin_unlock_irqrestore(&dsq->lock, flags); in destroy_dsq()
5495 struct scx_dispatch_q *dsq; in scx_sched_free_rcu_work() local
5555 while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) in scx_sched_free_rcu_work()
5556 destroy_dsq(sch, dsq->id); in scx_sched_free_rcu_work()
5559 } while (dsq == ERR_PTR(-EAGAIN)); in scx_sched_free_rcu_work()
6982 if (p->scx.dsq) in scx_dump_task()
6984 (unsigned long long)p->scx.dsq->id); in scx_dump_task()
9174 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; in scx_dsq_move()
9351 struct scx_dispatch_q *dsq; in scx_bpf_dsq_move_to_local___v2() local
9368 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_move_to_local___v2()
9369 if (unlikely(!dsq)) { in scx_bpf_dsq_move_to_local___v2()
9374 if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { in scx_bpf_dsq_move_to_local___v2()
9565 struct scx_dispatch_q *dsq; in scx_bpf_create_dsq() local
9576 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); in scx_bpf_create_dsq()
9577 if (!dsq) in scx_bpf_create_dsq()
9584 ret = scx_init_dsq(dsq, dsq_id, NULL); in scx_bpf_create_dsq()
9586 kfree(dsq); in scx_bpf_create_dsq()
9594 dsq->sched = sch; in scx_bpf_create_dsq()
9595 ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, in scx_bpf_create_dsq()
9603 exit_dsq(dsq); in scx_bpf_create_dsq()
9604 kfree(dsq); in scx_bpf_create_dsq()
9849 struct scx_dispatch_q *dsq; in scx_bpf_dsq_nr_queued() local
9871 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_nr_queued()
9872 if (dsq) { in scx_bpf_dsq_nr_queued()
9873 ret = READ_ONCE(dsq->nr); in scx_bpf_dsq_nr_queued()
9931 kit->dsq = NULL; in bpf_iter_scx_dsq_new()
9940 kit->dsq = find_user_dsq(sch, dsq_id); in bpf_iter_scx_dsq_new()
9941 if (!kit->dsq) in bpf_iter_scx_dsq_new()
9944 kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); in bpf_iter_scx_dsq_new()
9959 if (!kit->dsq) in bpf_iter_scx_dsq_next()
9962 guard(raw_spinlock_irqsave)(&kit->dsq->lock); in bpf_iter_scx_dsq_next()
9964 return nldsq_cursor_next_task(&kit->cursor, kit->dsq); in bpf_iter_scx_dsq_next()
9977 if (!kit->dsq) in bpf_iter_scx_dsq_destroy()
9983 raw_spin_lock_irqsave(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
9985 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
9987 kit->dsq = NULL; in bpf_iter_scx_dsq_destroy()
10006 struct scx_dispatch_q *dsq; in scx_bpf_dsq_peek() local
10017 dsq = find_user_dsq(sch, dsq_id); in scx_bpf_dsq_peek()
10018 if (unlikely(!dsq)) { in scx_bpf_dsq_peek()
10023 return rcu_dereference(dsq->first_task); in scx_bpf_dsq_peek()
10050 struct scx_dispatch_q *dsq; in scx_bpf_dsq_reenq() local
10067 dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); in scx_bpf_dsq_reenq()
10068 schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); in scx_bpf_dsq_reenq()