6616 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,  in scx_dsq_move()  argument
6619 	struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;  in scx_dsq_move()
6663 		     u32_before(kit->cursor.priv, p->scx.dsq_seq) ||  in scx_dsq_move()
6678 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)  in scx_dsq_move()
6679 		p->scx.dsq_vtime = kit->vtime;  in scx_dsq_move()
6680 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)  in scx_dsq_move()
6681 		p->scx.slice = kit->slice;  in scx_dsq_move()
6696 	kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |  in scx_dsq_move()
6798 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_slice()  local
6800 	kit->slice = slice;  in scx_bpf_dsq_move_set_slice()
6801 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;  in scx_bpf_dsq_move_set_slice()
6825 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_vtime()  local
6827 	kit->vtime = vtime;  in scx_bpf_dsq_move_set_vtime()
6828 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;  in scx_bpf_dsq_move_set_vtime()
7169 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_new()  local
7179 	kit->dsq = find_user_dsq(dsq_id);  in bpf_iter_scx_dsq_new()
7180 	if (!kit->dsq)  in bpf_iter_scx_dsq_new()
7183 	INIT_LIST_HEAD(&kit->cursor.node);  in bpf_iter_scx_dsq_new()
7184 	kit->cursor.flags = SCX_DSQ_LNODE_ITER_CURSOR | flags;  in bpf_iter_scx_dsq_new()
7185 	kit->cursor.priv = READ_ONCE(kit->dsq->seq);  in bpf_iter_scx_dsq_new()
7198 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_next()  local
7199 	bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;  in bpf_iter_scx_dsq_next()
7203 	if (!kit->dsq)  in bpf_iter_scx_dsq_next()
7206 	raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
7208 	if (list_empty(&kit->cursor.node))  in bpf_iter_scx_dsq_next()
7211 		p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);  in bpf_iter_scx_dsq_next()
7219 		p = nldsq_next_task(kit->dsq, p, rev);  in bpf_iter_scx_dsq_next()
7220 	} while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));  in bpf_iter_scx_dsq_next()
7224 			list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
7226 			list_move(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
7228 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_next()
7231 	raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
7244 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_destroy()  local
7246 	if (!kit->dsq)  in bpf_iter_scx_dsq_destroy()
7249 	if (!list_empty(&kit->cursor.node)) {  in bpf_iter_scx_dsq_destroy()
7252 		raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
7253 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_destroy()
7254 		raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
7256 	kit->dsq = NULL;  in bpf_iter_scx_dsq_destroy()