Lines Matching refs:pcpu

350 	return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;  in bypass_dsq()
1007 this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \
1020 __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \
1231 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); in schedule_dsq_reenq()
1252 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); in schedule_dsq_reenq()
2751 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in flush_dispatch_buf()
2787 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_dispatch_sched()
2814 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_dispatch_sched() local
2816 if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && in scx_dispatch_sched()
4593 dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); in init_dsq()
4594 if (!dsq->pcpu) in init_dsq()
4598 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); in init_dsq() local
4600 pcpu->dsq = dsq; in init_dsq()
4601 INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); in init_dsq()
4612 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); in exit_dsq() local
4613 struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; in exit_dsq()
4626 free_percpu(dsq->pcpu); in exit_dsq()
4844 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_sched_free_rcu_work() local
4851 WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); in scx_sched_free_rcu_work()
4856 free_percpu(sch->pcpu); in scx_sched_free_rcu_work()
5498 struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); in scx_bypass() local
5501 pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; in scx_bypass()
5503 pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; in scx_bypass()
6615 sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, in scx_alloc_and_add_sched()
6618 if (!sch->pcpu) { in scx_alloc_and_add_sched()
6632 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); in scx_alloc_and_add_sched() local
6634 pcpu->sch = sch; in scx_alloc_and_add_sched()
6635 INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); in scx_alloc_and_add_sched()
6741 free_percpu(sch->pcpu); in scx_alloc_and_add_sched()
8129 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_dsq_insert_commit()
8458 return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); in scx_bpf_dispatch_nr_slots()
8479 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_bpf_dispatch_cancel()
8528 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; in scx_bpf_dsq_move_to_local___v2()
9682 e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; in scx_read_events()