Lines Matching refs:pcpu
1117 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq));
1138 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
2916 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
5150 dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu);
5151 if (!dsq->pcpu)
5155 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
5157 pcpu->dsq = dsq;
5158 INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
5169 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
5170 struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
5183 free_percpu(dsq->pcpu);
5460 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
5467 WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
5469 /* remove the queued ecaps sync so the pcpu can be freed */
5470 scx_discard_ecaps_to_sync(cpu, pcpu);
5474 * pcpu's to_kick_node is off the list before it is freed.
5477 WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
5478 free_cpumask_var(pcpu->cpus_to_kick);
5479 free_cpumask_var(pcpu->cpus_to_kick_if_idle);
5480 free_cpumask_var(pcpu->cpus_to_preempt);
5481 free_cpumask_var(pcpu->cpus_to_wait);
5486 free_percpu(sch->pcpu);
6123 * @pcpu: @pos's per-cpu state for @rq's cpu
6139 struct scx_sched_pcpu *pcpu)
6146 pcpu->idle_renotify = true;
6233 struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
6234 bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING;
6237 pcpu->flags |= SCX_SCHED_PCPU_BYPASSING;
6239 pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
6241 unbypass_renotify_idle(rq, pos, pcpu);
6952 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
6984 if (!cpumask_empty(pcpu->cpus_to_kick))
6986 cpumask_pr_args(pcpu->cpus_to_kick));
6987 if (!cpumask_empty(pcpu->cpus_to_kick_if_idle))
6989 cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
6990 if (!cpumask_empty(pcpu->cpus_to_preempt))
6992 cpumask_pr_args(pcpu->cpus_to_preempt));
6993 if (!cpumask_empty(pcpu->cpus_to_wait))
6995 cpumask_pr_args(pcpu->cpus_to_wait));
7281 sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu,
7284 if (!sch->pcpu) {
7298 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
7301 pcpu->sch = sch;
7302 INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node);
7304 init_llist_node(&pcpu->ecaps_to_sync_node);
7306 INIT_LIST_HEAD(&pcpu->to_kick_node);
7307 if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) ||
7308 !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) ||
7309 !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) ||
7310 !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
7440 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
7442 free_cpumask_var(pcpu->cpus_to_kick);
7443 free_cpumask_var(pcpu->cpus_to_kick_if_idle);
7444 free_cpumask_var(pcpu->cpus_to_preempt);
7445 free_cpumask_var(pcpu->cpus_to_wait);
7452 free_percpu(sch->pcpu);
8567 static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq,
8591 if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
8592 if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
8594 u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
8596 if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps)))
8597 __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
8599 __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
8601 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
8604 if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) {
8610 cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
8617 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
8618 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
8619 cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
8628 static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu,
8639 if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)))
8642 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
8654 struct scx_sched_pcpu *pcpu, *tmp;
8670 list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
8671 list_del_init(&pcpu->to_kick_node);
8673 for_each_cpu(cpu, pcpu->cpus_to_kick) {
8674 should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
8675 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
8676 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
8679 for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) {
8680 kick_one_cpu_if_idle(cpu, pcpu, this_rq);
8681 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
8889 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9225 return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);
9246 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9295 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9651 struct scx_sched_pcpu *pcpu;
9667 pcpu = this_cpu_ptr(sch->pcpu);
9682 * The kick masks are owned by @sch->pcpu, so that a preempt kick can be
9700 cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
9702 cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
9705 cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
9707 cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
9710 if (list_empty(&pcpu->to_kick_node))
9711 list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick);
10746 struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats;