Lines Matching defs:pcpu

248  * Decay @pcpu's rescue usage average in place, halving per the knob-derived
252 static u64 scx_rescue_decay_avg(struct scx_sched_pcpu *pcpu)
255 	u64 n = div_u64(get_jiffies_64() - pcpu->rescue_avg_at, halflife);
258 		pcpu->rescue_avg = n < 64 ? pcpu->rescue_avg >> n : 0;
259 		pcpu->rescue_avg_at += n * halflife;
261 	return pcpu->rescue_avg;
275 	struct scx_sched_pcpu *pcpu;
288 	pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
289 	pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec;
500 		u64 avg = scx_rescue_decay_avg(per_cpu_ptr(pos->pcpu, cpu));
921 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
931 	if (llist_on_list(&pcpu->ecaps_to_sync_node))
933 	if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync))
953  * pshard->caps[] is the target configuration. pcpu->ecaps is the effective
992 		struct scx_sched_pcpu *pcpu =
994 		struct scx_pshard *ps = pcpu->sch->pshard[shard];
1002 		old = READ_ONCE(pcpu->ecaps);
1004 		WRITE_ONCE(pcpu->ecaps, ecaps);
1020 		if (ecaps != pcpu->reported_ecaps &&
1021 		    SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
1022 		    !scx_bypassing(pcpu->sch, cpu)) {
1023 			struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
1028 			SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
1029 				    pcpu->reported_ecaps, ecaps);
1031 			scx_flush_dispatch_buf(pcpu->sch, rq);
1032 			pcpu->reported_ecaps = ecaps;
1041 			pcpu->idle_renotify = true;
1044 			pcpu->idle_renotify = false;
1072 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
1082 	if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps)
1152 		WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0);
1157  * @pcpu's sched was unhashed before the grace period, so nothing re-queues its
1158  * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
1160 void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
1167 	 * llist can't unlink a single node. Take all queued nodes, drop @pcpu's
1171 	if (llist_on_list(&pcpu->ecaps_to_sync_node)) {
1174 				if (pos == &pcpu->ecaps_to_sync_node) {
1193 	 * @pcpu accesses share one contiguous lock hold, off-list under the rq
1194 	 * lock means @pcpu won't be accessed again.
1198 			if (!llist_on_list(&pcpu->ecaps_to_sync_node))