1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef _KERNEL_SCHED_PELT_H
3 #define _KERNEL_SCHED_PELT_H
4 #include "sched.h"
5
6 #include "sched-pelt.h"
7
8 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
9 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
10 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
11 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
12 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
13 bool update_other_load_avgs(struct rq *rq);
14
15 #ifdef CONFIG_SCHED_HW_PRESSURE
16 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
17
hw_load_avg(struct rq * rq)18 static inline u64 hw_load_avg(struct rq *rq)
19 {
20 return READ_ONCE(rq->avg_hw.load_avg);
21 }
22 #else /* !CONFIG_SCHED_HW_PRESSURE: */
23 static inline int
update_hw_load_avg(u64 now,struct rq * rq,u64 capacity)24 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
25 {
26 return 0;
27 }
28
hw_load_avg(struct rq * rq)29 static inline u64 hw_load_avg(struct rq *rq)
30 {
31 return 0;
32 }
33 #endif /* !CONFIG_SCHED_HW_PRESSURE */
34
35 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
36 int update_irq_load_avg(struct rq *rq, u64 running);
37 #else
38 static inline int
update_irq_load_avg(struct rq * rq,u64 running)39 update_irq_load_avg(struct rq *rq, u64 running)
40 {
41 return 0;
42 }
43 #endif
44
45 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
46
get_pelt_divider(struct sched_avg * avg)47 static inline u32 get_pelt_divider(struct sched_avg *avg)
48 {
49 return PELT_MIN_DIVIDER + avg->period_contrib;
50 }
51
cfs_se_util_change(struct sched_avg * avg)52 static inline void cfs_se_util_change(struct sched_avg *avg)
53 {
54 unsigned int enqueued;
55
56 if (!sched_feat(UTIL_EST))
57 return;
58
59 /* Avoid store if the flag has been already reset */
60 enqueued = avg->util_est;
61 if (!(enqueued & UTIL_AVG_UNCHANGED))
62 return;
63
64 /* Reset flag to report util_avg has been updated */
65 enqueued &= ~UTIL_AVG_UNCHANGED;
66 WRITE_ONCE(avg->util_est, enqueued);
67 }
68
rq_clock_pelt(struct rq * rq)69 static inline u64 rq_clock_pelt(struct rq *rq)
70 {
71 lockdep_assert_rq_held(rq);
72 assert_clock_updated(rq);
73
74 return rq->clock_pelt - rq->lost_idle_time;
75 }
76
77 /* The rq is idle, we can sync to clock_task */
_update_idle_rq_clock_pelt(struct rq * rq)78 static inline void _update_idle_rq_clock_pelt(struct rq *rq)
79 {
80 rq->clock_pelt = rq_clock_task(rq);
81
82 u64_u32_store(rq->clock_idle, rq_clock(rq));
83 /* Paired with smp_rmb in migrate_se_pelt_lag() */
84 smp_wmb();
85 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
86 }
87
88 /*
89 * The clock_pelt scales the time to reflect the effective amount of
90 * computation done during the running delta time but then sync back to
91 * clock_task when rq is idle.
92 *
93 *
94 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
95 * @ max capacity ------******---------------******---------------
96 * @ half capacity ------************---------************---------
97 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
98 *
99 */
update_rq_clock_pelt(struct rq * rq,s64 delta)100 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
101 {
102 if (unlikely(is_idle_task(rq->curr))) {
103 _update_idle_rq_clock_pelt(rq);
104 return;
105 }
106
107 /*
108 * When a rq runs at a lower compute capacity, it will need
109 * more time to do the same amount of work than at max
110 * capacity. In order to be invariant, we scale the delta to
111 * reflect how much work has been really done.
112 * Running longer results in stealing idle time that will
113 * disturb the load signal compared to max capacity. This
114 * stolen idle time will be automatically reflected when the
115 * rq will be idle and the clock will be synced with
116 * rq_clock_task.
117 */
118
119 /*
120 * Scale the elapsed time to reflect the real amount of
121 * computation
122 */
123 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
124 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
125
126 rq->clock_pelt += delta;
127 }
128
129 /*
130 * When rq becomes idle, we have to check if it has lost idle time
131 * because it was fully busy. A rq is fully used when the /Sum util_sum
132 * is greater or equal to:
133 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
134 * For optimization and computing rounding purpose, we don't take into account
135 * the position in the current window (period_contrib) and we use the higher
136 * bound of util_sum to decide.
137 */
update_idle_rq_clock_pelt(struct rq * rq)138 static inline void update_idle_rq_clock_pelt(struct rq *rq)
139 {
140 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
141 u32 util_sum = rq->cfs.avg.util_sum;
142 util_sum += rq->avg_rt.util_sum;
143 util_sum += rq->avg_dl.util_sum;
144
145 /*
146 * Reflecting stolen time makes sense only if the idle
147 * phase would be present at max capacity. As soon as the
148 * utilization of a rq has reached the maximum value, it is
149 * considered as an always running rq without idle time to
150 * steal. This potential idle time is considered as lost in
151 * this case. We keep track of this lost idle time compare to
152 * rq's clock_task.
153 */
154 if (util_sum >= divider)
155 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
156
157 _update_idle_rq_clock_pelt(rq);
158 }
159
160 #ifdef CONFIG_CFS_BANDWIDTH
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)161 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
162 {
163 u64 throttled;
164
165 if (unlikely(cfs_rq->throttle_count))
166 throttled = U64_MAX;
167 else
168 throttled = cfs_rq->throttled_clock_pelt_time;
169
170 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
171 }
172
173 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)174 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
175 {
176 if (unlikely(cfs_rq->throttle_count))
177 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
178
179 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
180 }
181 #else /* !CONFIG_CFS_BANDWIDTH: */
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)182 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)183 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
184 {
185 return rq_clock_pelt(rq_of(cfs_rq));
186 }
187 #endif /* !CONFIG_CFS_BANDWIDTH */
188
189 #endif /* _KERNEL_SCHED_PELT_H */
190