1 #ifdef CONFIG_SMP
2 #include "sched-pelt.h"
3
4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9 bool update_other_load_avgs(struct rq *rq);
10
11 #ifdef CONFIG_SCHED_HW_PRESSURE
12 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
13
hw_load_avg(struct rq * rq)14 static inline u64 hw_load_avg(struct rq *rq)
15 {
16 return READ_ONCE(rq->avg_hw.load_avg);
17 }
18 #else
19 static inline int
update_hw_load_avg(u64 now,struct rq * rq,u64 capacity)20 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
21 {
22 return 0;
23 }
24
hw_load_avg(struct rq * rq)25 static inline u64 hw_load_avg(struct rq *rq)
26 {
27 return 0;
28 }
29 #endif
30
31 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
32 int update_irq_load_avg(struct rq *rq, u64 running);
33 #else
34 static inline int
update_irq_load_avg(struct rq * rq,u64 running)35 update_irq_load_avg(struct rq *rq, u64 running)
36 {
37 return 0;
38 }
39 #endif
40
41 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
42
get_pelt_divider(struct sched_avg * avg)43 static inline u32 get_pelt_divider(struct sched_avg *avg)
44 {
45 return PELT_MIN_DIVIDER + avg->period_contrib;
46 }
47
cfs_se_util_change(struct sched_avg * avg)48 static inline void cfs_se_util_change(struct sched_avg *avg)
49 {
50 unsigned int enqueued;
51
52 if (!sched_feat(UTIL_EST))
53 return;
54
55 /* Avoid store if the flag has been already reset */
56 enqueued = avg->util_est;
57 if (!(enqueued & UTIL_AVG_UNCHANGED))
58 return;
59
60 /* Reset flag to report util_avg has been updated */
61 enqueued &= ~UTIL_AVG_UNCHANGED;
62 WRITE_ONCE(avg->util_est, enqueued);
63 }
64
rq_clock_pelt(struct rq * rq)65 static inline u64 rq_clock_pelt(struct rq *rq)
66 {
67 lockdep_assert_rq_held(rq);
68 assert_clock_updated(rq);
69
70 return rq->clock_pelt - rq->lost_idle_time;
71 }
72
73 /* The rq is idle, we can sync to clock_task */
_update_idle_rq_clock_pelt(struct rq * rq)74 static inline void _update_idle_rq_clock_pelt(struct rq *rq)
75 {
76 rq->clock_pelt = rq_clock_task(rq);
77
78 u64_u32_store(rq->clock_idle, rq_clock(rq));
79 /* Paired with smp_rmb in migrate_se_pelt_lag() */
80 smp_wmb();
81 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
82 }
83
84 /*
85 * The clock_pelt scales the time to reflect the effective amount of
86 * computation done during the running delta time but then sync back to
87 * clock_task when rq is idle.
88 *
89 *
90 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
91 * @ max capacity ------******---------------******---------------
92 * @ half capacity ------************---------************---------
93 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
94 *
95 */
update_rq_clock_pelt(struct rq * rq,s64 delta)96 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
97 {
98 if (unlikely(is_idle_task(rq->curr))) {
99 _update_idle_rq_clock_pelt(rq);
100 return;
101 }
102
103 /*
104 * When a rq runs at a lower compute capacity, it will need
105 * more time to do the same amount of work than at max
106 * capacity. In order to be invariant, we scale the delta to
107 * reflect how much work has been really done.
108 * Running longer results in stealing idle time that will
109 * disturb the load signal compared to max capacity. This
110 * stolen idle time will be automatically reflected when the
111 * rq will be idle and the clock will be synced with
112 * rq_clock_task.
113 */
114
115 /*
116 * Scale the elapsed time to reflect the real amount of
117 * computation
118 */
119 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
120 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
121
122 rq->clock_pelt += delta;
123 }
124
125 /*
126 * When rq becomes idle, we have to check if it has lost idle time
127 * because it was fully busy. A rq is fully used when the /Sum util_sum
128 * is greater or equal to:
129 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
130 * For optimization and computing rounding purpose, we don't take into account
131 * the position in the current window (period_contrib) and we use the higher
132 * bound of util_sum to decide.
133 */
update_idle_rq_clock_pelt(struct rq * rq)134 static inline void update_idle_rq_clock_pelt(struct rq *rq)
135 {
136 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
137 u32 util_sum = rq->cfs.avg.util_sum;
138 util_sum += rq->avg_rt.util_sum;
139 util_sum += rq->avg_dl.util_sum;
140
141 /*
142 * Reflecting stolen time makes sense only if the idle
143 * phase would be present at max capacity. As soon as the
144 * utilization of a rq has reached the maximum value, it is
145 * considered as an always running rq without idle time to
146 * steal. This potential idle time is considered as lost in
147 * this case. We keep track of this lost idle time compare to
148 * rq's clock_task.
149 */
150 if (util_sum >= divider)
151 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
152
153 _update_idle_rq_clock_pelt(rq);
154 }
155
156 #ifdef CONFIG_CFS_BANDWIDTH
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)157 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
158 {
159 u64 throttled;
160
161 if (unlikely(cfs_rq->throttle_count))
162 throttled = U64_MAX;
163 else
164 throttled = cfs_rq->throttled_clock_pelt_time;
165
166 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
167 }
168
169 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)170 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
171 {
172 if (unlikely(cfs_rq->throttle_count))
173 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
174
175 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
176 }
177 #else
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)178 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)179 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
180 {
181 return rq_clock_pelt(rq_of(cfs_rq));
182 }
183 #endif
184
185 #else
186
187 static inline int
update_cfs_rq_load_avg(u64 now,struct cfs_rq * cfs_rq)188 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
189 {
190 return 0;
191 }
192
193 static inline int
update_rt_rq_load_avg(u64 now,struct rq * rq,int running)194 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
195 {
196 return 0;
197 }
198
199 static inline int
update_dl_rq_load_avg(u64 now,struct rq * rq,int running)200 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
201 {
202 return 0;
203 }
204
205 static inline int
update_hw_load_avg(u64 now,struct rq * rq,u64 capacity)206 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
207 {
208 return 0;
209 }
210
hw_load_avg(struct rq * rq)211 static inline u64 hw_load_avg(struct rq *rq)
212 {
213 return 0;
214 }
215
216 static inline int
update_irq_load_avg(struct rq * rq,u64 running)217 update_irq_load_avg(struct rq *rq, u64 running)
218 {
219 return 0;
220 }
221
rq_clock_pelt(struct rq * rq)222 static inline u64 rq_clock_pelt(struct rq *rq)
223 {
224 return rq_clock_task(rq);
225 }
226
227 static inline void
update_rq_clock_pelt(struct rq * rq,s64 delta)228 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
229
230 static inline void
update_idle_rq_clock_pelt(struct rq * rq)231 update_idle_rq_clock_pelt(struct rq *rq) { }
232
update_idle_cfs_rq_clock_pelt(struct cfs_rq * cfs_rq)233 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
234 #endif
235
236
237