1 #ifdef CONFIG_SMP 2 #include "sched-pelt.h" 3 4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); 5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); 6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); 7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); 8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); 9 10 #ifdef CONFIG_SCHED_THERMAL_PRESSURE 11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity); 12 13 static inline u64 thermal_load_avg(struct rq *rq) 14 { 15 return READ_ONCE(rq->avg_thermal.load_avg); 16 } 17 #else 18 static inline int 19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) 20 { 21 return 0; 22 } 23 24 static inline u64 thermal_load_avg(struct rq *rq) 25 { 26 return 0; 27 } 28 #endif 29 30 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 31 int update_irq_load_avg(struct rq *rq, u64 running); 32 #else 33 static inline int 34 update_irq_load_avg(struct rq *rq, u64 running) 35 { 36 return 0; 37 } 38 #endif 39 40 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024) 41 42 static inline u32 get_pelt_divider(struct sched_avg *avg) 43 { 44 return PELT_MIN_DIVIDER + avg->period_contrib; 45 } 46 47 static inline void cfs_se_util_change(struct sched_avg *avg) 48 { 49 unsigned int enqueued; 50 51 if (!sched_feat(UTIL_EST)) 52 return; 53 54 /* Avoid store if the flag has been already reset */ 55 enqueued = avg->util_est; 56 if (!(enqueued & UTIL_AVG_UNCHANGED)) 57 return; 58 59 /* Reset flag to report util_avg has been updated */ 60 enqueued &= ~UTIL_AVG_UNCHANGED; 61 WRITE_ONCE(avg->util_est, enqueued); 62 } 63 64 static inline u64 rq_clock_pelt(struct rq *rq) 65 { 66 lockdep_assert_rq_held(rq); 67 assert_clock_updated(rq); 68 69 return rq->clock_pelt - rq->lost_idle_time; 70 } 71 72 /* The rq is idle, we can sync to clock_task */ 73 static inline void _update_idle_rq_clock_pelt(struct rq *rq) 74 { 75 rq->clock_pelt = rq_clock_task(rq); 76 77 u64_u32_store(rq->clock_idle, rq_clock(rq)); 78 /* Paired with smp_rmb in migrate_se_pelt_lag() */ 79 smp_wmb(); 80 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq)); 81 } 82 83 /* 84 * The clock_pelt scales the time to reflect the effective amount of 85 * computation done during the running delta time but then sync back to 86 * clock_task when rq is idle. 87 * 88 * 89 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 90 * @ max capacity ------******---------------******--------------- 91 * @ half capacity ------************---------************--------- 92 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 93 * 94 */ 95 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) 96 { 97 if (unlikely(is_idle_task(rq->curr))) { 98 _update_idle_rq_clock_pelt(rq); 99 return; 100 } 101 102 /* 103 * When a rq runs at a lower compute capacity, it will need 104 * more time to do the same amount of work than at max 105 * capacity. In order to be invariant, we scale the delta to 106 * reflect how much work has been really done. 107 * Running longer results in stealing idle time that will 108 * disturb the load signal compared to max capacity. This 109 * stolen idle time will be automatically reflected when the 110 * rq will be idle and the clock will be synced with 111 * rq_clock_task. 112 */ 113 114 /* 115 * Scale the elapsed time to reflect the real amount of 116 * computation 117 */ 118 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 119 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 120 121 rq->clock_pelt += delta; 122 } 123 124 /* 125 * When rq becomes idle, we have to check if it has lost idle time 126 * because it was fully busy. A rq is fully used when the /Sum util_sum 127 * is greater or equal to: 128 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; 129 * For optimization and computing rounding purpose, we don't take into account 130 * the position in the current window (period_contrib) and we use the higher 131 * bound of util_sum to decide. 132 */ 133 static inline void update_idle_rq_clock_pelt(struct rq *rq) 134 { 135 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; 136 u32 util_sum = rq->cfs.avg.util_sum; 137 util_sum += rq->avg_rt.util_sum; 138 util_sum += rq->avg_dl.util_sum; 139 140 /* 141 * Reflecting stolen time makes sense only if the idle 142 * phase would be present at max capacity. As soon as the 143 * utilization of a rq has reached the maximum value, it is 144 * considered as an always running rq without idle time to 145 * steal. This potential idle time is considered as lost in 146 * this case. We keep track of this lost idle time compare to 147 * rq's clock_task. 148 */ 149 if (util_sum >= divider) 150 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; 151 152 _update_idle_rq_clock_pelt(rq); 153 } 154 155 #ifdef CONFIG_CFS_BANDWIDTH 156 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 157 { 158 u64 throttled; 159 160 if (unlikely(cfs_rq->throttle_count)) 161 throttled = U64_MAX; 162 else 163 throttled = cfs_rq->throttled_clock_pelt_time; 164 165 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled); 166 } 167 168 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 169 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 170 { 171 if (unlikely(cfs_rq->throttle_count)) 172 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time; 173 174 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time; 175 } 176 #else 177 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } 178 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 179 { 180 return rq_clock_pelt(rq_of(cfs_rq)); 181 } 182 #endif 183 184 #else 185 186 static inline int 187 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 188 { 189 return 0; 190 } 191 192 static inline int 193 update_rt_rq_load_avg(u64 now, struct rq *rq, int running) 194 { 195 return 0; 196 } 197 198 static inline int 199 update_dl_rq_load_avg(u64 now, struct rq *rq, int running) 200 { 201 return 0; 202 } 203 204 static inline int 205 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) 206 { 207 return 0; 208 } 209 210 static inline u64 thermal_load_avg(struct rq *rq) 211 { 212 return 0; 213 } 214 215 static inline int 216 update_irq_load_avg(struct rq *rq, u64 running) 217 { 218 return 0; 219 } 220 221 static inline u64 rq_clock_pelt(struct rq *rq) 222 { 223 return rq_clock_task(rq); 224 } 225 226 static inline void 227 update_rq_clock_pelt(struct rq *rq, s64 delta) { } 228 229 static inline void 230 update_idle_rq_clock_pelt(struct rq *rq) { } 231 232 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } 233 #endif 234 235 236