1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef _KERNEL_SCHED_PELT_H 3 #define _KERNEL_SCHED_PELT_H 4 #include "sched.h" 5 6 #include "sched-pelt.h" 7 8 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); 9 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); 10 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); 11 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); 12 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); 13 bool update_other_load_avgs(struct rq *rq); 14 15 #ifdef CONFIG_SCHED_HW_PRESSURE 16 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity); 17 18 static inline u64 hw_load_avg(struct rq *rq) 19 { 20 return READ_ONCE(rq->avg_hw.load_avg); 21 } 22 #else /* !CONFIG_SCHED_HW_PRESSURE: */ 23 static inline int 24 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity) 25 { 26 return 0; 27 } 28 29 static inline u64 hw_load_avg(struct rq *rq) 30 { 31 return 0; 32 } 33 #endif /* !CONFIG_SCHED_HW_PRESSURE */ 34 35 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 36 int update_irq_load_avg(struct rq *rq, u64 running); 37 #else 38 static inline int 39 update_irq_load_avg(struct rq *rq, u64 running) 40 { 41 return 0; 42 } 43 #endif 44 45 #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024) 46 47 static inline u32 get_pelt_divider(struct sched_avg *avg) 48 { 49 return PELT_MIN_DIVIDER + avg->period_contrib; 50 } 51 52 static inline void cfs_se_util_change(struct sched_avg *avg) 53 { 54 unsigned int enqueued; 55 56 if (!sched_feat(UTIL_EST)) 57 return; 58 59 /* Avoid store if the flag has been already reset */ 60 enqueued = avg->util_est; 61 if (!(enqueued & UTIL_AVG_UNCHANGED)) 62 return; 63 64 /* Reset flag to report util_avg has been updated */ 65 enqueued &= ~UTIL_AVG_UNCHANGED; 66 WRITE_ONCE(avg->util_est, enqueued); 67 } 68 69 static inline u64 rq_clock_pelt(struct rq *rq) 70 { 71 lockdep_assert_rq_held(rq); 72 assert_clock_updated(rq); 73 74 return rq->clock_pelt - rq->lost_idle_time; 75 } 76 77 /* The rq is idle, we can sync to clock_task */ 78 static inline void _update_idle_rq_clock_pelt(struct rq *rq) 79 { 80 rq->clock_pelt = rq_clock_task(rq); 81 82 u64_u32_store(rq->clock_idle, rq_clock(rq)); 83 /* Paired with smp_rmb in migrate_se_pelt_lag() */ 84 smp_wmb(); 85 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq)); 86 } 87 88 /* 89 * The clock_pelt scales the time to reflect the effective amount of 90 * computation done during the running delta time but then sync back to 91 * clock_task when rq is idle. 92 * 93 * 94 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 95 * @ max capacity ------******---------------******--------------- 96 * @ half capacity ------************---------************--------- 97 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 98 * 99 */ 100 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) 101 { 102 if (unlikely(is_idle_task(rq->curr))) { 103 _update_idle_rq_clock_pelt(rq); 104 return; 105 } 106 107 /* 108 * When a rq runs at a lower compute capacity, it will need 109 * more time to do the same amount of work than at max 110 * capacity. In order to be invariant, we scale the delta to 111 * reflect how much work has been really done. 112 * Running longer results in stealing idle time that will 113 * disturb the load signal compared to max capacity. This 114 * stolen idle time will be automatically reflected when the 115 * rq will be idle and the clock will be synced with 116 * rq_clock_task. 117 */ 118 119 /* 120 * Scale the elapsed time to reflect the real amount of 121 * computation 122 */ 123 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 124 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 125 126 rq->clock_pelt += delta; 127 } 128 129 /* 130 * When rq becomes idle, we have to check if it has lost idle time 131 * because it was fully busy. A rq is fully used when the /Sum util_sum 132 * is greater or equal to: 133 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; 134 * For optimization and computing rounding purpose, we don't take into account 135 * the position in the current window (period_contrib) and we use the higher 136 * bound of util_sum to decide. 137 */ 138 static inline void update_idle_rq_clock_pelt(struct rq *rq) 139 { 140 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; 141 u32 util_sum = rq->cfs.avg.util_sum; 142 util_sum += rq->avg_rt.util_sum; 143 util_sum += rq->avg_dl.util_sum; 144 145 /* 146 * Reflecting stolen time makes sense only if the idle 147 * phase would be present at max capacity. As soon as the 148 * utilization of a rq has reached the maximum value, it is 149 * considered as an always running rq without idle time to 150 * steal. This potential idle time is considered as lost in 151 * this case. We keep track of this lost idle time compare to 152 * rq's clock_task. 153 */ 154 if (util_sum >= divider) 155 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; 156 157 _update_idle_rq_clock_pelt(rq); 158 } 159 160 #ifdef CONFIG_CFS_BANDWIDTH 161 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 162 { 163 u64 throttled; 164 165 if (unlikely(cfs_rq->throttle_count)) 166 throttled = U64_MAX; 167 else 168 throttled = cfs_rq->throttled_clock_pelt_time; 169 170 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled); 171 } 172 173 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 174 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 175 { 176 if (unlikely(cfs_rq->throttle_count)) 177 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time; 178 179 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time; 180 } 181 #else /* !CONFIG_CFS_BANDWIDTH: */ 182 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } 183 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) 184 { 185 return rq_clock_pelt(rq_of(cfs_rq)); 186 } 187 #endif /* !CONFIG_CFS_BANDWIDTH */ 188 189 #endif /* _KERNEL_SCHED_PELT_H */ 190