1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include "msm_gpu.h" 8 #include "msm_gpu_trace.h" 9 10 #include <linux/devfreq.h> 11 #include <linux/devfreq_cooling.h> 12 #include <linux/math64.h> 13 #include <linux/units.h> 14 15 /* 16 * Power Management: 17 */ 18 19 static int msm_devfreq_target(struct device *dev, unsigned long *freq, 20 u32 flags) 21 { 22 struct msm_gpu *gpu = dev_to_gpu(dev); 23 struct msm_gpu_devfreq *df = &gpu->devfreq; 24 struct dev_pm_opp *opp; 25 26 /* 27 * Note that devfreq_recommended_opp() can modify the freq 28 * to something that actually is in the opp table: 29 */ 30 opp = devfreq_recommended_opp(dev, freq, flags); 31 if (IS_ERR(opp)) 32 return PTR_ERR(opp); 33 34 trace_msm_gpu_freq_change(dev_pm_opp_get_freq(opp)); 35 36 if (gpu->funcs->gpu_set_freq) { 37 mutex_lock(&df->lock); 38 gpu->funcs->gpu_set_freq(gpu, opp, df->suspended); 39 mutex_unlock(&df->lock); 40 } else { 41 clk_set_rate(gpu->core_clk, *freq); 42 } 43 44 dev_pm_opp_put(opp); 45 46 return 0; 47 } 48 49 static unsigned long get_freq(struct msm_gpu *gpu) 50 { 51 if (gpu->funcs->gpu_get_freq) 52 return gpu->funcs->gpu_get_freq(gpu); 53 54 return clk_get_rate(gpu->core_clk); 55 } 56 57 static void get_raw_dev_status(struct msm_gpu *gpu, 58 struct devfreq_dev_status *status) 59 { 60 struct msm_gpu_devfreq *df = &gpu->devfreq; 61 u64 busy_cycles, busy_time; 62 unsigned long sample_rate; 63 ktime_t time; 64 65 mutex_lock(&df->lock); 66 67 status->current_frequency = get_freq(gpu); 68 time = ktime_get(); 69 status->total_time = ktime_us_delta(time, df->time); 70 df->time = time; 71 72 if (df->suspended) { 73 mutex_unlock(&df->lock); 74 status->busy_time = 0; 75 return; 76 } 77 78 busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); 79 busy_time = busy_cycles - df->busy_cycles; 80 df->busy_cycles = busy_cycles; 81 82 mutex_unlock(&df->lock); 83 84 busy_time *= USEC_PER_SEC; 85 busy_time = div64_ul(busy_time, sample_rate); 86 if (WARN_ON(busy_time > ~0LU)) 87 busy_time = ~0LU; 88 89 status->busy_time = busy_time; 90 } 91 92 static void update_average_dev_status(struct msm_gpu *gpu, 93 const struct devfreq_dev_status *raw) 94 { 95 struct msm_gpu_devfreq *df = &gpu->devfreq; 96 const u32 polling_ms = df->devfreq->profile->polling_ms; 97 const u32 max_history_ms = polling_ms * 11 / 10; 98 struct devfreq_dev_status *avg = &df->average_status; 99 u64 avg_freq; 100 101 /* simple_ondemand governor interacts poorly with gpu->clamp_to_idle. 102 * When we enforce the constraint on idle, it calls get_dev_status 103 * which would normally reset the stats. When we remove the 104 * constraint on active, it calls get_dev_status again where busy_time 105 * would be 0. 106 * 107 * To remedy this, we always return the average load over the past 108 * polling_ms. 109 */ 110 111 /* raw is longer than polling_ms or avg has no history */ 112 if (div_u64(raw->total_time, USEC_PER_MSEC) >= polling_ms || 113 !avg->total_time) { 114 *avg = *raw; 115 return; 116 } 117 118 /* Truncate the oldest history first. 119 * 120 * Because we keep the history with a single devfreq_dev_status, 121 * rather than a list of devfreq_dev_status, we have to assume freq 122 * and load are the same over avg->total_time. We can scale down 123 * avg->busy_time and avg->total_time by the same factor to drop 124 * history. 125 */ 126 if (div_u64(avg->total_time + raw->total_time, USEC_PER_MSEC) >= 127 max_history_ms) { 128 const u32 new_total_time = polling_ms * USEC_PER_MSEC - 129 raw->total_time; 130 avg->busy_time = div_u64( 131 mul_u32_u32(avg->busy_time, new_total_time), 132 avg->total_time); 133 avg->total_time = new_total_time; 134 } 135 136 /* compute the average freq over avg->total_time + raw->total_time */ 137 avg_freq = mul_u32_u32(avg->current_frequency, avg->total_time); 138 avg_freq += mul_u32_u32(raw->current_frequency, raw->total_time); 139 do_div(avg_freq, avg->total_time + raw->total_time); 140 141 avg->current_frequency = avg_freq; 142 avg->busy_time += raw->busy_time; 143 avg->total_time += raw->total_time; 144 } 145 146 static int msm_devfreq_get_dev_status(struct device *dev, 147 struct devfreq_dev_status *status) 148 { 149 struct msm_gpu *gpu = dev_to_gpu(dev); 150 struct devfreq_dev_status raw; 151 152 get_raw_dev_status(gpu, &raw); 153 update_average_dev_status(gpu, &raw); 154 *status = gpu->devfreq.average_status; 155 156 return 0; 157 } 158 159 static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) 160 { 161 *freq = get_freq(dev_to_gpu(dev)); 162 163 return 0; 164 } 165 166 static struct devfreq_dev_profile msm_devfreq_profile = { 167 .timer = DEVFREQ_TIMER_DELAYED, 168 .polling_ms = 50, 169 .target = msm_devfreq_target, 170 .get_dev_status = msm_devfreq_get_dev_status, 171 .get_cur_freq = msm_devfreq_get_cur_freq, 172 }; 173 174 static void msm_devfreq_boost_work(struct kthread_work *work); 175 static void msm_devfreq_idle_work(struct kthread_work *work); 176 177 static bool has_devfreq(struct msm_gpu *gpu) 178 { 179 struct msm_gpu_devfreq *df = &gpu->devfreq; 180 return !!df->devfreq; 181 } 182 183 void msm_devfreq_init(struct msm_gpu *gpu) 184 { 185 struct msm_gpu_devfreq *df = &gpu->devfreq; 186 187 /* We need target support to do devfreq */ 188 if (!gpu->funcs->gpu_busy) 189 return; 190 191 mutex_init(&df->lock); 192 193 dev_pm_qos_add_request(&gpu->pdev->dev, &df->idle_freq, 194 DEV_PM_QOS_MAX_FREQUENCY, 195 PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); 196 dev_pm_qos_add_request(&gpu->pdev->dev, &df->boost_freq, 197 DEV_PM_QOS_MIN_FREQUENCY, 0); 198 199 msm_devfreq_profile.initial_freq = gpu->fast_rate; 200 201 /* 202 * Don't set the freq_table or max_state and let devfreq build the table 203 * from OPP 204 * After a deferred probe, these may have be left to non-zero values, 205 * so set them back to zero before creating the devfreq device 206 */ 207 msm_devfreq_profile.freq_table = NULL; 208 msm_devfreq_profile.max_state = 0; 209 210 df->devfreq = devm_devfreq_add_device(&gpu->pdev->dev, 211 &msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, 212 NULL); 213 214 if (IS_ERR(df->devfreq)) { 215 DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n"); 216 df->devfreq = NULL; 217 return; 218 } 219 220 devfreq_suspend_device(df->devfreq); 221 222 gpu->cooling = of_devfreq_cooling_register(gpu->pdev->dev.of_node, df->devfreq); 223 if (IS_ERR(gpu->cooling)) { 224 DRM_DEV_ERROR(&gpu->pdev->dev, 225 "Couldn't register GPU cooling device\n"); 226 gpu->cooling = NULL; 227 } 228 229 msm_hrtimer_work_init(&df->boost_work, gpu->worker, msm_devfreq_boost_work, 230 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 231 msm_hrtimer_work_init(&df->idle_work, gpu->worker, msm_devfreq_idle_work, 232 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 233 } 234 235 static void cancel_idle_work(struct msm_gpu_devfreq *df) 236 { 237 hrtimer_cancel(&df->idle_work.timer); 238 kthread_cancel_work_sync(&df->idle_work.work); 239 } 240 241 static void cancel_boost_work(struct msm_gpu_devfreq *df) 242 { 243 hrtimer_cancel(&df->boost_work.timer); 244 kthread_cancel_work_sync(&df->boost_work.work); 245 } 246 247 void msm_devfreq_cleanup(struct msm_gpu *gpu) 248 { 249 struct msm_gpu_devfreq *df = &gpu->devfreq; 250 251 if (!has_devfreq(gpu)) 252 return; 253 254 devfreq_cooling_unregister(gpu->cooling); 255 dev_pm_qos_remove_request(&df->boost_freq); 256 dev_pm_qos_remove_request(&df->idle_freq); 257 } 258 259 void msm_devfreq_resume(struct msm_gpu *gpu) 260 { 261 struct msm_gpu_devfreq *df = &gpu->devfreq; 262 unsigned long sample_rate; 263 264 if (!has_devfreq(gpu)) 265 return; 266 267 mutex_lock(&df->lock); 268 df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); 269 df->time = ktime_get(); 270 df->suspended = false; 271 mutex_unlock(&df->lock); 272 273 devfreq_resume_device(df->devfreq); 274 } 275 276 void msm_devfreq_suspend(struct msm_gpu *gpu) 277 { 278 struct msm_gpu_devfreq *df = &gpu->devfreq; 279 280 if (!has_devfreq(gpu)) 281 return; 282 283 mutex_lock(&df->lock); 284 df->suspended = true; 285 mutex_unlock(&df->lock); 286 287 devfreq_suspend_device(df->devfreq); 288 289 cancel_idle_work(df); 290 cancel_boost_work(df); 291 } 292 293 static void msm_devfreq_boost_work(struct kthread_work *work) 294 { 295 struct msm_gpu_devfreq *df = container_of(work, 296 struct msm_gpu_devfreq, boost_work.work); 297 298 dev_pm_qos_update_request(&df->boost_freq, 0); 299 } 300 301 void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor) 302 { 303 struct msm_gpu_devfreq *df = &gpu->devfreq; 304 uint64_t freq; 305 306 if (!has_devfreq(gpu)) 307 return; 308 309 freq = get_freq(gpu); 310 freq *= factor; 311 312 /* 313 * A nice little trap is that PM QoS operates in terms of KHz, 314 * while devfreq operates in terms of Hz: 315 */ 316 do_div(freq, HZ_PER_KHZ); 317 318 dev_pm_qos_update_request(&df->boost_freq, freq); 319 320 msm_hrtimer_queue_work(&df->boost_work, 321 ms_to_ktime(msm_devfreq_profile.polling_ms), 322 HRTIMER_MODE_REL); 323 } 324 325 void msm_devfreq_active(struct msm_gpu *gpu) 326 { 327 struct msm_gpu_devfreq *df = &gpu->devfreq; 328 unsigned int idle_time; 329 330 if (!has_devfreq(gpu)) 331 return; 332 333 /* 334 * Cancel any pending transition to idle frequency: 335 */ 336 cancel_idle_work(df); 337 338 idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time)); 339 340 /* 341 * If we've been idle for a significant fraction of a polling 342 * interval, then we won't meet the threshold of busyness for 343 * the governor to ramp up the freq.. so give some boost 344 */ 345 if (idle_time > msm_devfreq_profile.polling_ms) { 346 msm_devfreq_boost(gpu, 2); 347 } 348 349 dev_pm_qos_update_request(&df->idle_freq, 350 PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); 351 } 352 353 354 static void msm_devfreq_idle_work(struct kthread_work *work) 355 { 356 struct msm_gpu_devfreq *df = container_of(work, 357 struct msm_gpu_devfreq, idle_work.work); 358 struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq); 359 360 df->idle_time = ktime_get(); 361 362 if (gpu->clamp_to_idle) 363 dev_pm_qos_update_request(&df->idle_freq, 0); 364 } 365 366 void msm_devfreq_idle(struct msm_gpu *gpu) 367 { 368 struct msm_gpu_devfreq *df = &gpu->devfreq; 369 370 if (!has_devfreq(gpu)) 371 return; 372 373 msm_hrtimer_queue_work(&df->idle_work, ms_to_ktime(1), 374 HRTIMER_MODE_REL); 375 } 376