1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include "msm_gpu.h" 8 #include "msm_gpu_trace.h" 9 10 #include <linux/devfreq.h> 11 #include <linux/devfreq_cooling.h> 12 #include <linux/math64.h> 13 #include <linux/units.h> 14 15 /* 16 * Power Management: 17 */ 18 19 static int msm_devfreq_target(struct device *dev, unsigned long *freq, 20 u32 flags) 21 { 22 struct msm_gpu *gpu = dev_to_gpu(dev); 23 struct msm_gpu_devfreq *df = &gpu->devfreq; 24 struct dev_pm_opp *opp; 25 26 /* 27 * Note that devfreq_recommended_opp() can modify the freq 28 * to something that actually is in the opp table: 29 */ 30 opp = devfreq_recommended_opp(dev, freq, flags); 31 if (IS_ERR(opp)) 32 return PTR_ERR(opp); 33 34 trace_msm_gpu_freq_change(dev_pm_opp_get_freq(opp)); 35 36 if (gpu->funcs->gpu_set_freq) { 37 mutex_lock(&df->lock); 38 gpu->funcs->gpu_set_freq(gpu, opp, df->suspended); 39 mutex_unlock(&df->lock); 40 } else { 41 clk_set_rate(gpu->core_clk, *freq); 42 } 43 44 dev_pm_opp_put(opp); 45 46 return 0; 47 } 48 49 static unsigned long get_freq(struct msm_gpu *gpu) 50 { 51 if (gpu->funcs->gpu_get_freq) 52 return gpu->funcs->gpu_get_freq(gpu); 53 54 return clk_get_rate(gpu->core_clk); 55 } 56 57 static void get_raw_dev_status(struct msm_gpu *gpu, 58 struct devfreq_dev_status *status) 59 { 60 struct msm_gpu_devfreq *df = &gpu->devfreq; 61 u64 busy_cycles, busy_time; 62 unsigned long sample_rate; 63 ktime_t time; 64 65 mutex_lock(&df->lock); 66 67 status->current_frequency = get_freq(gpu); 68 time = ktime_get(); 69 status->total_time = ktime_us_delta(time, df->time); 70 df->time = time; 71 72 if (df->suspended) { 73 mutex_unlock(&df->lock); 74 status->busy_time = 0; 75 return; 76 } 77 78 busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); 79 busy_time = busy_cycles - df->busy_cycles; 80 df->busy_cycles = busy_cycles; 81 82 mutex_unlock(&df->lock); 83 84 busy_time *= USEC_PER_SEC; 85 busy_time = div64_ul(busy_time, sample_rate); 86 if (WARN_ON(busy_time > ~0LU)) 87 busy_time = ~0LU; 88 89 status->busy_time = busy_time; 90 } 91 92 static void update_average_dev_status(struct msm_gpu *gpu, 93 const struct devfreq_dev_status *raw) 94 { 95 struct msm_gpu_devfreq *df = &gpu->devfreq; 96 const u32 polling_ms = df->devfreq->profile->polling_ms; 97 const u32 max_history_ms = polling_ms * 11 / 10; 98 struct devfreq_dev_status *avg = &df->average_status; 99 u64 avg_freq; 100 101 /* simple_ondemand governor interacts poorly with gpu->clamp_to_idle. 102 * When we enforce the constraint on idle, it calls get_dev_status 103 * which would normally reset the stats. When we remove the 104 * constraint on active, it calls get_dev_status again where busy_time 105 * would be 0. 106 * 107 * To remedy this, we always return the average load over the past 108 * polling_ms. 109 */ 110 111 /* raw is longer than polling_ms or avg has no history */ 112 if (div_u64(raw->total_time, USEC_PER_MSEC) >= polling_ms || 113 !avg->total_time) { 114 *avg = *raw; 115 return; 116 } 117 118 /* Truncate the oldest history first. 119 * 120 * Because we keep the history with a single devfreq_dev_status, 121 * rather than a list of devfreq_dev_status, we have to assume freq 122 * and load are the same over avg->total_time. We can scale down 123 * avg->busy_time and avg->total_time by the same factor to drop 124 * history. 125 */ 126 if (div_u64(avg->total_time + raw->total_time, USEC_PER_MSEC) >= 127 max_history_ms) { 128 const u32 new_total_time = polling_ms * USEC_PER_MSEC - 129 raw->total_time; 130 avg->busy_time = div_u64( 131 mul_u32_u32(avg->busy_time, new_total_time), 132 avg->total_time); 133 avg->total_time = new_total_time; 134 } 135 136 /* compute the average freq over avg->total_time + raw->total_time */ 137 avg_freq = mul_u32_u32(avg->current_frequency, avg->total_time); 138 avg_freq += mul_u32_u32(raw->current_frequency, raw->total_time); 139 do_div(avg_freq, avg->total_time + raw->total_time); 140 141 avg->current_frequency = avg_freq; 142 avg->busy_time += raw->busy_time; 143 avg->total_time += raw->total_time; 144 } 145 146 static int msm_devfreq_get_dev_status(struct device *dev, 147 struct devfreq_dev_status *status) 148 { 149 struct msm_gpu *gpu = dev_to_gpu(dev); 150 struct devfreq_dev_status raw; 151 152 get_raw_dev_status(gpu, &raw); 153 update_average_dev_status(gpu, &raw); 154 *status = gpu->devfreq.average_status; 155 156 return 0; 157 } 158 159 static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) 160 { 161 *freq = get_freq(dev_to_gpu(dev)); 162 163 return 0; 164 } 165 166 static struct devfreq_dev_profile msm_devfreq_profile = { 167 .timer = DEVFREQ_TIMER_DELAYED, 168 .polling_ms = 50, 169 .target = msm_devfreq_target, 170 .get_dev_status = msm_devfreq_get_dev_status, 171 .get_cur_freq = msm_devfreq_get_cur_freq, 172 }; 173 174 static void msm_devfreq_boost_work(struct kthread_work *work); 175 static void msm_devfreq_idle_work(struct kthread_work *work); 176 177 static bool has_devfreq(struct msm_gpu *gpu) 178 { 179 struct msm_gpu_devfreq *df = &gpu->devfreq; 180 return !!df->devfreq; 181 } 182 183 void msm_devfreq_init(struct msm_gpu *gpu) 184 { 185 struct msm_gpu_devfreq *df = &gpu->devfreq; 186 187 /* We need target support to do devfreq */ 188 if (!gpu->funcs->gpu_busy) 189 return; 190 191 mutex_init(&df->lock); 192 193 dev_pm_qos_add_request(&gpu->pdev->dev, &df->idle_freq, 194 DEV_PM_QOS_MAX_FREQUENCY, 195 PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); 196 dev_pm_qos_add_request(&gpu->pdev->dev, &df->boost_freq, 197 DEV_PM_QOS_MIN_FREQUENCY, 0); 198 199 msm_devfreq_profile.initial_freq = gpu->fast_rate; 200 201 /* 202 * Don't set the freq_table or max_state and let devfreq build the table 203 * from OPP 204 * After a deferred probe, these may have be left to non-zero values, 205 * so set them back to zero before creating the devfreq device 206 */ 207 msm_devfreq_profile.freq_table = NULL; 208 msm_devfreq_profile.max_state = 0; 209 210 df->devfreq = devm_devfreq_add_device(&gpu->pdev->dev, 211 &msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, 212 NULL); 213 214 if (IS_ERR(df->devfreq)) { 215 DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n"); 216 dev_pm_qos_remove_request(&df->idle_freq); 217 dev_pm_qos_remove_request(&df->boost_freq); 218 df->devfreq = NULL; 219 return; 220 } 221 222 devfreq_suspend_device(df->devfreq); 223 224 gpu->cooling = of_devfreq_cooling_register(gpu->pdev->dev.of_node, df->devfreq); 225 if (IS_ERR(gpu->cooling)) { 226 DRM_DEV_ERROR(&gpu->pdev->dev, 227 "Couldn't register GPU cooling device\n"); 228 gpu->cooling = NULL; 229 } 230 231 msm_hrtimer_work_init(&df->boost_work, gpu->worker, msm_devfreq_boost_work, 232 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 233 msm_hrtimer_work_init(&df->idle_work, gpu->worker, msm_devfreq_idle_work, 234 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 235 } 236 237 static void cancel_idle_work(struct msm_gpu_devfreq *df) 238 { 239 hrtimer_cancel(&df->idle_work.timer); 240 kthread_cancel_work_sync(&df->idle_work.work); 241 } 242 243 static void cancel_boost_work(struct msm_gpu_devfreq *df) 244 { 245 hrtimer_cancel(&df->boost_work.timer); 246 kthread_cancel_work_sync(&df->boost_work.work); 247 } 248 249 void msm_devfreq_cleanup(struct msm_gpu *gpu) 250 { 251 struct msm_gpu_devfreq *df = &gpu->devfreq; 252 253 if (!has_devfreq(gpu)) 254 return; 255 256 devfreq_cooling_unregister(gpu->cooling); 257 dev_pm_qos_remove_request(&df->boost_freq); 258 dev_pm_qos_remove_request(&df->idle_freq); 259 } 260 261 void msm_devfreq_resume(struct msm_gpu *gpu) 262 { 263 struct msm_gpu_devfreq *df = &gpu->devfreq; 264 unsigned long sample_rate; 265 266 if (!has_devfreq(gpu)) 267 return; 268 269 mutex_lock(&df->lock); 270 df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); 271 df->time = ktime_get(); 272 df->suspended = false; 273 mutex_unlock(&df->lock); 274 275 devfreq_resume_device(df->devfreq); 276 } 277 278 void msm_devfreq_suspend(struct msm_gpu *gpu) 279 { 280 struct msm_gpu_devfreq *df = &gpu->devfreq; 281 282 if (!has_devfreq(gpu)) 283 return; 284 285 mutex_lock(&df->lock); 286 df->suspended = true; 287 mutex_unlock(&df->lock); 288 289 devfreq_suspend_device(df->devfreq); 290 291 cancel_idle_work(df); 292 cancel_boost_work(df); 293 } 294 295 static void msm_devfreq_boost_work(struct kthread_work *work) 296 { 297 struct msm_gpu_devfreq *df = container_of(work, 298 struct msm_gpu_devfreq, boost_work.work); 299 300 dev_pm_qos_update_request(&df->boost_freq, 0); 301 } 302 303 void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor) 304 { 305 struct msm_gpu_devfreq *df = &gpu->devfreq; 306 uint64_t freq; 307 308 if (!has_devfreq(gpu)) 309 return; 310 311 freq = get_freq(gpu); 312 freq *= factor; 313 314 /* 315 * A nice little trap is that PM QoS operates in terms of KHz, 316 * while devfreq operates in terms of Hz: 317 */ 318 do_div(freq, HZ_PER_KHZ); 319 320 dev_pm_qos_update_request(&df->boost_freq, freq); 321 322 msm_hrtimer_queue_work(&df->boost_work, 323 ms_to_ktime(msm_devfreq_profile.polling_ms), 324 HRTIMER_MODE_REL); 325 } 326 327 void msm_devfreq_active(struct msm_gpu *gpu) 328 { 329 struct msm_gpu_devfreq *df = &gpu->devfreq; 330 unsigned int idle_time; 331 332 if (!has_devfreq(gpu)) 333 return; 334 335 /* 336 * Cancel any pending transition to idle frequency: 337 */ 338 cancel_idle_work(df); 339 340 idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time)); 341 342 /* 343 * If we've been idle for a significant fraction of a polling 344 * interval, then we won't meet the threshold of busyness for 345 * the governor to ramp up the freq.. so give some boost 346 */ 347 if (idle_time > msm_devfreq_profile.polling_ms) { 348 msm_devfreq_boost(gpu, 2); 349 } 350 351 dev_pm_qos_update_request(&df->idle_freq, 352 PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); 353 } 354 355 356 static void msm_devfreq_idle_work(struct kthread_work *work) 357 { 358 struct msm_gpu_devfreq *df = container_of(work, 359 struct msm_gpu_devfreq, idle_work.work); 360 struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq); 361 362 df->idle_time = ktime_get(); 363 364 if (gpu->clamp_to_idle) 365 dev_pm_qos_update_request(&df->idle_freq, 0); 366 } 367 368 void msm_devfreq_idle(struct msm_gpu *gpu) 369 { 370 struct msm_gpu_devfreq *df = &gpu->devfreq; 371 372 if (!has_devfreq(gpu)) 373 return; 374 375 msm_hrtimer_queue_work(&df->idle_work, ms_to_ktime(1), 376 HRTIMER_MODE_REL); 377 } 378