1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 #include <linux/suspend.h> 8 9 #include "i915_drv.h" 10 #include "i915_irq.h" 11 #include "i915_params.h" 12 #include "intel_context.h" 13 #include "intel_engine_pm.h" 14 #include "intel_gt.h" 15 #include "intel_gt_clock_utils.h" 16 #include "intel_gt_mcr.h" 17 #include "intel_gt_pm.h" 18 #include "intel_gt_print.h" 19 #include "intel_gt_requests.h" 20 #include "intel_llc.h" 21 #include "intel_rc6.h" 22 #include "intel_rps.h" 23 #include "intel_wakeref.h" 24 #include "pxp/intel_pxp_pm.h" 25 26 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) 27 28 static void user_forcewake(struct intel_gt *gt, bool suspend) 29 { 30 int count = atomic_read(>->user_wakeref); 31 32 /* Inside suspend/resume so single threaded, no races to worry about. */ 33 if (likely(!count)) 34 return; 35 36 intel_gt_pm_get(gt); 37 if (suspend) { 38 GEM_BUG_ON(count > atomic_read(>->wakeref.count)); 39 atomic_sub(count, >->wakeref.count); 40 } else { 41 atomic_add(count, >->wakeref.count); 42 } 43 intel_gt_pm_put(gt); 44 } 45 46 static void runtime_begin(struct intel_gt *gt) 47 { 48 local_irq_disable(); 49 write_seqcount_begin(>->stats.lock); 50 gt->stats.start = ktime_get(); 51 gt->stats.active = true; 52 write_seqcount_end(>->stats.lock); 53 local_irq_enable(); 54 } 55 56 static void runtime_end(struct intel_gt *gt) 57 { 58 local_irq_disable(); 59 write_seqcount_begin(>->stats.lock); 60 gt->stats.active = false; 61 gt->stats.total = 62 ktime_add(gt->stats.total, 63 ktime_sub(ktime_get(), gt->stats.start)); 64 write_seqcount_end(>->stats.lock); 65 local_irq_enable(); 66 } 67 68 static int __gt_unpark(struct intel_wakeref *wf) 69 { 70 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 71 struct drm_i915_private *i915 = gt->i915; 72 73 GT_TRACE(gt, "\n"); 74 75 /* 76 * It seems that the DMC likes to transition between the DC states a lot 77 * when there are no connected displays (no active power domains) during 78 * command submission. 79 * 80 * This activity has negative impact on the performance of the chip with 81 * huge latencies observed in the interrupt handler and elsewhere. 82 * 83 * Work around it by grabbing a GT IRQ power domain whilst there is any 84 * GT activity, preventing any DC state transitions. 85 */ 86 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 87 GEM_BUG_ON(!gt->awake); 88 89 intel_rc6_unpark(>->rc6); 90 intel_rps_unpark(>->rps); 91 i915_pmu_gt_unparked(gt); 92 intel_guc_busyness_unpark(gt); 93 94 intel_gt_unpark_requests(gt); 95 runtime_begin(gt); 96 97 return 0; 98 } 99 100 static int __gt_park(struct intel_wakeref *wf) 101 { 102 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 103 intel_wakeref_t wakeref = fetch_and_zero(>->awake); 104 struct drm_i915_private *i915 = gt->i915; 105 106 GT_TRACE(gt, "\n"); 107 108 runtime_end(gt); 109 intel_gt_park_requests(gt); 110 111 intel_guc_busyness_park(gt); 112 i915_vma_parked(gt); 113 i915_pmu_gt_parked(gt); 114 intel_rps_park(>->rps); 115 intel_rc6_park(>->rc6); 116 117 /* Everything switched off, flush any residual interrupt just in case */ 118 intel_synchronize_irq(i915); 119 120 /* Defer dropping the display power well for 100ms, it's slow! */ 121 GEM_BUG_ON(!wakeref); 122 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); 123 124 return 0; 125 } 126 127 static const struct intel_wakeref_ops wf_ops = { 128 .get = __gt_unpark, 129 .put = __gt_park, 130 }; 131 132 void intel_gt_pm_init_early(struct intel_gt *gt) 133 { 134 /* 135 * We access the runtime_pm structure via gt->i915 here rather than 136 * gt->uncore as we do elsewhere in the file because gt->uncore is not 137 * yet initialized for all tiles at this point in the driver startup. 138 * runtime_pm is per-device rather than per-tile, so this is still the 139 * correct structure. 140 */ 141 intel_wakeref_init(>->wakeref, gt->i915, &wf_ops); 142 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); 143 } 144 145 void intel_gt_pm_init(struct intel_gt *gt) 146 { 147 /* 148 * Enabling power-management should be "self-healing". If we cannot 149 * enable a feature, simply leave it disabled with a notice to the 150 * user. 151 */ 152 intel_rc6_init(>->rc6); 153 intel_rps_init(>->rps); 154 } 155 156 static bool reset_engines(struct intel_gt *gt) 157 { 158 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 159 return false; 160 161 return __intel_gt_reset(gt, ALL_ENGINES) == 0; 162 } 163 164 static void gt_sanitize(struct intel_gt *gt, bool force) 165 { 166 struct intel_engine_cs *engine; 167 enum intel_engine_id id; 168 intel_wakeref_t wakeref; 169 170 GT_TRACE(gt, "force:%s", str_yes_no(force)); 171 172 /* Use a raw wakeref to avoid calling intel_display_power_get early */ 173 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 174 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 175 176 intel_gt_check_clock_frequency(gt); 177 178 /* 179 * As we have just resumed the machine and woken the device up from 180 * deep PCI sleep (presumably D3_cold), assume the HW has been reset 181 * back to defaults, recovering from whatever wedged state we left it 182 * in and so worth trying to use the device once more. 183 */ 184 if (intel_gt_is_wedged(gt)) 185 intel_gt_unset_wedged(gt); 186 187 /* For GuC mode, ensure submission is disabled before stopping ring */ 188 intel_uc_reset_prepare(>->uc); 189 190 for_each_engine(engine, gt, id) { 191 if (engine->reset.prepare) 192 engine->reset.prepare(engine); 193 194 if (engine->sanitize) 195 engine->sanitize(engine); 196 } 197 198 if (reset_engines(gt) || force) { 199 for_each_engine(engine, gt, id) 200 __intel_engine_reset(engine, false); 201 } 202 203 intel_uc_reset(>->uc, false); 204 205 for_each_engine(engine, gt, id) 206 if (engine->reset.finish) 207 engine->reset.finish(engine); 208 209 intel_rps_sanitize(>->rps); 210 211 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 212 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 213 } 214 215 void intel_gt_pm_fini(struct intel_gt *gt) 216 { 217 intel_rc6_fini(>->rc6); 218 } 219 220 void intel_gt_resume_early(struct intel_gt *gt) 221 { 222 /* 223 * Sanitize steer semaphores during driver resume. This is necessary 224 * to address observed cases of steer semaphores being 225 * held after a suspend operation. Confirmation from the hardware team 226 * assures the safety of this operation, as no lock acquisitions 227 * by other agents occur during driver load/resume process. 228 */ 229 intel_gt_mcr_lock_sanitize(gt); 230 231 intel_uncore_resume_early(gt->uncore); 232 intel_gt_check_and_clear_faults(gt); 233 } 234 235 int intel_gt_resume(struct intel_gt *gt) 236 { 237 struct intel_engine_cs *engine; 238 enum intel_engine_id id; 239 int err; 240 241 err = intel_gt_has_unrecoverable_error(gt); 242 if (err) 243 return err; 244 245 GT_TRACE(gt, "\n"); 246 247 /* 248 * After resume, we may need to poke into the pinned kernel 249 * contexts to paper over any damage caused by the sudden suspend. 250 * Only the kernel contexts should remain pinned over suspend, 251 * allowing us to fixup the user contexts on their first pin. 252 */ 253 gt_sanitize(gt, true); 254 255 intel_gt_pm_get(gt); 256 257 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 258 intel_rc6_sanitize(>->rc6); 259 if (intel_gt_is_wedged(gt)) { 260 err = -EIO; 261 goto out_fw; 262 } 263 264 /* Only when the HW is re-initialised, can we replay the requests */ 265 err = intel_gt_init_hw(gt); 266 if (err) { 267 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n"); 268 goto err_wedged; 269 } 270 271 intel_uc_reset_finish(>->uc); 272 273 intel_rps_enable(>->rps); 274 intel_llc_enable(>->llc); 275 276 for_each_engine(engine, gt, id) { 277 intel_engine_pm_get(engine); 278 279 engine->serial++; /* kernel context lost */ 280 err = intel_engine_resume(engine); 281 282 intel_engine_pm_put(engine); 283 if (err) { 284 gt_err(gt, "Failed to restart %s (%d)\n", 285 engine->name, err); 286 goto err_wedged; 287 } 288 } 289 290 intel_rc6_enable(>->rc6); 291 292 intel_uc_resume(>->uc); 293 294 user_forcewake(gt, false); 295 296 out_fw: 297 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 298 intel_gt_pm_put(gt); 299 intel_gt_bind_context_set_ready(gt); 300 return err; 301 302 err_wedged: 303 intel_gt_set_wedged(gt); 304 goto out_fw; 305 } 306 307 static void wait_for_suspend(struct intel_gt *gt) 308 { 309 if (!intel_gt_pm_is_awake(gt)) 310 return; 311 312 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { 313 /* 314 * Forcibly cancel outstanding work and leave 315 * the gpu quiet. 316 */ 317 intel_gt_set_wedged(gt); 318 intel_gt_retire_requests(gt); 319 } 320 321 intel_gt_pm_wait_for_idle(gt); 322 } 323 324 void intel_gt_suspend_prepare(struct intel_gt *gt) 325 { 326 intel_gt_bind_context_set_unready(gt); 327 user_forcewake(gt, true); 328 wait_for_suspend(gt); 329 } 330 331 static suspend_state_t pm_suspend_target(void) 332 { 333 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) 334 return pm_suspend_target_state; 335 #else 336 return PM_SUSPEND_TO_IDLE; 337 #endif 338 } 339 340 void intel_gt_suspend_late(struct intel_gt *gt) 341 { 342 intel_wakeref_t wakeref; 343 344 /* We expect to be idle already; but also want to be independent */ 345 wait_for_suspend(gt); 346 347 if (is_mock_gt(gt)) 348 return; 349 350 GEM_BUG_ON(gt->awake); 351 352 intel_uc_suspend(>->uc); 353 354 /* 355 * On disabling the device, we want to turn off HW access to memory 356 * that we no longer own. 357 * 358 * However, not all suspend-states disable the device. S0 (s2idle) 359 * is effectively runtime-suspend, the device is left powered on 360 * but needs to be put into a low power state. We need to keep 361 * powermanagement enabled, but we also retain system state and so 362 * it remains safe to keep on using our allocated memory. 363 */ 364 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) 365 return; 366 367 with_intel_runtime_pm(gt->uncore->rpm, wakeref) { 368 intel_rps_disable(>->rps); 369 intel_rc6_disable(>->rc6); 370 intel_llc_disable(>->llc); 371 } 372 373 gt_sanitize(gt, false); 374 375 GT_TRACE(gt, "\n"); 376 } 377 378 void intel_gt_runtime_suspend(struct intel_gt *gt) 379 { 380 intel_gt_bind_context_set_unready(gt); 381 intel_uc_runtime_suspend(>->uc); 382 383 GT_TRACE(gt, "\n"); 384 } 385 386 int intel_gt_runtime_resume(struct intel_gt *gt) 387 { 388 int ret; 389 390 GT_TRACE(gt, "\n"); 391 intel_gt_init_swizzling(gt); 392 intel_ggtt_restore_fences(gt->ggtt); 393 394 ret = intel_uc_runtime_resume(>->uc); 395 if (ret) 396 return ret; 397 398 intel_gt_bind_context_set_ready(gt); 399 return 0; 400 } 401 402 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) 403 { 404 ktime_t total = gt->stats.total; 405 406 if (gt->stats.active) 407 total = ktime_add(total, 408 ktime_sub(ktime_get(), gt->stats.start)); 409 410 return total; 411 } 412 413 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt) 414 { 415 unsigned int seq; 416 ktime_t total; 417 418 do { 419 seq = read_seqcount_begin(>->stats.lock); 420 total = __intel_gt_get_awake_time(gt); 421 } while (read_seqcount_retry(>->stats.lock, seq)); 422 423 return total; 424 } 425 426 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 427 #include "selftest_gt_pm.c" 428 #endif 429