53 static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm)  in rpm_to_i915()  argument
55 	return container_of(rpm, struct drm_i915_private, runtime_pm);  in rpm_to_i915()
60 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
62 	if (!rpm->debug.class)  in init_intel_runtime_pm_wakeref()
63 		ref_tracker_dir_init(&rpm->debug, INTEL_REFTRACK_DEAD_COUNT,  in init_intel_runtime_pm_wakeref()
68 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
70 	if (!rpm->available || rpm->no_wakeref_tracking)  in track_intel_runtime_pm_wakeref()
73 	return intel_ref_tracker_alloc(&rpm->debug);  in track_intel_runtime_pm_wakeref()
76 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
79 	if (!rpm->available || rpm->no_wakeref_tracking)  in untrack_intel_runtime_pm_wakeref()
82 	intel_ref_tracker_free(&rpm->debug, wakeref);  in untrack_intel_runtime_pm_wakeref()
85 static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
87 	ref_tracker_dir_exit(&rpm->debug);  in untrack_all_intel_runtime_pm_wakerefs()
91 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
95 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
96 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
100 	ref_tracker_dir_print_locked(&rpm->debug, INTEL_REFTRACK_PRINT_LIMIT);  in __intel_wakeref_dec_and_check_tracking()
101 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
104 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
107 	intel_ref_tracker_show(&rpm->debug, p);  in print_intel_runtime_pm_wakeref()
112 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
117 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
122 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
128 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
130 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
134 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
141 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
144 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
145 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
147 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
148 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
153 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
156 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
157 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
159 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
162 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
165 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
168 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in __intel_runtime_pm_get()
171 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
175 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
177 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
182  * @rpm: the intel_runtime_pm structure
197 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
199 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
204  * @rpm: the intel_runtime_pm structure
214 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
216 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
221  * @rpm: the intel_runtime_pm structure
241 static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get_if_active()  argument
246 		 * In cases runtime PM is disabled by the RPM core and we get  in __intel_runtime_pm_get_if_active()
252 		     pm_runtime_get_if_active(rpm->kdev) <= 0) ||  in __intel_runtime_pm_get_if_active()
254 		     pm_runtime_get_if_in_use(rpm->kdev) <= 0))  in __intel_runtime_pm_get_if_active()
258 	intel_runtime_pm_acquire(rpm, true);  in __intel_runtime_pm_get_if_active()
260 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get_if_active()
263 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
265 	return __intel_runtime_pm_get_if_active(rpm, false);  in intel_runtime_pm_get_if_in_use()
268 intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_active()  argument
270 	return __intel_runtime_pm_get_if_active(rpm, true);  in intel_runtime_pm_get_if_active()
275  * @rpm: the intel_runtime_pm structure
288 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
290 	assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_get_noresume()
291 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
293 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
295 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
298 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
302 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
304 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
306 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
314  * @rpm: the intel_runtime_pm structure
322 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
324 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
329  * @rpm: the intel_runtime_pm structure
339 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
341 	__intel_runtime_pm_put(rpm, INTEL_WAKEREF_DEF, true);  in intel_runtime_pm_put_unchecked()
347  * @rpm: the intel_runtime_pm structure
354 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
356 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
362  * @rpm: the intel_runtime_pm structure
370 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
372 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_enable()
373 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
389 	 * Take a permanent reference to disable the RPM functionality and drop  in intel_runtime_pm_enable()
391 	 * so the driver's own RPM reference tracking asserts also work on  in intel_runtime_pm_enable()
392 	 * platforms without RPM support.  in intel_runtime_pm_enable()
394 	if (!rpm->available) {  in intel_runtime_pm_enable()
416 	 * The core calls the driver load handler with an RPM reference held.  in intel_runtime_pm_enable()
423 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
425 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_disable()
426 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
428 	/* Transfer rpm ownership back to core */  in intel_runtime_pm_disable()
430 		 "Failed to pass rpm ownership back to core\n");  in intel_runtime_pm_disable()
434 	if (!rpm->available)  in intel_runtime_pm_disable()
438 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
440 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_driver_release()
441 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
443 	intel_wakeref_auto_fini(&rpm->userfault_wakeref);  in intel_runtime_pm_driver_release()
451 void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_last_release()  argument
453 	intel_runtime_pm_driver_release(rpm);  in intel_runtime_pm_driver_last_release()
454 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_last_release()
457 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
459 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_init_early()
463 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
464 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
465 	atomic_set(&rpm->wakeref_count, 0);  in intel_runtime_pm_init_early()
467 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()
468 	INIT_LIST_HEAD(&rpm->lmem_userfault_list);  in intel_runtime_pm_init_early()
469 	spin_lock_init(&rpm->lmem_userfault_lock);  in intel_runtime_pm_init_early()
470 	intel_wakeref_auto_init(&rpm->userfault_wakeref, i915);  in intel_runtime_pm_init_early()