1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/wait_bit.h> 8 9 #include "intel_runtime_pm.h" 10 #include "intel_wakeref.h" 11 #include "i915_drv.h" 12 13 int __intel_wakeref_get_first(struct intel_wakeref *wf) 14 { 15 intel_wakeref_t wakeref; 16 int ret = 0; 17 18 wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm); 19 /* 20 * Treat get/put as different subclasses, as we may need to run 21 * the put callback from under the shrinker and do not want to 22 * cross-contanimate that callback with any extra work performed 23 * upon acquiring the wakeref. 24 */ 25 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING); 26 27 if (!atomic_read(&wf->count)) { 28 INTEL_WAKEREF_BUG_ON(wf->wakeref); 29 wf->wakeref = wakeref; 30 wakeref = NULL; 31 32 ret = wf->ops->get(wf); 33 if (ret) { 34 wakeref = xchg(&wf->wakeref, NULL); 35 wake_up_var(&wf->wakeref); 36 goto unlock; 37 } 38 39 smp_mb__before_atomic(); /* release wf->count */ 40 } 41 42 atomic_inc(&wf->count); 43 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); 44 45 unlock: 46 mutex_unlock(&wf->mutex); 47 if (unlikely(wakeref)) 48 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); 49 50 return ret; 51 } 52 53 static void ____intel_wakeref_put_last(struct intel_wakeref *wf) 54 { 55 intel_wakeref_t wakeref = NULL; 56 57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); 58 if (unlikely(!atomic_dec_and_test(&wf->count))) 59 goto unlock; 60 61 /* ops->put() must reschedule its own release on error/deferral */ 62 if (likely(!wf->ops->put(wf))) { 63 INTEL_WAKEREF_BUG_ON(!wf->wakeref); 64 wakeref = xchg(&wf->wakeref, NULL); 65 wake_up_var(&wf->wakeref); 66 } 67 68 unlock: 69 mutex_unlock(&wf->mutex); 70 if (wakeref) 71 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); 72 } 73 74 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) 75 { 76 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); 77 78 /* Assume we are not in process context and so cannot sleep. */ 79 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { 80 mod_delayed_work(wf->i915->unordered_wq, &wf->work, 81 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); 82 return; 83 } 84 85 ____intel_wakeref_put_last(wf); 86 } 87 88 static void __intel_wakeref_put_work(struct work_struct *wrk) 89 { 90 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); 91 92 if (atomic_add_unless(&wf->count, -1, 1)) 93 return; 94 95 mutex_lock(&wf->mutex); 96 ____intel_wakeref_put_last(wf); 97 } 98 99 void __intel_wakeref_init(struct intel_wakeref *wf, 100 struct drm_i915_private *i915, 101 const struct intel_wakeref_ops *ops, 102 struct intel_wakeref_lockclass *key, 103 const char *name) 104 { 105 wf->i915 = i915; 106 wf->ops = ops; 107 108 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex); 109 atomic_set(&wf->count, 0); 110 wf->wakeref = NULL; 111 112 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); 113 lockdep_init_map(&wf->work.work.lockdep_map, 114 "wakeref.work", &key->work, 0); 115 116 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF) 117 if (!wf->debug.class) 118 ref_tracker_dir_init(&wf->debug, INTEL_REFTRACK_DEAD_COUNT, "intel_wakeref"); 119 #endif 120 } 121 122 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) 123 { 124 int err; 125 126 might_sleep(); 127 128 err = wait_var_event_killable(&wf->wakeref, 129 !intel_wakeref_is_active(wf)); 130 if (err) 131 return err; 132 133 intel_wakeref_unlock_wait(wf); 134 return 0; 135 } 136 137 static void wakeref_auto_timeout(struct timer_list *t) 138 { 139 struct intel_wakeref_auto *wf = timer_container_of(wf, t, timer); 140 intel_wakeref_t wakeref; 141 unsigned long flags; 142 143 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags)) 144 return; 145 146 wakeref = xchg(&wf->wakeref, NULL); 147 spin_unlock_irqrestore(&wf->lock, flags); 148 149 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); 150 } 151 152 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, 153 struct drm_i915_private *i915) 154 { 155 spin_lock_init(&wf->lock); 156 timer_setup(&wf->timer, wakeref_auto_timeout, 0); 157 refcount_set(&wf->count, 0); 158 wf->wakeref = NULL; 159 wf->i915 = i915; 160 } 161 162 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) 163 { 164 unsigned long flags; 165 166 if (!timeout) { 167 if (timer_delete_sync(&wf->timer)) 168 wakeref_auto_timeout(&wf->timer); 169 return; 170 } 171 172 /* Our mission is that we only extend an already active wakeref */ 173 assert_rpm_wakelock_held(&wf->i915->runtime_pm); 174 175 if (!refcount_inc_not_zero(&wf->count)) { 176 spin_lock_irqsave(&wf->lock, flags); 177 if (!refcount_inc_not_zero(&wf->count)) { 178 INTEL_WAKEREF_BUG_ON(wf->wakeref); 179 wf->wakeref = 180 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm); 181 refcount_set(&wf->count, 1); 182 } 183 spin_unlock_irqrestore(&wf->lock, flags); 184 } 185 186 /* 187 * If we extend a pending timer, we will only get a single timer 188 * callback and so need to cancel the local inc by running the 189 * elided callback to keep the wf->count balanced. 190 */ 191 if (mod_timer(&wf->timer, jiffies + timeout)) 192 wakeref_auto_timeout(&wf->timer); 193 } 194 195 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) 196 { 197 intel_wakeref_auto(wf, 0); 198 INTEL_WAKEREF_BUG_ON(wf->wakeref); 199 } 200 201 void intel_ref_tracker_show(struct ref_tracker_dir *dir, 202 struct drm_printer *p) 203 { 204 const size_t buf_size = PAGE_SIZE; 205 char *buf, *sb, *se; 206 size_t count; 207 208 buf = kmalloc(buf_size, GFP_NOWAIT); 209 if (!buf) 210 return; 211 212 count = ref_tracker_dir_snprint(dir, buf, buf_size); 213 if (!count) 214 goto free; 215 /* printk does not like big buffers, so we split it */ 216 for (sb = buf; *sb; sb = se + 1) { 217 se = strchrnul(sb, '\n'); 218 drm_printf(p, "%.*s", (int)(se - sb + 1), sb); 219 if (!*se) 220 break; 221 } 222 if (count >= buf_size) 223 drm_printf(p, "\n...dropped %zd extra bytes of leak report.\n", 224 count + 1 - buf_size); 225 free: 226 kfree(buf); 227 } 228