1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/dma-fence-array.h> 8 #include <linux/dma-fence-chain.h> 9 #include <linux/jiffies.h> 10 11 #include "gt/intel_engine.h" 12 13 #include "dma_resv_utils.h" 14 #include "i915_gem_ioctls.h" 15 #include "i915_gem_object.h" 16 17 static long 18 i915_gem_object_wait_fence(struct dma_fence *fence, 19 unsigned int flags, 20 long timeout) 21 { 22 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 23 24 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 25 return timeout; 26 27 if (dma_fence_is_i915(fence)) 28 return i915_request_wait(to_request(fence), flags, timeout); 29 30 return dma_fence_wait_timeout(fence, 31 flags & I915_WAIT_INTERRUPTIBLE, 32 timeout); 33 } 34 35 static long 36 i915_gem_object_wait_reservation(struct dma_resv *resv, 37 unsigned int flags, 38 long timeout) 39 { 40 struct dma_fence *excl; 41 bool prune_fences = false; 42 43 if (flags & I915_WAIT_ALL) { 44 struct dma_fence **shared; 45 unsigned int count, i; 46 int ret; 47 48 ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared); 49 if (ret) 50 return ret; 51 52 for (i = 0; i < count; i++) { 53 timeout = i915_gem_object_wait_fence(shared[i], 54 flags, timeout); 55 if (timeout < 0) 56 break; 57 58 dma_fence_put(shared[i]); 59 } 60 61 for (; i < count; i++) 62 dma_fence_put(shared[i]); 63 kfree(shared); 64 65 /* 66 * If both shared fences and an exclusive fence exist, 67 * then by construction the shared fences must be later 68 * than the exclusive fence. If we successfully wait for 69 * all the shared fences, we know that the exclusive fence 70 * must all be signaled. If all the shared fences are 71 * signaled, we can prune the array and recover the 72 * floating references on the fences/requests. 73 */ 74 prune_fences = count && timeout >= 0; 75 } else { 76 excl = dma_resv_get_excl_rcu(resv); 77 } 78 79 if (excl && timeout >= 0) 80 timeout = i915_gem_object_wait_fence(excl, flags, timeout); 81 82 dma_fence_put(excl); 83 84 /* 85 * Opportunistically prune the fences iff we know they have *all* been 86 * signaled. 87 */ 88 if (prune_fences) 89 dma_resv_prune(resv); 90 91 return timeout; 92 } 93 94 static void fence_set_priority(struct dma_fence *fence, 95 const struct i915_sched_attr *attr) 96 { 97 struct i915_request *rq; 98 struct intel_engine_cs *engine; 99 100 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 101 return; 102 103 rq = to_request(fence); 104 engine = rq->engine; 105 106 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 107 if (engine->schedule) 108 engine->schedule(rq, attr); 109 rcu_read_unlock(); 110 } 111 112 static inline bool __dma_fence_is_chain(const struct dma_fence *fence) 113 { 114 return fence->ops == &dma_fence_chain_ops; 115 } 116 117 void i915_gem_fence_wait_priority(struct dma_fence *fence, 118 const struct i915_sched_attr *attr) 119 { 120 if (dma_fence_is_signaled(fence)) 121 return; 122 123 local_bh_disable(); 124 125 /* Recurse once into a fence-array */ 126 if (dma_fence_is_array(fence)) { 127 struct dma_fence_array *array = to_dma_fence_array(fence); 128 int i; 129 130 for (i = 0; i < array->num_fences; i++) 131 fence_set_priority(array->fences[i], attr); 132 } else if (__dma_fence_is_chain(fence)) { 133 struct dma_fence *iter; 134 135 /* The chain is ordered; if we boost the last, we boost all */ 136 dma_fence_chain_for_each(iter, fence) { 137 fence_set_priority(to_dma_fence_chain(iter)->fence, 138 attr); 139 break; 140 } 141 dma_fence_put(iter); 142 } else { 143 fence_set_priority(fence, attr); 144 } 145 146 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 147 } 148 149 int 150 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 151 unsigned int flags, 152 const struct i915_sched_attr *attr) 153 { 154 struct dma_fence *excl; 155 156 if (flags & I915_WAIT_ALL) { 157 struct dma_fence **shared; 158 unsigned int count, i; 159 int ret; 160 161 ret = dma_resv_get_fences_rcu(obj->base.resv, 162 &excl, &count, &shared); 163 if (ret) 164 return ret; 165 166 for (i = 0; i < count; i++) { 167 i915_gem_fence_wait_priority(shared[i], attr); 168 dma_fence_put(shared[i]); 169 } 170 171 kfree(shared); 172 } else { 173 excl = dma_resv_get_excl_rcu(obj->base.resv); 174 } 175 176 if (excl) { 177 i915_gem_fence_wait_priority(excl, attr); 178 dma_fence_put(excl); 179 } 180 return 0; 181 } 182 183 /** 184 * Waits for rendering to the object to be completed 185 * @obj: i915 gem object 186 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 187 * @timeout: how long to wait 188 */ 189 int 190 i915_gem_object_wait(struct drm_i915_gem_object *obj, 191 unsigned int flags, 192 long timeout) 193 { 194 might_sleep(); 195 GEM_BUG_ON(timeout < 0); 196 197 timeout = i915_gem_object_wait_reservation(obj->base.resv, 198 flags, timeout); 199 return timeout < 0 ? timeout : 0; 200 } 201 202 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 203 { 204 /* nsecs_to_jiffies64() does not guard against overflow */ 205 if (NSEC_PER_SEC % HZ && 206 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 207 return MAX_JIFFY_OFFSET; 208 209 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 210 } 211 212 static unsigned long to_wait_timeout(s64 timeout_ns) 213 { 214 if (timeout_ns < 0) 215 return MAX_SCHEDULE_TIMEOUT; 216 217 if (timeout_ns == 0) 218 return 0; 219 220 return nsecs_to_jiffies_timeout(timeout_ns); 221 } 222 223 /** 224 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 225 * @dev: drm device pointer 226 * @data: ioctl data blob 227 * @file: drm file pointer 228 * 229 * Returns 0 if successful, else an error is returned with the remaining time in 230 * the timeout parameter. 231 * -ETIME: object is still busy after timeout 232 * -ERESTARTSYS: signal interrupted the wait 233 * -ENONENT: object doesn't exist 234 * Also possible, but rare: 235 * -EAGAIN: incomplete, restart syscall 236 * -ENOMEM: damn 237 * -ENODEV: Internal IRQ fail 238 * -E?: The add request failed 239 * 240 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 241 * non-zero timeout parameter the wait ioctl will wait for the given number of 242 * nanoseconds on an object becoming unbusy. Since the wait itself does so 243 * without holding struct_mutex the object may become re-busied before this 244 * function completes. A similar but shorter * race condition exists in the busy 245 * ioctl 246 */ 247 int 248 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 249 { 250 struct drm_i915_gem_wait *args = data; 251 struct drm_i915_gem_object *obj; 252 ktime_t start; 253 long ret; 254 255 if (args->flags != 0) 256 return -EINVAL; 257 258 obj = i915_gem_object_lookup(file, args->bo_handle); 259 if (!obj) 260 return -ENOENT; 261 262 start = ktime_get(); 263 264 ret = i915_gem_object_wait(obj, 265 I915_WAIT_INTERRUPTIBLE | 266 I915_WAIT_PRIORITY | 267 I915_WAIT_ALL, 268 to_wait_timeout(args->timeout_ns)); 269 270 if (args->timeout_ns > 0) { 271 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 272 if (args->timeout_ns < 0) 273 args->timeout_ns = 0; 274 275 /* 276 * Apparently ktime isn't accurate enough and occasionally has a 277 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 278 * things up to make the test happy. We allow up to 1 jiffy. 279 * 280 * This is a regression from the timespec->ktime conversion. 281 */ 282 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 283 args->timeout_ns = 0; 284 285 /* Asked to wait beyond the jiffie/scheduler precision? */ 286 if (ret == -ETIME && args->timeout_ns) 287 ret = -EAGAIN; 288 } 289 290 i915_gem_object_put(obj); 291 return ret; 292 } 293