1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/dma-fence-array.h> 8 #include <linux/dma-fence-chain.h> 9 #include <linux/jiffies.h> 10 11 #include "gt/intel_engine.h" 12 13 #include "i915_gem_ioctls.h" 14 #include "i915_gem_object.h" 15 16 static long 17 i915_gem_object_wait_fence(struct dma_fence *fence, 18 unsigned int flags, 19 long timeout) 20 { 21 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 22 23 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 24 return timeout; 25 26 if (dma_fence_is_i915(fence)) 27 return i915_request_wait_timeout(to_request(fence), flags, timeout); 28 29 return dma_fence_wait_timeout(fence, 30 flags & I915_WAIT_INTERRUPTIBLE, 31 timeout); 32 } 33 34 static long 35 i915_gem_object_wait_reservation(struct dma_resv *resv, 36 unsigned int flags, 37 long timeout) 38 { 39 struct dma_resv_iter cursor; 40 struct dma_fence *fence; 41 long ret = timeout ?: 1; 42 43 dma_resv_iter_begin(&cursor, resv, 44 dma_resv_usage_rw(flags & I915_WAIT_ALL)); 45 dma_resv_for_each_fence_unlocked(&cursor, fence) { 46 ret = i915_gem_object_wait_fence(fence, flags, timeout); 47 if (ret <= 0) 48 break; 49 50 if (timeout) 51 timeout = ret; 52 } 53 dma_resv_iter_end(&cursor); 54 55 return ret; 56 } 57 58 static void fence_set_priority(struct dma_fence *fence, 59 const struct i915_sched_attr *attr) 60 { 61 struct i915_request *rq; 62 struct intel_engine_cs *engine; 63 64 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 65 return; 66 67 rq = to_request(fence); 68 engine = rq->engine; 69 70 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 71 if (engine->sched_engine->schedule) 72 engine->sched_engine->schedule(rq, attr); 73 rcu_read_unlock(); 74 } 75 76 static inline bool __dma_fence_is_chain(const struct dma_fence *fence) 77 { 78 return fence->ops == &dma_fence_chain_ops; 79 } 80 81 void i915_gem_fence_wait_priority(struct dma_fence *fence, 82 const struct i915_sched_attr *attr) 83 { 84 if (dma_fence_is_signaled(fence)) 85 return; 86 87 local_bh_disable(); 88 89 /* Recurse once into a fence-array */ 90 if (dma_fence_is_array(fence)) { 91 struct dma_fence_array *array = to_dma_fence_array(fence); 92 int i; 93 94 for (i = 0; i < array->num_fences; i++) 95 fence_set_priority(array->fences[i], attr); 96 } else if (__dma_fence_is_chain(fence)) { 97 struct dma_fence *iter; 98 99 /* The chain is ordered; if we boost the last, we boost all */ 100 dma_fence_chain_for_each(iter, fence) { 101 fence_set_priority(to_dma_fence_chain(iter)->fence, 102 attr); 103 break; 104 } 105 dma_fence_put(iter); 106 } else { 107 fence_set_priority(fence, attr); 108 } 109 110 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 111 } 112 113 int 114 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 115 unsigned int flags, 116 const struct i915_sched_attr *attr) 117 { 118 struct dma_resv_iter cursor; 119 struct dma_fence *fence; 120 121 dma_resv_iter_begin(&cursor, obj->base.resv, 122 dma_resv_usage_rw(flags & I915_WAIT_ALL)); 123 dma_resv_for_each_fence_unlocked(&cursor, fence) 124 i915_gem_fence_wait_priority(fence, attr); 125 dma_resv_iter_end(&cursor); 126 return 0; 127 } 128 129 /** 130 * Waits for rendering to the object to be completed 131 * @obj: i915 gem object 132 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 133 * @timeout: how long to wait 134 */ 135 int 136 i915_gem_object_wait(struct drm_i915_gem_object *obj, 137 unsigned int flags, 138 long timeout) 139 { 140 might_sleep(); 141 GEM_BUG_ON(timeout < 0); 142 143 timeout = i915_gem_object_wait_reservation(obj->base.resv, 144 flags, timeout); 145 146 if (timeout < 0) 147 return timeout; 148 149 return !timeout ? -ETIME : 0; 150 } 151 152 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 153 { 154 /* nsecs_to_jiffies64() does not guard against overflow */ 155 if (NSEC_PER_SEC % HZ && 156 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 157 return MAX_JIFFY_OFFSET; 158 159 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 160 } 161 162 static unsigned long to_wait_timeout(s64 timeout_ns) 163 { 164 if (timeout_ns < 0) 165 return MAX_SCHEDULE_TIMEOUT; 166 167 if (timeout_ns == 0) 168 return 0; 169 170 return nsecs_to_jiffies_timeout(timeout_ns); 171 } 172 173 /** 174 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 175 * @dev: drm device pointer 176 * @data: ioctl data blob 177 * @file: drm file pointer 178 * 179 * Returns 0 if successful, else an error is returned with the remaining time in 180 * the timeout parameter. 181 * -ETIME: object is still busy after timeout 182 * -ERESTARTSYS: signal interrupted the wait 183 * -ENONENT: object doesn't exist 184 * Also possible, but rare: 185 * -EAGAIN: incomplete, restart syscall 186 * -ENOMEM: damn 187 * -ENODEV: Internal IRQ fail 188 * -E?: The add request failed 189 * 190 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 191 * non-zero timeout parameter the wait ioctl will wait for the given number of 192 * nanoseconds on an object becoming unbusy. Since the wait itself does so 193 * without holding struct_mutex the object may become re-busied before this 194 * function completes. A similar but shorter * race condition exists in the busy 195 * ioctl 196 */ 197 int 198 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 199 { 200 struct drm_i915_gem_wait *args = data; 201 struct drm_i915_gem_object *obj; 202 ktime_t start; 203 long ret; 204 205 if (args->flags != 0) 206 return -EINVAL; 207 208 obj = i915_gem_object_lookup(file, args->bo_handle); 209 if (!obj) 210 return -ENOENT; 211 212 start = ktime_get(); 213 214 ret = i915_gem_object_wait(obj, 215 I915_WAIT_INTERRUPTIBLE | 216 I915_WAIT_PRIORITY | 217 I915_WAIT_ALL, 218 to_wait_timeout(args->timeout_ns)); 219 220 if (args->timeout_ns > 0) { 221 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 222 if (args->timeout_ns < 0) 223 args->timeout_ns = 0; 224 225 /* 226 * Apparently ktime isn't accurate enough and occasionally has a 227 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 228 * things up to make the test happy. We allow up to 1 jiffy. 229 * 230 * This is a regression from the timespec->ktime conversion. 231 */ 232 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 233 args->timeout_ns = 0; 234 235 /* Asked to wait beyond the jiffie/scheduler precision? */ 236 if (ret == -ETIME && args->timeout_ns) 237 ret = -EAGAIN; 238 } 239 240 i915_gem_object_put(obj); 241 return ret; 242 } 243 244 /** 245 * i915_gem_object_wait_migration - Sync an accelerated migration operation 246 * @obj: The migrating object. 247 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE. 248 * 249 * Wait for any pending async migration operation on the object, 250 * whether it's explicitly (i915_gem_object_migrate()) or implicitly 251 * (swapin, initial clearing) initiated. 252 * 253 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting. 254 */ 255 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 256 unsigned int flags) 257 { 258 might_sleep(); 259 260 return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE)); 261 } 262