Lines Matching +full:reference +full:- +full:sync
1 // SPDX-License-Identifier: MIT
8 #include <linux/dma-fence-array.h>
38 mmdrop(ufence->mm); in user_fence_destroy()
44 kref_get(&ufence->refcount); in user_fence_get()
49 kref_put(&ufence->refcount, user_fence_destroy); in user_fence_put()
60 return ERR_PTR(-EFAULT); in user_fence_create()
64 return ERR_PTR(-ENOMEM); in user_fence_create()
66 ufence->xe = xe; in user_fence_create()
67 kref_init(&ufence->refcount); in user_fence_create()
68 ufence->addr = ptr; in user_fence_create()
69 ufence->value = value; in user_fence_create()
70 ufence->mm = current->mm; in user_fence_create()
71 mmgrab(ufence->mm); in user_fence_create()
80 WRITE_ONCE(ufence->signalled, 1); in user_fence_worker()
81 if (mmget_not_zero(ufence->mm)) { in user_fence_worker()
82 kthread_use_mm(ufence->mm); in user_fence_worker()
83 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) in user_fence_worker()
85 kthread_unuse_mm(ufence->mm); in user_fence_worker()
86 mmput(ufence->mm); in user_fence_worker()
88 drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n"); in user_fence_worker()
93 * to safely reuse the same ufence without encountering -EBUSY errors. in user_fence_worker()
95 wake_up_all(&ufence->xe->ufence_wq); in user_fence_worker()
101 INIT_WORK(&ufence->worker, user_fence_worker); in kick_ufence()
102 queue_work(ufence->xe->ordered_wq, &ufence->worker); in kick_ufence()
114 struct xe_sync_entry *sync, in xe_sync_entry_parse() argument
128 return -EFAULT; in xe_sync_entry_parse()
132 return -EINVAL; in xe_sync_entry_parse()
138 return -EOPNOTSUPP; in xe_sync_entry_parse()
141 return -EINVAL; in xe_sync_entry_parse()
143 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); in xe_sync_entry_parse()
144 if (XE_IOCTL_DBG(xe, !sync->syncobj)) in xe_sync_entry_parse()
145 return -ENOENT; in xe_sync_entry_parse()
148 sync->fence = drm_syncobj_fence_get(sync->syncobj); in xe_sync_entry_parse()
149 if (XE_IOCTL_DBG(xe, !sync->fence)) { in xe_sync_entry_parse()
150 err = -EINVAL; in xe_sync_entry_parse()
158 return -EOPNOTSUPP; in xe_sync_entry_parse()
161 return -EINVAL; in xe_sync_entry_parse()
164 return -EINVAL; in xe_sync_entry_parse()
166 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); in xe_sync_entry_parse()
167 if (XE_IOCTL_DBG(xe, !sync->syncobj)) in xe_sync_entry_parse()
168 return -ENOENT; in xe_sync_entry_parse()
171 sync->chain_fence = dma_fence_chain_alloc(); in xe_sync_entry_parse()
172 if (!sync->chain_fence) { in xe_sync_entry_parse()
173 err = -ENOMEM; in xe_sync_entry_parse()
177 sync->fence = drm_syncobj_fence_get(sync->syncobj); in xe_sync_entry_parse()
178 if (XE_IOCTL_DBG(xe, !sync->fence)) { in xe_sync_entry_parse()
179 err = -EINVAL; in xe_sync_entry_parse()
183 err = dma_fence_chain_find_seqno(&sync->fence, in xe_sync_entry_parse()
192 return -EOPNOTSUPP; in xe_sync_entry_parse()
195 return -EOPNOTSUPP; in xe_sync_entry_parse()
198 return -EINVAL; in xe_sync_entry_parse()
201 sync->addr = sync_in.addr; in xe_sync_entry_parse()
203 sync->ufence_timeline_value = ufence_timeline_value; in xe_sync_entry_parse()
204 sync->ufence = user_fence_create(xe, sync_in.addr, in xe_sync_entry_parse()
206 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) in xe_sync_entry_parse()
207 return PTR_ERR(sync->ufence); in xe_sync_entry_parse()
208 sync->ufence_chain_fence = dma_fence_chain_alloc(); in xe_sync_entry_parse()
209 if (!sync->ufence_chain_fence) { in xe_sync_entry_parse()
210 err = -ENOMEM; in xe_sync_entry_parse()
213 sync->ufence_syncobj = ufence_syncobj; in xe_sync_entry_parse()
219 return -EINVAL; in xe_sync_entry_parse()
222 sync->type = sync_in.type; in xe_sync_entry_parse()
223 sync->flags = sync_in.flags; in xe_sync_entry_parse()
224 sync->timeline_value = sync_in.timeline_value; in xe_sync_entry_parse()
229 xe_sync_entry_cleanup(sync); in xe_sync_entry_parse()
234 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) in xe_sync_entry_add_deps() argument
236 if (sync->fence) in xe_sync_entry_add_deps()
237 return drm_sched_job_add_dependency(&job->drm, in xe_sync_entry_add_deps()
238 dma_fence_get(sync->fence)); in xe_sync_entry_add_deps()
244 * xe_sync_entry_wait() - Wait on in-sync
245 * @sync: Sync object
247 * If the sync is in an in-sync, wait on the sync to signal.
249 * Return: 0 on success, -ERESTARTSYS on failure (interruption)
251 int xe_sync_entry_wait(struct xe_sync_entry *sync) in xe_sync_entry_wait() argument
253 return xe_sync_needs_wait(sync) ? in xe_sync_entry_wait()
254 dma_fence_wait(sync->fence, true) : 0; in xe_sync_entry_wait()
258 * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
259 * @sync: Sync object
261 * Return: True if sync needs a wait, False otherwise
263 bool xe_sync_needs_wait(struct xe_sync_entry *sync) in xe_sync_needs_wait() argument
265 return sync->fence && in xe_sync_needs_wait()
266 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); in xe_sync_needs_wait()
269 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) in xe_sync_entry_signal() argument
271 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) in xe_sync_entry_signal()
274 if (sync->chain_fence) { in xe_sync_entry_signal()
275 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, in xe_sync_entry_signal()
276 fence, sync->timeline_value); in xe_sync_entry_signal()
281 sync->chain_fence = NULL; in xe_sync_entry_signal()
282 } else if (sync->syncobj) { in xe_sync_entry_signal()
283 drm_syncobj_replace_fence(sync->syncobj, fence); in xe_sync_entry_signal()
284 } else if (sync->ufence) { in xe_sync_entry_signal()
287 drm_syncobj_add_point(sync->ufence_syncobj, in xe_sync_entry_signal()
288 sync->ufence_chain_fence, in xe_sync_entry_signal()
289 fence, sync->ufence_timeline_value); in xe_sync_entry_signal()
290 sync->ufence_chain_fence = NULL; in xe_sync_entry_signal()
292 fence = drm_syncobj_fence_get(sync->ufence_syncobj); in xe_sync_entry_signal()
293 user_fence_get(sync->ufence); in xe_sync_entry_signal()
294 err = dma_fence_add_callback(fence, &sync->ufence->cb, in xe_sync_entry_signal()
296 if (err == -ENOENT) { in xe_sync_entry_signal()
297 kick_ufence(sync->ufence, fence); in xe_sync_entry_signal()
300 user_fence_put(sync->ufence); in xe_sync_entry_signal()
306 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) in xe_sync_entry_cleanup() argument
308 if (sync->syncobj) in xe_sync_entry_cleanup()
309 drm_syncobj_put(sync->syncobj); in xe_sync_entry_cleanup()
310 dma_fence_put(sync->fence); in xe_sync_entry_cleanup()
311 dma_fence_chain_free(sync->chain_fence); in xe_sync_entry_cleanup()
312 dma_fence_chain_free(sync->ufence_chain_fence); in xe_sync_entry_cleanup()
313 if (!IS_ERR_OR_NULL(sync->ufence)) in xe_sync_entry_cleanup()
314 user_fence_put(sync->ufence); in xe_sync_entry_cleanup()
318 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
319 * @sync: input syncs
324 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
325 * and return a composite fence of all in-fences + last fence. If no in-fences
326 * return last fence on input exec queue. Caller must drop reference to
329 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
332 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, in xe_sync_in_fence_get() argument
340 lockdep_assert_held(&vm->lock); in xe_sync_in_fence_get()
344 if (sync[i].fence) in xe_sync_in_fence_get()
345 return ERR_PTR(-EOPNOTSUPP); in xe_sync_in_fence_get()
347 if (q->flags & EXEC_QUEUE_FLAG_VM) { in xe_sync_in_fence_get()
352 for_each_tile(tile, vm->xe, id) { in xe_sync_in_fence_get()
360 return ERR_PTR(-ENOMEM); in xe_sync_in_fence_get()
367 list_for_each_entry(__q, &q->multi_gt_list, in xe_sync_in_fence_get()
376 xe_assert(vm->xe, current_fence == num_fence); in xe_sync_in_fence_get()
383 return &cf->base; in xe_sync_in_fence_get()
391 dma_fence_put(fences[--current_fence]); in xe_sync_in_fence_get()
394 return ERR_PTR(-ENOMEM); in xe_sync_in_fence_get()
398 * __xe_sync_ufence_get() - Get user fence from user fence
401 * Get a user fence reference from user fence
403 * Return: xe_user_fence pointer with reference
413 * xe_sync_ufence_get() - Get user fence from sync
414 * @sync: input sync
416 * Get a user fence reference from sync.
418 * Return: xe_user_fence pointer with reference
420 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) in xe_sync_ufence_get() argument
422 user_fence_get(sync->ufence); in xe_sync_ufence_get()
424 return sync->ufence; in xe_sync_ufence_get()
428 * xe_sync_ufence_put() - Put user fence reference
429 * @ufence: user fence reference
438 * xe_sync_ufence_get_status() - Get user fence status
445 return READ_ONCE(ufence->signalled); in xe_sync_ufence_get_status()