Lines Matching +full:sync +full:- +full:1
1 // SPDX-License-Identifier: MIT
8 #include <linux/dma-fence-array.h>
38 mmdrop(ufence->mm); in user_fence_destroy()
44 kref_get(&ufence->refcount); in user_fence_get()
49 kref_put(&ufence->refcount, user_fence_destroy); in user_fence_put()
60 return ERR_PTR(-EFAULT); in user_fence_create()
64 return ERR_PTR(-ENOMEM); in user_fence_create()
66 ufence->xe = xe; in user_fence_create()
67 kref_init(&ufence->refcount); in user_fence_create()
68 ufence->addr = ptr; in user_fence_create()
69 ufence->value = value; in user_fence_create()
70 ufence->mm = current->mm; in user_fence_create()
71 mmgrab(ufence->mm); in user_fence_create()
80 WRITE_ONCE(ufence->signalled, 1); in user_fence_worker()
81 if (mmget_not_zero(ufence->mm)) { in user_fence_worker()
82 kthread_use_mm(ufence->mm); in user_fence_worker()
83 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) in user_fence_worker()
85 kthread_unuse_mm(ufence->mm); in user_fence_worker()
86 mmput(ufence->mm); in user_fence_worker()
88 drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n"); in user_fence_worker()
93 * to safely reuse the same ufence without encountering -EBUSY errors. in user_fence_worker()
95 wake_up_all(&ufence->xe->ufence_wq); in user_fence_worker()
101 INIT_WORK(&ufence->worker, user_fence_worker); in kick_ufence()
102 queue_work(ufence->xe->ordered_wq, &ufence->worker); in kick_ufence()
114 struct xe_sync_entry *sync, in xe_sync_entry_parse() argument
128 return -EFAULT; in xe_sync_entry_parse()
131 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) in xe_sync_entry_parse()
132 return -EINVAL; in xe_sync_entry_parse()
138 return -EOPNOTSUPP; in xe_sync_entry_parse()
141 return -EINVAL; in xe_sync_entry_parse()
143 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); in xe_sync_entry_parse()
144 if (XE_IOCTL_DBG(xe, !sync->syncobj)) in xe_sync_entry_parse()
145 return -ENOENT; in xe_sync_entry_parse()
148 sync->fence = drm_syncobj_fence_get(sync->syncobj); in xe_sync_entry_parse()
149 if (XE_IOCTL_DBG(xe, !sync->fence)) in xe_sync_entry_parse()
150 return -EINVAL; in xe_sync_entry_parse()
156 return -EOPNOTSUPP; in xe_sync_entry_parse()
159 return -EINVAL; in xe_sync_entry_parse()
162 return -EINVAL; in xe_sync_entry_parse()
164 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); in xe_sync_entry_parse()
165 if (XE_IOCTL_DBG(xe, !sync->syncobj)) in xe_sync_entry_parse()
166 return -ENOENT; in xe_sync_entry_parse()
169 sync->chain_fence = dma_fence_chain_alloc(); in xe_sync_entry_parse()
170 if (!sync->chain_fence) in xe_sync_entry_parse()
171 return -ENOMEM; in xe_sync_entry_parse()
173 sync->fence = drm_syncobj_fence_get(sync->syncobj); in xe_sync_entry_parse()
174 if (XE_IOCTL_DBG(xe, !sync->fence)) in xe_sync_entry_parse()
175 return -EINVAL; in xe_sync_entry_parse()
177 err = dma_fence_chain_find_seqno(&sync->fence, in xe_sync_entry_parse()
186 return -EOPNOTSUPP; in xe_sync_entry_parse()
189 return -EOPNOTSUPP; in xe_sync_entry_parse()
192 return -EINVAL; in xe_sync_entry_parse()
195 sync->addr = sync_in.addr; in xe_sync_entry_parse()
197 sync->ufence_timeline_value = ufence_timeline_value; in xe_sync_entry_parse()
198 sync->ufence = user_fence_create(xe, sync_in.addr, in xe_sync_entry_parse()
200 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) in xe_sync_entry_parse()
201 return PTR_ERR(sync->ufence); in xe_sync_entry_parse()
202 sync->ufence_chain_fence = dma_fence_chain_alloc(); in xe_sync_entry_parse()
203 if (!sync->ufence_chain_fence) in xe_sync_entry_parse()
204 return -ENOMEM; in xe_sync_entry_parse()
205 sync->ufence_syncobj = ufence_syncobj; in xe_sync_entry_parse()
211 return -EINVAL; in xe_sync_entry_parse()
214 sync->type = sync_in.type; in xe_sync_entry_parse()
215 sync->flags = sync_in.flags; in xe_sync_entry_parse()
216 sync->timeline_value = sync_in.timeline_value; in xe_sync_entry_parse()
222 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) in xe_sync_entry_add_deps() argument
224 if (sync->fence) in xe_sync_entry_add_deps()
225 return drm_sched_job_add_dependency(&job->drm, in xe_sync_entry_add_deps()
226 dma_fence_get(sync->fence)); in xe_sync_entry_add_deps()
231 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) in xe_sync_entry_signal() argument
233 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) in xe_sync_entry_signal()
236 if (sync->chain_fence) { in xe_sync_entry_signal()
237 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, in xe_sync_entry_signal()
238 fence, sync->timeline_value); in xe_sync_entry_signal()
243 sync->chain_fence = NULL; in xe_sync_entry_signal()
244 } else if (sync->syncobj) { in xe_sync_entry_signal()
245 drm_syncobj_replace_fence(sync->syncobj, fence); in xe_sync_entry_signal()
246 } else if (sync->ufence) { in xe_sync_entry_signal()
249 drm_syncobj_add_point(sync->ufence_syncobj, in xe_sync_entry_signal()
250 sync->ufence_chain_fence, in xe_sync_entry_signal()
251 fence, sync->ufence_timeline_value); in xe_sync_entry_signal()
252 sync->ufence_chain_fence = NULL; in xe_sync_entry_signal()
254 fence = drm_syncobj_fence_get(sync->ufence_syncobj); in xe_sync_entry_signal()
255 user_fence_get(sync->ufence); in xe_sync_entry_signal()
256 err = dma_fence_add_callback(fence, &sync->ufence->cb, in xe_sync_entry_signal()
258 if (err == -ENOENT) { in xe_sync_entry_signal()
259 kick_ufence(sync->ufence, fence); in xe_sync_entry_signal()
262 user_fence_put(sync->ufence); in xe_sync_entry_signal()
268 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) in xe_sync_entry_cleanup() argument
270 if (sync->syncobj) in xe_sync_entry_cleanup()
271 drm_syncobj_put(sync->syncobj); in xe_sync_entry_cleanup()
272 dma_fence_put(sync->fence); in xe_sync_entry_cleanup()
273 dma_fence_chain_free(sync->chain_fence); in xe_sync_entry_cleanup()
274 dma_fence_chain_free(sync->ufence_chain_fence); in xe_sync_entry_cleanup()
275 if (!IS_ERR_OR_NULL(sync->ufence)) in xe_sync_entry_cleanup()
276 user_fence_put(sync->ufence); in xe_sync_entry_cleanup()
280 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
281 * @sync: input syncs
286 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
287 * and return a composite fence of all in-fences + last fence. If no in-fences
291 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
294 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, in xe_sync_in_fence_get() argument
302 lockdep_assert_held(&vm->lock); in xe_sync_in_fence_get()
304 /* Count in-fences */ in xe_sync_in_fence_get()
306 if (sync[i].fence) { in xe_sync_in_fence_get()
308 fence = sync[i].fence; in xe_sync_in_fence_get()
319 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); in xe_sync_in_fence_get()
321 return ERR_PTR(-ENOMEM); in xe_sync_in_fence_get()
323 if (sync[i].fence) { in xe_sync_in_fence_get()
324 dma_fence_get(sync[i].fence); in xe_sync_in_fence_get()
325 fences[current_fence++] = sync[i].fence; in xe_sync_in_fence_get()
330 vm->composite_fence_ctx, in xe_sync_in_fence_get()
331 vm->composite_fence_seqno++, in xe_sync_in_fence_get()
334 --vm->composite_fence_seqno; in xe_sync_in_fence_get()
338 return &cf->base; in xe_sync_in_fence_get()
342 dma_fence_put(fences[--current_fence]); in xe_sync_in_fence_get()
346 return ERR_PTR(-ENOMEM); in xe_sync_in_fence_get()
350 * __xe_sync_ufence_get() - Get user fence from user fence
365 * xe_sync_ufence_get() - Get user fence from sync
366 * @sync: input sync
368 * Get a user fence reference from sync.
372 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) in xe_sync_ufence_get() argument
374 user_fence_get(sync->ufence); in xe_sync_ufence_get()
376 return sync->ufence; in xe_sync_ufence_get()
380 * xe_sync_ufence_put() - Put user fence reference
390 * xe_sync_ufence_get_status() - Get user fence status
393 * Return: 1 if signalled, 0 not signalled, <0 on error
397 return READ_ONCE(ufence->signalled); in xe_sync_ufence_get_status()