1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "xe_sync.h"
7
8 #include <linux/dma-fence-array.h>
9 #include <linux/kthread.h>
10 #include <linux/sched/mm.h>
11 #include <linux/uaccess.h>
12
13 #include <drm/drm_print.h>
14 #include <drm/drm_syncobj.h>
15 #include <uapi/drm/xe_drm.h>
16
17 #include "xe_device_types.h"
18 #include "xe_exec_queue.h"
19 #include "xe_macros.h"
20 #include "xe_sched_job_types.h"
21
22 struct xe_user_fence {
23 struct xe_device *xe;
24 struct kref refcount;
25 struct dma_fence_cb cb;
26 struct work_struct worker;
27 struct mm_struct *mm;
28 u64 __user *addr;
29 u64 value;
30 int signalled;
31 };
32
user_fence_destroy(struct kref * kref)33 static void user_fence_destroy(struct kref *kref)
34 {
35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
36 refcount);
37
38 mmdrop(ufence->mm);
39 kfree(ufence);
40 }
41
user_fence_get(struct xe_user_fence * ufence)42 static void user_fence_get(struct xe_user_fence *ufence)
43 {
44 kref_get(&ufence->refcount);
45 }
46
user_fence_put(struct xe_user_fence * ufence)47 static void user_fence_put(struct xe_user_fence *ufence)
48 {
49 kref_put(&ufence->refcount, user_fence_destroy);
50 }
51
user_fence_create(struct xe_device * xe,u64 addr,u64 value)52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
53 u64 value)
54 {
55 struct xe_user_fence *ufence;
56 u64 __user *ptr = u64_to_user_ptr(addr);
57 u64 __maybe_unused prefetch_val;
58
59 if (get_user(prefetch_val, ptr))
60 return ERR_PTR(-EFAULT);
61
62 ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
63 if (!ufence)
64 return ERR_PTR(-ENOMEM);
65
66 ufence->xe = xe;
67 kref_init(&ufence->refcount);
68 ufence->addr = ptr;
69 ufence->value = value;
70 ufence->mm = current->mm;
71 mmgrab(ufence->mm);
72
73 return ufence;
74 }
75
user_fence_worker(struct work_struct * w)76 static void user_fence_worker(struct work_struct *w)
77 {
78 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
79
80 if (mmget_not_zero(ufence->mm)) {
81 kthread_use_mm(ufence->mm);
82 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
83 XE_WARN_ON("Copy to user failed");
84 kthread_unuse_mm(ufence->mm);
85 mmput(ufence->mm);
86 }
87
88 wake_up_all(&ufence->xe->ufence_wq);
89 WRITE_ONCE(ufence->signalled, 1);
90 user_fence_put(ufence);
91 }
92
kick_ufence(struct xe_user_fence * ufence,struct dma_fence * fence)93 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
94 {
95 INIT_WORK(&ufence->worker, user_fence_worker);
96 queue_work(ufence->xe->ordered_wq, &ufence->worker);
97 dma_fence_put(fence);
98 }
99
user_fence_cb(struct dma_fence * fence,struct dma_fence_cb * cb)100 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
101 {
102 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
103
104 kick_ufence(ufence, fence);
105 }
106
xe_sync_entry_parse(struct xe_device * xe,struct xe_file * xef,struct xe_sync_entry * sync,struct drm_xe_sync __user * sync_user,unsigned int flags)107 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
108 struct xe_sync_entry *sync,
109 struct drm_xe_sync __user *sync_user,
110 unsigned int flags)
111 {
112 struct drm_xe_sync sync_in;
113 int err;
114 bool exec = flags & SYNC_PARSE_FLAG_EXEC;
115 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
116 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
117 bool signal;
118
119 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
120 return -EFAULT;
121
122 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
123 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
124 return -EINVAL;
125
126 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
127 switch (sync_in.type) {
128 case DRM_XE_SYNC_TYPE_SYNCOBJ:
129 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
130 return -EOPNOTSUPP;
131
132 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
133 return -EINVAL;
134
135 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
136 if (XE_IOCTL_DBG(xe, !sync->syncobj))
137 return -ENOENT;
138
139 if (!signal) {
140 sync->fence = drm_syncobj_fence_get(sync->syncobj);
141 if (XE_IOCTL_DBG(xe, !sync->fence))
142 return -EINVAL;
143 }
144 break;
145
146 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
147 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
148 return -EOPNOTSUPP;
149
150 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
151 return -EINVAL;
152
153 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
154 return -EINVAL;
155
156 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
157 if (XE_IOCTL_DBG(xe, !sync->syncobj))
158 return -ENOENT;
159
160 if (signal) {
161 sync->chain_fence = dma_fence_chain_alloc();
162 if (!sync->chain_fence)
163 return -ENOMEM;
164 } else {
165 sync->fence = drm_syncobj_fence_get(sync->syncobj);
166 if (XE_IOCTL_DBG(xe, !sync->fence))
167 return -EINVAL;
168
169 err = dma_fence_chain_find_seqno(&sync->fence,
170 sync_in.timeline_value);
171 if (err)
172 return err;
173 }
174 break;
175
176 case DRM_XE_SYNC_TYPE_USER_FENCE:
177 if (XE_IOCTL_DBG(xe, disallow_user_fence))
178 return -EOPNOTSUPP;
179
180 if (XE_IOCTL_DBG(xe, !signal))
181 return -EOPNOTSUPP;
182
183 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
184 return -EINVAL;
185
186 if (exec) {
187 sync->addr = sync_in.addr;
188 } else {
189 sync->ufence = user_fence_create(xe, sync_in.addr,
190 sync_in.timeline_value);
191 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))
192 return PTR_ERR(sync->ufence);
193 }
194
195 break;
196
197 default:
198 return -EINVAL;
199 }
200
201 sync->type = sync_in.type;
202 sync->flags = sync_in.flags;
203 sync->timeline_value = sync_in.timeline_value;
204
205 return 0;
206 }
207
xe_sync_entry_add_deps(struct xe_sync_entry * sync,struct xe_sched_job * job)208 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
209 {
210 if (sync->fence)
211 return drm_sched_job_add_dependency(&job->drm,
212 dma_fence_get(sync->fence));
213
214 return 0;
215 }
216
xe_sync_entry_signal(struct xe_sync_entry * sync,struct dma_fence * fence)217 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
218 {
219 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
220 return;
221
222 if (sync->chain_fence) {
223 drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
224 fence, sync->timeline_value);
225 /*
226 * The chain's ownership is transferred to the
227 * timeline.
228 */
229 sync->chain_fence = NULL;
230 } else if (sync->syncobj) {
231 drm_syncobj_replace_fence(sync->syncobj, fence);
232 } else if (sync->ufence) {
233 int err;
234
235 dma_fence_get(fence);
236 user_fence_get(sync->ufence);
237 err = dma_fence_add_callback(fence, &sync->ufence->cb,
238 user_fence_cb);
239 if (err == -ENOENT) {
240 kick_ufence(sync->ufence, fence);
241 } else if (err) {
242 XE_WARN_ON("failed to add user fence");
243 user_fence_put(sync->ufence);
244 dma_fence_put(fence);
245 }
246 }
247 }
248
xe_sync_entry_cleanup(struct xe_sync_entry * sync)249 void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
250 {
251 if (sync->syncobj)
252 drm_syncobj_put(sync->syncobj);
253 dma_fence_put(sync->fence);
254 dma_fence_chain_free(sync->chain_fence);
255 if (sync->ufence)
256 user_fence_put(sync->ufence);
257 }
258
259 /**
260 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
261 * @sync: input syncs
262 * @num_sync: number of syncs
263 * @q: exec queue
264 * @vm: VM
265 *
266 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
267 * and return a composite fence of all in-fences + last fence. If no in-fences
268 * return last fence on input exec queue. Caller must drop reference to
269 * returned fence.
270 *
271 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
272 */
273 struct dma_fence *
xe_sync_in_fence_get(struct xe_sync_entry * sync,int num_sync,struct xe_exec_queue * q,struct xe_vm * vm)274 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
275 struct xe_exec_queue *q, struct xe_vm *vm)
276 {
277 struct dma_fence **fences = NULL;
278 struct dma_fence_array *cf = NULL;
279 struct dma_fence *fence;
280 int i, num_in_fence = 0, current_fence = 0;
281
282 lockdep_assert_held(&vm->lock);
283
284 /* Count in-fences */
285 for (i = 0; i < num_sync; ++i) {
286 if (sync[i].fence) {
287 ++num_in_fence;
288 fence = sync[i].fence;
289 }
290 }
291
292 /* Easy case... */
293 if (!num_in_fence) {
294 fence = xe_exec_queue_last_fence_get(q, vm);
295 return fence;
296 }
297
298 /* Create composite fence */
299 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);
300 if (!fences)
301 return ERR_PTR(-ENOMEM);
302 for (i = 0; i < num_sync; ++i) {
303 if (sync[i].fence) {
304 dma_fence_get(sync[i].fence);
305 fences[current_fence++] = sync[i].fence;
306 }
307 }
308 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);
309 cf = dma_fence_array_create(num_in_fence, fences,
310 vm->composite_fence_ctx,
311 vm->composite_fence_seqno++,
312 false);
313 if (!cf) {
314 --vm->composite_fence_seqno;
315 goto err_out;
316 }
317
318 return &cf->base;
319
320 err_out:
321 while (current_fence)
322 dma_fence_put(fences[--current_fence]);
323 kfree(fences);
324 kfree(cf);
325
326 return ERR_PTR(-ENOMEM);
327 }
328
329 /**
330 * __xe_sync_ufence_get() - Get user fence from user fence
331 * @ufence: input user fence
332 *
333 * Get a user fence reference from user fence
334 *
335 * Return: xe_user_fence pointer with reference
336 */
__xe_sync_ufence_get(struct xe_user_fence * ufence)337 struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence)
338 {
339 user_fence_get(ufence);
340
341 return ufence;
342 }
343
344 /**
345 * xe_sync_ufence_get() - Get user fence from sync
346 * @sync: input sync
347 *
348 * Get a user fence reference from sync.
349 *
350 * Return: xe_user_fence pointer with reference
351 */
xe_sync_ufence_get(struct xe_sync_entry * sync)352 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
353 {
354 user_fence_get(sync->ufence);
355
356 return sync->ufence;
357 }
358
359 /**
360 * xe_sync_ufence_put() - Put user fence reference
361 * @ufence: user fence reference
362 *
363 */
xe_sync_ufence_put(struct xe_user_fence * ufence)364 void xe_sync_ufence_put(struct xe_user_fence *ufence)
365 {
366 user_fence_put(ufence);
367 }
368
369 /**
370 * xe_sync_ufence_get_status() - Get user fence status
371 * @ufence: user fence
372 *
373 * Return: 1 if signalled, 0 not signalled, <0 on error
374 */
xe_sync_ufence_get_status(struct xe_user_fence * ufence)375 int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
376 {
377 return READ_ONCE(ufence->signalled);
378 }
379