1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright (C) 2015 Red Hat, Inc.
4 * All Rights Reserved.
5 *
6 * Authors:
7 * Dave Airlie
8 * Alon Levy
9 */
10
11 #include <linux/dma-fence-unwrap.h>
12 #include <linux/file.h>
13 #include <linux/sync_file.h>
14 #include <linux/uaccess.h>
15
16 #include <drm/drm_file.h>
17 #include <drm/drm_syncobj.h>
18 #include <drm/virtgpu_drm.h>
19
20 #include "virtgpu_drv.h"
21
22 struct virtio_gpu_submit_post_dep {
23 struct drm_syncobj *syncobj;
24 struct dma_fence_chain *chain;
25 u64 point;
26 };
27
28 struct virtio_gpu_submit {
29 struct virtio_gpu_submit_post_dep *post_deps;
30 unsigned int num_out_syncobjs;
31
32 struct drm_syncobj **in_syncobjs;
33 unsigned int num_in_syncobjs;
34
35 struct virtio_gpu_object_array *buflist;
36 struct drm_virtgpu_execbuffer *exbuf;
37 struct virtio_gpu_fence *out_fence;
38 struct virtio_gpu_fpriv *vfpriv;
39 struct virtio_gpu_device *vgdev;
40 struct sync_file *sync_file;
41 struct drm_file *file;
42 int out_fence_fd;
43 u64 fence_ctx;
44 u32 ring_idx;
45 void *buf;
46 };
47
virtio_gpu_do_fence_wait(struct virtio_gpu_submit * submit,struct dma_fence * in_fence)48 static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit,
49 struct dma_fence *in_fence)
50 {
51 u64 context = submit->fence_ctx + submit->ring_idx;
52
53 if (dma_fence_match_context(in_fence, context))
54 return 0;
55
56 return dma_fence_wait(in_fence, true);
57 }
58
virtio_gpu_dma_fence_wait(struct virtio_gpu_submit * submit,struct dma_fence * fence)59 static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit,
60 struct dma_fence *fence)
61 {
62 struct dma_fence_unwrap itr;
63 struct dma_fence *f;
64 int err;
65
66 dma_fence_unwrap_for_each(f, &itr, fence) {
67 err = virtio_gpu_do_fence_wait(submit, f);
68 if (err) {
69 dma_fence_put(itr.chain);
70 return err;
71 }
72 }
73
74 return 0;
75 }
76
virtio_gpu_free_syncobjs(struct drm_syncobj ** syncobjs,u32 nr_syncobjs)77 static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs,
78 u32 nr_syncobjs)
79 {
80 u32 i = nr_syncobjs;
81
82 while (i--) {
83 if (syncobjs[i])
84 drm_syncobj_put(syncobjs[i]);
85 }
86
87 kvfree(syncobjs);
88 }
89
90 static int
virtio_gpu_parse_deps(struct virtio_gpu_submit * submit)91 virtio_gpu_parse_deps(struct virtio_gpu_submit *submit)
92 {
93 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
94 struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
95 size_t syncobj_stride = exbuf->syncobj_stride;
96 u32 num_in_syncobjs = exbuf->num_in_syncobjs;
97 struct drm_syncobj **syncobjs;
98 int ret = 0, i;
99
100 if (!num_in_syncobjs)
101 return 0;
102
103 /*
104 * kvmalloc() at first tries to allocate memory using kmalloc() and
105 * falls back to vmalloc() only on failure. It also uses __GFP_NOWARN
106 * internally for allocations larger than a page size, preventing
107 * storm of KMSG warnings.
108 */
109 syncobjs = kvzalloc_objs(*syncobjs, num_in_syncobjs);
110 if (!syncobjs)
111 return -ENOMEM;
112
113 for (i = 0; i < num_in_syncobjs; i++) {
114 u64 address = exbuf->in_syncobjs + i * syncobj_stride;
115 struct dma_fence *fence;
116
117 memset(&syncobj_desc, 0, sizeof(syncobj_desc));
118
119 if (copy_from_user(&syncobj_desc,
120 u64_to_user_ptr(address),
121 min(syncobj_stride, sizeof(syncobj_desc)))) {
122 ret = -EFAULT;
123 break;
124 }
125
126 if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) {
127 ret = -EINVAL;
128 break;
129 }
130
131 ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle,
132 syncobj_desc.point, 0, &fence);
133 if (ret)
134 break;
135
136 ret = virtio_gpu_dma_fence_wait(submit, fence);
137
138 dma_fence_put(fence);
139 if (ret)
140 break;
141
142 if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) {
143 syncobjs[i] = drm_syncobj_find(submit->file,
144 syncobj_desc.handle);
145 if (!syncobjs[i]) {
146 ret = -EINVAL;
147 break;
148 }
149 }
150 }
151
152 if (ret) {
153 virtio_gpu_free_syncobjs(syncobjs, i);
154 return ret;
155 }
156
157 submit->num_in_syncobjs = num_in_syncobjs;
158 submit->in_syncobjs = syncobjs;
159
160 return ret;
161 }
162
virtio_gpu_reset_syncobjs(struct drm_syncobj ** syncobjs,u32 nr_syncobjs)163 static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs,
164 u32 nr_syncobjs)
165 {
166 u32 i;
167
168 for (i = 0; i < nr_syncobjs; i++) {
169 if (syncobjs[i])
170 drm_syncobj_replace_fence(syncobjs[i], NULL);
171 }
172 }
173
174 static void
virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep * post_deps,u32 nr_syncobjs)175 virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps,
176 u32 nr_syncobjs)
177 {
178 u32 i = nr_syncobjs;
179
180 while (i--) {
181 kfree(post_deps[i].chain);
182 drm_syncobj_put(post_deps[i].syncobj);
183 }
184
185 kvfree(post_deps);
186 }
187
virtio_gpu_parse_post_deps(struct virtio_gpu_submit * submit)188 static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit)
189 {
190 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
191 struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
192 struct virtio_gpu_submit_post_dep *post_deps;
193 u32 num_out_syncobjs = exbuf->num_out_syncobjs;
194 size_t syncobj_stride = exbuf->syncobj_stride;
195 int ret = 0, i;
196
197 if (!num_out_syncobjs)
198 return 0;
199
200 post_deps = kvzalloc_objs(*post_deps, num_out_syncobjs);
201 if (!post_deps)
202 return -ENOMEM;
203
204 for (i = 0; i < num_out_syncobjs; i++) {
205 u64 address = exbuf->out_syncobjs + i * syncobj_stride;
206
207 memset(&syncobj_desc, 0, sizeof(syncobj_desc));
208
209 if (copy_from_user(&syncobj_desc,
210 u64_to_user_ptr(address),
211 min(syncobj_stride, sizeof(syncobj_desc)))) {
212 ret = -EFAULT;
213 break;
214 }
215
216 post_deps[i].point = syncobj_desc.point;
217
218 if (syncobj_desc.flags) {
219 ret = -EINVAL;
220 break;
221 }
222
223 if (syncobj_desc.point) {
224 post_deps[i].chain = dma_fence_chain_alloc();
225 if (!post_deps[i].chain) {
226 ret = -ENOMEM;
227 break;
228 }
229 }
230
231 post_deps[i].syncobj = drm_syncobj_find(submit->file,
232 syncobj_desc.handle);
233 if (!post_deps[i].syncobj) {
234 kfree(post_deps[i].chain);
235 ret = -EINVAL;
236 break;
237 }
238 }
239
240 if (ret) {
241 virtio_gpu_free_post_deps(post_deps, i);
242 return ret;
243 }
244
245 submit->num_out_syncobjs = num_out_syncobjs;
246 submit->post_deps = post_deps;
247
248 return 0;
249 }
250
251 static void
virtio_gpu_process_post_deps(struct virtio_gpu_submit * submit)252 virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit)
253 {
254 struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps;
255
256 if (post_deps) {
257 struct dma_fence *fence = &submit->out_fence->f;
258 u32 i;
259
260 for (i = 0; i < submit->num_out_syncobjs; i++) {
261 if (post_deps[i].chain) {
262 drm_syncobj_add_point(post_deps[i].syncobj,
263 post_deps[i].chain,
264 fence, post_deps[i].point);
265 post_deps[i].chain = NULL;
266 } else {
267 drm_syncobj_replace_fence(post_deps[i].syncobj,
268 fence);
269 }
270 }
271 }
272 }
273
virtio_gpu_fence_event_create(struct drm_device * dev,struct drm_file * file,struct virtio_gpu_fence * fence,u32 ring_idx)274 static int virtio_gpu_fence_event_create(struct drm_device *dev,
275 struct drm_file *file,
276 struct virtio_gpu_fence *fence,
277 u32 ring_idx)
278 {
279 struct virtio_gpu_fence_event *e = NULL;
280 int ret;
281
282 e = kzalloc_obj(*e);
283 if (!e)
284 return -ENOMEM;
285
286 e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED;
287 e->event.length = sizeof(e->event);
288
289 ret = drm_event_reserve_init(dev, file, &e->base, &e->event);
290 if (ret) {
291 kfree(e);
292 return ret;
293 }
294
295 fence->e = e;
296
297 return 0;
298 }
299
virtio_gpu_init_submit_buflist(struct virtio_gpu_submit * submit)300 static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit)
301 {
302 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
303 u32 *bo_handles;
304
305 if (!exbuf->num_bo_handles)
306 return 0;
307
308 bo_handles = kvmalloc_array(exbuf->num_bo_handles, sizeof(*bo_handles),
309 GFP_KERNEL);
310 if (!bo_handles)
311 return -ENOMEM;
312
313 if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles),
314 exbuf->num_bo_handles * sizeof(*bo_handles))) {
315 kvfree(bo_handles);
316 return -EFAULT;
317 }
318
319 submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles,
320 exbuf->num_bo_handles);
321 if (!submit->buflist) {
322 kvfree(bo_handles);
323 return -ENOENT;
324 }
325
326 kvfree(bo_handles);
327
328 return 0;
329 }
330
virtio_gpu_cleanup_submit(struct virtio_gpu_submit * submit)331 static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit)
332 {
333 virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
334 virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
335 virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs);
336
337 if (!IS_ERR(submit->buf))
338 kvfree(submit->buf);
339
340 if (submit->buflist)
341 virtio_gpu_array_put_free(submit->buflist);
342
343 if (submit->out_fence_fd >= 0)
344 put_unused_fd(submit->out_fence_fd);
345
346 if (submit->out_fence)
347 dma_fence_put(&submit->out_fence->f);
348
349 if (submit->sync_file)
350 fput(submit->sync_file->file);
351 }
352
virtio_gpu_submit(struct virtio_gpu_submit * submit)353 static void virtio_gpu_submit(struct virtio_gpu_submit *submit)
354 {
355 virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size,
356 submit->vfpriv->ctx_id, submit->buflist,
357 submit->out_fence);
358 virtio_gpu_notify(submit->vgdev);
359 }
360
virtio_gpu_complete_submit(struct virtio_gpu_submit * submit)361 static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit)
362 {
363 submit->buf = NULL;
364 submit->buflist = NULL;
365 submit->sync_file = NULL;
366 submit->out_fence_fd = -1;
367 }
368
virtio_gpu_init_submit(struct virtio_gpu_submit * submit,struct drm_virtgpu_execbuffer * exbuf,struct drm_device * dev,struct drm_file * file,u64 fence_ctx,u32 ring_idx)369 static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit,
370 struct drm_virtgpu_execbuffer *exbuf,
371 struct drm_device *dev,
372 struct drm_file *file,
373 u64 fence_ctx, u32 ring_idx)
374 {
375 struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
376 struct virtio_gpu_device *vgdev = dev->dev_private;
377 struct virtio_gpu_fence *out_fence;
378 bool drm_fence_event;
379 int err;
380
381 memset(submit, 0, sizeof(*submit));
382
383 if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) &&
384 (vfpriv->ring_idx_mask & BIT_ULL(ring_idx)))
385 drm_fence_event = true;
386 else
387 drm_fence_event = false;
388
389 if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) ||
390 exbuf->num_out_syncobjs ||
391 exbuf->num_bo_handles ||
392 drm_fence_event) {
393 out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx);
394 if (!out_fence)
395 return -ENOMEM;
396 } else {
397 out_fence = NULL;
398 }
399
400 if (drm_fence_event) {
401 err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx);
402 if (err) {
403 dma_fence_put(&out_fence->f);
404 return err;
405 }
406 }
407
408 submit->out_fence = out_fence;
409 submit->fence_ctx = fence_ctx;
410 submit->ring_idx = ring_idx;
411 submit->out_fence_fd = -1;
412 submit->vfpriv = vfpriv;
413 submit->vgdev = vgdev;
414 submit->exbuf = exbuf;
415 submit->file = file;
416
417 err = virtio_gpu_init_submit_buflist(submit);
418 if (err)
419 return err;
420
421 submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size);
422 if (IS_ERR(submit->buf))
423 return PTR_ERR(submit->buf);
424
425 if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) {
426 err = get_unused_fd_flags(O_CLOEXEC);
427 if (err < 0)
428 return err;
429
430 submit->out_fence_fd = err;
431
432 submit->sync_file = sync_file_create(&out_fence->f);
433 if (!submit->sync_file)
434 return -ENOMEM;
435 }
436
437 return 0;
438 }
439
virtio_gpu_wait_in_fence(struct virtio_gpu_submit * submit)440 static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit)
441 {
442 int ret = 0;
443
444 if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) {
445 struct dma_fence *in_fence =
446 sync_file_get_fence(submit->exbuf->fence_fd);
447 if (!in_fence)
448 return -EINVAL;
449
450 /*
451 * Wait if the fence is from a foreign context, or if the
452 * fence array contains any fence from a foreign context.
453 */
454 ret = virtio_gpu_dma_fence_wait(submit, in_fence);
455
456 dma_fence_put(in_fence);
457 }
458
459 return ret;
460 }
461
virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit * submit)462 static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit)
463 {
464 if (submit->sync_file) {
465 submit->exbuf->fence_fd = submit->out_fence_fd;
466 fd_install(submit->out_fence_fd, submit->sync_file->file);
467 }
468 }
469
virtio_gpu_lock_buflist(struct virtio_gpu_submit * submit)470 static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit)
471 {
472 if (submit->buflist)
473 return virtio_gpu_array_lock_resv(submit->buflist);
474
475 return 0;
476 }
477
virtio_gpu_execbuffer_ioctl(struct drm_device * dev,void * data,struct drm_file * file)478 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
479 struct drm_file *file)
480 {
481 struct virtio_gpu_device *vgdev = dev->dev_private;
482 struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
483 u64 fence_ctx = vgdev->fence_drv.context;
484 struct drm_virtgpu_execbuffer *exbuf = data;
485 struct virtio_gpu_submit submit;
486 u32 ring_idx = 0;
487 int ret = -EINVAL;
488
489 if (!vgdev->has_virgl_3d)
490 return -ENOSYS;
491
492 if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS)
493 return ret;
494
495 if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) {
496 if (exbuf->ring_idx >= vfpriv->num_rings)
497 return ret;
498
499 if (!vfpriv->base_fence_ctx)
500 return ret;
501
502 fence_ctx = vfpriv->base_fence_ctx;
503 ring_idx = exbuf->ring_idx;
504 }
505
506 virtio_gpu_create_context(dev, file);
507
508 ret = virtio_gpu_init_submit(&submit, exbuf, dev, file,
509 fence_ctx, ring_idx);
510 if (ret)
511 goto cleanup;
512
513 ret = virtio_gpu_parse_post_deps(&submit);
514 if (ret)
515 goto cleanup;
516
517 ret = virtio_gpu_parse_deps(&submit);
518 if (ret)
519 goto cleanup;
520
521 /*
522 * Await in-fences in the end of the job submission path to
523 * optimize the path by proceeding directly to the submission
524 * to virtio after the waits.
525 */
526 ret = virtio_gpu_wait_in_fence(&submit);
527 if (ret)
528 goto cleanup;
529
530 ret = virtio_gpu_lock_buflist(&submit);
531 if (ret)
532 goto cleanup;
533
534 virtio_gpu_submit(&submit);
535
536 /*
537 * Set up user-out data after submitting the job to optimize
538 * the job submission path.
539 */
540 virtio_gpu_install_out_fence_fd(&submit);
541 virtio_gpu_process_post_deps(&submit);
542 virtio_gpu_complete_submit(&submit);
543 cleanup:
544 if (ret && submit.out_fence && submit.out_fence->e) {
545 drm_event_cancel_free(dev, &submit.out_fence->e->base);
546 submit.out_fence->e = NULL;
547 }
548 virtio_gpu_cleanup_submit(&submit);
549
550 return ret;
551 }
552