xref: /linux/drivers/gpu/drm/virtio/virtgpu_submit.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright (C) 2015 Red Hat, Inc.
4  * All Rights Reserved.
5  *
6  * Authors:
7  *    Dave Airlie
8  *    Alon Levy
9  */
10 
11 #include <linux/dma-fence-unwrap.h>
12 #include <linux/file.h>
13 #include <linux/sync_file.h>
14 #include <linux/uaccess.h>
15 
16 #include <drm/drm_file.h>
17 #include <drm/drm_syncobj.h>
18 #include <drm/virtgpu_drm.h>
19 
20 #include "virtgpu_drv.h"
21 
22 struct virtio_gpu_submit_post_dep {
23 	struct drm_syncobj *syncobj;
24 	struct dma_fence_chain *chain;
25 	u64 point;
26 };
27 
28 struct virtio_gpu_submit {
29 	struct virtio_gpu_submit_post_dep *post_deps;
30 	unsigned int num_out_syncobjs;
31 
32 	struct drm_syncobj **in_syncobjs;
33 	unsigned int num_in_syncobjs;
34 
35 	struct virtio_gpu_object_array *buflist;
36 	struct drm_virtgpu_execbuffer *exbuf;
37 	struct virtio_gpu_fence *out_fence;
38 	struct virtio_gpu_fpriv *vfpriv;
39 	struct virtio_gpu_device *vgdev;
40 	struct sync_file *sync_file;
41 	struct drm_file *file;
42 	int out_fence_fd;
43 	u64 fence_ctx;
44 	u32 ring_idx;
45 	void *buf;
46 };
47 
48 static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit,
49 				    struct dma_fence *in_fence)
50 {
51 	u64 context = submit->fence_ctx + submit->ring_idx;
52 
53 	if (dma_fence_match_context(in_fence, context))
54 		return 0;
55 
56 	return dma_fence_wait(in_fence, true);
57 }
58 
59 static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit,
60 				     struct dma_fence *fence)
61 {
62 	struct dma_fence_unwrap itr;
63 	struct dma_fence *f;
64 	int err;
65 
66 	dma_fence_unwrap_for_each(f, &itr, fence) {
67 		err = virtio_gpu_do_fence_wait(submit, f);
68 		if (err) {
69 			dma_fence_put(itr.chain);
70 			return err;
71 		}
72 	}
73 
74 	return 0;
75 }
76 
77 static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs,
78 				     u32 nr_syncobjs)
79 {
80 	u32 i = nr_syncobjs;
81 
82 	while (i--) {
83 		if (syncobjs[i])
84 			drm_syncobj_put(syncobjs[i]);
85 	}
86 
87 	kvfree(syncobjs);
88 }
89 
90 static int
91 virtio_gpu_parse_deps(struct virtio_gpu_submit *submit)
92 {
93 	struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
94 	struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
95 	size_t syncobj_stride = exbuf->syncobj_stride;
96 	u32 num_in_syncobjs = exbuf->num_in_syncobjs;
97 	struct drm_syncobj **syncobjs;
98 	int ret = 0, i;
99 
100 	if (!num_in_syncobjs)
101 		return 0;
102 
103 	/*
104 	 * kvmalloc() at first tries to allocate memory using kmalloc() and
105 	 * falls back to vmalloc() only on failure. It also uses __GFP_NOWARN
106 	 * internally for allocations larger than a page size, preventing
107 	 * storm of KMSG warnings.
108 	 */
109 	syncobjs = kvzalloc_objs(*syncobjs, num_in_syncobjs);
110 	if (!syncobjs)
111 		return -ENOMEM;
112 
113 	for (i = 0; i < num_in_syncobjs; i++) {
114 		u64 address = exbuf->in_syncobjs + i * syncobj_stride;
115 		struct dma_fence *fence;
116 
117 		memset(&syncobj_desc, 0, sizeof(syncobj_desc));
118 
119 		if (copy_from_user(&syncobj_desc,
120 				   u64_to_user_ptr(address),
121 				   min(syncobj_stride, sizeof(syncobj_desc)))) {
122 			ret = -EFAULT;
123 			break;
124 		}
125 
126 		if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) {
127 			ret = -EINVAL;
128 			break;
129 		}
130 
131 		ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle,
132 					     syncobj_desc.point, 0, &fence);
133 		if (ret)
134 			break;
135 
136 		ret = virtio_gpu_dma_fence_wait(submit, fence);
137 
138 		dma_fence_put(fence);
139 		if (ret)
140 			break;
141 
142 		if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) {
143 			syncobjs[i] = drm_syncobj_find(submit->file,
144 						       syncobj_desc.handle);
145 			if (!syncobjs[i]) {
146 				ret = -EINVAL;
147 				break;
148 			}
149 		}
150 	}
151 
152 	if (ret) {
153 		virtio_gpu_free_syncobjs(syncobjs, i);
154 		return ret;
155 	}
156 
157 	submit->num_in_syncobjs = num_in_syncobjs;
158 	submit->in_syncobjs = syncobjs;
159 
160 	return ret;
161 }
162 
163 static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs,
164 				      u32 nr_syncobjs)
165 {
166 	u32 i;
167 
168 	for (i = 0; i < nr_syncobjs; i++) {
169 		if (syncobjs[i])
170 			drm_syncobj_replace_fence(syncobjs[i], NULL);
171 	}
172 }
173 
174 static void
175 virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps,
176 			  u32 nr_syncobjs)
177 {
178 	u32 i = nr_syncobjs;
179 
180 	while (i--) {
181 		kfree(post_deps[i].chain);
182 		drm_syncobj_put(post_deps[i].syncobj);
183 	}
184 
185 	kvfree(post_deps);
186 }
187 
188 static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit)
189 {
190 	struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
191 	struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
192 	struct virtio_gpu_submit_post_dep *post_deps;
193 	u32 num_out_syncobjs = exbuf->num_out_syncobjs;
194 	size_t syncobj_stride = exbuf->syncobj_stride;
195 	int ret = 0, i;
196 
197 	if (!num_out_syncobjs)
198 		return 0;
199 
200 	post_deps = kvzalloc_objs(*post_deps, num_out_syncobjs);
201 	if (!post_deps)
202 		return -ENOMEM;
203 
204 	for (i = 0; i < num_out_syncobjs; i++) {
205 		u64 address = exbuf->out_syncobjs + i * syncobj_stride;
206 
207 		memset(&syncobj_desc, 0, sizeof(syncobj_desc));
208 
209 		if (copy_from_user(&syncobj_desc,
210 				   u64_to_user_ptr(address),
211 				   min(syncobj_stride, sizeof(syncobj_desc)))) {
212 			ret = -EFAULT;
213 			break;
214 		}
215 
216 		post_deps[i].point = syncobj_desc.point;
217 
218 		if (syncobj_desc.flags) {
219 			ret = -EINVAL;
220 			break;
221 		}
222 
223 		if (syncobj_desc.point) {
224 			post_deps[i].chain = dma_fence_chain_alloc();
225 			if (!post_deps[i].chain) {
226 				ret = -ENOMEM;
227 				break;
228 			}
229 		}
230 
231 		post_deps[i].syncobj = drm_syncobj_find(submit->file,
232 							syncobj_desc.handle);
233 		if (!post_deps[i].syncobj) {
234 			kfree(post_deps[i].chain);
235 			ret = -EINVAL;
236 			break;
237 		}
238 	}
239 
240 	if (ret) {
241 		virtio_gpu_free_post_deps(post_deps, i);
242 		return ret;
243 	}
244 
245 	submit->num_out_syncobjs = num_out_syncobjs;
246 	submit->post_deps = post_deps;
247 
248 	return 0;
249 }
250 
251 static void
252 virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit)
253 {
254 	struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps;
255 
256 	if (post_deps) {
257 		struct dma_fence *fence = &submit->out_fence->f;
258 		u32 i;
259 
260 		for (i = 0; i < submit->num_out_syncobjs; i++) {
261 			if (post_deps[i].chain) {
262 				drm_syncobj_add_point(post_deps[i].syncobj,
263 						      post_deps[i].chain,
264 						      fence, post_deps[i].point);
265 				post_deps[i].chain = NULL;
266 			} else {
267 				drm_syncobj_replace_fence(post_deps[i].syncobj,
268 							  fence);
269 			}
270 		}
271 	}
272 }
273 
274 static int virtio_gpu_fence_event_create(struct drm_device *dev,
275 					 struct drm_file *file,
276 					 struct virtio_gpu_fence *fence,
277 					 u32 ring_idx)
278 {
279 	struct virtio_gpu_fence_event *e = NULL;
280 	int ret;
281 
282 	e = kzalloc_obj(*e);
283 	if (!e)
284 		return -ENOMEM;
285 
286 	e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED;
287 	e->event.length = sizeof(e->event);
288 
289 	ret = drm_event_reserve_init(dev, file, &e->base, &e->event);
290 	if (ret) {
291 		kfree(e);
292 		return ret;
293 	}
294 
295 	fence->e = e;
296 
297 	return 0;
298 }
299 
300 static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit)
301 {
302 	struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
303 	u32 *bo_handles;
304 
305 	if (!exbuf->num_bo_handles)
306 		return 0;
307 
308 	bo_handles = kvmalloc_objs(*bo_handles, exbuf->num_bo_handles);
309 	if (!bo_handles)
310 		return -ENOMEM;
311 
312 	if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles),
313 			   exbuf->num_bo_handles * sizeof(*bo_handles))) {
314 		kvfree(bo_handles);
315 		return -EFAULT;
316 	}
317 
318 	submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles,
319 							exbuf->num_bo_handles);
320 	if (!submit->buflist) {
321 		kvfree(bo_handles);
322 		return -ENOENT;
323 	}
324 
325 	kvfree(bo_handles);
326 
327 	return 0;
328 }
329 
330 static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit)
331 {
332 	virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
333 	virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
334 	virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs);
335 
336 	if (!IS_ERR(submit->buf))
337 		kvfree(submit->buf);
338 
339 	if (submit->buflist)
340 		virtio_gpu_array_put_free(submit->buflist);
341 
342 	if (submit->out_fence_fd >= 0)
343 		put_unused_fd(submit->out_fence_fd);
344 
345 	if (submit->out_fence)
346 		dma_fence_put(&submit->out_fence->f);
347 
348 	if (submit->sync_file)
349 		fput(submit->sync_file->file);
350 }
351 
352 static void virtio_gpu_submit(struct virtio_gpu_submit *submit)
353 {
354 	virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size,
355 			      submit->vfpriv->ctx_id, submit->buflist,
356 			      submit->out_fence);
357 	virtio_gpu_notify(submit->vgdev);
358 }
359 
360 static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit)
361 {
362 	submit->buf = NULL;
363 	submit->buflist = NULL;
364 	submit->sync_file = NULL;
365 	submit->out_fence_fd = -1;
366 }
367 
368 static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit,
369 				  struct drm_virtgpu_execbuffer *exbuf,
370 				  struct drm_device *dev,
371 				  struct drm_file *file,
372 				  u64 fence_ctx, u32 ring_idx)
373 {
374 	struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
375 	struct virtio_gpu_device *vgdev = dev->dev_private;
376 	struct virtio_gpu_fence *out_fence;
377 	bool drm_fence_event;
378 	int err;
379 
380 	memset(submit, 0, sizeof(*submit));
381 
382 	if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) &&
383 	    (vfpriv->ring_idx_mask & BIT_ULL(ring_idx)))
384 		drm_fence_event = true;
385 	else
386 		drm_fence_event = false;
387 
388 	if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) ||
389 	    exbuf->num_out_syncobjs ||
390 	    exbuf->num_bo_handles ||
391 	    drm_fence_event) {
392 		out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx);
393 		if (!out_fence)
394 			return -ENOMEM;
395 	} else {
396 		out_fence = NULL;
397 	}
398 
399 	if (drm_fence_event) {
400 		err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx);
401 		if (err) {
402 			dma_fence_put(&out_fence->f);
403 			return err;
404 		}
405 	}
406 
407 	submit->out_fence = out_fence;
408 	submit->fence_ctx = fence_ctx;
409 	submit->ring_idx = ring_idx;
410 	submit->out_fence_fd = -1;
411 	submit->vfpriv = vfpriv;
412 	submit->vgdev = vgdev;
413 	submit->exbuf = exbuf;
414 	submit->file = file;
415 
416 	err = virtio_gpu_init_submit_buflist(submit);
417 	if (err)
418 		return err;
419 
420 	submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size);
421 	if (IS_ERR(submit->buf))
422 		return PTR_ERR(submit->buf);
423 
424 	if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) {
425 		err = get_unused_fd_flags(O_CLOEXEC);
426 		if (err < 0)
427 			return err;
428 
429 		submit->out_fence_fd = err;
430 
431 		submit->sync_file = sync_file_create(&out_fence->f);
432 		if (!submit->sync_file)
433 			return -ENOMEM;
434 	}
435 
436 	return 0;
437 }
438 
439 static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit)
440 {
441 	int ret = 0;
442 
443 	if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) {
444 		struct dma_fence *in_fence =
445 				sync_file_get_fence(submit->exbuf->fence_fd);
446 		if (!in_fence)
447 			return -EINVAL;
448 
449 		/*
450 		 * Wait if the fence is from a foreign context, or if the
451 		 * fence array contains any fence from a foreign context.
452 		 */
453 		ret = virtio_gpu_dma_fence_wait(submit, in_fence);
454 
455 		dma_fence_put(in_fence);
456 	}
457 
458 	return ret;
459 }
460 
461 static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit)
462 {
463 	if (submit->sync_file) {
464 		submit->exbuf->fence_fd = submit->out_fence_fd;
465 		fd_install(submit->out_fence_fd, submit->sync_file->file);
466 	}
467 }
468 
469 static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit)
470 {
471 	if (submit->buflist)
472 		return virtio_gpu_array_lock_resv(submit->buflist);
473 
474 	return 0;
475 }
476 
477 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
478 				struct drm_file *file)
479 {
480 	struct virtio_gpu_device *vgdev = dev->dev_private;
481 	struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
482 	u64 fence_ctx = vgdev->fence_drv.context;
483 	struct drm_virtgpu_execbuffer *exbuf = data;
484 	struct virtio_gpu_submit submit;
485 	u32 ring_idx = 0;
486 	int ret = -EINVAL;
487 
488 	if (!vgdev->has_virgl_3d)
489 		return -ENOSYS;
490 
491 	if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS)
492 		return ret;
493 
494 	if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) {
495 		if (exbuf->ring_idx >= vfpriv->num_rings)
496 			return ret;
497 
498 		if (!vfpriv->base_fence_ctx)
499 			return ret;
500 
501 		fence_ctx = vfpriv->base_fence_ctx;
502 		ring_idx = exbuf->ring_idx;
503 	}
504 
505 	virtio_gpu_create_context(dev, file);
506 
507 	ret = virtio_gpu_init_submit(&submit, exbuf, dev, file,
508 				     fence_ctx, ring_idx);
509 	if (ret)
510 		goto cleanup;
511 
512 	ret = virtio_gpu_parse_post_deps(&submit);
513 	if (ret)
514 		goto cleanup;
515 
516 	ret = virtio_gpu_parse_deps(&submit);
517 	if (ret)
518 		goto cleanup;
519 
520 	/*
521 	 * Await in-fences in the end of the job submission path to
522 	 * optimize the path by proceeding directly to the submission
523 	 * to virtio after the waits.
524 	 */
525 	ret = virtio_gpu_wait_in_fence(&submit);
526 	if (ret)
527 		goto cleanup;
528 
529 	ret = virtio_gpu_lock_buflist(&submit);
530 	if (ret)
531 		goto cleanup;
532 
533 	virtio_gpu_submit(&submit);
534 
535 	/*
536 	 * Set up user-out data after submitting the job to optimize
537 	 * the job submission path.
538 	 */
539 	virtio_gpu_install_out_fence_fd(&submit);
540 	virtio_gpu_process_post_deps(&submit);
541 	virtio_gpu_complete_submit(&submit);
542 cleanup:
543 	if (ret && submit.out_fence && submit.out_fence->e) {
544 		drm_event_cancel_free(dev, &submit.out_fence->e->base);
545 		submit.out_fence->e = NULL;
546 	}
547 	virtio_gpu_cleanup_submit(&submit);
548 
549 	return ret;
550 }
551