1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright (C) 2015 Red Hat, Inc. 4 * All Rights Reserved. 5 * 6 * Authors: 7 * Dave Airlie 8 * Alon Levy 9 */ 10 11 #include <linux/dma-fence-unwrap.h> 12 #include <linux/file.h> 13 #include <linux/sync_file.h> 14 #include <linux/uaccess.h> 15 16 #include <drm/drm_file.h> 17 #include <drm/drm_syncobj.h> 18 #include <drm/virtgpu_drm.h> 19 20 #include "virtgpu_drv.h" 21 22 struct virtio_gpu_submit_post_dep { 23 struct drm_syncobj *syncobj; 24 struct dma_fence_chain *chain; 25 u64 point; 26 }; 27 28 struct virtio_gpu_submit { 29 struct virtio_gpu_submit_post_dep *post_deps; 30 unsigned int num_out_syncobjs; 31 32 struct drm_syncobj **in_syncobjs; 33 unsigned int num_in_syncobjs; 34 35 struct virtio_gpu_object_array *buflist; 36 struct drm_virtgpu_execbuffer *exbuf; 37 struct virtio_gpu_fence *out_fence; 38 struct virtio_gpu_fpriv *vfpriv; 39 struct virtio_gpu_device *vgdev; 40 struct sync_file *sync_file; 41 struct drm_file *file; 42 int out_fence_fd; 43 u64 fence_ctx; 44 u32 ring_idx; 45 void *buf; 46 }; 47 48 static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit, 49 struct dma_fence *in_fence) 50 { 51 u64 context = submit->fence_ctx + submit->ring_idx; 52 53 if (dma_fence_match_context(in_fence, context)) 54 return 0; 55 56 return dma_fence_wait(in_fence, true); 57 } 58 59 static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit, 60 struct dma_fence *fence) 61 { 62 struct dma_fence_unwrap itr; 63 struct dma_fence *f; 64 int err; 65 66 dma_fence_unwrap_for_each(f, &itr, fence) { 67 err = virtio_gpu_do_fence_wait(submit, f); 68 if (err) { 69 dma_fence_put(itr.chain); 70 return err; 71 } 72 } 73 74 return 0; 75 } 76 77 static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs, 78 u32 nr_syncobjs) 79 { 80 u32 i = nr_syncobjs; 81 82 while (i--) { 83 if (syncobjs[i]) 84 drm_syncobj_put(syncobjs[i]); 85 } 86 87 kvfree(syncobjs); 88 } 89 90 static int 91 virtio_gpu_parse_deps(struct virtio_gpu_submit *submit) 92 { 93 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 94 struct drm_virtgpu_execbuffer_syncobj syncobj_desc; 95 size_t syncobj_stride = exbuf->syncobj_stride; 96 u32 num_in_syncobjs = exbuf->num_in_syncobjs; 97 struct drm_syncobj **syncobjs; 98 int ret = 0, i; 99 100 if (!num_in_syncobjs) 101 return 0; 102 103 /* 104 * kvmalloc() at first tries to allocate memory using kmalloc() and 105 * falls back to vmalloc() only on failure. It also uses __GFP_NOWARN 106 * internally for allocations larger than a page size, preventing 107 * storm of KMSG warnings. 108 */ 109 syncobjs = kvzalloc_objs(*syncobjs, num_in_syncobjs); 110 if (!syncobjs) 111 return -ENOMEM; 112 113 for (i = 0; i < num_in_syncobjs; i++) { 114 u64 address = exbuf->in_syncobjs + i * syncobj_stride; 115 struct dma_fence *fence; 116 117 memset(&syncobj_desc, 0, sizeof(syncobj_desc)); 118 119 if (copy_from_user(&syncobj_desc, 120 u64_to_user_ptr(address), 121 min(syncobj_stride, sizeof(syncobj_desc)))) { 122 ret = -EFAULT; 123 break; 124 } 125 126 if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) { 127 ret = -EINVAL; 128 break; 129 } 130 131 ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle, 132 syncobj_desc.point, 0, &fence); 133 if (ret) 134 break; 135 136 ret = virtio_gpu_dma_fence_wait(submit, fence); 137 138 dma_fence_put(fence); 139 if (ret) 140 break; 141 142 if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) { 143 syncobjs[i] = drm_syncobj_find(submit->file, 144 syncobj_desc.handle); 145 if (!syncobjs[i]) { 146 ret = -EINVAL; 147 break; 148 } 149 } 150 } 151 152 if (ret) { 153 virtio_gpu_free_syncobjs(syncobjs, i); 154 return ret; 155 } 156 157 submit->num_in_syncobjs = num_in_syncobjs; 158 submit->in_syncobjs = syncobjs; 159 160 return ret; 161 } 162 163 static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs, 164 u32 nr_syncobjs) 165 { 166 u32 i; 167 168 for (i = 0; i < nr_syncobjs; i++) { 169 if (syncobjs[i]) 170 drm_syncobj_replace_fence(syncobjs[i], NULL); 171 } 172 } 173 174 static void 175 virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps, 176 u32 nr_syncobjs) 177 { 178 u32 i = nr_syncobjs; 179 180 while (i--) { 181 kfree(post_deps[i].chain); 182 drm_syncobj_put(post_deps[i].syncobj); 183 } 184 185 kvfree(post_deps); 186 } 187 188 static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit) 189 { 190 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 191 struct drm_virtgpu_execbuffer_syncobj syncobj_desc; 192 struct virtio_gpu_submit_post_dep *post_deps; 193 u32 num_out_syncobjs = exbuf->num_out_syncobjs; 194 size_t syncobj_stride = exbuf->syncobj_stride; 195 int ret = 0, i; 196 197 if (!num_out_syncobjs) 198 return 0; 199 200 post_deps = kvzalloc_objs(*post_deps, num_out_syncobjs); 201 if (!post_deps) 202 return -ENOMEM; 203 204 for (i = 0; i < num_out_syncobjs; i++) { 205 u64 address = exbuf->out_syncobjs + i * syncobj_stride; 206 207 memset(&syncobj_desc, 0, sizeof(syncobj_desc)); 208 209 if (copy_from_user(&syncobj_desc, 210 u64_to_user_ptr(address), 211 min(syncobj_stride, sizeof(syncobj_desc)))) { 212 ret = -EFAULT; 213 break; 214 } 215 216 post_deps[i].point = syncobj_desc.point; 217 218 if (syncobj_desc.flags) { 219 ret = -EINVAL; 220 break; 221 } 222 223 if (syncobj_desc.point) { 224 post_deps[i].chain = dma_fence_chain_alloc(); 225 if (!post_deps[i].chain) { 226 ret = -ENOMEM; 227 break; 228 } 229 } 230 231 post_deps[i].syncobj = drm_syncobj_find(submit->file, 232 syncobj_desc.handle); 233 if (!post_deps[i].syncobj) { 234 kfree(post_deps[i].chain); 235 ret = -EINVAL; 236 break; 237 } 238 } 239 240 if (ret) { 241 virtio_gpu_free_post_deps(post_deps, i); 242 return ret; 243 } 244 245 submit->num_out_syncobjs = num_out_syncobjs; 246 submit->post_deps = post_deps; 247 248 return 0; 249 } 250 251 static void 252 virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit) 253 { 254 struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps; 255 256 if (post_deps) { 257 struct dma_fence *fence = &submit->out_fence->f; 258 u32 i; 259 260 for (i = 0; i < submit->num_out_syncobjs; i++) { 261 if (post_deps[i].chain) { 262 drm_syncobj_add_point(post_deps[i].syncobj, 263 post_deps[i].chain, 264 fence, post_deps[i].point); 265 post_deps[i].chain = NULL; 266 } else { 267 drm_syncobj_replace_fence(post_deps[i].syncobj, 268 fence); 269 } 270 } 271 } 272 } 273 274 static int virtio_gpu_fence_event_create(struct drm_device *dev, 275 struct drm_file *file, 276 struct virtio_gpu_fence *fence, 277 u32 ring_idx) 278 { 279 struct virtio_gpu_fence_event *e = NULL; 280 int ret; 281 282 e = kzalloc_obj(*e); 283 if (!e) 284 return -ENOMEM; 285 286 e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED; 287 e->event.length = sizeof(e->event); 288 289 ret = drm_event_reserve_init(dev, file, &e->base, &e->event); 290 if (ret) { 291 kfree(e); 292 return ret; 293 } 294 295 fence->e = e; 296 297 return 0; 298 } 299 300 static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit) 301 { 302 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 303 u32 *bo_handles; 304 305 if (!exbuf->num_bo_handles) 306 return 0; 307 308 bo_handles = kvmalloc_array(exbuf->num_bo_handles, sizeof(*bo_handles), 309 GFP_KERNEL); 310 if (!bo_handles) 311 return -ENOMEM; 312 313 if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles), 314 exbuf->num_bo_handles * sizeof(*bo_handles))) { 315 kvfree(bo_handles); 316 return -EFAULT; 317 } 318 319 submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles, 320 exbuf->num_bo_handles); 321 if (!submit->buflist) { 322 kvfree(bo_handles); 323 return -ENOENT; 324 } 325 326 kvfree(bo_handles); 327 328 return 0; 329 } 330 331 static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit) 332 { 333 virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs); 334 virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs); 335 virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs); 336 337 if (!IS_ERR(submit->buf)) 338 kvfree(submit->buf); 339 340 if (submit->buflist) 341 virtio_gpu_array_put_free(submit->buflist); 342 343 if (submit->out_fence_fd >= 0) 344 put_unused_fd(submit->out_fence_fd); 345 346 if (submit->out_fence) 347 dma_fence_put(&submit->out_fence->f); 348 349 if (submit->sync_file) 350 fput(submit->sync_file->file); 351 } 352 353 static void virtio_gpu_submit(struct virtio_gpu_submit *submit) 354 { 355 virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size, 356 submit->vfpriv->ctx_id, submit->buflist, 357 submit->out_fence); 358 virtio_gpu_notify(submit->vgdev); 359 } 360 361 static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit) 362 { 363 submit->buf = NULL; 364 submit->buflist = NULL; 365 submit->sync_file = NULL; 366 submit->out_fence_fd = -1; 367 } 368 369 static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit, 370 struct drm_virtgpu_execbuffer *exbuf, 371 struct drm_device *dev, 372 struct drm_file *file, 373 u64 fence_ctx, u32 ring_idx) 374 { 375 struct virtio_gpu_fpriv *vfpriv = file->driver_priv; 376 struct virtio_gpu_device *vgdev = dev->dev_private; 377 struct virtio_gpu_fence *out_fence; 378 bool drm_fence_event; 379 int err; 380 381 memset(submit, 0, sizeof(*submit)); 382 383 if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) && 384 (vfpriv->ring_idx_mask & BIT_ULL(ring_idx))) 385 drm_fence_event = true; 386 else 387 drm_fence_event = false; 388 389 if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) || 390 exbuf->num_out_syncobjs || 391 exbuf->num_bo_handles || 392 drm_fence_event) 393 out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx); 394 else 395 out_fence = NULL; 396 397 if (drm_fence_event) { 398 err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx); 399 if (err) { 400 dma_fence_put(&out_fence->f); 401 return err; 402 } 403 } 404 405 submit->out_fence = out_fence; 406 submit->fence_ctx = fence_ctx; 407 submit->ring_idx = ring_idx; 408 submit->out_fence_fd = -1; 409 submit->vfpriv = vfpriv; 410 submit->vgdev = vgdev; 411 submit->exbuf = exbuf; 412 submit->file = file; 413 414 err = virtio_gpu_init_submit_buflist(submit); 415 if (err) 416 return err; 417 418 submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size); 419 if (IS_ERR(submit->buf)) 420 return PTR_ERR(submit->buf); 421 422 if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) { 423 err = get_unused_fd_flags(O_CLOEXEC); 424 if (err < 0) 425 return err; 426 427 submit->out_fence_fd = err; 428 429 submit->sync_file = sync_file_create(&out_fence->f); 430 if (!submit->sync_file) 431 return -ENOMEM; 432 } 433 434 return 0; 435 } 436 437 static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit) 438 { 439 int ret = 0; 440 441 if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) { 442 struct dma_fence *in_fence = 443 sync_file_get_fence(submit->exbuf->fence_fd); 444 if (!in_fence) 445 return -EINVAL; 446 447 /* 448 * Wait if the fence is from a foreign context, or if the 449 * fence array contains any fence from a foreign context. 450 */ 451 ret = virtio_gpu_dma_fence_wait(submit, in_fence); 452 453 dma_fence_put(in_fence); 454 } 455 456 return ret; 457 } 458 459 static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit) 460 { 461 if (submit->sync_file) { 462 submit->exbuf->fence_fd = submit->out_fence_fd; 463 fd_install(submit->out_fence_fd, submit->sync_file->file); 464 } 465 } 466 467 static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit) 468 { 469 if (submit->buflist) 470 return virtio_gpu_array_lock_resv(submit->buflist); 471 472 return 0; 473 } 474 475 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, 476 struct drm_file *file) 477 { 478 struct virtio_gpu_device *vgdev = dev->dev_private; 479 struct virtio_gpu_fpriv *vfpriv = file->driver_priv; 480 u64 fence_ctx = vgdev->fence_drv.context; 481 struct drm_virtgpu_execbuffer *exbuf = data; 482 struct virtio_gpu_submit submit; 483 u32 ring_idx = 0; 484 int ret = -EINVAL; 485 486 if (!vgdev->has_virgl_3d) 487 return -ENOSYS; 488 489 if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS) 490 return ret; 491 492 if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) { 493 if (exbuf->ring_idx >= vfpriv->num_rings) 494 return ret; 495 496 if (!vfpriv->base_fence_ctx) 497 return ret; 498 499 fence_ctx = vfpriv->base_fence_ctx; 500 ring_idx = exbuf->ring_idx; 501 } 502 503 virtio_gpu_create_context(dev, file); 504 505 ret = virtio_gpu_init_submit(&submit, exbuf, dev, file, 506 fence_ctx, ring_idx); 507 if (ret) 508 goto cleanup; 509 510 ret = virtio_gpu_parse_post_deps(&submit); 511 if (ret) 512 goto cleanup; 513 514 ret = virtio_gpu_parse_deps(&submit); 515 if (ret) 516 goto cleanup; 517 518 /* 519 * Await in-fences in the end of the job submission path to 520 * optimize the path by proceeding directly to the submission 521 * to virtio after the waits. 522 */ 523 ret = virtio_gpu_wait_in_fence(&submit); 524 if (ret) 525 goto cleanup; 526 527 ret = virtio_gpu_lock_buflist(&submit); 528 if (ret) 529 goto cleanup; 530 531 virtio_gpu_submit(&submit); 532 533 /* 534 * Set up user-out data after submitting the job to optimize 535 * the job submission path. 536 */ 537 virtio_gpu_install_out_fence_fd(&submit); 538 virtio_gpu_process_post_deps(&submit); 539 virtio_gpu_complete_submit(&submit); 540 cleanup: 541 virtio_gpu_cleanup_submit(&submit); 542 543 return ret; 544 } 545