1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright (C) 2015 Red Hat, Inc. 4 * All Rights Reserved. 5 * 6 * Authors: 7 * Dave Airlie 8 * Alon Levy 9 */ 10 11 #include <linux/dma-fence-unwrap.h> 12 #include <linux/file.h> 13 #include <linux/sync_file.h> 14 #include <linux/uaccess.h> 15 16 #include <drm/drm_file.h> 17 #include <drm/drm_syncobj.h> 18 #include <drm/virtgpu_drm.h> 19 20 #include "virtgpu_drv.h" 21 22 struct virtio_gpu_submit_post_dep { 23 struct drm_syncobj *syncobj; 24 struct dma_fence_chain *chain; 25 u64 point; 26 }; 27 28 struct virtio_gpu_submit { 29 struct virtio_gpu_submit_post_dep *post_deps; 30 unsigned int num_out_syncobjs; 31 32 struct drm_syncobj **in_syncobjs; 33 unsigned int num_in_syncobjs; 34 35 struct virtio_gpu_object_array *buflist; 36 struct drm_virtgpu_execbuffer *exbuf; 37 struct virtio_gpu_fence *out_fence; 38 struct virtio_gpu_fpriv *vfpriv; 39 struct virtio_gpu_device *vgdev; 40 struct sync_file *sync_file; 41 struct drm_file *file; 42 int out_fence_fd; 43 u64 fence_ctx; 44 u32 ring_idx; 45 void *buf; 46 }; 47 48 static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit, 49 struct dma_fence *in_fence) 50 { 51 u64 context = submit->fence_ctx + submit->ring_idx; 52 53 if (dma_fence_match_context(in_fence, context)) 54 return 0; 55 56 return dma_fence_wait(in_fence, true); 57 } 58 59 static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit, 60 struct dma_fence *fence) 61 { 62 struct dma_fence_unwrap itr; 63 struct dma_fence *f; 64 int err; 65 66 dma_fence_unwrap_for_each(f, &itr, fence) { 67 err = virtio_gpu_do_fence_wait(submit, f); 68 if (err) { 69 dma_fence_put(itr.chain); 70 return err; 71 } 72 } 73 74 return 0; 75 } 76 77 static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs, 78 u32 nr_syncobjs) 79 { 80 u32 i = nr_syncobjs; 81 82 while (i--) { 83 if (syncobjs[i]) 84 drm_syncobj_put(syncobjs[i]); 85 } 86 87 kvfree(syncobjs); 88 } 89 90 static int 91 virtio_gpu_parse_deps(struct virtio_gpu_submit *submit) 92 { 93 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 94 struct drm_virtgpu_execbuffer_syncobj syncobj_desc; 95 size_t syncobj_stride = exbuf->syncobj_stride; 96 u32 num_in_syncobjs = exbuf->num_in_syncobjs; 97 struct drm_syncobj **syncobjs; 98 int ret = 0, i; 99 100 if (!num_in_syncobjs) 101 return 0; 102 103 /* 104 * kvmalloc() at first tries to allocate memory using kmalloc() and 105 * falls back to vmalloc() only on failure. It also uses __GFP_NOWARN 106 * internally for allocations larger than a page size, preventing 107 * storm of KMSG warnings. 108 */ 109 syncobjs = kvzalloc_objs(*syncobjs, num_in_syncobjs); 110 if (!syncobjs) 111 return -ENOMEM; 112 113 for (i = 0; i < num_in_syncobjs; i++) { 114 u64 address = exbuf->in_syncobjs + i * syncobj_stride; 115 struct dma_fence *fence; 116 117 memset(&syncobj_desc, 0, sizeof(syncobj_desc)); 118 119 if (copy_from_user(&syncobj_desc, 120 u64_to_user_ptr(address), 121 min(syncobj_stride, sizeof(syncobj_desc)))) { 122 ret = -EFAULT; 123 break; 124 } 125 126 if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) { 127 ret = -EINVAL; 128 break; 129 } 130 131 ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle, 132 syncobj_desc.point, 0, &fence); 133 if (ret) 134 break; 135 136 ret = virtio_gpu_dma_fence_wait(submit, fence); 137 138 dma_fence_put(fence); 139 if (ret) 140 break; 141 142 if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) { 143 syncobjs[i] = drm_syncobj_find(submit->file, 144 syncobj_desc.handle); 145 if (!syncobjs[i]) { 146 ret = -EINVAL; 147 break; 148 } 149 } 150 } 151 152 if (ret) { 153 virtio_gpu_free_syncobjs(syncobjs, i); 154 return ret; 155 } 156 157 submit->num_in_syncobjs = num_in_syncobjs; 158 submit->in_syncobjs = syncobjs; 159 160 return ret; 161 } 162 163 static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs, 164 u32 nr_syncobjs) 165 { 166 u32 i; 167 168 for (i = 0; i < nr_syncobjs; i++) { 169 if (syncobjs[i]) 170 drm_syncobj_replace_fence(syncobjs[i], NULL); 171 } 172 } 173 174 static void 175 virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps, 176 u32 nr_syncobjs) 177 { 178 u32 i = nr_syncobjs; 179 180 while (i--) { 181 kfree(post_deps[i].chain); 182 drm_syncobj_put(post_deps[i].syncobj); 183 } 184 185 kvfree(post_deps); 186 } 187 188 static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit) 189 { 190 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 191 struct drm_virtgpu_execbuffer_syncobj syncobj_desc; 192 struct virtio_gpu_submit_post_dep *post_deps; 193 u32 num_out_syncobjs = exbuf->num_out_syncobjs; 194 size_t syncobj_stride = exbuf->syncobj_stride; 195 int ret = 0, i; 196 197 if (!num_out_syncobjs) 198 return 0; 199 200 post_deps = kvzalloc_objs(*post_deps, num_out_syncobjs); 201 if (!post_deps) 202 return -ENOMEM; 203 204 for (i = 0; i < num_out_syncobjs; i++) { 205 u64 address = exbuf->out_syncobjs + i * syncobj_stride; 206 207 memset(&syncobj_desc, 0, sizeof(syncobj_desc)); 208 209 if (copy_from_user(&syncobj_desc, 210 u64_to_user_ptr(address), 211 min(syncobj_stride, sizeof(syncobj_desc)))) { 212 ret = -EFAULT; 213 break; 214 } 215 216 post_deps[i].point = syncobj_desc.point; 217 218 if (syncobj_desc.flags) { 219 ret = -EINVAL; 220 break; 221 } 222 223 if (syncobj_desc.point) { 224 post_deps[i].chain = dma_fence_chain_alloc(); 225 if (!post_deps[i].chain) { 226 ret = -ENOMEM; 227 break; 228 } 229 } 230 231 post_deps[i].syncobj = drm_syncobj_find(submit->file, 232 syncobj_desc.handle); 233 if (!post_deps[i].syncobj) { 234 kfree(post_deps[i].chain); 235 ret = -EINVAL; 236 break; 237 } 238 } 239 240 if (ret) { 241 virtio_gpu_free_post_deps(post_deps, i); 242 return ret; 243 } 244 245 submit->num_out_syncobjs = num_out_syncobjs; 246 submit->post_deps = post_deps; 247 248 return 0; 249 } 250 251 static void 252 virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit) 253 { 254 struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps; 255 256 if (post_deps) { 257 struct dma_fence *fence = &submit->out_fence->f; 258 u32 i; 259 260 for (i = 0; i < submit->num_out_syncobjs; i++) { 261 if (post_deps[i].chain) { 262 drm_syncobj_add_point(post_deps[i].syncobj, 263 post_deps[i].chain, 264 fence, post_deps[i].point); 265 post_deps[i].chain = NULL; 266 } else { 267 drm_syncobj_replace_fence(post_deps[i].syncobj, 268 fence); 269 } 270 } 271 } 272 } 273 274 static int virtio_gpu_fence_event_create(struct drm_device *dev, 275 struct drm_file *file, 276 struct virtio_gpu_fence *fence, 277 u32 ring_idx) 278 { 279 struct virtio_gpu_fence_event *e = NULL; 280 int ret; 281 282 e = kzalloc_obj(*e); 283 if (!e) 284 return -ENOMEM; 285 286 e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED; 287 e->event.length = sizeof(e->event); 288 289 ret = drm_event_reserve_init(dev, file, &e->base, &e->event); 290 if (ret) { 291 kfree(e); 292 return ret; 293 } 294 295 fence->e = e; 296 297 return 0; 298 } 299 300 static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit) 301 { 302 struct drm_virtgpu_execbuffer *exbuf = submit->exbuf; 303 u32 *bo_handles; 304 305 if (!exbuf->num_bo_handles) 306 return 0; 307 308 bo_handles = kvmalloc_objs(*bo_handles, exbuf->num_bo_handles); 309 if (!bo_handles) 310 return -ENOMEM; 311 312 if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles), 313 exbuf->num_bo_handles * sizeof(*bo_handles))) { 314 kvfree(bo_handles); 315 return -EFAULT; 316 } 317 318 submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles, 319 exbuf->num_bo_handles); 320 if (!submit->buflist) { 321 kvfree(bo_handles); 322 return -ENOENT; 323 } 324 325 kvfree(bo_handles); 326 327 return 0; 328 } 329 330 static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit) 331 { 332 virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs); 333 virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs); 334 virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs); 335 336 if (!IS_ERR(submit->buf)) 337 kvfree(submit->buf); 338 339 if (submit->buflist) 340 virtio_gpu_array_put_free(submit->buflist); 341 342 if (submit->out_fence_fd >= 0) 343 put_unused_fd(submit->out_fence_fd); 344 345 if (submit->out_fence) 346 dma_fence_put(&submit->out_fence->f); 347 348 if (submit->sync_file) 349 fput(submit->sync_file->file); 350 } 351 352 static void virtio_gpu_submit(struct virtio_gpu_submit *submit) 353 { 354 virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size, 355 submit->vfpriv->ctx_id, submit->buflist, 356 submit->out_fence); 357 virtio_gpu_notify(submit->vgdev); 358 } 359 360 static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit) 361 { 362 submit->buf = NULL; 363 submit->buflist = NULL; 364 submit->sync_file = NULL; 365 submit->out_fence_fd = -1; 366 } 367 368 static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit, 369 struct drm_virtgpu_execbuffer *exbuf, 370 struct drm_device *dev, 371 struct drm_file *file, 372 u64 fence_ctx, u32 ring_idx) 373 { 374 struct virtio_gpu_fpriv *vfpriv = file->driver_priv; 375 struct virtio_gpu_device *vgdev = dev->dev_private; 376 struct virtio_gpu_fence *out_fence; 377 bool drm_fence_event; 378 int err; 379 380 memset(submit, 0, sizeof(*submit)); 381 382 if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) && 383 (vfpriv->ring_idx_mask & BIT_ULL(ring_idx))) 384 drm_fence_event = true; 385 else 386 drm_fence_event = false; 387 388 if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) || 389 exbuf->num_out_syncobjs || 390 exbuf->num_bo_handles || 391 drm_fence_event) { 392 out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx); 393 if (!out_fence) 394 return -ENOMEM; 395 } else { 396 out_fence = NULL; 397 } 398 399 if (drm_fence_event) { 400 err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx); 401 if (err) { 402 dma_fence_put(&out_fence->f); 403 return err; 404 } 405 } 406 407 submit->out_fence = out_fence; 408 submit->fence_ctx = fence_ctx; 409 submit->ring_idx = ring_idx; 410 submit->out_fence_fd = -1; 411 submit->vfpriv = vfpriv; 412 submit->vgdev = vgdev; 413 submit->exbuf = exbuf; 414 submit->file = file; 415 416 err = virtio_gpu_init_submit_buflist(submit); 417 if (err) 418 return err; 419 420 submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size); 421 if (IS_ERR(submit->buf)) 422 return PTR_ERR(submit->buf); 423 424 if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) { 425 err = get_unused_fd_flags(O_CLOEXEC); 426 if (err < 0) 427 return err; 428 429 submit->out_fence_fd = err; 430 431 submit->sync_file = sync_file_create(&out_fence->f); 432 if (!submit->sync_file) 433 return -ENOMEM; 434 } 435 436 return 0; 437 } 438 439 static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit) 440 { 441 int ret = 0; 442 443 if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) { 444 struct dma_fence *in_fence = 445 sync_file_get_fence(submit->exbuf->fence_fd); 446 if (!in_fence) 447 return -EINVAL; 448 449 /* 450 * Wait if the fence is from a foreign context, or if the 451 * fence array contains any fence from a foreign context. 452 */ 453 ret = virtio_gpu_dma_fence_wait(submit, in_fence); 454 455 dma_fence_put(in_fence); 456 } 457 458 return ret; 459 } 460 461 static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit) 462 { 463 if (submit->sync_file) { 464 submit->exbuf->fence_fd = submit->out_fence_fd; 465 fd_install(submit->out_fence_fd, submit->sync_file->file); 466 } 467 } 468 469 static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit) 470 { 471 if (submit->buflist) 472 return virtio_gpu_array_lock_resv(submit->buflist); 473 474 return 0; 475 } 476 477 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, 478 struct drm_file *file) 479 { 480 struct virtio_gpu_device *vgdev = dev->dev_private; 481 struct virtio_gpu_fpriv *vfpriv = file->driver_priv; 482 u64 fence_ctx = vgdev->fence_drv.context; 483 struct drm_virtgpu_execbuffer *exbuf = data; 484 struct virtio_gpu_submit submit; 485 u32 ring_idx = 0; 486 int ret = -EINVAL; 487 488 if (!vgdev->has_virgl_3d) 489 return -ENOSYS; 490 491 if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS) 492 return ret; 493 494 if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) { 495 if (exbuf->ring_idx >= vfpriv->num_rings) 496 return ret; 497 498 if (!vfpriv->base_fence_ctx) 499 return ret; 500 501 fence_ctx = vfpriv->base_fence_ctx; 502 ring_idx = exbuf->ring_idx; 503 } 504 505 virtio_gpu_create_context(dev, file); 506 507 ret = virtio_gpu_init_submit(&submit, exbuf, dev, file, 508 fence_ctx, ring_idx); 509 if (ret) 510 goto cleanup; 511 512 ret = virtio_gpu_parse_post_deps(&submit); 513 if (ret) 514 goto cleanup; 515 516 ret = virtio_gpu_parse_deps(&submit); 517 if (ret) 518 goto cleanup; 519 520 /* 521 * Await in-fences in the end of the job submission path to 522 * optimize the path by proceeding directly to the submission 523 * to virtio after the waits. 524 */ 525 ret = virtio_gpu_wait_in_fence(&submit); 526 if (ret) 527 goto cleanup; 528 529 ret = virtio_gpu_lock_buflist(&submit); 530 if (ret) 531 goto cleanup; 532 533 virtio_gpu_submit(&submit); 534 535 /* 536 * Set up user-out data after submitting the job to optimize 537 * the job submission path. 538 */ 539 virtio_gpu_install_out_fence_fd(&submit); 540 virtio_gpu_process_post_deps(&submit); 541 virtio_gpu_complete_submit(&submit); 542 cleanup: 543 if (ret && submit.out_fence && submit.out_fence->e) { 544 drm_event_cancel_free(dev, &submit.out_fence->e->base); 545 submit.out_fence->e = NULL; 546 } 547 virtio_gpu_cleanup_submit(&submit); 548 549 return ret; 550 } 551