1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <drm/drm_print.h> 8 #include <drm/drm_syncobj.h> 9 10 #include "v3d_drv.h" 11 #include "v3d_regs.h" 12 #include "v3d_trace.h" 13 14 /* Takes the reservation lock on all the BOs being referenced, so that 15 * we can attach fences and update the reservations after pushing the job 16 * to the queue. 17 * 18 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 19 * (all of which are on render->unref_list). They're entirely private 20 * to v3d, so we don't attach dma-buf fences to them. 21 */ 22 static int 23 v3d_lock_bo_reservations(struct v3d_job *job, 24 struct ww_acquire_ctx *acquire_ctx) 25 { 26 int i, ret; 27 28 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); 29 if (ret) 30 return ret; 31 32 for (i = 0; i < job->bo_count; i++) { 33 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); 34 if (ret) 35 goto fail; 36 37 ret = drm_sched_job_add_implicit_dependencies(&job->base, 38 job->bo[i], true); 39 if (ret) 40 goto fail; 41 } 42 43 return 0; 44 45 fail: 46 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 47 return ret; 48 } 49 50 /** 51 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 52 * referenced by the job. 53 * @dev: DRM device 54 * @file_priv: DRM file for this fd 55 * @job: V3D job being set up 56 * @bo_handles: GEM handles 57 * @bo_count: Number of GEM handles passed in 58 * 59 * The command validator needs to reference BOs by their index within 60 * the submitted job's BO list. This does the validation of the job's 61 * BO list and reference counting for the lifetime of the job. 62 * 63 * Note that this function doesn't need to unreference the BOs on 64 * failure, because that will happen at `v3d_job_free()`. 65 */ 66 static int 67 v3d_lookup_bos(struct drm_device *dev, 68 struct drm_file *file_priv, 69 struct v3d_job *job, 70 u64 bo_handles, 71 u32 bo_count) 72 { 73 job->bo_count = bo_count; 74 75 if (!job->bo_count) { 76 /* See comment on bo_index for why we have to check 77 * this. 78 */ 79 drm_warn(dev, "Rendering requires BOs\n"); 80 return -EINVAL; 81 } 82 83 return drm_gem_objects_lookup(file_priv, 84 (void __user *)(uintptr_t)bo_handles, 85 job->bo_count, &job->bo); 86 } 87 88 static void 89 v3d_job_free(struct kref *ref) 90 { 91 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 92 int i; 93 94 if (job->bo) { 95 for (i = 0; i < job->bo_count; i++) 96 drm_gem_object_put(job->bo[i]); 97 kvfree(job->bo); 98 } 99 100 dma_fence_put(job->irq_fence); 101 dma_fence_put(job->done_fence); 102 103 if (job->perfmon) 104 v3d_perfmon_put(job->perfmon); 105 106 v3d_stats_put(job->client_stats); 107 v3d_stats_put(job->global_stats); 108 109 if (job->has_pm_ref) 110 v3d_pm_runtime_put(job->v3d); 111 112 kfree(job); 113 } 114 115 static void 116 v3d_render_job_free(struct kref *ref) 117 { 118 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 119 base.refcount); 120 struct v3d_bo *bo, *save; 121 122 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 123 drm_gem_object_put(&bo->base.base); 124 } 125 126 v3d_job_free(ref); 127 } 128 129 static void 130 v3d_cpu_job_free(struct kref *ref) 131 { 132 struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job, 133 base.refcount); 134 135 v3d_timestamp_query_info_free(&job->timestamp_query, 136 job->timestamp_query.count); 137 138 v3d_performance_query_info_free(&job->performance_query, 139 job->performance_query.count); 140 141 if (job->indirect_csd.indirect) 142 drm_gem_object_put(job->indirect_csd.indirect); 143 144 v3d_job_free(ref); 145 } 146 147 void v3d_job_cleanup(struct v3d_job *job) 148 { 149 if (!job) 150 return; 151 152 drm_sched_job_cleanup(&job->base); 153 v3d_job_put(job); 154 } 155 156 void v3d_job_put(struct v3d_job *job) 157 { 158 if (!job) 159 return; 160 161 kref_put(&job->refcount, job->free); 162 } 163 164 static int 165 v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) 166 { 167 *container = kcalloc(1, size, GFP_KERNEL); 168 if (!*container) { 169 drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); 170 return -ENOMEM; 171 } 172 173 return 0; 174 } 175 176 static void 177 v3d_job_deallocate(void **container) 178 { 179 kfree(*container); 180 *container = NULL; 181 } 182 183 static int 184 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, 185 struct v3d_job *job, void (*free)(struct kref *ref), 186 u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) 187 { 188 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 189 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 190 int ret, i; 191 192 job->v3d = v3d; 193 job->free = free; 194 job->file_priv = v3d_priv; 195 196 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 197 1, v3d_priv, file_priv->client_id); 198 if (ret) 199 return ret; 200 201 if (has_multisync) { 202 if (se->in_sync_count && se->wait_stage == queue) { 203 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 204 205 for (i = 0; i < se->in_sync_count; i++) { 206 struct drm_v3d_sem in; 207 208 if (copy_from_user(&in, handle++, sizeof(in))) { 209 ret = -EFAULT; 210 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); 211 goto fail_job_init; 212 } 213 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); 214 215 // TODO: Investigate why this was filtered out for the IOCTL. 216 if (ret && ret != -ENOENT) 217 goto fail_job_init; 218 } 219 } 220 } else { 221 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); 222 223 // TODO: Investigate why this was filtered out for the IOCTL. 224 if (ret && ret != -ENOENT) 225 goto fail_job_init; 226 } 227 228 /* CPU jobs don't require hardware resources */ 229 if (queue != V3D_CPU) { 230 ret = v3d_pm_runtime_get(v3d); 231 if (ret) 232 goto fail_job_init; 233 job->has_pm_ref = true; 234 } 235 236 kref_init(&job->refcount); 237 238 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); 239 job->global_stats = v3d_stats_get(v3d->queue[queue].stats); 240 241 return 0; 242 243 fail_job_init: 244 drm_sched_job_cleanup(&job->base); 245 return ret; 246 } 247 248 static void 249 v3d_push_job(struct v3d_job *job) 250 { 251 drm_sched_job_arm(&job->base); 252 253 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 254 255 /* put by scheduler job completion */ 256 kref_get(&job->refcount); 257 258 drm_sched_entity_push_job(&job->base); 259 } 260 261 static void 262 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, 263 struct v3d_job *job, 264 struct ww_acquire_ctx *acquire_ctx, 265 u32 out_sync, 266 struct v3d_submit_ext *se, 267 struct dma_fence *done_fence) 268 { 269 struct drm_syncobj *sync_out; 270 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 271 int i; 272 273 for (i = 0; i < job->bo_count; i++) { 274 /* XXX: Use shared fences for read-only objects. */ 275 dma_resv_add_fence(job->bo[i]->resv, job->done_fence, 276 DMA_RESV_USAGE_WRITE); 277 } 278 279 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 280 281 /* Update the return sync object for the job */ 282 /* If it only supports a single signal semaphore*/ 283 if (!has_multisync) { 284 sync_out = drm_syncobj_find(file_priv, out_sync); 285 if (sync_out) { 286 drm_syncobj_replace_fence(sync_out, done_fence); 287 drm_syncobj_put(sync_out); 288 } 289 return; 290 } 291 292 /* If multiple semaphores extension is supported */ 293 if (se->out_sync_count) { 294 for (i = 0; i < se->out_sync_count; i++) { 295 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 296 done_fence); 297 drm_syncobj_put(se->out_syncs[i].syncobj); 298 } 299 kvfree(se->out_syncs); 300 } 301 } 302 303 static int 304 v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, 305 struct v3d_dev *v3d, 306 struct drm_v3d_submit_csd *args, 307 struct v3d_csd_job **job, 308 struct v3d_job **clean_job, 309 struct v3d_submit_ext *se, 310 struct ww_acquire_ctx *acquire_ctx) 311 { 312 int ret; 313 314 ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); 315 if (ret) 316 return ret; 317 318 ret = v3d_job_init(v3d, file_priv, &(*job)->base, 319 v3d_job_free, args->in_sync, se, V3D_CSD); 320 if (ret) { 321 v3d_job_deallocate((void *)job); 322 return ret; 323 } 324 325 ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); 326 if (ret) 327 return ret; 328 329 ret = v3d_job_init(v3d, file_priv, *clean_job, 330 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 331 if (ret) { 332 v3d_job_deallocate((void *)clean_job); 333 return ret; 334 } 335 336 (*job)->args = *args; 337 338 ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, 339 args->bo_handles, args->bo_handle_count); 340 if (ret) 341 return ret; 342 343 return v3d_lock_bo_reservations(*clean_job, acquire_ctx); 344 } 345 346 static void 347 v3d_put_multisync_post_deps(struct v3d_submit_ext *se) 348 { 349 unsigned int i; 350 351 if (!(se && se->out_sync_count)) 352 return; 353 354 for (i = 0; i < se->out_sync_count; i++) 355 drm_syncobj_put(se->out_syncs[i].syncobj); 356 kvfree(se->out_syncs); 357 } 358 359 static int 360 v3d_get_multisync_post_deps(struct drm_file *file_priv, 361 struct v3d_submit_ext *se, 362 u32 count, u64 handles) 363 { 364 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 365 struct v3d_dev *v3d = v3d_priv->v3d; 366 struct drm_v3d_sem __user *post_deps; 367 int i, ret; 368 369 if (!count) 370 return 0; 371 372 se->out_syncs = (struct v3d_submit_outsync *) 373 kvmalloc_objs(struct v3d_submit_outsync, count); 374 if (!se->out_syncs) 375 return -ENOMEM; 376 377 post_deps = u64_to_user_ptr(handles); 378 379 for (i = 0; i < count; i++) { 380 struct drm_v3d_sem out; 381 382 if (copy_from_user(&out, post_deps++, sizeof(out))) { 383 ret = -EFAULT; 384 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n"); 385 goto fail; 386 } 387 388 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 389 out.handle); 390 if (!se->out_syncs[i].syncobj) { 391 ret = -EINVAL; 392 goto fail; 393 } 394 } 395 se->out_sync_count = count; 396 397 return 0; 398 399 fail: 400 for (i--; i >= 0; i--) 401 drm_syncobj_put(se->out_syncs[i].syncobj); 402 kvfree(se->out_syncs); 403 404 return ret; 405 } 406 407 /* Get data for multiple binary semaphores synchronization. Parse syncobj 408 * to be signaled when job completes (out_sync). 409 */ 410 static int 411 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 412 struct drm_v3d_extension __user *ext, 413 struct v3d_submit_ext *se) 414 { 415 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 416 struct v3d_dev *v3d = v3d_priv->v3d; 417 struct drm_v3d_multi_sync multisync; 418 int ret; 419 420 if (se->in_sync_count || se->out_sync_count) { 421 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job."); 422 return -EINVAL; 423 } 424 425 if (copy_from_user(&multisync, ext, sizeof(multisync))) 426 return -EFAULT; 427 428 if (multisync.pad) 429 return -EINVAL; 430 431 if (!multisync.in_sync_count && !multisync.out_sync_count) { 432 drm_dbg(&v3d->drm, "Empty multisync extension\n"); 433 return -EINVAL; 434 } 435 436 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 437 multisync.out_syncs); 438 if (ret) 439 return ret; 440 441 se->in_sync_count = multisync.in_sync_count; 442 se->in_syncs = multisync.in_syncs; 443 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 444 se->wait_stage = multisync.wait_stage; 445 446 return 0; 447 } 448 449 /* Returns false if the CPU job has an invalid configuration. */ 450 static bool 451 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job) 452 { 453 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 454 struct v3d_dev *v3d = v3d_priv->v3d; 455 456 if (!job) { 457 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n"); 458 return false; 459 } 460 461 if (job->job_type) { 462 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n"); 463 return false; 464 } 465 466 return true; 467 } 468 469 /* Get data for the indirect CSD job submission. */ 470 static int 471 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 472 struct drm_v3d_extension __user *ext, 473 struct v3d_cpu_job *job) 474 { 475 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 476 struct v3d_dev *v3d = v3d_priv->v3d; 477 struct drm_v3d_indirect_csd indirect_csd; 478 struct v3d_indirect_csd_info *info = &job->indirect_csd; 479 480 if (!v3d_validate_cpu_job(file_priv, job)) 481 return -EINVAL; 482 483 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 484 return -EFAULT; 485 486 if (!v3d_has_csd(v3d)) { 487 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n"); 488 return -EINVAL; 489 } 490 491 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 492 info->offset = indirect_csd.offset; 493 info->wg_size = indirect_csd.wg_size; 494 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 495 sizeof(indirect_csd.wg_uniform_offsets)); 496 497 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 498 499 return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, 500 &info->job, &info->clean_job, 501 NULL, &info->acquire_ctx); 502 } 503 504 /* Get data for the query timestamp job submission. */ 505 static int 506 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 507 struct drm_v3d_extension __user *ext, 508 struct v3d_cpu_job *job) 509 { 510 u32 __user *offsets, *syncs; 511 struct drm_v3d_timestamp_query timestamp; 512 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 513 unsigned int i; 514 int err; 515 516 if (!v3d_validate_cpu_job(file_priv, job)) 517 return -EINVAL; 518 519 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 520 return -EFAULT; 521 522 if (timestamp.pad) 523 return -EINVAL; 524 525 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 526 527 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 528 timestamp.count); 529 if (!query_info->queries) 530 return -ENOMEM; 531 532 offsets = u64_to_user_ptr(timestamp.offsets); 533 syncs = u64_to_user_ptr(timestamp.syncs); 534 535 for (i = 0; i < timestamp.count; i++) { 536 u32 offset, sync; 537 538 if (get_user(offset, offsets++)) { 539 err = -EFAULT; 540 goto error; 541 } 542 543 query_info->queries[i].offset = offset; 544 545 if (get_user(sync, syncs++)) { 546 err = -EFAULT; 547 goto error; 548 } 549 550 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 551 sync); 552 if (!query_info->queries[i].syncobj) { 553 err = -ENOENT; 554 goto error; 555 } 556 } 557 query_info->count = timestamp.count; 558 559 return 0; 560 561 error: 562 v3d_timestamp_query_info_free(&job->timestamp_query, i); 563 return err; 564 } 565 566 static int 567 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 568 struct drm_v3d_extension __user *ext, 569 struct v3d_cpu_job *job) 570 { 571 u32 __user *syncs; 572 struct drm_v3d_reset_timestamp_query reset; 573 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 574 unsigned int i; 575 int err; 576 577 if (!v3d_validate_cpu_job(file_priv, job)) 578 return -EINVAL; 579 580 if (copy_from_user(&reset, ext, sizeof(reset))) 581 return -EFAULT; 582 583 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 584 585 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 586 reset.count); 587 if (!query_info->queries) 588 return -ENOMEM; 589 590 syncs = u64_to_user_ptr(reset.syncs); 591 592 for (i = 0; i < reset.count; i++) { 593 u32 sync; 594 595 query_info->queries[i].offset = reset.offset + 8 * i; 596 597 if (get_user(sync, syncs++)) { 598 err = -EFAULT; 599 goto error; 600 } 601 602 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 603 sync); 604 if (!query_info->queries[i].syncobj) { 605 err = -ENOENT; 606 goto error; 607 } 608 } 609 query_info->count = reset.count; 610 611 return 0; 612 613 error: 614 v3d_timestamp_query_info_free(&job->timestamp_query, i); 615 return err; 616 } 617 618 /* Get data for the copy timestamp query results job submission. */ 619 static int 620 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 621 struct drm_v3d_extension __user *ext, 622 struct v3d_cpu_job *job) 623 { 624 u32 __user *offsets, *syncs; 625 struct drm_v3d_copy_timestamp_query copy; 626 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 627 unsigned int i; 628 int err; 629 630 if (!v3d_validate_cpu_job(file_priv, job)) 631 return -EINVAL; 632 633 if (copy_from_user(©, ext, sizeof(copy))) 634 return -EFAULT; 635 636 if (copy.pad) 637 return -EINVAL; 638 639 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 640 641 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 642 copy.count); 643 if (!query_info->queries) 644 return -ENOMEM; 645 646 offsets = u64_to_user_ptr(copy.offsets); 647 syncs = u64_to_user_ptr(copy.syncs); 648 649 for (i = 0; i < copy.count; i++) { 650 u32 offset, sync; 651 652 if (get_user(offset, offsets++)) { 653 err = -EFAULT; 654 goto error; 655 } 656 657 query_info->queries[i].offset = offset; 658 659 if (get_user(sync, syncs++)) { 660 err = -EFAULT; 661 goto error; 662 } 663 664 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 665 sync); 666 if (!query_info->queries[i].syncobj) { 667 err = -ENOENT; 668 goto error; 669 } 670 } 671 query_info->count = copy.count; 672 673 job->copy.do_64bit = copy.do_64bit; 674 job->copy.do_partial = copy.do_partial; 675 job->copy.availability_bit = copy.availability_bit; 676 job->copy.offset = copy.offset; 677 job->copy.stride = copy.stride; 678 679 return 0; 680 681 error: 682 v3d_timestamp_query_info_free(&job->timestamp_query, i); 683 return err; 684 } 685 686 static int 687 v3d_copy_query_info(struct v3d_performance_query_info *query_info, 688 unsigned int count, 689 unsigned int nperfmons, 690 u32 __user *syncs, 691 u64 __user *kperfmon_ids, 692 struct drm_file *file_priv) 693 { 694 unsigned int i, j; 695 int err; 696 697 for (i = 0; i < count; i++) { 698 struct v3d_performance_query *query = &query_info->queries[i]; 699 u32 __user *ids_pointer; 700 u32 sync, id; 701 u64 ids; 702 703 if (get_user(sync, syncs++)) { 704 err = -EFAULT; 705 goto error; 706 } 707 708 if (get_user(ids, kperfmon_ids++)) { 709 err = -EFAULT; 710 goto error; 711 } 712 713 query->kperfmon_ids = 714 kvmalloc_array(nperfmons, 715 sizeof(struct v3d_performance_query *), 716 GFP_KERNEL); 717 if (!query->kperfmon_ids) { 718 err = -ENOMEM; 719 goto error; 720 } 721 722 ids_pointer = u64_to_user_ptr(ids); 723 724 for (j = 0; j < nperfmons; j++) { 725 if (get_user(id, ids_pointer++)) { 726 kvfree(query->kperfmon_ids); 727 err = -EFAULT; 728 goto error; 729 } 730 731 query->kperfmon_ids[j] = id; 732 } 733 734 query->syncobj = drm_syncobj_find(file_priv, sync); 735 if (!query->syncobj) { 736 kvfree(query->kperfmon_ids); 737 err = -ENOENT; 738 goto error; 739 } 740 } 741 742 return 0; 743 744 error: 745 v3d_performance_query_info_free(query_info, i); 746 return err; 747 } 748 749 static int 750 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 751 struct drm_v3d_extension __user *ext, 752 struct v3d_cpu_job *job) 753 { 754 struct v3d_performance_query_info *query_info = &job->performance_query; 755 struct drm_v3d_reset_performance_query reset; 756 int err; 757 758 if (!v3d_validate_cpu_job(file_priv, job)) 759 return -EINVAL; 760 761 if (copy_from_user(&reset, ext, sizeof(reset))) 762 return -EFAULT; 763 764 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 765 766 query_info->queries = 767 kvmalloc_objs(struct v3d_performance_query, reset.count); 768 if (!query_info->queries) 769 return -ENOMEM; 770 771 err = v3d_copy_query_info(query_info, 772 reset.count, 773 reset.nperfmons, 774 u64_to_user_ptr(reset.syncs), 775 u64_to_user_ptr(reset.kperfmon_ids), 776 file_priv); 777 if (err) 778 return err; 779 780 query_info->count = reset.count; 781 query_info->nperfmons = reset.nperfmons; 782 783 return 0; 784 } 785 786 static int 787 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 788 struct drm_v3d_extension __user *ext, 789 struct v3d_cpu_job *job) 790 { 791 struct v3d_performance_query_info *query_info = &job->performance_query; 792 struct drm_v3d_copy_performance_query copy; 793 int err; 794 795 if (!v3d_validate_cpu_job(file_priv, job)) 796 return -EINVAL; 797 798 if (copy_from_user(©, ext, sizeof(copy))) 799 return -EFAULT; 800 801 if (copy.pad) 802 return -EINVAL; 803 804 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 805 806 query_info->queries = 807 kvmalloc_objs(struct v3d_performance_query, copy.count); 808 if (!query_info->queries) 809 return -ENOMEM; 810 811 err = v3d_copy_query_info(query_info, 812 copy.count, 813 copy.nperfmons, 814 u64_to_user_ptr(copy.syncs), 815 u64_to_user_ptr(copy.kperfmon_ids), 816 file_priv); 817 if (err) 818 return err; 819 820 query_info->count = copy.count; 821 query_info->nperfmons = copy.nperfmons; 822 query_info->ncounters = copy.ncounters; 823 824 job->copy.do_64bit = copy.do_64bit; 825 job->copy.do_partial = copy.do_partial; 826 job->copy.availability_bit = copy.availability_bit; 827 job->copy.offset = copy.offset; 828 job->copy.stride = copy.stride; 829 830 return 0; 831 } 832 833 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 834 * according to the extension id (name). 835 */ 836 static int 837 v3d_get_extensions(struct drm_file *file_priv, 838 u64 ext_handles, 839 struct v3d_submit_ext *se, 840 struct v3d_cpu_job *job) 841 { 842 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 843 struct v3d_dev *v3d = v3d_priv->v3d; 844 struct drm_v3d_extension __user *user_ext; 845 int ret; 846 847 user_ext = u64_to_user_ptr(ext_handles); 848 while (user_ext) { 849 struct drm_v3d_extension ext; 850 851 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 852 drm_dbg(&v3d->drm, "Failed to copy submit extension\n"); 853 return -EFAULT; 854 } 855 856 switch (ext.id) { 857 case DRM_V3D_EXT_ID_MULTI_SYNC: 858 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 859 break; 860 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 861 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 862 break; 863 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 864 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 865 break; 866 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 867 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 868 break; 869 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 870 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 871 break; 872 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 873 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 874 break; 875 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 876 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 877 break; 878 default: 879 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id); 880 return -EINVAL; 881 } 882 883 if (ret) 884 return ret; 885 886 user_ext = u64_to_user_ptr(ext.next); 887 } 888 889 return 0; 890 } 891 892 /** 893 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 894 * @dev: DRM device 895 * @data: ioctl argument 896 * @file_priv: DRM file for this fd 897 * 898 * This is the main entrypoint for userspace to submit a 3D frame to 899 * the GPU. Userspace provides the binner command list (if 900 * applicable), and the kernel sets up the render command list to draw 901 * to the framebuffer described in the ioctl, using the command lists 902 * that the 3D engine's binner will produce. 903 */ 904 int 905 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 906 struct drm_file *file_priv) 907 { 908 struct v3d_dev *v3d = to_v3d_dev(dev); 909 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 910 struct drm_v3d_submit_cl *args = data; 911 struct v3d_submit_ext se = {0}; 912 struct v3d_bin_job *bin = NULL; 913 struct v3d_render_job *render = NULL; 914 struct v3d_job *clean_job = NULL; 915 struct v3d_job *last_job; 916 struct ww_acquire_ctx acquire_ctx; 917 int ret = 0; 918 919 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); 920 921 if (args->pad) 922 return -EINVAL; 923 924 if (args->flags && 925 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 926 DRM_V3D_SUBMIT_EXTENSION)) { 927 drm_dbg(dev, "invalid flags: %d\n", args->flags); 928 return -EINVAL; 929 } 930 931 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 932 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 933 if (ret) { 934 drm_dbg(dev, "Failed to get extensions.\n"); 935 return ret; 936 } 937 } 938 939 ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); 940 if (ret) 941 return ret; 942 943 ret = v3d_job_init(v3d, file_priv, &render->base, 944 v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); 945 if (ret) { 946 v3d_job_deallocate((void *)&render); 947 goto fail; 948 } 949 950 render->start = args->rcl_start; 951 render->end = args->rcl_end; 952 INIT_LIST_HEAD(&render->unref_list); 953 954 if (args->bcl_start != args->bcl_end) { 955 ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); 956 if (ret) 957 goto fail; 958 959 ret = v3d_job_init(v3d, file_priv, &bin->base, 960 v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); 961 if (ret) { 962 v3d_job_deallocate((void *)&bin); 963 goto fail; 964 } 965 966 bin->start = args->bcl_start; 967 bin->end = args->bcl_end; 968 bin->qma = args->qma; 969 bin->qms = args->qms; 970 bin->qts = args->qts; 971 bin->render = render; 972 } 973 974 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 975 ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); 976 if (ret) 977 goto fail; 978 979 ret = v3d_job_init(v3d, file_priv, clean_job, 980 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 981 if (ret) { 982 v3d_job_deallocate((void *)&clean_job); 983 goto fail; 984 } 985 986 last_job = clean_job; 987 } else { 988 last_job = &render->base; 989 } 990 991 ret = v3d_lookup_bos(dev, file_priv, last_job, 992 args->bo_handles, args->bo_handle_count); 993 if (ret) 994 goto fail; 995 996 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); 997 if (ret) 998 goto fail; 999 1000 if (args->perfmon_id) { 1001 if (v3d->global_perfmon) { 1002 ret = -EAGAIN; 1003 goto fail_perfmon; 1004 } 1005 1006 render->base.perfmon = v3d_perfmon_find(v3d_priv, 1007 args->perfmon_id); 1008 1009 if (!render->base.perfmon) { 1010 ret = -ENOENT; 1011 goto fail_perfmon; 1012 } 1013 } 1014 1015 mutex_lock(&v3d->sched_lock); 1016 if (bin) { 1017 bin->base.perfmon = render->base.perfmon; 1018 v3d_perfmon_get(bin->base.perfmon); 1019 v3d_push_job(&bin->base); 1020 1021 ret = drm_sched_job_add_dependency(&render->base.base, 1022 dma_fence_get(bin->base.done_fence)); 1023 if (ret) 1024 goto fail_unreserve; 1025 } 1026 1027 v3d_push_job(&render->base); 1028 1029 if (clean_job) { 1030 struct dma_fence *render_fence = 1031 dma_fence_get(render->base.done_fence); 1032 ret = drm_sched_job_add_dependency(&clean_job->base, 1033 render_fence); 1034 if (ret) 1035 goto fail_unreserve; 1036 clean_job->perfmon = render->base.perfmon; 1037 v3d_perfmon_get(clean_job->perfmon); 1038 v3d_push_job(clean_job); 1039 } 1040 1041 mutex_unlock(&v3d->sched_lock); 1042 1043 v3d_attach_fences_and_unlock_reservation(file_priv, 1044 last_job, 1045 &acquire_ctx, 1046 args->out_sync, 1047 &se, 1048 last_job->done_fence); 1049 1050 v3d_job_put(&bin->base); 1051 v3d_job_put(&render->base); 1052 v3d_job_put(clean_job); 1053 1054 return 0; 1055 1056 fail_unreserve: 1057 mutex_unlock(&v3d->sched_lock); 1058 fail_perfmon: 1059 drm_gem_unlock_reservations(last_job->bo, 1060 last_job->bo_count, &acquire_ctx); 1061 fail: 1062 v3d_job_cleanup((void *)bin); 1063 v3d_job_cleanup((void *)render); 1064 v3d_job_cleanup(clean_job); 1065 v3d_put_multisync_post_deps(&se); 1066 1067 return ret; 1068 } 1069 1070 /** 1071 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1072 * @dev: DRM device 1073 * @data: ioctl argument 1074 * @file_priv: DRM file for this fd 1075 * 1076 * Userspace provides the register setup for the TFU, which we don't 1077 * need to validate since the TFU is behind the MMU. 1078 */ 1079 int 1080 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1081 struct drm_file *file_priv) 1082 { 1083 struct v3d_dev *v3d = to_v3d_dev(dev); 1084 struct drm_v3d_submit_tfu *args = data; 1085 struct v3d_submit_ext se = {0}; 1086 struct v3d_tfu_job *job = NULL; 1087 struct ww_acquire_ctx acquire_ctx; 1088 int ret = 0; 1089 1090 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); 1091 1092 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1093 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1094 return -EINVAL; 1095 } 1096 1097 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1098 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1099 if (ret) { 1100 drm_dbg(dev, "Failed to get extensions.\n"); 1101 return ret; 1102 } 1103 } 1104 1105 ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); 1106 if (ret) 1107 return ret; 1108 1109 ret = v3d_job_init(v3d, file_priv, &job->base, 1110 v3d_job_free, args->in_sync, &se, V3D_TFU); 1111 if (ret) { 1112 v3d_job_deallocate((void *)&job); 1113 goto fail; 1114 } 1115 1116 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); 1117 if (!job->base.bo) { 1118 ret = -ENOMEM; 1119 goto fail; 1120 } 1121 1122 job->args = *args; 1123 1124 for (job->base.bo_count = 0; 1125 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1126 job->base.bo_count++) { 1127 struct drm_gem_object *bo; 1128 1129 if (!args->bo_handles[job->base.bo_count]) 1130 break; 1131 1132 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1133 if (!bo) { 1134 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n", 1135 job->base.bo_count, 1136 args->bo_handles[job->base.bo_count]); 1137 ret = -ENOENT; 1138 goto fail; 1139 } 1140 job->base.bo[job->base.bo_count] = bo; 1141 } 1142 1143 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); 1144 if (ret) 1145 goto fail; 1146 1147 mutex_lock(&v3d->sched_lock); 1148 v3d_push_job(&job->base); 1149 mutex_unlock(&v3d->sched_lock); 1150 1151 v3d_attach_fences_and_unlock_reservation(file_priv, 1152 &job->base, &acquire_ctx, 1153 args->out_sync, 1154 &se, 1155 job->base.done_fence); 1156 1157 v3d_job_put(&job->base); 1158 1159 return 0; 1160 1161 fail: 1162 v3d_job_cleanup((void *)job); 1163 v3d_put_multisync_post_deps(&se); 1164 1165 return ret; 1166 } 1167 1168 /** 1169 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1170 * @dev: DRM device 1171 * @data: ioctl argument 1172 * @file_priv: DRM file for this fd 1173 * 1174 * Userspace provides the register setup for the CSD, which we don't 1175 * need to validate since the CSD is behind the MMU. 1176 */ 1177 int 1178 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1179 struct drm_file *file_priv) 1180 { 1181 struct v3d_dev *v3d = to_v3d_dev(dev); 1182 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 1183 struct drm_v3d_submit_csd *args = data; 1184 struct v3d_submit_ext se = {0}; 1185 struct v3d_csd_job *job = NULL; 1186 struct v3d_job *clean_job = NULL; 1187 struct ww_acquire_ctx acquire_ctx; 1188 int ret; 1189 1190 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); 1191 1192 if (args->pad) 1193 return -EINVAL; 1194 1195 if (!v3d_has_csd(v3d)) { 1196 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); 1197 return -EINVAL; 1198 } 1199 1200 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1201 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1202 return -EINVAL; 1203 } 1204 1205 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1206 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1207 if (ret) { 1208 drm_dbg(dev, "Failed to get extensions.\n"); 1209 return ret; 1210 } 1211 } 1212 1213 ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, 1214 &job, &clean_job, &se, 1215 &acquire_ctx); 1216 if (ret) 1217 goto fail; 1218 1219 if (args->perfmon_id) { 1220 if (v3d->global_perfmon) { 1221 ret = -EAGAIN; 1222 goto fail_perfmon; 1223 } 1224 1225 job->base.perfmon = v3d_perfmon_find(v3d_priv, 1226 args->perfmon_id); 1227 if (!job->base.perfmon) { 1228 ret = -ENOENT; 1229 goto fail_perfmon; 1230 } 1231 } 1232 1233 mutex_lock(&v3d->sched_lock); 1234 v3d_push_job(&job->base); 1235 1236 ret = drm_sched_job_add_dependency(&clean_job->base, 1237 dma_fence_get(job->base.done_fence)); 1238 if (ret) 1239 goto fail_unreserve; 1240 1241 v3d_push_job(clean_job); 1242 mutex_unlock(&v3d->sched_lock); 1243 1244 v3d_attach_fences_and_unlock_reservation(file_priv, 1245 clean_job, 1246 &acquire_ctx, 1247 args->out_sync, 1248 &se, 1249 clean_job->done_fence); 1250 1251 v3d_job_put(&job->base); 1252 v3d_job_put(clean_job); 1253 1254 return 0; 1255 1256 fail_unreserve: 1257 mutex_unlock(&v3d->sched_lock); 1258 fail_perfmon: 1259 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1260 &acquire_ctx); 1261 fail: 1262 v3d_job_cleanup((void *)job); 1263 v3d_job_cleanup(clean_job); 1264 v3d_put_multisync_post_deps(&se); 1265 1266 return ret; 1267 } 1268 1269 static const unsigned int cpu_job_bo_handle_count[] = { 1270 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1271 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1272 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1273 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1274 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1275 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1276 }; 1277 1278 /** 1279 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1280 * @dev: DRM device 1281 * @data: ioctl argument 1282 * @file_priv: DRM file for this fd 1283 * 1284 * Userspace specifies the CPU job type and data required to perform its 1285 * operations through the drm_v3d_extension struct. 1286 */ 1287 int 1288 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1289 struct drm_file *file_priv) 1290 { 1291 struct v3d_dev *v3d = to_v3d_dev(dev); 1292 struct drm_v3d_submit_cpu *args = data; 1293 struct v3d_submit_ext se = {0}; 1294 struct v3d_submit_ext *out_se = NULL; 1295 struct v3d_cpu_job *cpu_job = NULL; 1296 struct v3d_csd_job *csd_job = NULL; 1297 struct v3d_job *clean_job = NULL; 1298 struct ww_acquire_ctx acquire_ctx; 1299 int ret; 1300 1301 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1302 drm_dbg(dev, "Invalid flags: %d\n", args->flags); 1303 return -EINVAL; 1304 } 1305 1306 ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); 1307 if (ret) 1308 return ret; 1309 1310 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1311 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1312 if (ret) { 1313 drm_dbg(dev, "Failed to get extensions.\n"); 1314 goto fail; 1315 } 1316 } 1317 1318 /* Every CPU job must have a CPU job user extension */ 1319 if (!cpu_job->job_type) { 1320 drm_dbg(dev, "CPU job must have a CPU job user extension.\n"); 1321 ret = -EINVAL; 1322 goto fail; 1323 } 1324 1325 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1326 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n"); 1327 ret = -EINVAL; 1328 goto fail; 1329 } 1330 1331 trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); 1332 1333 ret = v3d_job_init(v3d, file_priv, &cpu_job->base, 1334 v3d_cpu_job_free, 0, &se, V3D_CPU); 1335 if (ret) { 1336 v3d_job_deallocate((void *)&cpu_job); 1337 goto fail; 1338 } 1339 1340 clean_job = cpu_job->indirect_csd.clean_job; 1341 csd_job = cpu_job->indirect_csd.job; 1342 1343 if (args->bo_handle_count) { 1344 ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, 1345 args->bo_handles, args->bo_handle_count); 1346 if (ret) 1347 goto fail; 1348 1349 ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); 1350 if (ret) 1351 goto fail; 1352 } 1353 1354 mutex_lock(&v3d->sched_lock); 1355 v3d_push_job(&cpu_job->base); 1356 1357 switch (cpu_job->job_type) { 1358 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1359 ret = drm_sched_job_add_dependency(&csd_job->base.base, 1360 dma_fence_get(cpu_job->base.done_fence)); 1361 if (ret) 1362 goto fail_unreserve; 1363 1364 v3d_push_job(&csd_job->base); 1365 1366 ret = drm_sched_job_add_dependency(&clean_job->base, 1367 dma_fence_get(csd_job->base.done_fence)); 1368 if (ret) 1369 goto fail_unreserve; 1370 1371 v3d_push_job(clean_job); 1372 1373 break; 1374 default: 1375 break; 1376 } 1377 mutex_unlock(&v3d->sched_lock); 1378 1379 out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; 1380 1381 v3d_attach_fences_and_unlock_reservation(file_priv, 1382 &cpu_job->base, 1383 &acquire_ctx, 0, 1384 out_se, cpu_job->base.done_fence); 1385 1386 switch (cpu_job->job_type) { 1387 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1388 v3d_attach_fences_and_unlock_reservation(file_priv, 1389 clean_job, 1390 &cpu_job->indirect_csd.acquire_ctx, 1391 0, &se, clean_job->done_fence); 1392 break; 1393 default: 1394 break; 1395 } 1396 1397 v3d_job_put(&cpu_job->base); 1398 v3d_job_put(&csd_job->base); 1399 v3d_job_put(clean_job); 1400 1401 return 0; 1402 1403 fail_unreserve: 1404 mutex_unlock(&v3d->sched_lock); 1405 1406 drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, 1407 &acquire_ctx); 1408 1409 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1410 &cpu_job->indirect_csd.acquire_ctx); 1411 1412 fail: 1413 v3d_job_cleanup((void *)cpu_job); 1414 v3d_job_cleanup((void *)csd_job); 1415 v3d_job_cleanup(clean_job); 1416 v3d_put_multisync_post_deps(&se); 1417 1418 return ret; 1419 } 1420