1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <drm/drm_print.h> 8 #include <drm/drm_syncobj.h> 9 10 #include "v3d_drv.h" 11 #include "v3d_regs.h" 12 #include "v3d_trace.h" 13 14 /* Takes the reservation lock on all the BOs being referenced, so that 15 * we can attach fences and update the reservations after pushing the job 16 * to the queue. 17 * 18 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 19 * (all of which are on render->unref_list). They're entirely private 20 * to v3d, so we don't attach dma-buf fences to them. 21 */ 22 static int 23 v3d_lock_bo_reservations(struct v3d_job *job, 24 struct ww_acquire_ctx *acquire_ctx) 25 { 26 int i, ret; 27 28 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); 29 if (ret) 30 return ret; 31 32 for (i = 0; i < job->bo_count; i++) { 33 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); 34 if (ret) 35 goto fail; 36 37 ret = drm_sched_job_add_implicit_dependencies(&job->base, 38 job->bo[i], true); 39 if (ret) 40 goto fail; 41 } 42 43 return 0; 44 45 fail: 46 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 47 return ret; 48 } 49 50 /** 51 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 52 * referenced by the job. 53 * @dev: DRM device 54 * @file_priv: DRM file for this fd 55 * @job: V3D job being set up 56 * @bo_handles: GEM handles 57 * @bo_count: Number of GEM handles passed in 58 * 59 * The command validator needs to reference BOs by their index within 60 * the submitted job's BO list. This does the validation of the job's 61 * BO list and reference counting for the lifetime of the job. 62 * 63 * Note that this function doesn't need to unreference the BOs on 64 * failure, because that will happen at `v3d_job_free()`. 65 */ 66 static int 67 v3d_lookup_bos(struct drm_device *dev, 68 struct drm_file *file_priv, 69 struct v3d_job *job, 70 u64 bo_handles, 71 u32 bo_count) 72 { 73 job->bo_count = bo_count; 74 75 if (!job->bo_count) { 76 /* See comment on bo_index for why we have to check 77 * this. 78 */ 79 drm_warn(dev, "Rendering requires BOs\n"); 80 return -EINVAL; 81 } 82 83 return drm_gem_objects_lookup(file_priv, 84 (void __user *)(uintptr_t)bo_handles, 85 job->bo_count, &job->bo); 86 } 87 88 static void 89 v3d_job_free(struct kref *ref) 90 { 91 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 92 int i; 93 94 if (job->bo) { 95 for (i = 0; i < job->bo_count; i++) 96 drm_gem_object_put(job->bo[i]); 97 kvfree(job->bo); 98 } 99 100 dma_fence_put(job->irq_fence); 101 dma_fence_put(job->done_fence); 102 103 if (job->perfmon) 104 v3d_perfmon_put(job->perfmon); 105 106 v3d_stats_put(job->client_stats); 107 v3d_stats_put(job->global_stats); 108 109 if (job->has_pm_ref) 110 v3d_pm_runtime_put(job->v3d); 111 112 kfree(job); 113 } 114 115 static void 116 v3d_render_job_free(struct kref *ref) 117 { 118 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 119 base.refcount); 120 struct v3d_bo *bo, *save; 121 122 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 123 drm_gem_object_put(&bo->base.base); 124 } 125 126 v3d_job_free(ref); 127 } 128 129 void v3d_job_cleanup(struct v3d_job *job) 130 { 131 if (!job) 132 return; 133 134 drm_sched_job_cleanup(&job->base); 135 v3d_job_put(job); 136 } 137 138 void v3d_job_put(struct v3d_job *job) 139 { 140 if (!job) 141 return; 142 143 kref_put(&job->refcount, job->free); 144 } 145 146 static int 147 v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) 148 { 149 *container = kcalloc(1, size, GFP_KERNEL); 150 if (!*container) { 151 drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); 152 return -ENOMEM; 153 } 154 155 return 0; 156 } 157 158 static void 159 v3d_job_deallocate(void **container) 160 { 161 kfree(*container); 162 *container = NULL; 163 } 164 165 static int 166 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, 167 struct v3d_job *job, void (*free)(struct kref *ref), 168 u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) 169 { 170 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 171 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 172 int ret, i; 173 174 job->v3d = v3d; 175 job->free = free; 176 job->file_priv = v3d_priv; 177 178 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 179 1, v3d_priv, file_priv->client_id); 180 if (ret) 181 return ret; 182 183 if (has_multisync) { 184 if (se->in_sync_count && se->wait_stage == queue) { 185 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 186 187 for (i = 0; i < se->in_sync_count; i++) { 188 struct drm_v3d_sem in; 189 190 if (copy_from_user(&in, handle++, sizeof(in))) { 191 ret = -EFAULT; 192 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); 193 goto fail_job_init; 194 } 195 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); 196 197 // TODO: Investigate why this was filtered out for the IOCTL. 198 if (ret && ret != -ENOENT) 199 goto fail_job_init; 200 } 201 } 202 } else { 203 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); 204 205 // TODO: Investigate why this was filtered out for the IOCTL. 206 if (ret && ret != -ENOENT) 207 goto fail_job_init; 208 } 209 210 /* CPU jobs don't require hardware resources */ 211 if (queue != V3D_CPU) { 212 ret = v3d_pm_runtime_get(v3d); 213 if (ret) 214 goto fail_job_init; 215 job->has_pm_ref = true; 216 } 217 218 kref_init(&job->refcount); 219 220 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); 221 job->global_stats = v3d_stats_get(v3d->queue[queue].stats); 222 223 return 0; 224 225 fail_job_init: 226 drm_sched_job_cleanup(&job->base); 227 return ret; 228 } 229 230 static void 231 v3d_push_job(struct v3d_job *job) 232 { 233 drm_sched_job_arm(&job->base); 234 235 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 236 237 /* put by scheduler job completion */ 238 kref_get(&job->refcount); 239 240 drm_sched_entity_push_job(&job->base); 241 } 242 243 static void 244 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, 245 struct v3d_job *job, 246 struct ww_acquire_ctx *acquire_ctx, 247 u32 out_sync, 248 struct v3d_submit_ext *se, 249 struct dma_fence *done_fence) 250 { 251 struct drm_syncobj *sync_out; 252 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 253 int i; 254 255 for (i = 0; i < job->bo_count; i++) { 256 /* XXX: Use shared fences for read-only objects. */ 257 dma_resv_add_fence(job->bo[i]->resv, job->done_fence, 258 DMA_RESV_USAGE_WRITE); 259 } 260 261 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 262 263 /* Update the return sync object for the job */ 264 /* If it only supports a single signal semaphore*/ 265 if (!has_multisync) { 266 sync_out = drm_syncobj_find(file_priv, out_sync); 267 if (sync_out) { 268 drm_syncobj_replace_fence(sync_out, done_fence); 269 drm_syncobj_put(sync_out); 270 } 271 return; 272 } 273 274 /* If multiple semaphores extension is supported */ 275 if (se->out_sync_count) { 276 for (i = 0; i < se->out_sync_count; i++) { 277 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 278 done_fence); 279 drm_syncobj_put(se->out_syncs[i].syncobj); 280 } 281 kvfree(se->out_syncs); 282 } 283 } 284 285 static int 286 v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, 287 struct v3d_dev *v3d, 288 struct drm_v3d_submit_csd *args, 289 struct v3d_csd_job **job, 290 struct v3d_job **clean_job, 291 struct v3d_submit_ext *se, 292 struct ww_acquire_ctx *acquire_ctx) 293 { 294 int ret; 295 296 ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); 297 if (ret) 298 return ret; 299 300 ret = v3d_job_init(v3d, file_priv, &(*job)->base, 301 v3d_job_free, args->in_sync, se, V3D_CSD); 302 if (ret) { 303 v3d_job_deallocate((void *)job); 304 return ret; 305 } 306 307 ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); 308 if (ret) 309 return ret; 310 311 ret = v3d_job_init(v3d, file_priv, *clean_job, 312 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 313 if (ret) { 314 v3d_job_deallocate((void *)clean_job); 315 return ret; 316 } 317 318 (*job)->args = *args; 319 320 ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, 321 args->bo_handles, args->bo_handle_count); 322 if (ret) 323 return ret; 324 325 return v3d_lock_bo_reservations(*clean_job, acquire_ctx); 326 } 327 328 static void 329 v3d_put_multisync_post_deps(struct v3d_submit_ext *se) 330 { 331 unsigned int i; 332 333 if (!(se && se->out_sync_count)) 334 return; 335 336 for (i = 0; i < se->out_sync_count; i++) 337 drm_syncobj_put(se->out_syncs[i].syncobj); 338 kvfree(se->out_syncs); 339 } 340 341 static int 342 v3d_get_multisync_post_deps(struct drm_file *file_priv, 343 struct v3d_submit_ext *se, 344 u32 count, u64 handles) 345 { 346 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 347 struct v3d_dev *v3d = v3d_priv->v3d; 348 struct drm_v3d_sem __user *post_deps; 349 int i, ret; 350 351 if (!count) 352 return 0; 353 354 se->out_syncs = (struct v3d_submit_outsync *) 355 kvmalloc_objs(struct v3d_submit_outsync, count); 356 if (!se->out_syncs) 357 return -ENOMEM; 358 359 post_deps = u64_to_user_ptr(handles); 360 361 for (i = 0; i < count; i++) { 362 struct drm_v3d_sem out; 363 364 if (copy_from_user(&out, post_deps++, sizeof(out))) { 365 ret = -EFAULT; 366 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n"); 367 goto fail; 368 } 369 370 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 371 out.handle); 372 if (!se->out_syncs[i].syncobj) { 373 ret = -EINVAL; 374 goto fail; 375 } 376 } 377 se->out_sync_count = count; 378 379 return 0; 380 381 fail: 382 for (i--; i >= 0; i--) 383 drm_syncobj_put(se->out_syncs[i].syncobj); 384 kvfree(se->out_syncs); 385 386 return ret; 387 } 388 389 /* Get data for multiple binary semaphores synchronization. Parse syncobj 390 * to be signaled when job completes (out_sync). 391 */ 392 static int 393 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 394 struct drm_v3d_extension __user *ext, 395 struct v3d_submit_ext *se) 396 { 397 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 398 struct v3d_dev *v3d = v3d_priv->v3d; 399 struct drm_v3d_multi_sync multisync; 400 int ret; 401 402 if (se->in_sync_count || se->out_sync_count) { 403 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job."); 404 return -EINVAL; 405 } 406 407 if (copy_from_user(&multisync, ext, sizeof(multisync))) 408 return -EFAULT; 409 410 if (multisync.pad) 411 return -EINVAL; 412 413 if (!multisync.in_sync_count && !multisync.out_sync_count) { 414 drm_dbg(&v3d->drm, "Empty multisync extension\n"); 415 return -EINVAL; 416 } 417 418 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 419 multisync.out_syncs); 420 if (ret) 421 return ret; 422 423 se->in_sync_count = multisync.in_sync_count; 424 se->in_syncs = multisync.in_syncs; 425 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 426 se->wait_stage = multisync.wait_stage; 427 428 return 0; 429 } 430 431 /* Returns false if the CPU job has an invalid configuration. */ 432 static bool 433 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job) 434 { 435 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 436 struct v3d_dev *v3d = v3d_priv->v3d; 437 438 if (!job) { 439 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n"); 440 return false; 441 } 442 443 if (job->job_type) { 444 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n"); 445 return false; 446 } 447 448 return true; 449 } 450 451 /* Get data for the indirect CSD job submission. */ 452 static int 453 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 454 struct drm_v3d_extension __user *ext, 455 struct v3d_cpu_job *job) 456 { 457 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 458 struct v3d_dev *v3d = v3d_priv->v3d; 459 struct drm_v3d_indirect_csd indirect_csd; 460 struct v3d_indirect_csd_info *info = &job->indirect_csd; 461 462 if (!v3d_validate_cpu_job(file_priv, job)) 463 return -EINVAL; 464 465 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 466 return -EFAULT; 467 468 if (!v3d_has_csd(v3d)) { 469 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n"); 470 return -EINVAL; 471 } 472 473 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 474 info->offset = indirect_csd.offset; 475 info->wg_size = indirect_csd.wg_size; 476 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 477 sizeof(indirect_csd.wg_uniform_offsets)); 478 479 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 480 481 return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, 482 &info->job, &info->clean_job, 483 NULL, &info->acquire_ctx); 484 } 485 486 /* Get data for the query timestamp job submission. */ 487 static int 488 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 489 struct drm_v3d_extension __user *ext, 490 struct v3d_cpu_job *job) 491 { 492 u32 __user *offsets, *syncs; 493 struct drm_v3d_timestamp_query timestamp; 494 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 495 unsigned int i; 496 int err; 497 498 if (!v3d_validate_cpu_job(file_priv, job)) 499 return -EINVAL; 500 501 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 502 return -EFAULT; 503 504 if (timestamp.pad) 505 return -EINVAL; 506 507 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 508 509 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 510 timestamp.count); 511 if (!query_info->queries) 512 return -ENOMEM; 513 514 offsets = u64_to_user_ptr(timestamp.offsets); 515 syncs = u64_to_user_ptr(timestamp.syncs); 516 517 for (i = 0; i < timestamp.count; i++) { 518 u32 offset, sync; 519 520 if (get_user(offset, offsets++)) { 521 err = -EFAULT; 522 goto error; 523 } 524 525 query_info->queries[i].offset = offset; 526 527 if (get_user(sync, syncs++)) { 528 err = -EFAULT; 529 goto error; 530 } 531 532 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 533 sync); 534 if (!query_info->queries[i].syncobj) { 535 err = -ENOENT; 536 goto error; 537 } 538 } 539 query_info->count = timestamp.count; 540 541 return 0; 542 543 error: 544 v3d_timestamp_query_info_free(&job->timestamp_query, i); 545 return err; 546 } 547 548 static int 549 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 550 struct drm_v3d_extension __user *ext, 551 struct v3d_cpu_job *job) 552 { 553 u32 __user *syncs; 554 struct drm_v3d_reset_timestamp_query reset; 555 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 556 unsigned int i; 557 int err; 558 559 if (!v3d_validate_cpu_job(file_priv, job)) 560 return -EINVAL; 561 562 if (copy_from_user(&reset, ext, sizeof(reset))) 563 return -EFAULT; 564 565 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 566 567 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 568 reset.count); 569 if (!query_info->queries) 570 return -ENOMEM; 571 572 syncs = u64_to_user_ptr(reset.syncs); 573 574 for (i = 0; i < reset.count; i++) { 575 u32 sync; 576 577 query_info->queries[i].offset = reset.offset + 8 * i; 578 579 if (get_user(sync, syncs++)) { 580 err = -EFAULT; 581 goto error; 582 } 583 584 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 585 sync); 586 if (!query_info->queries[i].syncobj) { 587 err = -ENOENT; 588 goto error; 589 } 590 } 591 query_info->count = reset.count; 592 593 return 0; 594 595 error: 596 v3d_timestamp_query_info_free(&job->timestamp_query, i); 597 return err; 598 } 599 600 /* Get data for the copy timestamp query results job submission. */ 601 static int 602 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 603 struct drm_v3d_extension __user *ext, 604 struct v3d_cpu_job *job) 605 { 606 u32 __user *offsets, *syncs; 607 struct drm_v3d_copy_timestamp_query copy; 608 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 609 unsigned int i; 610 int err; 611 612 if (!v3d_validate_cpu_job(file_priv, job)) 613 return -EINVAL; 614 615 if (copy_from_user(©, ext, sizeof(copy))) 616 return -EFAULT; 617 618 if (copy.pad) 619 return -EINVAL; 620 621 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 622 623 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 624 copy.count); 625 if (!query_info->queries) 626 return -ENOMEM; 627 628 offsets = u64_to_user_ptr(copy.offsets); 629 syncs = u64_to_user_ptr(copy.syncs); 630 631 for (i = 0; i < copy.count; i++) { 632 u32 offset, sync; 633 634 if (get_user(offset, offsets++)) { 635 err = -EFAULT; 636 goto error; 637 } 638 639 query_info->queries[i].offset = offset; 640 641 if (get_user(sync, syncs++)) { 642 err = -EFAULT; 643 goto error; 644 } 645 646 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 647 sync); 648 if (!query_info->queries[i].syncobj) { 649 err = -ENOENT; 650 goto error; 651 } 652 } 653 query_info->count = copy.count; 654 655 job->copy.do_64bit = copy.do_64bit; 656 job->copy.do_partial = copy.do_partial; 657 job->copy.availability_bit = copy.availability_bit; 658 job->copy.offset = copy.offset; 659 job->copy.stride = copy.stride; 660 661 return 0; 662 663 error: 664 v3d_timestamp_query_info_free(&job->timestamp_query, i); 665 return err; 666 } 667 668 static int 669 v3d_copy_query_info(struct v3d_performance_query_info *query_info, 670 unsigned int count, 671 unsigned int nperfmons, 672 u32 __user *syncs, 673 u64 __user *kperfmon_ids, 674 struct drm_file *file_priv) 675 { 676 unsigned int i, j; 677 int err; 678 679 for (i = 0; i < count; i++) { 680 struct v3d_performance_query *query = &query_info->queries[i]; 681 u32 __user *ids_pointer; 682 u32 sync, id; 683 u64 ids; 684 685 if (get_user(sync, syncs++)) { 686 err = -EFAULT; 687 goto error; 688 } 689 690 if (get_user(ids, kperfmon_ids++)) { 691 err = -EFAULT; 692 goto error; 693 } 694 695 query->kperfmon_ids = 696 kvmalloc_array(nperfmons, 697 sizeof(struct v3d_performance_query *), 698 GFP_KERNEL); 699 if (!query->kperfmon_ids) { 700 err = -ENOMEM; 701 goto error; 702 } 703 704 ids_pointer = u64_to_user_ptr(ids); 705 706 for (j = 0; j < nperfmons; j++) { 707 if (get_user(id, ids_pointer++)) { 708 kvfree(query->kperfmon_ids); 709 err = -EFAULT; 710 goto error; 711 } 712 713 query->kperfmon_ids[j] = id; 714 } 715 716 query->syncobj = drm_syncobj_find(file_priv, sync); 717 if (!query->syncobj) { 718 kvfree(query->kperfmon_ids); 719 err = -ENOENT; 720 goto error; 721 } 722 } 723 724 return 0; 725 726 error: 727 v3d_performance_query_info_free(query_info, i); 728 return err; 729 } 730 731 static int 732 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 733 struct drm_v3d_extension __user *ext, 734 struct v3d_cpu_job *job) 735 { 736 struct v3d_performance_query_info *query_info = &job->performance_query; 737 struct drm_v3d_reset_performance_query reset; 738 int err; 739 740 if (!v3d_validate_cpu_job(file_priv, job)) 741 return -EINVAL; 742 743 if (copy_from_user(&reset, ext, sizeof(reset))) 744 return -EFAULT; 745 746 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 747 748 query_info->queries = 749 kvmalloc_objs(struct v3d_performance_query, reset.count); 750 if (!query_info->queries) 751 return -ENOMEM; 752 753 err = v3d_copy_query_info(query_info, 754 reset.count, 755 reset.nperfmons, 756 u64_to_user_ptr(reset.syncs), 757 u64_to_user_ptr(reset.kperfmon_ids), 758 file_priv); 759 if (err) 760 return err; 761 762 query_info->count = reset.count; 763 query_info->nperfmons = reset.nperfmons; 764 765 return 0; 766 } 767 768 static int 769 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 770 struct drm_v3d_extension __user *ext, 771 struct v3d_cpu_job *job) 772 { 773 struct v3d_performance_query_info *query_info = &job->performance_query; 774 struct drm_v3d_copy_performance_query copy; 775 int err; 776 777 if (!v3d_validate_cpu_job(file_priv, job)) 778 return -EINVAL; 779 780 if (copy_from_user(©, ext, sizeof(copy))) 781 return -EFAULT; 782 783 if (copy.pad) 784 return -EINVAL; 785 786 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 787 788 query_info->queries = 789 kvmalloc_objs(struct v3d_performance_query, copy.count); 790 if (!query_info->queries) 791 return -ENOMEM; 792 793 err = v3d_copy_query_info(query_info, 794 copy.count, 795 copy.nperfmons, 796 u64_to_user_ptr(copy.syncs), 797 u64_to_user_ptr(copy.kperfmon_ids), 798 file_priv); 799 if (err) 800 return err; 801 802 query_info->count = copy.count; 803 query_info->nperfmons = copy.nperfmons; 804 query_info->ncounters = copy.ncounters; 805 806 job->copy.do_64bit = copy.do_64bit; 807 job->copy.do_partial = copy.do_partial; 808 job->copy.availability_bit = copy.availability_bit; 809 job->copy.offset = copy.offset; 810 job->copy.stride = copy.stride; 811 812 return 0; 813 } 814 815 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 816 * according to the extension id (name). 817 */ 818 static int 819 v3d_get_extensions(struct drm_file *file_priv, 820 u64 ext_handles, 821 struct v3d_submit_ext *se, 822 struct v3d_cpu_job *job) 823 { 824 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 825 struct v3d_dev *v3d = v3d_priv->v3d; 826 struct drm_v3d_extension __user *user_ext; 827 int ret; 828 829 user_ext = u64_to_user_ptr(ext_handles); 830 while (user_ext) { 831 struct drm_v3d_extension ext; 832 833 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 834 drm_dbg(&v3d->drm, "Failed to copy submit extension\n"); 835 return -EFAULT; 836 } 837 838 switch (ext.id) { 839 case DRM_V3D_EXT_ID_MULTI_SYNC: 840 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 841 break; 842 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 843 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 844 break; 845 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 846 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 847 break; 848 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 849 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 850 break; 851 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 852 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 853 break; 854 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 855 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 856 break; 857 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 858 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 859 break; 860 default: 861 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id); 862 return -EINVAL; 863 } 864 865 if (ret) 866 return ret; 867 868 user_ext = u64_to_user_ptr(ext.next); 869 } 870 871 return 0; 872 } 873 874 /** 875 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 876 * @dev: DRM device 877 * @data: ioctl argument 878 * @file_priv: DRM file for this fd 879 * 880 * This is the main entrypoint for userspace to submit a 3D frame to 881 * the GPU. Userspace provides the binner command list (if 882 * applicable), and the kernel sets up the render command list to draw 883 * to the framebuffer described in the ioctl, using the command lists 884 * that the 3D engine's binner will produce. 885 */ 886 int 887 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 888 struct drm_file *file_priv) 889 { 890 struct v3d_dev *v3d = to_v3d_dev(dev); 891 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 892 struct drm_v3d_submit_cl *args = data; 893 struct v3d_submit_ext se = {0}; 894 struct v3d_bin_job *bin = NULL; 895 struct v3d_render_job *render = NULL; 896 struct v3d_job *clean_job = NULL; 897 struct v3d_job *last_job; 898 struct ww_acquire_ctx acquire_ctx; 899 int ret = 0; 900 901 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); 902 903 if (args->pad) 904 return -EINVAL; 905 906 if (args->flags && 907 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 908 DRM_V3D_SUBMIT_EXTENSION)) { 909 drm_dbg(dev, "invalid flags: %d\n", args->flags); 910 return -EINVAL; 911 } 912 913 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 914 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 915 if (ret) { 916 drm_dbg(dev, "Failed to get extensions.\n"); 917 return ret; 918 } 919 } 920 921 ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); 922 if (ret) 923 return ret; 924 925 ret = v3d_job_init(v3d, file_priv, &render->base, 926 v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); 927 if (ret) { 928 v3d_job_deallocate((void *)&render); 929 goto fail; 930 } 931 932 render->start = args->rcl_start; 933 render->end = args->rcl_end; 934 INIT_LIST_HEAD(&render->unref_list); 935 936 if (args->bcl_start != args->bcl_end) { 937 ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); 938 if (ret) 939 goto fail; 940 941 ret = v3d_job_init(v3d, file_priv, &bin->base, 942 v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); 943 if (ret) { 944 v3d_job_deallocate((void *)&bin); 945 goto fail; 946 } 947 948 bin->start = args->bcl_start; 949 bin->end = args->bcl_end; 950 bin->qma = args->qma; 951 bin->qms = args->qms; 952 bin->qts = args->qts; 953 bin->render = render; 954 } 955 956 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 957 ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); 958 if (ret) 959 goto fail; 960 961 ret = v3d_job_init(v3d, file_priv, clean_job, 962 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 963 if (ret) { 964 v3d_job_deallocate((void *)&clean_job); 965 goto fail; 966 } 967 968 last_job = clean_job; 969 } else { 970 last_job = &render->base; 971 } 972 973 ret = v3d_lookup_bos(dev, file_priv, last_job, 974 args->bo_handles, args->bo_handle_count); 975 if (ret) 976 goto fail; 977 978 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); 979 if (ret) 980 goto fail; 981 982 if (args->perfmon_id) { 983 if (v3d->global_perfmon) { 984 ret = -EAGAIN; 985 goto fail_perfmon; 986 } 987 988 render->base.perfmon = v3d_perfmon_find(v3d_priv, 989 args->perfmon_id); 990 991 if (!render->base.perfmon) { 992 ret = -ENOENT; 993 goto fail_perfmon; 994 } 995 } 996 997 mutex_lock(&v3d->sched_lock); 998 if (bin) { 999 bin->base.perfmon = render->base.perfmon; 1000 v3d_perfmon_get(bin->base.perfmon); 1001 v3d_push_job(&bin->base); 1002 1003 ret = drm_sched_job_add_dependency(&render->base.base, 1004 dma_fence_get(bin->base.done_fence)); 1005 if (ret) 1006 goto fail_unreserve; 1007 } 1008 1009 v3d_push_job(&render->base); 1010 1011 if (clean_job) { 1012 struct dma_fence *render_fence = 1013 dma_fence_get(render->base.done_fence); 1014 ret = drm_sched_job_add_dependency(&clean_job->base, 1015 render_fence); 1016 if (ret) 1017 goto fail_unreserve; 1018 clean_job->perfmon = render->base.perfmon; 1019 v3d_perfmon_get(clean_job->perfmon); 1020 v3d_push_job(clean_job); 1021 } 1022 1023 mutex_unlock(&v3d->sched_lock); 1024 1025 v3d_attach_fences_and_unlock_reservation(file_priv, 1026 last_job, 1027 &acquire_ctx, 1028 args->out_sync, 1029 &se, 1030 last_job->done_fence); 1031 1032 v3d_job_put(&bin->base); 1033 v3d_job_put(&render->base); 1034 v3d_job_put(clean_job); 1035 1036 return 0; 1037 1038 fail_unreserve: 1039 mutex_unlock(&v3d->sched_lock); 1040 fail_perfmon: 1041 drm_gem_unlock_reservations(last_job->bo, 1042 last_job->bo_count, &acquire_ctx); 1043 fail: 1044 v3d_job_cleanup((void *)bin); 1045 v3d_job_cleanup((void *)render); 1046 v3d_job_cleanup(clean_job); 1047 v3d_put_multisync_post_deps(&se); 1048 1049 return ret; 1050 } 1051 1052 /** 1053 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1054 * @dev: DRM device 1055 * @data: ioctl argument 1056 * @file_priv: DRM file for this fd 1057 * 1058 * Userspace provides the register setup for the TFU, which we don't 1059 * need to validate since the TFU is behind the MMU. 1060 */ 1061 int 1062 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1063 struct drm_file *file_priv) 1064 { 1065 struct v3d_dev *v3d = to_v3d_dev(dev); 1066 struct drm_v3d_submit_tfu *args = data; 1067 struct v3d_submit_ext se = {0}; 1068 struct v3d_tfu_job *job = NULL; 1069 struct ww_acquire_ctx acquire_ctx; 1070 int ret = 0; 1071 1072 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); 1073 1074 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1075 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1076 return -EINVAL; 1077 } 1078 1079 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1080 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1081 if (ret) { 1082 drm_dbg(dev, "Failed to get extensions.\n"); 1083 return ret; 1084 } 1085 } 1086 1087 ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); 1088 if (ret) 1089 return ret; 1090 1091 ret = v3d_job_init(v3d, file_priv, &job->base, 1092 v3d_job_free, args->in_sync, &se, V3D_TFU); 1093 if (ret) { 1094 v3d_job_deallocate((void *)&job); 1095 goto fail; 1096 } 1097 1098 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); 1099 if (!job->base.bo) { 1100 ret = -ENOMEM; 1101 goto fail; 1102 } 1103 1104 job->args = *args; 1105 1106 for (job->base.bo_count = 0; 1107 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1108 job->base.bo_count++) { 1109 struct drm_gem_object *bo; 1110 1111 if (!args->bo_handles[job->base.bo_count]) 1112 break; 1113 1114 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1115 if (!bo) { 1116 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n", 1117 job->base.bo_count, 1118 args->bo_handles[job->base.bo_count]); 1119 ret = -ENOENT; 1120 goto fail; 1121 } 1122 job->base.bo[job->base.bo_count] = bo; 1123 } 1124 1125 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); 1126 if (ret) 1127 goto fail; 1128 1129 mutex_lock(&v3d->sched_lock); 1130 v3d_push_job(&job->base); 1131 mutex_unlock(&v3d->sched_lock); 1132 1133 v3d_attach_fences_and_unlock_reservation(file_priv, 1134 &job->base, &acquire_ctx, 1135 args->out_sync, 1136 &se, 1137 job->base.done_fence); 1138 1139 v3d_job_put(&job->base); 1140 1141 return 0; 1142 1143 fail: 1144 v3d_job_cleanup((void *)job); 1145 v3d_put_multisync_post_deps(&se); 1146 1147 return ret; 1148 } 1149 1150 /** 1151 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1152 * @dev: DRM device 1153 * @data: ioctl argument 1154 * @file_priv: DRM file for this fd 1155 * 1156 * Userspace provides the register setup for the CSD, which we don't 1157 * need to validate since the CSD is behind the MMU. 1158 */ 1159 int 1160 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1161 struct drm_file *file_priv) 1162 { 1163 struct v3d_dev *v3d = to_v3d_dev(dev); 1164 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 1165 struct drm_v3d_submit_csd *args = data; 1166 struct v3d_submit_ext se = {0}; 1167 struct v3d_csd_job *job = NULL; 1168 struct v3d_job *clean_job = NULL; 1169 struct ww_acquire_ctx acquire_ctx; 1170 int ret; 1171 1172 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); 1173 1174 if (args->pad) 1175 return -EINVAL; 1176 1177 if (!v3d_has_csd(v3d)) { 1178 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); 1179 return -EINVAL; 1180 } 1181 1182 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1183 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1184 return -EINVAL; 1185 } 1186 1187 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1188 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1189 if (ret) { 1190 drm_dbg(dev, "Failed to get extensions.\n"); 1191 return ret; 1192 } 1193 } 1194 1195 ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, 1196 &job, &clean_job, &se, 1197 &acquire_ctx); 1198 if (ret) 1199 goto fail; 1200 1201 if (args->perfmon_id) { 1202 if (v3d->global_perfmon) { 1203 ret = -EAGAIN; 1204 goto fail_perfmon; 1205 } 1206 1207 job->base.perfmon = v3d_perfmon_find(v3d_priv, 1208 args->perfmon_id); 1209 if (!job->base.perfmon) { 1210 ret = -ENOENT; 1211 goto fail_perfmon; 1212 } 1213 } 1214 1215 mutex_lock(&v3d->sched_lock); 1216 v3d_push_job(&job->base); 1217 1218 ret = drm_sched_job_add_dependency(&clean_job->base, 1219 dma_fence_get(job->base.done_fence)); 1220 if (ret) 1221 goto fail_unreserve; 1222 1223 v3d_push_job(clean_job); 1224 mutex_unlock(&v3d->sched_lock); 1225 1226 v3d_attach_fences_and_unlock_reservation(file_priv, 1227 clean_job, 1228 &acquire_ctx, 1229 args->out_sync, 1230 &se, 1231 clean_job->done_fence); 1232 1233 v3d_job_put(&job->base); 1234 v3d_job_put(clean_job); 1235 1236 return 0; 1237 1238 fail_unreserve: 1239 mutex_unlock(&v3d->sched_lock); 1240 fail_perfmon: 1241 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1242 &acquire_ctx); 1243 fail: 1244 v3d_job_cleanup((void *)job); 1245 v3d_job_cleanup(clean_job); 1246 v3d_put_multisync_post_deps(&se); 1247 1248 return ret; 1249 } 1250 1251 static const unsigned int cpu_job_bo_handle_count[] = { 1252 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1253 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1254 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1255 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1256 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1257 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1258 }; 1259 1260 /** 1261 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1262 * @dev: DRM device 1263 * @data: ioctl argument 1264 * @file_priv: DRM file for this fd 1265 * 1266 * Userspace specifies the CPU job type and data required to perform its 1267 * operations through the drm_v3d_extension struct. 1268 */ 1269 int 1270 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1271 struct drm_file *file_priv) 1272 { 1273 struct v3d_dev *v3d = to_v3d_dev(dev); 1274 struct drm_v3d_submit_cpu *args = data; 1275 struct v3d_submit_ext se = {0}; 1276 struct v3d_submit_ext *out_se = NULL; 1277 struct v3d_cpu_job *cpu_job = NULL; 1278 struct v3d_csd_job *csd_job = NULL; 1279 struct v3d_job *clean_job = NULL; 1280 struct ww_acquire_ctx acquire_ctx; 1281 int ret; 1282 1283 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1284 drm_dbg(dev, "Invalid flags: %d\n", args->flags); 1285 return -EINVAL; 1286 } 1287 1288 ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); 1289 if (ret) 1290 return ret; 1291 1292 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1293 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1294 if (ret) { 1295 drm_dbg(dev, "Failed to get extensions.\n"); 1296 goto fail; 1297 } 1298 } 1299 1300 /* Every CPU job must have a CPU job user extension */ 1301 if (!cpu_job->job_type) { 1302 drm_dbg(dev, "CPU job must have a CPU job user extension.\n"); 1303 ret = -EINVAL; 1304 goto fail; 1305 } 1306 1307 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1308 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n"); 1309 ret = -EINVAL; 1310 goto fail; 1311 } 1312 1313 trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); 1314 1315 ret = v3d_job_init(v3d, file_priv, &cpu_job->base, 1316 v3d_job_free, 0, &se, V3D_CPU); 1317 if (ret) { 1318 v3d_job_deallocate((void *)&cpu_job); 1319 goto fail; 1320 } 1321 1322 clean_job = cpu_job->indirect_csd.clean_job; 1323 csd_job = cpu_job->indirect_csd.job; 1324 1325 if (args->bo_handle_count) { 1326 ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, 1327 args->bo_handles, args->bo_handle_count); 1328 if (ret) 1329 goto fail; 1330 1331 ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); 1332 if (ret) 1333 goto fail; 1334 } 1335 1336 mutex_lock(&v3d->sched_lock); 1337 v3d_push_job(&cpu_job->base); 1338 1339 switch (cpu_job->job_type) { 1340 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1341 ret = drm_sched_job_add_dependency(&csd_job->base.base, 1342 dma_fence_get(cpu_job->base.done_fence)); 1343 if (ret) 1344 goto fail_unreserve; 1345 1346 v3d_push_job(&csd_job->base); 1347 1348 ret = drm_sched_job_add_dependency(&clean_job->base, 1349 dma_fence_get(csd_job->base.done_fence)); 1350 if (ret) 1351 goto fail_unreserve; 1352 1353 v3d_push_job(clean_job); 1354 1355 break; 1356 default: 1357 break; 1358 } 1359 mutex_unlock(&v3d->sched_lock); 1360 1361 out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; 1362 1363 v3d_attach_fences_and_unlock_reservation(file_priv, 1364 &cpu_job->base, 1365 &acquire_ctx, 0, 1366 out_se, cpu_job->base.done_fence); 1367 1368 switch (cpu_job->job_type) { 1369 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1370 v3d_attach_fences_and_unlock_reservation(file_priv, 1371 clean_job, 1372 &cpu_job->indirect_csd.acquire_ctx, 1373 0, &se, clean_job->done_fence); 1374 break; 1375 default: 1376 break; 1377 } 1378 1379 v3d_job_put(&cpu_job->base); 1380 v3d_job_put(&csd_job->base); 1381 v3d_job_put(clean_job); 1382 1383 return 0; 1384 1385 fail_unreserve: 1386 mutex_unlock(&v3d->sched_lock); 1387 1388 drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, 1389 &acquire_ctx); 1390 1391 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1392 &cpu_job->indirect_csd.acquire_ctx); 1393 1394 fail: 1395 v3d_job_cleanup((void *)cpu_job); 1396 v3d_job_cleanup((void *)csd_job); 1397 v3d_job_cleanup(clean_job); 1398 v3d_put_multisync_post_deps(&se); 1399 kvfree(cpu_job->timestamp_query.queries); 1400 kvfree(cpu_job->performance_query.queries); 1401 1402 return ret; 1403 } 1404