1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <drm/drm_print.h> 8 #include <drm/drm_syncobj.h> 9 10 #include "v3d_drv.h" 11 #include "v3d_regs.h" 12 #include "v3d_trace.h" 13 14 /* Takes the reservation lock on all the BOs being referenced, so that 15 * we can attach fences and update the reservations after pushing the job 16 * to the queue. 17 * 18 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 19 * (all of which are on render->unref_list). They're entirely private 20 * to v3d, so we don't attach dma-buf fences to them. 21 */ 22 static int 23 v3d_lock_bo_reservations(struct v3d_job *job, 24 struct ww_acquire_ctx *acquire_ctx) 25 { 26 int i, ret; 27 28 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); 29 if (ret) 30 return ret; 31 32 for (i = 0; i < job->bo_count; i++) { 33 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); 34 if (ret) 35 goto fail; 36 37 ret = drm_sched_job_add_implicit_dependencies(&job->base, 38 job->bo[i], true); 39 if (ret) 40 goto fail; 41 } 42 43 return 0; 44 45 fail: 46 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 47 return ret; 48 } 49 50 /** 51 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 52 * referenced by the job. 53 * @dev: DRM device 54 * @file_priv: DRM file for this fd 55 * @job: V3D job being set up 56 * @bo_handles: GEM handles 57 * @bo_count: Number of GEM handles passed in 58 * 59 * The command validator needs to reference BOs by their index within 60 * the submitted job's BO list. This does the validation of the job's 61 * BO list and reference counting for the lifetime of the job. 62 * 63 * Note that this function doesn't need to unreference the BOs on 64 * failure, because that will happen at `v3d_job_free()`. 65 */ 66 static int 67 v3d_lookup_bos(struct drm_device *dev, 68 struct drm_file *file_priv, 69 struct v3d_job *job, 70 u64 bo_handles, 71 u32 bo_count) 72 { 73 job->bo_count = bo_count; 74 75 if (!job->bo_count) { 76 /* See comment on bo_index for why we have to check 77 * this. 78 */ 79 drm_warn(dev, "Rendering requires BOs\n"); 80 return -EINVAL; 81 } 82 83 return drm_gem_objects_lookup(file_priv, 84 (void __user *)(uintptr_t)bo_handles, 85 job->bo_count, &job->bo); 86 } 87 88 static void 89 v3d_job_free(struct kref *ref) 90 { 91 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 92 int i; 93 94 if (job->bo) { 95 for (i = 0; i < job->bo_count; i++) 96 drm_gem_object_put(job->bo[i]); 97 kvfree(job->bo); 98 } 99 100 dma_fence_put(job->irq_fence); 101 dma_fence_put(job->done_fence); 102 103 if (job->perfmon) 104 v3d_perfmon_put(job->perfmon); 105 106 v3d_stats_put(job->client_stats); 107 v3d_stats_put(job->global_stats); 108 109 kfree(job); 110 } 111 112 static void 113 v3d_render_job_free(struct kref *ref) 114 { 115 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 116 base.refcount); 117 struct v3d_bo *bo, *save; 118 119 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 120 drm_gem_object_put(&bo->base.base); 121 } 122 123 v3d_job_free(ref); 124 } 125 126 void v3d_job_cleanup(struct v3d_job *job) 127 { 128 if (!job) 129 return; 130 131 drm_sched_job_cleanup(&job->base); 132 v3d_job_put(job); 133 } 134 135 void v3d_job_put(struct v3d_job *job) 136 { 137 if (!job) 138 return; 139 140 kref_put(&job->refcount, job->free); 141 } 142 143 static int 144 v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) 145 { 146 *container = kcalloc(1, size, GFP_KERNEL); 147 if (!*container) { 148 drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); 149 return -ENOMEM; 150 } 151 152 return 0; 153 } 154 155 static void 156 v3d_job_deallocate(void **container) 157 { 158 kfree(*container); 159 *container = NULL; 160 } 161 162 static int 163 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, 164 struct v3d_job *job, void (*free)(struct kref *ref), 165 u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) 166 { 167 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 168 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 169 int ret, i; 170 171 job->v3d = v3d; 172 job->free = free; 173 job->file_priv = v3d_priv; 174 175 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 176 1, v3d_priv, file_priv->client_id); 177 if (ret) 178 return ret; 179 180 if (has_multisync) { 181 if (se->in_sync_count && se->wait_stage == queue) { 182 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 183 184 for (i = 0; i < se->in_sync_count; i++) { 185 struct drm_v3d_sem in; 186 187 if (copy_from_user(&in, handle++, sizeof(in))) { 188 ret = -EFAULT; 189 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); 190 goto fail_deps; 191 } 192 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); 193 194 // TODO: Investigate why this was filtered out for the IOCTL. 195 if (ret && ret != -ENOENT) 196 goto fail_deps; 197 } 198 } 199 } else { 200 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); 201 202 // TODO: Investigate why this was filtered out for the IOCTL. 203 if (ret && ret != -ENOENT) 204 goto fail_deps; 205 } 206 207 kref_init(&job->refcount); 208 209 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); 210 job->global_stats = v3d_stats_get(v3d->queue[queue].stats); 211 212 return 0; 213 214 fail_deps: 215 drm_sched_job_cleanup(&job->base); 216 return ret; 217 } 218 219 static void 220 v3d_push_job(struct v3d_job *job) 221 { 222 drm_sched_job_arm(&job->base); 223 224 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 225 226 /* put by scheduler job completion */ 227 kref_get(&job->refcount); 228 229 drm_sched_entity_push_job(&job->base); 230 } 231 232 static void 233 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, 234 struct v3d_job *job, 235 struct ww_acquire_ctx *acquire_ctx, 236 u32 out_sync, 237 struct v3d_submit_ext *se, 238 struct dma_fence *done_fence) 239 { 240 struct drm_syncobj *sync_out; 241 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 242 int i; 243 244 for (i = 0; i < job->bo_count; i++) { 245 /* XXX: Use shared fences for read-only objects. */ 246 dma_resv_add_fence(job->bo[i]->resv, job->done_fence, 247 DMA_RESV_USAGE_WRITE); 248 } 249 250 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 251 252 /* Update the return sync object for the job */ 253 /* If it only supports a single signal semaphore*/ 254 if (!has_multisync) { 255 sync_out = drm_syncobj_find(file_priv, out_sync); 256 if (sync_out) { 257 drm_syncobj_replace_fence(sync_out, done_fence); 258 drm_syncobj_put(sync_out); 259 } 260 return; 261 } 262 263 /* If multiple semaphores extension is supported */ 264 if (se->out_sync_count) { 265 for (i = 0; i < se->out_sync_count; i++) { 266 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 267 done_fence); 268 drm_syncobj_put(se->out_syncs[i].syncobj); 269 } 270 kvfree(se->out_syncs); 271 } 272 } 273 274 static int 275 v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, 276 struct v3d_dev *v3d, 277 struct drm_v3d_submit_csd *args, 278 struct v3d_csd_job **job, 279 struct v3d_job **clean_job, 280 struct v3d_submit_ext *se, 281 struct ww_acquire_ctx *acquire_ctx) 282 { 283 int ret; 284 285 ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); 286 if (ret) 287 return ret; 288 289 ret = v3d_job_init(v3d, file_priv, &(*job)->base, 290 v3d_job_free, args->in_sync, se, V3D_CSD); 291 if (ret) { 292 v3d_job_deallocate((void *)job); 293 return ret; 294 } 295 296 ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); 297 if (ret) 298 return ret; 299 300 ret = v3d_job_init(v3d, file_priv, *clean_job, 301 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 302 if (ret) { 303 v3d_job_deallocate((void *)clean_job); 304 return ret; 305 } 306 307 (*job)->args = *args; 308 309 ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, 310 args->bo_handles, args->bo_handle_count); 311 if (ret) 312 return ret; 313 314 return v3d_lock_bo_reservations(*clean_job, acquire_ctx); 315 } 316 317 static void 318 v3d_put_multisync_post_deps(struct v3d_submit_ext *se) 319 { 320 unsigned int i; 321 322 if (!(se && se->out_sync_count)) 323 return; 324 325 for (i = 0; i < se->out_sync_count; i++) 326 drm_syncobj_put(se->out_syncs[i].syncobj); 327 kvfree(se->out_syncs); 328 } 329 330 static int 331 v3d_get_multisync_post_deps(struct drm_file *file_priv, 332 struct v3d_submit_ext *se, 333 u32 count, u64 handles) 334 { 335 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 336 struct v3d_dev *v3d = v3d_priv->v3d; 337 struct drm_v3d_sem __user *post_deps; 338 int i, ret; 339 340 if (!count) 341 return 0; 342 343 se->out_syncs = (struct v3d_submit_outsync *) 344 kvmalloc_objs(struct v3d_submit_outsync, count); 345 if (!se->out_syncs) 346 return -ENOMEM; 347 348 post_deps = u64_to_user_ptr(handles); 349 350 for (i = 0; i < count; i++) { 351 struct drm_v3d_sem out; 352 353 if (copy_from_user(&out, post_deps++, sizeof(out))) { 354 ret = -EFAULT; 355 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n"); 356 goto fail; 357 } 358 359 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 360 out.handle); 361 if (!se->out_syncs[i].syncobj) { 362 ret = -EINVAL; 363 goto fail; 364 } 365 } 366 se->out_sync_count = count; 367 368 return 0; 369 370 fail: 371 for (i--; i >= 0; i--) 372 drm_syncobj_put(se->out_syncs[i].syncobj); 373 kvfree(se->out_syncs); 374 375 return ret; 376 } 377 378 /* Get data for multiple binary semaphores synchronization. Parse syncobj 379 * to be signaled when job completes (out_sync). 380 */ 381 static int 382 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 383 struct drm_v3d_extension __user *ext, 384 struct v3d_submit_ext *se) 385 { 386 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 387 struct v3d_dev *v3d = v3d_priv->v3d; 388 struct drm_v3d_multi_sync multisync; 389 int ret; 390 391 if (se->in_sync_count || se->out_sync_count) { 392 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job."); 393 return -EINVAL; 394 } 395 396 if (copy_from_user(&multisync, ext, sizeof(multisync))) 397 return -EFAULT; 398 399 if (multisync.pad) 400 return -EINVAL; 401 402 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 403 multisync.out_syncs); 404 if (ret) 405 return ret; 406 407 se->in_sync_count = multisync.in_sync_count; 408 se->in_syncs = multisync.in_syncs; 409 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 410 se->wait_stage = multisync.wait_stage; 411 412 return 0; 413 } 414 415 /* Returns false if the CPU job has an invalid configuration. */ 416 static bool 417 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job) 418 { 419 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 420 struct v3d_dev *v3d = v3d_priv->v3d; 421 422 if (!job) { 423 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n"); 424 return false; 425 } 426 427 if (job->job_type) { 428 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n"); 429 return false; 430 } 431 432 return true; 433 } 434 435 /* Get data for the indirect CSD job submission. */ 436 static int 437 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 438 struct drm_v3d_extension __user *ext, 439 struct v3d_cpu_job *job) 440 { 441 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 442 struct v3d_dev *v3d = v3d_priv->v3d; 443 struct drm_v3d_indirect_csd indirect_csd; 444 struct v3d_indirect_csd_info *info = &job->indirect_csd; 445 446 if (!v3d_validate_cpu_job(file_priv, job)) 447 return -EINVAL; 448 449 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 450 return -EFAULT; 451 452 if (!v3d_has_csd(v3d)) { 453 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n"); 454 return -EINVAL; 455 } 456 457 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 458 info->offset = indirect_csd.offset; 459 info->wg_size = indirect_csd.wg_size; 460 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 461 sizeof(indirect_csd.wg_uniform_offsets)); 462 463 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 464 465 return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, 466 &info->job, &info->clean_job, 467 NULL, &info->acquire_ctx); 468 } 469 470 /* Get data for the query timestamp job submission. */ 471 static int 472 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 473 struct drm_v3d_extension __user *ext, 474 struct v3d_cpu_job *job) 475 { 476 u32 __user *offsets, *syncs; 477 struct drm_v3d_timestamp_query timestamp; 478 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 479 unsigned int i; 480 int err; 481 482 if (!v3d_validate_cpu_job(file_priv, job)) 483 return -EINVAL; 484 485 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 486 return -EFAULT; 487 488 if (timestamp.pad) 489 return -EINVAL; 490 491 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 492 493 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 494 timestamp.count); 495 if (!query_info->queries) 496 return -ENOMEM; 497 498 offsets = u64_to_user_ptr(timestamp.offsets); 499 syncs = u64_to_user_ptr(timestamp.syncs); 500 501 for (i = 0; i < timestamp.count; i++) { 502 u32 offset, sync; 503 504 if (get_user(offset, offsets++)) { 505 err = -EFAULT; 506 goto error; 507 } 508 509 query_info->queries[i].offset = offset; 510 511 if (get_user(sync, syncs++)) { 512 err = -EFAULT; 513 goto error; 514 } 515 516 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 517 sync); 518 if (!query_info->queries[i].syncobj) { 519 err = -ENOENT; 520 goto error; 521 } 522 } 523 query_info->count = timestamp.count; 524 525 return 0; 526 527 error: 528 v3d_timestamp_query_info_free(&job->timestamp_query, i); 529 return err; 530 } 531 532 static int 533 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 534 struct drm_v3d_extension __user *ext, 535 struct v3d_cpu_job *job) 536 { 537 u32 __user *syncs; 538 struct drm_v3d_reset_timestamp_query reset; 539 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 540 unsigned int i; 541 int err; 542 543 if (!v3d_validate_cpu_job(file_priv, job)) 544 return -EINVAL; 545 546 if (copy_from_user(&reset, ext, sizeof(reset))) 547 return -EFAULT; 548 549 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 550 551 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 552 reset.count); 553 if (!query_info->queries) 554 return -ENOMEM; 555 556 syncs = u64_to_user_ptr(reset.syncs); 557 558 for (i = 0; i < reset.count; i++) { 559 u32 sync; 560 561 query_info->queries[i].offset = reset.offset + 8 * i; 562 563 if (get_user(sync, syncs++)) { 564 err = -EFAULT; 565 goto error; 566 } 567 568 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 569 sync); 570 if (!query_info->queries[i].syncobj) { 571 err = -ENOENT; 572 goto error; 573 } 574 } 575 query_info->count = reset.count; 576 577 return 0; 578 579 error: 580 v3d_timestamp_query_info_free(&job->timestamp_query, i); 581 return err; 582 } 583 584 /* Get data for the copy timestamp query results job submission. */ 585 static int 586 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 587 struct drm_v3d_extension __user *ext, 588 struct v3d_cpu_job *job) 589 { 590 u32 __user *offsets, *syncs; 591 struct drm_v3d_copy_timestamp_query copy; 592 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 593 unsigned int i; 594 int err; 595 596 if (!v3d_validate_cpu_job(file_priv, job)) 597 return -EINVAL; 598 599 if (copy_from_user(©, ext, sizeof(copy))) 600 return -EFAULT; 601 602 if (copy.pad) 603 return -EINVAL; 604 605 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 606 607 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 608 copy.count); 609 if (!query_info->queries) 610 return -ENOMEM; 611 612 offsets = u64_to_user_ptr(copy.offsets); 613 syncs = u64_to_user_ptr(copy.syncs); 614 615 for (i = 0; i < copy.count; i++) { 616 u32 offset, sync; 617 618 if (get_user(offset, offsets++)) { 619 err = -EFAULT; 620 goto error; 621 } 622 623 query_info->queries[i].offset = offset; 624 625 if (get_user(sync, syncs++)) { 626 err = -EFAULT; 627 goto error; 628 } 629 630 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 631 sync); 632 if (!query_info->queries[i].syncobj) { 633 err = -ENOENT; 634 goto error; 635 } 636 } 637 query_info->count = copy.count; 638 639 job->copy.do_64bit = copy.do_64bit; 640 job->copy.do_partial = copy.do_partial; 641 job->copy.availability_bit = copy.availability_bit; 642 job->copy.offset = copy.offset; 643 job->copy.stride = copy.stride; 644 645 return 0; 646 647 error: 648 v3d_timestamp_query_info_free(&job->timestamp_query, i); 649 return err; 650 } 651 652 static int 653 v3d_copy_query_info(struct v3d_performance_query_info *query_info, 654 unsigned int count, 655 unsigned int nperfmons, 656 u32 __user *syncs, 657 u64 __user *kperfmon_ids, 658 struct drm_file *file_priv) 659 { 660 unsigned int i, j; 661 int err; 662 663 for (i = 0; i < count; i++) { 664 struct v3d_performance_query *query = &query_info->queries[i]; 665 u32 __user *ids_pointer; 666 u32 sync, id; 667 u64 ids; 668 669 if (get_user(sync, syncs++)) { 670 err = -EFAULT; 671 goto error; 672 } 673 674 if (get_user(ids, kperfmon_ids++)) { 675 err = -EFAULT; 676 goto error; 677 } 678 679 query->kperfmon_ids = 680 kvmalloc_array(nperfmons, 681 sizeof(struct v3d_performance_query *), 682 GFP_KERNEL); 683 if (!query->kperfmon_ids) { 684 err = -ENOMEM; 685 goto error; 686 } 687 688 ids_pointer = u64_to_user_ptr(ids); 689 690 for (j = 0; j < nperfmons; j++) { 691 if (get_user(id, ids_pointer++)) { 692 kvfree(query->kperfmon_ids); 693 err = -EFAULT; 694 goto error; 695 } 696 697 query->kperfmon_ids[j] = id; 698 } 699 700 query->syncobj = drm_syncobj_find(file_priv, sync); 701 if (!query->syncobj) { 702 kvfree(query->kperfmon_ids); 703 err = -ENOENT; 704 goto error; 705 } 706 } 707 708 return 0; 709 710 error: 711 v3d_performance_query_info_free(query_info, i); 712 return err; 713 } 714 715 static int 716 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 717 struct drm_v3d_extension __user *ext, 718 struct v3d_cpu_job *job) 719 { 720 struct v3d_performance_query_info *query_info = &job->performance_query; 721 struct drm_v3d_reset_performance_query reset; 722 int err; 723 724 if (!v3d_validate_cpu_job(file_priv, job)) 725 return -EINVAL; 726 727 if (copy_from_user(&reset, ext, sizeof(reset))) 728 return -EFAULT; 729 730 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 731 732 query_info->queries = 733 kvmalloc_objs(struct v3d_performance_query, reset.count); 734 if (!query_info->queries) 735 return -ENOMEM; 736 737 err = v3d_copy_query_info(query_info, 738 reset.count, 739 reset.nperfmons, 740 u64_to_user_ptr(reset.syncs), 741 u64_to_user_ptr(reset.kperfmon_ids), 742 file_priv); 743 if (err) 744 return err; 745 746 query_info->count = reset.count; 747 query_info->nperfmons = reset.nperfmons; 748 749 return 0; 750 } 751 752 static int 753 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 754 struct drm_v3d_extension __user *ext, 755 struct v3d_cpu_job *job) 756 { 757 struct v3d_performance_query_info *query_info = &job->performance_query; 758 struct drm_v3d_copy_performance_query copy; 759 int err; 760 761 if (!v3d_validate_cpu_job(file_priv, job)) 762 return -EINVAL; 763 764 if (copy_from_user(©, ext, sizeof(copy))) 765 return -EFAULT; 766 767 if (copy.pad) 768 return -EINVAL; 769 770 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 771 772 query_info->queries = 773 kvmalloc_objs(struct v3d_performance_query, copy.count); 774 if (!query_info->queries) 775 return -ENOMEM; 776 777 err = v3d_copy_query_info(query_info, 778 copy.count, 779 copy.nperfmons, 780 u64_to_user_ptr(copy.syncs), 781 u64_to_user_ptr(copy.kperfmon_ids), 782 file_priv); 783 if (err) 784 return err; 785 786 query_info->count = copy.count; 787 query_info->nperfmons = copy.nperfmons; 788 query_info->ncounters = copy.ncounters; 789 790 job->copy.do_64bit = copy.do_64bit; 791 job->copy.do_partial = copy.do_partial; 792 job->copy.availability_bit = copy.availability_bit; 793 job->copy.offset = copy.offset; 794 job->copy.stride = copy.stride; 795 796 return 0; 797 } 798 799 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 800 * according to the extension id (name). 801 */ 802 static int 803 v3d_get_extensions(struct drm_file *file_priv, 804 u64 ext_handles, 805 struct v3d_submit_ext *se, 806 struct v3d_cpu_job *job) 807 { 808 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 809 struct v3d_dev *v3d = v3d_priv->v3d; 810 struct drm_v3d_extension __user *user_ext; 811 int ret; 812 813 user_ext = u64_to_user_ptr(ext_handles); 814 while (user_ext) { 815 struct drm_v3d_extension ext; 816 817 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 818 drm_dbg(&v3d->drm, "Failed to copy submit extension\n"); 819 return -EFAULT; 820 } 821 822 switch (ext.id) { 823 case DRM_V3D_EXT_ID_MULTI_SYNC: 824 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 825 break; 826 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 827 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 828 break; 829 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 830 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 831 break; 832 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 833 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 834 break; 835 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 836 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 837 break; 838 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 839 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 840 break; 841 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 842 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 843 break; 844 default: 845 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id); 846 return -EINVAL; 847 } 848 849 if (ret) 850 return ret; 851 852 user_ext = u64_to_user_ptr(ext.next); 853 } 854 855 return 0; 856 } 857 858 /** 859 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 860 * @dev: DRM device 861 * @data: ioctl argument 862 * @file_priv: DRM file for this fd 863 * 864 * This is the main entrypoint for userspace to submit a 3D frame to 865 * the GPU. Userspace provides the binner command list (if 866 * applicable), and the kernel sets up the render command list to draw 867 * to the framebuffer described in the ioctl, using the command lists 868 * that the 3D engine's binner will produce. 869 */ 870 int 871 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 872 struct drm_file *file_priv) 873 { 874 struct v3d_dev *v3d = to_v3d_dev(dev); 875 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 876 struct drm_v3d_submit_cl *args = data; 877 struct v3d_submit_ext se = {0}; 878 struct v3d_bin_job *bin = NULL; 879 struct v3d_render_job *render = NULL; 880 struct v3d_job *clean_job = NULL; 881 struct v3d_job *last_job; 882 struct ww_acquire_ctx acquire_ctx; 883 int ret = 0; 884 885 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); 886 887 if (args->pad) 888 return -EINVAL; 889 890 if (args->flags && 891 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 892 DRM_V3D_SUBMIT_EXTENSION)) { 893 drm_dbg(dev, "invalid flags: %d\n", args->flags); 894 return -EINVAL; 895 } 896 897 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 898 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 899 if (ret) { 900 drm_dbg(dev, "Failed to get extensions.\n"); 901 return ret; 902 } 903 } 904 905 ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); 906 if (ret) 907 return ret; 908 909 ret = v3d_job_init(v3d, file_priv, &render->base, 910 v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); 911 if (ret) { 912 v3d_job_deallocate((void *)&render); 913 goto fail; 914 } 915 916 render->start = args->rcl_start; 917 render->end = args->rcl_end; 918 INIT_LIST_HEAD(&render->unref_list); 919 920 if (args->bcl_start != args->bcl_end) { 921 ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); 922 if (ret) 923 goto fail; 924 925 ret = v3d_job_init(v3d, file_priv, &bin->base, 926 v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); 927 if (ret) { 928 v3d_job_deallocate((void *)&bin); 929 goto fail; 930 } 931 932 bin->start = args->bcl_start; 933 bin->end = args->bcl_end; 934 bin->qma = args->qma; 935 bin->qms = args->qms; 936 bin->qts = args->qts; 937 bin->render = render; 938 } 939 940 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 941 ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); 942 if (ret) 943 goto fail; 944 945 ret = v3d_job_init(v3d, file_priv, clean_job, 946 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 947 if (ret) { 948 v3d_job_deallocate((void *)&clean_job); 949 goto fail; 950 } 951 952 last_job = clean_job; 953 } else { 954 last_job = &render->base; 955 } 956 957 ret = v3d_lookup_bos(dev, file_priv, last_job, 958 args->bo_handles, args->bo_handle_count); 959 if (ret) 960 goto fail; 961 962 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); 963 if (ret) 964 goto fail; 965 966 if (args->perfmon_id) { 967 if (v3d->global_perfmon) { 968 ret = -EAGAIN; 969 goto fail_perfmon; 970 } 971 972 render->base.perfmon = v3d_perfmon_find(v3d_priv, 973 args->perfmon_id); 974 975 if (!render->base.perfmon) { 976 ret = -ENOENT; 977 goto fail_perfmon; 978 } 979 } 980 981 mutex_lock(&v3d->sched_lock); 982 if (bin) { 983 bin->base.perfmon = render->base.perfmon; 984 v3d_perfmon_get(bin->base.perfmon); 985 v3d_push_job(&bin->base); 986 987 ret = drm_sched_job_add_dependency(&render->base.base, 988 dma_fence_get(bin->base.done_fence)); 989 if (ret) 990 goto fail_unreserve; 991 } 992 993 v3d_push_job(&render->base); 994 995 if (clean_job) { 996 struct dma_fence *render_fence = 997 dma_fence_get(render->base.done_fence); 998 ret = drm_sched_job_add_dependency(&clean_job->base, 999 render_fence); 1000 if (ret) 1001 goto fail_unreserve; 1002 clean_job->perfmon = render->base.perfmon; 1003 v3d_perfmon_get(clean_job->perfmon); 1004 v3d_push_job(clean_job); 1005 } 1006 1007 mutex_unlock(&v3d->sched_lock); 1008 1009 v3d_attach_fences_and_unlock_reservation(file_priv, 1010 last_job, 1011 &acquire_ctx, 1012 args->out_sync, 1013 &se, 1014 last_job->done_fence); 1015 1016 v3d_job_put(&bin->base); 1017 v3d_job_put(&render->base); 1018 v3d_job_put(clean_job); 1019 1020 return 0; 1021 1022 fail_unreserve: 1023 mutex_unlock(&v3d->sched_lock); 1024 fail_perfmon: 1025 drm_gem_unlock_reservations(last_job->bo, 1026 last_job->bo_count, &acquire_ctx); 1027 fail: 1028 v3d_job_cleanup((void *)bin); 1029 v3d_job_cleanup((void *)render); 1030 v3d_job_cleanup(clean_job); 1031 v3d_put_multisync_post_deps(&se); 1032 1033 return ret; 1034 } 1035 1036 /** 1037 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1038 * @dev: DRM device 1039 * @data: ioctl argument 1040 * @file_priv: DRM file for this fd 1041 * 1042 * Userspace provides the register setup for the TFU, which we don't 1043 * need to validate since the TFU is behind the MMU. 1044 */ 1045 int 1046 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1047 struct drm_file *file_priv) 1048 { 1049 struct v3d_dev *v3d = to_v3d_dev(dev); 1050 struct drm_v3d_submit_tfu *args = data; 1051 struct v3d_submit_ext se = {0}; 1052 struct v3d_tfu_job *job = NULL; 1053 struct ww_acquire_ctx acquire_ctx; 1054 int ret = 0; 1055 1056 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); 1057 1058 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1059 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1060 return -EINVAL; 1061 } 1062 1063 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1064 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1065 if (ret) { 1066 drm_dbg(dev, "Failed to get extensions.\n"); 1067 return ret; 1068 } 1069 } 1070 1071 ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); 1072 if (ret) 1073 return ret; 1074 1075 ret = v3d_job_init(v3d, file_priv, &job->base, 1076 v3d_job_free, args->in_sync, &se, V3D_TFU); 1077 if (ret) { 1078 v3d_job_deallocate((void *)&job); 1079 goto fail; 1080 } 1081 1082 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); 1083 if (!job->base.bo) { 1084 ret = -ENOMEM; 1085 goto fail; 1086 } 1087 1088 job->args = *args; 1089 1090 for (job->base.bo_count = 0; 1091 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1092 job->base.bo_count++) { 1093 struct drm_gem_object *bo; 1094 1095 if (!args->bo_handles[job->base.bo_count]) 1096 break; 1097 1098 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1099 if (!bo) { 1100 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n", 1101 job->base.bo_count, 1102 args->bo_handles[job->base.bo_count]); 1103 ret = -ENOENT; 1104 goto fail; 1105 } 1106 job->base.bo[job->base.bo_count] = bo; 1107 } 1108 1109 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); 1110 if (ret) 1111 goto fail; 1112 1113 mutex_lock(&v3d->sched_lock); 1114 v3d_push_job(&job->base); 1115 mutex_unlock(&v3d->sched_lock); 1116 1117 v3d_attach_fences_and_unlock_reservation(file_priv, 1118 &job->base, &acquire_ctx, 1119 args->out_sync, 1120 &se, 1121 job->base.done_fence); 1122 1123 v3d_job_put(&job->base); 1124 1125 return 0; 1126 1127 fail: 1128 v3d_job_cleanup((void *)job); 1129 v3d_put_multisync_post_deps(&se); 1130 1131 return ret; 1132 } 1133 1134 /** 1135 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1136 * @dev: DRM device 1137 * @data: ioctl argument 1138 * @file_priv: DRM file for this fd 1139 * 1140 * Userspace provides the register setup for the CSD, which we don't 1141 * need to validate since the CSD is behind the MMU. 1142 */ 1143 int 1144 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1145 struct drm_file *file_priv) 1146 { 1147 struct v3d_dev *v3d = to_v3d_dev(dev); 1148 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 1149 struct drm_v3d_submit_csd *args = data; 1150 struct v3d_submit_ext se = {0}; 1151 struct v3d_csd_job *job = NULL; 1152 struct v3d_job *clean_job = NULL; 1153 struct ww_acquire_ctx acquire_ctx; 1154 int ret; 1155 1156 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); 1157 1158 if (args->pad) 1159 return -EINVAL; 1160 1161 if (!v3d_has_csd(v3d)) { 1162 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); 1163 return -EINVAL; 1164 } 1165 1166 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1167 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1168 return -EINVAL; 1169 } 1170 1171 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1172 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1173 if (ret) { 1174 drm_dbg(dev, "Failed to get extensions.\n"); 1175 return ret; 1176 } 1177 } 1178 1179 ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, 1180 &job, &clean_job, &se, 1181 &acquire_ctx); 1182 if (ret) 1183 goto fail; 1184 1185 if (args->perfmon_id) { 1186 if (v3d->global_perfmon) { 1187 ret = -EAGAIN; 1188 goto fail_perfmon; 1189 } 1190 1191 job->base.perfmon = v3d_perfmon_find(v3d_priv, 1192 args->perfmon_id); 1193 if (!job->base.perfmon) { 1194 ret = -ENOENT; 1195 goto fail_perfmon; 1196 } 1197 } 1198 1199 mutex_lock(&v3d->sched_lock); 1200 v3d_push_job(&job->base); 1201 1202 ret = drm_sched_job_add_dependency(&clean_job->base, 1203 dma_fence_get(job->base.done_fence)); 1204 if (ret) 1205 goto fail_unreserve; 1206 1207 v3d_push_job(clean_job); 1208 mutex_unlock(&v3d->sched_lock); 1209 1210 v3d_attach_fences_and_unlock_reservation(file_priv, 1211 clean_job, 1212 &acquire_ctx, 1213 args->out_sync, 1214 &se, 1215 clean_job->done_fence); 1216 1217 v3d_job_put(&job->base); 1218 v3d_job_put(clean_job); 1219 1220 return 0; 1221 1222 fail_unreserve: 1223 mutex_unlock(&v3d->sched_lock); 1224 fail_perfmon: 1225 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1226 &acquire_ctx); 1227 fail: 1228 v3d_job_cleanup((void *)job); 1229 v3d_job_cleanup(clean_job); 1230 v3d_put_multisync_post_deps(&se); 1231 1232 return ret; 1233 } 1234 1235 static const unsigned int cpu_job_bo_handle_count[] = { 1236 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1237 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1238 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1239 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1240 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1241 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1242 }; 1243 1244 /** 1245 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1246 * @dev: DRM device 1247 * @data: ioctl argument 1248 * @file_priv: DRM file for this fd 1249 * 1250 * Userspace specifies the CPU job type and data required to perform its 1251 * operations through the drm_v3d_extension struct. 1252 */ 1253 int 1254 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1255 struct drm_file *file_priv) 1256 { 1257 struct v3d_dev *v3d = to_v3d_dev(dev); 1258 struct drm_v3d_submit_cpu *args = data; 1259 struct v3d_submit_ext se = {0}; 1260 struct v3d_submit_ext *out_se = NULL; 1261 struct v3d_cpu_job *cpu_job = NULL; 1262 struct v3d_csd_job *csd_job = NULL; 1263 struct v3d_job *clean_job = NULL; 1264 struct ww_acquire_ctx acquire_ctx; 1265 int ret; 1266 1267 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1268 drm_dbg(dev, "Invalid flags: %d\n", args->flags); 1269 return -EINVAL; 1270 } 1271 1272 ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); 1273 if (ret) 1274 return ret; 1275 1276 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1277 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1278 if (ret) { 1279 drm_dbg(dev, "Failed to get extensions.\n"); 1280 goto fail; 1281 } 1282 } 1283 1284 /* Every CPU job must have a CPU job user extension */ 1285 if (!cpu_job->job_type) { 1286 drm_dbg(dev, "CPU job must have a CPU job user extension.\n"); 1287 ret = -EINVAL; 1288 goto fail; 1289 } 1290 1291 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1292 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n"); 1293 ret = -EINVAL; 1294 goto fail; 1295 } 1296 1297 trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); 1298 1299 ret = v3d_job_init(v3d, file_priv, &cpu_job->base, 1300 v3d_job_free, 0, &se, V3D_CPU); 1301 if (ret) { 1302 v3d_job_deallocate((void *)&cpu_job); 1303 goto fail; 1304 } 1305 1306 clean_job = cpu_job->indirect_csd.clean_job; 1307 csd_job = cpu_job->indirect_csd.job; 1308 1309 if (args->bo_handle_count) { 1310 ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, 1311 args->bo_handles, args->bo_handle_count); 1312 if (ret) 1313 goto fail; 1314 1315 ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); 1316 if (ret) 1317 goto fail; 1318 } 1319 1320 mutex_lock(&v3d->sched_lock); 1321 v3d_push_job(&cpu_job->base); 1322 1323 switch (cpu_job->job_type) { 1324 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1325 ret = drm_sched_job_add_dependency(&csd_job->base.base, 1326 dma_fence_get(cpu_job->base.done_fence)); 1327 if (ret) 1328 goto fail_unreserve; 1329 1330 v3d_push_job(&csd_job->base); 1331 1332 ret = drm_sched_job_add_dependency(&clean_job->base, 1333 dma_fence_get(csd_job->base.done_fence)); 1334 if (ret) 1335 goto fail_unreserve; 1336 1337 v3d_push_job(clean_job); 1338 1339 break; 1340 default: 1341 break; 1342 } 1343 mutex_unlock(&v3d->sched_lock); 1344 1345 out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; 1346 1347 v3d_attach_fences_and_unlock_reservation(file_priv, 1348 &cpu_job->base, 1349 &acquire_ctx, 0, 1350 out_se, cpu_job->base.done_fence); 1351 1352 switch (cpu_job->job_type) { 1353 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1354 v3d_attach_fences_and_unlock_reservation(file_priv, 1355 clean_job, 1356 &cpu_job->indirect_csd.acquire_ctx, 1357 0, &se, clean_job->done_fence); 1358 break; 1359 default: 1360 break; 1361 } 1362 1363 v3d_job_put(&cpu_job->base); 1364 v3d_job_put(&csd_job->base); 1365 v3d_job_put(clean_job); 1366 1367 return 0; 1368 1369 fail_unreserve: 1370 mutex_unlock(&v3d->sched_lock); 1371 1372 drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, 1373 &acquire_ctx); 1374 1375 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1376 &cpu_job->indirect_csd.acquire_ctx); 1377 1378 fail: 1379 v3d_job_cleanup((void *)cpu_job); 1380 v3d_job_cleanup((void *)csd_job); 1381 v3d_job_cleanup(clean_job); 1382 v3d_put_multisync_post_deps(&se); 1383 kvfree(cpu_job->timestamp_query.queries); 1384 kvfree(cpu_job->performance_query.queries); 1385 1386 return ret; 1387 } 1388