1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <drm/drm_print.h> 8 #include <drm/drm_syncobj.h> 9 10 #include "v3d_drv.h" 11 #include "v3d_regs.h" 12 #include "v3d_trace.h" 13 14 /* Takes the reservation lock on all the BOs being referenced, so that 15 * we can attach fences and update the reservations after pushing the job 16 * to the queue. 17 * 18 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 19 * (all of which are on render->unref_list). They're entirely private 20 * to v3d, so we don't attach dma-buf fences to them. 21 */ 22 static int 23 v3d_submit_lock_reservations(struct v3d_submit *submit) 24 { 25 int i, j, ret; 26 27 drm_exec_init(&submit->exec, 28 DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); 29 drm_exec_until_all_locked(&submit->exec) { 30 for (i = 0; i < submit->job_count; i++) { 31 struct v3d_job *job = submit->jobs[i]; 32 33 ret = drm_exec_prepare_array(&submit->exec, job->bo, 34 job->bo_count, 1); 35 if (ret) 36 break; 37 } 38 drm_exec_retry_on_contention(&submit->exec); 39 if (ret) 40 goto fail; 41 } 42 43 for (i = 0; i < submit->job_count; i++) { 44 struct v3d_job *job = submit->jobs[i]; 45 46 for (j = 0; j < job->bo_count; j++) { 47 ret = drm_sched_job_add_implicit_dependencies(&job->base, 48 job->bo[j], 49 true); 50 if (ret) 51 goto fail; 52 } 53 } 54 55 return 0; 56 57 fail: 58 drm_exec_fini(&submit->exec); 59 return ret; 60 } 61 62 static void 63 v3d_submit_unlock_reservations(struct v3d_submit *submit) 64 { 65 drm_exec_fini(&submit->exec); 66 } 67 68 /** 69 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 70 * referenced by the job. 71 * @dev: DRM device 72 * @file_priv: DRM file for this fd 73 * @job: V3D job being set up 74 * @bo_handles: GEM handles 75 * @bo_count: Number of GEM handles passed in 76 * 77 * The command validator needs to reference BOs by their index within 78 * the submitted job's BO list. This does the validation of the job's 79 * BO list and reference counting for the lifetime of the job. 80 * 81 * Note that this function doesn't need to unreference the BOs on 82 * failure, because that will happen at `v3d_job_free()`. 83 */ 84 static int 85 v3d_lookup_bos(struct v3d_submit *submit, u64 bo_handles, u32 bo_count) 86 { 87 struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; 88 89 last_job->bo_count = bo_count; 90 91 if (!last_job->bo_count) { 92 /* See comment on bo_index for why we have to check 93 * this. 94 */ 95 drm_warn(&submit->v3d->drm, "Rendering requires BOs\n"); 96 return -EINVAL; 97 } 98 99 return drm_gem_objects_lookup(submit->file_priv, 100 (void __user *)(uintptr_t)bo_handles, 101 last_job->bo_count, &last_job->bo); 102 } 103 104 static void 105 v3d_job_free(struct kref *ref) 106 { 107 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 108 int i; 109 110 if (job->bo) { 111 for (i = 0; i < job->bo_count; i++) 112 drm_gem_object_put(job->bo[i]); 113 kvfree(job->bo); 114 } 115 116 dma_fence_put(job->irq_fence); 117 dma_fence_put(job->done_fence); 118 119 if (job->perfmon) 120 v3d_perfmon_put(job->perfmon); 121 122 v3d_stats_put(job->client_stats); 123 v3d_stats_put(job->global_stats); 124 125 if (job->has_pm_ref) 126 v3d_pm_runtime_put(job->v3d); 127 128 kfree(job); 129 } 130 131 static void 132 v3d_render_job_free(struct kref *ref) 133 { 134 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 135 base.refcount); 136 struct v3d_bo *bo, *save; 137 138 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 139 drm_gem_object_put(&bo->base.base); 140 } 141 142 v3d_job_free(ref); 143 } 144 145 static void 146 v3d_cpu_job_free(struct kref *ref) 147 { 148 struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job, 149 base.refcount); 150 151 v3d_timestamp_query_info_free(&job->timestamp_query, 152 job->timestamp_query.count); 153 154 v3d_performance_query_info_free(&job->performance_query, 155 job->performance_query.count); 156 157 if (job->indirect_csd.indirect) 158 drm_gem_object_put(job->indirect_csd.indirect); 159 160 v3d_job_free(ref); 161 } 162 163 void v3d_job_cleanup(struct v3d_job *job) 164 { 165 if (!job) 166 return; 167 168 drm_sched_job_cleanup(&job->base); 169 v3d_job_put(job); 170 } 171 172 void v3d_job_put(struct v3d_job *job) 173 { 174 if (!job) 175 return; 176 177 kref_put(&job->refcount, job->free); 178 } 179 180 static int 181 v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv, 182 u32 in_sync, struct v3d_submit_ext *se) 183 { 184 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 185 struct v3d_dev *v3d = job->v3d; 186 int ret = 0; 187 188 if (!has_multisync) { 189 /* Ignore syncobj if its handle is zero */ 190 if (in_sync) 191 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 192 in_sync, 0); 193 return ret; 194 } 195 196 if (se->in_sync_count && se->wait_stage == job->queue) { 197 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 198 199 for (int i = 0; i < se->in_sync_count; i++) { 200 struct drm_v3d_sem in; 201 202 if (copy_from_user(&in, handle++, sizeof(in))) { 203 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); 204 return -EFAULT; 205 } 206 207 /* Ignore syncobj if its handle is zero */ 208 if (in.handle) { 209 ret = drm_sched_job_add_syncobj_dependency(&job->base, 210 file_priv, in.handle, 0); 211 if (ret) 212 return ret; 213 } 214 } 215 } 216 217 return 0; 218 } 219 220 static const struct { 221 size_t size; 222 void (*free)(struct kref *ref); 223 } v3d_job_types[] = { 224 [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free }, 225 [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free }, 226 [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free }, 227 [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free }, 228 [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free }, 229 [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free }, 230 }; 231 232 static struct v3d_job * 233 v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue) 234 { 235 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; 236 struct v3d_dev *v3d = submit->v3d; 237 struct v3d_job *job; 238 int ret; 239 240 if (queue >= V3D_MAX_QUEUES) 241 return ERR_PTR(-EINVAL); 242 243 job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL); 244 if (!job) 245 return ERR_PTR(-ENOMEM); 246 247 job->v3d = v3d; 248 job->queue = queue; 249 job->file_priv = v3d_priv; 250 job->free = v3d_job_types[queue].free; 251 252 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 253 1, v3d_priv, submit->file_priv->client_id); 254 if (ret) 255 goto fail_free; 256 257 /* CPU jobs don't require hardware resources */ 258 if (queue != V3D_CPU) { 259 ret = v3d_pm_runtime_get(v3d); 260 if (ret) 261 goto fail_sched_job; 262 job->has_pm_ref = true; 263 } 264 265 kref_init(&job->refcount); 266 267 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); 268 job->global_stats = v3d_stats_get(v3d->queue[queue].stats); 269 270 submit->jobs[submit->job_count++] = job; 271 272 return job; 273 274 fail_sched_job: 275 drm_sched_job_cleanup(&job->base); 276 fail_free: 277 kfree(job); 278 return ERR_PTR(ret); 279 } 280 281 static void 282 v3d_submit_put_jobs(struct v3d_submit *submit) 283 { 284 for (int i = 0; i < submit->job_count; i++) 285 v3d_job_put(submit->jobs[i]); 286 } 287 288 static void 289 v3d_submit_cleanup_jobs(struct v3d_submit *submit) 290 { 291 for (int i = 0; i < submit->job_count; i++) 292 v3d_job_cleanup(submit->jobs[i]); 293 } 294 295 static int 296 v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id) 297 { 298 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; 299 struct v3d_dev *v3d = submit->v3d; 300 struct v3d_perfmon *perfmon; 301 302 if (!perfmon_id) 303 return 0; 304 305 if (v3d->global_perfmon) 306 return -EAGAIN; 307 308 perfmon = v3d_perfmon_find(v3d_priv, perfmon_id); 309 if (!perfmon) 310 return -ENOENT; 311 312 for (int i = 0; i < submit->job_count; i++) { 313 submit->jobs[i]->perfmon = perfmon; 314 if (i != 0) 315 v3d_perfmon_get(perfmon); 316 } 317 318 return 0; 319 } 320 321 static void 322 v3d_submit_attach_object_fences(struct v3d_submit *submit) 323 { 324 struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; 325 326 /* The submission's last fence covers the entire submission. Attach it 327 * to every BO touched by any job in the submission. 328 */ 329 for (int i = 0; i < submit->job_count; i++) { 330 struct v3d_job *job = submit->jobs[i]; 331 332 for (int j = 0; j < job->bo_count; j++) { 333 /* XXX: Use shared fences for read-only objects. */ 334 dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence, 335 DMA_RESV_USAGE_WRITE); 336 } 337 } 338 } 339 340 static void 341 v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out, 342 struct v3d_submit_ext *se) 343 { 344 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 345 struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; 346 347 /* Make sure single syncobj and multisync are mutually exclusive */ 348 WARN_ON_ONCE(sync_out && has_multisync); 349 350 /* Update the return sync object for the job */ 351 /* If it only supports a single signal semaphore*/ 352 if (!has_multisync) { 353 if (sync_out) { 354 drm_syncobj_replace_fence(sync_out, last_job->done_fence); 355 drm_syncobj_put(sync_out); 356 } 357 return; 358 } 359 360 /* If multiple semaphores extension is supported */ 361 if (se->out_sync_count) { 362 for (int i = 0; i < se->out_sync_count; i++) { 363 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 364 last_job->done_fence); 365 drm_syncobj_put(se->out_syncs[i].syncobj); 366 } 367 kvfree(se->out_syncs); 368 } 369 } 370 371 static int 372 v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out, 373 struct v3d_submit_ext *se) 374 { 375 struct v3d_dev *v3d = submit->v3d; 376 int ret = 0; 377 378 mutex_lock(&v3d->sched_lock); 379 380 for (int i = 0; i < submit->job_count; i++) { 381 struct v3d_job *job = submit->jobs[i]; 382 383 drm_sched_job_arm(&job->base); 384 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 385 386 /* put by scheduler job completion */ 387 kref_get(&job->refcount); 388 } 389 390 for (int i = 1; i < submit->job_count; i++) { 391 ret = drm_sched_job_add_dependency(&submit->jobs[i]->base, 392 dma_fence_get(submit->jobs[i - 1]->done_fence)); 393 if (ret) 394 goto err; 395 } 396 397 for (int i = 0; i < submit->job_count; i++) 398 drm_sched_entity_push_job(&submit->jobs[i]->base); 399 400 mutex_unlock(&v3d->sched_lock); 401 402 v3d_submit_attach_object_fences(submit); 403 v3d_submit_unlock_reservations(submit); 404 v3d_submit_process_post_deps(submit, sync_out, se); 405 406 v3d_submit_put_jobs(submit); 407 408 return 0; 409 410 err: 411 /* Mark every armed job as failed so run_job() skips execution */ 412 for (int i = 0; i < submit->job_count; i++) 413 dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret); 414 415 for (int i = 0; i < submit->job_count; i++) 416 drm_sched_entity_push_job(&submit->jobs[i]->base); 417 418 mutex_unlock(&v3d->sched_lock); 419 420 v3d_submit_unlock_reservations(submit); 421 v3d_submit_put_jobs(submit); 422 423 return ret; 424 } 425 426 static int 427 v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit, 428 struct drm_v3d_submit_csd *args, 429 struct v3d_submit_ext *se) 430 { 431 struct v3d_csd_job *job; 432 struct v3d_job *clean_job; 433 int ret; 434 435 job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD); 436 if (IS_ERR(job)) 437 return PTR_ERR(job); 438 439 ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se); 440 if (ret) 441 return ret; 442 443 job->args = *args; 444 445 clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN); 446 if (IS_ERR(clean_job)) 447 return PTR_ERR(clean_job); 448 449 return v3d_lookup_bos(submit, args->bo_handles, args->bo_handle_count); 450 } 451 452 static void 453 v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se) 454 { 455 unsigned int i; 456 457 if (sync_out) 458 drm_syncobj_put(sync_out); 459 460 if (!(se && se->out_sync_count)) 461 return; 462 463 for (i = 0; i < se->out_sync_count; i++) 464 drm_syncobj_put(se->out_syncs[i].syncobj); 465 kvfree(se->out_syncs); 466 } 467 468 static int 469 v3d_get_multisync_post_deps(struct drm_file *file_priv, 470 struct v3d_submit_ext *se, 471 u32 count, u64 handles) 472 { 473 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 474 struct v3d_dev *v3d = v3d_priv->v3d; 475 struct drm_v3d_sem __user *post_deps; 476 int i, ret; 477 478 if (!count) 479 return 0; 480 481 se->out_syncs = (struct v3d_submit_outsync *) 482 kvmalloc_objs(struct v3d_submit_outsync, count); 483 if (!se->out_syncs) 484 return -ENOMEM; 485 486 post_deps = u64_to_user_ptr(handles); 487 488 for (i = 0; i < count; i++) { 489 struct drm_v3d_sem out; 490 491 if (copy_from_user(&out, post_deps++, sizeof(out))) { 492 ret = -EFAULT; 493 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n"); 494 goto fail; 495 } 496 497 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 498 out.handle); 499 if (!se->out_syncs[i].syncobj) { 500 ret = -EINVAL; 501 goto fail; 502 } 503 } 504 se->out_sync_count = count; 505 506 return 0; 507 508 fail: 509 for (i--; i >= 0; i--) 510 drm_syncobj_put(se->out_syncs[i].syncobj); 511 kvfree(se->out_syncs); 512 513 return ret; 514 } 515 516 /* Get data for multiple binary semaphores synchronization. Parse syncobj 517 * to be signaled when job completes (out_sync). 518 */ 519 static int 520 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 521 struct drm_v3d_extension __user *ext, 522 struct v3d_submit_ext *se) 523 { 524 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 525 struct v3d_dev *v3d = v3d_priv->v3d; 526 struct drm_v3d_multi_sync multisync; 527 int ret; 528 529 if (se->in_sync_count || se->out_sync_count) { 530 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job."); 531 return -EINVAL; 532 } 533 534 if (copy_from_user(&multisync, ext, sizeof(multisync))) 535 return -EFAULT; 536 537 if (multisync.pad) 538 return -EINVAL; 539 540 if (!multisync.in_sync_count && !multisync.out_sync_count) { 541 drm_dbg(&v3d->drm, "Empty multisync extension\n"); 542 return -EINVAL; 543 } 544 545 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 546 multisync.out_syncs); 547 if (ret) 548 return ret; 549 550 se->in_sync_count = multisync.in_sync_count; 551 se->in_syncs = multisync.in_syncs; 552 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 553 se->wait_stage = multisync.wait_stage; 554 555 return 0; 556 } 557 558 /* Returns false if the CPU job has an invalid configuration. */ 559 static bool 560 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job) 561 { 562 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 563 struct v3d_dev *v3d = v3d_priv->v3d; 564 565 if (!job) { 566 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n"); 567 return false; 568 } 569 570 if (job->job_type) { 571 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n"); 572 return false; 573 } 574 575 return true; 576 } 577 578 /* Get data for the indirect CSD job submission. */ 579 static int 580 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 581 struct drm_v3d_extension __user *ext, 582 struct v3d_cpu_job *job) 583 { 584 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 585 struct v3d_dev *v3d = v3d_priv->v3d; 586 struct drm_v3d_indirect_csd indirect_csd; 587 struct v3d_indirect_csd_info *info = &job->indirect_csd; 588 589 if (!v3d_validate_cpu_job(file_priv, job)) 590 return -EINVAL; 591 592 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 593 return -EFAULT; 594 595 if (!v3d_has_csd(v3d)) { 596 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n"); 597 return -EINVAL; 598 } 599 600 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 601 info->args = indirect_csd.submit; 602 info->offset = indirect_csd.offset; 603 info->wg_size = indirect_csd.wg_size; 604 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 605 sizeof(indirect_csd.wg_uniform_offsets)); 606 607 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 608 609 return 0; 610 } 611 612 /* Get data for the query timestamp job submission. */ 613 static int 614 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 615 struct drm_v3d_extension __user *ext, 616 struct v3d_cpu_job *job) 617 { 618 u32 __user *offsets, *syncs; 619 struct drm_v3d_timestamp_query timestamp; 620 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 621 unsigned int i; 622 int err; 623 624 if (!v3d_validate_cpu_job(file_priv, job)) 625 return -EINVAL; 626 627 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 628 return -EFAULT; 629 630 if (timestamp.pad) 631 return -EINVAL; 632 633 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 634 635 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 636 timestamp.count); 637 if (!query_info->queries) 638 return -ENOMEM; 639 640 offsets = u64_to_user_ptr(timestamp.offsets); 641 syncs = u64_to_user_ptr(timestamp.syncs); 642 643 for (i = 0; i < timestamp.count; i++) { 644 u32 offset, sync; 645 646 if (get_user(offset, offsets++)) { 647 err = -EFAULT; 648 goto error; 649 } 650 651 query_info->queries[i].offset = offset; 652 653 if (get_user(sync, syncs++)) { 654 err = -EFAULT; 655 goto error; 656 } 657 658 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 659 sync); 660 if (!query_info->queries[i].syncobj) { 661 err = -ENOENT; 662 goto error; 663 } 664 } 665 query_info->count = timestamp.count; 666 667 return 0; 668 669 error: 670 v3d_timestamp_query_info_free(&job->timestamp_query, i); 671 return err; 672 } 673 674 static int 675 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 676 struct drm_v3d_extension __user *ext, 677 struct v3d_cpu_job *job) 678 { 679 u32 __user *syncs; 680 struct drm_v3d_reset_timestamp_query reset; 681 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 682 unsigned int i; 683 int err; 684 685 if (!v3d_validate_cpu_job(file_priv, job)) 686 return -EINVAL; 687 688 if (copy_from_user(&reset, ext, sizeof(reset))) 689 return -EFAULT; 690 691 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 692 693 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 694 reset.count); 695 if (!query_info->queries) 696 return -ENOMEM; 697 698 syncs = u64_to_user_ptr(reset.syncs); 699 700 for (i = 0; i < reset.count; i++) { 701 u32 sync; 702 703 query_info->queries[i].offset = reset.offset + 8 * i; 704 705 if (get_user(sync, syncs++)) { 706 err = -EFAULT; 707 goto error; 708 } 709 710 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 711 sync); 712 if (!query_info->queries[i].syncobj) { 713 err = -ENOENT; 714 goto error; 715 } 716 } 717 query_info->count = reset.count; 718 719 return 0; 720 721 error: 722 v3d_timestamp_query_info_free(&job->timestamp_query, i); 723 return err; 724 } 725 726 /* Get data for the copy timestamp query results job submission. */ 727 static int 728 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 729 struct drm_v3d_extension __user *ext, 730 struct v3d_cpu_job *job) 731 { 732 u32 __user *offsets, *syncs; 733 struct drm_v3d_copy_timestamp_query copy; 734 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 735 unsigned int i; 736 int err; 737 738 if (!v3d_validate_cpu_job(file_priv, job)) 739 return -EINVAL; 740 741 if (copy_from_user(©, ext, sizeof(copy))) 742 return -EFAULT; 743 744 if (copy.pad) 745 return -EINVAL; 746 747 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 748 749 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 750 copy.count); 751 if (!query_info->queries) 752 return -ENOMEM; 753 754 offsets = u64_to_user_ptr(copy.offsets); 755 syncs = u64_to_user_ptr(copy.syncs); 756 757 for (i = 0; i < copy.count; i++) { 758 u32 offset, sync; 759 760 if (get_user(offset, offsets++)) { 761 err = -EFAULT; 762 goto error; 763 } 764 765 query_info->queries[i].offset = offset; 766 767 if (get_user(sync, syncs++)) { 768 err = -EFAULT; 769 goto error; 770 } 771 772 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 773 sync); 774 if (!query_info->queries[i].syncobj) { 775 err = -ENOENT; 776 goto error; 777 } 778 } 779 query_info->count = copy.count; 780 781 job->copy.do_64bit = copy.do_64bit; 782 job->copy.do_partial = copy.do_partial; 783 job->copy.availability_bit = copy.availability_bit; 784 job->copy.offset = copy.offset; 785 job->copy.stride = copy.stride; 786 787 return 0; 788 789 error: 790 v3d_timestamp_query_info_free(&job->timestamp_query, i); 791 return err; 792 } 793 794 static int 795 v3d_copy_query_info(struct v3d_performance_query_info *query_info, 796 unsigned int count, 797 unsigned int nperfmons, 798 u32 __user *syncs, 799 u64 __user *kperfmon_ids, 800 struct drm_file *file_priv) 801 { 802 unsigned int i, j; 803 int err; 804 805 for (i = 0; i < count; i++) { 806 struct v3d_performance_query *query = &query_info->queries[i]; 807 u32 __user *ids_pointer; 808 u32 sync, id; 809 u64 ids; 810 811 if (get_user(sync, syncs++)) { 812 err = -EFAULT; 813 goto error; 814 } 815 816 if (get_user(ids, kperfmon_ids++)) { 817 err = -EFAULT; 818 goto error; 819 } 820 821 query->kperfmon_ids = 822 kvmalloc_array(nperfmons, 823 sizeof(struct v3d_performance_query *), 824 GFP_KERNEL); 825 if (!query->kperfmon_ids) { 826 err = -ENOMEM; 827 goto error; 828 } 829 830 ids_pointer = u64_to_user_ptr(ids); 831 832 for (j = 0; j < nperfmons; j++) { 833 if (get_user(id, ids_pointer++)) { 834 kvfree(query->kperfmon_ids); 835 err = -EFAULT; 836 goto error; 837 } 838 839 query->kperfmon_ids[j] = id; 840 } 841 842 query->syncobj = drm_syncobj_find(file_priv, sync); 843 if (!query->syncobj) { 844 kvfree(query->kperfmon_ids); 845 err = -ENOENT; 846 goto error; 847 } 848 } 849 850 return 0; 851 852 error: 853 v3d_performance_query_info_free(query_info, i); 854 return err; 855 } 856 857 static int 858 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 859 struct drm_v3d_extension __user *ext, 860 struct v3d_cpu_job *job) 861 { 862 struct v3d_performance_query_info *query_info = &job->performance_query; 863 struct drm_v3d_reset_performance_query reset; 864 int err; 865 866 if (!v3d_validate_cpu_job(file_priv, job)) 867 return -EINVAL; 868 869 if (copy_from_user(&reset, ext, sizeof(reset))) 870 return -EFAULT; 871 872 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 873 874 query_info->queries = 875 kvmalloc_objs(struct v3d_performance_query, reset.count); 876 if (!query_info->queries) 877 return -ENOMEM; 878 879 err = v3d_copy_query_info(query_info, 880 reset.count, 881 reset.nperfmons, 882 u64_to_user_ptr(reset.syncs), 883 u64_to_user_ptr(reset.kperfmon_ids), 884 file_priv); 885 if (err) 886 return err; 887 888 query_info->count = reset.count; 889 query_info->nperfmons = reset.nperfmons; 890 891 return 0; 892 } 893 894 static int 895 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 896 struct drm_v3d_extension __user *ext, 897 struct v3d_cpu_job *job) 898 { 899 struct v3d_performance_query_info *query_info = &job->performance_query; 900 struct drm_v3d_copy_performance_query copy; 901 int err; 902 903 if (!v3d_validate_cpu_job(file_priv, job)) 904 return -EINVAL; 905 906 if (copy_from_user(©, ext, sizeof(copy))) 907 return -EFAULT; 908 909 if (copy.pad) 910 return -EINVAL; 911 912 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 913 914 query_info->queries = 915 kvmalloc_objs(struct v3d_performance_query, copy.count); 916 if (!query_info->queries) 917 return -ENOMEM; 918 919 err = v3d_copy_query_info(query_info, 920 copy.count, 921 copy.nperfmons, 922 u64_to_user_ptr(copy.syncs), 923 u64_to_user_ptr(copy.kperfmon_ids), 924 file_priv); 925 if (err) 926 return err; 927 928 query_info->count = copy.count; 929 query_info->nperfmons = copy.nperfmons; 930 query_info->ncounters = copy.ncounters; 931 932 job->copy.do_64bit = copy.do_64bit; 933 job->copy.do_partial = copy.do_partial; 934 job->copy.availability_bit = copy.availability_bit; 935 job->copy.offset = copy.offset; 936 job->copy.stride = copy.stride; 937 938 return 0; 939 } 940 941 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 942 * according to the extension id (name). 943 */ 944 static int 945 v3d_get_extensions(struct drm_file *file_priv, 946 u64 ext_handles, 947 struct v3d_submit_ext *se, 948 struct v3d_cpu_job *job) 949 { 950 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 951 struct v3d_dev *v3d = v3d_priv->v3d; 952 struct drm_v3d_extension __user *user_ext; 953 int ret; 954 955 user_ext = u64_to_user_ptr(ext_handles); 956 while (user_ext) { 957 struct drm_v3d_extension ext; 958 959 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 960 drm_dbg(&v3d->drm, "Failed to copy submit extension\n"); 961 return -EFAULT; 962 } 963 964 switch (ext.id) { 965 case DRM_V3D_EXT_ID_MULTI_SYNC: 966 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 967 break; 968 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 969 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 970 break; 971 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 972 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 973 break; 974 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 975 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 976 break; 977 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 978 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 979 break; 980 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 981 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 982 break; 983 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 984 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 985 break; 986 default: 987 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id); 988 return -EINVAL; 989 } 990 991 if (ret) 992 return ret; 993 994 user_ext = u64_to_user_ptr(ext.next); 995 } 996 997 return 0; 998 } 999 1000 /** 1001 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 1002 * @dev: DRM device 1003 * @data: ioctl argument 1004 * @file_priv: DRM file for this fd 1005 * 1006 * This is the main entrypoint for userspace to submit a 3D frame to 1007 * the GPU. Userspace provides the binner command list (if 1008 * applicable), and the kernel sets up the render command list to draw 1009 * to the framebuffer described in the ioctl, using the command lists 1010 * that the 3D engine's binner will produce. 1011 */ 1012 int 1013 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 1014 struct drm_file *file_priv) 1015 { 1016 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1017 struct drm_v3d_submit_cl *args = data; 1018 struct drm_syncobj *sync_out = NULL; 1019 struct v3d_submit_ext se = {0}; 1020 struct v3d_bin_job *bin = NULL; 1021 struct v3d_render_job *render; 1022 struct v3d_job *clean_job; 1023 int ret; 1024 1025 trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end); 1026 1027 if (args->pad) 1028 return -EINVAL; 1029 1030 if (args->flags && 1031 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 1032 DRM_V3D_SUBMIT_EXTENSION)) { 1033 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1034 return -EINVAL; 1035 } 1036 1037 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1038 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1039 if (ret) { 1040 drm_dbg(dev, "Failed to get extensions.\n"); 1041 return ret; 1042 } 1043 } 1044 1045 /* If multisync is configured, give priority to it and ignore out_sync. */ 1046 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1047 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1048 if (!sync_out) 1049 return -ENOENT; 1050 } 1051 1052 if (args->bcl_start != args->bcl_end) { 1053 bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN); 1054 if (IS_ERR(bin)) { 1055 ret = PTR_ERR(bin); 1056 goto fail; 1057 } 1058 1059 bin->start = args->bcl_start; 1060 bin->end = args->bcl_end; 1061 bin->qma = args->qma; 1062 bin->qms = args->qms; 1063 bin->qts = args->qts; 1064 1065 ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl, 1066 &se); 1067 if (ret) 1068 goto fail; 1069 } 1070 1071 render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER); 1072 if (IS_ERR(render)) { 1073 ret = PTR_ERR(render); 1074 goto fail; 1075 } 1076 1077 INIT_LIST_HEAD(&render->unref_list); 1078 render->start = args->rcl_start; 1079 render->end = args->rcl_end; 1080 1081 if (bin) 1082 bin->render = render; 1083 1084 ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se); 1085 if (ret) 1086 goto fail; 1087 1088 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 1089 clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN); 1090 if (IS_ERR(clean_job)) { 1091 ret = PTR_ERR(clean_job); 1092 goto fail; 1093 } 1094 } 1095 1096 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); 1097 if (ret) 1098 goto fail; 1099 1100 ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count); 1101 if (ret) 1102 goto fail; 1103 1104 ret = v3d_submit_lock_reservations(&submit); 1105 if (ret) 1106 goto fail; 1107 1108 ret = v3d_submit_jobs(&submit, sync_out, &se); 1109 if (ret) 1110 goto fail_submit; 1111 1112 return 0; 1113 1114 fail: 1115 v3d_submit_cleanup_jobs(&submit); 1116 fail_submit: 1117 v3d_submit_put_post_deps(sync_out, &se); 1118 1119 return ret; 1120 } 1121 1122 /** 1123 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1124 * @dev: DRM device 1125 * @data: ioctl argument 1126 * @file_priv: DRM file for this fd 1127 * 1128 * Userspace provides the register setup for the TFU, which we don't 1129 * need to validate since the TFU is behind the MMU. 1130 */ 1131 int 1132 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1133 struct drm_file *file_priv) 1134 { 1135 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1136 struct drm_v3d_submit_tfu *args = data; 1137 struct drm_syncobj *sync_out = NULL; 1138 struct v3d_submit_ext se = {0}; 1139 struct v3d_tfu_job *job; 1140 int ret = 0; 1141 1142 trace_v3d_submit_tfu_ioctl(dev, args->iia); 1143 1144 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1145 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1146 return -EINVAL; 1147 } 1148 1149 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1150 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1151 if (ret) { 1152 drm_dbg(dev, "Failed to get extensions.\n"); 1153 return ret; 1154 } 1155 } 1156 1157 /* If multisync is configured, give priority to it and ignore out_sync. */ 1158 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1159 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1160 if (!sync_out) 1161 return -ENOENT; 1162 } 1163 1164 job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU); 1165 if (IS_ERR(job)) { 1166 ret = PTR_ERR(job); 1167 goto fail; 1168 } 1169 1170 ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se); 1171 if (ret) 1172 goto fail; 1173 1174 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); 1175 if (!job->base.bo) { 1176 ret = -ENOMEM; 1177 goto fail; 1178 } 1179 1180 job->args = *args; 1181 1182 for (job->base.bo_count = 0; 1183 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1184 job->base.bo_count++) { 1185 struct drm_gem_object *bo; 1186 1187 if (!args->bo_handles[job->base.bo_count]) 1188 break; 1189 1190 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1191 if (!bo) { 1192 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n", 1193 job->base.bo_count, 1194 args->bo_handles[job->base.bo_count]); 1195 ret = -ENOENT; 1196 goto fail; 1197 } 1198 job->base.bo[job->base.bo_count] = bo; 1199 } 1200 1201 ret = v3d_submit_lock_reservations(&submit); 1202 if (ret) 1203 goto fail; 1204 1205 ret = v3d_submit_jobs(&submit, sync_out, &se); 1206 if (ret) 1207 goto fail_submit; 1208 1209 return 0; 1210 1211 fail: 1212 v3d_submit_cleanup_jobs(&submit); 1213 fail_submit: 1214 v3d_submit_put_post_deps(sync_out, &se); 1215 1216 return ret; 1217 } 1218 1219 /** 1220 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1221 * @dev: DRM device 1222 * @data: ioctl argument 1223 * @file_priv: DRM file for this fd 1224 * 1225 * Userspace provides the register setup for the CSD, which we don't 1226 * need to validate since the CSD is behind the MMU. 1227 */ 1228 int 1229 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1230 struct drm_file *file_priv) 1231 { 1232 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1233 struct drm_v3d_submit_csd *args = data; 1234 struct drm_syncobj *sync_out = NULL; 1235 struct v3d_submit_ext se = {0}; 1236 int ret; 1237 1238 trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]); 1239 1240 if (args->pad) 1241 return -EINVAL; 1242 1243 if (!v3d_has_csd(submit.v3d)) { 1244 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); 1245 return -EINVAL; 1246 } 1247 1248 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1249 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1250 return -EINVAL; 1251 } 1252 1253 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1254 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1255 if (ret) { 1256 drm_dbg(dev, "Failed to get extensions.\n"); 1257 return ret; 1258 } 1259 } 1260 1261 /* If multisync is configured, give priority to it and ignore out_sync. */ 1262 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1263 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1264 if (!sync_out) 1265 return -ENOENT; 1266 } 1267 1268 ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se); 1269 if (ret) 1270 goto fail; 1271 1272 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); 1273 if (ret) 1274 goto fail; 1275 1276 ret = v3d_submit_lock_reservations(&submit); 1277 if (ret) 1278 goto fail; 1279 1280 ret = v3d_submit_jobs(&submit, sync_out, &se); 1281 if (ret) 1282 goto fail_submit; 1283 1284 return 0; 1285 1286 fail: 1287 v3d_submit_cleanup_jobs(&submit); 1288 fail_submit: 1289 v3d_submit_put_post_deps(sync_out, &se); 1290 1291 return ret; 1292 } 1293 1294 static const unsigned int cpu_job_bo_handle_count[] = { 1295 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1296 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1297 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1298 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1299 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1300 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1301 }; 1302 1303 /** 1304 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1305 * @dev: DRM device 1306 * @data: ioctl argument 1307 * @file_priv: DRM file for this fd 1308 * 1309 * Userspace specifies the CPU job type and data required to perform its 1310 * operations through the drm_v3d_extension struct. 1311 */ 1312 int 1313 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1314 struct drm_file *file_priv) 1315 { 1316 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1317 struct drm_v3d_submit_cpu *args = data; 1318 struct v3d_submit_ext se = {0}; 1319 struct v3d_cpu_job *cpu_job = NULL; 1320 int ret; 1321 1322 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1323 drm_dbg(dev, "Invalid flags: %d\n", args->flags); 1324 return -EINVAL; 1325 } 1326 1327 cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU); 1328 if (IS_ERR(cpu_job)) 1329 return PTR_ERR(cpu_job); 1330 1331 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1332 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1333 if (ret) { 1334 drm_dbg(dev, "Failed to get extensions.\n"); 1335 goto fail; 1336 } 1337 } 1338 1339 /* Every CPU job must have a CPU job user extension */ 1340 if (!cpu_job->job_type) { 1341 drm_dbg(dev, "CPU job must have a CPU job user extension.\n"); 1342 ret = -EINVAL; 1343 goto fail; 1344 } 1345 1346 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1347 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n"); 1348 ret = -EINVAL; 1349 goto fail; 1350 } 1351 1352 trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type); 1353 1354 ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se); 1355 if (ret) 1356 goto fail; 1357 1358 /* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends 1359 * the CSD and clean jobs in the case of indirect CSD job. 1360 */ 1361 if (args->bo_handle_count) { 1362 ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count); 1363 if (ret) 1364 goto fail; 1365 } 1366 1367 if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) { 1368 ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args, 1369 NULL); 1370 if (ret) 1371 goto fail; 1372 1373 /* The CSD job was appended at jobs[1] */ 1374 if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) { 1375 ret = -EINVAL; 1376 goto fail; 1377 } 1378 1379 cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job, 1380 base); 1381 } 1382 1383 ret = v3d_submit_lock_reservations(&submit); 1384 if (ret) 1385 goto fail; 1386 1387 ret = v3d_submit_jobs(&submit, NULL, &se); 1388 if (ret) 1389 goto fail_submit; 1390 1391 return 0; 1392 1393 fail: 1394 v3d_submit_cleanup_jobs(&submit); 1395 fail_submit: 1396 v3d_submit_put_post_deps(NULL, &se); 1397 1398 return ret; 1399 } 1400