1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <linux/dma-fence-unwrap.h> 8 #include <linux/overflow.h> 9 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 13 #include "v3d_drv.h" 14 #include "v3d_regs.h" 15 #include "v3d_trace.h" 16 17 /* Takes the reservation lock on all the BOs being referenced, so that 18 * we can attach fences and update the reservations after pushing the job 19 * to the queue. 20 * 21 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 22 * (all of which are on render->unref_list). They're entirely private 23 * to v3d, so we don't attach dma-buf fences to them. 24 */ 25 static int 26 v3d_submit_lock_reservations(struct v3d_submit *submit) 27 { 28 int i, j, ret; 29 30 drm_exec_init(&submit->exec, 31 DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); 32 drm_exec_until_all_locked(&submit->exec) { 33 for (i = 0; i < submit->job_count; i++) { 34 struct v3d_job *job = submit->jobs[i]; 35 36 ret = drm_exec_prepare_array(&submit->exec, job->bo, 37 job->bo_count, 1); 38 if (ret) 39 break; 40 } 41 drm_exec_retry_on_contention(&submit->exec); 42 if (ret) 43 goto fail; 44 } 45 46 for (i = 0; i < submit->job_count; i++) { 47 struct v3d_job *job = submit->jobs[i]; 48 49 if (!job->has_implicit_dep) 50 continue; 51 52 for (j = 0; j < job->bo_count; j++) { 53 ret = drm_sched_job_add_implicit_dependencies(&job->base, 54 job->bo[j], 55 true); 56 if (ret) 57 goto fail; 58 } 59 } 60 61 return 0; 62 63 fail: 64 drm_exec_fini(&submit->exec); 65 return ret; 66 } 67 68 static void 69 v3d_submit_unlock_reservations(struct v3d_submit *submit) 70 { 71 drm_exec_fini(&submit->exec); 72 } 73 74 /** 75 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 76 * referenced by the job. 77 * @file_priv: DRM file for this fd 78 * @job: V3D job being set up 79 * @bo_handles: GEM handles 80 * @bo_count: Number of GEM handles passed in 81 * 82 * The command validator needs to reference BOs by their index within 83 * the submitted job's BO list. This does the validation of the job's 84 * BO list and reference counting for the lifetime of the job. 85 * 86 * Note that this function doesn't need to unreference the BOs on 87 * failure, because that will happen at `v3d_job_free()`. 88 */ 89 static int 90 v3d_lookup_bos(struct drm_file *file_priv, struct v3d_job *job, 91 u64 bo_handles, u32 bo_count) 92 { 93 if (!bo_count) { 94 drm_warn(&job->v3d->drm, "Rendering requires BOs\n"); 95 return -EINVAL; 96 } 97 98 job->bo_count = bo_count; 99 100 return drm_gem_objects_lookup(file_priv, 101 (void __user *)(uintptr_t)bo_handles, 102 job->bo_count, &job->bo); 103 } 104 105 /** 106 * v3d_job_reference_bos() - Share another job's BOs with @dst 107 * @dst: job that acquires references to the BOs 108 * @src: job whose already-resolved BO list is shared 109 * 110 * For submissions with multiple jobs that use the same BOs, a trailing job 111 * shouldn't look the handles up again, as it could cause inconsistencies. 112 * Instead, it should reference the previous job's BOs. 113 */ 114 static int 115 v3d_job_reference_bos(struct v3d_job *dst, struct v3d_job *src) 116 { 117 dst->bo = kvmalloc_objs(*dst->bo, src->bo_count); 118 if (!dst->bo) 119 return -ENOMEM; 120 121 dst->bo_count = src->bo_count; 122 for (int i = 0; i < dst->bo_count; i++) { 123 dst->bo[i] = src->bo[i]; 124 drm_gem_object_get(dst->bo[i]); 125 } 126 127 return 0; 128 } 129 130 static void 131 v3d_job_free(struct kref *ref) 132 { 133 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 134 int i; 135 136 if (job->bo) { 137 for (i = 0; i < job->bo_count; i++) 138 drm_gem_object_put(job->bo[i]); 139 kvfree(job->bo); 140 } 141 142 dma_fence_put(job->irq_fence); 143 dma_fence_put(job->done_fence); 144 145 if (job->perfmon) 146 v3d_perfmon_put(job->perfmon); 147 148 v3d_stats_put(job->client_stats); 149 v3d_stats_put(job->global_stats); 150 151 if (job->has_pm_ref) 152 v3d_pm_runtime_put(job->v3d); 153 154 kfree(job); 155 } 156 157 static void 158 v3d_render_job_free(struct kref *ref) 159 { 160 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 161 base.refcount); 162 struct v3d_bo *bo, *save; 163 164 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 165 drm_gem_object_put(&bo->base.base); 166 } 167 168 v3d_job_free(ref); 169 } 170 171 static void 172 v3d_cpu_job_free(struct kref *ref) 173 { 174 struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job, 175 base.refcount); 176 177 v3d_timestamp_query_info_free(&job->timestamp_query, 178 job->timestamp_query.count); 179 180 v3d_performance_query_info_free(&job->performance_query, 181 job->performance_query.count); 182 183 if (job->indirect_csd.indirect) 184 drm_gem_object_put(job->indirect_csd.indirect); 185 186 v3d_job_free(ref); 187 } 188 189 void v3d_job_cleanup(struct v3d_job *job) 190 { 191 if (!job) 192 return; 193 194 drm_sched_job_cleanup(&job->base); 195 v3d_job_put(job); 196 } 197 198 void v3d_job_put(struct v3d_job *job) 199 { 200 if (!job) 201 return; 202 203 kref_put(&job->refcount, job->free); 204 } 205 206 static int 207 v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv, 208 u32 in_sync, struct v3d_submit_ext *se) 209 { 210 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 211 struct v3d_dev *v3d = job->v3d; 212 int ret = 0; 213 214 if (!has_multisync) { 215 /* Ignore syncobj if its handle is zero */ 216 if (in_sync) 217 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 218 in_sync, 0); 219 return ret; 220 } 221 222 if (se->in_sync_count && se->wait_stage == job->queue) { 223 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 224 225 for (int i = 0; i < se->in_sync_count; i++) { 226 struct drm_v3d_sem in; 227 228 if (copy_from_user(&in, handle++, sizeof(in))) { 229 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); 230 return -EFAULT; 231 } 232 233 /* Ignore syncobj if its handle is zero */ 234 if (in.handle) { 235 ret = drm_sched_job_add_syncobj_dependency(&job->base, 236 file_priv, in.handle, 0); 237 if (ret) 238 return ret; 239 } 240 } 241 } 242 243 return 0; 244 } 245 246 static const struct { 247 size_t size; 248 void (*free)(struct kref *ref); 249 bool has_implicit_dep; 250 } v3d_job_types[] = { 251 [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free, false }, 252 [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free, true }, 253 [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free, true }, 254 [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free, true }, 255 [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free, false }, 256 [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free, true }, 257 }; 258 259 static struct v3d_job * 260 v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue) 261 { 262 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; 263 struct v3d_dev *v3d = submit->v3d; 264 struct v3d_job *job; 265 int ret; 266 267 if (queue >= V3D_MAX_QUEUES) 268 return ERR_PTR(-EINVAL); 269 270 job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL); 271 if (!job) 272 return ERR_PTR(-ENOMEM); 273 274 job->v3d = v3d; 275 job->queue = queue; 276 job->file_priv = v3d_priv; 277 job->free = v3d_job_types[queue].free; 278 job->has_implicit_dep = v3d_job_types[queue].has_implicit_dep; 279 280 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 281 1, v3d_priv, submit->file_priv->client_id); 282 if (ret) 283 goto fail_free; 284 285 /* CPU jobs don't require hardware resources */ 286 if (queue != V3D_CPU) { 287 ret = v3d_pm_runtime_get(v3d); 288 if (ret) 289 goto fail_sched_job; 290 job->has_pm_ref = true; 291 } 292 293 kref_init(&job->refcount); 294 295 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); 296 job->global_stats = v3d_stats_get(v3d->queue[queue].stats); 297 298 submit->jobs[submit->job_count++] = job; 299 300 return job; 301 302 fail_sched_job: 303 drm_sched_job_cleanup(&job->base); 304 fail_free: 305 kfree(job); 306 return ERR_PTR(ret); 307 } 308 309 static void 310 v3d_submit_put_jobs(struct v3d_submit *submit) 311 { 312 for (int i = 0; i < submit->job_count; i++) 313 v3d_job_put(submit->jobs[i]); 314 } 315 316 static void 317 v3d_submit_cleanup_jobs(struct v3d_submit *submit) 318 { 319 for (int i = 0; i < submit->job_count; i++) 320 v3d_job_cleanup(submit->jobs[i]); 321 } 322 323 static int 324 v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id) 325 { 326 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; 327 struct v3d_dev *v3d = submit->v3d; 328 struct v3d_perfmon *perfmon; 329 330 if (!perfmon_id) 331 return 0; 332 333 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 334 if (v3d->global_perfmon) 335 return -EAGAIN; 336 } 337 338 perfmon = v3d_perfmon_find(v3d_priv, perfmon_id); 339 if (!perfmon) 340 return -ENOENT; 341 342 for (int i = 0; i < submit->job_count; i++) { 343 submit->jobs[i]->perfmon = perfmon; 344 if (i != 0) 345 v3d_perfmon_get(perfmon); 346 } 347 348 return 0; 349 } 350 351 /* 352 * Prepare fences to enforce job serialization when a perfmon is active. A job 353 * that carries a non-global perfmon must wait for every job currently in-flight 354 * across all HW queues to finish, otherwise concurrent unrelated work on the 355 * same core would pollute the performance counters. Symmetrically, while such a 356 * job is still in-flight, all subsequently submitted jobs must wait for it. 357 * 358 * We don't serialize the jobs when using a global perfmon as it's expected to 359 * track concurrent activity from all jobs. 360 * 361 * Keeping track of the in-flight jobs costs a fence merge per job, so it is 362 * only done while at least one perfmon is alive, and is skipped for the jobs 363 * carrying no perfmon while a global perfmon is set. Jobs submitted while 364 * tracking is off go untracked and may overlap the first measured job. 365 */ 366 static int 367 v3d_serialize_for_perfmon(struct v3d_job *job) 368 { 369 struct v3d_dev *v3d = job->v3d; 370 struct dma_fence *merged; 371 int ret; 372 373 lockdep_assert_held(&v3d->sched_lock); 374 375 if (!atomic_read(&v3d->perfmon_state.nperfmons)) 376 return 0; 377 378 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 379 if (!job->perfmon && v3d->global_perfmon) 380 return 0; 381 } 382 383 if (job->perfmon) { 384 for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) { 385 struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q]; 386 387 if (!f || dma_fence_is_signaled(f)) 388 continue; 389 390 ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f)); 391 if (ret) 392 return ret; 393 } 394 } else if (v3d->perfmon_state.fence && 395 !dma_fence_is_signaled(v3d->perfmon_state.fence)) { 396 ret = drm_sched_job_add_dependency(&job->base, 397 dma_fence_get(v3d->perfmon_state.fence)); 398 if (ret) 399 return ret; 400 } 401 402 /* 403 * Accumulate every in-flight job on this queue into one merged fence. 404 * A HW queue is fed by several scheduler entities (one per-fd), so jobs 405 * on it can complete out of order. 406 */ 407 merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue], 408 job->done_fence); 409 if (!merged) 410 return -ENOMEM; 411 412 dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]); 413 v3d->perfmon_state.last_hw_fence[job->queue] = merged; 414 415 if (job->perfmon) { 416 dma_fence_put(v3d->perfmon_state.fence); 417 v3d->perfmon_state.fence = dma_fence_get(job->done_fence); 418 } 419 420 return 0; 421 } 422 423 static void 424 v3d_submit_attach_object_fences(struct v3d_submit *submit) 425 { 426 struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; 427 428 /* The submission's last fence covers the entire submission. Attach it 429 * to every BO touched by any job in the submission. 430 */ 431 for (int i = 0; i < submit->job_count; i++) { 432 struct v3d_job *job = submit->jobs[i]; 433 434 for (int j = 0; j < job->bo_count; j++) { 435 /* XXX: Use shared fences for read-only objects. */ 436 dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence, 437 DMA_RESV_USAGE_WRITE); 438 } 439 } 440 } 441 442 static void 443 v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out, 444 struct v3d_submit_ext *se) 445 { 446 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 447 struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; 448 449 /* Make sure single syncobj and multisync are mutually exclusive */ 450 WARN_ON_ONCE(sync_out && has_multisync); 451 452 /* Update the return sync object for the job */ 453 /* If it only supports a single signal semaphore*/ 454 if (!has_multisync) { 455 if (sync_out) { 456 drm_syncobj_replace_fence(sync_out, last_job->done_fence); 457 drm_syncobj_put(sync_out); 458 } 459 return; 460 } 461 462 /* If multiple semaphores extension is supported */ 463 if (se->out_sync_count) { 464 for (int i = 0; i < se->out_sync_count; i++) { 465 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 466 last_job->done_fence); 467 drm_syncobj_put(se->out_syncs[i].syncobj); 468 } 469 kvfree(se->out_syncs); 470 } 471 } 472 473 static int 474 v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out, 475 struct v3d_submit_ext *se) 476 { 477 struct v3d_dev *v3d = submit->v3d; 478 int ret = 0; 479 480 mutex_lock(&v3d->sched_lock); 481 482 for (int i = 0; i < submit->job_count; i++) { 483 struct v3d_job *job = submit->jobs[i]; 484 485 drm_sched_job_arm(&job->base); 486 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 487 488 /* put by scheduler job completion */ 489 kref_get(&job->refcount); 490 } 491 492 for (int i = 1; i < submit->job_count; i++) { 493 ret = drm_sched_job_add_dependency(&submit->jobs[i]->base, 494 dma_fence_get(submit->jobs[i - 1]->done_fence)); 495 if (ret) 496 goto err; 497 } 498 499 for (int i = 0; i < submit->job_count; i++) { 500 ret = v3d_serialize_for_perfmon(submit->jobs[i]); 501 if (ret) 502 goto err; 503 } 504 505 for (int i = 0; i < submit->job_count; i++) 506 drm_sched_entity_push_job(&submit->jobs[i]->base); 507 508 mutex_unlock(&v3d->sched_lock); 509 510 v3d_submit_attach_object_fences(submit); 511 v3d_submit_unlock_reservations(submit); 512 v3d_submit_process_post_deps(submit, sync_out, se); 513 514 v3d_submit_put_jobs(submit); 515 516 return 0; 517 518 err: 519 /* Mark every armed job as failed so run_job() skips execution */ 520 for (int i = 0; i < submit->job_count; i++) 521 dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret); 522 523 for (int i = 0; i < submit->job_count; i++) 524 drm_sched_entity_push_job(&submit->jobs[i]->base); 525 526 mutex_unlock(&v3d->sched_lock); 527 528 v3d_submit_unlock_reservations(submit); 529 v3d_submit_put_jobs(submit); 530 531 return ret; 532 } 533 534 static int 535 v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit, 536 struct drm_v3d_submit_csd *args, 537 struct v3d_submit_ext *se) 538 { 539 struct v3d_csd_job *job; 540 struct v3d_job *clean_job; 541 int ret; 542 543 job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD); 544 if (IS_ERR(job)) 545 return PTR_ERR(job); 546 547 ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se); 548 if (ret) 549 return ret; 550 551 ret = v3d_lookup_bos(submit->file_priv, &job->base, args->bo_handles, 552 args->bo_handle_count); 553 if (ret) 554 return ret; 555 556 job->args = *args; 557 558 clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN); 559 if (IS_ERR(clean_job)) 560 return PTR_ERR(clean_job); 561 562 return v3d_job_reference_bos(clean_job, &job->base); 563 } 564 565 static void 566 v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se) 567 { 568 unsigned int i; 569 570 if (sync_out) 571 drm_syncobj_put(sync_out); 572 573 if (!(se && se->out_sync_count)) 574 return; 575 576 for (i = 0; i < se->out_sync_count; i++) 577 drm_syncobj_put(se->out_syncs[i].syncobj); 578 kvfree(se->out_syncs); 579 } 580 581 static int 582 v3d_get_multisync_post_deps(struct drm_file *file_priv, 583 struct v3d_submit_ext *se, 584 u32 count, u64 handles) 585 { 586 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 587 struct v3d_dev *v3d = v3d_priv->v3d; 588 struct drm_v3d_sem __user *post_deps; 589 int i, ret; 590 591 if (!count) 592 return 0; 593 594 se->out_syncs = (struct v3d_submit_outsync *) 595 kvmalloc_objs(struct v3d_submit_outsync, count); 596 if (!se->out_syncs) 597 return -ENOMEM; 598 599 post_deps = u64_to_user_ptr(handles); 600 601 for (i = 0; i < count; i++) { 602 struct drm_v3d_sem out; 603 604 if (copy_from_user(&out, post_deps++, sizeof(out))) { 605 ret = -EFAULT; 606 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n"); 607 goto fail; 608 } 609 610 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 611 out.handle); 612 if (!se->out_syncs[i].syncobj) { 613 ret = -EINVAL; 614 goto fail; 615 } 616 } 617 se->out_sync_count = count; 618 619 return 0; 620 621 fail: 622 for (i--; i >= 0; i--) 623 drm_syncobj_put(se->out_syncs[i].syncobj); 624 kvfree(se->out_syncs); 625 626 return ret; 627 } 628 629 /* Get data for multiple binary semaphores synchronization. Parse syncobj 630 * to be signaled when job completes (out_sync). 631 */ 632 static int 633 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 634 struct drm_v3d_extension __user *ext, 635 struct v3d_submit_ext *se) 636 { 637 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 638 struct v3d_dev *v3d = v3d_priv->v3d; 639 struct drm_v3d_multi_sync multisync; 640 int ret; 641 642 if (se->in_sync_count || se->out_sync_count) { 643 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job."); 644 return -EINVAL; 645 } 646 647 if (copy_from_user(&multisync, ext, sizeof(multisync))) 648 return -EFAULT; 649 650 if (multisync.pad) 651 return -EINVAL; 652 653 if (!multisync.in_sync_count && !multisync.out_sync_count) { 654 drm_dbg(&v3d->drm, "Empty multisync extension\n"); 655 return -EINVAL; 656 } 657 658 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 659 multisync.out_syncs); 660 if (ret) 661 return ret; 662 663 se->in_sync_count = multisync.in_sync_count; 664 se->in_syncs = multisync.in_syncs; 665 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 666 se->wait_stage = multisync.wait_stage; 667 668 return 0; 669 } 670 671 /* Returns false if the CPU job has an invalid configuration. */ 672 static bool 673 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job) 674 { 675 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 676 struct v3d_dev *v3d = v3d_priv->v3d; 677 678 if (!job) { 679 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n"); 680 return false; 681 } 682 683 if (job->job_type) { 684 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n"); 685 return false; 686 } 687 688 return true; 689 } 690 691 /* Get data for the indirect CSD job submission. */ 692 static int 693 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 694 struct drm_v3d_extension __user *ext, 695 struct v3d_cpu_job *job) 696 { 697 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 698 struct v3d_dev *v3d = v3d_priv->v3d; 699 struct drm_v3d_indirect_csd indirect_csd; 700 struct v3d_indirect_csd_info *info = &job->indirect_csd; 701 702 if (!v3d_validate_cpu_job(file_priv, job)) 703 return -EINVAL; 704 705 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 706 return -EFAULT; 707 708 if (!v3d_has_csd(v3d)) { 709 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n"); 710 return -EINVAL; 711 } 712 713 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 714 info->args = indirect_csd.submit; 715 info->offset = indirect_csd.offset; 716 info->wg_size = indirect_csd.wg_size; 717 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 718 sizeof(indirect_csd.wg_uniform_offsets)); 719 720 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 721 if (!info->indirect) 722 return -ENOENT; 723 724 return 0; 725 } 726 727 /* Get data for the query timestamp job submission. */ 728 static int 729 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 730 struct drm_v3d_extension __user *ext, 731 struct v3d_cpu_job *job) 732 { 733 u32 __user *offsets, *syncs; 734 struct drm_v3d_timestamp_query timestamp; 735 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 736 unsigned int i; 737 int err; 738 739 if (!v3d_validate_cpu_job(file_priv, job)) 740 return -EINVAL; 741 742 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 743 return -EFAULT; 744 745 if (timestamp.pad) 746 return -EINVAL; 747 748 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 749 750 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 751 timestamp.count); 752 if (!query_info->queries) 753 return -ENOMEM; 754 755 offsets = u64_to_user_ptr(timestamp.offsets); 756 syncs = u64_to_user_ptr(timestamp.syncs); 757 758 for (i = 0; i < timestamp.count; i++) { 759 u32 offset, sync; 760 761 if (get_user(offset, offsets++)) { 762 err = -EFAULT; 763 goto error; 764 } 765 766 query_info->queries[i].offset = offset; 767 768 if (get_user(sync, syncs++)) { 769 err = -EFAULT; 770 goto error; 771 } 772 773 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 774 sync); 775 if (!query_info->queries[i].syncobj) { 776 err = -ENOENT; 777 goto error; 778 } 779 } 780 query_info->count = timestamp.count; 781 782 return 0; 783 784 error: 785 v3d_timestamp_query_info_free(&job->timestamp_query, i); 786 return err; 787 } 788 789 static int 790 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 791 struct drm_v3d_extension __user *ext, 792 struct v3d_cpu_job *job) 793 { 794 u32 __user *syncs; 795 struct drm_v3d_reset_timestamp_query reset; 796 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 797 unsigned int i; 798 int err; 799 800 if (!v3d_validate_cpu_job(file_priv, job)) 801 return -EINVAL; 802 803 if (copy_from_user(&reset, ext, sizeof(reset))) 804 return -EFAULT; 805 806 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 807 808 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 809 reset.count); 810 if (!query_info->queries) 811 return -ENOMEM; 812 813 syncs = u64_to_user_ptr(reset.syncs); 814 815 for (i = 0; i < reset.count; i++) { 816 u32 sync; 817 818 query_info->queries[i].offset = reset.offset + 8 * i; 819 820 if (get_user(sync, syncs++)) { 821 err = -EFAULT; 822 goto error; 823 } 824 825 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 826 sync); 827 if (!query_info->queries[i].syncobj) { 828 err = -ENOENT; 829 goto error; 830 } 831 } 832 query_info->count = reset.count; 833 834 return 0; 835 836 error: 837 v3d_timestamp_query_info_free(&job->timestamp_query, i); 838 return err; 839 } 840 841 /* Get data for the copy timestamp query results job submission. */ 842 static int 843 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 844 struct drm_v3d_extension __user *ext, 845 struct v3d_cpu_job *job) 846 { 847 u32 __user *offsets, *syncs; 848 struct drm_v3d_copy_timestamp_query copy; 849 struct v3d_timestamp_query_info *query_info = &job->timestamp_query; 850 unsigned int i; 851 int err; 852 853 if (!v3d_validate_cpu_job(file_priv, job)) 854 return -EINVAL; 855 856 if (copy_from_user(©, ext, sizeof(copy))) 857 return -EFAULT; 858 859 if (copy.pad) 860 return -EINVAL; 861 862 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 863 864 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query, 865 copy.count); 866 if (!query_info->queries) 867 return -ENOMEM; 868 869 offsets = u64_to_user_ptr(copy.offsets); 870 syncs = u64_to_user_ptr(copy.syncs); 871 872 for (i = 0; i < copy.count; i++) { 873 u32 offset, sync; 874 875 if (get_user(offset, offsets++)) { 876 err = -EFAULT; 877 goto error; 878 } 879 880 query_info->queries[i].offset = offset; 881 882 if (get_user(sync, syncs++)) { 883 err = -EFAULT; 884 goto error; 885 } 886 887 query_info->queries[i].syncobj = drm_syncobj_find(file_priv, 888 sync); 889 if (!query_info->queries[i].syncobj) { 890 err = -ENOENT; 891 goto error; 892 } 893 } 894 query_info->count = copy.count; 895 896 job->copy.do_64bit = copy.do_64bit; 897 job->copy.do_partial = copy.do_partial; 898 job->copy.availability_bit = copy.availability_bit; 899 job->copy.offset = copy.offset; 900 job->copy.stride = copy.stride; 901 902 return 0; 903 904 error: 905 v3d_timestamp_query_info_free(&job->timestamp_query, i); 906 return err; 907 } 908 909 static int 910 v3d_copy_query_info(struct v3d_performance_query_info *query_info, 911 unsigned int count, 912 unsigned int nperfmons, 913 u32 __user *syncs, 914 u64 __user *kperfmon_ids, 915 struct drm_file *file_priv) 916 { 917 unsigned int i, j; 918 int err; 919 920 for (i = 0; i < count; i++) { 921 struct v3d_performance_query *query = &query_info->queries[i]; 922 u32 __user *ids_pointer; 923 u32 sync, id; 924 u64 ids; 925 926 if (get_user(sync, syncs++)) { 927 err = -EFAULT; 928 goto error; 929 } 930 931 if (get_user(ids, kperfmon_ids++)) { 932 err = -EFAULT; 933 goto error; 934 } 935 936 query->kperfmon_ids = 937 kvmalloc_array(nperfmons, 938 sizeof(struct v3d_performance_query *), 939 GFP_KERNEL); 940 if (!query->kperfmon_ids) { 941 err = -ENOMEM; 942 goto error; 943 } 944 945 ids_pointer = u64_to_user_ptr(ids); 946 947 for (j = 0; j < nperfmons; j++) { 948 if (get_user(id, ids_pointer++)) { 949 kvfree(query->kperfmon_ids); 950 err = -EFAULT; 951 goto error; 952 } 953 954 query->kperfmon_ids[j] = id; 955 } 956 957 query->syncobj = drm_syncobj_find(file_priv, sync); 958 if (!query->syncobj) { 959 kvfree(query->kperfmon_ids); 960 err = -ENOENT; 961 goto error; 962 } 963 } 964 965 return 0; 966 967 error: 968 v3d_performance_query_info_free(query_info, i); 969 return err; 970 } 971 972 static int 973 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 974 struct drm_v3d_extension __user *ext, 975 struct v3d_cpu_job *job) 976 { 977 struct v3d_performance_query_info *query_info = &job->performance_query; 978 struct drm_v3d_reset_performance_query reset; 979 int err; 980 981 if (!v3d_validate_cpu_job(file_priv, job)) 982 return -EINVAL; 983 984 if (copy_from_user(&reset, ext, sizeof(reset))) 985 return -EFAULT; 986 987 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 988 989 query_info->queries = 990 kvmalloc_objs(struct v3d_performance_query, reset.count); 991 if (!query_info->queries) 992 return -ENOMEM; 993 994 err = v3d_copy_query_info(query_info, 995 reset.count, 996 reset.nperfmons, 997 u64_to_user_ptr(reset.syncs), 998 u64_to_user_ptr(reset.kperfmon_ids), 999 file_priv); 1000 if (err) 1001 return err; 1002 1003 query_info->count = reset.count; 1004 query_info->nperfmons = reset.nperfmons; 1005 1006 return 0; 1007 } 1008 1009 static int 1010 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 1011 struct drm_v3d_extension __user *ext, 1012 struct v3d_cpu_job *job) 1013 { 1014 struct v3d_performance_query_info *query_info = &job->performance_query; 1015 struct drm_v3d_copy_performance_query copy; 1016 int err; 1017 1018 if (!v3d_validate_cpu_job(file_priv, job)) 1019 return -EINVAL; 1020 1021 if (copy_from_user(©, ext, sizeof(copy))) 1022 return -EFAULT; 1023 1024 if (copy.pad) 1025 return -EINVAL; 1026 1027 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 1028 1029 query_info->queries = 1030 kvmalloc_objs(struct v3d_performance_query, copy.count); 1031 if (!query_info->queries) 1032 return -ENOMEM; 1033 1034 err = v3d_copy_query_info(query_info, 1035 copy.count, 1036 copy.nperfmons, 1037 u64_to_user_ptr(copy.syncs), 1038 u64_to_user_ptr(copy.kperfmon_ids), 1039 file_priv); 1040 if (err) 1041 return err; 1042 1043 query_info->count = copy.count; 1044 query_info->nperfmons = copy.nperfmons; 1045 query_info->ncounters = copy.ncounters; 1046 1047 job->copy.do_64bit = copy.do_64bit; 1048 job->copy.do_partial = copy.do_partial; 1049 job->copy.availability_bit = copy.availability_bit; 1050 job->copy.offset = copy.offset; 1051 job->copy.stride = copy.stride; 1052 1053 return 0; 1054 } 1055 1056 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 1057 * according to the extension id (name). 1058 */ 1059 static int 1060 v3d_get_extensions(struct drm_file *file_priv, 1061 u64 ext_handles, 1062 struct v3d_submit_ext *se, 1063 struct v3d_cpu_job *job) 1064 { 1065 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 1066 struct v3d_dev *v3d = v3d_priv->v3d; 1067 struct drm_v3d_extension __user *user_ext; 1068 int ret; 1069 1070 user_ext = u64_to_user_ptr(ext_handles); 1071 while (user_ext) { 1072 struct drm_v3d_extension ext; 1073 1074 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 1075 drm_dbg(&v3d->drm, "Failed to copy submit extension\n"); 1076 return -EFAULT; 1077 } 1078 1079 switch (ext.id) { 1080 case DRM_V3D_EXT_ID_MULTI_SYNC: 1081 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 1082 break; 1083 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 1084 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 1085 break; 1086 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 1087 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 1088 break; 1089 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 1090 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 1091 break; 1092 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 1093 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 1094 break; 1095 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 1096 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 1097 break; 1098 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 1099 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 1100 break; 1101 default: 1102 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id); 1103 return -EINVAL; 1104 } 1105 1106 if (ret) 1107 return ret; 1108 1109 user_ext = u64_to_user_ptr(ext.next); 1110 } 1111 1112 return 0; 1113 } 1114 1115 /** 1116 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 1117 * @dev: DRM device 1118 * @data: ioctl argument 1119 * @file_priv: DRM file for this fd 1120 * 1121 * This is the main entrypoint for userspace to submit a 3D frame to 1122 * the GPU. Userspace provides the binner command list (if 1123 * applicable), and the kernel sets up the render command list to draw 1124 * to the framebuffer described in the ioctl, using the command lists 1125 * that the 3D engine's binner will produce. 1126 */ 1127 int 1128 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 1129 struct drm_file *file_priv) 1130 { 1131 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1132 struct drm_v3d_submit_cl *args = data; 1133 struct drm_syncobj *sync_out = NULL; 1134 struct v3d_submit_ext se = {0}; 1135 struct v3d_bin_job *bin = NULL; 1136 struct v3d_render_job *render; 1137 struct v3d_job *clean_job; 1138 int ret; 1139 1140 trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end); 1141 1142 if (args->pad) 1143 return -EINVAL; 1144 1145 if (args->flags && 1146 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 1147 DRM_V3D_SUBMIT_EXTENSION)) { 1148 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1149 return -EINVAL; 1150 } 1151 1152 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1153 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1154 if (ret) { 1155 drm_dbg(dev, "Failed to get extensions.\n"); 1156 return ret; 1157 } 1158 } 1159 1160 /* If multisync is configured, give priority to it and ignore out_sync. */ 1161 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1162 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1163 if (!sync_out) 1164 return -ENOENT; 1165 } 1166 1167 if (args->bcl_start != args->bcl_end) { 1168 bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN); 1169 if (IS_ERR(bin)) { 1170 ret = PTR_ERR(bin); 1171 goto fail; 1172 } 1173 1174 bin->start = args->bcl_start; 1175 bin->end = args->bcl_end; 1176 bin->qma = args->qma; 1177 bin->qms = args->qms; 1178 bin->qts = args->qts; 1179 1180 ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl, 1181 &se); 1182 if (ret) 1183 goto fail; 1184 } 1185 1186 render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER); 1187 if (IS_ERR(render)) { 1188 ret = PTR_ERR(render); 1189 goto fail; 1190 } 1191 1192 INIT_LIST_HEAD(&render->unref_list); 1193 render->start = args->rcl_start; 1194 render->end = args->rcl_end; 1195 1196 if (bin) 1197 bin->render = render; 1198 1199 ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se); 1200 if (ret) 1201 goto fail; 1202 1203 /* 1204 * We don't associate the BOs with the BIN job. Fences are only 1205 * attached to the last job in the submission chain, and BIN jobs 1206 * don't need implicit dependencies since depending on results from 1207 * another context is not a realistic scenario for binning. 1208 */ 1209 ret = v3d_lookup_bos(submit.file_priv, &render->base, 1210 args->bo_handles, args->bo_handle_count); 1211 if (ret) 1212 goto fail; 1213 1214 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 1215 clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN); 1216 if (IS_ERR(clean_job)) { 1217 ret = PTR_ERR(clean_job); 1218 goto fail; 1219 } 1220 1221 ret = v3d_job_reference_bos(clean_job, &render->base); 1222 if (ret) 1223 goto fail; 1224 } 1225 1226 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); 1227 if (ret) 1228 goto fail; 1229 1230 ret = v3d_submit_lock_reservations(&submit); 1231 if (ret) 1232 goto fail; 1233 1234 ret = v3d_submit_jobs(&submit, sync_out, &se); 1235 if (ret) 1236 goto fail_submit; 1237 1238 return 0; 1239 1240 fail: 1241 v3d_submit_cleanup_jobs(&submit); 1242 fail_submit: 1243 v3d_submit_put_post_deps(sync_out, &se); 1244 1245 return ret; 1246 } 1247 1248 /** 1249 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1250 * @dev: DRM device 1251 * @data: ioctl argument 1252 * @file_priv: DRM file for this fd 1253 * 1254 * Userspace provides the register setup for the TFU, which we don't 1255 * need to validate since the TFU is behind the MMU. 1256 */ 1257 int 1258 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1259 struct drm_file *file_priv) 1260 { 1261 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1262 struct drm_v3d_submit_tfu *args = data; 1263 struct drm_syncobj *sync_out = NULL; 1264 struct v3d_submit_ext se = {0}; 1265 struct v3d_tfu_job *job; 1266 int ret = 0; 1267 1268 trace_v3d_submit_tfu_ioctl(dev, args->iia); 1269 1270 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1271 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1272 return -EINVAL; 1273 } 1274 1275 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1276 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1277 if (ret) { 1278 drm_dbg(dev, "Failed to get extensions.\n"); 1279 return ret; 1280 } 1281 } 1282 1283 /* If multisync is configured, give priority to it and ignore out_sync. */ 1284 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1285 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1286 if (!sync_out) 1287 return -ENOENT; 1288 } 1289 1290 job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU); 1291 if (IS_ERR(job)) { 1292 ret = PTR_ERR(job); 1293 goto fail; 1294 } 1295 1296 ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se); 1297 if (ret) 1298 goto fail; 1299 1300 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); 1301 if (!job->base.bo) { 1302 ret = -ENOMEM; 1303 goto fail; 1304 } 1305 1306 job->args = *args; 1307 1308 for (job->base.bo_count = 0; 1309 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1310 job->base.bo_count++) { 1311 struct drm_gem_object *bo; 1312 1313 if (!args->bo_handles[job->base.bo_count]) 1314 break; 1315 1316 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1317 if (!bo) { 1318 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n", 1319 job->base.bo_count, 1320 args->bo_handles[job->base.bo_count]); 1321 ret = -ENOENT; 1322 goto fail; 1323 } 1324 job->base.bo[job->base.bo_count] = bo; 1325 } 1326 1327 ret = v3d_submit_lock_reservations(&submit); 1328 if (ret) 1329 goto fail; 1330 1331 ret = v3d_submit_jobs(&submit, sync_out, &se); 1332 if (ret) 1333 goto fail_submit; 1334 1335 return 0; 1336 1337 fail: 1338 v3d_submit_cleanup_jobs(&submit); 1339 fail_submit: 1340 v3d_submit_put_post_deps(sync_out, &se); 1341 1342 return ret; 1343 } 1344 1345 /** 1346 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1347 * @dev: DRM device 1348 * @data: ioctl argument 1349 * @file_priv: DRM file for this fd 1350 * 1351 * Userspace provides the register setup for the CSD, which we don't 1352 * need to validate since the CSD is behind the MMU. 1353 */ 1354 int 1355 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1356 struct drm_file *file_priv) 1357 { 1358 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1359 struct drm_v3d_submit_csd *args = data; 1360 struct drm_syncobj *sync_out = NULL; 1361 struct v3d_submit_ext se = {0}; 1362 int ret; 1363 1364 trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]); 1365 1366 if (args->pad) 1367 return -EINVAL; 1368 1369 if (!v3d_has_csd(submit.v3d)) { 1370 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); 1371 return -EINVAL; 1372 } 1373 1374 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1375 drm_dbg(dev, "invalid flags: %d\n", args->flags); 1376 return -EINVAL; 1377 } 1378 1379 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1380 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1381 if (ret) { 1382 drm_dbg(dev, "Failed to get extensions.\n"); 1383 return ret; 1384 } 1385 } 1386 1387 /* If multisync is configured, give priority to it and ignore out_sync. */ 1388 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { 1389 sync_out = drm_syncobj_find(file_priv, args->out_sync); 1390 if (!sync_out) 1391 return -ENOENT; 1392 } 1393 1394 ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se); 1395 if (ret) 1396 goto fail; 1397 1398 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); 1399 if (ret) 1400 goto fail; 1401 1402 ret = v3d_submit_lock_reservations(&submit); 1403 if (ret) 1404 goto fail; 1405 1406 ret = v3d_submit_jobs(&submit, sync_out, &se); 1407 if (ret) 1408 goto fail_submit; 1409 1410 return 0; 1411 1412 fail: 1413 v3d_submit_cleanup_jobs(&submit); 1414 fail_submit: 1415 v3d_submit_put_post_deps(sync_out, &se); 1416 1417 return ret; 1418 } 1419 1420 static const unsigned int cpu_job_bo_handle_count[] = { 1421 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1422 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1423 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1424 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1425 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1426 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1427 }; 1428 1429 /* Reject offset + (count - 1) * stride + write_size if it leaves the BO. */ 1430 static int 1431 v3d_check_copy_extent(struct drm_device *dev, size_t bo_size, 1432 u32 offset, u32 stride, u32 count, u64 write_size) 1433 { 1434 u64 last; 1435 1436 if (!count) 1437 return 0; 1438 1439 /* 1440 * The executors walk a u8 * cursor, so the furthest written byte is 1441 * offset + (count - 1) * stride + write_size, matching the pointer 1442 * arithmetic in v3d_copy_query_results()/v3d_copy_performance_query(). 1443 * (count - 1) * stride is a u32 * u32 product that is exact in u64, 1444 * and offset + write_size stays far below the u64 range, so a single 1445 * overflow check guards the total. 1446 */ 1447 last = write_size + offset; 1448 if (check_add_overflow((u64)(count - 1) * stride, last, &last) || 1449 last > bo_size) { 1450 drm_dbg(dev, "CPU job copy buffer exceeds the destination BO.\n"); 1451 return -EINVAL; 1452 } 1453 1454 return 0; 1455 } 1456 1457 /* Reject a query CPU job whose writes would land outside their BO. */ 1458 static int 1459 v3d_cpu_job_bounds_check(struct v3d_cpu_job *job) 1460 { 1461 struct drm_device *dev = &job->base.v3d->drm; 1462 struct v3d_timestamp_query_info *tquery = &job->timestamp_query; 1463 struct v3d_copy_query_results_info *copy = &job->copy; 1464 u32 elem = copy->do_64bit ? sizeof(u64) : sizeof(u32); 1465 struct v3d_bo *dst, *src; 1466 u64 slots, write_size; 1467 u32 i; 1468 1469 switch (job->job_type) { 1470 case V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY: 1471 case V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY: 1472 /* Each query writes one u64 timestamp slot into bo[0]. */ 1473 dst = to_v3d_bo(job->base.bo[0]); 1474 1475 for (i = 0; i < tquery->count; i++) { 1476 if ((u64)tquery->queries[i].offset + sizeof(u64) > 1477 dst->base.base.size) 1478 goto err_range; 1479 } 1480 return 0; 1481 case V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY: 1482 /* Copies one u64 per query from bo[1] into bo[0]. */ 1483 dst = to_v3d_bo(job->base.bo[0]); 1484 src = to_v3d_bo(job->base.bo[1]); 1485 1486 for (i = 0; i < tquery->count; i++) { 1487 if ((u64)tquery->queries[i].offset + sizeof(u64) > 1488 src->base.base.size) 1489 goto err_range; 1490 } 1491 1492 write_size = (copy->availability_bit ? 2 : 1) * elem; 1493 return v3d_check_copy_extent(dev, dst->base.base.size, 1494 copy->offset, copy->stride, 1495 tquery->count, write_size); 1496 case V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY: 1497 /* 1498 * Each query writes nperfmons * DRM_V3D_MAX_PERF_COUNTERS 1499 * counter slots into bo[0], plus an availability slot at 1500 * index ncounters. nperfmons and ncounters are user values, 1501 * so the slot count is computed overflow-safe. 1502 */ 1503 dst = to_v3d_bo(job->base.bo[0]); 1504 1505 slots = (u64)job->performance_query.nperfmons * 1506 DRM_V3D_MAX_PERF_COUNTERS; 1507 if (copy->availability_bit) 1508 slots = max(slots, 1509 (u64)job->performance_query.ncounters + 1); 1510 1511 write_size = slots * elem; 1512 return v3d_check_copy_extent(dev, dst->base.base.size, 1513 copy->offset, copy->stride, 1514 job->performance_query.count, 1515 write_size); 1516 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: { 1517 struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd; 1518 1519 /* 3 is the three dimensions (x, y, z) of the workgroup counts. */ 1520 src = to_v3d_bo(job->base.bo[0]); 1521 if ((u64)indirect_csd->offset + 3 * sizeof(u32) > 1522 src->base.base.size) 1523 goto err_range; 1524 1525 dst = to_v3d_bo(indirect_csd->indirect); 1526 for (i = 0; i < 3; i++) { 1527 u32 uidx = indirect_csd->wg_uniform_offsets[i]; 1528 1529 /* 1530 * 0xffffffff means "skip this rewrite", so the exec 1531 * path never writes that index and it needs no check. 1532 */ 1533 if (uidx != 0xffffffff && 1534 (u64)uidx * sizeof(u32) + sizeof(u32) > 1535 dst->base.base.size) 1536 goto err_range; 1537 } 1538 return 0; 1539 } 1540 default: 1541 return 0; 1542 } 1543 1544 err_range: 1545 drm_dbg(dev, "CPU job query offset exceeds the BO.\n"); 1546 return -EINVAL; 1547 } 1548 1549 /** 1550 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1551 * @dev: DRM device 1552 * @data: ioctl argument 1553 * @file_priv: DRM file for this fd 1554 * 1555 * Userspace specifies the CPU job type and data required to perform its 1556 * operations through the drm_v3d_extension struct. 1557 */ 1558 int 1559 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1560 struct drm_file *file_priv) 1561 { 1562 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; 1563 struct drm_v3d_submit_cpu *args = data; 1564 struct v3d_submit_ext se = {0}; 1565 struct v3d_cpu_job *cpu_job = NULL; 1566 int ret; 1567 1568 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1569 drm_dbg(dev, "Invalid flags: %d\n", args->flags); 1570 return -EINVAL; 1571 } 1572 1573 cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU); 1574 if (IS_ERR(cpu_job)) 1575 return PTR_ERR(cpu_job); 1576 1577 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1578 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1579 if (ret) { 1580 drm_dbg(dev, "Failed to get extensions.\n"); 1581 goto fail; 1582 } 1583 } 1584 1585 /* Every CPU job must have a CPU job user extension */ 1586 if (!cpu_job->job_type) { 1587 drm_dbg(dev, "CPU job must have a CPU job user extension.\n"); 1588 ret = -EINVAL; 1589 goto fail; 1590 } 1591 1592 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1593 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n"); 1594 ret = -EINVAL; 1595 goto fail; 1596 } 1597 1598 trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type); 1599 1600 ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se); 1601 if (ret) 1602 goto fail; 1603 1604 /* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends 1605 * the CSD and clean jobs in the case of indirect CSD job. 1606 */ 1607 if (args->bo_handle_count) { 1608 ret = v3d_lookup_bos(submit.file_priv, &cpu_job->base, 1609 args->bo_handles, args->bo_handle_count); 1610 if (ret) 1611 goto fail; 1612 1613 ret = v3d_cpu_job_bounds_check(cpu_job); 1614 if (ret) 1615 goto fail; 1616 } 1617 1618 if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) { 1619 ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args, 1620 NULL); 1621 if (ret) 1622 goto fail; 1623 1624 /* The CSD job was appended at jobs[1] */ 1625 if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) { 1626 ret = -EINVAL; 1627 goto fail; 1628 } 1629 1630 cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job, 1631 base); 1632 } 1633 1634 ret = v3d_submit_lock_reservations(&submit); 1635 if (ret) 1636 goto fail; 1637 1638 ret = v3d_submit_jobs(&submit, NULL, &se); 1639 if (ret) 1640 goto fail_submit; 1641 1642 return 0; 1643 1644 fail: 1645 v3d_submit_cleanup_jobs(&submit); 1646 fail_submit: 1647 v3d_submit_put_post_deps(NULL, &se); 1648 1649 return ret; 1650 } 1651