1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4 #include "pvr_context.h" 5 #include "pvr_device.h" 6 #include "pvr_drv.h" 7 #include "pvr_gem.h" 8 #include "pvr_hwrt.h" 9 #include "pvr_job.h" 10 #include "pvr_mmu.h" 11 #include "pvr_power.h" 12 #include "pvr_rogue_fwif.h" 13 #include "pvr_rogue_fwif_client.h" 14 #include "pvr_stream.h" 15 #include "pvr_stream_defs.h" 16 #include "pvr_sync.h" 17 #include "pvr_trace.h" 18 19 #include <drm/drm_exec.h> 20 #include <drm/drm_gem.h> 21 #include <linux/types.h> 22 #include <uapi/drm/pvr_drm.h> 23 24 static void pvr_job_release(struct kref *kref) 25 { 26 struct pvr_job *job = container_of(kref, struct pvr_job, ref_count); 27 28 xa_erase(&job->pvr_dev->job_ids, job->id); 29 30 pvr_hwrt_data_put(job->hwrt); 31 pvr_context_put(job->ctx); 32 33 WARN_ON(job->paired_job); 34 35 pvr_queue_job_cleanup(job); 36 pvr_job_release_pm_ref(job); 37 38 kfree(job->cmd); 39 kfree(job); 40 } 41 42 /** 43 * pvr_job_put() - Release reference on job 44 * @job: Target job. 45 */ 46 void 47 pvr_job_put(struct pvr_job *job) 48 { 49 if (job) 50 kref_put(&job->ref_count, pvr_job_release); 51 } 52 53 /** 54 * pvr_job_process_stream() - Build job FW structure from stream 55 * @pvr_dev: Device pointer. 56 * @cmd_defs: Stream definition. 57 * @stream: Pointer to command stream. 58 * @stream_size: Size of command stream, in bytes. 59 * @job: Pointer to job. 60 * 61 * Caller is responsible for freeing the output structure. 62 * 63 * Returns: 64 * * 0 on success, 65 * * -%ENOMEM on out of memory, or 66 * * -%EINVAL on malformed stream. 67 */ 68 static int 69 pvr_job_process_stream(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs, 70 void *stream, u32 stream_size, struct pvr_job *job) 71 { 72 int err; 73 74 job->cmd = kzalloc(cmd_defs->dest_size, GFP_KERNEL); 75 if (!job->cmd) 76 return -ENOMEM; 77 78 job->cmd_len = cmd_defs->dest_size; 79 80 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, job->cmd); 81 if (err) 82 kfree(job->cmd); 83 84 return err; 85 } 86 87 static int pvr_fw_cmd_init(struct pvr_device *pvr_dev, struct pvr_job *job, 88 const struct pvr_stream_cmd_defs *stream_def, 89 u64 stream_userptr, u32 stream_len) 90 { 91 void *stream; 92 int err; 93 94 stream = memdup_user(u64_to_user_ptr(stream_userptr), stream_len); 95 if (IS_ERR(stream)) 96 return PTR_ERR(stream); 97 98 err = pvr_job_process_stream(pvr_dev, stream_def, stream, stream_len, job); 99 100 kfree(stream); 101 return err; 102 } 103 104 static u32 105 convert_geom_flags(u32 in_flags) 106 { 107 u32 out_flags = 0; 108 109 if (in_flags & DRM_PVR_SUBMIT_JOB_GEOM_CMD_FIRST) 110 out_flags |= ROGUE_GEOM_FLAGS_FIRSTKICK; 111 if (in_flags & DRM_PVR_SUBMIT_JOB_GEOM_CMD_LAST) 112 out_flags |= ROGUE_GEOM_FLAGS_LASTKICK; 113 if (in_flags & DRM_PVR_SUBMIT_JOB_GEOM_CMD_SINGLE_CORE) 114 out_flags |= ROGUE_GEOM_FLAGS_SINGLE_CORE; 115 116 return out_flags; 117 } 118 119 static u32 120 convert_frag_flags(u32 in_flags) 121 { 122 u32 out_flags = 0; 123 124 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_SINGLE_CORE) 125 out_flags |= ROGUE_FRAG_FLAGS_SINGLE_CORE; 126 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_DEPTHBUFFER) 127 out_flags |= ROGUE_FRAG_FLAGS_DEPTHBUFFER; 128 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_STENCILBUFFER) 129 out_flags |= ROGUE_FRAG_FLAGS_STENCILBUFFER; 130 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_PREVENT_CDM_OVERLAP) 131 out_flags |= ROGUE_FRAG_FLAGS_PREVENT_CDM_OVERLAP; 132 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_SCRATCHBUFFER) 133 out_flags |= ROGUE_FRAG_FLAGS_SCRATCHBUFFER; 134 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_GET_VIS_RESULTS) 135 out_flags |= ROGUE_FRAG_FLAGS_GET_VIS_RESULTS; 136 if (in_flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_DISABLE_PIXELMERGE) 137 out_flags |= ROGUE_FRAG_FLAGS_DISABLE_PIXELMERGE; 138 139 return out_flags; 140 } 141 142 static int 143 pvr_geom_job_fw_cmd_init(struct pvr_job *job, 144 struct drm_pvr_job *args) 145 { 146 struct rogue_fwif_cmd_geom *cmd; 147 int err; 148 149 if (args->flags & ~DRM_PVR_SUBMIT_JOB_GEOM_CMD_FLAGS_MASK) 150 return -EINVAL; 151 152 if (job->ctx->type != DRM_PVR_CTX_TYPE_RENDER) 153 return -EINVAL; 154 155 if (!job->hwrt) 156 return -EINVAL; 157 158 job->fw_ccb_cmd_type = ROGUE_FWIF_CCB_CMD_TYPE_GEOM; 159 err = pvr_fw_cmd_init(job->pvr_dev, job, &pvr_cmd_geom_stream, 160 args->cmd_stream, args->cmd_stream_len); 161 if (err) 162 return err; 163 164 cmd = job->cmd; 165 cmd->cmd_shared.cmn.frame_num = 0; 166 cmd->flags = convert_geom_flags(args->flags); 167 pvr_fw_object_get_fw_addr(job->hwrt->fw_obj, &cmd->cmd_shared.hwrt_data_fw_addr); 168 return 0; 169 } 170 171 static int 172 pvr_frag_job_fw_cmd_init(struct pvr_job *job, 173 struct drm_pvr_job *args) 174 { 175 struct rogue_fwif_cmd_frag *cmd; 176 int err; 177 178 if (args->flags & ~DRM_PVR_SUBMIT_JOB_FRAG_CMD_FLAGS_MASK) 179 return -EINVAL; 180 181 if (job->ctx->type != DRM_PVR_CTX_TYPE_RENDER) 182 return -EINVAL; 183 184 if (!job->hwrt) 185 return -EINVAL; 186 187 job->fw_ccb_cmd_type = (args->flags & DRM_PVR_SUBMIT_JOB_FRAG_CMD_PARTIAL_RENDER) ? 188 ROGUE_FWIF_CCB_CMD_TYPE_FRAG_PR : 189 ROGUE_FWIF_CCB_CMD_TYPE_FRAG; 190 err = pvr_fw_cmd_init(job->pvr_dev, job, &pvr_cmd_frag_stream, 191 args->cmd_stream, args->cmd_stream_len); 192 if (err) 193 return err; 194 195 cmd = job->cmd; 196 cmd->cmd_shared.cmn.frame_num = 0; 197 cmd->flags = convert_frag_flags(args->flags); 198 pvr_fw_object_get_fw_addr(job->hwrt->fw_obj, &cmd->cmd_shared.hwrt_data_fw_addr); 199 return 0; 200 } 201 202 static u32 203 convert_compute_flags(u32 in_flags) 204 { 205 u32 out_flags = 0; 206 207 if (in_flags & DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_PREVENT_ALL_OVERLAP) 208 out_flags |= ROGUE_COMPUTE_FLAG_PREVENT_ALL_OVERLAP; 209 if (in_flags & DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_SINGLE_CORE) 210 out_flags |= ROGUE_COMPUTE_FLAG_SINGLE_CORE; 211 212 return out_flags; 213 } 214 215 static int 216 pvr_compute_job_fw_cmd_init(struct pvr_job *job, 217 struct drm_pvr_job *args) 218 { 219 struct rogue_fwif_cmd_compute *cmd; 220 int err; 221 222 if (args->flags & ~DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_FLAGS_MASK) 223 return -EINVAL; 224 225 if (job->ctx->type != DRM_PVR_CTX_TYPE_COMPUTE) 226 return -EINVAL; 227 228 job->fw_ccb_cmd_type = ROGUE_FWIF_CCB_CMD_TYPE_CDM; 229 err = pvr_fw_cmd_init(job->pvr_dev, job, &pvr_cmd_compute_stream, 230 args->cmd_stream, args->cmd_stream_len); 231 if (err) 232 return err; 233 234 cmd = job->cmd; 235 cmd->common.frame_num = 0; 236 cmd->flags = convert_compute_flags(args->flags); 237 return 0; 238 } 239 240 static u32 241 convert_transfer_flags(u32 in_flags) 242 { 243 u32 out_flags = 0; 244 245 if (in_flags & DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_SINGLE_CORE) 246 out_flags |= ROGUE_TRANSFER_FLAGS_SINGLE_CORE; 247 248 return out_flags; 249 } 250 251 static int 252 pvr_transfer_job_fw_cmd_init(struct pvr_job *job, 253 struct drm_pvr_job *args) 254 { 255 struct rogue_fwif_cmd_transfer *cmd; 256 int err; 257 258 if (args->flags & ~DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_FLAGS_MASK) 259 return -EINVAL; 260 261 if (job->ctx->type != DRM_PVR_CTX_TYPE_TRANSFER_FRAG) 262 return -EINVAL; 263 264 job->fw_ccb_cmd_type = ROGUE_FWIF_CCB_CMD_TYPE_TQ_3D; 265 err = pvr_fw_cmd_init(job->pvr_dev, job, &pvr_cmd_transfer_stream, 266 args->cmd_stream, args->cmd_stream_len); 267 if (err) 268 return err; 269 270 cmd = job->cmd; 271 cmd->common.frame_num = 0; 272 cmd->flags = convert_transfer_flags(args->flags); 273 return 0; 274 } 275 276 static int 277 pvr_job_fw_cmd_init(struct pvr_job *job, 278 struct drm_pvr_job *args) 279 { 280 switch (args->type) { 281 case DRM_PVR_JOB_TYPE_GEOMETRY: 282 return pvr_geom_job_fw_cmd_init(job, args); 283 284 case DRM_PVR_JOB_TYPE_FRAGMENT: 285 return pvr_frag_job_fw_cmd_init(job, args); 286 287 case DRM_PVR_JOB_TYPE_COMPUTE: 288 return pvr_compute_job_fw_cmd_init(job, args); 289 290 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG: 291 return pvr_transfer_job_fw_cmd_init(job, args); 292 293 default: 294 return -EINVAL; 295 } 296 } 297 298 /** 299 * struct pvr_job_data - Helper container for pairing jobs with the 300 * sync_ops supplied for them by the user. 301 */ 302 struct pvr_job_data { 303 /** @job: Pointer to the job. */ 304 struct pvr_job *job; 305 306 /** @sync_ops: Pointer to the sync_ops associated with @job. */ 307 struct drm_pvr_sync_op *sync_ops; 308 309 /** @sync_op_count: Number of members of @sync_ops. */ 310 u32 sync_op_count; 311 }; 312 313 /** 314 * prepare_job_syncs() - Prepare all sync objects for a single job. 315 * @pvr_file: PowerVR file. 316 * @job_data: Precreated job and sync_ops array. 317 * @signal_array: xarray to receive signal sync objects. 318 * 319 * Returns: 320 * * 0 on success, or 321 * * Any error code returned by pvr_sync_signal_array_collect_ops(), 322 * pvr_sync_add_deps_to_job(), drm_sched_job_add_resv_dependencies() or 323 * pvr_sync_signal_array_update_fences(). 324 */ 325 static int 326 prepare_job_syncs(struct pvr_file *pvr_file, 327 struct pvr_job_data *job_data, 328 struct xarray *signal_array) 329 { 330 struct dma_fence *finished_fence; 331 int err = pvr_sync_signal_array_collect_ops(signal_array, 332 from_pvr_file(pvr_file), 333 job_data->sync_op_count, 334 job_data->sync_ops); 335 336 if (err) 337 return err; 338 339 err = pvr_sync_add_deps_to_job(pvr_file, &job_data->job->base, 340 job_data->sync_op_count, 341 job_data->sync_ops, signal_array); 342 if (err) 343 return err; 344 345 if (job_data->job->hwrt) { 346 /* The geometry job writes the HWRT region headers, which are 347 * then read by the fragment job. 348 */ 349 struct drm_gem_object *obj = 350 gem_from_pvr_gem(job_data->job->hwrt->fw_obj->gem); 351 enum dma_resv_usage usage = 352 dma_resv_usage_rw(job_data->job->type == 353 DRM_PVR_JOB_TYPE_GEOMETRY); 354 355 dma_resv_lock(obj->resv, NULL); 356 err = drm_sched_job_add_resv_dependencies(&job_data->job->base, 357 obj->resv, usage); 358 dma_resv_unlock(obj->resv); 359 if (err) 360 return err; 361 } 362 363 /* We need to arm the job to get the job finished fence. */ 364 finished_fence = pvr_queue_job_arm(job_data->job); 365 366 err = pvr_sync_signal_array_update_fences(signal_array, 367 job_data->sync_op_count, 368 job_data->sync_ops, 369 finished_fence); 370 return err; 371 } 372 373 /** 374 * prepare_job_syncs_for_each() - Prepare all sync objects for an array of jobs. 375 * @pvr_file: PowerVR file. 376 * @job_data: Array of precreated jobs and their sync_ops. 377 * @job_count: Number of jobs. 378 * @signal_array: xarray to receive signal sync objects. 379 * 380 * Returns: 381 * * 0 on success, or 382 * * Any error code returned by pvr_vm_bind_job_prepare_syncs(). 383 */ 384 static int 385 prepare_job_syncs_for_each(struct pvr_file *pvr_file, 386 struct pvr_job_data *job_data, 387 u32 *job_count, 388 struct xarray *signal_array) 389 { 390 for (u32 i = 0; i < *job_count; i++) { 391 int err = prepare_job_syncs(pvr_file, &job_data[i], 392 signal_array); 393 394 if (err) { 395 *job_count = i; 396 return err; 397 } 398 } 399 400 return 0; 401 } 402 403 static struct pvr_job * 404 create_job(struct pvr_device *pvr_dev, 405 struct pvr_file *pvr_file, 406 struct drm_pvr_job *args) 407 { 408 struct pvr_job *job = NULL; 409 int err; 410 411 if (!args->cmd_stream || !args->cmd_stream_len) 412 return ERR_PTR(-EINVAL); 413 414 if (args->type != DRM_PVR_JOB_TYPE_GEOMETRY && 415 args->type != DRM_PVR_JOB_TYPE_FRAGMENT && 416 (args->hwrt.set_handle || args->hwrt.data_index)) 417 return ERR_PTR(-EINVAL); 418 419 job = kzalloc_obj(*job); 420 if (!job) 421 return ERR_PTR(-ENOMEM); 422 423 kref_init(&job->ref_count); 424 job->type = args->type; 425 job->pvr_dev = pvr_dev; 426 427 err = xa_alloc(&pvr_dev->job_ids, &job->id, job, xa_limit_32b, GFP_KERNEL); 428 if (err) 429 goto err_put_job; 430 431 job->ctx = pvr_context_lookup(pvr_file, args->context_handle); 432 if (!job->ctx) { 433 err = -EINVAL; 434 goto err_put_job; 435 } 436 437 if (args->hwrt.set_handle) { 438 job->hwrt = pvr_hwrt_data_lookup(pvr_file, args->hwrt.set_handle, 439 args->hwrt.data_index); 440 if (!job->hwrt) { 441 err = -EINVAL; 442 goto err_put_job; 443 } 444 } 445 446 err = pvr_job_fw_cmd_init(job, args); 447 if (err) 448 goto err_put_job; 449 450 err = pvr_queue_job_init(job, pvr_file->file->client_id); 451 if (err) 452 goto err_put_job; 453 454 return job; 455 456 err_put_job: 457 pvr_job_put(job); 458 return ERR_PTR(err); 459 } 460 461 /** 462 * pvr_job_data_fini() - Cleanup all allocs used to set up job submission. 463 * @job_data: Job data array. 464 * @job_count: Number of members of @job_data. 465 */ 466 static void 467 pvr_job_data_fini(struct pvr_job_data *job_data, u32 job_count) 468 { 469 for (u32 i = 0; i < job_count; i++) { 470 pvr_job_put(job_data[i].job); 471 kvfree(job_data[i].sync_ops); 472 } 473 } 474 475 /** 476 * pvr_job_data_init() - Init an array of created jobs, associating them with 477 * the appropriate sync_ops args, which will be copied in. 478 * @pvr_dev: Target PowerVR device. 479 * @pvr_file: Pointer to PowerVR file structure. 480 * @job_args: Job args array copied from user. 481 * @job_count: Number of members of @job_args. 482 * @job_data_out: Job data array. 483 */ 484 static int pvr_job_data_init(struct pvr_device *pvr_dev, 485 struct pvr_file *pvr_file, 486 struct drm_pvr_job *job_args, 487 u32 *job_count, 488 struct pvr_job_data *job_data_out) 489 { 490 int err = 0, i = 0; 491 492 for (; i < *job_count; i++) { 493 job_data_out[i].job = 494 create_job(pvr_dev, pvr_file, &job_args[i]); 495 err = PTR_ERR_OR_ZERO(job_data_out[i].job); 496 497 if (err) { 498 *job_count = i; 499 job_data_out[i].job = NULL; 500 goto err_cleanup; 501 } 502 503 err = PVR_UOBJ_GET_ARRAY(job_data_out[i].sync_ops, 504 &job_args[i].sync_ops); 505 if (err) { 506 *job_count = i; 507 508 /* Ensure the job created above is also cleaned up. */ 509 i++; 510 goto err_cleanup; 511 } 512 513 job_data_out[i].sync_op_count = job_args[i].sync_ops.count; 514 515 trace_pvr_job_create(pvr_dev, job_data_out[i].job, job_data_out[i].sync_op_count); 516 } 517 518 return 0; 519 520 err_cleanup: 521 pvr_job_data_fini(job_data_out, i); 522 523 return err; 524 } 525 526 static void 527 push_jobs(struct pvr_job_data *job_data, u32 job_count) 528 { 529 for (u32 i = 0; i < job_count; i++) 530 pvr_queue_job_push(job_data[i].job); 531 } 532 533 static int 534 prepare_fw_obj_resv(struct drm_exec *exec, struct pvr_fw_object *fw_obj) 535 { 536 return drm_exec_prepare_obj(exec, gem_from_pvr_gem(fw_obj->gem), 1); 537 } 538 539 static int 540 jobs_lock_all_objs(struct drm_exec *exec, struct pvr_job_data *job_data, 541 u32 job_count) 542 { 543 for (u32 i = 0; i < job_count; i++) { 544 struct pvr_job *job = job_data[i].job; 545 546 /* Grab a lock on a the context, to guard against 547 * concurrent submission to the same queue. 548 */ 549 int err = drm_exec_lock_obj(exec, 550 gem_from_pvr_gem(job->ctx->fw_obj->gem)); 551 552 if (err) 553 return err; 554 555 if (job->hwrt) { 556 err = prepare_fw_obj_resv(exec, 557 job->hwrt->fw_obj); 558 if (err) 559 return err; 560 } 561 } 562 563 return 0; 564 } 565 566 static int 567 prepare_job_resvs_for_each(struct drm_exec *exec, struct pvr_job_data *job_data, 568 u32 job_count) 569 { 570 drm_exec_until_all_locked(exec) { 571 int err = jobs_lock_all_objs(exec, job_data, job_count); 572 573 drm_exec_retry_on_contention(exec); 574 if (err) 575 return err; 576 } 577 578 return 0; 579 } 580 581 static void 582 update_job_resvs(struct pvr_job *job) 583 { 584 if (job->hwrt) { 585 enum dma_resv_usage usage = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ? 586 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ; 587 struct drm_gem_object *obj = gem_from_pvr_gem(job->hwrt->fw_obj->gem); 588 589 dma_resv_add_fence(obj->resv, &job->base.s_fence->finished, usage); 590 } 591 } 592 593 static void 594 update_job_resvs_for_each(struct pvr_job_data *job_data, u32 job_count) 595 { 596 for (u32 i = 0; i < job_count; i++) 597 update_job_resvs(job_data[i].job); 598 } 599 600 static bool can_combine_jobs(struct pvr_job *a, struct pvr_job *b) 601 { 602 struct pvr_job *geom_job = a, *frag_job = b; 603 604 /* Geometry and fragment jobs can be combined if they are queued to the 605 * same context and targeting the same HWRT. 606 */ 607 if (a->type != DRM_PVR_JOB_TYPE_GEOMETRY || 608 b->type != DRM_PVR_JOB_TYPE_FRAGMENT || 609 a->ctx != b->ctx || 610 a->hwrt != b->hwrt) 611 return false; 612 613 /* We combine when we see an explicit geom -> frag dep. */ 614 return drm_sched_job_has_dependency(&frag_job->base, 615 &geom_job->base.s_fence->scheduled); 616 } 617 618 static struct dma_fence * 619 get_last_queued_job_scheduled_fence(struct pvr_queue *queue, 620 struct pvr_job_data *job_data, 621 u32 cur_job_pos) 622 { 623 /* We iterate over the current job array in reverse order to grab the 624 * last to-be-queued job targeting the same queue. 625 */ 626 for (u32 i = cur_job_pos; i > 0; i--) { 627 struct pvr_job *job = job_data[i - 1].job; 628 629 if (job->ctx == queue->ctx && job->type == queue->type) 630 return dma_fence_get(&job->base.s_fence->scheduled); 631 } 632 633 /* If we didn't find any, we just return the last queued job scheduled 634 * fence attached to the queue. 635 */ 636 return dma_fence_get(queue->last_queued_job_scheduled_fence); 637 } 638 639 static int 640 pvr_jobs_link_geom_frag(struct pvr_job_data *job_data, u32 *job_count) 641 { 642 for (u32 i = 0; i < *job_count - 1; i++) { 643 struct pvr_job *geom_job = job_data[i].job; 644 struct pvr_job *frag_job = job_data[i + 1].job; 645 struct pvr_queue *frag_queue; 646 struct dma_fence *f; 647 648 if (!can_combine_jobs(job_data[i].job, job_data[i + 1].job)) 649 continue; 650 651 /* The fragment job will be submitted by the geometry queue. We 652 * need to make sure it comes after all the other fragment jobs 653 * queued before it. 654 */ 655 frag_queue = pvr_context_get_queue_for_job(frag_job->ctx, 656 frag_job->type); 657 f = get_last_queued_job_scheduled_fence(frag_queue, job_data, 658 i); 659 if (f) { 660 int err = drm_sched_job_add_dependency(&geom_job->base, 661 f); 662 if (err) { 663 *job_count = i; 664 return err; 665 } 666 } 667 668 /* The KCCB slot will be reserved by the geometry job, so we can 669 * drop the KCCB fence on the fragment job. 670 */ 671 pvr_kccb_fence_put(frag_job->kccb_fence); 672 frag_job->kccb_fence = NULL; 673 674 geom_job->paired_job = frag_job; 675 frag_job->paired_job = geom_job; 676 677 /* The geometry job pvr_job structure is used when the fragment 678 * job is being prepared by the GPU scheduler. Have the fragment 679 * job hold a reference on the geometry job to prevent it being 680 * freed until the fragment job has finished with it. 681 */ 682 pvr_job_get(geom_job); 683 684 /* Skip the fragment job we just paired to the geometry job. */ 685 i++; 686 } 687 688 return 0; 689 } 690 691 /** 692 * pvr_submit_jobs() - Submit jobs to the GPU 693 * @pvr_dev: Target PowerVR device. 694 * @pvr_file: Pointer to PowerVR file structure. 695 * @args: Ioctl args. 696 * 697 * This initial implementation is entirely synchronous; on return the GPU will 698 * be idle. This will not be the case for future implementations. 699 * 700 * Returns: 701 * * 0 on success, 702 * * -%EFAULT if arguments can not be copied from user space, or 703 * * -%EINVAL on invalid arguments, or 704 * * Any other error. 705 */ 706 int 707 pvr_submit_jobs(struct pvr_device *pvr_dev, struct pvr_file *pvr_file, 708 struct drm_pvr_ioctl_submit_jobs_args *args) 709 { 710 struct pvr_job_data *job_data = NULL; 711 struct drm_pvr_job *job_args; 712 struct xarray signal_array; 713 u32 jobs_alloced = 0; 714 struct drm_exec exec; 715 int err; 716 717 if (!args->jobs.count) 718 return -EINVAL; 719 720 err = PVR_UOBJ_GET_ARRAY(job_args, &args->jobs); 721 if (err) 722 return err; 723 724 job_data = kvmalloc_objs(*job_data, args->jobs.count, 725 GFP_KERNEL | __GFP_ZERO); 726 if (!job_data) { 727 err = -ENOMEM; 728 goto out_free; 729 } 730 731 err = pvr_job_data_init(pvr_dev, pvr_file, job_args, &args->jobs.count, 732 job_data); 733 if (err) 734 goto out_free; 735 736 jobs_alloced = args->jobs.count; 737 738 /* 739 * Flush MMU if needed - this has been deferred until now to avoid 740 * overuse of this expensive operation. 741 */ 742 err = pvr_mmu_flush_exec(pvr_dev, false); 743 if (err) 744 goto out_job_data_cleanup; 745 746 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); 747 748 xa_init_flags(&signal_array, XA_FLAGS_ALLOC); 749 750 err = prepare_job_syncs_for_each(pvr_file, job_data, &args->jobs.count, 751 &signal_array); 752 if (err) 753 goto out_exec_fini; 754 755 err = prepare_job_resvs_for_each(&exec, job_data, args->jobs.count); 756 if (err) 757 goto out_exec_fini; 758 759 err = pvr_jobs_link_geom_frag(job_data, &args->jobs.count); 760 if (err) 761 goto out_exec_fini; 762 763 /* Anything after that point must succeed because we start exposing job 764 * finished fences to the outside world. 765 */ 766 update_job_resvs_for_each(job_data, args->jobs.count); 767 push_jobs(job_data, args->jobs.count); 768 pvr_sync_signal_array_push_fences(&signal_array); 769 err = 0; 770 771 out_exec_fini: 772 drm_exec_fini(&exec); 773 pvr_sync_signal_array_cleanup(&signal_array); 774 775 out_job_data_cleanup: 776 pvr_job_data_fini(job_data, jobs_alloced); 777 778 out_free: 779 kvfree(job_data); 780 kvfree(job_args); 781 782 return err; 783 } 784