1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2024 Intel Corporation 4 */ 5 6 #include <drm/drm_file.h> 7 8 #include <linux/bitfield.h> 9 #include <linux/highmem.h> 10 #include <linux/pci.h> 11 #include <linux/module.h> 12 #include <uapi/drm/ivpu_accel.h> 13 14 #include "ivpu_drv.h" 15 #include "ivpu_fw.h" 16 #include "ivpu_hw.h" 17 #include "ivpu_ipc.h" 18 #include "ivpu_job.h" 19 #include "ivpu_jsm_msg.h" 20 #include "ivpu_pm.h" 21 #include "ivpu_trace.h" 22 #include "vpu_boot_api.h" 23 24 #define CMD_BUF_IDX 0 25 #define JOB_MAX_BUFFER_COUNT 65535 26 27 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) 28 { 29 ivpu_hw_db_set(vdev, cmdq->db_id); 30 } 31 32 static int ivpu_preemption_buffers_create(struct ivpu_device *vdev, 33 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 34 { 35 u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE); 36 u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE); 37 38 if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW || 39 ivpu_test_mode & IVPU_TEST_MODE_MIP_DISABLE) 40 return 0; 41 42 cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user, 43 primary_size, DRM_IVPU_BO_WC); 44 if (!cmdq->primary_preempt_buf) { 45 ivpu_err(vdev, "Failed to create primary preemption buffer\n"); 46 return -ENOMEM; 47 } 48 49 cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma, 50 secondary_size, DRM_IVPU_BO_WC); 51 if (!cmdq->secondary_preempt_buf) { 52 ivpu_err(vdev, "Failed to create secondary preemption buffer\n"); 53 goto err_free_primary; 54 } 55 56 return 0; 57 58 err_free_primary: 59 ivpu_bo_free(cmdq->primary_preempt_buf); 60 cmdq->primary_preempt_buf = NULL; 61 return -ENOMEM; 62 } 63 64 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev, 65 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 66 { 67 if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW) 68 return; 69 70 if (cmdq->primary_preempt_buf) 71 ivpu_bo_free(cmdq->primary_preempt_buf); 72 if (cmdq->secondary_preempt_buf) 73 ivpu_bo_free(cmdq->secondary_preempt_buf); 74 } 75 76 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv) 77 { 78 struct ivpu_device *vdev = file_priv->vdev; 79 struct ivpu_cmdq *cmdq; 80 int ret; 81 82 cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL); 83 if (!cmdq) 84 return NULL; 85 86 ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next, 87 GFP_KERNEL); 88 if (ret < 0) { 89 ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret); 90 goto err_free_cmdq; 91 } 92 93 ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit, 94 &file_priv->cmdq_id_next, GFP_KERNEL); 95 if (ret < 0) { 96 ivpu_err(vdev, "Failed to allocate command queue id: %d\n", ret); 97 goto err_erase_db_xa; 98 } 99 100 cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); 101 if (!cmdq->mem) 102 goto err_erase_cmdq_xa; 103 104 ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq); 105 if (ret) 106 ivpu_warn(vdev, "Failed to allocate preemption buffers, preemption limited\n"); 107 108 return cmdq; 109 110 err_erase_cmdq_xa: 111 xa_erase(&file_priv->cmdq_xa, cmdq->id); 112 err_erase_db_xa: 113 xa_erase(&vdev->db_xa, cmdq->db_id); 114 err_free_cmdq: 115 kfree(cmdq); 116 return NULL; 117 } 118 119 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 120 { 121 if (!cmdq) 122 return; 123 124 ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq); 125 ivpu_bo_free(cmdq->mem); 126 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); 127 kfree(cmdq); 128 } 129 130 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, 131 u8 priority) 132 { 133 struct ivpu_device *vdev = file_priv->vdev; 134 int ret; 135 136 ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id, 137 task_pid_nr(current), engine, 138 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 139 if (ret) 140 return ret; 141 142 ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id, 143 priority); 144 if (ret) 145 return ret; 146 147 return 0; 148 } 149 150 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 151 { 152 struct ivpu_device *vdev = file_priv->vdev; 153 int ret; 154 155 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) 156 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id, 157 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 158 else 159 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id, 160 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 161 162 if (!ret) 163 ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d\n", 164 cmdq->db_id, cmdq->id, file_priv->ctx.id); 165 166 return ret; 167 } 168 169 static int 170 ivpu_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u8 priority) 171 { 172 struct ivpu_device *vdev = file_priv->vdev; 173 struct vpu_job_queue_header *jobq_header; 174 int ret; 175 176 lockdep_assert_held(&file_priv->lock); 177 178 if (cmdq->db_registered) 179 return 0; 180 181 cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) / 182 sizeof(struct vpu_job_queue_entry)); 183 184 cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem); 185 jobq_header = &cmdq->jobq->header; 186 jobq_header->engine_idx = VPU_ENGINE_COMPUTE; 187 jobq_header->head = 0; 188 jobq_header->tail = 0; 189 if (ivpu_test_mode & IVPU_TEST_MODE_TURBO) { 190 ivpu_dbg(vdev, JOB, "Turbo mode enabled"); 191 jobq_header->flags = VPU_JOB_QUEUE_FLAGS_TURBO_MODE; 192 } 193 194 wmb(); /* Flush WC buffer for jobq->header */ 195 196 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 197 ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, priority); 198 if (ret) 199 return ret; 200 } 201 202 ret = ivpu_register_db(file_priv, cmdq); 203 if (ret) 204 return ret; 205 206 cmdq->db_registered = true; 207 208 return 0; 209 } 210 211 static int ivpu_cmdq_fini(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 212 { 213 struct ivpu_device *vdev = file_priv->vdev; 214 int ret; 215 216 lockdep_assert_held(&file_priv->lock); 217 218 if (!cmdq->db_registered) 219 return 0; 220 221 cmdq->db_registered = false; 222 223 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 224 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); 225 if (!ret) 226 ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->id); 227 } 228 229 ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id); 230 if (!ret) 231 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id); 232 233 return 0; 234 } 235 236 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u8 priority) 237 { 238 struct ivpu_cmdq *cmdq; 239 unsigned long cmdq_id; 240 int ret; 241 242 lockdep_assert_held(&file_priv->lock); 243 244 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) 245 if (cmdq->priority == priority) 246 break; 247 248 if (!cmdq) { 249 cmdq = ivpu_cmdq_alloc(file_priv); 250 if (!cmdq) 251 return NULL; 252 cmdq->priority = priority; 253 } 254 255 ret = ivpu_cmdq_init(file_priv, cmdq, priority); 256 if (ret) 257 return NULL; 258 259 return cmdq; 260 } 261 262 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv) 263 { 264 struct ivpu_cmdq *cmdq; 265 unsigned long cmdq_id; 266 267 lockdep_assert_held(&file_priv->lock); 268 269 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) { 270 xa_erase(&file_priv->cmdq_xa, cmdq_id); 271 ivpu_cmdq_fini(file_priv, cmdq); 272 ivpu_cmdq_free(file_priv, cmdq); 273 } 274 } 275 276 /* 277 * Mark the doorbell as unregistered 278 * This function needs to be called when the VPU hardware is restarted 279 * and FW loses job queue state. The next time job queue is used it 280 * will be registered again. 281 */ 282 static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv) 283 { 284 struct ivpu_cmdq *cmdq; 285 unsigned long cmdq_id; 286 287 mutex_lock(&file_priv->lock); 288 289 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) 290 cmdq->db_registered = false; 291 292 mutex_unlock(&file_priv->lock); 293 } 294 295 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev) 296 { 297 struct ivpu_file_priv *file_priv; 298 unsigned long ctx_id; 299 300 mutex_lock(&vdev->context_list_lock); 301 302 xa_for_each(&vdev->context_xa, ctx_id, file_priv) 303 ivpu_cmdq_reset(file_priv); 304 305 mutex_unlock(&vdev->context_list_lock); 306 } 307 308 static void ivpu_cmdq_fini_all(struct ivpu_file_priv *file_priv) 309 { 310 struct ivpu_cmdq *cmdq; 311 unsigned long cmdq_id; 312 313 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) 314 ivpu_cmdq_fini(file_priv, cmdq); 315 } 316 317 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv) 318 { 319 struct ivpu_device *vdev = file_priv->vdev; 320 321 lockdep_assert_held(&file_priv->lock); 322 323 ivpu_cmdq_fini_all(file_priv); 324 325 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_OS) 326 ivpu_jsm_context_release(vdev, file_priv->ctx.id); 327 } 328 329 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job) 330 { 331 struct ivpu_device *vdev = job->vdev; 332 struct vpu_job_queue_header *header = &cmdq->jobq->header; 333 struct vpu_job_queue_entry *entry; 334 u32 tail = READ_ONCE(header->tail); 335 u32 next_entry = (tail + 1) % cmdq->entry_count; 336 337 /* Check if there is space left in job queue */ 338 if (next_entry == header->head) { 339 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n", 340 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail); 341 return -EBUSY; 342 } 343 344 entry = &cmdq->jobq->slot[tail].job; 345 entry->batch_buf_addr = job->cmd_buf_vpu_addr; 346 entry->job_id = job->job_id; 347 entry->flags = 0; 348 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION)) 349 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK; 350 351 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 352 if (cmdq->primary_preempt_buf) { 353 entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr; 354 entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf); 355 } 356 357 if (cmdq->secondary_preempt_buf) { 358 entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr; 359 entry->secondary_preempt_buf_size = 360 ivpu_bo_size(cmdq->secondary_preempt_buf); 361 } 362 } 363 364 wmb(); /* Ensure that tail is updated after filling entry */ 365 header->tail = next_entry; 366 wmb(); /* Flush WC buffer for jobq header */ 367 368 return 0; 369 } 370 371 struct ivpu_fence { 372 struct dma_fence base; 373 spinlock_t lock; /* protects base */ 374 struct ivpu_device *vdev; 375 }; 376 377 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence) 378 { 379 return container_of(fence, struct ivpu_fence, base); 380 } 381 382 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence) 383 { 384 return DRIVER_NAME; 385 } 386 387 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence) 388 { 389 struct ivpu_fence *ivpu_fence = to_vpu_fence(fence); 390 391 return dev_name(ivpu_fence->vdev->drm.dev); 392 } 393 394 static const struct dma_fence_ops ivpu_fence_ops = { 395 .get_driver_name = ivpu_fence_get_driver_name, 396 .get_timeline_name = ivpu_fence_get_timeline_name, 397 }; 398 399 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev) 400 { 401 struct ivpu_fence *fence; 402 403 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 404 if (!fence) 405 return NULL; 406 407 fence->vdev = vdev; 408 spin_lock_init(&fence->lock); 409 dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1); 410 411 return &fence->base; 412 } 413 414 static void ivpu_job_destroy(struct ivpu_job *job) 415 { 416 struct ivpu_device *vdev = job->vdev; 417 u32 i; 418 419 ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d", 420 job->job_id, job->file_priv->ctx.id, job->engine_idx); 421 422 for (i = 0; i < job->bo_count; i++) 423 if (job->bos[i]) 424 drm_gem_object_put(&job->bos[i]->base.base); 425 426 dma_fence_put(job->done_fence); 427 ivpu_file_priv_put(&job->file_priv); 428 kfree(job); 429 } 430 431 static struct ivpu_job * 432 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) 433 { 434 struct ivpu_device *vdev = file_priv->vdev; 435 struct ivpu_job *job; 436 437 job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL); 438 if (!job) 439 return NULL; 440 441 job->vdev = vdev; 442 job->engine_idx = engine_idx; 443 job->bo_count = bo_count; 444 job->done_fence = ivpu_fence_create(vdev); 445 if (!job->done_fence) { 446 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n"); 447 goto err_free_job; 448 } 449 450 job->file_priv = ivpu_file_priv_get(file_priv); 451 452 trace_job("create", job); 453 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx); 454 return job; 455 456 err_free_job: 457 kfree(job); 458 return NULL; 459 } 460 461 static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id) 462 { 463 struct ivpu_job *job; 464 465 xa_lock(&vdev->submitted_jobs_xa); 466 job = __xa_erase(&vdev->submitted_jobs_xa, job_id); 467 468 if (xa_empty(&vdev->submitted_jobs_xa) && job) { 469 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts), 470 vdev->busy_time); 471 } 472 473 xa_unlock(&vdev->submitted_jobs_xa); 474 475 return job; 476 } 477 478 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) 479 { 480 struct ivpu_job *job; 481 482 job = ivpu_job_remove_from_submitted_jobs(vdev, job_id); 483 if (!job) 484 return -ENOENT; 485 486 if (job->file_priv->has_mmu_faults) 487 job_status = DRM_IVPU_JOB_STATUS_ABORTED; 488 489 job->bos[CMD_BUF_IDX]->job_status = job_status; 490 dma_fence_signal(job->done_fence); 491 492 trace_job("done", job); 493 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n", 494 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status); 495 496 ivpu_job_destroy(job); 497 ivpu_stop_job_timeout_detection(vdev); 498 499 ivpu_rpm_put(vdev); 500 return 0; 501 } 502 503 void ivpu_jobs_abort_all(struct ivpu_device *vdev) 504 { 505 struct ivpu_job *job; 506 unsigned long id; 507 508 xa_for_each(&vdev->submitted_jobs_xa, id, job) 509 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); 510 } 511 512 static int ivpu_job_submit(struct ivpu_job *job, u8 priority) 513 { 514 struct ivpu_file_priv *file_priv = job->file_priv; 515 struct ivpu_device *vdev = job->vdev; 516 struct ivpu_cmdq *cmdq; 517 bool is_first_job; 518 int ret; 519 520 ret = ivpu_rpm_get(vdev); 521 if (ret < 0) 522 return ret; 523 524 mutex_lock(&file_priv->lock); 525 526 cmdq = ivpu_cmdq_acquire(file_priv, priority); 527 if (!cmdq) { 528 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d engine %d prio %d\n", 529 file_priv->ctx.id, job->engine_idx, priority); 530 ret = -EINVAL; 531 goto err_unlock_file_priv; 532 } 533 534 xa_lock(&vdev->submitted_jobs_xa); 535 is_first_job = xa_empty(&vdev->submitted_jobs_xa); 536 ret = __xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit, 537 &file_priv->job_id_next, GFP_KERNEL); 538 if (ret < 0) { 539 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n", 540 file_priv->ctx.id); 541 ret = -EBUSY; 542 goto err_unlock_submitted_jobs_xa; 543 } 544 545 ret = ivpu_cmdq_push_job(cmdq, job); 546 if (ret) 547 goto err_erase_xa; 548 549 ivpu_start_job_timeout_detection(vdev); 550 551 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) { 552 cmdq->jobq->header.head = cmdq->jobq->header.tail; 553 wmb(); /* Flush WC buffer for jobq header */ 554 } else { 555 ivpu_cmdq_ring_db(vdev, cmdq); 556 if (is_first_job) 557 vdev->busy_start_ts = ktime_get(); 558 } 559 560 trace_job("submit", job); 561 ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n", 562 job->job_id, file_priv->ctx.id, job->engine_idx, priority, 563 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail); 564 565 xa_unlock(&vdev->submitted_jobs_xa); 566 567 mutex_unlock(&file_priv->lock); 568 569 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) 570 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS); 571 572 return 0; 573 574 err_erase_xa: 575 __xa_erase(&vdev->submitted_jobs_xa, job->job_id); 576 err_unlock_submitted_jobs_xa: 577 xa_unlock(&vdev->submitted_jobs_xa); 578 err_unlock_file_priv: 579 mutex_unlock(&file_priv->lock); 580 ivpu_rpm_put(vdev); 581 return ret; 582 } 583 584 static int 585 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles, 586 u32 buf_count, u32 commands_offset) 587 { 588 struct ivpu_file_priv *file_priv = file->driver_priv; 589 struct ivpu_device *vdev = file_priv->vdev; 590 struct ww_acquire_ctx acquire_ctx; 591 enum dma_resv_usage usage; 592 struct ivpu_bo *bo; 593 int ret; 594 u32 i; 595 596 for (i = 0; i < buf_count; i++) { 597 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]); 598 599 if (!obj) 600 return -ENOENT; 601 602 job->bos[i] = to_ivpu_bo(obj); 603 604 ret = ivpu_bo_pin(job->bos[i]); 605 if (ret) 606 return ret; 607 } 608 609 bo = job->bos[CMD_BUF_IDX]; 610 if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) { 611 ivpu_warn(vdev, "Buffer is already in use\n"); 612 return -EBUSY; 613 } 614 615 if (commands_offset >= ivpu_bo_size(bo)) { 616 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset); 617 return -EINVAL; 618 } 619 620 job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset; 621 622 ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count, 623 &acquire_ctx); 624 if (ret) { 625 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret); 626 return ret; 627 } 628 629 for (i = 0; i < buf_count; i++) { 630 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1); 631 if (ret) { 632 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret); 633 goto unlock_reservations; 634 } 635 } 636 637 for (i = 0; i < buf_count; i++) { 638 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP; 639 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage); 640 } 641 642 unlock_reservations: 643 drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx); 644 645 wmb(); /* Flush write combining buffers */ 646 647 return ret; 648 } 649 650 static inline u8 ivpu_job_to_hws_priority(struct ivpu_file_priv *file_priv, u8 priority) 651 { 652 if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT) 653 return DRM_IVPU_JOB_PRIORITY_NORMAL; 654 655 return priority - 1; 656 } 657 658 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 659 { 660 struct ivpu_file_priv *file_priv = file->driver_priv; 661 struct ivpu_device *vdev = file_priv->vdev; 662 struct drm_ivpu_submit *params = data; 663 struct ivpu_job *job; 664 u32 *buf_handles; 665 int idx, ret; 666 u8 priority; 667 668 if (params->engine != DRM_IVPU_ENGINE_COMPUTE) 669 return -EINVAL; 670 671 if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME) 672 return -EINVAL; 673 674 if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT) 675 return -EINVAL; 676 677 if (!IS_ALIGNED(params->commands_offset, 8)) 678 return -EINVAL; 679 680 if (!file_priv->ctx.id) 681 return -EINVAL; 682 683 if (file_priv->has_mmu_faults) 684 return -EBADFD; 685 686 buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL); 687 if (!buf_handles) 688 return -ENOMEM; 689 690 ret = copy_from_user(buf_handles, 691 (void __user *)params->buffers_ptr, 692 params->buffer_count * sizeof(u32)); 693 if (ret) { 694 ret = -EFAULT; 695 goto err_free_handles; 696 } 697 698 if (!drm_dev_enter(&vdev->drm, &idx)) { 699 ret = -ENODEV; 700 goto err_free_handles; 701 } 702 703 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n", 704 file_priv->ctx.id, params->buffer_count); 705 706 job = ivpu_job_create(file_priv, params->engine, params->buffer_count); 707 if (!job) { 708 ivpu_err(vdev, "Failed to create job\n"); 709 ret = -ENOMEM; 710 goto err_exit_dev; 711 } 712 713 ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count, 714 params->commands_offset); 715 if (ret) { 716 ivpu_err(vdev, "Failed to prepare job: %d\n", ret); 717 goto err_destroy_job; 718 } 719 720 priority = ivpu_job_to_hws_priority(file_priv, params->priority); 721 722 down_read(&vdev->pm->reset_lock); 723 ret = ivpu_job_submit(job, priority); 724 up_read(&vdev->pm->reset_lock); 725 if (ret) 726 goto err_signal_fence; 727 728 drm_dev_exit(idx); 729 kfree(buf_handles); 730 return ret; 731 732 err_signal_fence: 733 dma_fence_signal(job->done_fence); 734 err_destroy_job: 735 ivpu_job_destroy(job); 736 err_exit_dev: 737 drm_dev_exit(idx); 738 err_free_handles: 739 kfree(buf_handles); 740 return ret; 741 } 742 743 static void 744 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, 745 struct vpu_jsm_msg *jsm_msg) 746 { 747 struct vpu_ipc_msg_payload_job_done *payload; 748 int ret; 749 750 if (!jsm_msg) { 751 ivpu_err(vdev, "IPC message has no JSM payload\n"); 752 return; 753 } 754 755 if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) { 756 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result); 757 return; 758 } 759 760 payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload; 761 ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status); 762 if (!ret && !xa_empty(&vdev->submitted_jobs_xa)) 763 ivpu_start_job_timeout_detection(vdev); 764 } 765 766 void ivpu_job_done_consumer_init(struct ivpu_device *vdev) 767 { 768 ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer, 769 VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback); 770 } 771 772 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev) 773 { 774 ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer); 775 } 776