1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2024 Intel Corporation 4 */ 5 6 #include <drm/drm_file.h> 7 8 #include <linux/bitfield.h> 9 #include <linux/highmem.h> 10 #include <linux/pci.h> 11 #include <linux/pm_runtime.h> 12 #include <linux/module.h> 13 #include <uapi/drm/ivpu_accel.h> 14 15 #include "ivpu_drv.h" 16 #include "ivpu_fw.h" 17 #include "ivpu_hw.h" 18 #include "ivpu_ipc.h" 19 #include "ivpu_job.h" 20 #include "ivpu_jsm_msg.h" 21 #include "ivpu_mmu.h" 22 #include "ivpu_pm.h" 23 #include "ivpu_trace.h" 24 #include "vpu_boot_api.h" 25 26 #define CMD_BUF_IDX 0 27 #define JOB_MAX_BUFFER_COUNT 65535 28 29 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) 30 { 31 ivpu_hw_db_set(vdev, cmdq->db_id); 32 } 33 34 static int ivpu_preemption_buffers_create(struct ivpu_device *vdev, 35 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 36 { 37 u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE); 38 u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE); 39 40 if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW || 41 ivpu_test_mode & IVPU_TEST_MODE_MIP_DISABLE) 42 return 0; 43 44 cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user, 45 primary_size, DRM_IVPU_BO_WC); 46 if (!cmdq->primary_preempt_buf) { 47 ivpu_err(vdev, "Failed to create primary preemption buffer\n"); 48 return -ENOMEM; 49 } 50 51 cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma, 52 secondary_size, DRM_IVPU_BO_WC); 53 if (!cmdq->secondary_preempt_buf) { 54 ivpu_err(vdev, "Failed to create secondary preemption buffer\n"); 55 goto err_free_primary; 56 } 57 58 return 0; 59 60 err_free_primary: 61 ivpu_bo_free(cmdq->primary_preempt_buf); 62 cmdq->primary_preempt_buf = NULL; 63 return -ENOMEM; 64 } 65 66 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev, 67 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 68 { 69 if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW) 70 return; 71 72 if (cmdq->primary_preempt_buf) 73 ivpu_bo_free(cmdq->primary_preempt_buf); 74 if (cmdq->secondary_preempt_buf) 75 ivpu_bo_free(cmdq->secondary_preempt_buf); 76 } 77 78 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv) 79 { 80 struct ivpu_device *vdev = file_priv->vdev; 81 struct ivpu_cmdq *cmdq; 82 int ret; 83 84 cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL); 85 if (!cmdq) 86 return NULL; 87 88 cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); 89 if (!cmdq->mem) 90 goto err_free_cmdq; 91 92 ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq); 93 if (ret) 94 ivpu_warn(vdev, "Failed to allocate preemption buffers, preemption limited\n"); 95 96 return cmdq; 97 98 err_free_cmdq: 99 kfree(cmdq); 100 return NULL; 101 } 102 103 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 104 { 105 ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq); 106 ivpu_bo_free(cmdq->mem); 107 kfree(cmdq); 108 } 109 110 static struct ivpu_cmdq *ivpu_cmdq_create(struct ivpu_file_priv *file_priv, u8 priority, 111 bool is_legacy) 112 { 113 struct ivpu_device *vdev = file_priv->vdev; 114 struct ivpu_cmdq *cmdq = NULL; 115 int ret; 116 117 lockdep_assert_held(&file_priv->lock); 118 119 cmdq = ivpu_cmdq_alloc(file_priv); 120 if (!cmdq) { 121 ivpu_err(vdev, "Failed to allocate command queue\n"); 122 return NULL; 123 } 124 125 cmdq->priority = priority; 126 cmdq->is_legacy = is_legacy; 127 128 ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit, 129 &file_priv->cmdq_id_next, GFP_KERNEL); 130 if (ret < 0) { 131 ivpu_err(vdev, "Failed to allocate command queue ID: %d\n", ret); 132 goto err_free_cmdq; 133 } 134 135 ivpu_dbg(vdev, JOB, "Command queue %d created, ctx %d\n", cmdq->id, file_priv->ctx.id); 136 return cmdq; 137 138 err_free_cmdq: 139 ivpu_cmdq_free(file_priv, cmdq); 140 return NULL; 141 } 142 143 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, 144 u8 priority) 145 { 146 struct ivpu_device *vdev = file_priv->vdev; 147 int ret; 148 149 ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id, 150 task_pid_nr(current), engine, 151 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 152 if (ret) 153 return ret; 154 155 ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id, 156 priority); 157 if (ret) 158 return ret; 159 160 return 0; 161 } 162 163 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 164 { 165 struct ivpu_device *vdev = file_priv->vdev; 166 int ret; 167 168 ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next, 169 GFP_KERNEL); 170 if (ret < 0) { 171 ivpu_err(vdev, "Failed to allocate doorbell ID: %d\n", ret); 172 return ret; 173 } 174 175 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) 176 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id, 177 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 178 else 179 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id, 180 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); 181 182 if (!ret) 183 ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n", 184 cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority); 185 else 186 xa_erase(&vdev->db_xa, cmdq->db_id); 187 188 return ret; 189 } 190 191 static void ivpu_cmdq_jobq_init(struct ivpu_device *vdev, struct vpu_job_queue *jobq) 192 { 193 jobq->header.engine_idx = VPU_ENGINE_COMPUTE; 194 jobq->header.head = 0; 195 jobq->header.tail = 0; 196 197 if (ivpu_test_mode & IVPU_TEST_MODE_TURBO) { 198 ivpu_dbg(vdev, JOB, "Turbo mode enabled"); 199 jobq->header.flags = VPU_JOB_QUEUE_FLAGS_TURBO_MODE; 200 } 201 202 wmb(); /* Flush WC buffer for jobq->header */ 203 } 204 205 static inline u32 ivpu_cmdq_get_entry_count(struct ivpu_cmdq *cmdq) 206 { 207 size_t size = ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header); 208 209 return size / sizeof(struct vpu_job_queue_entry); 210 } 211 212 static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 213 { 214 struct ivpu_device *vdev = file_priv->vdev; 215 int ret; 216 217 lockdep_assert_held(&file_priv->lock); 218 219 if (cmdq->db_id) 220 return 0; 221 222 cmdq->entry_count = ivpu_cmdq_get_entry_count(cmdq); 223 cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem); 224 225 ivpu_cmdq_jobq_init(vdev, cmdq->jobq); 226 227 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 228 ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, cmdq->priority); 229 if (ret) 230 return ret; 231 } 232 233 ret = ivpu_register_db(file_priv, cmdq); 234 if (ret) 235 return ret; 236 237 return 0; 238 } 239 240 static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 241 { 242 struct ivpu_device *vdev = file_priv->vdev; 243 int ret; 244 245 lockdep_assert_held(&file_priv->lock); 246 247 if (!cmdq->db_id) 248 return 0; 249 250 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 251 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); 252 if (!ret) 253 ivpu_dbg(vdev, JOB, "Command queue %d destroyed, ctx %d\n", 254 cmdq->id, file_priv->ctx.id); 255 } 256 257 ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id); 258 if (!ret) 259 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id); 260 261 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); 262 cmdq->db_id = 0; 263 264 return 0; 265 } 266 267 static inline u8 ivpu_job_to_jsm_priority(u8 priority) 268 { 269 if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT) 270 return VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL; 271 272 return priority - 1; 273 } 274 275 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) 276 { 277 ivpu_cmdq_unregister(file_priv, cmdq); 278 xa_erase(&file_priv->cmdq_xa, cmdq->id); 279 ivpu_cmdq_free(file_priv, cmdq); 280 } 281 282 static struct ivpu_cmdq *ivpu_cmdq_acquire_legacy(struct ivpu_file_priv *file_priv, u8 priority) 283 { 284 struct ivpu_cmdq *cmdq; 285 unsigned long id; 286 287 lockdep_assert_held(&file_priv->lock); 288 289 xa_for_each(&file_priv->cmdq_xa, id, cmdq) 290 if (cmdq->is_legacy && cmdq->priority == priority) 291 break; 292 293 if (!cmdq) { 294 cmdq = ivpu_cmdq_create(file_priv, priority, true); 295 if (!cmdq) 296 return NULL; 297 } 298 299 return cmdq; 300 } 301 302 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u32 cmdq_id) 303 { 304 struct ivpu_device *vdev = file_priv->vdev; 305 struct ivpu_cmdq *cmdq; 306 307 lockdep_assert_held(&file_priv->lock); 308 309 cmdq = xa_load(&file_priv->cmdq_xa, cmdq_id); 310 if (!cmdq) { 311 ivpu_warn_ratelimited(vdev, "Failed to find command queue with ID: %u\n", cmdq_id); 312 return NULL; 313 } 314 315 return cmdq; 316 } 317 318 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv) 319 { 320 struct ivpu_cmdq *cmdq; 321 unsigned long cmdq_id; 322 323 lockdep_assert_held(&file_priv->lock); 324 325 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) 326 ivpu_cmdq_destroy(file_priv, cmdq); 327 } 328 329 /* 330 * Mark the doorbell as unregistered 331 * This function needs to be called when the VPU hardware is restarted 332 * and FW loses job queue state. The next time job queue is used it 333 * will be registered again. 334 */ 335 static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv) 336 { 337 struct ivpu_cmdq *cmdq; 338 unsigned long cmdq_id; 339 340 mutex_lock(&file_priv->lock); 341 342 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) { 343 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); 344 cmdq->db_id = 0; 345 } 346 347 mutex_unlock(&file_priv->lock); 348 } 349 350 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev) 351 { 352 struct ivpu_file_priv *file_priv; 353 unsigned long ctx_id; 354 355 mutex_lock(&vdev->context_list_lock); 356 357 xa_for_each(&vdev->context_xa, ctx_id, file_priv) 358 ivpu_cmdq_reset(file_priv); 359 360 mutex_unlock(&vdev->context_list_lock); 361 } 362 363 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv) 364 { 365 struct ivpu_device *vdev = file_priv->vdev; 366 struct ivpu_cmdq *cmdq; 367 unsigned long cmdq_id; 368 369 lockdep_assert_held(&file_priv->lock); 370 ivpu_dbg(vdev, JOB, "Context ID: %u abort\n", file_priv->ctx.id); 371 372 xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) 373 ivpu_cmdq_unregister(file_priv, cmdq); 374 375 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_OS) 376 ivpu_jsm_context_release(vdev, file_priv->ctx.id); 377 378 ivpu_mmu_disable_ssid_events(vdev, file_priv->ctx.id); 379 380 file_priv->aborted = true; 381 } 382 383 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job) 384 { 385 struct ivpu_device *vdev = job->vdev; 386 struct vpu_job_queue_header *header = &cmdq->jobq->header; 387 struct vpu_job_queue_entry *entry; 388 u32 tail = READ_ONCE(header->tail); 389 u32 next_entry = (tail + 1) % cmdq->entry_count; 390 391 /* Check if there is space left in job queue */ 392 if (next_entry == header->head) { 393 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n", 394 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail); 395 return -EBUSY; 396 } 397 398 entry = &cmdq->jobq->slot[tail].job; 399 entry->batch_buf_addr = job->cmd_buf_vpu_addr; 400 entry->job_id = job->job_id; 401 entry->flags = 0; 402 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION)) 403 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK; 404 405 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { 406 if (cmdq->primary_preempt_buf) { 407 entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr; 408 entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf); 409 } 410 411 if (cmdq->secondary_preempt_buf) { 412 entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr; 413 entry->secondary_preempt_buf_size = 414 ivpu_bo_size(cmdq->secondary_preempt_buf); 415 } 416 } 417 418 wmb(); /* Ensure that tail is updated after filling entry */ 419 header->tail = next_entry; 420 wmb(); /* Flush WC buffer for jobq header */ 421 422 return 0; 423 } 424 425 struct ivpu_fence { 426 struct dma_fence base; 427 spinlock_t lock; /* protects base */ 428 struct ivpu_device *vdev; 429 }; 430 431 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence) 432 { 433 return container_of(fence, struct ivpu_fence, base); 434 } 435 436 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence) 437 { 438 return DRIVER_NAME; 439 } 440 441 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence) 442 { 443 struct ivpu_fence *ivpu_fence = to_vpu_fence(fence); 444 445 return dev_name(ivpu_fence->vdev->drm.dev); 446 } 447 448 static const struct dma_fence_ops ivpu_fence_ops = { 449 .get_driver_name = ivpu_fence_get_driver_name, 450 .get_timeline_name = ivpu_fence_get_timeline_name, 451 }; 452 453 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev) 454 { 455 struct ivpu_fence *fence; 456 457 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 458 if (!fence) 459 return NULL; 460 461 fence->vdev = vdev; 462 spin_lock_init(&fence->lock); 463 dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1); 464 465 return &fence->base; 466 } 467 468 static void ivpu_job_destroy(struct ivpu_job *job) 469 { 470 struct ivpu_device *vdev = job->vdev; 471 u32 i; 472 473 ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d cmdq_id %u engine %d", 474 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx); 475 476 for (i = 0; i < job->bo_count; i++) 477 if (job->bos[i]) 478 drm_gem_object_put(&job->bos[i]->base.base); 479 480 dma_fence_put(job->done_fence); 481 ivpu_file_priv_put(&job->file_priv); 482 kfree(job); 483 } 484 485 static struct ivpu_job * 486 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) 487 { 488 struct ivpu_device *vdev = file_priv->vdev; 489 struct ivpu_job *job; 490 491 job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL); 492 if (!job) 493 return NULL; 494 495 job->vdev = vdev; 496 job->engine_idx = engine_idx; 497 job->bo_count = bo_count; 498 job->done_fence = ivpu_fence_create(vdev); 499 if (!job->done_fence) { 500 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n"); 501 goto err_free_job; 502 } 503 504 job->file_priv = ivpu_file_priv_get(file_priv); 505 506 trace_job("create", job); 507 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx); 508 return job; 509 510 err_free_job: 511 kfree(job); 512 return NULL; 513 } 514 515 static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id) 516 { 517 struct ivpu_job *job; 518 519 lockdep_assert_held(&vdev->submitted_jobs_lock); 520 521 job = xa_erase(&vdev->submitted_jobs_xa, job_id); 522 if (xa_empty(&vdev->submitted_jobs_xa) && job) { 523 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts), 524 vdev->busy_time); 525 } 526 527 return job; 528 } 529 530 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) 531 { 532 struct ivpu_job *job; 533 534 lockdep_assert_held(&vdev->submitted_jobs_lock); 535 536 job = xa_load(&vdev->submitted_jobs_xa, job_id); 537 if (!job) 538 return -ENOENT; 539 540 if (job_status == VPU_JSM_STATUS_MVNCI_CONTEXT_VIOLATION_HW) { 541 guard(mutex)(&job->file_priv->lock); 542 543 if (job->file_priv->has_mmu_faults) 544 return 0; 545 546 /* 547 * Mark context as faulty and defer destruction of the job to jobs abort thread 548 * handler to synchronize between both faults and jobs returning context violation 549 * status and ensure both are handled in the same way 550 */ 551 job->file_priv->has_mmu_faults = true; 552 queue_work(system_wq, &vdev->context_abort_work); 553 return 0; 554 } 555 556 job = ivpu_job_remove_from_submitted_jobs(vdev, job_id); 557 if (!job) 558 return -ENOENT; 559 560 if (job->file_priv->has_mmu_faults) 561 job_status = DRM_IVPU_JOB_STATUS_ABORTED; 562 563 job->bos[CMD_BUF_IDX]->job_status = job_status; 564 dma_fence_signal(job->done_fence); 565 566 trace_job("done", job); 567 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d cmdq_id %u engine %d status 0x%x\n", 568 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx, job_status); 569 570 ivpu_job_destroy(job); 571 ivpu_stop_job_timeout_detection(vdev); 572 573 ivpu_rpm_put(vdev); 574 575 if (!xa_empty(&vdev->submitted_jobs_xa)) 576 ivpu_start_job_timeout_detection(vdev); 577 578 return 0; 579 } 580 581 void ivpu_jobs_abort_all(struct ivpu_device *vdev) 582 { 583 struct ivpu_job *job; 584 unsigned long id; 585 586 mutex_lock(&vdev->submitted_jobs_lock); 587 588 xa_for_each(&vdev->submitted_jobs_xa, id, job) 589 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); 590 591 mutex_unlock(&vdev->submitted_jobs_lock); 592 } 593 594 void ivpu_cmdq_abort_all_jobs(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id) 595 { 596 struct ivpu_job *job; 597 unsigned long id; 598 599 mutex_lock(&vdev->submitted_jobs_lock); 600 601 xa_for_each(&vdev->submitted_jobs_xa, id, job) 602 if (job->file_priv->ctx.id == ctx_id && job->cmdq_id == cmdq_id) 603 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); 604 605 mutex_unlock(&vdev->submitted_jobs_lock); 606 } 607 608 static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id) 609 { 610 struct ivpu_file_priv *file_priv = job->file_priv; 611 struct ivpu_device *vdev = job->vdev; 612 struct ivpu_cmdq *cmdq; 613 bool is_first_job; 614 int ret; 615 616 ret = ivpu_rpm_get(vdev); 617 if (ret < 0) 618 return ret; 619 620 mutex_lock(&vdev->submitted_jobs_lock); 621 mutex_lock(&file_priv->lock); 622 623 if (cmdq_id == 0) 624 cmdq = ivpu_cmdq_acquire_legacy(file_priv, priority); 625 else 626 cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id); 627 if (!cmdq) { 628 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d\n", file_priv->ctx.id); 629 ret = -EINVAL; 630 goto err_unlock; 631 } 632 633 ret = ivpu_cmdq_register(file_priv, cmdq); 634 if (ret) { 635 ivpu_err(vdev, "Failed to register command queue: %d\n", ret); 636 goto err_unlock; 637 } 638 639 job->cmdq_id = cmdq->id; 640 641 is_first_job = xa_empty(&vdev->submitted_jobs_xa); 642 ret = xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit, 643 &file_priv->job_id_next, GFP_KERNEL); 644 if (ret < 0) { 645 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n", 646 file_priv->ctx.id); 647 ret = -EBUSY; 648 goto err_unlock; 649 } 650 651 ret = ivpu_cmdq_push_job(cmdq, job); 652 if (ret) 653 goto err_erase_xa; 654 655 ivpu_start_job_timeout_detection(vdev); 656 657 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) { 658 cmdq->jobq->header.head = cmdq->jobq->header.tail; 659 wmb(); /* Flush WC buffer for jobq header */ 660 } else { 661 ivpu_cmdq_ring_db(vdev, cmdq); 662 if (is_first_job) 663 vdev->busy_start_ts = ktime_get(); 664 } 665 666 trace_job("submit", job); 667 ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d cmdq_id %u engine %d prio %d addr 0x%llx next %d\n", 668 job->job_id, file_priv->ctx.id, cmdq->id, job->engine_idx, cmdq->priority, 669 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail); 670 671 mutex_unlock(&file_priv->lock); 672 673 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) { 674 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS); 675 } 676 677 mutex_unlock(&vdev->submitted_jobs_lock); 678 679 return 0; 680 681 err_erase_xa: 682 xa_erase(&vdev->submitted_jobs_xa, job->job_id); 683 err_unlock: 684 mutex_unlock(&vdev->submitted_jobs_lock); 685 mutex_unlock(&file_priv->lock); 686 ivpu_rpm_put(vdev); 687 return ret; 688 } 689 690 static int 691 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles, 692 u32 buf_count, u32 commands_offset) 693 { 694 struct ivpu_file_priv *file_priv = job->file_priv; 695 struct ivpu_device *vdev = file_priv->vdev; 696 struct ww_acquire_ctx acquire_ctx; 697 enum dma_resv_usage usage; 698 struct ivpu_bo *bo; 699 int ret; 700 u32 i; 701 702 for (i = 0; i < buf_count; i++) { 703 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]); 704 705 if (!obj) 706 return -ENOENT; 707 708 job->bos[i] = to_ivpu_bo(obj); 709 710 ret = ivpu_bo_pin(job->bos[i]); 711 if (ret) 712 return ret; 713 } 714 715 bo = job->bos[CMD_BUF_IDX]; 716 if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) { 717 ivpu_warn(vdev, "Buffer is already in use\n"); 718 return -EBUSY; 719 } 720 721 if (commands_offset >= ivpu_bo_size(bo)) { 722 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset); 723 return -EINVAL; 724 } 725 726 job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset; 727 728 ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count, 729 &acquire_ctx); 730 if (ret) { 731 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret); 732 return ret; 733 } 734 735 for (i = 0; i < buf_count; i++) { 736 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1); 737 if (ret) { 738 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret); 739 goto unlock_reservations; 740 } 741 } 742 743 for (i = 0; i < buf_count; i++) { 744 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP; 745 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage); 746 } 747 748 unlock_reservations: 749 drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx); 750 751 wmb(); /* Flush write combining buffers */ 752 753 return ret; 754 } 755 756 static int ivpu_submit(struct drm_file *file, struct ivpu_file_priv *file_priv, u32 cmdq_id, 757 u32 buffer_count, u32 engine, void __user *buffers_ptr, u32 cmds_offset, 758 u8 priority) 759 { 760 struct ivpu_device *vdev = file_priv->vdev; 761 struct ivpu_job *job; 762 u32 *buf_handles; 763 int idx, ret; 764 765 buf_handles = kcalloc(buffer_count, sizeof(u32), GFP_KERNEL); 766 if (!buf_handles) 767 return -ENOMEM; 768 769 ret = copy_from_user(buf_handles, buffers_ptr, buffer_count * sizeof(u32)); 770 if (ret) { 771 ret = -EFAULT; 772 goto err_free_handles; 773 } 774 775 if (!drm_dev_enter(&vdev->drm, &idx)) { 776 ret = -ENODEV; 777 goto err_free_handles; 778 } 779 780 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u cmdq_id %u buf_count %u\n", 781 file_priv->ctx.id, cmdq_id, buffer_count); 782 783 job = ivpu_job_create(file_priv, engine, buffer_count); 784 if (!job) { 785 ivpu_err(vdev, "Failed to create job\n"); 786 ret = -ENOMEM; 787 goto err_exit_dev; 788 } 789 790 ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, buffer_count, cmds_offset); 791 if (ret) { 792 ivpu_err(vdev, "Failed to prepare job: %d\n", ret); 793 goto err_destroy_job; 794 } 795 796 down_read(&vdev->pm->reset_lock); 797 ret = ivpu_job_submit(job, priority, cmdq_id); 798 up_read(&vdev->pm->reset_lock); 799 if (ret) 800 goto err_signal_fence; 801 802 drm_dev_exit(idx); 803 kfree(buf_handles); 804 return ret; 805 806 err_signal_fence: 807 dma_fence_signal(job->done_fence); 808 err_destroy_job: 809 ivpu_job_destroy(job); 810 err_exit_dev: 811 drm_dev_exit(idx); 812 err_free_handles: 813 kfree(buf_handles); 814 return ret; 815 } 816 817 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 818 { 819 struct ivpu_file_priv *file_priv = file->driver_priv; 820 struct drm_ivpu_submit *args = data; 821 u8 priority; 822 823 if (args->engine != DRM_IVPU_ENGINE_COMPUTE) 824 return -EINVAL; 825 826 if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME) 827 return -EINVAL; 828 829 if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT) 830 return -EINVAL; 831 832 if (!IS_ALIGNED(args->commands_offset, 8)) 833 return -EINVAL; 834 835 if (!file_priv->ctx.id) 836 return -EINVAL; 837 838 if (file_priv->has_mmu_faults) 839 return -EBADFD; 840 841 priority = ivpu_job_to_jsm_priority(args->priority); 842 843 return ivpu_submit(file, file_priv, 0, args->buffer_count, args->engine, 844 (void __user *)args->buffers_ptr, args->commands_offset, priority); 845 } 846 847 int ivpu_cmdq_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 848 { 849 struct ivpu_file_priv *file_priv = file->driver_priv; 850 struct drm_ivpu_cmdq_submit *args = data; 851 852 if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) 853 return -ENODEV; 854 855 if (args->cmdq_id < IVPU_CMDQ_MIN_ID || args->cmdq_id > IVPU_CMDQ_MAX_ID) 856 return -EINVAL; 857 858 if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT) 859 return -EINVAL; 860 861 if (!IS_ALIGNED(args->commands_offset, 8)) 862 return -EINVAL; 863 864 if (!file_priv->ctx.id) 865 return -EINVAL; 866 867 if (file_priv->has_mmu_faults) 868 return -EBADFD; 869 870 return ivpu_submit(file, file_priv, args->cmdq_id, args->buffer_count, VPU_ENGINE_COMPUTE, 871 (void __user *)args->buffers_ptr, args->commands_offset, 0); 872 } 873 874 int ivpu_cmdq_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 875 { 876 struct ivpu_file_priv *file_priv = file->driver_priv; 877 struct drm_ivpu_cmdq_create *args = data; 878 struct ivpu_cmdq *cmdq; 879 880 if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) 881 return -ENODEV; 882 883 if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME) 884 return -EINVAL; 885 886 mutex_lock(&file_priv->lock); 887 888 cmdq = ivpu_cmdq_create(file_priv, ivpu_job_to_jsm_priority(args->priority), false); 889 if (cmdq) 890 args->cmdq_id = cmdq->id; 891 892 mutex_unlock(&file_priv->lock); 893 894 return cmdq ? 0 : -ENOMEM; 895 } 896 897 int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 898 { 899 struct ivpu_file_priv *file_priv = file->driver_priv; 900 struct ivpu_device *vdev = file_priv->vdev; 901 struct drm_ivpu_cmdq_destroy *args = data; 902 struct ivpu_cmdq *cmdq; 903 u32 cmdq_id; 904 int ret; 905 906 if (!ivpu_is_capable(vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) 907 return -ENODEV; 908 909 mutex_lock(&file_priv->lock); 910 911 cmdq = xa_load(&file_priv->cmdq_xa, args->cmdq_id); 912 if (!cmdq || cmdq->is_legacy) { 913 ret = -ENOENT; 914 goto err_unlock; 915 } 916 917 cmdq_id = cmdq->id; 918 ivpu_cmdq_destroy(file_priv, cmdq); 919 mutex_unlock(&file_priv->lock); 920 ivpu_cmdq_abort_all_jobs(vdev, file_priv->ctx.id, cmdq_id); 921 return 0; 922 923 err_unlock: 924 mutex_unlock(&file_priv->lock); 925 return ret; 926 } 927 928 static void 929 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, 930 struct vpu_jsm_msg *jsm_msg) 931 { 932 struct vpu_ipc_msg_payload_job_done *payload; 933 934 if (!jsm_msg) { 935 ivpu_err(vdev, "IPC message has no JSM payload\n"); 936 return; 937 } 938 939 if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) { 940 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result); 941 return; 942 } 943 944 payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload; 945 946 mutex_lock(&vdev->submitted_jobs_lock); 947 ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status); 948 mutex_unlock(&vdev->submitted_jobs_lock); 949 } 950 951 void ivpu_job_done_consumer_init(struct ivpu_device *vdev) 952 { 953 ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer, 954 VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback); 955 } 956 957 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev) 958 { 959 ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer); 960 } 961 962 void ivpu_context_abort_work_fn(struct work_struct *work) 963 { 964 struct ivpu_device *vdev = container_of(work, struct ivpu_device, context_abort_work); 965 struct ivpu_file_priv *file_priv; 966 struct ivpu_job *job; 967 unsigned long ctx_id; 968 unsigned long id; 969 970 if (drm_WARN_ON(&vdev->drm, pm_runtime_get_if_active(vdev->drm.dev) <= 0)) 971 return; 972 973 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) 974 ivpu_jsm_reset_engine(vdev, 0); 975 976 mutex_lock(&vdev->context_list_lock); 977 xa_for_each(&vdev->context_xa, ctx_id, file_priv) { 978 if (!file_priv->has_mmu_faults || file_priv->aborted) 979 continue; 980 981 mutex_lock(&file_priv->lock); 982 ivpu_context_abort_locked(file_priv); 983 mutex_unlock(&file_priv->lock); 984 } 985 mutex_unlock(&vdev->context_list_lock); 986 987 /* 988 * We will not receive new MMU event interrupts until existing events are discarded 989 * however, we want to discard these events only after aborting the faulty context 990 * to avoid generating new faults from that context 991 */ 992 ivpu_mmu_discard_events(vdev); 993 994 if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW) 995 goto runtime_put; 996 997 ivpu_jsm_hws_resume_engine(vdev, 0); 998 /* 999 * In hardware scheduling mode NPU already has stopped processing jobs 1000 * and won't send us any further notifications, thus we have to free job related resources 1001 * and notify userspace 1002 */ 1003 mutex_lock(&vdev->submitted_jobs_lock); 1004 xa_for_each(&vdev->submitted_jobs_xa, id, job) 1005 if (job->file_priv->aborted) 1006 ivpu_job_signal_and_destroy(vdev, job->job_id, DRM_IVPU_JOB_STATUS_ABORTED); 1007 mutex_unlock(&vdev->submitted_jobs_lock); 1008 1009 runtime_put: 1010 pm_runtime_mark_last_busy(vdev->drm.dev); 1011 pm_runtime_put_autosuspend(vdev->drm.dev); 1012 } 1013