1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 3 /* Copyright 2019 Collabora ltd. */ 4 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */ 5 6 #include <drm/drm_print.h> 7 #include <drm/drm_file.h> 8 #include <drm/drm_gem.h> 9 #include <drm/rocket_accel.h> 10 #include <linux/align.h> 11 #include <linux/interrupt.h> 12 #include <linux/overflow.h> 13 #include <linux/iommu.h> 14 #include <linux/platform_device.h> 15 #include <linux/pm_runtime.h> 16 17 #include "rocket_core.h" 18 #include "rocket_device.h" 19 #include "rocket_drv.h" 20 #include "rocket_job.h" 21 #include "rocket_registers.h" 22 23 #define JOB_TIMEOUT_MS 500 24 25 /* 26 * The PC unit fetches two 64-bit register commands per pc_data_amount unit, 27 * and the field holds (regcmd_count + 1) / 2 - 1. 28 */ 29 #define ROCKET_MAX_REGCMDS ((PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT__MASK + 1) * 2U) 30 31 /* Bits 3:0 of PC_BASE_ADDRESS hold the mode selection bit and reserved bits */ 32 #define ROCKET_REGCMD_ALIGN 16 33 34 static struct rocket_job * 35 to_rocket_job(struct drm_sched_job *sched_job) 36 { 37 return container_of(sched_job, struct rocket_job, base); 38 } 39 40 static const char *rocket_fence_get_driver_name(struct dma_fence *fence) 41 { 42 return "rocket"; 43 } 44 45 static const char *rocket_fence_get_timeline_name(struct dma_fence *fence) 46 { 47 return "rockchip-npu"; 48 } 49 50 static const struct dma_fence_ops rocket_fence_ops = { 51 .get_driver_name = rocket_fence_get_driver_name, 52 .get_timeline_name = rocket_fence_get_timeline_name, 53 }; 54 55 static struct dma_fence *rocket_fence_create(struct rocket_core *core) 56 { 57 struct dma_fence *fence; 58 59 fence = kzalloc_obj(*fence); 60 if (!fence) 61 return ERR_PTR(-ENOMEM); 62 63 dma_fence_init(fence, &rocket_fence_ops, &core->fence_lock, 64 core->fence_context, ++core->emit_seqno); 65 66 return fence; 67 } 68 69 static int 70 rocket_copy_tasks(struct drm_device *dev, 71 struct drm_file *file_priv, 72 struct drm_rocket_job *job, 73 struct rocket_job *rjob) 74 { 75 int ret = 0; 76 77 if (job->task_struct_size < sizeof(struct drm_rocket_task)) 78 return -EINVAL; 79 80 rjob->task_count = job->task_count; 81 82 if (!rjob->task_count) 83 return 0; 84 85 rjob->tasks = kvmalloc_objs(*rjob->tasks, job->task_count); 86 if (!rjob->tasks) { 87 drm_dbg(dev, "Failed to allocate task array\n"); 88 return -ENOMEM; 89 } 90 91 for (unsigned int i = 0; i < rjob->task_count; i++) { 92 struct drm_rocket_task task = {0}; 93 94 if (copy_from_user(&task, 95 u64_to_user_ptr(job->tasks) + i * job->task_struct_size, 96 sizeof(task))) { 97 drm_dbg(dev, "Failed to copy incoming tasks\n"); 98 ret = -EFAULT; 99 goto fail; 100 } 101 102 if (task.regcmd_count == 0) { 103 drm_dbg(dev, "regcmd_count field in drm_rocket_task should be > 0.\n"); 104 ret = -EINVAL; 105 goto fail; 106 } 107 108 if (task.regcmd_count > ROCKET_MAX_REGCMDS) { 109 drm_dbg(dev, "regcmd_count field in drm_rocket_task should be <= %u.\n", 110 ROCKET_MAX_REGCMDS); 111 ret = -EINVAL; 112 goto fail; 113 } 114 115 if (!IS_ALIGNED(task.regcmd, ROCKET_REGCMD_ALIGN)) { 116 drm_dbg(dev, "regcmd field in drm_rocket_task should be aligned to %u bytes.\n", 117 ROCKET_REGCMD_ALIGN); 118 ret = -EINVAL; 119 goto fail; 120 } 121 122 rjob->tasks[i].regcmd = task.regcmd; 123 rjob->tasks[i].regcmd_count = task.regcmd_count; 124 } 125 126 return 0; 127 128 fail: 129 kvfree(rjob->tasks); 130 rjob->tasks = NULL; 131 return ret; 132 } 133 134 static void rocket_job_hw_submit(struct rocket_core *core, struct rocket_job *job) 135 { 136 struct rocket_task *task; 137 unsigned int extra_bit; 138 139 /* Don't queue the job if a reset is in progress */ 140 if (atomic_read(&core->reset.pending)) 141 return; 142 143 /* GO ! */ 144 145 task = &job->tasks[job->next_task_idx]; 146 job->next_task_idx++; 147 148 rocket_pc_writel(core, BASE_ADDRESS, 0x1); 149 150 /* From rknpu, in the TRM this bit is marked as reserved */ 151 extra_bit = 0x10000000 * core->index; 152 rocket_cna_writel(core, S_POINTER, CNA_S_POINTER_POINTER_PP_EN(1) | 153 CNA_S_POINTER_EXECUTER_PP_EN(1) | 154 CNA_S_POINTER_POINTER_PP_MODE(1) | 155 extra_bit); 156 157 rocket_core_writel(core, S_POINTER, CORE_S_POINTER_POINTER_PP_EN(1) | 158 CORE_S_POINTER_EXECUTER_PP_EN(1) | 159 CORE_S_POINTER_POINTER_PP_MODE(1) | 160 extra_bit); 161 162 rocket_pc_writel(core, BASE_ADDRESS, task->regcmd); 163 rocket_pc_writel(core, REGISTER_AMOUNTS, 164 PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT((task->regcmd_count + 1) / 2 - 1)); 165 166 rocket_pc_writel(core, INTERRUPT_MASK, PC_INTERRUPT_MASK_DPU_0 | PC_INTERRUPT_MASK_DPU_1); 167 rocket_pc_writel(core, INTERRUPT_CLEAR, PC_INTERRUPT_CLEAR_DPU_0 | PC_INTERRUPT_CLEAR_DPU_1); 168 169 rocket_pc_writel(core, TASK_CON, PC_TASK_CON_RESERVED_0(1) | 170 PC_TASK_CON_TASK_COUNT_CLEAR(1) | 171 PC_TASK_CON_TASK_NUMBER(1) | 172 PC_TASK_CON_TASK_PP_EN(1)); 173 174 rocket_pc_writel(core, TASK_DMA_BASE_ADDR, PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR(0x0)); 175 176 rocket_pc_writel(core, OPERATION_ENABLE, PC_OPERATION_ENABLE_OP_EN(1)); 177 178 dev_dbg(core->dev, "Submitted regcmd at 0x%llx to core %d", task->regcmd, core->index); 179 } 180 181 static int rocket_acquire_object_fences(struct drm_gem_object **bos, 182 int bo_count, 183 struct drm_sched_job *job, 184 bool is_write) 185 { 186 int i, ret; 187 188 for (i = 0; i < bo_count; i++) { 189 ret = dma_resv_reserve_fences(bos[i]->resv, 1); 190 if (ret) 191 return ret; 192 193 ret = drm_sched_job_add_implicit_dependencies(job, bos[i], 194 is_write); 195 if (ret) 196 return ret; 197 } 198 199 return 0; 200 } 201 202 static void rocket_attach_object_fences(struct drm_gem_object **bos, 203 int bo_count, 204 struct dma_fence *fence) 205 { 206 int i; 207 208 for (i = 0; i < bo_count; i++) 209 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); 210 } 211 212 static int rocket_job_push(struct rocket_job *job) 213 { 214 struct rocket_device *rdev = job->rdev; 215 struct drm_gem_object **bos; 216 struct ww_acquire_ctx acquire_ctx; 217 u32 bo_count; 218 int ret = 0; 219 220 if (check_add_overflow(job->in_bo_count, job->out_bo_count, &bo_count)) 221 return -EINVAL; 222 223 bos = kvmalloc_objs(*bos, bo_count); 224 if (!bos) 225 return -ENOMEM; 226 memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *)); 227 memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *)); 228 229 ret = drm_gem_lock_reservations(bos, bo_count, &acquire_ctx); 230 if (ret) 231 goto err; 232 233 scoped_guard(mutex, &rdev->sched_lock) { 234 drm_sched_job_arm(&job->base); 235 236 job->inference_done_fence = dma_fence_get(&job->base.s_fence->finished); 237 238 ret = rocket_acquire_object_fences(job->in_bos, job->in_bo_count, &job->base, false); 239 if (ret) 240 goto err_unlock; 241 242 ret = rocket_acquire_object_fences(job->out_bos, job->out_bo_count, &job->base, true); 243 if (ret) 244 goto err_unlock; 245 246 kref_get(&job->refcount); /* put by scheduler job completion */ 247 248 drm_sched_entity_push_job(&job->base); 249 } 250 251 rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence); 252 253 err_unlock: 254 drm_gem_unlock_reservations(bos, bo_count, &acquire_ctx); 255 err: 256 kvfree(bos); 257 258 return ret; 259 } 260 261 static void rocket_job_cleanup(struct kref *ref) 262 { 263 struct rocket_job *job = container_of(ref, struct rocket_job, 264 refcount); 265 unsigned int i; 266 267 rocket_iommu_domain_put(job->domain); 268 269 dma_fence_put(job->done_fence); 270 dma_fence_put(job->inference_done_fence); 271 272 if (job->in_bos) { 273 for (i = 0; i < job->in_bo_count; i++) 274 drm_gem_object_put(job->in_bos[i]); 275 276 kvfree(job->in_bos); 277 } 278 279 if (job->out_bos) { 280 for (i = 0; i < job->out_bo_count; i++) 281 drm_gem_object_put(job->out_bos[i]); 282 283 kvfree(job->out_bos); 284 } 285 286 kvfree(job->tasks); 287 288 kfree(job); 289 } 290 291 static void rocket_job_put(struct rocket_job *job) 292 { 293 kref_put(&job->refcount, rocket_job_cleanup); 294 } 295 296 static void rocket_job_free(struct drm_sched_job *sched_job) 297 { 298 struct rocket_job *job = to_rocket_job(sched_job); 299 300 drm_sched_job_cleanup(sched_job); 301 302 rocket_job_put(job); 303 } 304 305 static struct rocket_core *sched_to_core(struct rocket_device *rdev, 306 struct drm_gpu_scheduler *sched) 307 { 308 unsigned int core; 309 310 for (core = 0; core < rdev->max_cores; core++) { 311 if (&rdev->cores[core].sched == sched) 312 return &rdev->cores[core]; 313 } 314 315 return NULL; 316 } 317 318 static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) 319 { 320 struct rocket_job *job = to_rocket_job(sched_job); 321 struct rocket_device *rdev = job->rdev; 322 struct rocket_core *core = sched_to_core(rdev, sched_job->sched); 323 struct dma_fence *fence = NULL; 324 int ret; 325 326 if (unlikely(job->base.s_fence->finished.error)) 327 return NULL; 328 329 /* 330 * Nothing to execute: can happen if the job has finished while 331 * we were resetting the NPU. 332 */ 333 if (job->next_task_idx == job->task_count) 334 return NULL; 335 336 fence = rocket_fence_create(core); 337 if (IS_ERR(fence)) 338 return fence; 339 340 if (job->done_fence) 341 dma_fence_put(job->done_fence); 342 job->done_fence = dma_fence_get(fence); 343 344 ret = pm_runtime_resume_and_get(core->dev); 345 if (ret < 0) 346 goto err_put_fences; 347 348 ret = iommu_attach_group(job->domain->domain, core->iommu_group); 349 if (ret < 0) 350 goto err_put_pm; 351 352 scoped_guard(mutex, &core->job_lock) { 353 core->in_flight_job = job; 354 rocket_job_hw_submit(core, job); 355 } 356 357 return fence; 358 359 err_put_pm: 360 pm_runtime_put(core->dev); 361 err_put_fences: 362 dma_fence_put(job->done_fence); 363 job->done_fence = NULL; 364 dma_fence_put(fence); 365 return ERR_PTR(ret); 366 } 367 368 static void rocket_job_handle_irq(struct rocket_core *core) 369 { 370 pm_runtime_mark_last_busy(core->dev); 371 372 rocket_pc_writel(core, OPERATION_ENABLE, 0x0); 373 rocket_pc_writel(core, INTERRUPT_CLEAR, 0x1ffff); 374 375 scoped_guard(mutex, &core->job_lock) 376 if (core->in_flight_job) { 377 if (core->in_flight_job->next_task_idx < core->in_flight_job->task_count) { 378 rocket_job_hw_submit(core, core->in_flight_job); 379 return; 380 } 381 382 iommu_detach_group(NULL, iommu_group_get(core->dev)); 383 dma_fence_signal(core->in_flight_job->done_fence); 384 pm_runtime_put_autosuspend(core->dev); 385 core->in_flight_job = NULL; 386 } 387 } 388 389 static void 390 rocket_reset(struct rocket_core *core, struct drm_sched_job *bad) 391 { 392 if (!atomic_read(&core->reset.pending)) 393 return; 394 395 drm_sched_stop(&core->sched, bad); 396 397 /* 398 * Remaining interrupts have been handled, but we might still have 399 * stuck jobs. Let's make sure the PM counters stay balanced by 400 * manually calling pm_runtime_put_noidle(). 401 */ 402 scoped_guard(mutex, &core->job_lock) { 403 if (core->in_flight_job) 404 pm_runtime_put_noidle(core->dev); 405 406 iommu_detach_group(NULL, core->iommu_group); 407 408 core->in_flight_job = NULL; 409 } 410 411 /* Proceed with reset now. */ 412 rocket_core_reset(core); 413 414 /* NPU has been reset, we can clear the reset pending bit. */ 415 atomic_set(&core->reset.pending, 0); 416 417 /* Restart the scheduler */ 418 drm_sched_start(&core->sched, 0); 419 } 420 421 static enum drm_gpu_sched_stat rocket_job_timedout(struct drm_sched_job *sched_job) 422 { 423 struct rocket_job *job = to_rocket_job(sched_job); 424 struct rocket_device *rdev = job->rdev; 425 struct rocket_core *core = sched_to_core(rdev, sched_job->sched); 426 427 dev_err(core->dev, "NPU job timed out"); 428 429 atomic_set(&core->reset.pending, 1); 430 rocket_reset(core, sched_job); 431 432 return DRM_GPU_SCHED_STAT_RESET; 433 } 434 435 static void rocket_reset_work(struct work_struct *work) 436 { 437 struct rocket_core *core; 438 439 core = container_of(work, struct rocket_core, reset.work); 440 rocket_reset(core, NULL); 441 } 442 443 static const struct drm_sched_backend_ops rocket_sched_ops = { 444 .run_job = rocket_job_run, 445 .timedout_job = rocket_job_timedout, 446 .free_job = rocket_job_free 447 }; 448 449 static irqreturn_t rocket_job_irq_handler_thread(int irq, void *data) 450 { 451 struct rocket_core *core = data; 452 453 rocket_job_handle_irq(core); 454 455 return IRQ_HANDLED; 456 } 457 458 static irqreturn_t rocket_job_irq_handler(int irq, void *data) 459 { 460 struct rocket_core *core = data; 461 u32 raw_status = rocket_pc_readl(core, INTERRUPT_RAW_STATUS); 462 463 WARN_ON(raw_status & PC_INTERRUPT_RAW_STATUS_DMA_READ_ERROR); 464 WARN_ON(raw_status & PC_INTERRUPT_RAW_STATUS_DMA_WRITE_ERROR); 465 466 if (!(raw_status & PC_INTERRUPT_RAW_STATUS_DPU_0 || 467 raw_status & PC_INTERRUPT_RAW_STATUS_DPU_1)) 468 return IRQ_NONE; 469 470 rocket_pc_writel(core, INTERRUPT_MASK, 0x0); 471 472 return IRQ_WAKE_THREAD; 473 } 474 475 int rocket_job_init(struct rocket_core *core) 476 { 477 struct drm_sched_init_args args = { 478 .ops = &rocket_sched_ops, 479 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 480 .credit_limit = 1, 481 .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), 482 .name = dev_name(core->dev), 483 .dev = core->dev, 484 }; 485 int ret; 486 487 INIT_WORK(&core->reset.work, rocket_reset_work); 488 spin_lock_init(&core->fence_lock); 489 mutex_init(&core->job_lock); 490 491 core->irq = platform_get_irq(to_platform_device(core->dev), 0); 492 if (core->irq < 0) 493 return core->irq; 494 495 ret = devm_request_threaded_irq(core->dev, core->irq, 496 rocket_job_irq_handler, 497 rocket_job_irq_handler_thread, 498 IRQF_SHARED, dev_name(core->dev), 499 core); 500 if (ret) { 501 dev_err(core->dev, "failed to request job irq"); 502 return ret; 503 } 504 505 core->reset.wq = alloc_ordered_workqueue("rocket-reset-%d", 0, core->index); 506 if (!core->reset.wq) 507 return -ENOMEM; 508 509 core->fence_context = dma_fence_context_alloc(1); 510 511 args.timeout_wq = core->reset.wq; 512 ret = drm_sched_init(&core->sched, &args); 513 if (ret) { 514 dev_err(core->dev, "Failed to create scheduler: %d.", ret); 515 goto err_sched; 516 } 517 518 return 0; 519 520 err_sched: 521 destroy_workqueue(core->reset.wq); 522 return ret; 523 } 524 525 void rocket_job_fini(struct rocket_core *core) 526 { 527 drm_sched_fini(&core->sched); 528 529 cancel_work_sync(&core->reset.work); 530 destroy_workqueue(core->reset.wq); 531 } 532 533 int rocket_job_open(struct rocket_file_priv *rocket_priv) 534 { 535 struct rocket_device *rdev = rocket_priv->rdev; 536 struct drm_gpu_scheduler **scheds = kmalloc_objs(*scheds, 537 rdev->num_cores); 538 unsigned int core; 539 int ret; 540 541 for (core = 0; core < rdev->num_cores; core++) 542 scheds[core] = &rdev->cores[core].sched; 543 544 ret = drm_sched_entity_init(&rocket_priv->sched_entity, 545 DRM_SCHED_PRIORITY_NORMAL, 546 scheds, 547 rdev->num_cores, NULL); 548 if (WARN_ON(ret)) 549 return ret; 550 551 return 0; 552 } 553 554 void rocket_job_close(struct rocket_file_priv *rocket_priv) 555 { 556 struct drm_sched_entity *entity = &rocket_priv->sched_entity; 557 558 kfree(entity->sched_list); 559 drm_sched_entity_destroy(entity); 560 } 561 562 int rocket_job_is_idle(struct rocket_core *core) 563 { 564 /* If there are any jobs in this HW queue, we're not idle */ 565 if (atomic_read(&core->sched.credit_count)) 566 return false; 567 568 return true; 569 } 570 571 static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file, 572 struct drm_rocket_job *job) 573 { 574 struct rocket_device *rdev = to_rocket_device(dev); 575 struct rocket_file_priv *file_priv = file->driver_priv; 576 struct rocket_job *rjob = NULL; 577 int ret = 0; 578 579 if (job->task_count == 0) 580 return -EINVAL; 581 582 rjob = kzalloc_obj(*rjob); 583 if (!rjob) 584 return -ENOMEM; 585 586 kref_init(&rjob->refcount); 587 588 rjob->rdev = rdev; 589 rjob->domain = rocket_iommu_domain_get(file_priv); 590 591 ret = drm_sched_job_init(&rjob->base, 592 &file_priv->sched_entity, 593 1, NULL, file->client_id); 594 if (ret) 595 goto out_put_job; 596 597 ret = rocket_copy_tasks(dev, file, job, rjob); 598 if (ret) 599 goto out_cleanup_job; 600 601 ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->in_bo_handles), 602 job->in_bo_handle_count, &rjob->in_bos); 603 if (ret) 604 goto out_cleanup_job; 605 606 rjob->in_bo_count = job->in_bo_handle_count; 607 608 ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->out_bo_handles), 609 job->out_bo_handle_count, &rjob->out_bos); 610 if (ret) 611 goto out_cleanup_job; 612 613 rjob->out_bo_count = job->out_bo_handle_count; 614 615 ret = rocket_job_push(rjob); 616 if (ret) 617 goto out_cleanup_job; 618 619 out_cleanup_job: 620 if (ret) 621 drm_sched_job_cleanup(&rjob->base); 622 out_put_job: 623 rocket_job_put(rjob); 624 625 return ret; 626 } 627 628 int rocket_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *file) 629 { 630 struct drm_rocket_submit *args = data; 631 struct drm_rocket_job *jobs; 632 int ret = 0; 633 unsigned int i = 0; 634 635 if (args->job_count == 0) 636 return 0; 637 638 if (args->job_struct_size < sizeof(struct drm_rocket_job)) { 639 drm_dbg(dev, "job_struct_size field in drm_rocket_submit struct is too small.\n"); 640 return -EINVAL; 641 } 642 643 if (args->reserved != 0) { 644 drm_dbg(dev, "Reserved field in drm_rocket_submit struct should be 0.\n"); 645 return -EINVAL; 646 } 647 648 jobs = kvmalloc_objs(*jobs, args->job_count); 649 if (!jobs) { 650 drm_dbg(dev, "Failed to allocate incoming job array\n"); 651 return -ENOMEM; 652 } 653 654 for (i = 0; i < args->job_count; i++) { 655 if (copy_from_user(&jobs[i], 656 u64_to_user_ptr(args->jobs) + i * args->job_struct_size, 657 sizeof(*jobs))) { 658 ret = -EFAULT; 659 drm_dbg(dev, "Failed to copy incoming job array\n"); 660 goto exit; 661 } 662 } 663 664 665 for (i = 0; i < args->job_count; i++) 666 rocket_ioctl_submit_job(dev, file, &jobs[i]); 667 668 exit: 669 kvfree(jobs); 670 671 return ret; 672 } 673