1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 3 /* Copyright 2019 Collabora ltd. */ 4 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */ 5 6 #include <drm/drm_print.h> 7 #include <drm/drm_file.h> 8 #include <drm/drm_gem.h> 9 #include <drm/rocket_accel.h> 10 #include <linux/interrupt.h> 11 #include <linux/overflow.h> 12 #include <linux/iommu.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include "rocket_core.h" 17 #include "rocket_device.h" 18 #include "rocket_drv.h" 19 #include "rocket_job.h" 20 #include "rocket_registers.h" 21 22 #define JOB_TIMEOUT_MS 500 23 24 static struct rocket_job * 25 to_rocket_job(struct drm_sched_job *sched_job) 26 { 27 return container_of(sched_job, struct rocket_job, base); 28 } 29 30 static const char *rocket_fence_get_driver_name(struct dma_fence *fence) 31 { 32 return "rocket"; 33 } 34 35 static const char *rocket_fence_get_timeline_name(struct dma_fence *fence) 36 { 37 return "rockchip-npu"; 38 } 39 40 static const struct dma_fence_ops rocket_fence_ops = { 41 .get_driver_name = rocket_fence_get_driver_name, 42 .get_timeline_name = rocket_fence_get_timeline_name, 43 }; 44 45 static struct dma_fence *rocket_fence_create(struct rocket_core *core) 46 { 47 struct dma_fence *fence; 48 49 fence = kzalloc_obj(*fence); 50 if (!fence) 51 return ERR_PTR(-ENOMEM); 52 53 dma_fence_init(fence, &rocket_fence_ops, &core->fence_lock, 54 core->fence_context, ++core->emit_seqno); 55 56 return fence; 57 } 58 59 static int 60 rocket_copy_tasks(struct drm_device *dev, 61 struct drm_file *file_priv, 62 struct drm_rocket_job *job, 63 struct rocket_job *rjob) 64 { 65 int ret = 0; 66 67 if (job->task_struct_size < sizeof(struct drm_rocket_task)) 68 return -EINVAL; 69 70 rjob->task_count = job->task_count; 71 72 if (!rjob->task_count) 73 return 0; 74 75 rjob->tasks = kvmalloc_objs(*rjob->tasks, job->task_count); 76 if (!rjob->tasks) { 77 drm_dbg(dev, "Failed to allocate task array\n"); 78 return -ENOMEM; 79 } 80 81 for (int i = 0; i < rjob->task_count; i++) { 82 struct drm_rocket_task task = {0}; 83 84 if (copy_from_user(&task, 85 u64_to_user_ptr(job->tasks) + i * job->task_struct_size, 86 sizeof(task))) { 87 drm_dbg(dev, "Failed to copy incoming tasks\n"); 88 ret = -EFAULT; 89 goto fail; 90 } 91 92 if (task.regcmd_count == 0) { 93 drm_dbg(dev, "regcmd_count field in drm_rocket_task should be > 0.\n"); 94 ret = -EINVAL; 95 goto fail; 96 } 97 98 rjob->tasks[i].regcmd = task.regcmd; 99 rjob->tasks[i].regcmd_count = task.regcmd_count; 100 } 101 102 return 0; 103 104 fail: 105 kvfree(rjob->tasks); 106 rjob->tasks = NULL; 107 return ret; 108 } 109 110 static void rocket_job_hw_submit(struct rocket_core *core, struct rocket_job *job) 111 { 112 struct rocket_task *task; 113 unsigned int extra_bit; 114 115 /* Don't queue the job if a reset is in progress */ 116 if (atomic_read(&core->reset.pending)) 117 return; 118 119 /* GO ! */ 120 121 task = &job->tasks[job->next_task_idx]; 122 job->next_task_idx++; 123 124 rocket_pc_writel(core, BASE_ADDRESS, 0x1); 125 126 /* From rknpu, in the TRM this bit is marked as reserved */ 127 extra_bit = 0x10000000 * core->index; 128 rocket_cna_writel(core, S_POINTER, CNA_S_POINTER_POINTER_PP_EN(1) | 129 CNA_S_POINTER_EXECUTER_PP_EN(1) | 130 CNA_S_POINTER_POINTER_PP_MODE(1) | 131 extra_bit); 132 133 rocket_core_writel(core, S_POINTER, CORE_S_POINTER_POINTER_PP_EN(1) | 134 CORE_S_POINTER_EXECUTER_PP_EN(1) | 135 CORE_S_POINTER_POINTER_PP_MODE(1) | 136 extra_bit); 137 138 rocket_pc_writel(core, BASE_ADDRESS, task->regcmd); 139 rocket_pc_writel(core, REGISTER_AMOUNTS, 140 PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT((task->regcmd_count + 1) / 2 - 1)); 141 142 rocket_pc_writel(core, INTERRUPT_MASK, PC_INTERRUPT_MASK_DPU_0 | PC_INTERRUPT_MASK_DPU_1); 143 rocket_pc_writel(core, INTERRUPT_CLEAR, PC_INTERRUPT_CLEAR_DPU_0 | PC_INTERRUPT_CLEAR_DPU_1); 144 145 rocket_pc_writel(core, TASK_CON, PC_TASK_CON_RESERVED_0(1) | 146 PC_TASK_CON_TASK_COUNT_CLEAR(1) | 147 PC_TASK_CON_TASK_NUMBER(1) | 148 PC_TASK_CON_TASK_PP_EN(1)); 149 150 rocket_pc_writel(core, TASK_DMA_BASE_ADDR, PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR(0x0)); 151 152 rocket_pc_writel(core, OPERATION_ENABLE, PC_OPERATION_ENABLE_OP_EN(1)); 153 154 dev_dbg(core->dev, "Submitted regcmd at 0x%llx to core %d", task->regcmd, core->index); 155 } 156 157 static int rocket_acquire_object_fences(struct drm_gem_object **bos, 158 int bo_count, 159 struct drm_sched_job *job, 160 bool is_write) 161 { 162 int i, ret; 163 164 for (i = 0; i < bo_count; i++) { 165 ret = dma_resv_reserve_fences(bos[i]->resv, 1); 166 if (ret) 167 return ret; 168 169 ret = drm_sched_job_add_implicit_dependencies(job, bos[i], 170 is_write); 171 if (ret) 172 return ret; 173 } 174 175 return 0; 176 } 177 178 static void rocket_attach_object_fences(struct drm_gem_object **bos, 179 int bo_count, 180 struct dma_fence *fence) 181 { 182 int i; 183 184 for (i = 0; i < bo_count; i++) 185 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); 186 } 187 188 static int rocket_job_push(struct rocket_job *job) 189 { 190 struct rocket_device *rdev = job->rdev; 191 struct drm_gem_object **bos; 192 struct ww_acquire_ctx acquire_ctx; 193 u32 bo_count; 194 int ret = 0; 195 196 if (check_add_overflow(job->in_bo_count, job->out_bo_count, &bo_count)) 197 return -EINVAL; 198 199 bos = kvmalloc_array(bo_count, sizeof(*bos), GFP_KERNEL); 200 if (!bos) 201 return -ENOMEM; 202 memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *)); 203 memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *)); 204 205 ret = drm_gem_lock_reservations(bos, bo_count, &acquire_ctx); 206 if (ret) 207 goto err; 208 209 scoped_guard(mutex, &rdev->sched_lock) { 210 drm_sched_job_arm(&job->base); 211 212 job->inference_done_fence = dma_fence_get(&job->base.s_fence->finished); 213 214 ret = rocket_acquire_object_fences(job->in_bos, job->in_bo_count, &job->base, false); 215 if (ret) 216 goto err_unlock; 217 218 ret = rocket_acquire_object_fences(job->out_bos, job->out_bo_count, &job->base, true); 219 if (ret) 220 goto err_unlock; 221 222 kref_get(&job->refcount); /* put by scheduler job completion */ 223 224 drm_sched_entity_push_job(&job->base); 225 } 226 227 rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence); 228 229 err_unlock: 230 drm_gem_unlock_reservations(bos, bo_count, &acquire_ctx); 231 err: 232 kvfree(bos); 233 234 return ret; 235 } 236 237 static void rocket_job_cleanup(struct kref *ref) 238 { 239 struct rocket_job *job = container_of(ref, struct rocket_job, 240 refcount); 241 unsigned int i; 242 243 rocket_iommu_domain_put(job->domain); 244 245 dma_fence_put(job->done_fence); 246 dma_fence_put(job->inference_done_fence); 247 248 if (job->in_bos) { 249 for (i = 0; i < job->in_bo_count; i++) 250 drm_gem_object_put(job->in_bos[i]); 251 252 kvfree(job->in_bos); 253 } 254 255 if (job->out_bos) { 256 for (i = 0; i < job->out_bo_count; i++) 257 drm_gem_object_put(job->out_bos[i]); 258 259 kvfree(job->out_bos); 260 } 261 262 kvfree(job->tasks); 263 264 kfree(job); 265 } 266 267 static void rocket_job_put(struct rocket_job *job) 268 { 269 kref_put(&job->refcount, rocket_job_cleanup); 270 } 271 272 static void rocket_job_free(struct drm_sched_job *sched_job) 273 { 274 struct rocket_job *job = to_rocket_job(sched_job); 275 276 drm_sched_job_cleanup(sched_job); 277 278 rocket_job_put(job); 279 } 280 281 static struct rocket_core *sched_to_core(struct rocket_device *rdev, 282 struct drm_gpu_scheduler *sched) 283 { 284 unsigned int core; 285 286 for (core = 0; core < rdev->num_cores; core++) { 287 if (&rdev->cores[core].sched == sched) 288 return &rdev->cores[core]; 289 } 290 291 return NULL; 292 } 293 294 static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) 295 { 296 struct rocket_job *job = to_rocket_job(sched_job); 297 struct rocket_device *rdev = job->rdev; 298 struct rocket_core *core = sched_to_core(rdev, sched_job->sched); 299 struct dma_fence *fence = NULL; 300 int ret; 301 302 if (unlikely(job->base.s_fence->finished.error)) 303 return NULL; 304 305 /* 306 * Nothing to execute: can happen if the job has finished while 307 * we were resetting the NPU. 308 */ 309 if (job->next_task_idx == job->task_count) 310 return NULL; 311 312 fence = rocket_fence_create(core); 313 if (IS_ERR(fence)) 314 return fence; 315 316 if (job->done_fence) 317 dma_fence_put(job->done_fence); 318 job->done_fence = dma_fence_get(fence); 319 320 ret = pm_runtime_resume_and_get(core->dev); 321 if (ret < 0) 322 goto err_put_fences; 323 324 ret = iommu_attach_group(job->domain->domain, core->iommu_group); 325 if (ret < 0) 326 goto err_put_pm; 327 328 scoped_guard(mutex, &core->job_lock) { 329 core->in_flight_job = job; 330 rocket_job_hw_submit(core, job); 331 } 332 333 return fence; 334 335 err_put_pm: 336 pm_runtime_put(core->dev); 337 err_put_fences: 338 dma_fence_put(job->done_fence); 339 job->done_fence = NULL; 340 dma_fence_put(fence); 341 return ERR_PTR(ret); 342 } 343 344 static void rocket_job_handle_irq(struct rocket_core *core) 345 { 346 pm_runtime_mark_last_busy(core->dev); 347 348 rocket_pc_writel(core, OPERATION_ENABLE, 0x0); 349 rocket_pc_writel(core, INTERRUPT_CLEAR, 0x1ffff); 350 351 scoped_guard(mutex, &core->job_lock) 352 if (core->in_flight_job) { 353 if (core->in_flight_job->next_task_idx < core->in_flight_job->task_count) { 354 rocket_job_hw_submit(core, core->in_flight_job); 355 return; 356 } 357 358 iommu_detach_group(NULL, iommu_group_get(core->dev)); 359 dma_fence_signal(core->in_flight_job->done_fence); 360 pm_runtime_put_autosuspend(core->dev); 361 core->in_flight_job = NULL; 362 } 363 } 364 365 static void 366 rocket_reset(struct rocket_core *core, struct drm_sched_job *bad) 367 { 368 if (!atomic_read(&core->reset.pending)) 369 return; 370 371 drm_sched_stop(&core->sched, bad); 372 373 /* 374 * Remaining interrupts have been handled, but we might still have 375 * stuck jobs. Let's make sure the PM counters stay balanced by 376 * manually calling pm_runtime_put_noidle(). 377 */ 378 scoped_guard(mutex, &core->job_lock) { 379 if (core->in_flight_job) 380 pm_runtime_put_noidle(core->dev); 381 382 iommu_detach_group(NULL, core->iommu_group); 383 384 core->in_flight_job = NULL; 385 } 386 387 /* Proceed with reset now. */ 388 rocket_core_reset(core); 389 390 /* NPU has been reset, we can clear the reset pending bit. */ 391 atomic_set(&core->reset.pending, 0); 392 393 /* Restart the scheduler */ 394 drm_sched_start(&core->sched, 0); 395 } 396 397 static enum drm_gpu_sched_stat rocket_job_timedout(struct drm_sched_job *sched_job) 398 { 399 struct rocket_job *job = to_rocket_job(sched_job); 400 struct rocket_device *rdev = job->rdev; 401 struct rocket_core *core = sched_to_core(rdev, sched_job->sched); 402 403 dev_err(core->dev, "NPU job timed out"); 404 405 atomic_set(&core->reset.pending, 1); 406 rocket_reset(core, sched_job); 407 408 return DRM_GPU_SCHED_STAT_RESET; 409 } 410 411 static void rocket_reset_work(struct work_struct *work) 412 { 413 struct rocket_core *core; 414 415 core = container_of(work, struct rocket_core, reset.work); 416 rocket_reset(core, NULL); 417 } 418 419 static const struct drm_sched_backend_ops rocket_sched_ops = { 420 .run_job = rocket_job_run, 421 .timedout_job = rocket_job_timedout, 422 .free_job = rocket_job_free 423 }; 424 425 static irqreturn_t rocket_job_irq_handler_thread(int irq, void *data) 426 { 427 struct rocket_core *core = data; 428 429 rocket_job_handle_irq(core); 430 431 return IRQ_HANDLED; 432 } 433 434 static irqreturn_t rocket_job_irq_handler(int irq, void *data) 435 { 436 struct rocket_core *core = data; 437 u32 raw_status = rocket_pc_readl(core, INTERRUPT_RAW_STATUS); 438 439 WARN_ON(raw_status & PC_INTERRUPT_RAW_STATUS_DMA_READ_ERROR); 440 WARN_ON(raw_status & PC_INTERRUPT_RAW_STATUS_DMA_WRITE_ERROR); 441 442 if (!(raw_status & PC_INTERRUPT_RAW_STATUS_DPU_0 || 443 raw_status & PC_INTERRUPT_RAW_STATUS_DPU_1)) 444 return IRQ_NONE; 445 446 rocket_pc_writel(core, INTERRUPT_MASK, 0x0); 447 448 return IRQ_WAKE_THREAD; 449 } 450 451 int rocket_job_init(struct rocket_core *core) 452 { 453 struct drm_sched_init_args args = { 454 .ops = &rocket_sched_ops, 455 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 456 .credit_limit = 1, 457 .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), 458 .name = dev_name(core->dev), 459 .dev = core->dev, 460 }; 461 int ret; 462 463 INIT_WORK(&core->reset.work, rocket_reset_work); 464 spin_lock_init(&core->fence_lock); 465 mutex_init(&core->job_lock); 466 467 core->irq = platform_get_irq(to_platform_device(core->dev), 0); 468 if (core->irq < 0) 469 return core->irq; 470 471 ret = devm_request_threaded_irq(core->dev, core->irq, 472 rocket_job_irq_handler, 473 rocket_job_irq_handler_thread, 474 IRQF_SHARED, dev_name(core->dev), 475 core); 476 if (ret) { 477 dev_err(core->dev, "failed to request job irq"); 478 return ret; 479 } 480 481 core->reset.wq = alloc_ordered_workqueue("rocket-reset-%d", 0, core->index); 482 if (!core->reset.wq) 483 return -ENOMEM; 484 485 core->fence_context = dma_fence_context_alloc(1); 486 487 args.timeout_wq = core->reset.wq; 488 ret = drm_sched_init(&core->sched, &args); 489 if (ret) { 490 dev_err(core->dev, "Failed to create scheduler: %d.", ret); 491 goto err_sched; 492 } 493 494 return 0; 495 496 err_sched: 497 drm_sched_fini(&core->sched); 498 499 destroy_workqueue(core->reset.wq); 500 return ret; 501 } 502 503 void rocket_job_fini(struct rocket_core *core) 504 { 505 drm_sched_fini(&core->sched); 506 507 cancel_work_sync(&core->reset.work); 508 destroy_workqueue(core->reset.wq); 509 } 510 511 int rocket_job_open(struct rocket_file_priv *rocket_priv) 512 { 513 struct rocket_device *rdev = rocket_priv->rdev; 514 struct drm_gpu_scheduler **scheds = kmalloc_objs(*scheds, 515 rdev->num_cores); 516 unsigned int core; 517 int ret; 518 519 for (core = 0; core < rdev->num_cores; core++) 520 scheds[core] = &rdev->cores[core].sched; 521 522 ret = drm_sched_entity_init(&rocket_priv->sched_entity, 523 DRM_SCHED_PRIORITY_NORMAL, 524 scheds, 525 rdev->num_cores, NULL); 526 if (WARN_ON(ret)) 527 return ret; 528 529 return 0; 530 } 531 532 void rocket_job_close(struct rocket_file_priv *rocket_priv) 533 { 534 struct drm_sched_entity *entity = &rocket_priv->sched_entity; 535 536 kfree(entity->sched_list); 537 drm_sched_entity_destroy(entity); 538 } 539 540 int rocket_job_is_idle(struct rocket_core *core) 541 { 542 /* If there are any jobs in this HW queue, we're not idle */ 543 if (atomic_read(&core->sched.credit_count)) 544 return false; 545 546 return true; 547 } 548 549 static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file, 550 struct drm_rocket_job *job) 551 { 552 struct rocket_device *rdev = to_rocket_device(dev); 553 struct rocket_file_priv *file_priv = file->driver_priv; 554 struct rocket_job *rjob = NULL; 555 int ret = 0; 556 557 if (job->task_count == 0) 558 return -EINVAL; 559 560 rjob = kzalloc_obj(*rjob); 561 if (!rjob) 562 return -ENOMEM; 563 564 kref_init(&rjob->refcount); 565 566 rjob->rdev = rdev; 567 rjob->domain = rocket_iommu_domain_get(file_priv); 568 569 ret = drm_sched_job_init(&rjob->base, 570 &file_priv->sched_entity, 571 1, NULL, file->client_id); 572 if (ret) 573 goto out_put_job; 574 575 ret = rocket_copy_tasks(dev, file, job, rjob); 576 if (ret) 577 goto out_cleanup_job; 578 579 ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->in_bo_handles), 580 job->in_bo_handle_count, &rjob->in_bos); 581 if (ret) 582 goto out_cleanup_job; 583 584 rjob->in_bo_count = job->in_bo_handle_count; 585 586 ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->out_bo_handles), 587 job->out_bo_handle_count, &rjob->out_bos); 588 if (ret) 589 goto out_cleanup_job; 590 591 rjob->out_bo_count = job->out_bo_handle_count; 592 593 ret = rocket_job_push(rjob); 594 if (ret) 595 goto out_cleanup_job; 596 597 out_cleanup_job: 598 if (ret) 599 drm_sched_job_cleanup(&rjob->base); 600 out_put_job: 601 rocket_job_put(rjob); 602 603 return ret; 604 } 605 606 int rocket_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *file) 607 { 608 struct drm_rocket_submit *args = data; 609 struct drm_rocket_job *jobs; 610 int ret = 0; 611 unsigned int i = 0; 612 613 if (args->job_count == 0) 614 return 0; 615 616 if (args->job_struct_size < sizeof(struct drm_rocket_job)) { 617 drm_dbg(dev, "job_struct_size field in drm_rocket_submit struct is too small.\n"); 618 return -EINVAL; 619 } 620 621 if (args->reserved != 0) { 622 drm_dbg(dev, "Reserved field in drm_rocket_submit struct should be 0.\n"); 623 return -EINVAL; 624 } 625 626 jobs = kvmalloc_objs(*jobs, args->job_count); 627 if (!jobs) { 628 drm_dbg(dev, "Failed to allocate incoming job array\n"); 629 return -ENOMEM; 630 } 631 632 for (i = 0; i < args->job_count; i++) { 633 if (copy_from_user(&jobs[i], 634 u64_to_user_ptr(args->jobs) + i * args->job_struct_size, 635 sizeof(*jobs))) { 636 ret = -EFAULT; 637 drm_dbg(dev, "Failed to copy incoming job array\n"); 638 goto exit; 639 } 640 } 641 642 643 for (i = 0; i < args->job_count; i++) 644 rocket_ioctl_submit_job(dev, file, &jobs[i]); 645 646 exit: 647 kvfree(jobs); 648 649 return ret; 650 } 651