1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */ 3 /* Copyright 2025-2026 Arm, Ltd. */ 4 5 #include <linux/bitfield.h> 6 #include <linux/genalloc.h> 7 #include <linux/interrupt.h> 8 #include <linux/iopoll.h> 9 #include <linux/platform_device.h> 10 #include <linux/pm_runtime.h> 11 12 #include <drm/drm_file.h> 13 #include <drm/drm_gem.h> 14 #include <drm/drm_gem_dma_helper.h> 15 #include <drm/drm_print.h> 16 #include <drm/ethosu_accel.h> 17 18 #include "ethosu_device.h" 19 #include "ethosu_drv.h" 20 #include "ethosu_gem.h" 21 #include "ethosu_job.h" 22 23 #define JOB_TIMEOUT_MS 500 24 25 static struct ethosu_job *to_ethosu_job(struct drm_sched_job *sched_job) 26 { 27 return container_of(sched_job, struct ethosu_job, base); 28 } 29 30 static const char *ethosu_fence_get_driver_name(struct dma_fence *fence) 31 { 32 return "ethosu"; 33 } 34 35 static const char *ethosu_fence_get_timeline_name(struct dma_fence *fence) 36 { 37 return "ethosu-npu"; 38 } 39 40 static const struct dma_fence_ops ethosu_fence_ops = { 41 .get_driver_name = ethosu_fence_get_driver_name, 42 .get_timeline_name = ethosu_fence_get_timeline_name, 43 }; 44 45 static void ethosu_job_hw_submit(struct ethosu_device *dev, struct ethosu_job *job) 46 { 47 struct drm_gem_dma_object *cmd_bo = to_drm_gem_dma_obj(job->cmd_bo); 48 struct ethosu_validated_cmdstream_info *cmd_info = to_ethosu_bo(job->cmd_bo)->info; 49 50 for (int i = 0; i < job->region_cnt; i++) { 51 struct drm_gem_dma_object *bo; 52 int region = job->region_bo_num[i]; 53 54 bo = to_drm_gem_dma_obj(job->region_bo[i]); 55 writel_relaxed(lower_32_bits(bo->dma_addr), dev->regs + NPU_REG_BASEP(region)); 56 writel_relaxed(upper_32_bits(bo->dma_addr), dev->regs + NPU_REG_BASEP_HI(region)); 57 dev_dbg(dev->base.dev, "Region %d base addr = %pad\n", region, &bo->dma_addr); 58 } 59 60 if (job->sram_size) { 61 writel_relaxed(lower_32_bits(dev->sramphys), 62 dev->regs + NPU_REG_BASEP(ETHOSU_SRAM_REGION)); 63 writel_relaxed(upper_32_bits(dev->sramphys), 64 dev->regs + NPU_REG_BASEP_HI(ETHOSU_SRAM_REGION)); 65 dev_dbg(dev->base.dev, "Region %d base addr = %pad (SRAM)\n", 66 ETHOSU_SRAM_REGION, &dev->sramphys); 67 } 68 69 writel_relaxed(lower_32_bits(cmd_bo->dma_addr), dev->regs + NPU_REG_QBASE); 70 writel_relaxed(upper_32_bits(cmd_bo->dma_addr), dev->regs + NPU_REG_QBASE_HI); 71 writel_relaxed(cmd_info->cmd_size, dev->regs + NPU_REG_QSIZE); 72 73 writel(CMD_TRANSITION_TO_RUN, dev->regs + NPU_REG_CMD); 74 75 dev_dbg(dev->base.dev, 76 "Submitted cmd at %pad to core\n", &cmd_bo->dma_addr); 77 } 78 79 static int ethosu_acquire_object_fences(struct ethosu_job *job) 80 { 81 int i, ret; 82 struct drm_gem_object **bos = job->region_bo; 83 struct ethosu_validated_cmdstream_info *info = to_ethosu_bo(job->cmd_bo)->info; 84 85 for (i = 0; i < job->region_cnt; i++) { 86 bool is_write; 87 88 if (!bos[i]) 89 break; 90 91 ret = dma_resv_reserve_fences(bos[i]->resv, 1); 92 if (ret) 93 return ret; 94 95 is_write = info->output_region[job->region_bo_num[i]]; 96 ret = drm_sched_job_add_implicit_dependencies(&job->base, bos[i], 97 is_write); 98 if (ret) 99 return ret; 100 } 101 102 return 0; 103 } 104 105 static void ethosu_attach_object_fences(struct ethosu_job *job) 106 { 107 int i; 108 struct dma_fence *fence = job->inference_done_fence; 109 struct drm_gem_object **bos = job->region_bo; 110 struct ethosu_validated_cmdstream_info *info = to_ethosu_bo(job->cmd_bo)->info; 111 112 for (i = 0; i < job->region_cnt; i++) 113 if (info->output_region[job->region_bo_num[i]]) 114 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); 115 } 116 117 static int ethosu_job_push(struct ethosu_job *job) 118 { 119 struct ww_acquire_ctx acquire_ctx; 120 int ret; 121 122 ret = drm_gem_lock_reservations(job->region_bo, job->region_cnt, &acquire_ctx); 123 if (ret) 124 return ret; 125 126 ret = ethosu_acquire_object_fences(job); 127 if (ret) 128 goto out; 129 130 ret = pm_runtime_resume_and_get(job->dev->base.dev); 131 if (!ret) { 132 guard(mutex)(&job->dev->sched_lock); 133 134 drm_sched_job_arm(&job->base); 135 job->inference_done_fence = dma_fence_get(&job->base.s_fence->finished); 136 kref_get(&job->refcount); /* put by scheduler job completion */ 137 drm_sched_entity_push_job(&job->base); 138 ethosu_attach_object_fences(job); 139 } 140 141 out: 142 drm_gem_unlock_reservations(job->region_bo, job->region_cnt, &acquire_ctx); 143 return ret; 144 } 145 146 static void ethosu_job_err_cleanup(struct ethosu_job *job) 147 { 148 unsigned int i; 149 150 ethosu_perfmon_put(job->perfmon); 151 152 for (i = 0; i < job->region_cnt; i++) 153 drm_gem_object_put(job->region_bo[i]); 154 155 drm_gem_object_put(job->cmd_bo); 156 157 kfree(job); 158 } 159 160 static void ethosu_job_cleanup(struct kref *ref) 161 { 162 struct ethosu_job *job = container_of(ref, struct ethosu_job, 163 refcount); 164 165 pm_runtime_put_autosuspend(job->dev->base.dev); 166 167 dma_fence_put(job->done_fence); 168 dma_fence_put(job->inference_done_fence); 169 170 ethosu_job_err_cleanup(job); 171 } 172 173 static void ethosu_job_put(struct ethosu_job *job) 174 { 175 kref_put(&job->refcount, ethosu_job_cleanup); 176 } 177 178 static void ethosu_job_free(struct drm_sched_job *sched_job) 179 { 180 struct ethosu_job *job = to_ethosu_job(sched_job); 181 182 drm_sched_job_cleanup(sched_job); 183 ethosu_job_put(job); 184 } 185 186 static void 187 ethosu_switch_perfmon(struct ethosu_device *ethosu, struct ethosu_job *job) 188 { 189 struct ethosu_perfmon *perfmon; 190 191 guard(mutex)(ðosu->perfmon_state.lock); 192 193 perfmon = ethosu->global_perfmon; 194 if (!perfmon) 195 perfmon = job->perfmon; 196 197 if (perfmon == ethosu->perfmon_state.active) 198 return; 199 200 ethosu_perfmon_stop_locked(ethosu, ethosu->perfmon_state.active, true); 201 202 if (perfmon) 203 ethosu_perfmon_start(ethosu, perfmon); 204 } 205 206 static struct dma_fence *ethosu_job_run(struct drm_sched_job *sched_job) 207 { 208 struct ethosu_job *job = to_ethosu_job(sched_job); 209 struct ethosu_device *dev = job->dev; 210 struct dma_fence *fence = job->done_fence; 211 212 if (unlikely(job->base.s_fence->finished.error)) 213 return NULL; 214 215 dma_fence_init(fence, ðosu_fence_ops, &dev->fence_lock, 216 dev->fence_context, ++dev->emit_seqno); 217 dma_fence_get(fence); 218 219 ethosu_switch_perfmon(dev, job); 220 221 WRITE_ONCE(dev->in_flight_job, job); 222 ethosu_job_hw_submit(dev, job); 223 224 return fence; 225 } 226 227 static void ethosu_job_handle_irq(struct ethosu_device *dev) 228 { 229 u32 status = readl_relaxed(dev->regs + NPU_REG_STATUS); 230 struct ethosu_job *job; 231 232 if (status & (STATUS_BUS_STATUS | STATUS_CMD_PARSE_ERR)) { 233 dev_err(dev->base.dev, "Error IRQ - %x\n", status); 234 drm_sched_fault(&dev->sched); 235 return; 236 } 237 238 job = READ_ONCE(dev->in_flight_job); 239 if (job) { 240 WRITE_ONCE(dev->in_flight_job, NULL); 241 dma_fence_signal(job->done_fence); 242 } 243 } 244 245 static irqreturn_t ethosu_job_irq_handler_thread(int irq, void *data) 246 { 247 struct ethosu_device *dev = data; 248 249 ethosu_job_handle_irq(dev); 250 251 return IRQ_HANDLED; 252 } 253 254 static irqreturn_t ethosu_job_irq_handler(int irq, void *data) 255 { 256 struct ethosu_device *dev = data; 257 u32 status = readl_relaxed(dev->regs + NPU_REG_STATUS); 258 259 if (!(status & STATUS_IRQ_RAISED)) 260 return IRQ_NONE; 261 262 writel_relaxed(CMD_CLEAR_IRQ, dev->regs + NPU_REG_CMD); 263 return IRQ_WAKE_THREAD; 264 } 265 266 static enum drm_gpu_sched_stat ethosu_job_timedout(struct drm_sched_job *bad) 267 { 268 struct ethosu_job *job = to_ethosu_job(bad); 269 struct ethosu_device *dev = job->dev; 270 bool running; 271 u32 *bocmds = to_drm_gem_dma_obj(job->cmd_bo)->vaddr; 272 u32 cmdaddr; 273 274 cmdaddr = readl_relaxed(dev->regs + NPU_REG_QREAD); 275 running = FIELD_GET(STATUS_STATE_RUNNING, readl_relaxed(dev->regs + NPU_REG_STATUS)); 276 277 if (running) { 278 int ret; 279 u32 reg; 280 281 ret = readl_relaxed_poll_timeout(dev->regs + NPU_REG_QREAD, 282 reg, 283 reg != cmdaddr, 284 USEC_PER_MSEC, 100 * USEC_PER_MSEC); 285 286 /* If still running and progress is being made, just return */ 287 if (!ret) 288 return DRM_GPU_SCHED_STAT_NO_HANG; 289 } 290 291 dev_err(dev->base.dev, "NPU sched timed out: NPU %s, cmdstream offset 0x%x: 0x%x\n", 292 running ? "running" : "stopped", 293 cmdaddr, bocmds[cmdaddr / 4]); 294 295 drm_sched_stop(&dev->sched, bad); 296 297 WRITE_ONCE(dev->in_flight_job, NULL); 298 299 /* Proceed with reset now. */ 300 pm_runtime_force_suspend(dev->base.dev); 301 pm_runtime_force_resume(dev->base.dev); 302 303 /* Restart the scheduler */ 304 drm_sched_start(&dev->sched, 0); 305 306 return DRM_GPU_SCHED_STAT_RESET; 307 } 308 309 static const struct drm_sched_backend_ops ethosu_sched_ops = { 310 .run_job = ethosu_job_run, 311 .timedout_job = ethosu_job_timedout, 312 .free_job = ethosu_job_free 313 }; 314 315 int ethosu_job_init(struct ethosu_device *edev) 316 { 317 struct device *dev = edev->base.dev; 318 struct drm_sched_init_args args = { 319 .ops = ðosu_sched_ops, 320 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 321 .credit_limit = 1, 322 .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), 323 .name = dev_name(dev), 324 .dev = dev, 325 }; 326 int ret; 327 328 spin_lock_init(&edev->fence_lock); 329 ret = devm_mutex_init(dev, &edev->sched_lock); 330 if (ret) 331 return ret; 332 333 edev->irq = platform_get_irq(to_platform_device(dev), 0); 334 if (edev->irq < 0) 335 return edev->irq; 336 337 ret = devm_request_threaded_irq(dev, edev->irq, 338 ethosu_job_irq_handler, 339 ethosu_job_irq_handler_thread, 340 IRQF_SHARED, KBUILD_MODNAME, 341 edev); 342 if (ret) { 343 dev_err(dev, "failed to request irq\n"); 344 return ret; 345 } 346 347 edev->fence_context = dma_fence_context_alloc(1); 348 349 ret = drm_sched_init(&edev->sched, &args); 350 if (ret) { 351 dev_err(dev, "Failed to create scheduler: %d\n", ret); 352 goto err_sched; 353 } 354 355 return 0; 356 357 err_sched: 358 drm_sched_fini(&edev->sched); 359 return ret; 360 } 361 362 void ethosu_job_fini(struct ethosu_device *dev) 363 { 364 drm_sched_fini(&dev->sched); 365 } 366 367 int ethosu_job_open(struct ethosu_file_priv *ethosu_priv) 368 { 369 struct ethosu_device *dev = ethosu_priv->edev; 370 struct drm_gpu_scheduler *sched = &dev->sched; 371 int ret; 372 373 ret = drm_sched_entity_init(ðosu_priv->sched_entity, 374 DRM_SCHED_PRIORITY_NORMAL, 375 &sched, 1, NULL); 376 return WARN_ON(ret); 377 } 378 379 void ethosu_job_close(struct ethosu_file_priv *ethosu_priv) 380 { 381 struct drm_sched_entity *entity = ðosu_priv->sched_entity; 382 383 drm_sched_entity_destroy(entity); 384 } 385 386 static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file, 387 struct drm_ethosu_job *job, 388 int perfmon_id) 389 { 390 struct ethosu_device *edev = to_ethosu_device(dev); 391 struct ethosu_file_priv *file_priv = file->driver_priv; 392 struct ethosu_job *ejob = NULL; 393 struct ethosu_validated_cmdstream_info *cmd_info; 394 int ret = 0; 395 396 /* BO region 2 is reserved if SRAM is used */ 397 if (job->region_bo_handles[ETHOSU_SRAM_REGION] && job->sram_size) 398 return -EINVAL; 399 400 if (edev->npu_info.sram_size < job->sram_size) 401 return -EINVAL; 402 403 ejob = kzalloc_obj(*ejob); 404 if (!ejob) 405 return -ENOMEM; 406 407 kref_init(&ejob->refcount); 408 409 ejob->dev = edev; 410 ejob->sram_size = job->sram_size; 411 412 if (perfmon_id) 413 ejob->perfmon = ethosu_perfmon_find(file_priv, perfmon_id); 414 415 ejob->done_fence = kzalloc_obj(*ejob->done_fence); 416 if (!ejob->done_fence) { 417 ret = -ENOMEM; 418 goto out_cleanup_job; 419 } 420 421 ret = drm_sched_job_init(&ejob->base, 422 &file_priv->sched_entity, 423 1, NULL, file->client_id); 424 if (ret) 425 goto out_put_job; 426 427 ejob->cmd_bo = drm_gem_object_lookup(file, job->cmd_bo); 428 if (!ejob->cmd_bo) { 429 ret = -ENOENT; 430 goto out_cleanup_job; 431 } 432 cmd_info = to_ethosu_bo(ejob->cmd_bo)->info; 433 if (!cmd_info) { 434 ret = -EINVAL; 435 goto out_cleanup_job; 436 } 437 438 for (int i = 0; i < NPU_BASEP_REGION_MAX; i++) { 439 struct drm_gem_object *gem; 440 441 /* Can only omit a BO handle if the region is not used or used for SRAM */ 442 if (!job->region_bo_handles[i]) { 443 if (!cmd_info->region_size[i]) 444 continue; 445 if (i == ETHOSU_SRAM_REGION) { 446 if (cmd_info->region_size[i] <= edev->npu_info.sram_size) 447 continue; 448 449 dev_err(dev->dev, 450 "cmd stream region %d size greater than SRAM size (%llu > %u)\n", 451 i, cmd_info->region_size[i], 452 edev->npu_info.sram_size); 453 ret = -EINVAL; 454 goto out_cleanup_job; 455 } 456 } 457 458 if (job->region_bo_handles[i] && !cmd_info->region_size[i]) { 459 dev_err(dev->dev, 460 "Cmdstream BO handle %d set for unused region %d\n", 461 job->region_bo_handles[i], i); 462 ret = -EINVAL; 463 goto out_cleanup_job; 464 } 465 466 gem = drm_gem_object_lookup(file, job->region_bo_handles[i]); 467 if (!gem) { 468 dev_err(dev->dev, 469 "Invalid BO handle %d for region %d\n", 470 job->region_bo_handles[i], i); 471 ret = -ENOENT; 472 goto out_cleanup_job; 473 } 474 475 ejob->region_bo[ejob->region_cnt] = gem; 476 ejob->region_bo_num[ejob->region_cnt] = i; 477 ejob->region_cnt++; 478 479 if (to_ethosu_bo(gem)->info) { 480 dev_err(dev->dev, 481 "Cmdstream BO handle %d used for region %d\n", 482 job->region_bo_handles[i], i); 483 ret = -EINVAL; 484 goto out_cleanup_job; 485 } 486 487 /* Verify the command stream doesn't have accesses outside the BO */ 488 if (cmd_info->region_size[i] > gem->size) { 489 dev_err(dev->dev, 490 "cmd stream region %d size greater than BO size (%llu > %zu)\n", 491 i, cmd_info->region_size[i], gem->size); 492 ret = -EOVERFLOW; 493 goto out_cleanup_job; 494 } 495 } 496 ret = ethosu_job_push(ejob); 497 if (!ret) { 498 ethosu_job_put(ejob); 499 return 0; 500 } 501 502 out_cleanup_job: 503 if (ret) 504 drm_sched_job_cleanup(&ejob->base); 505 out_put_job: 506 ethosu_job_err_cleanup(ejob); 507 508 return ret; 509 } 510 511 int ethosu_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *file) 512 { 513 struct drm_ethosu_submit *args = data; 514 int ret = 0; 515 unsigned int i = 0; 516 517 struct drm_ethosu_job __free(kvfree) *jobs = 518 kvmalloc_objs(*jobs, args->job_count); 519 if (!jobs) 520 return -ENOMEM; 521 522 if (copy_from_user(jobs, 523 (void __user *)(uintptr_t)args->jobs, 524 args->job_count * sizeof(*jobs))) { 525 drm_dbg(dev, "Failed to copy incoming job array\n"); 526 return -EFAULT; 527 } 528 529 for (i = 0; i < args->job_count; i++) { 530 ret = ethosu_ioctl_submit_job(dev, file, &jobs[i], args->perfmon_id); 531 if (ret) 532 return ret; 533 } 534 535 return 0; 536 } 537