1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 // Copyright (C) 2025 Arm, Ltd. 3 4 #include <linux/bitfield.h> 5 #include <linux/clk.h> 6 #include <linux/genalloc.h> 7 #include <linux/io.h> 8 #include <linux/iopoll.h> 9 #include <linux/module.h> 10 #include <linux/platform_device.h> 11 #include <linux/pm_runtime.h> 12 13 #include <drm/drm_drv.h> 14 #include <drm/drm_ioctl.h> 15 #include <drm/drm_utils.h> 16 #include <drm/drm_gem.h> 17 #include <drm/drm_accel.h> 18 #include <drm/drm_managed.h> 19 #include <drm/ethosu_accel.h> 20 21 #include "ethosu_drv.h" 22 #include "ethosu_device.h" 23 #include "ethosu_gem.h" 24 #include "ethosu_job.h" 25 26 static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data, 27 struct drm_file *file) 28 { 29 struct ethosu_device *ethosudev = to_ethosu_device(ddev); 30 struct drm_ethosu_dev_query *args = data; 31 32 if (!args->pointer) { 33 switch (args->type) { 34 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 35 args->size = sizeof(ethosudev->npu_info); 36 return 0; 37 default: 38 return -EINVAL; 39 } 40 } 41 42 switch (args->type) { 43 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 44 if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size)) 45 return -EINVAL; 46 return copy_struct_to_user(u64_to_user_ptr(args->pointer), 47 args->size, 48 ðosudev->npu_info, 49 sizeof(ethosudev->npu_info), NULL); 50 default: 51 return -EINVAL; 52 } 53 } 54 55 #define ETHOSU_BO_FLAGS DRM_ETHOSU_BO_NO_MMAP 56 57 static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data, 58 struct drm_file *file) 59 { 60 struct drm_ethosu_bo_create *args = data; 61 int cookie, ret; 62 63 if (!drm_dev_enter(ddev, &cookie)) 64 return -ENODEV; 65 66 if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) { 67 ret = -EINVAL; 68 goto out_dev_exit; 69 } 70 71 ret = ethosu_gem_create_with_handle(file, ddev, &args->size, 72 args->flags, &args->handle); 73 74 out_dev_exit: 75 drm_dev_exit(cookie); 76 return ret; 77 } 78 79 static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data, 80 struct drm_file *file) 81 { 82 struct drm_ethosu_bo_wait *args = data; 83 int cookie, ret; 84 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 85 86 if (args->pad) 87 return -EINVAL; 88 89 if (!drm_dev_enter(ddev, &cookie)) 90 return -ENODEV; 91 92 ret = drm_gem_dma_resv_wait(file, args->handle, true, timeout); 93 94 drm_dev_exit(cookie); 95 return ret; 96 } 97 98 static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data, 99 struct drm_file *file) 100 { 101 struct drm_ethosu_bo_mmap_offset *args = data; 102 struct drm_gem_object *obj; 103 104 if (args->pad) 105 return -EINVAL; 106 107 obj = drm_gem_object_lookup(file, args->handle); 108 if (!obj) 109 return -ENOENT; 110 111 args->offset = drm_vma_node_offset_addr(&obj->vma_node); 112 drm_gem_object_put(obj); 113 return 0; 114 } 115 116 static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data, 117 struct drm_file *file) 118 { 119 struct drm_ethosu_cmdstream_bo_create *args = data; 120 int cookie, ret; 121 122 if (!drm_dev_enter(ddev, &cookie)) 123 return -ENODEV; 124 125 if (!args->size || !args->data || args->pad || args->flags) { 126 ret = -EINVAL; 127 goto out_dev_exit; 128 } 129 130 args->flags |= DRM_ETHOSU_BO_NO_MMAP; 131 132 ret = ethosu_gem_cmdstream_create(file, ddev, args->size, args->data, 133 args->flags, &args->handle); 134 135 out_dev_exit: 136 drm_dev_exit(cookie); 137 return ret; 138 } 139 140 static int ethosu_open(struct drm_device *ddev, struct drm_file *file) 141 { 142 int ret = 0; 143 144 if (!try_module_get(THIS_MODULE)) 145 return -EINVAL; 146 147 struct ethosu_file_priv __free(kfree) *priv = kzalloc_obj(*priv); 148 if (!priv) { 149 ret = -ENOMEM; 150 goto err_put_mod; 151 } 152 priv->edev = to_ethosu_device(ddev); 153 154 ret = ethosu_job_open(priv); 155 if (ret) 156 goto err_put_mod; 157 158 ethosu_perfmon_open_file(priv); 159 file->driver_priv = no_free_ptr(priv); 160 return 0; 161 162 err_put_mod: 163 module_put(THIS_MODULE); 164 return ret; 165 } 166 167 static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file) 168 { 169 ethosu_job_close(file->driver_priv); 170 ethosu_perfmon_close_file(file->driver_priv); 171 kfree(file->driver_priv); 172 module_put(THIS_MODULE); 173 } 174 175 static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = { 176 #define ETHOSU_IOCTL(n, func, flags) \ 177 DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags) 178 179 ETHOSU_IOCTL(DEV_QUERY, dev_query, 0), 180 ETHOSU_IOCTL(BO_CREATE, bo_create, 0), 181 ETHOSU_IOCTL(BO_WAIT, bo_wait, 0), 182 ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0), 183 ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0), 184 ETHOSU_IOCTL(SUBMIT, submit, 0), 185 ETHOSU_IOCTL(PERFMON_CREATE, perfmon_create, 0), 186 ETHOSU_IOCTL(PERFMON_DESTROY, perfmon_destroy, 0), 187 ETHOSU_IOCTL(PERFMON_GET_VALUES, perfmon_get_values, 0), 188 ETHOSU_IOCTL(PERFMON_SET_GLOBAL, perfmon_set_global, 0), 189 }; 190 191 DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops); 192 193 /* 194 * Ethosu driver version: 195 * - 1.0 - initial interface 196 */ 197 static const struct drm_driver ethosu_drm_driver = { 198 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM, 199 .open = ethosu_open, 200 .postclose = ethosu_postclose, 201 .ioctls = ethosu_drm_driver_ioctls, 202 .num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls), 203 .fops = ðosu_drm_driver_fops, 204 .name = "ethosu", 205 .desc = "Arm Ethos-U Accel driver", 206 .major = 1, 207 .minor = 0, 208 209 .gem_create_object = ethosu_gem_create_object, 210 }; 211 212 #define U65_DRAM_AXI_LIMIT_CFG 0x1f3f0002 213 #define U65_SRAM_AXI_LIMIT_CFG 0x1f3f00b0 214 #define U85_AXI_EXT_CFG 0x00021f3f 215 #define U85_AXI_SRAM_CFG 0x00021f3f 216 #define U85_MEM_ATTR0_CFG 0x00000000 217 #define U85_MEM_ATTR2_CFG 0x000000b7 218 219 static int ethosu_reset(struct ethosu_device *ethosudev) 220 { 221 int ret; 222 u32 reg; 223 224 writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET); 225 ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg, 226 !FIELD_GET(STATUS_RESET_STATUS, reg), 227 USEC_PER_MSEC, USEC_PER_SEC); 228 if (ret) 229 return ret; 230 231 if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) { 232 dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n", 233 readl_relaxed(ethosudev->regs + NPU_REG_PROT)); 234 } 235 236 /* 237 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0), 238 * everything else to AXI M1 (AXILIMIT2) 239 */ 240 writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG); 241 if (ethosu_is_u65(ethosudev)) { 242 writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0); 243 writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2); 244 } else { 245 writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM); 246 writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT); 247 writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0); // SRAM 248 writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2); // DRAM 249 } 250 251 if (ethosudev->sram) 252 memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size); 253 254 return 0; 255 } 256 257 static int ethosu_device_resume(struct device *dev) 258 { 259 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 260 int ret; 261 262 ret = clk_bulk_prepare_enable(ethosudev->num_clks, ethosudev->clks); 263 if (ret) 264 return ret; 265 266 ret = ethosu_reset(ethosudev); 267 if (!ret) 268 return 0; 269 270 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 271 return ret; 272 } 273 274 static int ethosu_device_suspend(struct device *dev) 275 { 276 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 277 278 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 279 return 0; 280 } 281 282 static int ethosu_sram_init(struct ethosu_device *ethosudev) 283 { 284 ethosudev->npu_info.sram_size = 0; 285 286 ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0); 287 if (!ethosudev->srampool) 288 return 0; 289 290 ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool); 291 292 ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(ethosudev->srampool, 293 ethosudev->npu_info.sram_size, 294 ðosudev->sramphys); 295 if (!ethosudev->sram) { 296 dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n"); 297 return -ENOMEM; 298 } 299 300 return 0; 301 } 302 303 static int ethosu_init(struct ethosu_device *ethosudev) 304 { 305 int ret; 306 u32 id, config; 307 308 ret = ethosu_device_resume(ethosudev->base.dev); 309 if (ret) 310 return ret; 311 312 pm_runtime_set_autosuspend_delay(ethosudev->base.dev, 50); 313 pm_runtime_use_autosuspend(ethosudev->base.dev); 314 ret = devm_pm_runtime_set_active_enabled(ethosudev->base.dev); 315 if (ret) 316 return ret; 317 pm_runtime_get_noresume(ethosudev->base.dev); 318 319 ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID); 320 ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG); 321 322 ethosu_sram_init(ethosudev); 323 324 if (!ethosu_is_u65(ethosudev)) 325 ethosudev->pmu_regs += 0x1000; 326 327 ethosudev->npu_info.pmu_counters = FIELD_GET(PMCR_NUM_EVENT_CNT_MASK, 328 readl_relaxed(ethosudev->pmu_regs + NPU_REG_PMCR)); 329 330 dev_info(ethosudev->base.dev, 331 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM, %d PMU cntrs\n", 332 FIELD_GET(ID_ARCH_MAJOR_MASK, id), 333 FIELD_GET(ID_ARCH_MINOR_MASK, id), 334 FIELD_GET(ID_ARCH_PATCH_MASK, id), 335 FIELD_GET(ID_VER_MAJOR_MASK, id), 336 FIELD_GET(ID_VER_MINOR_MASK, id), 337 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config), 338 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config), 339 ethosudev->npu_info.sram_size / 1024, 340 ethosudev->npu_info.pmu_counters); 341 342 return 0; 343 } 344 345 static int ethosu_probe(struct platform_device *pdev) 346 { 347 int ret; 348 struct ethosu_device *ethosudev; 349 350 ethosudev = devm_drm_dev_alloc(&pdev->dev, ðosu_drm_driver, 351 struct ethosu_device, base); 352 if (IS_ERR(ethosudev)) 353 return -ENOMEM; 354 platform_set_drvdata(pdev, ethosudev); 355 356 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); 357 358 ethosudev->regs = devm_platform_ioremap_resource(pdev, 0); 359 ethosudev->pmu_regs = ethosudev->regs; 360 361 ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, ðosudev->clks); 362 if (ethosudev->num_clks < 0) 363 return ethosudev->num_clks; 364 365 ret = drmm_mutex_init(ðosudev->base, ðosudev->perfmon_state.lock); 366 if (ret) 367 return ret; 368 369 ret = ethosu_job_init(ethosudev); 370 if (ret) 371 return ret; 372 373 ret = ethosu_init(ethosudev); 374 if (ret) 375 return ret; 376 377 ret = drm_dev_register(ðosudev->base, 0); 378 if (ret) 379 pm_runtime_dont_use_autosuspend(ethosudev->base.dev); 380 381 pm_runtime_put_autosuspend(ethosudev->base.dev); 382 return ret; 383 } 384 385 static void ethosu_remove(struct platform_device *pdev) 386 { 387 struct ethosu_device *ethosudev = dev_get_drvdata(&pdev->dev); 388 389 drm_dev_unregister(ðosudev->base); 390 ethosu_job_fini(ethosudev); 391 if (ethosudev->sram) 392 gen_pool_free(ethosudev->srampool, (unsigned long)ethosudev->sram, 393 ethosudev->npu_info.sram_size); 394 } 395 396 static const struct of_device_id dt_match[] = { 397 { .compatible = "arm,ethos-u65" }, 398 { .compatible = "arm,ethos-u85" }, 399 {} 400 }; 401 MODULE_DEVICE_TABLE(of, dt_match); 402 403 static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops, 404 ethosu_device_suspend, 405 ethosu_device_resume, 406 NULL); 407 408 static struct platform_driver ethosu_driver = { 409 .probe = ethosu_probe, 410 .remove = ethosu_remove, 411 .driver = { 412 .name = "ethosu", 413 .pm = pm_ptr(ðosu_pm_ops), 414 .of_match_table = dt_match, 415 }, 416 }; 417 module_platform_driver(ethosu_driver); 418 419 MODULE_AUTHOR("Rob Herring <robh@kernel.org>"); 420 MODULE_DESCRIPTION("Arm Ethos-U Accel Driver"); 421 MODULE_LICENSE("Dual MIT/GPL"); 422