1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 // Copyright (C) 2025 Arm, Ltd. 3 4 #include <linux/bitfield.h> 5 #include <linux/clk.h> 6 #include <linux/genalloc.h> 7 #include <linux/io.h> 8 #include <linux/iopoll.h> 9 #include <linux/module.h> 10 #include <linux/platform_device.h> 11 #include <linux/pm_runtime.h> 12 13 #include <drm/drm_drv.h> 14 #include <drm/drm_ioctl.h> 15 #include <drm/drm_utils.h> 16 #include <drm/drm_gem.h> 17 #include <drm/drm_accel.h> 18 #include <drm/ethosu_accel.h> 19 20 #include "ethosu_drv.h" 21 #include "ethosu_device.h" 22 #include "ethosu_gem.h" 23 #include "ethosu_job.h" 24 25 static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data, 26 struct drm_file *file) 27 { 28 struct ethosu_device *ethosudev = to_ethosu_device(ddev); 29 struct drm_ethosu_dev_query *args = data; 30 31 if (!args->pointer) { 32 switch (args->type) { 33 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 34 args->size = sizeof(ethosudev->npu_info); 35 return 0; 36 default: 37 return -EINVAL; 38 } 39 } 40 41 switch (args->type) { 42 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 43 if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size)) 44 return -EINVAL; 45 return copy_struct_to_user(u64_to_user_ptr(args->pointer), 46 args->size, 47 ðosudev->npu_info, 48 sizeof(ethosudev->npu_info), NULL); 49 default: 50 return -EINVAL; 51 } 52 } 53 54 #define ETHOSU_BO_FLAGS DRM_ETHOSU_BO_NO_MMAP 55 56 static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data, 57 struct drm_file *file) 58 { 59 struct drm_ethosu_bo_create *args = data; 60 int cookie, ret; 61 62 if (!drm_dev_enter(ddev, &cookie)) 63 return -ENODEV; 64 65 if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) { 66 ret = -EINVAL; 67 goto out_dev_exit; 68 } 69 70 ret = ethosu_gem_create_with_handle(file, ddev, &args->size, 71 args->flags, &args->handle); 72 73 out_dev_exit: 74 drm_dev_exit(cookie); 75 return ret; 76 } 77 78 static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data, 79 struct drm_file *file) 80 { 81 struct drm_ethosu_bo_wait *args = data; 82 int cookie, ret; 83 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 84 85 if (args->pad) 86 return -EINVAL; 87 88 if (!drm_dev_enter(ddev, &cookie)) 89 return -ENODEV; 90 91 ret = drm_gem_dma_resv_wait(file, args->handle, true, timeout); 92 93 drm_dev_exit(cookie); 94 return ret; 95 } 96 97 static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data, 98 struct drm_file *file) 99 { 100 struct drm_ethosu_bo_mmap_offset *args = data; 101 struct drm_gem_object *obj; 102 103 if (args->pad) 104 return -EINVAL; 105 106 obj = drm_gem_object_lookup(file, args->handle); 107 if (!obj) 108 return -ENOENT; 109 110 args->offset = drm_vma_node_offset_addr(&obj->vma_node); 111 drm_gem_object_put(obj); 112 return 0; 113 } 114 115 static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data, 116 struct drm_file *file) 117 { 118 struct drm_ethosu_cmdstream_bo_create *args = data; 119 int cookie, ret; 120 121 if (!drm_dev_enter(ddev, &cookie)) 122 return -ENODEV; 123 124 if (!args->size || !args->data || args->pad || args->flags) { 125 ret = -EINVAL; 126 goto out_dev_exit; 127 } 128 129 args->flags |= DRM_ETHOSU_BO_NO_MMAP; 130 131 ret = ethosu_gem_cmdstream_create(file, ddev, args->size, args->data, 132 args->flags, &args->handle); 133 134 out_dev_exit: 135 drm_dev_exit(cookie); 136 return ret; 137 } 138 139 static int ethosu_open(struct drm_device *ddev, struct drm_file *file) 140 { 141 int ret = 0; 142 143 if (!try_module_get(THIS_MODULE)) 144 return -EINVAL; 145 146 struct ethosu_file_priv __free(kfree) *priv = kzalloc_obj(*priv); 147 if (!priv) { 148 ret = -ENOMEM; 149 goto err_put_mod; 150 } 151 priv->edev = to_ethosu_device(ddev); 152 153 ret = ethosu_job_open(priv); 154 if (ret) 155 goto err_put_mod; 156 157 file->driver_priv = no_free_ptr(priv); 158 return 0; 159 160 err_put_mod: 161 module_put(THIS_MODULE); 162 return ret; 163 } 164 165 static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file) 166 { 167 ethosu_job_close(file->driver_priv); 168 kfree(file->driver_priv); 169 module_put(THIS_MODULE); 170 } 171 172 static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = { 173 #define ETHOSU_IOCTL(n, func, flags) \ 174 DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags) 175 176 ETHOSU_IOCTL(DEV_QUERY, dev_query, 0), 177 ETHOSU_IOCTL(BO_CREATE, bo_create, 0), 178 ETHOSU_IOCTL(BO_WAIT, bo_wait, 0), 179 ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0), 180 ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0), 181 ETHOSU_IOCTL(SUBMIT, submit, 0), 182 }; 183 184 DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops); 185 186 /* 187 * Ethosu driver version: 188 * - 1.0 - initial interface 189 */ 190 static const struct drm_driver ethosu_drm_driver = { 191 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM, 192 .open = ethosu_open, 193 .postclose = ethosu_postclose, 194 .ioctls = ethosu_drm_driver_ioctls, 195 .num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls), 196 .fops = ðosu_drm_driver_fops, 197 .name = "ethosu", 198 .desc = "Arm Ethos-U Accel driver", 199 .major = 1, 200 .minor = 0, 201 202 .gem_create_object = ethosu_gem_create_object, 203 }; 204 205 #define U65_DRAM_AXI_LIMIT_CFG 0x1f3f0002 206 #define U65_SRAM_AXI_LIMIT_CFG 0x1f3f00b0 207 #define U85_AXI_EXT_CFG 0x00021f3f 208 #define U85_AXI_SRAM_CFG 0x00021f3f 209 #define U85_MEM_ATTR0_CFG 0x00000000 210 #define U85_MEM_ATTR2_CFG 0x000000b7 211 212 static int ethosu_reset(struct ethosu_device *ethosudev) 213 { 214 int ret; 215 u32 reg; 216 217 writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET); 218 ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg, 219 !FIELD_GET(STATUS_RESET_STATUS, reg), 220 USEC_PER_MSEC, USEC_PER_SEC); 221 if (ret) 222 return ret; 223 224 if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) { 225 dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n", 226 readl_relaxed(ethosudev->regs + NPU_REG_PROT)); 227 } 228 229 /* 230 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0), 231 * everything else to AXI M1 (AXILIMIT2) 232 */ 233 writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG); 234 if (ethosu_is_u65(ethosudev)) { 235 writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0); 236 writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2); 237 } else { 238 writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM); 239 writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT); 240 writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0); // SRAM 241 writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2); // DRAM 242 } 243 244 if (ethosudev->sram) 245 memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size); 246 247 return 0; 248 } 249 250 static int ethosu_device_resume(struct device *dev) 251 { 252 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 253 int ret; 254 255 ret = clk_bulk_prepare_enable(ethosudev->num_clks, ethosudev->clks); 256 if (ret) 257 return ret; 258 259 ret = ethosu_reset(ethosudev); 260 if (!ret) 261 return 0; 262 263 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 264 return ret; 265 } 266 267 static int ethosu_device_suspend(struct device *dev) 268 { 269 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 270 271 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 272 return 0; 273 } 274 275 static int ethosu_sram_init(struct ethosu_device *ethosudev) 276 { 277 ethosudev->npu_info.sram_size = 0; 278 279 ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0); 280 if (!ethosudev->srampool) 281 return 0; 282 283 ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool); 284 285 ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(ethosudev->srampool, 286 ethosudev->npu_info.sram_size, 287 ðosudev->sramphys); 288 if (!ethosudev->sram) { 289 dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n"); 290 return -ENOMEM; 291 } 292 293 return 0; 294 } 295 296 static int ethosu_init(struct ethosu_device *ethosudev) 297 { 298 int ret; 299 u32 id, config; 300 301 ret = ethosu_device_resume(ethosudev->base.dev); 302 if (ret) 303 return ret; 304 305 pm_runtime_set_autosuspend_delay(ethosudev->base.dev, 50); 306 pm_runtime_use_autosuspend(ethosudev->base.dev); 307 ret = devm_pm_runtime_set_active_enabled(ethosudev->base.dev); 308 if (ret) 309 return ret; 310 pm_runtime_get_noresume(ethosudev->base.dev); 311 312 ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID); 313 ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG); 314 315 ethosu_sram_init(ethosudev); 316 317 dev_info(ethosudev->base.dev, 318 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM\n", 319 FIELD_GET(ID_ARCH_MAJOR_MASK, id), 320 FIELD_GET(ID_ARCH_MINOR_MASK, id), 321 FIELD_GET(ID_ARCH_PATCH_MASK, id), 322 FIELD_GET(ID_VER_MAJOR_MASK, id), 323 FIELD_GET(ID_VER_MINOR_MASK, id), 324 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config), 325 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config), 326 ethosudev->npu_info.sram_size / 1024); 327 328 return 0; 329 } 330 331 static int ethosu_probe(struct platform_device *pdev) 332 { 333 int ret; 334 struct ethosu_device *ethosudev; 335 336 ethosudev = devm_drm_dev_alloc(&pdev->dev, ðosu_drm_driver, 337 struct ethosu_device, base); 338 if (IS_ERR(ethosudev)) 339 return -ENOMEM; 340 platform_set_drvdata(pdev, ethosudev); 341 342 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); 343 344 ethosudev->regs = devm_platform_ioremap_resource(pdev, 0); 345 346 ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, ðosudev->clks); 347 if (ethosudev->num_clks < 0) 348 return ethosudev->num_clks; 349 350 ret = ethosu_job_init(ethosudev); 351 if (ret) 352 return ret; 353 354 ret = ethosu_init(ethosudev); 355 if (ret) 356 return ret; 357 358 ret = drm_dev_register(ðosudev->base, 0); 359 if (ret) 360 pm_runtime_dont_use_autosuspend(ethosudev->base.dev); 361 362 pm_runtime_put_autosuspend(ethosudev->base.dev); 363 return ret; 364 } 365 366 static void ethosu_remove(struct platform_device *pdev) 367 { 368 struct ethosu_device *ethosudev = dev_get_drvdata(&pdev->dev); 369 370 drm_dev_unregister(ðosudev->base); 371 ethosu_job_fini(ethosudev); 372 if (ethosudev->sram) 373 gen_pool_free(ethosudev->srampool, (unsigned long)ethosudev->sram, 374 ethosudev->npu_info.sram_size); 375 } 376 377 static const struct of_device_id dt_match[] = { 378 { .compatible = "arm,ethos-u65" }, 379 { .compatible = "arm,ethos-u85" }, 380 {} 381 }; 382 MODULE_DEVICE_TABLE(of, dt_match); 383 384 static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops, 385 ethosu_device_suspend, 386 ethosu_device_resume, 387 NULL); 388 389 static struct platform_driver ethosu_driver = { 390 .probe = ethosu_probe, 391 .remove = ethosu_remove, 392 .driver = { 393 .name = "ethosu", 394 .pm = pm_ptr(ðosu_pm_ops), 395 .of_match_table = dt_match, 396 }, 397 }; 398 module_platform_driver(ethosu_driver); 399 400 MODULE_AUTHOR("Rob Herring <robh@kernel.org>"); 401 MODULE_DESCRIPTION("Arm Ethos-U Accel Driver"); 402 MODULE_LICENSE("Dual MIT/GPL"); 403