1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 // Copyright (C) 2025 Arm, Ltd. 3 4 #include <linux/bitfield.h> 5 #include <linux/clk.h> 6 #include <linux/genalloc.h> 7 #include <linux/io.h> 8 #include <linux/iopoll.h> 9 #include <linux/module.h> 10 #include <linux/mod_devicetable.h> 11 #include <linux/platform_device.h> 12 #include <linux/pm_runtime.h> 13 14 #include <drm/drm_drv.h> 15 #include <drm/drm_ioctl.h> 16 #include <drm/drm_utils.h> 17 #include <drm/drm_gem.h> 18 #include <drm/drm_accel.h> 19 #include <drm/ethosu_accel.h> 20 21 #include "ethosu_drv.h" 22 #include "ethosu_device.h" 23 #include "ethosu_gem.h" 24 #include "ethosu_job.h" 25 26 static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data, 27 struct drm_file *file) 28 { 29 struct ethosu_device *ethosudev = to_ethosu_device(ddev); 30 struct drm_ethosu_dev_query *args = data; 31 32 if (!args->pointer) { 33 switch (args->type) { 34 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 35 args->size = sizeof(ethosudev->npu_info); 36 return 0; 37 default: 38 return -EINVAL; 39 } 40 } 41 42 switch (args->type) { 43 case DRM_ETHOSU_DEV_QUERY_NPU_INFO: 44 if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size)) 45 return -EINVAL; 46 return copy_struct_to_user(u64_to_user_ptr(args->pointer), 47 args->size, 48 ðosudev->npu_info, 49 sizeof(ethosudev->npu_info), NULL); 50 default: 51 return -EINVAL; 52 } 53 } 54 55 #define ETHOSU_BO_FLAGS DRM_ETHOSU_BO_NO_MMAP 56 57 static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data, 58 struct drm_file *file) 59 { 60 struct drm_ethosu_bo_create *args = data; 61 int cookie, ret; 62 63 if (!drm_dev_enter(ddev, &cookie)) 64 return -ENODEV; 65 66 if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) { 67 ret = -EINVAL; 68 goto out_dev_exit; 69 } 70 71 ret = ethosu_gem_create_with_handle(file, ddev, &args->size, 72 args->flags, &args->handle); 73 74 out_dev_exit: 75 drm_dev_exit(cookie); 76 return ret; 77 } 78 79 static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data, 80 struct drm_file *file) 81 { 82 struct drm_ethosu_bo_wait *args = data; 83 int cookie, ret; 84 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 85 86 if (args->pad) 87 return -EINVAL; 88 89 if (!drm_dev_enter(ddev, &cookie)) 90 return -ENODEV; 91 92 ret = drm_gem_dma_resv_wait(file, args->handle, true, timeout); 93 94 drm_dev_exit(cookie); 95 return ret; 96 } 97 98 static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data, 99 struct drm_file *file) 100 { 101 struct drm_ethosu_bo_mmap_offset *args = data; 102 struct drm_gem_object *obj; 103 104 if (args->pad) 105 return -EINVAL; 106 107 obj = drm_gem_object_lookup(file, args->handle); 108 if (!obj) 109 return -ENOENT; 110 111 args->offset = drm_vma_node_offset_addr(&obj->vma_node); 112 drm_gem_object_put(obj); 113 return 0; 114 } 115 116 static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data, 117 struct drm_file *file) 118 { 119 struct drm_ethosu_cmdstream_bo_create *args = data; 120 int cookie, ret; 121 122 if (!drm_dev_enter(ddev, &cookie)) 123 return -ENODEV; 124 125 if (!args->size || !args->data || args->pad || args->flags) { 126 ret = -EINVAL; 127 goto out_dev_exit; 128 } 129 130 args->flags |= DRM_ETHOSU_BO_NO_MMAP; 131 132 ret = ethosu_gem_cmdstream_create(file, ddev, args->size, args->data, 133 args->flags, &args->handle); 134 135 out_dev_exit: 136 drm_dev_exit(cookie); 137 return ret; 138 } 139 140 static int ethosu_open(struct drm_device *ddev, struct drm_file *file) 141 { 142 int ret = 0; 143 144 if (!try_module_get(THIS_MODULE)) 145 return -EINVAL; 146 147 struct ethosu_file_priv __free(kfree) *priv = kzalloc(sizeof(*priv), GFP_KERNEL); 148 if (!priv) { 149 ret = -ENOMEM; 150 goto err_put_mod; 151 } 152 priv->edev = to_ethosu_device(ddev); 153 154 ret = ethosu_job_open(priv); 155 if (ret) 156 goto err_put_mod; 157 158 file->driver_priv = no_free_ptr(priv); 159 return 0; 160 161 err_put_mod: 162 module_put(THIS_MODULE); 163 return ret; 164 } 165 166 static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file) 167 { 168 ethosu_job_close(file->driver_priv); 169 kfree(file->driver_priv); 170 module_put(THIS_MODULE); 171 } 172 173 static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = { 174 #define ETHOSU_IOCTL(n, func, flags) \ 175 DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags) 176 177 ETHOSU_IOCTL(DEV_QUERY, dev_query, 0), 178 ETHOSU_IOCTL(BO_CREATE, bo_create, 0), 179 ETHOSU_IOCTL(BO_WAIT, bo_wait, 0), 180 ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0), 181 ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0), 182 ETHOSU_IOCTL(SUBMIT, submit, 0), 183 }; 184 185 DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops); 186 187 /* 188 * Ethosu driver version: 189 * - 1.0 - initial interface 190 */ 191 static const struct drm_driver ethosu_drm_driver = { 192 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM, 193 .open = ethosu_open, 194 .postclose = ethosu_postclose, 195 .ioctls = ethosu_drm_driver_ioctls, 196 .num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls), 197 .fops = ðosu_drm_driver_fops, 198 .name = "ethosu", 199 .desc = "Arm Ethos-U Accel driver", 200 .major = 1, 201 .minor = 0, 202 203 .gem_create_object = ethosu_gem_create_object, 204 }; 205 206 #define U65_DRAM_AXI_LIMIT_CFG 0x1f3f0002 207 #define U65_SRAM_AXI_LIMIT_CFG 0x1f3f00b0 208 #define U85_AXI_EXT_CFG 0x00021f3f 209 #define U85_AXI_SRAM_CFG 0x00021f3f 210 #define U85_MEM_ATTR0_CFG 0x00000000 211 #define U85_MEM_ATTR2_CFG 0x000000b7 212 213 static int ethosu_reset(struct ethosu_device *ethosudev) 214 { 215 int ret; 216 u32 reg; 217 218 writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET); 219 ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg, 220 !FIELD_GET(STATUS_RESET_STATUS, reg), 221 USEC_PER_MSEC, USEC_PER_SEC); 222 if (ret) 223 return ret; 224 225 if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) { 226 dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n", 227 readl_relaxed(ethosudev->regs + NPU_REG_PROT)); 228 } 229 230 /* 231 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0), 232 * everything else to AXI M1 (AXILIMIT2) 233 */ 234 writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG); 235 if (ethosu_is_u65(ethosudev)) { 236 writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0); 237 writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2); 238 } else { 239 writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM); 240 writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT); 241 writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0); // SRAM 242 writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2); // DRAM 243 } 244 245 if (ethosudev->sram) 246 memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size); 247 248 return 0; 249 } 250 251 static int ethosu_device_resume(struct device *dev) 252 { 253 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 254 int ret; 255 256 ret = clk_bulk_prepare_enable(ethosudev->num_clks, ethosudev->clks); 257 if (ret) 258 return ret; 259 260 ret = ethosu_reset(ethosudev); 261 if (!ret) 262 return 0; 263 264 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 265 return ret; 266 } 267 268 static int ethosu_device_suspend(struct device *dev) 269 { 270 struct ethosu_device *ethosudev = dev_get_drvdata(dev); 271 272 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks); 273 return 0; 274 } 275 276 static int ethosu_sram_init(struct ethosu_device *ethosudev) 277 { 278 ethosudev->npu_info.sram_size = 0; 279 280 ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0); 281 if (!ethosudev->srampool) 282 return 0; 283 284 ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool); 285 286 ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(ethosudev->srampool, 287 ethosudev->npu_info.sram_size, 288 ðosudev->sramphys); 289 if (!ethosudev->sram) { 290 dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n"); 291 return -ENOMEM; 292 } 293 294 return 0; 295 } 296 297 static int ethosu_init(struct ethosu_device *ethosudev) 298 { 299 int ret; 300 u32 id, config; 301 302 ret = ethosu_device_resume(ethosudev->base.dev); 303 if (ret) 304 return ret; 305 306 pm_runtime_set_autosuspend_delay(ethosudev->base.dev, 50); 307 pm_runtime_use_autosuspend(ethosudev->base.dev); 308 ret = devm_pm_runtime_set_active_enabled(ethosudev->base.dev); 309 if (ret) 310 return ret; 311 pm_runtime_get_noresume(ethosudev->base.dev); 312 313 ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID); 314 ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG); 315 316 ethosu_sram_init(ethosudev); 317 318 dev_info(ethosudev->base.dev, 319 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM\n", 320 FIELD_GET(ID_ARCH_MAJOR_MASK, id), 321 FIELD_GET(ID_ARCH_MINOR_MASK, id), 322 FIELD_GET(ID_ARCH_PATCH_MASK, id), 323 FIELD_GET(ID_VER_MAJOR_MASK, id), 324 FIELD_GET(ID_VER_MINOR_MASK, id), 325 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config), 326 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config), 327 ethosudev->npu_info.sram_size / 1024); 328 329 return 0; 330 } 331 332 static int ethosu_probe(struct platform_device *pdev) 333 { 334 int ret; 335 struct ethosu_device *ethosudev; 336 337 ethosudev = devm_drm_dev_alloc(&pdev->dev, ðosu_drm_driver, 338 struct ethosu_device, base); 339 if (IS_ERR(ethosudev)) 340 return -ENOMEM; 341 platform_set_drvdata(pdev, ethosudev); 342 343 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); 344 345 ethosudev->regs = devm_platform_ioremap_resource(pdev, 0); 346 347 ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, ðosudev->clks); 348 if (ethosudev->num_clks < 0) 349 return ethosudev->num_clks; 350 351 ret = ethosu_job_init(ethosudev); 352 if (ret) 353 return ret; 354 355 ret = ethosu_init(ethosudev); 356 if (ret) 357 return ret; 358 359 ret = drm_dev_register(ðosudev->base, 0); 360 if (ret) 361 pm_runtime_dont_use_autosuspend(ethosudev->base.dev); 362 363 pm_runtime_put_autosuspend(ethosudev->base.dev); 364 return ret; 365 } 366 367 static void ethosu_remove(struct platform_device *pdev) 368 { 369 struct ethosu_device *ethosudev = dev_get_drvdata(&pdev->dev); 370 371 drm_dev_unregister(ðosudev->base); 372 ethosu_job_fini(ethosudev); 373 if (ethosudev->sram) 374 gen_pool_free(ethosudev->srampool, (unsigned long)ethosudev->sram, 375 ethosudev->npu_info.sram_size); 376 } 377 378 static const struct of_device_id dt_match[] = { 379 { .compatible = "arm,ethos-u65" }, 380 { .compatible = "arm,ethos-u85" }, 381 {} 382 }; 383 MODULE_DEVICE_TABLE(of, dt_match); 384 385 static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops, 386 ethosu_device_suspend, 387 ethosu_device_resume, 388 NULL); 389 390 static struct platform_driver ethosu_driver = { 391 .probe = ethosu_probe, 392 .remove = ethosu_remove, 393 .driver = { 394 .name = "ethosu", 395 .pm = pm_ptr(ðosu_pm_ops), 396 .of_match_table = dt_match, 397 }, 398 }; 399 module_platform_driver(ethosu_driver); 400 401 MODULE_AUTHOR("Rob Herring <robh@kernel.org>"); 402 MODULE_DESCRIPTION("Arm Ethos-U Accel Driver"); 403 MODULE_LICENSE("Dual MIT/GPL"); 404