1 // SPDX-License-Identifier: GPL-2.0-only or MIT
2 // Copyright (C) 2025 Arm, Ltd.
3
4 #include <linux/bitfield.h>
5 #include <linux/clk.h>
6 #include <linux/genalloc.h>
7 #include <linux/io.h>
8 #include <linux/iopoll.h>
9 #include <linux/module.h>
10 #include <linux/platform_device.h>
11 #include <linux/pm_runtime.h>
12
13 #include <drm/drm_drv.h>
14 #include <drm/drm_ioctl.h>
15 #include <drm/drm_utils.h>
16 #include <drm/drm_gem.h>
17 #include <drm/drm_accel.h>
18 #include <drm/drm_managed.h>
19 #include <drm/ethosu_accel.h>
20
21 #include "ethosu_drv.h"
22 #include "ethosu_device.h"
23 #include "ethosu_gem.h"
24 #include "ethosu_job.h"
25
ethosu_ioctl_dev_query(struct drm_device * ddev,void * data,struct drm_file * file)26 static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data,
27 struct drm_file *file)
28 {
29 struct ethosu_device *ethosudev = to_ethosu_device(ddev);
30 struct drm_ethosu_dev_query *args = data;
31
32 if (!args->pointer) {
33 switch (args->type) {
34 case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
35 args->size = sizeof(ethosudev->npu_info);
36 return 0;
37 default:
38 return -EINVAL;
39 }
40 }
41
42 switch (args->type) {
43 case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
44 if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size))
45 return -EINVAL;
46 return copy_struct_to_user(u64_to_user_ptr(args->pointer),
47 args->size,
48 ðosudev->npu_info,
49 sizeof(ethosudev->npu_info), NULL);
50 default:
51 return -EINVAL;
52 }
53 }
54
55 #define ETHOSU_BO_FLAGS DRM_ETHOSU_BO_NO_MMAP
56
ethosu_ioctl_bo_create(struct drm_device * ddev,void * data,struct drm_file * file)57 static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data,
58 struct drm_file *file)
59 {
60 struct drm_ethosu_bo_create *args = data;
61 int cookie, ret;
62
63 if (!drm_dev_enter(ddev, &cookie))
64 return -ENODEV;
65
66 if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) {
67 ret = -EINVAL;
68 goto out_dev_exit;
69 }
70
71 ret = ethosu_gem_create_with_handle(file, ddev, &args->size,
72 args->flags, &args->handle);
73
74 out_dev_exit:
75 drm_dev_exit(cookie);
76 return ret;
77 }
78
ethosu_ioctl_bo_wait(struct drm_device * ddev,void * data,struct drm_file * file)79 static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data,
80 struct drm_file *file)
81 {
82 struct drm_ethosu_bo_wait *args = data;
83 int cookie, ret;
84 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
85
86 if (args->pad)
87 return -EINVAL;
88
89 if (!drm_dev_enter(ddev, &cookie))
90 return -ENODEV;
91
92 ret = drm_gem_dma_resv_wait(file, args->handle, true, timeout);
93
94 drm_dev_exit(cookie);
95 return ret;
96 }
97
ethosu_ioctl_bo_mmap_offset(struct drm_device * ddev,void * data,struct drm_file * file)98 static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data,
99 struct drm_file *file)
100 {
101 struct drm_ethosu_bo_mmap_offset *args = data;
102 struct drm_gem_object *obj;
103
104 if (args->pad)
105 return -EINVAL;
106
107 obj = drm_gem_object_lookup(file, args->handle);
108 if (!obj)
109 return -ENOENT;
110
111 args->offset = drm_vma_node_offset_addr(&obj->vma_node);
112 drm_gem_object_put(obj);
113 return 0;
114 }
115
ethosu_ioctl_cmdstream_bo_create(struct drm_device * ddev,void * data,struct drm_file * file)116 static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data,
117 struct drm_file *file)
118 {
119 struct drm_ethosu_cmdstream_bo_create *args = data;
120 int cookie, ret;
121
122 if (!drm_dev_enter(ddev, &cookie))
123 return -ENODEV;
124
125 if (!args->size || !args->data || args->pad || args->flags) {
126 ret = -EINVAL;
127 goto out_dev_exit;
128 }
129
130 args->flags |= DRM_ETHOSU_BO_NO_MMAP;
131
132 ret = ethosu_gem_cmdstream_create(file, ddev, args->size, args->data,
133 args->flags, &args->handle);
134
135 out_dev_exit:
136 drm_dev_exit(cookie);
137 return ret;
138 }
139
ethosu_open(struct drm_device * ddev,struct drm_file * file)140 static int ethosu_open(struct drm_device *ddev, struct drm_file *file)
141 {
142 int ret = 0;
143
144 if (!try_module_get(THIS_MODULE))
145 return -EINVAL;
146
147 struct ethosu_file_priv __free(kfree) *priv = kzalloc_obj(*priv);
148 if (!priv) {
149 ret = -ENOMEM;
150 goto err_put_mod;
151 }
152 priv->edev = to_ethosu_device(ddev);
153
154 ret = ethosu_job_open(priv);
155 if (ret)
156 goto err_put_mod;
157
158 ethosu_perfmon_open_file(priv);
159 file->driver_priv = no_free_ptr(priv);
160 return 0;
161
162 err_put_mod:
163 module_put(THIS_MODULE);
164 return ret;
165 }
166
ethosu_postclose(struct drm_device * ddev,struct drm_file * file)167 static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file)
168 {
169 ethosu_job_close(file->driver_priv);
170 ethosu_perfmon_close_file(file->driver_priv);
171 kfree(file->driver_priv);
172 module_put(THIS_MODULE);
173 }
174
175 static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = {
176 #define ETHOSU_IOCTL(n, func, flags) \
177 DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags)
178
179 ETHOSU_IOCTL(DEV_QUERY, dev_query, 0),
180 ETHOSU_IOCTL(BO_CREATE, bo_create, 0),
181 ETHOSU_IOCTL(BO_WAIT, bo_wait, 0),
182 ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0),
183 ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0),
184 ETHOSU_IOCTL(SUBMIT, submit, 0),
185 ETHOSU_IOCTL(PERFMON_CREATE, perfmon_create, 0),
186 ETHOSU_IOCTL(PERFMON_DESTROY, perfmon_destroy, 0),
187 ETHOSU_IOCTL(PERFMON_GET_VALUES, perfmon_get_values, 0),
188 ETHOSU_IOCTL(PERFMON_SET_GLOBAL, perfmon_set_global, 0),
189 };
190
191 DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops);
192
193 /*
194 * Ethosu driver version:
195 * - 1.0 - initial interface
196 */
197 static const struct drm_driver ethosu_drm_driver = {
198 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM,
199 .open = ethosu_open,
200 .postclose = ethosu_postclose,
201 .ioctls = ethosu_drm_driver_ioctls,
202 .num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls),
203 .fops = ðosu_drm_driver_fops,
204 .name = "ethosu",
205 .desc = "Arm Ethos-U Accel driver",
206 .major = 1,
207 .minor = 0,
208
209 .gem_create_object = ethosu_gem_create_object,
210 };
211
212 #define U65_DRAM_AXI_LIMIT_CFG 0x1f3f0002
213 #define U65_SRAM_AXI_LIMIT_CFG 0x1f3f00b0
214 #define U85_AXI_EXT_CFG 0x00021f3f
215 #define U85_AXI_SRAM_CFG 0x00021f3f
216 #define U85_MEM_ATTR0_CFG 0x00000000
217 #define U85_MEM_ATTR2_CFG 0x000000b7
218
ethosu_reset(struct ethosu_device * ethosudev)219 static int ethosu_reset(struct ethosu_device *ethosudev)
220 {
221 int ret;
222 u32 reg;
223
224 writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET);
225 ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg,
226 !FIELD_GET(STATUS_RESET_STATUS, reg),
227 USEC_PER_MSEC, USEC_PER_SEC);
228 if (ret)
229 return ret;
230
231 if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) {
232 dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n",
233 readl_relaxed(ethosudev->regs + NPU_REG_PROT));
234 }
235
236 /*
237 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0),
238 * everything else to AXI M1 (AXILIMIT2)
239 */
240 writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG);
241 if (ethosu_is_u65(ethosudev)) {
242 writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0);
243 writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2);
244 } else {
245 writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM);
246 writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT);
247 writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0); // SRAM
248 writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2); // DRAM
249 }
250
251 if (ethosudev->sram)
252 memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size);
253
254 return 0;
255 }
256
ethosu_device_resume(struct device * dev)257 static int ethosu_device_resume(struct device *dev)
258 {
259 struct ethosu_device *ethosudev = dev_get_drvdata(dev);
260 int ret;
261
262 ret = clk_bulk_prepare_enable(ethosudev->num_clks, ethosudev->clks);
263 if (ret)
264 return ret;
265
266 ret = ethosu_reset(ethosudev);
267 if (!ret)
268 return 0;
269
270 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks);
271 return ret;
272 }
273
ethosu_device_suspend(struct device * dev)274 static int ethosu_device_suspend(struct device *dev)
275 {
276 struct ethosu_device *ethosudev = dev_get_drvdata(dev);
277
278 clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks);
279 return 0;
280 }
281
ethosu_sram_init(struct ethosu_device * ethosudev)282 static int ethosu_sram_init(struct ethosu_device *ethosudev)
283 {
284 ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0);
285 if (!ethosudev->srampool)
286 return 0;
287
288 ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool);
289
290 ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(ethosudev->srampool,
291 ethosudev->npu_info.sram_size,
292 ðosudev->sramphys);
293 if (!ethosudev->sram) {
294 ethosudev->npu_info.sram_size = 0;
295 dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n");
296 return -ENOMEM;
297 }
298
299 return 0;
300 }
301
ethosu_init(struct ethosu_device * ethosudev)302 static int ethosu_init(struct ethosu_device *ethosudev)
303 {
304 int ret;
305 u32 id, config;
306
307 ret = ethosu_device_resume(ethosudev->base.dev);
308 if (ret)
309 return ret;
310
311 pm_runtime_set_autosuspend_delay(ethosudev->base.dev, 50);
312 pm_runtime_use_autosuspend(ethosudev->base.dev);
313 ret = devm_pm_runtime_set_active_enabled(ethosudev->base.dev);
314 if (ret)
315 return ret;
316 pm_runtime_get_noresume(ethosudev->base.dev);
317
318 ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID);
319 ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG);
320
321 ethosu_sram_init(ethosudev);
322
323 if (!ethosu_is_u65(ethosudev))
324 ethosudev->pmu_regs += 0x1000;
325
326 ethosudev->npu_info.pmu_counters = FIELD_GET(PMCR_NUM_EVENT_CNT_MASK,
327 readl_relaxed(ethosudev->pmu_regs + NPU_REG_PMCR));
328
329 dev_info(ethosudev->base.dev,
330 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM, %d PMU cntrs\n",
331 FIELD_GET(ID_ARCH_MAJOR_MASK, id),
332 FIELD_GET(ID_ARCH_MINOR_MASK, id),
333 FIELD_GET(ID_ARCH_PATCH_MASK, id),
334 FIELD_GET(ID_VER_MAJOR_MASK, id),
335 FIELD_GET(ID_VER_MINOR_MASK, id),
336 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config),
337 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config),
338 ethosudev->npu_info.sram_size / 1024,
339 ethosudev->npu_info.pmu_counters);
340
341 return 0;
342 }
343
ethosu_probe(struct platform_device * pdev)344 static int ethosu_probe(struct platform_device *pdev)
345 {
346 int ret;
347 struct ethosu_device *ethosudev;
348
349 ethosudev = devm_drm_dev_alloc(&pdev->dev, ðosu_drm_driver,
350 struct ethosu_device, base);
351 if (IS_ERR(ethosudev))
352 return -ENOMEM;
353 platform_set_drvdata(pdev, ethosudev);
354
355 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
356
357 ethosudev->regs = devm_platform_ioremap_resource(pdev, 0);
358 if (IS_ERR(ethosudev->regs))
359 return PTR_ERR(ethosudev->regs);
360 ethosudev->pmu_regs = ethosudev->regs;
361
362 ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, ðosudev->clks);
363 if (ethosudev->num_clks < 0)
364 return ethosudev->num_clks;
365
366 ret = drmm_mutex_init(ðosudev->base, ðosudev->perfmon_state.lock);
367 if (ret)
368 return ret;
369
370 ret = ethosu_job_init(ethosudev);
371 if (ret)
372 return ret;
373
374 ret = ethosu_init(ethosudev);
375 if (ret)
376 return ret;
377
378 ret = drm_dev_register(ðosudev->base, 0);
379 if (ret)
380 pm_runtime_dont_use_autosuspend(ethosudev->base.dev);
381
382 pm_runtime_put_autosuspend(ethosudev->base.dev);
383 return ret;
384 }
385
ethosu_remove(struct platform_device * pdev)386 static void ethosu_remove(struct platform_device *pdev)
387 {
388 struct ethosu_device *ethosudev = dev_get_drvdata(&pdev->dev);
389
390 drm_dev_unregister(ðosudev->base);
391 ethosu_job_fini(ethosudev);
392 if (ethosudev->sram)
393 gen_pool_free(ethosudev->srampool, (unsigned long)ethosudev->sram,
394 ethosudev->npu_info.sram_size);
395 }
396
397 static const struct of_device_id dt_match[] = {
398 { .compatible = "arm,ethos-u65" },
399 { .compatible = "arm,ethos-u85" },
400 {}
401 };
402 MODULE_DEVICE_TABLE(of, dt_match);
403
404 static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops,
405 ethosu_device_suspend,
406 ethosu_device_resume,
407 NULL);
408
409 static struct platform_driver ethosu_driver = {
410 .probe = ethosu_probe,
411 .remove = ethosu_remove,
412 .driver = {
413 .name = "ethosu",
414 .pm = pm_ptr(ðosu_pm_ops),
415 .of_match_table = dt_match,
416 },
417 };
418 module_platform_driver(ethosu_driver);
419
420 MODULE_AUTHOR("Rob Herring <robh@kernel.org>");
421 MODULE_DESCRIPTION("Arm Ethos-U Accel Driver");
422 MODULE_LICENSE("Dual MIT/GPL");
423