1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */
3
4 #include <drm/drm_accel.h>
5 #include <drm/drm_drv.h>
6 #include <drm/drm_gem.h>
7 #include <drm/drm_ioctl.h>
8 #include <drm/rocket_accel.h>
9 #include <linux/clk.h>
10 #include <linux/err.h>
11 #include <linux/iommu.h>
12 #include <linux/of.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15
16 #include "rocket_device.h"
17 #include "rocket_drv.h"
18 #include "rocket_gem.h"
19 #include "rocket_job.h"
20
21 /*
22 * Facade device, used to expose a single DRM device to userspace, that
23 * schedules jobs to any RKNN cores in the system.
24 */
25 static struct platform_device *drm_dev;
26 static struct rocket_device *rdev;
27
28 static void
rocket_iommu_domain_destroy(struct kref * kref)29 rocket_iommu_domain_destroy(struct kref *kref)
30 {
31 struct rocket_iommu_domain *domain = container_of(kref, struct rocket_iommu_domain, kref);
32
33 iommu_domain_free(domain->domain);
34 domain->domain = NULL;
35 kfree(domain);
36 }
37
38 static struct rocket_iommu_domain*
rocket_iommu_domain_create(struct device * dev)39 rocket_iommu_domain_create(struct device *dev)
40 {
41 struct rocket_iommu_domain *domain = kmalloc_obj(*domain);
42 void *err;
43
44 if (!domain)
45 return ERR_PTR(-ENOMEM);
46
47 domain->domain = iommu_paging_domain_alloc(dev);
48 if (IS_ERR(domain->domain)) {
49 err = ERR_CAST(domain->domain);
50 kfree(domain);
51 return err;
52 }
53 kref_init(&domain->kref);
54
55 return domain;
56 }
57
58 struct rocket_iommu_domain *
rocket_iommu_domain_get(struct rocket_file_priv * rocket_priv)59 rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv)
60 {
61 kref_get(&rocket_priv->domain->kref);
62 return rocket_priv->domain;
63 }
64
65 void
rocket_iommu_domain_put(struct rocket_iommu_domain * domain)66 rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
67 {
68 kref_put(&domain->kref, rocket_iommu_domain_destroy);
69 }
70
71 static int
rocket_open(struct drm_device * dev,struct drm_file * file)72 rocket_open(struct drm_device *dev, struct drm_file *file)
73 {
74 struct rocket_device *rdev = to_rocket_device(dev);
75 struct rocket_file_priv *rocket_priv;
76 u64 start, end;
77 int ret;
78
79 if (!try_module_get(THIS_MODULE))
80 return -EINVAL;
81
82 rocket_priv = kzalloc_obj(*rocket_priv);
83 if (!rocket_priv) {
84 ret = -ENOMEM;
85 goto err_put_mod;
86 }
87
88 rocket_priv->rdev = rdev;
89 rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
90 if (IS_ERR(rocket_priv->domain)) {
91 ret = PTR_ERR(rocket_priv->domain);
92 goto err_free;
93 }
94
95 file->driver_priv = rocket_priv;
96
97 start = rocket_priv->domain->domain->geometry.aperture_start;
98 end = rocket_priv->domain->domain->geometry.aperture_end;
99 drm_mm_init(&rocket_priv->mm, start, end - start + 1);
100 mutex_init(&rocket_priv->mm_lock);
101
102 ret = rocket_job_open(rocket_priv);
103 if (ret)
104 goto err_mm_takedown;
105
106 return 0;
107
108 err_mm_takedown:
109 mutex_destroy(&rocket_priv->mm_lock);
110 drm_mm_takedown(&rocket_priv->mm);
111 rocket_iommu_domain_put(rocket_priv->domain);
112 err_free:
113 kfree(rocket_priv);
114 err_put_mod:
115 module_put(THIS_MODULE);
116 return ret;
117 }
118
119 static void
rocket_postclose(struct drm_device * dev,struct drm_file * file)120 rocket_postclose(struct drm_device *dev, struct drm_file *file)
121 {
122 struct rocket_file_priv *rocket_priv = file->driver_priv;
123
124 rocket_job_close(rocket_priv);
125 mutex_destroy(&rocket_priv->mm_lock);
126 drm_mm_takedown(&rocket_priv->mm);
127 rocket_iommu_domain_put(rocket_priv->domain);
128 kfree(rocket_priv);
129 module_put(THIS_MODULE);
130 }
131
132 static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
133 #define ROCKET_IOCTL(n, func) \
134 DRM_IOCTL_DEF_DRV(ROCKET_##n, rocket_ioctl_##func, 0)
135
136 ROCKET_IOCTL(CREATE_BO, create_bo),
137 ROCKET_IOCTL(SUBMIT, submit),
138 ROCKET_IOCTL(PREP_BO, prep_bo),
139 ROCKET_IOCTL(FINI_BO, fini_bo),
140 };
141
142 DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
143
144 /*
145 * Rocket driver version:
146 * - 1.0 - initial interface
147 */
148 static const struct drm_driver rocket_drm_driver = {
149 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM,
150 .open = rocket_open,
151 .postclose = rocket_postclose,
152 .gem_create_object = rocket_gem_create_object,
153 .ioctls = rocket_drm_driver_ioctls,
154 .num_ioctls = ARRAY_SIZE(rocket_drm_driver_ioctls),
155 .fops = &rocket_accel_driver_fops,
156 .name = "rocket",
157 .desc = "rocket DRM",
158 };
159
rocket_probe(struct platform_device * pdev)160 static int rocket_probe(struct platform_device *pdev)
161 {
162 int ret;
163
164 if (rdev == NULL) {
165 /* First core probing, initialize DRM device. */
166 rdev = rocket_device_init(drm_dev, &rocket_drm_driver);
167 if (IS_ERR(rdev)) {
168 dev_err(&pdev->dev, "failed to initialize rocket device\n");
169 return PTR_ERR(rdev);
170 }
171 }
172
173 unsigned int core = rdev->num_cores;
174
175 dev_set_drvdata(&pdev->dev, rdev);
176
177 rdev->cores[core].rdev = rdev;
178 rdev->cores[core].dev = &pdev->dev;
179 rdev->cores[core].index = core;
180
181 rdev->num_cores++;
182
183 ret = rocket_core_init(&rdev->cores[core]);
184 if (ret) {
185 rdev->num_cores--;
186
187 if (rdev->num_cores == 0) {
188 rocket_device_fini(rdev);
189 rdev = NULL;
190 }
191 }
192
193 return ret;
194 }
195
196 static int find_core_for_dev(struct device *dev);
197
rocket_remove(struct platform_device * pdev)198 static void rocket_remove(struct platform_device *pdev)
199 {
200 struct device *dev = &pdev->dev;
201 int core = find_core_for_dev(dev);
202
203 if (core < 0)
204 return;
205
206 rocket_core_fini(&rdev->cores[core]);
207 rdev->num_cores--;
208
209 if (rdev->num_cores == 0) {
210 /* Last core removed, deinitialize DRM device. */
211 rocket_device_fini(rdev);
212 rdev = NULL;
213 }
214 }
215
216 static const struct of_device_id dt_match[] = {
217 { .compatible = "rockchip,rk3588-rknn-core" },
218 {}
219 };
220 MODULE_DEVICE_TABLE(of, dt_match);
221
find_core_for_dev(struct device * dev)222 static int find_core_for_dev(struct device *dev)
223 {
224 struct rocket_device *rdev = dev_get_drvdata(dev);
225
226 for (unsigned int core = 0; core < rdev->num_cores; core++) {
227 if (dev == rdev->cores[core].dev)
228 return core;
229 }
230
231 return -1;
232 }
233
rocket_device_runtime_resume(struct device * dev)234 static int rocket_device_runtime_resume(struct device *dev)
235 {
236 struct rocket_device *rdev = dev_get_drvdata(dev);
237 int core = find_core_for_dev(dev);
238 int err = 0;
239
240 if (core < 0)
241 return -ENODEV;
242
243 err = clk_bulk_prepare_enable(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks);
244 if (err) {
245 dev_err(dev, "failed to enable (%d) clocks for core %d\n", err, core);
246 return err;
247 }
248
249 return 0;
250 }
251
rocket_device_runtime_suspend(struct device * dev)252 static int rocket_device_runtime_suspend(struct device *dev)
253 {
254 struct rocket_device *rdev = dev_get_drvdata(dev);
255 int core = find_core_for_dev(dev);
256
257 if (core < 0)
258 return -ENODEV;
259
260 if (!rocket_job_is_idle(&rdev->cores[core]))
261 return -EBUSY;
262
263 clk_bulk_disable_unprepare(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks);
264
265 return 0;
266 }
267
268 EXPORT_GPL_DEV_PM_OPS(rocket_pm_ops) = {
269 RUNTIME_PM_OPS(rocket_device_runtime_suspend, rocket_device_runtime_resume, NULL)
270 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
271 };
272
273 static struct platform_driver rocket_driver = {
274 .probe = rocket_probe,
275 .remove = rocket_remove,
276 .driver = {
277 .name = "rocket",
278 .pm = pm_ptr(&rocket_pm_ops),
279 .of_match_table = dt_match,
280 },
281 };
282
rocket_register(void)283 static int __init rocket_register(void)
284 {
285 drm_dev = platform_device_register_simple("rknn", -1, NULL, 0);
286 if (IS_ERR(drm_dev))
287 return PTR_ERR(drm_dev);
288
289 return platform_driver_register(&rocket_driver);
290 }
291
rocket_unregister(void)292 static void __exit rocket_unregister(void)
293 {
294 platform_driver_unregister(&rocket_driver);
295
296 platform_device_unregister(drm_dev);
297 }
298
299 module_init(rocket_register);
300 module_exit(rocket_unregister);
301
302 MODULE_LICENSE("GPL");
303 MODULE_DESCRIPTION("DRM driver for the Rockchip NPU IP");
304 MODULE_AUTHOR("Tomeu Vizoso");
305