1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */ 3 4 #include <drm/drm_accel.h> 5 #include <drm/drm_drv.h> 6 #include <drm/drm_gem.h> 7 #include <drm/drm_ioctl.h> 8 #include <drm/rocket_accel.h> 9 #include <linux/clk.h> 10 #include <linux/err.h> 11 #include <linux/iommu.h> 12 #include <linux/of.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include "rocket_device.h" 17 #include "rocket_drv.h" 18 #include "rocket_gem.h" 19 #include "rocket_job.h" 20 21 /* 22 * Facade device, used to expose a single DRM device to userspace, that 23 * schedules jobs to any RKNN cores in the system. 24 */ 25 static struct platform_device *drm_dev; 26 static struct rocket_device *rdev; 27 28 static void 29 rocket_iommu_domain_destroy(struct kref *kref) 30 { 31 struct rocket_iommu_domain *domain = container_of(kref, struct rocket_iommu_domain, kref); 32 33 iommu_domain_free(domain->domain); 34 domain->domain = NULL; 35 kfree(domain); 36 } 37 38 static struct rocket_iommu_domain* 39 rocket_iommu_domain_create(struct device *dev) 40 { 41 struct rocket_iommu_domain *domain = kmalloc(sizeof(*domain), GFP_KERNEL); 42 void *err; 43 44 if (!domain) 45 return ERR_PTR(-ENOMEM); 46 47 domain->domain = iommu_paging_domain_alloc(dev); 48 if (IS_ERR(domain->domain)) { 49 err = ERR_CAST(domain->domain); 50 kfree(domain); 51 return err; 52 } 53 kref_init(&domain->kref); 54 55 return domain; 56 } 57 58 struct rocket_iommu_domain * 59 rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv) 60 { 61 kref_get(&rocket_priv->domain->kref); 62 return rocket_priv->domain; 63 } 64 65 void 66 rocket_iommu_domain_put(struct rocket_iommu_domain *domain) 67 { 68 kref_put(&domain->kref, rocket_iommu_domain_destroy); 69 } 70 71 static int 72 rocket_open(struct drm_device *dev, struct drm_file *file) 73 { 74 struct rocket_device *rdev = to_rocket_device(dev); 75 struct rocket_file_priv *rocket_priv; 76 u64 start, end; 77 int ret; 78 79 if (!try_module_get(THIS_MODULE)) 80 return -EINVAL; 81 82 rocket_priv = kzalloc(sizeof(*rocket_priv), GFP_KERNEL); 83 if (!rocket_priv) { 84 ret = -ENOMEM; 85 goto err_put_mod; 86 } 87 88 rocket_priv->rdev = rdev; 89 rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev); 90 if (IS_ERR(rocket_priv->domain)) { 91 ret = PTR_ERR(rocket_priv->domain); 92 goto err_free; 93 } 94 95 file->driver_priv = rocket_priv; 96 97 start = rocket_priv->domain->domain->geometry.aperture_start; 98 end = rocket_priv->domain->domain->geometry.aperture_end; 99 drm_mm_init(&rocket_priv->mm, start, end - start + 1); 100 mutex_init(&rocket_priv->mm_lock); 101 102 ret = rocket_job_open(rocket_priv); 103 if (ret) 104 goto err_mm_takedown; 105 106 return 0; 107 108 err_mm_takedown: 109 mutex_destroy(&rocket_priv->mm_lock); 110 drm_mm_takedown(&rocket_priv->mm); 111 rocket_iommu_domain_put(rocket_priv->domain); 112 err_free: 113 kfree(rocket_priv); 114 err_put_mod: 115 module_put(THIS_MODULE); 116 return ret; 117 } 118 119 static void 120 rocket_postclose(struct drm_device *dev, struct drm_file *file) 121 { 122 struct rocket_file_priv *rocket_priv = file->driver_priv; 123 124 rocket_job_close(rocket_priv); 125 mutex_destroy(&rocket_priv->mm_lock); 126 drm_mm_takedown(&rocket_priv->mm); 127 rocket_iommu_domain_put(rocket_priv->domain); 128 kfree(rocket_priv); 129 module_put(THIS_MODULE); 130 } 131 132 static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = { 133 #define ROCKET_IOCTL(n, func) \ 134 DRM_IOCTL_DEF_DRV(ROCKET_##n, rocket_ioctl_##func, 0) 135 136 ROCKET_IOCTL(CREATE_BO, create_bo), 137 ROCKET_IOCTL(SUBMIT, submit), 138 ROCKET_IOCTL(PREP_BO, prep_bo), 139 ROCKET_IOCTL(FINI_BO, fini_bo), 140 }; 141 142 DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops); 143 144 /* 145 * Rocket driver version: 146 * - 1.0 - initial interface 147 */ 148 static const struct drm_driver rocket_drm_driver = { 149 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM, 150 .open = rocket_open, 151 .postclose = rocket_postclose, 152 .gem_create_object = rocket_gem_create_object, 153 .ioctls = rocket_drm_driver_ioctls, 154 .num_ioctls = ARRAY_SIZE(rocket_drm_driver_ioctls), 155 .fops = &rocket_accel_driver_fops, 156 .name = "rocket", 157 .desc = "rocket DRM", 158 }; 159 160 static int rocket_probe(struct platform_device *pdev) 161 { 162 int ret; 163 164 if (rdev == NULL) { 165 /* First core probing, initialize DRM device. */ 166 rdev = rocket_device_init(drm_dev, &rocket_drm_driver); 167 if (IS_ERR(rdev)) { 168 dev_err(&pdev->dev, "failed to initialize rocket device\n"); 169 return PTR_ERR(rdev); 170 } 171 } 172 173 unsigned int core = rdev->num_cores; 174 175 dev_set_drvdata(&pdev->dev, rdev); 176 177 rdev->cores[core].rdev = rdev; 178 rdev->cores[core].dev = &pdev->dev; 179 rdev->cores[core].index = core; 180 181 rdev->num_cores++; 182 183 ret = rocket_core_init(&rdev->cores[core]); 184 if (ret) { 185 rdev->num_cores--; 186 187 if (rdev->num_cores == 0) { 188 rocket_device_fini(rdev); 189 rdev = NULL; 190 } 191 } 192 193 return ret; 194 } 195 196 static int find_core_for_dev(struct device *dev); 197 198 static void rocket_remove(struct platform_device *pdev) 199 { 200 struct device *dev = &pdev->dev; 201 int core = find_core_for_dev(dev); 202 203 if (core < 0) 204 return; 205 206 rocket_core_fini(&rdev->cores[core]); 207 rdev->num_cores--; 208 209 if (rdev->num_cores == 0) { 210 /* Last core removed, deinitialize DRM device. */ 211 rocket_device_fini(rdev); 212 rdev = NULL; 213 } 214 } 215 216 static const struct of_device_id dt_match[] = { 217 { .compatible = "rockchip,rk3588-rknn-core" }, 218 {} 219 }; 220 MODULE_DEVICE_TABLE(of, dt_match); 221 222 static int find_core_for_dev(struct device *dev) 223 { 224 struct rocket_device *rdev = dev_get_drvdata(dev); 225 226 for (unsigned int core = 0; core < rdev->num_cores; core++) { 227 if (dev == rdev->cores[core].dev) 228 return core; 229 } 230 231 return -1; 232 } 233 234 static int rocket_device_runtime_resume(struct device *dev) 235 { 236 struct rocket_device *rdev = dev_get_drvdata(dev); 237 int core = find_core_for_dev(dev); 238 int err = 0; 239 240 if (core < 0) 241 return -ENODEV; 242 243 err = clk_bulk_prepare_enable(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks); 244 if (err) { 245 dev_err(dev, "failed to enable (%d) clocks for core %d\n", err, core); 246 return err; 247 } 248 249 return 0; 250 } 251 252 static int rocket_device_runtime_suspend(struct device *dev) 253 { 254 struct rocket_device *rdev = dev_get_drvdata(dev); 255 int core = find_core_for_dev(dev); 256 257 if (core < 0) 258 return -ENODEV; 259 260 if (!rocket_job_is_idle(&rdev->cores[core])) 261 return -EBUSY; 262 263 clk_bulk_disable_unprepare(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks); 264 265 return 0; 266 } 267 268 EXPORT_GPL_DEV_PM_OPS(rocket_pm_ops) = { 269 RUNTIME_PM_OPS(rocket_device_runtime_suspend, rocket_device_runtime_resume, NULL) 270 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 271 }; 272 273 static struct platform_driver rocket_driver = { 274 .probe = rocket_probe, 275 .remove = rocket_remove, 276 .driver = { 277 .name = "rocket", 278 .pm = pm_ptr(&rocket_pm_ops), 279 .of_match_table = dt_match, 280 }, 281 }; 282 283 static int __init rocket_register(void) 284 { 285 drm_dev = platform_device_register_simple("rknn", -1, NULL, 0); 286 if (IS_ERR(drm_dev)) 287 return PTR_ERR(drm_dev); 288 289 return platform_driver_register(&rocket_driver); 290 } 291 292 static void __exit rocket_unregister(void) 293 { 294 platform_driver_unregister(&rocket_driver); 295 296 platform_device_unregister(drm_dev); 297 } 298 299 module_init(rocket_register); 300 module_exit(rocket_unregister); 301 302 MODULE_LICENSE("GPL"); 303 MODULE_DESCRIPTION("DRM driver for the Rockchip NPU IP"); 304 MODULE_AUTHOR("Tomeu Vizoso"); 305