1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */ 3 4 #include <drm/drm_accel.h> 5 #include <drm/drm_drv.h> 6 #include <drm/drm_gem.h> 7 #include <drm/drm_ioctl.h> 8 #include <drm/rocket_accel.h> 9 #include <linux/clk.h> 10 #include <linux/err.h> 11 #include <linux/iommu.h> 12 #include <linux/of.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include "rocket_drv.h" 17 #include "rocket_gem.h" 18 #include "rocket_job.h" 19 20 /* 21 * Facade device, used to expose a single DRM device to userspace, that 22 * schedules jobs to any RKNN cores in the system. 23 */ 24 static struct platform_device *drm_dev; 25 static struct rocket_device *rdev; 26 27 static void 28 rocket_iommu_domain_destroy(struct kref *kref) 29 { 30 struct rocket_iommu_domain *domain = container_of(kref, struct rocket_iommu_domain, kref); 31 32 iommu_domain_free(domain->domain); 33 domain->domain = NULL; 34 kfree(domain); 35 } 36 37 static struct rocket_iommu_domain* 38 rocket_iommu_domain_create(struct device *dev) 39 { 40 struct rocket_iommu_domain *domain = kmalloc(sizeof(*domain), GFP_KERNEL); 41 void *err; 42 43 if (!domain) 44 return ERR_PTR(-ENOMEM); 45 46 domain->domain = iommu_paging_domain_alloc(dev); 47 if (IS_ERR(domain->domain)) { 48 err = ERR_CAST(domain->domain); 49 kfree(domain); 50 return err; 51 } 52 kref_init(&domain->kref); 53 54 return domain; 55 } 56 57 struct rocket_iommu_domain * 58 rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv) 59 { 60 kref_get(&rocket_priv->domain->kref); 61 return rocket_priv->domain; 62 } 63 64 void 65 rocket_iommu_domain_put(struct rocket_iommu_domain *domain) 66 { 67 kref_put(&domain->kref, rocket_iommu_domain_destroy); 68 } 69 70 static int 71 rocket_open(struct drm_device *dev, struct drm_file *file) 72 { 73 struct rocket_device *rdev = to_rocket_device(dev); 74 struct rocket_file_priv *rocket_priv; 75 u64 start, end; 76 int ret; 77 78 if (!try_module_get(THIS_MODULE)) 79 return -EINVAL; 80 81 rocket_priv = kzalloc(sizeof(*rocket_priv), GFP_KERNEL); 82 if (!rocket_priv) { 83 ret = -ENOMEM; 84 goto err_put_mod; 85 } 86 87 rocket_priv->rdev = rdev; 88 rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev); 89 if (IS_ERR(rocket_priv->domain)) { 90 ret = PTR_ERR(rocket_priv->domain); 91 goto err_free; 92 } 93 94 file->driver_priv = rocket_priv; 95 96 start = rocket_priv->domain->domain->geometry.aperture_start; 97 end = rocket_priv->domain->domain->geometry.aperture_end; 98 drm_mm_init(&rocket_priv->mm, start, end - start + 1); 99 mutex_init(&rocket_priv->mm_lock); 100 101 ret = rocket_job_open(rocket_priv); 102 if (ret) 103 goto err_mm_takedown; 104 105 return 0; 106 107 err_mm_takedown: 108 mutex_destroy(&rocket_priv->mm_lock); 109 drm_mm_takedown(&rocket_priv->mm); 110 rocket_iommu_domain_put(rocket_priv->domain); 111 err_free: 112 kfree(rocket_priv); 113 err_put_mod: 114 module_put(THIS_MODULE); 115 return ret; 116 } 117 118 static void 119 rocket_postclose(struct drm_device *dev, struct drm_file *file) 120 { 121 struct rocket_file_priv *rocket_priv = file->driver_priv; 122 123 rocket_job_close(rocket_priv); 124 mutex_destroy(&rocket_priv->mm_lock); 125 drm_mm_takedown(&rocket_priv->mm); 126 rocket_iommu_domain_put(rocket_priv->domain); 127 kfree(rocket_priv); 128 module_put(THIS_MODULE); 129 } 130 131 static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = { 132 #define ROCKET_IOCTL(n, func) \ 133 DRM_IOCTL_DEF_DRV(ROCKET_##n, rocket_ioctl_##func, 0) 134 135 ROCKET_IOCTL(CREATE_BO, create_bo), 136 ROCKET_IOCTL(SUBMIT, submit), 137 }; 138 139 DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops); 140 141 /* 142 * Rocket driver version: 143 * - 1.0 - initial interface 144 */ 145 static const struct drm_driver rocket_drm_driver = { 146 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM, 147 .open = rocket_open, 148 .postclose = rocket_postclose, 149 .gem_create_object = rocket_gem_create_object, 150 .ioctls = rocket_drm_driver_ioctls, 151 .num_ioctls = ARRAY_SIZE(rocket_drm_driver_ioctls), 152 .fops = &rocket_accel_driver_fops, 153 .name = "rocket", 154 .desc = "rocket DRM", 155 }; 156 157 static int rocket_probe(struct platform_device *pdev) 158 { 159 if (rdev == NULL) { 160 /* First core probing, initialize DRM device. */ 161 rdev = rocket_device_init(drm_dev, &rocket_drm_driver); 162 if (IS_ERR(rdev)) { 163 dev_err(&pdev->dev, "failed to initialize rocket device\n"); 164 return PTR_ERR(rdev); 165 } 166 } 167 168 unsigned int core = rdev->num_cores; 169 170 dev_set_drvdata(&pdev->dev, rdev); 171 172 rdev->cores[core].rdev = rdev; 173 rdev->cores[core].dev = &pdev->dev; 174 rdev->cores[core].index = core; 175 176 rdev->num_cores++; 177 178 return rocket_core_init(&rdev->cores[core]); 179 } 180 181 static void rocket_remove(struct platform_device *pdev) 182 { 183 struct device *dev = &pdev->dev; 184 185 for (unsigned int core = 0; core < rdev->num_cores; core++) { 186 if (rdev->cores[core].dev == dev) { 187 rocket_core_fini(&rdev->cores[core]); 188 rdev->num_cores--; 189 break; 190 } 191 } 192 193 if (rdev->num_cores == 0) { 194 /* Last core removed, deinitialize DRM device. */ 195 rocket_device_fini(rdev); 196 rdev = NULL; 197 } 198 } 199 200 static const struct of_device_id dt_match[] = { 201 { .compatible = "rockchip,rk3588-rknn-core" }, 202 {} 203 }; 204 MODULE_DEVICE_TABLE(of, dt_match); 205 206 static int find_core_for_dev(struct device *dev) 207 { 208 struct rocket_device *rdev = dev_get_drvdata(dev); 209 210 for (unsigned int core = 0; core < rdev->num_cores; core++) { 211 if (dev == rdev->cores[core].dev) 212 return core; 213 } 214 215 return -1; 216 } 217 218 static int rocket_device_runtime_resume(struct device *dev) 219 { 220 struct rocket_device *rdev = dev_get_drvdata(dev); 221 int core = find_core_for_dev(dev); 222 int err = 0; 223 224 if (core < 0) 225 return -ENODEV; 226 227 err = clk_bulk_prepare_enable(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks); 228 if (err) { 229 dev_err(dev, "failed to enable (%d) clocks for core %d\n", err, core); 230 return err; 231 } 232 233 return 0; 234 } 235 236 static int rocket_device_runtime_suspend(struct device *dev) 237 { 238 struct rocket_device *rdev = dev_get_drvdata(dev); 239 int core = find_core_for_dev(dev); 240 241 if (core < 0) 242 return -ENODEV; 243 244 if (!rocket_job_is_idle(&rdev->cores[core])) 245 return -EBUSY; 246 247 clk_bulk_disable_unprepare(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks); 248 249 return 0; 250 } 251 252 EXPORT_GPL_DEV_PM_OPS(rocket_pm_ops) = { 253 RUNTIME_PM_OPS(rocket_device_runtime_suspend, rocket_device_runtime_resume, NULL) 254 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 255 }; 256 257 static struct platform_driver rocket_driver = { 258 .probe = rocket_probe, 259 .remove = rocket_remove, 260 .driver = { 261 .name = "rocket", 262 .pm = pm_ptr(&rocket_pm_ops), 263 .of_match_table = dt_match, 264 }, 265 }; 266 267 static int __init rocket_register(void) 268 { 269 drm_dev = platform_device_register_simple("rknn", -1, NULL, 0); 270 if (IS_ERR(drm_dev)) 271 return PTR_ERR(drm_dev); 272 273 return platform_driver_register(&rocket_driver); 274 } 275 276 static void __exit rocket_unregister(void) 277 { 278 platform_driver_unregister(&rocket_driver); 279 280 platform_device_unregister(drm_dev); 281 } 282 283 module_init(rocket_register); 284 module_exit(rocket_unregister); 285 286 MODULE_LICENSE("GPL"); 287 MODULE_DESCRIPTION("DRM driver for the Rockchip NPU IP"); 288 MODULE_AUTHOR("Tomeu Vizoso"); 289