1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/bitfield.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/module.h> 9 #include <linux/of_platform.h> 10 #include <linux/pagemap.h> 11 #include <linux/pm_runtime.h> 12 #include <drm/panfrost_drm.h> 13 #include <drm/drm_drv.h> 14 #include <drm/drm_ioctl.h> 15 #include <drm/drm_syncobj.h> 16 #include <drm/drm_utils.h> 17 18 #include "panfrost_device.h" 19 #include "panfrost_devfreq.h" 20 #include "panfrost_gem.h" 21 #include "panfrost_mmu.h" 22 #include "panfrost_job.h" 23 #include "panfrost_gpu.h" 24 25 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 26 { 27 struct drm_panfrost_get_param *param = data; 28 struct panfrost_device *pfdev = ddev->dev_private; 29 30 if (param->pad != 0) 31 return -EINVAL; 32 33 switch (param->param) { 34 case DRM_PANFROST_PARAM_GPU_PROD_ID: 35 param->value = pfdev->features.id; 36 break; 37 default: 38 return -EINVAL; 39 } 40 41 return 0; 42 } 43 44 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 45 struct drm_file *file) 46 { 47 int ret; 48 struct drm_gem_shmem_object *shmem; 49 struct drm_panfrost_create_bo *args = data; 50 51 if (!args->size || args->flags || args->pad) 52 return -EINVAL; 53 54 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, 55 &args->handle); 56 if (IS_ERR(shmem)) 57 return PTR_ERR(shmem); 58 59 ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base)); 60 if (ret) 61 goto err_free; 62 63 args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; 64 65 return 0; 66 67 err_free: 68 drm_gem_object_put_unlocked(&shmem->base); 69 return ret; 70 } 71 72 /** 73 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 74 * referenced by the job. 75 * @dev: DRM device 76 * @file_priv: DRM file for this fd 77 * @args: IOCTL args 78 * @job: job being set up 79 * 80 * Resolve handles from userspace to BOs and attach them to job. 81 * 82 * Note that this function doesn't need to unreference the BOs on 83 * failure, because that will happen at panfrost_job_cleanup() time. 84 */ 85 static int 86 panfrost_lookup_bos(struct drm_device *dev, 87 struct drm_file *file_priv, 88 struct drm_panfrost_submit *args, 89 struct panfrost_job *job) 90 { 91 job->bo_count = args->bo_handle_count; 92 93 if (!job->bo_count) 94 return 0; 95 96 job->implicit_fences = kvmalloc_array(job->bo_count, 97 sizeof(struct dma_fence *), 98 GFP_KERNEL | __GFP_ZERO); 99 if (!job->implicit_fences) 100 return -ENOMEM; 101 102 return drm_gem_objects_lookup(file_priv, 103 (void __user *)(uintptr_t)args->bo_handles, 104 job->bo_count, &job->bos); 105 } 106 107 /** 108 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects 109 * referenced by the job. 110 * @dev: DRM device 111 * @file_priv: DRM file for this fd 112 * @args: IOCTL args 113 * @job: job being set up 114 * 115 * Resolve syncobjs from userspace to fences and attach them to job. 116 * 117 * Note that this function doesn't need to unreference the fences on 118 * failure, because that will happen at panfrost_job_cleanup() time. 119 */ 120 static int 121 panfrost_copy_in_sync(struct drm_device *dev, 122 struct drm_file *file_priv, 123 struct drm_panfrost_submit *args, 124 struct panfrost_job *job) 125 { 126 u32 *handles; 127 int ret = 0; 128 int i; 129 130 job->in_fence_count = args->in_sync_count; 131 132 if (!job->in_fence_count) 133 return 0; 134 135 job->in_fences = kvmalloc_array(job->in_fence_count, 136 sizeof(struct dma_fence *), 137 GFP_KERNEL | __GFP_ZERO); 138 if (!job->in_fences) { 139 DRM_DEBUG("Failed to allocate job in fences\n"); 140 return -ENOMEM; 141 } 142 143 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL); 144 if (!handles) { 145 ret = -ENOMEM; 146 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 147 goto fail; 148 } 149 150 if (copy_from_user(handles, 151 (void __user *)(uintptr_t)args->in_syncs, 152 job->in_fence_count * sizeof(u32))) { 153 ret = -EFAULT; 154 DRM_DEBUG("Failed to copy in syncobj handles\n"); 155 goto fail; 156 } 157 158 for (i = 0; i < job->in_fence_count; i++) { 159 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0, 160 &job->in_fences[i]); 161 if (ret == -EINVAL) 162 goto fail; 163 } 164 165 fail: 166 kvfree(handles); 167 return ret; 168 } 169 170 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 171 struct drm_file *file) 172 { 173 struct panfrost_device *pfdev = dev->dev_private; 174 struct drm_panfrost_submit *args = data; 175 struct drm_syncobj *sync_out; 176 struct panfrost_job *job; 177 int ret = 0; 178 179 job = kzalloc(sizeof(*job), GFP_KERNEL); 180 if (!job) 181 return -ENOMEM; 182 183 kref_init(&job->refcount); 184 185 job->pfdev = pfdev; 186 job->jc = args->jc; 187 job->requirements = args->requirements; 188 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 189 job->file_priv = file->driver_priv; 190 191 ret = panfrost_copy_in_sync(dev, file, args, job); 192 if (ret) 193 goto fail; 194 195 ret = panfrost_lookup_bos(dev, file, args, job); 196 if (ret) 197 goto fail; 198 199 ret = panfrost_job_push(job); 200 if (ret) 201 goto fail; 202 203 /* Update the return sync object for the job */ 204 sync_out = drm_syncobj_find(file, args->out_sync); 205 if (sync_out) { 206 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 207 drm_syncobj_put(sync_out); 208 } 209 210 fail: 211 panfrost_job_put(job); 212 213 return ret; 214 } 215 216 static int 217 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 218 struct drm_file *file_priv) 219 { 220 long ret; 221 struct drm_panfrost_wait_bo *args = data; 222 struct drm_gem_object *gem_obj; 223 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 224 225 if (args->pad) 226 return -EINVAL; 227 228 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 229 if (!gem_obj) 230 return -ENOENT; 231 232 ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true, 233 true, timeout); 234 if (!ret) 235 ret = timeout ? -ETIMEDOUT : -EBUSY; 236 237 drm_gem_object_put_unlocked(gem_obj); 238 239 return ret; 240 } 241 242 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 243 struct drm_file *file_priv) 244 { 245 struct drm_panfrost_mmap_bo *args = data; 246 struct drm_gem_object *gem_obj; 247 int ret; 248 249 if (args->flags != 0) { 250 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 251 return -EINVAL; 252 } 253 254 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 255 if (!gem_obj) { 256 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 257 return -ENOENT; 258 } 259 260 ret = drm_gem_create_mmap_offset(gem_obj); 261 if (ret == 0) 262 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 263 drm_gem_object_put_unlocked(gem_obj); 264 265 return ret; 266 } 267 268 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 269 struct drm_file *file_priv) 270 { 271 struct drm_panfrost_get_bo_offset *args = data; 272 struct drm_gem_object *gem_obj; 273 struct panfrost_gem_object *bo; 274 275 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 276 if (!gem_obj) { 277 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 278 return -ENOENT; 279 } 280 bo = to_panfrost_bo(gem_obj); 281 282 args->offset = bo->node.start << PAGE_SHIFT; 283 284 drm_gem_object_put_unlocked(gem_obj); 285 return 0; 286 } 287 288 static int 289 panfrost_open(struct drm_device *dev, struct drm_file *file) 290 { 291 struct panfrost_device *pfdev = dev->dev_private; 292 struct panfrost_file_priv *panfrost_priv; 293 294 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 295 if (!panfrost_priv) 296 return -ENOMEM; 297 298 panfrost_priv->pfdev = pfdev; 299 file->driver_priv = panfrost_priv; 300 301 return panfrost_job_open(panfrost_priv); 302 } 303 304 static void 305 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 306 { 307 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 308 309 panfrost_job_close(panfrost_priv); 310 311 kfree(panfrost_priv); 312 } 313 314 /* DRM_AUTH is required on SUBMIT for now, while all clients share a single 315 * address space. Note that render nodes would be able to submit jobs that 316 * could access BOs from clients authenticated with the master node. 317 */ 318 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 319 #define PANFROST_IOCTL(n, func, flags) \ 320 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 321 322 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH), 323 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 324 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 325 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 326 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 327 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 328 }; 329 330 DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); 331 332 static struct drm_driver panfrost_drm_driver = { 333 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME | 334 DRIVER_SYNCOBJ, 335 .open = panfrost_open, 336 .postclose = panfrost_postclose, 337 .ioctls = panfrost_drm_driver_ioctls, 338 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 339 .fops = &panfrost_drm_driver_fops, 340 .name = "panfrost", 341 .desc = "panfrost DRM", 342 .date = "20180908", 343 .major = 1, 344 .minor = 0, 345 346 .gem_create_object = panfrost_gem_create_object, 347 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 348 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 349 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 350 .gem_prime_mmap = drm_gem_prime_mmap, 351 }; 352 353 static int panfrost_probe(struct platform_device *pdev) 354 { 355 struct panfrost_device *pfdev; 356 struct drm_device *ddev; 357 int err; 358 359 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); 360 if (!pfdev) 361 return -ENOMEM; 362 363 pfdev->pdev = pdev; 364 pfdev->dev = &pdev->dev; 365 366 platform_set_drvdata(pdev, pfdev); 367 368 /* Allocate and initialze the DRM device. */ 369 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); 370 if (IS_ERR(ddev)) 371 return PTR_ERR(ddev); 372 373 ddev->dev_private = pfdev; 374 pfdev->ddev = ddev; 375 376 spin_lock_init(&pfdev->mm_lock); 377 378 /* 4G enough for now. can be 48-bit */ 379 drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); 380 381 pm_runtime_use_autosuspend(pfdev->dev); 382 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ 383 pm_runtime_enable(pfdev->dev); 384 385 err = panfrost_device_init(pfdev); 386 if (err) { 387 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 388 goto err_out0; 389 } 390 391 dma_set_mask_and_coherent(pfdev->dev, 392 DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features))); 393 394 err = panfrost_devfreq_init(pfdev); 395 if (err) { 396 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 397 goto err_out1; 398 } 399 400 /* 401 * Register the DRM device with the core and the connectors with 402 * sysfs 403 */ 404 err = drm_dev_register(ddev, 0); 405 if (err < 0) 406 goto err_out1; 407 408 return 0; 409 410 err_out1: 411 panfrost_device_fini(pfdev); 412 err_out0: 413 drm_dev_put(ddev); 414 return err; 415 } 416 417 static int panfrost_remove(struct platform_device *pdev) 418 { 419 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 420 struct drm_device *ddev = pfdev->ddev; 421 422 drm_dev_unregister(ddev); 423 pm_runtime_get_sync(pfdev->dev); 424 pm_runtime_put_sync_autosuspend(pfdev->dev); 425 pm_runtime_disable(pfdev->dev); 426 panfrost_device_fini(pfdev); 427 drm_dev_put(ddev); 428 return 0; 429 } 430 431 static const struct of_device_id dt_match[] = { 432 { .compatible = "arm,mali-t604" }, 433 { .compatible = "arm,mali-t624" }, 434 { .compatible = "arm,mali-t628" }, 435 { .compatible = "arm,mali-t720" }, 436 { .compatible = "arm,mali-t760" }, 437 { .compatible = "arm,mali-t820" }, 438 { .compatible = "arm,mali-t830" }, 439 { .compatible = "arm,mali-t860" }, 440 { .compatible = "arm,mali-t880" }, 441 {} 442 }; 443 MODULE_DEVICE_TABLE(of, dt_match); 444 445 static const struct dev_pm_ops panfrost_pm_ops = { 446 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 447 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL) 448 }; 449 450 static struct platform_driver panfrost_driver = { 451 .probe = panfrost_probe, 452 .remove = panfrost_remove, 453 .driver = { 454 .name = "panfrost", 455 .pm = &panfrost_pm_ops, 456 .of_match_table = dt_match, 457 }, 458 }; 459 module_platform_driver(panfrost_driver); 460 461 MODULE_AUTHOR("Panfrost Project Developers"); 462 MODULE_DESCRIPTION("Panfrost DRM Driver"); 463 MODULE_LICENSE("GPL v2"); 464