1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2018 Etnaviv Project 4 */ 5 6 #include <linux/component.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/module.h> 9 #include <linux/of_platform.h> 10 #include <linux/uaccess.h> 11 12 #include <drm/drm_debugfs.h> 13 #include <drm/drm_drv.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_ioctl.h> 16 #include <drm/drm_of.h> 17 #include <drm/drm_prime.h> 18 19 #include "etnaviv_cmdbuf.h" 20 #include "etnaviv_drv.h" 21 #include "etnaviv_gpu.h" 22 #include "etnaviv_gem.h" 23 #include "etnaviv_mmu.h" 24 #include "etnaviv_perfmon.h" 25 26 /* 27 * DRM operations: 28 */ 29 30 31 static void load_gpu(struct drm_device *dev) 32 { 33 struct etnaviv_drm_private *priv = dev->dev_private; 34 unsigned int i; 35 36 for (i = 0; i < ETNA_MAX_PIPES; i++) { 37 struct etnaviv_gpu *g = priv->gpu[i]; 38 39 if (g) { 40 int ret; 41 42 ret = etnaviv_gpu_init(g); 43 if (ret) 44 priv->gpu[i] = NULL; 45 } 46 } 47 } 48 49 static int etnaviv_open(struct drm_device *dev, struct drm_file *file) 50 { 51 struct etnaviv_drm_private *priv = dev->dev_private; 52 struct etnaviv_file_private *ctx; 53 int i; 54 55 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 56 if (!ctx) 57 return -ENOMEM; 58 59 for (i = 0; i < ETNA_MAX_PIPES; i++) { 60 struct etnaviv_gpu *gpu = priv->gpu[i]; 61 struct drm_sched_rq *rq; 62 63 if (gpu) { 64 rq = &gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; 65 drm_sched_entity_init(&ctx->sched_entity[i], 66 &rq, 1, NULL); 67 } 68 } 69 70 file->driver_priv = ctx; 71 72 return 0; 73 } 74 75 static void etnaviv_postclose(struct drm_device *dev, struct drm_file *file) 76 { 77 struct etnaviv_drm_private *priv = dev->dev_private; 78 struct etnaviv_file_private *ctx = file->driver_priv; 79 unsigned int i; 80 81 for (i = 0; i < ETNA_MAX_PIPES; i++) { 82 struct etnaviv_gpu *gpu = priv->gpu[i]; 83 84 if (gpu) 85 drm_sched_entity_destroy(&ctx->sched_entity[i]); 86 } 87 88 kfree(ctx); 89 } 90 91 /* 92 * DRM debugfs: 93 */ 94 95 #ifdef CONFIG_DEBUG_FS 96 static int etnaviv_gem_show(struct drm_device *dev, struct seq_file *m) 97 { 98 struct etnaviv_drm_private *priv = dev->dev_private; 99 100 etnaviv_gem_describe_objects(priv, m); 101 102 return 0; 103 } 104 105 static int etnaviv_mm_show(struct drm_device *dev, struct seq_file *m) 106 { 107 struct drm_printer p = drm_seq_file_printer(m); 108 109 read_lock(&dev->vma_offset_manager->vm_lock); 110 drm_mm_print(&dev->vma_offset_manager->vm_addr_space_mm, &p); 111 read_unlock(&dev->vma_offset_manager->vm_lock); 112 113 return 0; 114 } 115 116 static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m) 117 { 118 struct drm_printer p = drm_seq_file_printer(m); 119 120 seq_printf(m, "Active Objects (%s):\n", dev_name(gpu->dev)); 121 122 mutex_lock(&gpu->mmu->lock); 123 drm_mm_print(&gpu->mmu->mm, &p); 124 mutex_unlock(&gpu->mmu->lock); 125 126 return 0; 127 } 128 129 static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, struct seq_file *m) 130 { 131 struct etnaviv_cmdbuf *buf = &gpu->buffer; 132 u32 size = buf->size; 133 u32 *ptr = buf->vaddr; 134 u32 i; 135 136 seq_printf(m, "virt %p - phys 0x%llx - free 0x%08x\n", 137 buf->vaddr, (u64)etnaviv_cmdbuf_get_pa(buf), 138 size - buf->user_size); 139 140 for (i = 0; i < size / 4; i++) { 141 if (i && !(i % 4)) 142 seq_puts(m, "\n"); 143 if (i % 4 == 0) 144 seq_printf(m, "\t0x%p: ", ptr + i); 145 seq_printf(m, "%08x ", *(ptr + i)); 146 } 147 seq_puts(m, "\n"); 148 } 149 150 static int etnaviv_ring_show(struct etnaviv_gpu *gpu, struct seq_file *m) 151 { 152 seq_printf(m, "Ring Buffer (%s): ", dev_name(gpu->dev)); 153 154 mutex_lock(&gpu->lock); 155 etnaviv_buffer_dump(gpu, m); 156 mutex_unlock(&gpu->lock); 157 158 return 0; 159 } 160 161 static int show_unlocked(struct seq_file *m, void *arg) 162 { 163 struct drm_info_node *node = (struct drm_info_node *) m->private; 164 struct drm_device *dev = node->minor->dev; 165 int (*show)(struct drm_device *dev, struct seq_file *m) = 166 node->info_ent->data; 167 168 return show(dev, m); 169 } 170 171 static int show_each_gpu(struct seq_file *m, void *arg) 172 { 173 struct drm_info_node *node = (struct drm_info_node *) m->private; 174 struct drm_device *dev = node->minor->dev; 175 struct etnaviv_drm_private *priv = dev->dev_private; 176 struct etnaviv_gpu *gpu; 177 int (*show)(struct etnaviv_gpu *gpu, struct seq_file *m) = 178 node->info_ent->data; 179 unsigned int i; 180 int ret = 0; 181 182 for (i = 0; i < ETNA_MAX_PIPES; i++) { 183 gpu = priv->gpu[i]; 184 if (!gpu) 185 continue; 186 187 ret = show(gpu, m); 188 if (ret < 0) 189 break; 190 } 191 192 return ret; 193 } 194 195 static struct drm_info_list etnaviv_debugfs_list[] = { 196 {"gpu", show_each_gpu, 0, etnaviv_gpu_debugfs}, 197 {"gem", show_unlocked, 0, etnaviv_gem_show}, 198 { "mm", show_unlocked, 0, etnaviv_mm_show }, 199 {"mmu", show_each_gpu, 0, etnaviv_mmu_show}, 200 {"ring", show_each_gpu, 0, etnaviv_ring_show}, 201 }; 202 203 static int etnaviv_debugfs_init(struct drm_minor *minor) 204 { 205 struct drm_device *dev = minor->dev; 206 int ret; 207 208 ret = drm_debugfs_create_files(etnaviv_debugfs_list, 209 ARRAY_SIZE(etnaviv_debugfs_list), 210 minor->debugfs_root, minor); 211 212 if (ret) { 213 dev_err(dev->dev, "could not install etnaviv_debugfs_list\n"); 214 return ret; 215 } 216 217 return ret; 218 } 219 #endif 220 221 /* 222 * DRM ioctls: 223 */ 224 225 static int etnaviv_ioctl_get_param(struct drm_device *dev, void *data, 226 struct drm_file *file) 227 { 228 struct etnaviv_drm_private *priv = dev->dev_private; 229 struct drm_etnaviv_param *args = data; 230 struct etnaviv_gpu *gpu; 231 232 if (args->pipe >= ETNA_MAX_PIPES) 233 return -EINVAL; 234 235 gpu = priv->gpu[args->pipe]; 236 if (!gpu) 237 return -ENXIO; 238 239 return etnaviv_gpu_get_param(gpu, args->param, &args->value); 240 } 241 242 static int etnaviv_ioctl_gem_new(struct drm_device *dev, void *data, 243 struct drm_file *file) 244 { 245 struct drm_etnaviv_gem_new *args = data; 246 247 if (args->flags & ~(ETNA_BO_CACHED | ETNA_BO_WC | ETNA_BO_UNCACHED | 248 ETNA_BO_FORCE_MMU)) 249 return -EINVAL; 250 251 return etnaviv_gem_new_handle(dev, file, args->size, 252 args->flags, &args->handle); 253 } 254 255 #define TS(t) ((struct timespec){ \ 256 .tv_sec = (t).tv_sec, \ 257 .tv_nsec = (t).tv_nsec \ 258 }) 259 260 static int etnaviv_ioctl_gem_cpu_prep(struct drm_device *dev, void *data, 261 struct drm_file *file) 262 { 263 struct drm_etnaviv_gem_cpu_prep *args = data; 264 struct drm_gem_object *obj; 265 int ret; 266 267 if (args->op & ~(ETNA_PREP_READ | ETNA_PREP_WRITE | ETNA_PREP_NOSYNC)) 268 return -EINVAL; 269 270 obj = drm_gem_object_lookup(file, args->handle); 271 if (!obj) 272 return -ENOENT; 273 274 ret = etnaviv_gem_cpu_prep(obj, args->op, &TS(args->timeout)); 275 276 drm_gem_object_put_unlocked(obj); 277 278 return ret; 279 } 280 281 static int etnaviv_ioctl_gem_cpu_fini(struct drm_device *dev, void *data, 282 struct drm_file *file) 283 { 284 struct drm_etnaviv_gem_cpu_fini *args = data; 285 struct drm_gem_object *obj; 286 int ret; 287 288 if (args->flags) 289 return -EINVAL; 290 291 obj = drm_gem_object_lookup(file, args->handle); 292 if (!obj) 293 return -ENOENT; 294 295 ret = etnaviv_gem_cpu_fini(obj); 296 297 drm_gem_object_put_unlocked(obj); 298 299 return ret; 300 } 301 302 static int etnaviv_ioctl_gem_info(struct drm_device *dev, void *data, 303 struct drm_file *file) 304 { 305 struct drm_etnaviv_gem_info *args = data; 306 struct drm_gem_object *obj; 307 int ret; 308 309 if (args->pad) 310 return -EINVAL; 311 312 obj = drm_gem_object_lookup(file, args->handle); 313 if (!obj) 314 return -ENOENT; 315 316 ret = etnaviv_gem_mmap_offset(obj, &args->offset); 317 drm_gem_object_put_unlocked(obj); 318 319 return ret; 320 } 321 322 static int etnaviv_ioctl_wait_fence(struct drm_device *dev, void *data, 323 struct drm_file *file) 324 { 325 struct drm_etnaviv_wait_fence *args = data; 326 struct etnaviv_drm_private *priv = dev->dev_private; 327 struct timespec *timeout = &TS(args->timeout); 328 struct etnaviv_gpu *gpu; 329 330 if (args->flags & ~(ETNA_WAIT_NONBLOCK)) 331 return -EINVAL; 332 333 if (args->pipe >= ETNA_MAX_PIPES) 334 return -EINVAL; 335 336 gpu = priv->gpu[args->pipe]; 337 if (!gpu) 338 return -ENXIO; 339 340 if (args->flags & ETNA_WAIT_NONBLOCK) 341 timeout = NULL; 342 343 return etnaviv_gpu_wait_fence_interruptible(gpu, args->fence, 344 timeout); 345 } 346 347 static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data, 348 struct drm_file *file) 349 { 350 struct drm_etnaviv_gem_userptr *args = data; 351 352 if (args->flags & ~(ETNA_USERPTR_READ|ETNA_USERPTR_WRITE) || 353 args->flags == 0) 354 return -EINVAL; 355 356 if (offset_in_page(args->user_ptr | args->user_size) || 357 (uintptr_t)args->user_ptr != args->user_ptr || 358 (u32)args->user_size != args->user_size || 359 args->user_ptr & ~PAGE_MASK) 360 return -EINVAL; 361 362 if (!access_ok((void __user *)(unsigned long)args->user_ptr, 363 args->user_size)) 364 return -EFAULT; 365 366 return etnaviv_gem_new_userptr(dev, file, args->user_ptr, 367 args->user_size, args->flags, 368 &args->handle); 369 } 370 371 static int etnaviv_ioctl_gem_wait(struct drm_device *dev, void *data, 372 struct drm_file *file) 373 { 374 struct etnaviv_drm_private *priv = dev->dev_private; 375 struct drm_etnaviv_gem_wait *args = data; 376 struct timespec *timeout = &TS(args->timeout); 377 struct drm_gem_object *obj; 378 struct etnaviv_gpu *gpu; 379 int ret; 380 381 if (args->flags & ~(ETNA_WAIT_NONBLOCK)) 382 return -EINVAL; 383 384 if (args->pipe >= ETNA_MAX_PIPES) 385 return -EINVAL; 386 387 gpu = priv->gpu[args->pipe]; 388 if (!gpu) 389 return -ENXIO; 390 391 obj = drm_gem_object_lookup(file, args->handle); 392 if (!obj) 393 return -ENOENT; 394 395 if (args->flags & ETNA_WAIT_NONBLOCK) 396 timeout = NULL; 397 398 ret = etnaviv_gem_wait_bo(gpu, obj, timeout); 399 400 drm_gem_object_put_unlocked(obj); 401 402 return ret; 403 } 404 405 static int etnaviv_ioctl_pm_query_dom(struct drm_device *dev, void *data, 406 struct drm_file *file) 407 { 408 struct etnaviv_drm_private *priv = dev->dev_private; 409 struct drm_etnaviv_pm_domain *args = data; 410 struct etnaviv_gpu *gpu; 411 412 if (args->pipe >= ETNA_MAX_PIPES) 413 return -EINVAL; 414 415 gpu = priv->gpu[args->pipe]; 416 if (!gpu) 417 return -ENXIO; 418 419 return etnaviv_pm_query_dom(gpu, args); 420 } 421 422 static int etnaviv_ioctl_pm_query_sig(struct drm_device *dev, void *data, 423 struct drm_file *file) 424 { 425 struct etnaviv_drm_private *priv = dev->dev_private; 426 struct drm_etnaviv_pm_signal *args = data; 427 struct etnaviv_gpu *gpu; 428 429 if (args->pipe >= ETNA_MAX_PIPES) 430 return -EINVAL; 431 432 gpu = priv->gpu[args->pipe]; 433 if (!gpu) 434 return -ENXIO; 435 436 return etnaviv_pm_query_sig(gpu, args); 437 } 438 439 static const struct drm_ioctl_desc etnaviv_ioctls[] = { 440 #define ETNA_IOCTL(n, func, flags) \ 441 DRM_IOCTL_DEF_DRV(ETNAVIV_##n, etnaviv_ioctl_##func, flags) 442 ETNA_IOCTL(GET_PARAM, get_param, DRM_AUTH|DRM_RENDER_ALLOW), 443 ETNA_IOCTL(GEM_NEW, gem_new, DRM_AUTH|DRM_RENDER_ALLOW), 444 ETNA_IOCTL(GEM_INFO, gem_info, DRM_AUTH|DRM_RENDER_ALLOW), 445 ETNA_IOCTL(GEM_CPU_PREP, gem_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW), 446 ETNA_IOCTL(GEM_CPU_FINI, gem_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW), 447 ETNA_IOCTL(GEM_SUBMIT, gem_submit, DRM_AUTH|DRM_RENDER_ALLOW), 448 ETNA_IOCTL(WAIT_FENCE, wait_fence, DRM_AUTH|DRM_RENDER_ALLOW), 449 ETNA_IOCTL(GEM_USERPTR, gem_userptr, DRM_AUTH|DRM_RENDER_ALLOW), 450 ETNA_IOCTL(GEM_WAIT, gem_wait, DRM_AUTH|DRM_RENDER_ALLOW), 451 ETNA_IOCTL(PM_QUERY_DOM, pm_query_dom, DRM_AUTH|DRM_RENDER_ALLOW), 452 ETNA_IOCTL(PM_QUERY_SIG, pm_query_sig, DRM_AUTH|DRM_RENDER_ALLOW), 453 }; 454 455 static const struct vm_operations_struct vm_ops = { 456 .fault = etnaviv_gem_fault, 457 .open = drm_gem_vm_open, 458 .close = drm_gem_vm_close, 459 }; 460 461 static const struct file_operations fops = { 462 .owner = THIS_MODULE, 463 .open = drm_open, 464 .release = drm_release, 465 .unlocked_ioctl = drm_ioctl, 466 .compat_ioctl = drm_compat_ioctl, 467 .poll = drm_poll, 468 .read = drm_read, 469 .llseek = no_llseek, 470 .mmap = etnaviv_gem_mmap, 471 }; 472 473 static struct drm_driver etnaviv_drm_driver = { 474 .driver_features = DRIVER_GEM | 475 DRIVER_PRIME | 476 DRIVER_RENDER, 477 .open = etnaviv_open, 478 .postclose = etnaviv_postclose, 479 .gem_free_object_unlocked = etnaviv_gem_free_object, 480 .gem_vm_ops = &vm_ops, 481 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 482 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 483 .gem_prime_export = drm_gem_prime_export, 484 .gem_prime_import = drm_gem_prime_import, 485 .gem_prime_pin = etnaviv_gem_prime_pin, 486 .gem_prime_unpin = etnaviv_gem_prime_unpin, 487 .gem_prime_get_sg_table = etnaviv_gem_prime_get_sg_table, 488 .gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table, 489 .gem_prime_vmap = etnaviv_gem_prime_vmap, 490 .gem_prime_vunmap = etnaviv_gem_prime_vunmap, 491 .gem_prime_mmap = etnaviv_gem_prime_mmap, 492 #ifdef CONFIG_DEBUG_FS 493 .debugfs_init = etnaviv_debugfs_init, 494 #endif 495 .ioctls = etnaviv_ioctls, 496 .num_ioctls = DRM_ETNAVIV_NUM_IOCTLS, 497 .fops = &fops, 498 .name = "etnaviv", 499 .desc = "etnaviv DRM", 500 .date = "20151214", 501 .major = 1, 502 .minor = 2, 503 }; 504 505 /* 506 * Platform driver: 507 */ 508 static int etnaviv_bind(struct device *dev) 509 { 510 struct etnaviv_drm_private *priv; 511 struct drm_device *drm; 512 int ret; 513 514 drm = drm_dev_alloc(&etnaviv_drm_driver, dev); 515 if (IS_ERR(drm)) 516 return PTR_ERR(drm); 517 518 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 519 if (!priv) { 520 dev_err(dev, "failed to allocate private data\n"); 521 ret = -ENOMEM; 522 goto out_put; 523 } 524 drm->dev_private = priv; 525 526 dev->dma_parms = &priv->dma_parms; 527 dma_set_max_seg_size(dev, SZ_2G); 528 529 mutex_init(&priv->gem_lock); 530 INIT_LIST_HEAD(&priv->gem_list); 531 priv->num_gpus = 0; 532 533 dev_set_drvdata(dev, drm); 534 535 ret = component_bind_all(dev, drm); 536 if (ret < 0) 537 goto out_bind; 538 539 load_gpu(drm); 540 541 ret = drm_dev_register(drm, 0); 542 if (ret) 543 goto out_register; 544 545 return 0; 546 547 out_register: 548 component_unbind_all(dev, drm); 549 out_bind: 550 kfree(priv); 551 out_put: 552 drm_dev_put(drm); 553 554 return ret; 555 } 556 557 static void etnaviv_unbind(struct device *dev) 558 { 559 struct drm_device *drm = dev_get_drvdata(dev); 560 struct etnaviv_drm_private *priv = drm->dev_private; 561 562 drm_dev_unregister(drm); 563 564 component_unbind_all(dev, drm); 565 566 dev->dma_parms = NULL; 567 568 drm->dev_private = NULL; 569 kfree(priv); 570 571 drm_dev_put(drm); 572 } 573 574 static const struct component_master_ops etnaviv_master_ops = { 575 .bind = etnaviv_bind, 576 .unbind = etnaviv_unbind, 577 }; 578 579 static int compare_of(struct device *dev, void *data) 580 { 581 struct device_node *np = data; 582 583 return dev->of_node == np; 584 } 585 586 static int compare_str(struct device *dev, void *data) 587 { 588 return !strcmp(dev_name(dev), data); 589 } 590 591 static int etnaviv_pdev_probe(struct platform_device *pdev) 592 { 593 struct device *dev = &pdev->dev; 594 struct component_match *match = NULL; 595 596 if (!dev->platform_data) { 597 struct device_node *core_node; 598 599 for_each_compatible_node(core_node, NULL, "vivante,gc") { 600 if (!of_device_is_available(core_node)) 601 continue; 602 603 drm_of_component_match_add(&pdev->dev, &match, 604 compare_of, core_node); 605 } 606 } else { 607 char **names = dev->platform_data; 608 unsigned i; 609 610 for (i = 0; names[i]; i++) 611 component_match_add(dev, &match, compare_str, names[i]); 612 } 613 614 return component_master_add_with_match(dev, &etnaviv_master_ops, match); 615 } 616 617 static int etnaviv_pdev_remove(struct platform_device *pdev) 618 { 619 component_master_del(&pdev->dev, &etnaviv_master_ops); 620 621 return 0; 622 } 623 624 static struct platform_driver etnaviv_platform_driver = { 625 .probe = etnaviv_pdev_probe, 626 .remove = etnaviv_pdev_remove, 627 .driver = { 628 .name = "etnaviv", 629 }, 630 }; 631 632 static struct platform_device *etnaviv_drm; 633 634 static int __init etnaviv_init(void) 635 { 636 struct platform_device *pdev; 637 int ret; 638 struct device_node *np; 639 640 etnaviv_validate_init(); 641 642 ret = platform_driver_register(&etnaviv_gpu_driver); 643 if (ret != 0) 644 return ret; 645 646 ret = platform_driver_register(&etnaviv_platform_driver); 647 if (ret != 0) 648 goto unregister_gpu_driver; 649 650 /* 651 * If the DT contains at least one available GPU device, instantiate 652 * the DRM platform device. 653 */ 654 for_each_compatible_node(np, NULL, "vivante,gc") { 655 if (!of_device_is_available(np)) 656 continue; 657 658 pdev = platform_device_alloc("etnaviv", -1); 659 if (!pdev) { 660 ret = -ENOMEM; 661 of_node_put(np); 662 goto unregister_platform_driver; 663 } 664 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40); 665 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; 666 667 /* 668 * Apply the same DMA configuration to the virtual etnaviv 669 * device as the GPU we found. This assumes that all Vivante 670 * GPUs in the system share the same DMA constraints. 671 */ 672 of_dma_configure(&pdev->dev, np, true); 673 674 ret = platform_device_add(pdev); 675 if (ret) { 676 platform_device_put(pdev); 677 of_node_put(np); 678 goto unregister_platform_driver; 679 } 680 681 etnaviv_drm = pdev; 682 of_node_put(np); 683 break; 684 } 685 686 return 0; 687 688 unregister_platform_driver: 689 platform_driver_unregister(&etnaviv_platform_driver); 690 unregister_gpu_driver: 691 platform_driver_unregister(&etnaviv_gpu_driver); 692 return ret; 693 } 694 module_init(etnaviv_init); 695 696 static void __exit etnaviv_exit(void) 697 { 698 platform_device_unregister(etnaviv_drm); 699 platform_driver_unregister(&etnaviv_platform_driver); 700 platform_driver_unregister(&etnaviv_gpu_driver); 701 } 702 module_exit(etnaviv_exit); 703 704 MODULE_AUTHOR("Christian Gmeiner <christian.gmeiner@gmail.com>"); 705 MODULE_AUTHOR("Russell King <rmk+kernel@arm.linux.org.uk>"); 706 MODULE_AUTHOR("Lucas Stach <l.stach@pengutronix.de>"); 707 MODULE_DESCRIPTION("etnaviv DRM Driver"); 708 MODULE_LICENSE("GPL v2"); 709 MODULE_ALIAS("platform:etnaviv"); 710