1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #ifdef CONFIG_ARM_ARCH_TIMER 7 #include <asm/arch_timer.h> 8 #endif 9 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/pagemap.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <drm/panfrost_drm.h> 16 #include <drm/drm_debugfs.h> 17 #include <drm/drm_drv.h> 18 #include <drm/drm_ioctl.h> 19 #include <drm/drm_print.h> 20 #include <drm/drm_syncobj.h> 21 #include <drm/drm_utils.h> 22 23 #include "panfrost_device.h" 24 #include "panfrost_gem.h" 25 #include "panfrost_mmu.h" 26 #include "panfrost_job.h" 27 #include "panfrost_gpu.h" 28 #include "panfrost_perfcnt.h" 29 30 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT) 31 32 static bool unstable_ioctls; 33 module_param_unsafe(unstable_ioctls, bool, 0600); 34 35 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, 36 u64 *arg) 37 { 38 int ret; 39 40 ret = pm_runtime_resume_and_get(pfdev->base.dev); 41 if (ret) 42 return ret; 43 44 panfrost_cycle_counter_get(pfdev); 45 *arg = panfrost_timestamp_read(pfdev); 46 panfrost_cycle_counter_put(pfdev); 47 48 pm_runtime_put(pfdev->base.dev); 49 return 0; 50 } 51 52 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 53 { 54 struct drm_panfrost_get_param *param = data; 55 struct panfrost_device *pfdev = to_panfrost_device(ddev); 56 int ret; 57 58 if (param->pad != 0) 59 return -EINVAL; 60 61 #define PANFROST_FEATURE(name, member) \ 62 case DRM_PANFROST_PARAM_ ## name: \ 63 param->value = pfdev->features.member; \ 64 break 65 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 66 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 67 DRM_PANFROST_PARAM_ ## name ## max: \ 68 param->value = pfdev->features.member[param->param - \ 69 DRM_PANFROST_PARAM_ ## name ## 0]; \ 70 break 71 72 switch (param->param) { 73 PANFROST_FEATURE(GPU_PROD_ID, id); 74 PANFROST_FEATURE(GPU_REVISION, revision); 75 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 76 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 77 PANFROST_FEATURE(L2_PRESENT, l2_present); 78 PANFROST_FEATURE(STACK_PRESENT, stack_present); 79 PANFROST_FEATURE(AS_PRESENT, as_present); 80 PANFROST_FEATURE(JS_PRESENT, js_present); 81 PANFROST_FEATURE(L2_FEATURES, l2_features); 82 PANFROST_FEATURE(CORE_FEATURES, core_features); 83 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 84 PANFROST_FEATURE(MEM_FEATURES, mem_features); 85 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 86 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 87 PANFROST_FEATURE(MAX_THREADS, max_threads); 88 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 89 thread_max_workgroup_sz); 90 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 91 thread_max_barrier_sz); 92 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 93 PANFROST_FEATURE(AFBC_FEATURES, afbc_features); 94 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 95 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 96 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 97 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 98 PANFROST_FEATURE(SELECTED_COHERENCY, selected_coherency); 99 100 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP: 101 ret = panfrost_ioctl_query_timestamp(pfdev, ¶m->value); 102 if (ret) 103 return ret; 104 break; 105 106 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY: 107 #ifdef CONFIG_ARM_ARCH_TIMER 108 param->value = arch_timer_get_cntfrq(); 109 #else 110 param->value = 0; 111 #endif 112 break; 113 114 case DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES: 115 param->value = BIT(PANFROST_JM_CTX_PRIORITY_LOW) | 116 BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM); 117 118 if (panfrost_high_prio_allowed(file)) 119 param->value |= BIT(PANFROST_JM_CTX_PRIORITY_HIGH); 120 break; 121 122 default: 123 return -EINVAL; 124 } 125 126 return 0; 127 } 128 129 #define PANFROST_BO_FLAGS (PANFROST_BO_NOEXEC | \ 130 PANFROST_BO_HEAP | \ 131 PANFROST_BO_WB_MMAP) 132 133 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 134 struct drm_file *file) 135 { 136 struct panfrost_file_priv *priv = file->driver_priv; 137 struct panfrost_gem_object *bo; 138 struct drm_panfrost_create_bo *args = data; 139 struct panfrost_gem_mapping *mapping; 140 int ret; 141 142 if (!args->size || args->pad || (args->flags & ~PANFROST_BO_FLAGS)) 143 return -EINVAL; 144 145 /* Heaps should never be executable */ 146 if ((args->flags & PANFROST_BO_HEAP) && 147 !(args->flags & PANFROST_BO_NOEXEC)) 148 return -EINVAL; 149 150 bo = panfrost_gem_create(dev, args->size, args->flags); 151 if (IS_ERR(bo)) 152 return PTR_ERR(bo); 153 154 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 155 if (ret) 156 goto out; 157 158 mapping = panfrost_gem_mapping_get(bo, priv); 159 if (mapping) { 160 args->offset = mapping->mmnode.start << PAGE_SHIFT; 161 panfrost_gem_mapping_put(mapping); 162 } else { 163 /* This can only happen if the handle from 164 * drm_gem_handle_create() has already been guessed and freed 165 * by user space 166 */ 167 ret = -EINVAL; 168 } 169 170 out: 171 drm_gem_object_put(&bo->base.base); 172 return ret; 173 } 174 175 /** 176 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 177 * referenced by the job. 178 * @dev: DRM device 179 * @file_priv: DRM file for this fd 180 * @args: IOCTL args 181 * @job: job being set up 182 * 183 * Resolve handles from userspace to BOs and attach them to job. 184 * 185 * Note that this function doesn't need to unreference the BOs on 186 * failure, because that will happen at panfrost_job_cleanup() time. 187 */ 188 static int 189 panfrost_lookup_bos(struct drm_device *dev, 190 struct drm_file *file_priv, 191 struct drm_panfrost_submit *args, 192 struct panfrost_job *job) 193 { 194 struct panfrost_file_priv *priv = file_priv->driver_priv; 195 struct panfrost_gem_object *bo; 196 unsigned int i; 197 int ret; 198 199 job->bo_count = args->bo_handle_count; 200 201 if (!job->bo_count) 202 return 0; 203 204 ret = drm_gem_objects_lookup(file_priv, 205 (void __user *)(uintptr_t)args->bo_handles, 206 job->bo_count, &job->bos); 207 if (ret) 208 return ret; 209 210 job->mappings = kvmalloc_array(job->bo_count, 211 sizeof(struct panfrost_gem_mapping *), 212 GFP_KERNEL | __GFP_ZERO); 213 if (!job->mappings) 214 return -ENOMEM; 215 216 for (i = 0; i < job->bo_count; i++) { 217 struct panfrost_gem_mapping *mapping; 218 219 bo = to_panfrost_bo(job->bos[i]); 220 mapping = panfrost_gem_mapping_get(bo, priv); 221 if (!mapping) { 222 ret = -EINVAL; 223 break; 224 } 225 226 atomic_inc(&bo->gpu_usecount); 227 job->mappings[i] = mapping; 228 } 229 230 return ret; 231 } 232 233 /** 234 * panfrost_copy_in_sync() - Sets up job->deps with the sync objects 235 * referenced by the job. 236 * @dev: DRM device 237 * @file_priv: DRM file for this fd 238 * @args: IOCTL args 239 * @job: job being set up 240 * 241 * Resolve syncobjs from userspace to fences and attach them to job. 242 * 243 * Note that this function doesn't need to unreference the fences on 244 * failure, because that will happen at panfrost_job_cleanup() time. 245 */ 246 static int 247 panfrost_copy_in_sync(struct drm_device *dev, 248 struct drm_file *file_priv, 249 struct drm_panfrost_submit *args, 250 struct panfrost_job *job) 251 { 252 u32 *handles; 253 int ret = 0; 254 int i, in_fence_count; 255 256 in_fence_count = args->in_sync_count; 257 258 if (!in_fence_count) 259 return 0; 260 261 handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL); 262 if (!handles) { 263 ret = -ENOMEM; 264 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 265 goto fail; 266 } 267 268 if (copy_from_user(handles, 269 (void __user *)(uintptr_t)args->in_syncs, 270 in_fence_count * sizeof(u32))) { 271 ret = -EFAULT; 272 DRM_DEBUG("Failed to copy in syncobj handles\n"); 273 goto fail; 274 } 275 276 for (i = 0; i < in_fence_count; i++) { 277 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 278 handles[i], 0); 279 if (ret) 280 goto fail; 281 } 282 283 fail: 284 kvfree(handles); 285 return ret; 286 } 287 288 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 289 struct drm_file *file) 290 { 291 struct panfrost_device *pfdev = to_panfrost_device(dev); 292 struct panfrost_file_priv *file_priv = file->driver_priv; 293 struct drm_panfrost_submit *args = data; 294 struct drm_syncobj *sync_out = NULL; 295 struct panfrost_jm_ctx *jm_ctx; 296 struct panfrost_job *job; 297 int ret = 0, slot; 298 299 if (args->pad) 300 return -EINVAL; 301 302 if (!args->jc) 303 return -EINVAL; 304 305 if (args->requirements & ~JOB_REQUIREMENTS) 306 return -EINVAL; 307 308 if (args->out_sync > 0) { 309 sync_out = drm_syncobj_find(file, args->out_sync); 310 if (!sync_out) 311 return -ENODEV; 312 } 313 314 jm_ctx = panfrost_jm_ctx_from_handle(file, args->jm_ctx_handle); 315 if (!jm_ctx) { 316 ret = -EINVAL; 317 goto out_put_syncout; 318 } 319 320 job = kzalloc(sizeof(*job), GFP_KERNEL); 321 if (!job) { 322 ret = -ENOMEM; 323 goto out_put_jm_ctx; 324 } 325 326 kref_init(&job->refcount); 327 328 job->pfdev = pfdev; 329 job->jc = args->jc; 330 job->requirements = args->requirements; 331 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 332 job->mmu = file_priv->mmu; 333 job->ctx = panfrost_jm_ctx_get(jm_ctx); 334 job->engine_usage = &file_priv->engine_usage; 335 336 slot = panfrost_job_get_slot(job); 337 338 ret = drm_sched_job_init(&job->base, 339 &jm_ctx->slot_entity[slot], 340 1, NULL, file->client_id); 341 if (ret) 342 goto out_put_job; 343 344 ret = panfrost_copy_in_sync(dev, file, args, job); 345 if (ret) 346 goto out_cleanup_job; 347 348 ret = panfrost_lookup_bos(dev, file, args, job); 349 if (ret) 350 goto out_cleanup_job; 351 352 ret = panfrost_job_push(job); 353 if (ret) 354 goto out_cleanup_job; 355 356 /* Update the return sync object for the job */ 357 if (sync_out) 358 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 359 360 out_cleanup_job: 361 if (ret) 362 drm_sched_job_cleanup(&job->base); 363 out_put_job: 364 panfrost_job_put(job); 365 out_put_jm_ctx: 366 panfrost_jm_ctx_put(jm_ctx); 367 out_put_syncout: 368 if (sync_out) 369 drm_syncobj_put(sync_out); 370 371 return ret; 372 } 373 374 static int 375 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 376 struct drm_file *file_priv) 377 { 378 long ret; 379 struct drm_panfrost_wait_bo *args = data; 380 struct drm_gem_object *gem_obj; 381 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 382 383 if (args->pad) 384 return -EINVAL; 385 386 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 387 if (!gem_obj) 388 return -ENOENT; 389 390 ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ, 391 true, timeout); 392 if (!ret) 393 ret = timeout ? -ETIMEDOUT : -EBUSY; 394 395 drm_gem_object_put(gem_obj); 396 397 return ret; 398 } 399 400 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 401 struct drm_file *file_priv) 402 { 403 struct drm_panfrost_mmap_bo *args = data; 404 struct drm_gem_object *gem_obj; 405 int ret; 406 407 if (args->flags != 0) { 408 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 409 return -EINVAL; 410 } 411 412 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 413 if (!gem_obj) { 414 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 415 return -ENOENT; 416 } 417 418 /* Don't allow mmapping of heap objects as pages are not pinned. */ 419 if (to_panfrost_bo(gem_obj)->is_heap) { 420 ret = -EINVAL; 421 goto out; 422 } 423 424 ret = drm_gem_create_mmap_offset(gem_obj); 425 if (ret == 0) 426 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 427 428 out: 429 drm_gem_object_put(gem_obj); 430 return ret; 431 } 432 433 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 434 struct drm_file *file_priv) 435 { 436 struct panfrost_file_priv *priv = file_priv->driver_priv; 437 struct drm_panfrost_get_bo_offset *args = data; 438 struct panfrost_gem_mapping *mapping; 439 struct drm_gem_object *gem_obj; 440 struct panfrost_gem_object *bo; 441 442 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 443 if (!gem_obj) { 444 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 445 return -ENOENT; 446 } 447 bo = to_panfrost_bo(gem_obj); 448 449 mapping = panfrost_gem_mapping_get(bo, priv); 450 drm_gem_object_put(gem_obj); 451 452 if (!mapping) 453 return -EINVAL; 454 455 args->offset = mapping->mmnode.start << PAGE_SHIFT; 456 panfrost_gem_mapping_put(mapping); 457 return 0; 458 } 459 460 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, 461 struct drm_file *file_priv) 462 { 463 struct panfrost_file_priv *priv = file_priv->driver_priv; 464 struct drm_panfrost_madvise *args = data; 465 struct panfrost_device *pfdev = to_panfrost_device(dev); 466 struct drm_gem_object *gem_obj; 467 struct panfrost_gem_object *bo; 468 int ret = 0; 469 470 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 471 if (!gem_obj) { 472 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 473 return -ENOENT; 474 } 475 476 bo = to_panfrost_bo(gem_obj); 477 478 ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); 479 if (ret) 480 goto out_put_object; 481 482 mutex_lock(&pfdev->shrinker_lock); 483 mutex_lock(&bo->mappings.lock); 484 if (args->madv == PANFROST_MADV_DONTNEED) { 485 struct panfrost_gem_mapping *first; 486 487 first = list_first_entry(&bo->mappings.list, 488 struct panfrost_gem_mapping, 489 node); 490 491 /* 492 * If we want to mark the BO purgeable, there must be only one 493 * user: the caller FD. 494 * We could do something smarter and mark the BO purgeable only 495 * when all its users have marked it purgeable, but globally 496 * visible/shared BOs are likely to never be marked purgeable 497 * anyway, so let's not bother. 498 */ 499 if (!list_is_singular(&bo->mappings.list) || 500 WARN_ON_ONCE(first->mmu != priv->mmu)) { 501 ret = -EINVAL; 502 goto out_unlock_mappings; 503 } 504 } 505 506 args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv); 507 508 if (args->retained) { 509 if (args->madv == PANFROST_MADV_DONTNEED) 510 list_move_tail(&bo->base.madv_list, 511 &pfdev->shrinker_list); 512 else if (args->madv == PANFROST_MADV_WILLNEED) 513 list_del_init(&bo->base.madv_list); 514 } 515 516 out_unlock_mappings: 517 mutex_unlock(&bo->mappings.lock); 518 mutex_unlock(&pfdev->shrinker_lock); 519 dma_resv_unlock(bo->base.base.resv); 520 out_put_object: 521 drm_gem_object_put(gem_obj); 522 return ret; 523 } 524 525 static int panfrost_ioctl_set_label_bo(struct drm_device *ddev, void *data, 526 struct drm_file *file) 527 { 528 struct drm_panfrost_set_label_bo *args = data; 529 struct drm_gem_object *obj; 530 const char *label = NULL; 531 int ret = 0; 532 533 if (args->pad) 534 return -EINVAL; 535 536 obj = drm_gem_object_lookup(file, args->handle); 537 if (!obj) 538 return -ENOENT; 539 540 if (args->label) { 541 label = strndup_user(u64_to_user_ptr(args->label), 542 PANFROST_BO_LABEL_MAXLEN); 543 if (IS_ERR(label)) { 544 ret = PTR_ERR(label); 545 if (ret == -EINVAL) 546 ret = -E2BIG; 547 goto err_put_obj; 548 } 549 } 550 551 /* 552 * We treat passing a label of length 0 and passing a NULL label 553 * differently, because even though they might seem conceptually 554 * similar, future uses of the BO label might expect a different 555 * behaviour in each case. 556 */ 557 panfrost_gem_set_label(obj, label); 558 559 err_put_obj: 560 drm_gem_object_put(obj); 561 562 return ret; 563 } 564 565 static int panfrost_ioctl_jm_ctx_create(struct drm_device *dev, void *data, 566 struct drm_file *file) 567 { 568 return panfrost_jm_ctx_create(file, data); 569 } 570 571 static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data, 572 struct drm_file *file) 573 { 574 const struct drm_panfrost_jm_ctx_destroy *args = data; 575 576 if (args->pad) 577 return -EINVAL; 578 579 /* We can't destroy the default context created when the file is opened. */ 580 if (!args->handle) 581 return -EINVAL; 582 583 return panfrost_jm_ctx_destroy(file, args->handle); 584 } 585 586 static int panfrost_ioctl_sync_bo(struct drm_device *ddev, void *data, 587 struct drm_file *file) 588 { 589 struct drm_panfrost_sync_bo *args = data; 590 struct drm_panfrost_bo_sync_op *ops; 591 struct drm_gem_object *obj; 592 int ret; 593 u32 i; 594 595 if (args->pad) 596 return -EINVAL; 597 598 if (!args->op_count) 599 return 0; 600 601 ops = kvmalloc_array(args->op_count, sizeof(*ops), GFP_KERNEL); 602 if (!ops) { 603 DRM_DEBUG("Failed to allocate incoming BO sync ops array\n"); 604 return -ENOMEM; 605 } 606 607 if (copy_from_user(ops, (void __user *)(uintptr_t)args->ops, 608 args->op_count * sizeof(*ops))) { 609 DRM_DEBUG("Failed to copy in BO sync ops\n"); 610 ret = -EFAULT; 611 goto err_ops; 612 } 613 614 for (i = 0; i < args->op_count; i++) { 615 obj = drm_gem_object_lookup(file, ops[i].handle); 616 if (!obj) { 617 ret = -ENOENT; 618 goto err_ops; 619 } 620 621 ret = panfrost_gem_sync(obj, ops[i].type, 622 ops[i].offset, ops[i].size); 623 624 drm_gem_object_put(obj); 625 626 if (ret) 627 goto err_ops; 628 } 629 630 err_ops: 631 kvfree(ops); 632 633 return ret; 634 } 635 636 static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data, 637 struct drm_file *file_priv) 638 { 639 struct drm_panfrost_query_bo_info *args = data; 640 struct drm_gem_object *gem_obj; 641 struct panfrost_gem_object *bo; 642 643 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 644 if (!gem_obj) { 645 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 646 return -ENOENT; 647 } 648 649 bo = to_panfrost_bo(gem_obj); 650 args->pad = 0; 651 args->create_flags = 0; 652 args->extra_flags = 0; 653 654 if (drm_gem_is_imported(gem_obj)) { 655 args->extra_flags |= DRM_PANFROST_BO_IS_IMPORTED; 656 } else { 657 if (bo->noexec) 658 args->create_flags |= PANFROST_BO_NOEXEC; 659 660 if (bo->is_heap) 661 args->create_flags |= PANFROST_BO_HEAP; 662 663 if (!bo->base.map_wc) 664 args->create_flags |= PANFROST_BO_WB_MMAP; 665 } 666 667 drm_gem_object_put(gem_obj); 668 return 0; 669 } 670 671 int panfrost_unstable_ioctl_check(void) 672 { 673 if (!unstable_ioctls) 674 return -ENOSYS; 675 676 return 0; 677 } 678 679 static int 680 panfrost_open(struct drm_device *dev, struct drm_file *file) 681 { 682 int ret; 683 struct panfrost_device *pfdev = to_panfrost_device(dev); 684 struct panfrost_file_priv *panfrost_priv; 685 686 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 687 if (!panfrost_priv) 688 return -ENOMEM; 689 690 panfrost_priv->pfdev = pfdev; 691 file->driver_priv = panfrost_priv; 692 693 panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev); 694 if (IS_ERR(panfrost_priv->mmu)) { 695 ret = PTR_ERR(panfrost_priv->mmu); 696 goto err_free; 697 } 698 699 ret = panfrost_jm_open(file); 700 if (ret) 701 goto err_job; 702 703 return 0; 704 705 err_job: 706 panfrost_mmu_ctx_put(panfrost_priv->mmu); 707 err_free: 708 kfree(panfrost_priv); 709 return ret; 710 } 711 712 static void 713 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 714 { 715 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 716 717 panfrost_perfcnt_close(file); 718 panfrost_jm_close(file); 719 720 panfrost_mmu_ctx_put(panfrost_priv->mmu); 721 kfree(panfrost_priv); 722 } 723 724 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 725 #define PANFROST_IOCTL(n, func, flags) \ 726 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 727 728 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW), 729 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 730 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 731 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 732 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 733 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 734 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 735 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 736 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), 737 PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW), 738 PANFROST_IOCTL(JM_CTX_CREATE, jm_ctx_create, DRM_RENDER_ALLOW), 739 PANFROST_IOCTL(JM_CTX_DESTROY, jm_ctx_destroy, DRM_RENDER_ALLOW), 740 PANFROST_IOCTL(SYNC_BO, sync_bo, DRM_RENDER_ALLOW), 741 PANFROST_IOCTL(QUERY_BO_INFO, query_bo_info, DRM_RENDER_ALLOW), 742 }; 743 744 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, 745 struct panfrost_file_priv *panfrost_priv, 746 struct drm_printer *p) 747 { 748 int i; 749 750 /* 751 * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not 752 * accurate, as they only provide a rough estimation of the number of 753 * GPU cycles and CPU time spent in a given context. This is due to two 754 * different factors: 755 * - Firstly, we must consider the time the CPU and then the kernel 756 * takes to process the GPU interrupt, which means additional time and 757 * GPU cycles will be added in excess to the real figure. 758 * - Secondly, the pipelining done by the Job Manager (2 job slots per 759 * engine) implies there is no way to know exactly how much time each 760 * job spent on the GPU. 761 */ 762 763 for (i = 0; i < NUM_JOB_SLOTS - 1; i++) { 764 if (pfdev->profile_mode) { 765 drm_printf(p, "drm-engine-%s:\t%llu ns\n", 766 panfrost_engine_names[i], 767 panfrost_priv->engine_usage.elapsed_ns[i]); 768 drm_printf(p, "drm-cycles-%s:\t%llu\n", 769 panfrost_engine_names[i], 770 panfrost_priv->engine_usage.cycles[i]); 771 } 772 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n", 773 panfrost_engine_names[i], pfdev->pfdevfreq.fast_rate); 774 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n", 775 panfrost_engine_names[i], pfdev->pfdevfreq.current_frequency); 776 } 777 } 778 779 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file) 780 { 781 struct panfrost_device *pfdev = to_panfrost_device(file->minor->dev); 782 783 panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p); 784 785 drm_show_memory_stats(p, file); 786 } 787 788 static const struct file_operations panfrost_drm_driver_fops = { 789 .owner = THIS_MODULE, 790 DRM_GEM_FOPS, 791 .show_fdinfo = drm_show_fdinfo, 792 }; 793 794 #ifdef CONFIG_DEBUG_FS 795 static int panthor_gems_show(struct seq_file *m, void *data) 796 { 797 struct drm_info_node *node = m->private; 798 struct panfrost_device *pfdev = to_panfrost_device(node->minor->dev); 799 800 panfrost_gem_debugfs_print_bos(pfdev, m); 801 802 return 0; 803 } 804 805 static void show_panfrost_jm_ctx(struct panfrost_jm_ctx *jm_ctx, u32 handle, 806 struct seq_file *m) 807 { 808 struct drm_device *ddev = ((struct drm_info_node *)m->private)->minor->dev; 809 const char *prio = "UNKNOWN"; 810 811 static const char * const prios[] = { 812 [DRM_SCHED_PRIORITY_HIGH] = "HIGH", 813 [DRM_SCHED_PRIORITY_NORMAL] = "NORMAL", 814 [DRM_SCHED_PRIORITY_LOW] = "LOW", 815 }; 816 817 if (jm_ctx->slot_entity[0].priority != 818 jm_ctx->slot_entity[1].priority) 819 drm_warn(ddev, "Slot priorities should be the same in a single context"); 820 821 if (jm_ctx->slot_entity[0].priority < ARRAY_SIZE(prios)) 822 prio = prios[jm_ctx->slot_entity[0].priority]; 823 824 seq_printf(m, " JM context %u: priority %s\n", handle, prio); 825 } 826 827 static int show_file_jm_ctxs(struct panfrost_file_priv *pfile, 828 struct seq_file *m) 829 { 830 struct panfrost_jm_ctx *jm_ctx; 831 unsigned long i; 832 833 xa_lock(&pfile->jm_ctxs); 834 xa_for_each(&pfile->jm_ctxs, i, jm_ctx) { 835 jm_ctx = panfrost_jm_ctx_get(jm_ctx); 836 xa_unlock(&pfile->jm_ctxs); 837 show_panfrost_jm_ctx(jm_ctx, i, m); 838 panfrost_jm_ctx_put(jm_ctx); 839 xa_lock(&pfile->jm_ctxs); 840 } 841 xa_unlock(&pfile->jm_ctxs); 842 843 return 0; 844 } 845 846 static struct drm_info_list panthor_debugfs_list[] = { 847 {"gems", 848 panthor_gems_show, 0, NULL}, 849 }; 850 851 static int panthor_gems_debugfs_init(struct drm_minor *minor) 852 { 853 drm_debugfs_create_files(panthor_debugfs_list, 854 ARRAY_SIZE(panthor_debugfs_list), 855 minor->debugfs_root, minor); 856 857 return 0; 858 } 859 860 static int show_each_file(struct seq_file *m, void *arg) 861 { 862 struct drm_info_node *node = (struct drm_info_node *)m->private; 863 struct drm_device *ddev = node->minor->dev; 864 int (*show)(struct panfrost_file_priv *, struct seq_file *) = 865 node->info_ent->data; 866 struct drm_file *file; 867 int ret; 868 869 ret = mutex_lock_interruptible(&ddev->filelist_mutex); 870 if (ret) 871 return ret; 872 873 list_for_each_entry(file, &ddev->filelist, lhead) { 874 struct task_struct *task; 875 struct panfrost_file_priv *pfile = file->driver_priv; 876 struct pid *pid; 877 878 /* 879 * Although we have a valid reference on file->pid, that does 880 * not guarantee that the task_struct who called get_pid() is 881 * still alive (e.g. get_pid(current) => fork() => exit()). 882 * Therefore, we need to protect this ->comm access using RCU. 883 */ 884 rcu_read_lock(); 885 pid = rcu_dereference(file->pid); 886 task = pid_task(pid, PIDTYPE_TGID); 887 seq_printf(m, "client_id %8llu pid %8d command %s:\n", 888 file->client_id, pid_nr(pid), 889 task ? task->comm : "<unknown>"); 890 rcu_read_unlock(); 891 892 ret = show(pfile, m); 893 if (ret < 0) 894 break; 895 896 seq_puts(m, "\n"); 897 } 898 899 mutex_unlock(&ddev->filelist_mutex); 900 return ret; 901 } 902 903 static struct drm_info_list panfrost_sched_debugfs_list[] = { 904 { "sched_ctxs", show_each_file, 0, show_file_jm_ctxs }, 905 }; 906 907 static void panfrost_sched_debugfs_init(struct drm_minor *minor) 908 { 909 drm_debugfs_create_files(panfrost_sched_debugfs_list, 910 ARRAY_SIZE(panfrost_sched_debugfs_list), 911 minor->debugfs_root, minor); 912 } 913 914 static void panfrost_debugfs_init(struct drm_minor *minor) 915 { 916 panthor_gems_debugfs_init(minor); 917 panfrost_sched_debugfs_init(minor); 918 } 919 #endif 920 921 /* 922 * Panfrost driver version: 923 * - 1.0 - initial interface 924 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO 925 * - 1.2 - adds AFBC_FEATURES query 926 * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT 927 * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries 928 * - 1.4 - adds SET_LABEL_BO 929 * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow 930 * context creation with configurable priorities/affinity 931 * - 1.6 - adds PANFROST_BO_MAP_WB, PANFROST_IOCTL_SYNC_BO, 932 * PANFROST_IOCTL_QUERY_BO_INFO and 933 * DRM_PANFROST_PARAM_SELECTED_COHERENCY 934 */ 935 static const struct drm_driver panfrost_drm_driver = { 936 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 937 .open = panfrost_open, 938 .postclose = panfrost_postclose, 939 .show_fdinfo = panfrost_show_fdinfo, 940 .ioctls = panfrost_drm_driver_ioctls, 941 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 942 .fops = &panfrost_drm_driver_fops, 943 .name = "panfrost", 944 .desc = "panfrost DRM", 945 .major = 1, 946 .minor = 6, 947 948 .gem_create_object = panfrost_gem_create_object, 949 .gem_prime_import = panfrost_gem_prime_import, 950 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 951 #ifdef CONFIG_DEBUG_FS 952 .debugfs_init = panfrost_debugfs_init, 953 #endif 954 }; 955 956 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 957 bool panfrost_transparent_hugepage = true; 958 module_param_named(transparent_hugepage, panfrost_transparent_hugepage, bool, 0400); 959 MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)"); 960 #endif 961 962 static int panfrost_probe(struct platform_device *pdev) 963 { 964 struct panfrost_device *pfdev; 965 int err; 966 967 pfdev = devm_drm_dev_alloc(&pdev->dev, &panfrost_drm_driver, 968 struct panfrost_device, base); 969 if (IS_ERR(pfdev)) 970 return PTR_ERR(pfdev); 971 972 platform_set_drvdata(pdev, pfdev); 973 974 pfdev->comp = of_device_get_match_data(&pdev->dev); 975 if (!pfdev->comp) 976 return -ENODEV; 977 978 pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT; 979 980 mutex_init(&pfdev->shrinker_lock); 981 INIT_LIST_HEAD(&pfdev->shrinker_list); 982 983 err = panfrost_device_init(pfdev); 984 if (err) { 985 if (err != -EPROBE_DEFER) 986 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 987 goto err_out0; 988 } 989 990 pm_runtime_set_active(pfdev->base.dev); 991 pm_runtime_mark_last_busy(pfdev->base.dev); 992 pm_runtime_enable(pfdev->base.dev); 993 pm_runtime_set_autosuspend_delay(pfdev->base.dev, 50); /* ~3 frames */ 994 pm_runtime_use_autosuspend(pfdev->base.dev); 995 996 /* 997 * Register the DRM device with the core and the connectors with 998 * sysfs 999 */ 1000 err = drm_dev_register(&pfdev->base, 0); 1001 if (err < 0) 1002 goto err_out1; 1003 1004 err = panfrost_gem_shrinker_init(&pfdev->base); 1005 if (err) 1006 goto err_out2; 1007 1008 return 0; 1009 1010 err_out2: 1011 drm_dev_unregister(&pfdev->base); 1012 err_out1: 1013 pm_runtime_disable(pfdev->base.dev); 1014 panfrost_device_fini(pfdev); 1015 pm_runtime_set_suspended(pfdev->base.dev); 1016 err_out0: 1017 return err; 1018 } 1019 1020 static void panfrost_remove(struct platform_device *pdev) 1021 { 1022 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 1023 1024 drm_dev_unregister(&pfdev->base); 1025 panfrost_gem_shrinker_cleanup(&pfdev->base); 1026 1027 pm_runtime_get_sync(pfdev->base.dev); 1028 pm_runtime_disable(pfdev->base.dev); 1029 panfrost_device_fini(pfdev); 1030 pm_runtime_set_suspended(pfdev->base.dev); 1031 } 1032 1033 static ssize_t profiling_show(struct device *dev, 1034 struct device_attribute *attr, char *buf) 1035 { 1036 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1037 1038 return sysfs_emit(buf, "%d\n", pfdev->profile_mode); 1039 } 1040 1041 static ssize_t profiling_store(struct device *dev, 1042 struct device_attribute *attr, 1043 const char *buf, size_t len) 1044 { 1045 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1046 bool value; 1047 int err; 1048 1049 err = kstrtobool(buf, &value); 1050 if (err) 1051 return err; 1052 1053 pfdev->profile_mode = value; 1054 1055 return len; 1056 } 1057 1058 static DEVICE_ATTR_RW(profiling); 1059 1060 static struct attribute *panfrost_attrs[] = { 1061 &dev_attr_profiling.attr, 1062 NULL, 1063 }; 1064 1065 ATTRIBUTE_GROUPS(panfrost); 1066 1067 /* 1068 * The OPP core wants the supply names to be NULL terminated, but we need the 1069 * correct num_supplies value for regulator core. Hence, we NULL terminate here 1070 * and then initialize num_supplies with ARRAY_SIZE - 1. 1071 */ 1072 static const char * const default_supplies[] = { "mali", NULL }; 1073 static const struct panfrost_compatible default_data = { 1074 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1075 .supply_names = default_supplies, 1076 .num_pm_domains = 1, /* optional */ 1077 .pm_domain_names = NULL, 1078 }; 1079 1080 static const struct panfrost_compatible default_pm_rt_data = { 1081 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1082 .supply_names = default_supplies, 1083 .num_pm_domains = 1, 1084 .pm_features = BIT(GPU_PM_RT), 1085 }; 1086 1087 static const struct panfrost_compatible amlogic_data = { 1088 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1089 .supply_names = default_supplies, 1090 .vendor_quirk = panfrost_gpu_amlogic_quirk, 1091 }; 1092 1093 static const char * const mediatek_pm_domains[] = { "core0", "core1", "core2", 1094 "core3", "core4" }; 1095 /* 1096 * The old data with two power supplies for MT8183 is here only to 1097 * keep retro-compatibility with older devicetrees, as DVFS will 1098 * not work with this one. 1099 * 1100 * On new devicetrees please use the _b variant with a single and 1101 * coupled regulators instead. 1102 */ 1103 static const char * const legacy_supplies[] = { "mali", "sram", NULL }; 1104 static const struct panfrost_compatible mediatek_mt8183_data = { 1105 .num_supplies = ARRAY_SIZE(legacy_supplies) - 1, 1106 .supply_names = legacy_supplies, 1107 .num_pm_domains = 3, 1108 .pm_domain_names = mediatek_pm_domains, 1109 }; 1110 1111 static const struct panfrost_compatible mediatek_mt8183_b_data = { 1112 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1113 .supply_names = default_supplies, 1114 .num_pm_domains = 3, 1115 .pm_domain_names = mediatek_pm_domains, 1116 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1117 }; 1118 1119 static const struct panfrost_compatible mediatek_mt8186_data = { 1120 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1121 .supply_names = default_supplies, 1122 .num_pm_domains = 2, 1123 .pm_domain_names = mediatek_pm_domains, 1124 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1125 }; 1126 1127 static const struct panfrost_compatible mediatek_mt8188_data = { 1128 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1129 .supply_names = default_supplies, 1130 .num_pm_domains = 3, 1131 .pm_domain_names = mediatek_pm_domains, 1132 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1133 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1134 }; 1135 1136 static const struct panfrost_compatible mediatek_mt8192_data = { 1137 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1138 .supply_names = default_supplies, 1139 .num_pm_domains = 5, 1140 .pm_domain_names = mediatek_pm_domains, 1141 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1142 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1143 }; 1144 1145 static const struct panfrost_compatible mediatek_mt8370_data = { 1146 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1147 .supply_names = default_supplies, 1148 .num_pm_domains = 2, 1149 .pm_domain_names = mediatek_pm_domains, 1150 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1151 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1152 }; 1153 1154 static const struct of_device_id dt_match[] = { 1155 /* Set first to probe before the generic compatibles */ 1156 { .compatible = "amlogic,meson-gxm-mali", 1157 .data = &amlogic_data, }, 1158 { .compatible = "amlogic,meson-g12a-mali", 1159 .data = &amlogic_data, }, 1160 { .compatible = "renesas,r9a09g047-mali", .data = &default_pm_rt_data }, 1161 { .compatible = "arm,mali-t604", .data = &default_data, }, 1162 { .compatible = "arm,mali-t624", .data = &default_data, }, 1163 { .compatible = "arm,mali-t628", .data = &default_data, }, 1164 { .compatible = "arm,mali-t720", .data = &default_data, }, 1165 { .compatible = "arm,mali-t760", .data = &default_data, }, 1166 { .compatible = "arm,mali-t820", .data = &default_data, }, 1167 { .compatible = "arm,mali-t830", .data = &default_data, }, 1168 { .compatible = "arm,mali-t860", .data = &default_data, }, 1169 { .compatible = "arm,mali-t880", .data = &default_data, }, 1170 { .compatible = "arm,mali-bifrost", .data = &default_data, }, 1171 { .compatible = "arm,mali-valhall-jm", .data = &default_data, }, 1172 { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data }, 1173 { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data }, 1174 { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data }, 1175 { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data }, 1176 { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data }, 1177 { .compatible = "mediatek,mt8370-mali", .data = &mediatek_mt8370_data }, 1178 { .compatible = "allwinner,sun50i-h616-mali", .data = &default_pm_rt_data }, 1179 {} 1180 }; 1181 MODULE_DEVICE_TABLE(of, dt_match); 1182 1183 static struct platform_driver panfrost_driver = { 1184 .probe = panfrost_probe, 1185 .remove = panfrost_remove, 1186 .driver = { 1187 .name = "panfrost", 1188 .pm = pm_ptr(&panfrost_pm_ops), 1189 .of_match_table = dt_match, 1190 .dev_groups = panfrost_groups, 1191 }, 1192 }; 1193 module_platform_driver(panfrost_driver); 1194 1195 MODULE_AUTHOR("Panfrost Project Developers"); 1196 MODULE_DESCRIPTION("Panfrost DRM Driver"); 1197 MODULE_LICENSE("GPL v2"); 1198 MODULE_SOFTDEP("pre: governor_simpleondemand"); 1199