1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #ifdef CONFIG_ARM_ARCH_TIMER 7 #include <asm/arch_timer.h> 8 #endif 9 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/pagemap.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <drm/panfrost_drm.h> 16 #include <drm/drm_debugfs.h> 17 #include <drm/drm_drv.h> 18 #include <drm/drm_ioctl.h> 19 #include <drm/drm_print.h> 20 #include <drm/drm_syncobj.h> 21 #include <drm/drm_utils.h> 22 23 #include "panfrost_device.h" 24 #include "panfrost_gem.h" 25 #include "panfrost_mmu.h" 26 #include "panfrost_job.h" 27 #include "panfrost_gpu.h" 28 #include "panfrost_perfcnt.h" 29 30 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT) 31 32 static bool unstable_ioctls; 33 module_param_unsafe(unstable_ioctls, bool, 0600); 34 35 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, 36 u64 *arg) 37 { 38 int ret; 39 40 ret = pm_runtime_resume_and_get(pfdev->base.dev); 41 if (ret) 42 return ret; 43 44 panfrost_cycle_counter_get(pfdev); 45 *arg = panfrost_timestamp_read(pfdev); 46 panfrost_cycle_counter_put(pfdev); 47 48 pm_runtime_put(pfdev->base.dev); 49 return 0; 50 } 51 52 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 53 { 54 struct drm_panfrost_get_param *param = data; 55 struct panfrost_device *pfdev = to_panfrost_device(ddev); 56 int ret; 57 58 if (param->pad != 0) 59 return -EINVAL; 60 61 #define PANFROST_FEATURE(name, member) \ 62 case DRM_PANFROST_PARAM_ ## name: \ 63 param->value = pfdev->features.member; \ 64 break 65 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 66 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 67 DRM_PANFROST_PARAM_ ## name ## max: \ 68 param->value = pfdev->features.member[param->param - \ 69 DRM_PANFROST_PARAM_ ## name ## 0]; \ 70 break 71 72 switch (param->param) { 73 PANFROST_FEATURE(GPU_PROD_ID, id); 74 PANFROST_FEATURE(GPU_REVISION, revision); 75 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 76 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 77 PANFROST_FEATURE(L2_PRESENT, l2_present); 78 PANFROST_FEATURE(STACK_PRESENT, stack_present); 79 PANFROST_FEATURE(AS_PRESENT, as_present); 80 PANFROST_FEATURE(JS_PRESENT, js_present); 81 PANFROST_FEATURE(L2_FEATURES, l2_features); 82 PANFROST_FEATURE(CORE_FEATURES, core_features); 83 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 84 PANFROST_FEATURE(MEM_FEATURES, mem_features); 85 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 86 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 87 PANFROST_FEATURE(MAX_THREADS, max_threads); 88 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 89 thread_max_workgroup_sz); 90 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 91 thread_max_barrier_sz); 92 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 93 PANFROST_FEATURE(AFBC_FEATURES, afbc_features); 94 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 95 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 96 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 97 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 98 PANFROST_FEATURE(SELECTED_COHERENCY, selected_coherency); 99 100 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP: 101 ret = panfrost_ioctl_query_timestamp(pfdev, ¶m->value); 102 if (ret) 103 return ret; 104 break; 105 106 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY: 107 #ifdef CONFIG_ARM_ARCH_TIMER 108 param->value = arch_timer_get_cntfrq(); 109 #else 110 param->value = 0; 111 #endif 112 break; 113 114 case DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES: 115 param->value = BIT(PANFROST_JM_CTX_PRIORITY_LOW) | 116 BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM); 117 118 if (panfrost_high_prio_allowed(file)) 119 param->value |= BIT(PANFROST_JM_CTX_PRIORITY_HIGH); 120 break; 121 122 default: 123 return -EINVAL; 124 } 125 126 return 0; 127 } 128 129 #define PANFROST_BO_FLAGS (PANFROST_BO_NOEXEC | \ 130 PANFROST_BO_HEAP | \ 131 PANFROST_BO_WB_MMAP) 132 133 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 134 struct drm_file *file) 135 { 136 struct panfrost_file_priv *priv = file->driver_priv; 137 struct panfrost_gem_object *bo; 138 struct drm_panfrost_create_bo *args = data; 139 struct panfrost_gem_mapping *mapping; 140 int ret; 141 142 if (!args->size || args->pad || (args->flags & ~PANFROST_BO_FLAGS)) 143 return -EINVAL; 144 145 /* Heaps should never be executable */ 146 if ((args->flags & PANFROST_BO_HEAP) && 147 !(args->flags & PANFROST_BO_NOEXEC)) 148 return -EINVAL; 149 150 bo = panfrost_gem_create(dev, args->size, args->flags); 151 if (IS_ERR(bo)) 152 return PTR_ERR(bo); 153 154 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 155 if (ret) 156 goto out; 157 158 mapping = panfrost_gem_mapping_get(bo, priv); 159 if (mapping) { 160 args->offset = mapping->mmnode.start << PAGE_SHIFT; 161 panfrost_gem_mapping_put(mapping); 162 } else { 163 /* This can only happen if the handle from 164 * drm_gem_handle_create() has already been guessed and freed 165 * by user space 166 */ 167 ret = -EINVAL; 168 } 169 170 out: 171 drm_gem_object_put(&bo->base.base); 172 return ret; 173 } 174 175 /** 176 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 177 * referenced by the job. 178 * @dev: DRM device 179 * @file_priv: DRM file for this fd 180 * @args: IOCTL args 181 * @job: job being set up 182 * 183 * Resolve handles from userspace to BOs and attach them to job. 184 * 185 * Note that this function doesn't need to unreference the BOs on 186 * failure, because that will happen at panfrost_job_cleanup() time. 187 */ 188 static int 189 panfrost_lookup_bos(struct drm_device *dev, 190 struct drm_file *file_priv, 191 struct drm_panfrost_submit *args, 192 struct panfrost_job *job) 193 { 194 struct panfrost_file_priv *priv = file_priv->driver_priv; 195 struct panfrost_gem_object *bo; 196 unsigned int i; 197 int ret; 198 199 job->bo_count = args->bo_handle_count; 200 201 if (!job->bo_count) 202 return 0; 203 204 ret = drm_gem_objects_lookup(file_priv, 205 (void __user *)(uintptr_t)args->bo_handles, 206 job->bo_count, &job->bos); 207 if (ret) 208 return ret; 209 210 job->mappings = kvmalloc_objs(struct panfrost_gem_mapping *, 211 job->bo_count, GFP_KERNEL | __GFP_ZERO); 212 if (!job->mappings) 213 return -ENOMEM; 214 215 for (i = 0; i < job->bo_count; i++) { 216 struct panfrost_gem_mapping *mapping; 217 218 bo = to_panfrost_bo(job->bos[i]); 219 mapping = panfrost_gem_mapping_get(bo, priv); 220 if (!mapping) { 221 ret = -EINVAL; 222 break; 223 } 224 225 atomic_inc(&bo->gpu_usecount); 226 job->mappings[i] = mapping; 227 } 228 229 return ret; 230 } 231 232 /** 233 * panfrost_copy_in_sync() - Sets up job->deps with the sync objects 234 * referenced by the job. 235 * @dev: DRM device 236 * @file_priv: DRM file for this fd 237 * @args: IOCTL args 238 * @job: job being set up 239 * 240 * Resolve syncobjs from userspace to fences and attach them to job. 241 * 242 * Note that this function doesn't need to unreference the fences on 243 * failure, because that will happen at panfrost_job_cleanup() time. 244 */ 245 static int 246 panfrost_copy_in_sync(struct drm_device *dev, 247 struct drm_file *file_priv, 248 struct drm_panfrost_submit *args, 249 struct panfrost_job *job) 250 { 251 u32 *handles; 252 int ret = 0; 253 int i, in_fence_count; 254 255 in_fence_count = args->in_sync_count; 256 257 if (!in_fence_count) 258 return 0; 259 260 handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL); 261 if (!handles) { 262 ret = -ENOMEM; 263 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 264 goto fail; 265 } 266 267 if (copy_from_user(handles, 268 (void __user *)(uintptr_t)args->in_syncs, 269 in_fence_count * sizeof(u32))) { 270 ret = -EFAULT; 271 DRM_DEBUG("Failed to copy in syncobj handles\n"); 272 goto fail; 273 } 274 275 for (i = 0; i < in_fence_count; i++) { 276 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 277 handles[i], 0); 278 if (ret) 279 goto fail; 280 } 281 282 fail: 283 kvfree(handles); 284 return ret; 285 } 286 287 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 288 struct drm_file *file) 289 { 290 struct panfrost_device *pfdev = to_panfrost_device(dev); 291 struct panfrost_file_priv *file_priv = file->driver_priv; 292 struct drm_panfrost_submit *args = data; 293 struct drm_syncobj *sync_out = NULL; 294 struct panfrost_jm_ctx *jm_ctx; 295 struct panfrost_job *job; 296 int ret = 0, slot; 297 298 if (args->pad) 299 return -EINVAL; 300 301 if (!args->jc) 302 return -EINVAL; 303 304 if (args->requirements & ~JOB_REQUIREMENTS) 305 return -EINVAL; 306 307 if (args->out_sync > 0) { 308 sync_out = drm_syncobj_find(file, args->out_sync); 309 if (!sync_out) 310 return -ENODEV; 311 } 312 313 jm_ctx = panfrost_jm_ctx_from_handle(file, args->jm_ctx_handle); 314 if (!jm_ctx) { 315 ret = -EINVAL; 316 goto out_put_syncout; 317 } 318 319 job = kzalloc_obj(*job); 320 if (!job) { 321 ret = -ENOMEM; 322 goto out_put_jm_ctx; 323 } 324 325 kref_init(&job->refcount); 326 327 job->pfdev = pfdev; 328 job->jc = args->jc; 329 job->requirements = args->requirements; 330 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 331 job->mmu = file_priv->mmu; 332 job->ctx = panfrost_jm_ctx_get(jm_ctx); 333 job->engine_usage = &file_priv->engine_usage; 334 335 slot = panfrost_job_get_slot(job); 336 337 ret = drm_sched_job_init(&job->base, 338 &jm_ctx->slot_entity[slot], 339 1, NULL, file->client_id); 340 if (ret) 341 goto out_put_job; 342 343 ret = panfrost_copy_in_sync(dev, file, args, job); 344 if (ret) 345 goto out_cleanup_job; 346 347 ret = panfrost_lookup_bos(dev, file, args, job); 348 if (ret) 349 goto out_cleanup_job; 350 351 ret = panfrost_job_push(job); 352 if (ret) 353 goto out_cleanup_job; 354 355 /* Update the return sync object for the job */ 356 if (sync_out) 357 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 358 359 out_cleanup_job: 360 if (ret) 361 drm_sched_job_cleanup(&job->base); 362 out_put_job: 363 panfrost_job_put(job); 364 out_put_jm_ctx: 365 panfrost_jm_ctx_put(jm_ctx); 366 out_put_syncout: 367 if (sync_out) 368 drm_syncobj_put(sync_out); 369 370 return ret; 371 } 372 373 static int 374 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 375 struct drm_file *file_priv) 376 { 377 long ret; 378 struct drm_panfrost_wait_bo *args = data; 379 struct drm_gem_object *gem_obj; 380 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 381 382 if (args->pad) 383 return -EINVAL; 384 385 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 386 if (!gem_obj) 387 return -ENOENT; 388 389 ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ, 390 true, timeout); 391 if (!ret) 392 ret = timeout ? -ETIMEDOUT : -EBUSY; 393 else if (ret > 0) 394 ret = 0; 395 396 drm_gem_object_put(gem_obj); 397 398 return ret; 399 } 400 401 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 402 struct drm_file *file_priv) 403 { 404 struct drm_panfrost_mmap_bo *args = data; 405 struct drm_gem_object *gem_obj; 406 int ret; 407 408 if (args->flags != 0) { 409 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 410 return -EINVAL; 411 } 412 413 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 414 if (!gem_obj) { 415 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 416 return -ENOENT; 417 } 418 419 /* Don't allow mmapping of heap objects as pages are not pinned. */ 420 if (to_panfrost_bo(gem_obj)->is_heap) { 421 ret = -EINVAL; 422 goto out; 423 } 424 425 ret = drm_gem_create_mmap_offset(gem_obj); 426 if (ret == 0) 427 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 428 429 out: 430 drm_gem_object_put(gem_obj); 431 return ret; 432 } 433 434 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 435 struct drm_file *file_priv) 436 { 437 struct panfrost_file_priv *priv = file_priv->driver_priv; 438 struct drm_panfrost_get_bo_offset *args = data; 439 struct panfrost_gem_mapping *mapping; 440 struct drm_gem_object *gem_obj; 441 struct panfrost_gem_object *bo; 442 443 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 444 if (!gem_obj) { 445 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 446 return -ENOENT; 447 } 448 bo = to_panfrost_bo(gem_obj); 449 450 mapping = panfrost_gem_mapping_get(bo, priv); 451 drm_gem_object_put(gem_obj); 452 453 if (!mapping) 454 return -EINVAL; 455 456 args->offset = mapping->mmnode.start << PAGE_SHIFT; 457 panfrost_gem_mapping_put(mapping); 458 return 0; 459 } 460 461 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, 462 struct drm_file *file_priv) 463 { 464 struct panfrost_file_priv *priv = file_priv->driver_priv; 465 struct drm_panfrost_madvise *args = data; 466 struct panfrost_device *pfdev = to_panfrost_device(dev); 467 struct drm_gem_object *gem_obj; 468 struct panfrost_gem_object *bo; 469 int ret = 0; 470 471 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 472 if (!gem_obj) { 473 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 474 return -ENOENT; 475 } 476 477 bo = to_panfrost_bo(gem_obj); 478 479 ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); 480 if (ret) 481 goto out_put_object; 482 483 mutex_lock(&pfdev->shrinker_lock); 484 mutex_lock(&bo->mappings.lock); 485 if (args->madv == PANFROST_MADV_DONTNEED) { 486 struct panfrost_gem_mapping *first; 487 488 first = list_first_entry(&bo->mappings.list, 489 struct panfrost_gem_mapping, 490 node); 491 492 /* 493 * If we want to mark the BO purgeable, there must be only one 494 * user: the caller FD. 495 * We could do something smarter and mark the BO purgeable only 496 * when all its users have marked it purgeable, but globally 497 * visible/shared BOs are likely to never be marked purgeable 498 * anyway, so let's not bother. 499 */ 500 if (!list_is_singular(&bo->mappings.list) || 501 WARN_ON_ONCE(first->mmu != priv->mmu)) { 502 ret = -EINVAL; 503 goto out_unlock_mappings; 504 } 505 } 506 507 args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv); 508 509 if (args->retained) { 510 if (args->madv == PANFROST_MADV_DONTNEED) 511 list_move_tail(&bo->base.madv_list, 512 &pfdev->shrinker_list); 513 else if (args->madv == PANFROST_MADV_WILLNEED) 514 list_del_init(&bo->base.madv_list); 515 } 516 517 out_unlock_mappings: 518 mutex_unlock(&bo->mappings.lock); 519 mutex_unlock(&pfdev->shrinker_lock); 520 dma_resv_unlock(bo->base.base.resv); 521 out_put_object: 522 drm_gem_object_put(gem_obj); 523 return ret; 524 } 525 526 static int panfrost_ioctl_set_label_bo(struct drm_device *ddev, void *data, 527 struct drm_file *file) 528 { 529 struct drm_panfrost_set_label_bo *args = data; 530 struct drm_gem_object *obj; 531 const char *label = NULL; 532 int ret = 0; 533 534 if (args->pad) 535 return -EINVAL; 536 537 obj = drm_gem_object_lookup(file, args->handle); 538 if (!obj) 539 return -ENOENT; 540 541 if (args->label) { 542 label = strndup_user(u64_to_user_ptr(args->label), 543 PANFROST_BO_LABEL_MAXLEN); 544 if (IS_ERR(label)) { 545 ret = PTR_ERR(label); 546 if (ret == -EINVAL) 547 ret = -E2BIG; 548 goto err_put_obj; 549 } 550 } 551 552 /* 553 * We treat passing a label of length 0 and passing a NULL label 554 * differently, because even though they might seem conceptually 555 * similar, future uses of the BO label might expect a different 556 * behaviour in each case. 557 */ 558 panfrost_gem_set_label(obj, label); 559 560 err_put_obj: 561 drm_gem_object_put(obj); 562 563 return ret; 564 } 565 566 static int panfrost_ioctl_jm_ctx_create(struct drm_device *dev, void *data, 567 struct drm_file *file) 568 { 569 return panfrost_jm_ctx_create(file, data); 570 } 571 572 static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data, 573 struct drm_file *file) 574 { 575 const struct drm_panfrost_jm_ctx_destroy *args = data; 576 577 if (args->pad) 578 return -EINVAL; 579 580 /* We can't destroy the default context created when the file is opened. */ 581 if (!args->handle) 582 return -EINVAL; 583 584 return panfrost_jm_ctx_destroy(file, args->handle); 585 } 586 587 static int panfrost_ioctl_sync_bo(struct drm_device *ddev, void *data, 588 struct drm_file *file) 589 { 590 struct drm_panfrost_sync_bo *args = data; 591 struct drm_panfrost_bo_sync_op *ops; 592 struct drm_gem_object *obj; 593 int ret; 594 u32 i; 595 596 if (args->pad) 597 return -EINVAL; 598 599 if (!args->op_count) 600 return 0; 601 602 ops = kvmalloc_objs(*ops, args->op_count); 603 if (!ops) { 604 DRM_DEBUG("Failed to allocate incoming BO sync ops array\n"); 605 return -ENOMEM; 606 } 607 608 if (copy_from_user(ops, (void __user *)(uintptr_t)args->ops, 609 args->op_count * sizeof(*ops))) { 610 DRM_DEBUG("Failed to copy in BO sync ops\n"); 611 ret = -EFAULT; 612 goto err_ops; 613 } 614 615 for (i = 0; i < args->op_count; i++) { 616 obj = drm_gem_object_lookup(file, ops[i].handle); 617 if (!obj) { 618 ret = -ENOENT; 619 goto err_ops; 620 } 621 622 ret = panfrost_gem_sync(obj, ops[i].type, 623 ops[i].offset, ops[i].size); 624 625 drm_gem_object_put(obj); 626 627 if (ret) 628 goto err_ops; 629 } 630 631 err_ops: 632 kvfree(ops); 633 634 return ret; 635 } 636 637 static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data, 638 struct drm_file *file_priv) 639 { 640 struct drm_panfrost_query_bo_info *args = data; 641 struct drm_gem_object *gem_obj; 642 struct panfrost_gem_object *bo; 643 644 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 645 if (!gem_obj) { 646 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 647 return -ENOENT; 648 } 649 650 bo = to_panfrost_bo(gem_obj); 651 args->pad = 0; 652 args->create_flags = 0; 653 args->extra_flags = 0; 654 655 if (drm_gem_is_imported(gem_obj)) { 656 args->extra_flags |= DRM_PANFROST_BO_IS_IMPORTED; 657 } else { 658 if (bo->noexec) 659 args->create_flags |= PANFROST_BO_NOEXEC; 660 661 if (bo->is_heap) 662 args->create_flags |= PANFROST_BO_HEAP; 663 664 if (!bo->base.map_wc) 665 args->create_flags |= PANFROST_BO_WB_MMAP; 666 } 667 668 drm_gem_object_put(gem_obj); 669 return 0; 670 } 671 672 int panfrost_unstable_ioctl_check(void) 673 { 674 if (!unstable_ioctls) 675 return -ENOSYS; 676 677 return 0; 678 } 679 680 static int 681 panfrost_open(struct drm_device *dev, struct drm_file *file) 682 { 683 int ret; 684 struct panfrost_device *pfdev = to_panfrost_device(dev); 685 struct panfrost_file_priv *panfrost_priv; 686 687 panfrost_priv = kzalloc_obj(*panfrost_priv); 688 if (!panfrost_priv) 689 return -ENOMEM; 690 691 panfrost_priv->pfdev = pfdev; 692 file->driver_priv = panfrost_priv; 693 694 panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev); 695 if (IS_ERR(panfrost_priv->mmu)) { 696 ret = PTR_ERR(panfrost_priv->mmu); 697 goto err_free; 698 } 699 700 ret = panfrost_jm_open(file); 701 if (ret) 702 goto err_job; 703 704 return 0; 705 706 err_job: 707 panfrost_mmu_ctx_put(panfrost_priv->mmu); 708 err_free: 709 kfree(panfrost_priv); 710 return ret; 711 } 712 713 static void 714 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 715 { 716 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 717 718 panfrost_perfcnt_close(file); 719 panfrost_jm_close(file); 720 721 panfrost_mmu_ctx_put(panfrost_priv->mmu); 722 kfree(panfrost_priv); 723 } 724 725 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 726 #define PANFROST_IOCTL(n, func, flags) \ 727 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 728 729 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW), 730 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 731 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 732 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 733 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 734 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 735 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 736 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 737 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), 738 PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW), 739 PANFROST_IOCTL(JM_CTX_CREATE, jm_ctx_create, DRM_RENDER_ALLOW), 740 PANFROST_IOCTL(JM_CTX_DESTROY, jm_ctx_destroy, DRM_RENDER_ALLOW), 741 PANFROST_IOCTL(SYNC_BO, sync_bo, DRM_RENDER_ALLOW), 742 PANFROST_IOCTL(QUERY_BO_INFO, query_bo_info, DRM_RENDER_ALLOW), 743 }; 744 745 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, 746 struct panfrost_file_priv *panfrost_priv, 747 struct drm_printer *p) 748 { 749 int i; 750 751 /* 752 * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not 753 * accurate, as they only provide a rough estimation of the number of 754 * GPU cycles and CPU time spent in a given context. This is due to two 755 * different factors: 756 * - Firstly, we must consider the time the CPU and then the kernel 757 * takes to process the GPU interrupt, which means additional time and 758 * GPU cycles will be added in excess to the real figure. 759 * - Secondly, the pipelining done by the Job Manager (2 job slots per 760 * engine) implies there is no way to know exactly how much time each 761 * job spent on the GPU. 762 */ 763 764 for (i = 0; i < NUM_JOB_SLOTS - 1; i++) { 765 if (pfdev->profile_mode) { 766 drm_printf(p, "drm-engine-%s:\t%llu ns\n", 767 panfrost_engine_names[i], 768 panfrost_priv->engine_usage.elapsed_ns[i]); 769 drm_printf(p, "drm-cycles-%s:\t%llu\n", 770 panfrost_engine_names[i], 771 panfrost_priv->engine_usage.cycles[i]); 772 } 773 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n", 774 panfrost_engine_names[i], pfdev->pfdevfreq.fast_rate); 775 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n", 776 panfrost_engine_names[i], pfdev->pfdevfreq.current_frequency); 777 } 778 } 779 780 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file) 781 { 782 struct panfrost_device *pfdev = to_panfrost_device(file->minor->dev); 783 784 panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p); 785 786 drm_show_memory_stats(p, file); 787 } 788 789 static const struct file_operations panfrost_drm_driver_fops = { 790 .owner = THIS_MODULE, 791 DRM_GEM_FOPS, 792 .show_fdinfo = drm_show_fdinfo, 793 }; 794 795 #ifdef CONFIG_DEBUG_FS 796 static int panthor_gems_show(struct seq_file *m, void *data) 797 { 798 struct drm_info_node *node = m->private; 799 struct panfrost_device *pfdev = to_panfrost_device(node->minor->dev); 800 801 panfrost_gem_debugfs_print_bos(pfdev, m); 802 803 return 0; 804 } 805 806 static void show_panfrost_jm_ctx(struct panfrost_jm_ctx *jm_ctx, u32 handle, 807 struct seq_file *m) 808 { 809 struct drm_device *ddev = ((struct drm_info_node *)m->private)->minor->dev; 810 const char *prio = "UNKNOWN"; 811 812 static const char * const prios[] = { 813 [DRM_SCHED_PRIORITY_HIGH] = "HIGH", 814 [DRM_SCHED_PRIORITY_NORMAL] = "NORMAL", 815 [DRM_SCHED_PRIORITY_LOW] = "LOW", 816 }; 817 818 if (jm_ctx->slot_entity[0].priority != 819 jm_ctx->slot_entity[1].priority) 820 drm_warn(ddev, "Slot priorities should be the same in a single context"); 821 822 if (jm_ctx->slot_entity[0].priority < ARRAY_SIZE(prios)) 823 prio = prios[jm_ctx->slot_entity[0].priority]; 824 825 seq_printf(m, " JM context %u: priority %s\n", handle, prio); 826 } 827 828 static int show_file_jm_ctxs(struct panfrost_file_priv *pfile, 829 struct seq_file *m) 830 { 831 struct panfrost_jm_ctx *jm_ctx; 832 unsigned long i; 833 834 xa_lock(&pfile->jm_ctxs); 835 xa_for_each(&pfile->jm_ctxs, i, jm_ctx) { 836 jm_ctx = panfrost_jm_ctx_get(jm_ctx); 837 xa_unlock(&pfile->jm_ctxs); 838 show_panfrost_jm_ctx(jm_ctx, i, m); 839 panfrost_jm_ctx_put(jm_ctx); 840 xa_lock(&pfile->jm_ctxs); 841 } 842 xa_unlock(&pfile->jm_ctxs); 843 844 return 0; 845 } 846 847 static struct drm_info_list panthor_debugfs_list[] = { 848 {"gems", 849 panthor_gems_show, 0, NULL}, 850 }; 851 852 static int panthor_gems_debugfs_init(struct drm_minor *minor) 853 { 854 drm_debugfs_create_files(panthor_debugfs_list, 855 ARRAY_SIZE(panthor_debugfs_list), 856 minor->debugfs_root, minor); 857 858 return 0; 859 } 860 861 static int show_each_file(struct seq_file *m, void *arg) 862 { 863 struct drm_info_node *node = (struct drm_info_node *)m->private; 864 struct drm_device *ddev = node->minor->dev; 865 int (*show)(struct panfrost_file_priv *, struct seq_file *) = 866 node->info_ent->data; 867 struct drm_file *file; 868 int ret; 869 870 ret = mutex_lock_interruptible(&ddev->filelist_mutex); 871 if (ret) 872 return ret; 873 874 list_for_each_entry(file, &ddev->filelist, lhead) { 875 struct task_struct *task; 876 struct panfrost_file_priv *pfile = file->driver_priv; 877 struct pid *pid; 878 879 /* 880 * Although we have a valid reference on file->pid, that does 881 * not guarantee that the task_struct who called get_pid() is 882 * still alive (e.g. get_pid(current) => fork() => exit()). 883 * Therefore, we need to protect this ->comm access using RCU. 884 */ 885 rcu_read_lock(); 886 pid = rcu_dereference(file->pid); 887 task = pid_task(pid, PIDTYPE_TGID); 888 seq_printf(m, "client_id %8llu pid %8d command %s:\n", 889 file->client_id, pid_nr(pid), 890 task ? task->comm : "<unknown>"); 891 rcu_read_unlock(); 892 893 ret = show(pfile, m); 894 if (ret < 0) 895 break; 896 897 seq_puts(m, "\n"); 898 } 899 900 mutex_unlock(&ddev->filelist_mutex); 901 return ret; 902 } 903 904 static struct drm_info_list panfrost_sched_debugfs_list[] = { 905 { "sched_ctxs", show_each_file, 0, show_file_jm_ctxs }, 906 }; 907 908 static void panfrost_sched_debugfs_init(struct drm_minor *minor) 909 { 910 drm_debugfs_create_files(panfrost_sched_debugfs_list, 911 ARRAY_SIZE(panfrost_sched_debugfs_list), 912 minor->debugfs_root, minor); 913 } 914 915 static void panfrost_debugfs_init(struct drm_minor *minor) 916 { 917 panthor_gems_debugfs_init(minor); 918 panfrost_sched_debugfs_init(minor); 919 } 920 #endif 921 922 /* 923 * Panfrost driver version: 924 * - 1.0 - initial interface 925 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO 926 * - 1.2 - adds AFBC_FEATURES query 927 * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT 928 * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries 929 * - 1.4 - adds SET_LABEL_BO 930 * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow 931 * context creation with configurable priorities/affinity 932 * - 1.6 - adds PANFROST_BO_MAP_WB, PANFROST_IOCTL_SYNC_BO, 933 * PANFROST_IOCTL_QUERY_BO_INFO and 934 * DRM_PANFROST_PARAM_SELECTED_COHERENCY 935 */ 936 static const struct drm_driver panfrost_drm_driver = { 937 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 938 .open = panfrost_open, 939 .postclose = panfrost_postclose, 940 .show_fdinfo = panfrost_show_fdinfo, 941 .ioctls = panfrost_drm_driver_ioctls, 942 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 943 .fops = &panfrost_drm_driver_fops, 944 .name = "panfrost", 945 .desc = "panfrost DRM", 946 .major = 1, 947 .minor = 6, 948 949 .gem_create_object = panfrost_gem_create_object, 950 .gem_prime_import = panfrost_gem_prime_import, 951 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 952 #ifdef CONFIG_DEBUG_FS 953 .debugfs_init = panfrost_debugfs_init, 954 #endif 955 }; 956 957 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 958 bool panfrost_transparent_hugepage = true; 959 module_param_named(transparent_hugepage, panfrost_transparent_hugepage, bool, 0400); 960 MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)"); 961 #endif 962 963 static int panfrost_probe(struct platform_device *pdev) 964 { 965 struct panfrost_device *pfdev; 966 int err; 967 968 pfdev = devm_drm_dev_alloc(&pdev->dev, &panfrost_drm_driver, 969 struct panfrost_device, base); 970 if (IS_ERR(pfdev)) 971 return PTR_ERR(pfdev); 972 973 platform_set_drvdata(pdev, pfdev); 974 975 pfdev->comp = of_device_get_match_data(&pdev->dev); 976 if (!pfdev->comp) 977 return -ENODEV; 978 979 pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT; 980 981 mutex_init(&pfdev->shrinker_lock); 982 INIT_LIST_HEAD(&pfdev->shrinker_list); 983 984 err = panfrost_device_init(pfdev); 985 if (err) { 986 if (err != -EPROBE_DEFER) 987 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 988 goto err_out0; 989 } 990 991 pm_runtime_set_active(pfdev->base.dev); 992 pm_runtime_mark_last_busy(pfdev->base.dev); 993 pm_runtime_enable(pfdev->base.dev); 994 pm_runtime_set_autosuspend_delay(pfdev->base.dev, 50); /* ~3 frames */ 995 pm_runtime_use_autosuspend(pfdev->base.dev); 996 997 /* 998 * Register the DRM device with the core and the connectors with 999 * sysfs 1000 */ 1001 err = drm_dev_register(&pfdev->base, 0); 1002 if (err < 0) 1003 goto err_out1; 1004 1005 err = panfrost_gem_shrinker_init(&pfdev->base); 1006 if (err) 1007 goto err_out2; 1008 1009 return 0; 1010 1011 err_out2: 1012 drm_dev_unregister(&pfdev->base); 1013 err_out1: 1014 pm_runtime_disable(pfdev->base.dev); 1015 panfrost_device_fini(pfdev); 1016 pm_runtime_set_suspended(pfdev->base.dev); 1017 err_out0: 1018 return err; 1019 } 1020 1021 static void panfrost_remove(struct platform_device *pdev) 1022 { 1023 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 1024 1025 drm_dev_unregister(&pfdev->base); 1026 panfrost_gem_shrinker_cleanup(&pfdev->base); 1027 1028 pm_runtime_get_sync(pfdev->base.dev); 1029 pm_runtime_disable(pfdev->base.dev); 1030 panfrost_device_fini(pfdev); 1031 pm_runtime_set_suspended(pfdev->base.dev); 1032 } 1033 1034 static ssize_t profiling_show(struct device *dev, 1035 struct device_attribute *attr, char *buf) 1036 { 1037 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1038 1039 return sysfs_emit(buf, "%d\n", pfdev->profile_mode); 1040 } 1041 1042 static ssize_t profiling_store(struct device *dev, 1043 struct device_attribute *attr, 1044 const char *buf, size_t len) 1045 { 1046 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1047 bool value; 1048 int err; 1049 1050 err = kstrtobool(buf, &value); 1051 if (err) 1052 return err; 1053 1054 pfdev->profile_mode = value; 1055 1056 return len; 1057 } 1058 1059 static DEVICE_ATTR_RW(profiling); 1060 1061 static struct attribute *panfrost_attrs[] = { 1062 &dev_attr_profiling.attr, 1063 NULL, 1064 }; 1065 1066 ATTRIBUTE_GROUPS(panfrost); 1067 1068 /* 1069 * The OPP core wants the supply names to be NULL terminated, but we need the 1070 * correct num_supplies value for regulator core. Hence, we NULL terminate here 1071 * and then initialize num_supplies with ARRAY_SIZE - 1. 1072 */ 1073 static const char * const default_supplies[] = { "mali", NULL }; 1074 static const struct panfrost_compatible default_data = { 1075 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1076 .supply_names = default_supplies, 1077 .num_pm_domains = 1, /* optional */ 1078 .pm_domain_names = NULL, 1079 }; 1080 1081 static const struct panfrost_compatible default_pm_rt_data = { 1082 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1083 .supply_names = default_supplies, 1084 .num_pm_domains = 1, 1085 .pm_features = BIT(GPU_PM_RT), 1086 }; 1087 1088 static const struct panfrost_compatible amlogic_data = { 1089 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1090 .supply_names = default_supplies, 1091 .vendor_quirk = panfrost_gpu_amlogic_quirk, 1092 }; 1093 1094 static const char * const mediatek_pm_domains[] = { "core0", "core1", "core2", 1095 "core3", "core4" }; 1096 /* 1097 * The old data with two power supplies for MT8183 is here only to 1098 * keep retro-compatibility with older devicetrees, as DVFS will 1099 * not work with this one. 1100 * 1101 * On new devicetrees please use the _b variant with a single and 1102 * coupled regulators instead. 1103 */ 1104 static const char * const legacy_supplies[] = { "mali", "sram", NULL }; 1105 static const struct panfrost_compatible mediatek_mt8183_data = { 1106 .num_supplies = ARRAY_SIZE(legacy_supplies) - 1, 1107 .supply_names = legacy_supplies, 1108 .num_pm_domains = 3, 1109 .pm_domain_names = mediatek_pm_domains, 1110 }; 1111 1112 static const struct panfrost_compatible mediatek_mt8183_b_data = { 1113 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1114 .supply_names = default_supplies, 1115 .num_pm_domains = 3, 1116 .pm_domain_names = mediatek_pm_domains, 1117 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1118 }; 1119 1120 static const struct panfrost_compatible mediatek_mt8186_data = { 1121 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1122 .supply_names = default_supplies, 1123 .num_pm_domains = 2, 1124 .pm_domain_names = mediatek_pm_domains, 1125 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1126 }; 1127 1128 static const struct panfrost_compatible mediatek_mt8188_data = { 1129 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1130 .supply_names = default_supplies, 1131 .num_pm_domains = 3, 1132 .pm_domain_names = mediatek_pm_domains, 1133 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1134 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1135 }; 1136 1137 static const struct panfrost_compatible mediatek_mt8192_data = { 1138 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1139 .supply_names = default_supplies, 1140 .num_pm_domains = 5, 1141 .pm_domain_names = mediatek_pm_domains, 1142 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1143 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1144 }; 1145 1146 static const struct panfrost_compatible mediatek_mt8370_data = { 1147 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1148 .supply_names = default_supplies, 1149 .num_pm_domains = 2, 1150 .pm_domain_names = mediatek_pm_domains, 1151 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1152 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1153 }; 1154 1155 static const struct of_device_id dt_match[] = { 1156 /* Set first to probe before the generic compatibles */ 1157 { .compatible = "amlogic,meson-gxm-mali", 1158 .data = &amlogic_data, }, 1159 { .compatible = "amlogic,meson-g12a-mali", 1160 .data = &amlogic_data, }, 1161 { .compatible = "renesas,r9a09g047-mali", .data = &default_pm_rt_data }, 1162 { .compatible = "arm,mali-t604", .data = &default_data, }, 1163 { .compatible = "arm,mali-t624", .data = &default_data, }, 1164 { .compatible = "arm,mali-t628", .data = &default_data, }, 1165 { .compatible = "arm,mali-t720", .data = &default_data, }, 1166 { .compatible = "arm,mali-t760", .data = &default_data, }, 1167 { .compatible = "arm,mali-t820", .data = &default_data, }, 1168 { .compatible = "arm,mali-t830", .data = &default_data, }, 1169 { .compatible = "arm,mali-t860", .data = &default_data, }, 1170 { .compatible = "arm,mali-t880", .data = &default_data, }, 1171 { .compatible = "arm,mali-bifrost", .data = &default_data, }, 1172 { .compatible = "arm,mali-valhall-jm", .data = &default_data, }, 1173 { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data }, 1174 { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data }, 1175 { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data }, 1176 { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data }, 1177 { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data }, 1178 { .compatible = "mediatek,mt8370-mali", .data = &mediatek_mt8370_data }, 1179 { .compatible = "allwinner,sun50i-h616-mali", .data = &default_pm_rt_data }, 1180 {} 1181 }; 1182 MODULE_DEVICE_TABLE(of, dt_match); 1183 1184 static struct platform_driver panfrost_driver = { 1185 .probe = panfrost_probe, 1186 .remove = panfrost_remove, 1187 .driver = { 1188 .name = "panfrost", 1189 .pm = pm_ptr(&panfrost_pm_ops), 1190 .of_match_table = dt_match, 1191 .dev_groups = panfrost_groups, 1192 }, 1193 }; 1194 module_platform_driver(panfrost_driver); 1195 1196 MODULE_AUTHOR("Panfrost Project Developers"); 1197 MODULE_DESCRIPTION("Panfrost DRM Driver"); 1198 MODULE_LICENSE("GPL v2"); 1199 MODULE_SOFTDEP("pre: governor_simpleondemand"); 1200