1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #ifdef CONFIG_ARM_ARCH_TIMER 7 #include <asm/arch_timer.h> 8 #endif 9 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/pagemap.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <drm/panfrost_drm.h> 16 #include <drm/drm_debugfs.h> 17 #include <drm/drm_drv.h> 18 #include <drm/drm_ioctl.h> 19 #include <drm/drm_print.h> 20 #include <drm/drm_syncobj.h> 21 #include <drm/drm_utils.h> 22 23 #include "panfrost_device.h" 24 #include "panfrost_gem.h" 25 #include "panfrost_mmu.h" 26 #include "panfrost_job.h" 27 #include "panfrost_gpu.h" 28 #include "panfrost_perfcnt.h" 29 30 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT) 31 32 static bool unstable_ioctls; 33 module_param_unsafe(unstable_ioctls, bool, 0600); 34 35 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, 36 u64 *arg) 37 { 38 int ret; 39 40 ret = pm_runtime_resume_and_get(pfdev->base.dev); 41 if (ret) 42 return ret; 43 44 panfrost_cycle_counter_get(pfdev); 45 *arg = panfrost_timestamp_read(pfdev); 46 panfrost_cycle_counter_put(pfdev); 47 48 pm_runtime_put(pfdev->base.dev); 49 return 0; 50 } 51 52 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 53 { 54 struct drm_panfrost_get_param *param = data; 55 struct panfrost_device *pfdev = to_panfrost_device(ddev); 56 int ret; 57 58 if (param->pad != 0) 59 return -EINVAL; 60 61 #define PANFROST_FEATURE(name, member) \ 62 case DRM_PANFROST_PARAM_ ## name: \ 63 param->value = pfdev->features.member; \ 64 break 65 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 66 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 67 DRM_PANFROST_PARAM_ ## name ## max: \ 68 param->value = pfdev->features.member[param->param - \ 69 DRM_PANFROST_PARAM_ ## name ## 0]; \ 70 break 71 72 switch (param->param) { 73 PANFROST_FEATURE(GPU_PROD_ID, id); 74 PANFROST_FEATURE(GPU_REVISION, revision); 75 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 76 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 77 PANFROST_FEATURE(L2_PRESENT, l2_present); 78 PANFROST_FEATURE(STACK_PRESENT, stack_present); 79 PANFROST_FEATURE(AS_PRESENT, as_present); 80 PANFROST_FEATURE(JS_PRESENT, js_present); 81 PANFROST_FEATURE(L2_FEATURES, l2_features); 82 PANFROST_FEATURE(CORE_FEATURES, core_features); 83 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 84 PANFROST_FEATURE(MEM_FEATURES, mem_features); 85 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 86 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 87 PANFROST_FEATURE(MAX_THREADS, max_threads); 88 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 89 thread_max_workgroup_sz); 90 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 91 thread_max_barrier_sz); 92 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 93 PANFROST_FEATURE(AFBC_FEATURES, afbc_features); 94 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 95 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 96 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 97 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 98 PANFROST_FEATURE(SELECTED_COHERENCY, selected_coherency); 99 100 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP: 101 ret = panfrost_ioctl_query_timestamp(pfdev, ¶m->value); 102 if (ret) 103 return ret; 104 break; 105 106 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY: 107 #ifdef CONFIG_ARM_ARCH_TIMER 108 param->value = arch_timer_get_cntfrq(); 109 #else 110 param->value = 0; 111 #endif 112 break; 113 114 case DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES: 115 param->value = BIT(PANFROST_JM_CTX_PRIORITY_LOW) | 116 BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM); 117 118 if (panfrost_high_prio_allowed(file)) 119 param->value |= BIT(PANFROST_JM_CTX_PRIORITY_HIGH); 120 break; 121 122 default: 123 return -EINVAL; 124 } 125 126 return 0; 127 } 128 129 #define PANFROST_BO_FLAGS (PANFROST_BO_NOEXEC | \ 130 PANFROST_BO_HEAP | \ 131 PANFROST_BO_WB_MMAP) 132 133 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 134 struct drm_file *file) 135 { 136 struct panfrost_file_priv *priv = file->driver_priv; 137 struct panfrost_gem_object *bo; 138 struct drm_panfrost_create_bo *args = data; 139 struct panfrost_gem_mapping *mapping; 140 int ret; 141 142 if (!args->size || args->pad || (args->flags & ~PANFROST_BO_FLAGS)) 143 return -EINVAL; 144 145 /* Heaps should never be executable */ 146 if ((args->flags & PANFROST_BO_HEAP) && 147 !(args->flags & PANFROST_BO_NOEXEC)) 148 return -EINVAL; 149 150 bo = panfrost_gem_create(dev, args->size, args->flags); 151 if (IS_ERR(bo)) 152 return PTR_ERR(bo); 153 154 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 155 if (ret) 156 goto out; 157 158 mapping = panfrost_gem_mapping_get(bo, priv); 159 if (mapping) { 160 args->offset = mapping->mmnode.start << PAGE_SHIFT; 161 panfrost_gem_mapping_put(mapping); 162 } else { 163 /* This can only happen if the handle from 164 * drm_gem_handle_create() has already been guessed and freed 165 * by user space 166 */ 167 ret = -EINVAL; 168 } 169 170 out: 171 drm_gem_object_put(&bo->base.base); 172 return ret; 173 } 174 175 /** 176 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 177 * referenced by the job. 178 * @dev: DRM device 179 * @file_priv: DRM file for this fd 180 * @args: IOCTL args 181 * @job: job being set up 182 * 183 * Resolve handles from userspace to BOs and attach them to job. 184 * 185 * Note that this function doesn't need to unreference the BOs on 186 * failure, because that will happen at panfrost_job_cleanup() time. 187 */ 188 static int 189 panfrost_lookup_bos(struct drm_device *dev, 190 struct drm_file *file_priv, 191 struct drm_panfrost_submit *args, 192 struct panfrost_job *job) 193 { 194 struct panfrost_file_priv *priv = file_priv->driver_priv; 195 struct panfrost_gem_object *bo; 196 unsigned int i; 197 int ret; 198 199 job->bo_count = args->bo_handle_count; 200 201 if (!job->bo_count) 202 return 0; 203 204 ret = drm_gem_objects_lookup(file_priv, 205 (void __user *)(uintptr_t)args->bo_handles, 206 job->bo_count, &job->bos); 207 if (ret) 208 return ret; 209 210 job->mappings = kvmalloc_objs(struct panfrost_gem_mapping *, 211 job->bo_count, GFP_KERNEL | __GFP_ZERO); 212 if (!job->mappings) 213 return -ENOMEM; 214 215 for (i = 0; i < job->bo_count; i++) { 216 struct panfrost_gem_mapping *mapping; 217 218 bo = to_panfrost_bo(job->bos[i]); 219 mapping = panfrost_gem_mapping_get(bo, priv); 220 if (!mapping) { 221 ret = -EINVAL; 222 break; 223 } 224 225 atomic_inc(&bo->gpu_usecount); 226 job->mappings[i] = mapping; 227 } 228 229 return ret; 230 } 231 232 /** 233 * panfrost_copy_in_sync() - Sets up job->deps with the sync objects 234 * referenced by the job. 235 * @dev: DRM device 236 * @file_priv: DRM file for this fd 237 * @args: IOCTL args 238 * @job: job being set up 239 * 240 * Resolve syncobjs from userspace to fences and attach them to job. 241 * 242 * Note that this function doesn't need to unreference the fences on 243 * failure, because that will happen at panfrost_job_cleanup() time. 244 */ 245 static int 246 panfrost_copy_in_sync(struct drm_device *dev, 247 struct drm_file *file_priv, 248 struct drm_panfrost_submit *args, 249 struct panfrost_job *job) 250 { 251 u32 *handles; 252 int ret = 0; 253 int i, in_fence_count; 254 255 in_fence_count = args->in_sync_count; 256 257 if (!in_fence_count) 258 return 0; 259 260 handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL); 261 if (!handles) { 262 ret = -ENOMEM; 263 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 264 goto fail; 265 } 266 267 if (copy_from_user(handles, 268 (void __user *)(uintptr_t)args->in_syncs, 269 in_fence_count * sizeof(u32))) { 270 ret = -EFAULT; 271 DRM_DEBUG("Failed to copy in syncobj handles\n"); 272 goto fail; 273 } 274 275 for (i = 0; i < in_fence_count; i++) { 276 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 277 handles[i], 0); 278 if (ret) 279 goto fail; 280 } 281 282 fail: 283 kvfree(handles); 284 return ret; 285 } 286 287 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 288 struct drm_file *file) 289 { 290 struct panfrost_device *pfdev = to_panfrost_device(dev); 291 struct panfrost_file_priv *file_priv = file->driver_priv; 292 struct drm_panfrost_submit *args = data; 293 struct drm_syncobj *sync_out = NULL; 294 struct panfrost_jm_ctx *jm_ctx; 295 struct panfrost_job *job; 296 int ret = 0, slot; 297 298 if (args->pad) 299 return -EINVAL; 300 301 if (!args->jc) 302 return -EINVAL; 303 304 if (args->requirements & ~JOB_REQUIREMENTS) 305 return -EINVAL; 306 307 if (args->out_sync > 0) { 308 sync_out = drm_syncobj_find(file, args->out_sync); 309 if (!sync_out) 310 return -ENODEV; 311 } 312 313 jm_ctx = panfrost_jm_ctx_from_handle(file, args->jm_ctx_handle); 314 if (!jm_ctx) { 315 ret = -EINVAL; 316 goto out_put_syncout; 317 } 318 319 job = kzalloc_obj(*job); 320 if (!job) { 321 ret = -ENOMEM; 322 goto out_put_jm_ctx; 323 } 324 325 kref_init(&job->refcount); 326 327 job->pfdev = pfdev; 328 job->jc = args->jc; 329 job->requirements = args->requirements; 330 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 331 job->mmu = file_priv->mmu; 332 job->ctx = panfrost_jm_ctx_get(jm_ctx); 333 job->engine_usage = &file_priv->engine_usage; 334 335 slot = panfrost_job_get_slot(job); 336 337 ret = drm_sched_job_init(&job->base, 338 &jm_ctx->slot_entity[slot], 339 1, NULL, file->client_id); 340 if (ret) 341 goto out_put_job; 342 343 ret = panfrost_copy_in_sync(dev, file, args, job); 344 if (ret) 345 goto out_cleanup_job; 346 347 ret = panfrost_lookup_bos(dev, file, args, job); 348 if (ret) 349 goto out_cleanup_job; 350 351 ret = panfrost_job_push(job); 352 if (ret) 353 goto out_cleanup_job; 354 355 /* Update the return sync object for the job */ 356 if (sync_out) 357 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 358 359 out_cleanup_job: 360 if (ret) 361 drm_sched_job_cleanup(&job->base); 362 out_put_job: 363 panfrost_job_put(job); 364 out_put_jm_ctx: 365 panfrost_jm_ctx_put(jm_ctx); 366 out_put_syncout: 367 if (sync_out) 368 drm_syncobj_put(sync_out); 369 370 return ret; 371 } 372 373 static int 374 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 375 struct drm_file *file_priv) 376 { 377 long ret; 378 struct drm_panfrost_wait_bo *args = data; 379 struct drm_gem_object *gem_obj; 380 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 381 382 if (args->pad) 383 return -EINVAL; 384 385 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 386 if (!gem_obj) 387 return -ENOENT; 388 389 ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ, 390 true, timeout); 391 if (!ret) 392 ret = timeout ? -ETIMEDOUT : -EBUSY; 393 394 drm_gem_object_put(gem_obj); 395 396 return ret; 397 } 398 399 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 400 struct drm_file *file_priv) 401 { 402 struct drm_panfrost_mmap_bo *args = data; 403 struct drm_gem_object *gem_obj; 404 int ret; 405 406 if (args->flags != 0) { 407 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 408 return -EINVAL; 409 } 410 411 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 412 if (!gem_obj) { 413 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 414 return -ENOENT; 415 } 416 417 /* Don't allow mmapping of heap objects as pages are not pinned. */ 418 if (to_panfrost_bo(gem_obj)->is_heap) { 419 ret = -EINVAL; 420 goto out; 421 } 422 423 ret = drm_gem_create_mmap_offset(gem_obj); 424 if (ret == 0) 425 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 426 427 out: 428 drm_gem_object_put(gem_obj); 429 return ret; 430 } 431 432 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 433 struct drm_file *file_priv) 434 { 435 struct panfrost_file_priv *priv = file_priv->driver_priv; 436 struct drm_panfrost_get_bo_offset *args = data; 437 struct panfrost_gem_mapping *mapping; 438 struct drm_gem_object *gem_obj; 439 struct panfrost_gem_object *bo; 440 441 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 442 if (!gem_obj) { 443 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 444 return -ENOENT; 445 } 446 bo = to_panfrost_bo(gem_obj); 447 448 mapping = panfrost_gem_mapping_get(bo, priv); 449 drm_gem_object_put(gem_obj); 450 451 if (!mapping) 452 return -EINVAL; 453 454 args->offset = mapping->mmnode.start << PAGE_SHIFT; 455 panfrost_gem_mapping_put(mapping); 456 return 0; 457 } 458 459 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, 460 struct drm_file *file_priv) 461 { 462 struct panfrost_file_priv *priv = file_priv->driver_priv; 463 struct drm_panfrost_madvise *args = data; 464 struct panfrost_device *pfdev = to_panfrost_device(dev); 465 struct drm_gem_object *gem_obj; 466 struct panfrost_gem_object *bo; 467 int ret = 0; 468 469 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 470 if (!gem_obj) { 471 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 472 return -ENOENT; 473 } 474 475 bo = to_panfrost_bo(gem_obj); 476 477 ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); 478 if (ret) 479 goto out_put_object; 480 481 mutex_lock(&pfdev->shrinker_lock); 482 mutex_lock(&bo->mappings.lock); 483 if (args->madv == PANFROST_MADV_DONTNEED) { 484 struct panfrost_gem_mapping *first; 485 486 first = list_first_entry(&bo->mappings.list, 487 struct panfrost_gem_mapping, 488 node); 489 490 /* 491 * If we want to mark the BO purgeable, there must be only one 492 * user: the caller FD. 493 * We could do something smarter and mark the BO purgeable only 494 * when all its users have marked it purgeable, but globally 495 * visible/shared BOs are likely to never be marked purgeable 496 * anyway, so let's not bother. 497 */ 498 if (!list_is_singular(&bo->mappings.list) || 499 WARN_ON_ONCE(first->mmu != priv->mmu)) { 500 ret = -EINVAL; 501 goto out_unlock_mappings; 502 } 503 } 504 505 args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv); 506 507 if (args->retained) { 508 if (args->madv == PANFROST_MADV_DONTNEED) 509 list_move_tail(&bo->base.madv_list, 510 &pfdev->shrinker_list); 511 else if (args->madv == PANFROST_MADV_WILLNEED) 512 list_del_init(&bo->base.madv_list); 513 } 514 515 out_unlock_mappings: 516 mutex_unlock(&bo->mappings.lock); 517 mutex_unlock(&pfdev->shrinker_lock); 518 dma_resv_unlock(bo->base.base.resv); 519 out_put_object: 520 drm_gem_object_put(gem_obj); 521 return ret; 522 } 523 524 static int panfrost_ioctl_set_label_bo(struct drm_device *ddev, void *data, 525 struct drm_file *file) 526 { 527 struct drm_panfrost_set_label_bo *args = data; 528 struct drm_gem_object *obj; 529 const char *label = NULL; 530 int ret = 0; 531 532 if (args->pad) 533 return -EINVAL; 534 535 obj = drm_gem_object_lookup(file, args->handle); 536 if (!obj) 537 return -ENOENT; 538 539 if (args->label) { 540 label = strndup_user(u64_to_user_ptr(args->label), 541 PANFROST_BO_LABEL_MAXLEN); 542 if (IS_ERR(label)) { 543 ret = PTR_ERR(label); 544 if (ret == -EINVAL) 545 ret = -E2BIG; 546 goto err_put_obj; 547 } 548 } 549 550 /* 551 * We treat passing a label of length 0 and passing a NULL label 552 * differently, because even though they might seem conceptually 553 * similar, future uses of the BO label might expect a different 554 * behaviour in each case. 555 */ 556 panfrost_gem_set_label(obj, label); 557 558 err_put_obj: 559 drm_gem_object_put(obj); 560 561 return ret; 562 } 563 564 static int panfrost_ioctl_jm_ctx_create(struct drm_device *dev, void *data, 565 struct drm_file *file) 566 { 567 return panfrost_jm_ctx_create(file, data); 568 } 569 570 static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data, 571 struct drm_file *file) 572 { 573 const struct drm_panfrost_jm_ctx_destroy *args = data; 574 575 if (args->pad) 576 return -EINVAL; 577 578 /* We can't destroy the default context created when the file is opened. */ 579 if (!args->handle) 580 return -EINVAL; 581 582 return panfrost_jm_ctx_destroy(file, args->handle); 583 } 584 585 static int panfrost_ioctl_sync_bo(struct drm_device *ddev, void *data, 586 struct drm_file *file) 587 { 588 struct drm_panfrost_sync_bo *args = data; 589 struct drm_panfrost_bo_sync_op *ops; 590 struct drm_gem_object *obj; 591 int ret; 592 u32 i; 593 594 if (args->pad) 595 return -EINVAL; 596 597 if (!args->op_count) 598 return 0; 599 600 ops = kvmalloc_objs(*ops, args->op_count); 601 if (!ops) { 602 DRM_DEBUG("Failed to allocate incoming BO sync ops array\n"); 603 return -ENOMEM; 604 } 605 606 if (copy_from_user(ops, (void __user *)(uintptr_t)args->ops, 607 args->op_count * sizeof(*ops))) { 608 DRM_DEBUG("Failed to copy in BO sync ops\n"); 609 ret = -EFAULT; 610 goto err_ops; 611 } 612 613 for (i = 0; i < args->op_count; i++) { 614 obj = drm_gem_object_lookup(file, ops[i].handle); 615 if (!obj) { 616 ret = -ENOENT; 617 goto err_ops; 618 } 619 620 ret = panfrost_gem_sync(obj, ops[i].type, 621 ops[i].offset, ops[i].size); 622 623 drm_gem_object_put(obj); 624 625 if (ret) 626 goto err_ops; 627 } 628 629 err_ops: 630 kvfree(ops); 631 632 return ret; 633 } 634 635 static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data, 636 struct drm_file *file_priv) 637 { 638 struct drm_panfrost_query_bo_info *args = data; 639 struct drm_gem_object *gem_obj; 640 struct panfrost_gem_object *bo; 641 642 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 643 if (!gem_obj) { 644 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 645 return -ENOENT; 646 } 647 648 bo = to_panfrost_bo(gem_obj); 649 args->pad = 0; 650 args->create_flags = 0; 651 args->extra_flags = 0; 652 653 if (drm_gem_is_imported(gem_obj)) { 654 args->extra_flags |= DRM_PANFROST_BO_IS_IMPORTED; 655 } else { 656 if (bo->noexec) 657 args->create_flags |= PANFROST_BO_NOEXEC; 658 659 if (bo->is_heap) 660 args->create_flags |= PANFROST_BO_HEAP; 661 662 if (!bo->base.map_wc) 663 args->create_flags |= PANFROST_BO_WB_MMAP; 664 } 665 666 drm_gem_object_put(gem_obj); 667 return 0; 668 } 669 670 int panfrost_unstable_ioctl_check(void) 671 { 672 if (!unstable_ioctls) 673 return -ENOSYS; 674 675 return 0; 676 } 677 678 static int 679 panfrost_open(struct drm_device *dev, struct drm_file *file) 680 { 681 int ret; 682 struct panfrost_device *pfdev = to_panfrost_device(dev); 683 struct panfrost_file_priv *panfrost_priv; 684 685 panfrost_priv = kzalloc_obj(*panfrost_priv); 686 if (!panfrost_priv) 687 return -ENOMEM; 688 689 panfrost_priv->pfdev = pfdev; 690 file->driver_priv = panfrost_priv; 691 692 panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev); 693 if (IS_ERR(panfrost_priv->mmu)) { 694 ret = PTR_ERR(panfrost_priv->mmu); 695 goto err_free; 696 } 697 698 ret = panfrost_jm_open(file); 699 if (ret) 700 goto err_job; 701 702 return 0; 703 704 err_job: 705 panfrost_mmu_ctx_put(panfrost_priv->mmu); 706 err_free: 707 kfree(panfrost_priv); 708 return ret; 709 } 710 711 static void 712 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 713 { 714 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 715 716 panfrost_perfcnt_close(file); 717 panfrost_jm_close(file); 718 719 panfrost_mmu_ctx_put(panfrost_priv->mmu); 720 kfree(panfrost_priv); 721 } 722 723 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 724 #define PANFROST_IOCTL(n, func, flags) \ 725 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 726 727 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW), 728 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 729 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 730 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 731 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 732 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 733 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 734 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 735 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), 736 PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW), 737 PANFROST_IOCTL(JM_CTX_CREATE, jm_ctx_create, DRM_RENDER_ALLOW), 738 PANFROST_IOCTL(JM_CTX_DESTROY, jm_ctx_destroy, DRM_RENDER_ALLOW), 739 PANFROST_IOCTL(SYNC_BO, sync_bo, DRM_RENDER_ALLOW), 740 PANFROST_IOCTL(QUERY_BO_INFO, query_bo_info, DRM_RENDER_ALLOW), 741 }; 742 743 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, 744 struct panfrost_file_priv *panfrost_priv, 745 struct drm_printer *p) 746 { 747 int i; 748 749 /* 750 * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not 751 * accurate, as they only provide a rough estimation of the number of 752 * GPU cycles and CPU time spent in a given context. This is due to two 753 * different factors: 754 * - Firstly, we must consider the time the CPU and then the kernel 755 * takes to process the GPU interrupt, which means additional time and 756 * GPU cycles will be added in excess to the real figure. 757 * - Secondly, the pipelining done by the Job Manager (2 job slots per 758 * engine) implies there is no way to know exactly how much time each 759 * job spent on the GPU. 760 */ 761 762 for (i = 0; i < NUM_JOB_SLOTS - 1; i++) { 763 if (pfdev->profile_mode) { 764 drm_printf(p, "drm-engine-%s:\t%llu ns\n", 765 panfrost_engine_names[i], 766 panfrost_priv->engine_usage.elapsed_ns[i]); 767 drm_printf(p, "drm-cycles-%s:\t%llu\n", 768 panfrost_engine_names[i], 769 panfrost_priv->engine_usage.cycles[i]); 770 } 771 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n", 772 panfrost_engine_names[i], pfdev->pfdevfreq.fast_rate); 773 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n", 774 panfrost_engine_names[i], pfdev->pfdevfreq.current_frequency); 775 } 776 } 777 778 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file) 779 { 780 struct panfrost_device *pfdev = to_panfrost_device(file->minor->dev); 781 782 panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p); 783 784 drm_show_memory_stats(p, file); 785 } 786 787 static const struct file_operations panfrost_drm_driver_fops = { 788 .owner = THIS_MODULE, 789 DRM_GEM_FOPS, 790 .show_fdinfo = drm_show_fdinfo, 791 }; 792 793 #ifdef CONFIG_DEBUG_FS 794 static int panthor_gems_show(struct seq_file *m, void *data) 795 { 796 struct drm_info_node *node = m->private; 797 struct panfrost_device *pfdev = to_panfrost_device(node->minor->dev); 798 799 panfrost_gem_debugfs_print_bos(pfdev, m); 800 801 return 0; 802 } 803 804 static void show_panfrost_jm_ctx(struct panfrost_jm_ctx *jm_ctx, u32 handle, 805 struct seq_file *m) 806 { 807 struct drm_device *ddev = ((struct drm_info_node *)m->private)->minor->dev; 808 const char *prio = "UNKNOWN"; 809 810 static const char * const prios[] = { 811 [DRM_SCHED_PRIORITY_HIGH] = "HIGH", 812 [DRM_SCHED_PRIORITY_NORMAL] = "NORMAL", 813 [DRM_SCHED_PRIORITY_LOW] = "LOW", 814 }; 815 816 if (jm_ctx->slot_entity[0].priority != 817 jm_ctx->slot_entity[1].priority) 818 drm_warn(ddev, "Slot priorities should be the same in a single context"); 819 820 if (jm_ctx->slot_entity[0].priority < ARRAY_SIZE(prios)) 821 prio = prios[jm_ctx->slot_entity[0].priority]; 822 823 seq_printf(m, " JM context %u: priority %s\n", handle, prio); 824 } 825 826 static int show_file_jm_ctxs(struct panfrost_file_priv *pfile, 827 struct seq_file *m) 828 { 829 struct panfrost_jm_ctx *jm_ctx; 830 unsigned long i; 831 832 xa_lock(&pfile->jm_ctxs); 833 xa_for_each(&pfile->jm_ctxs, i, jm_ctx) { 834 jm_ctx = panfrost_jm_ctx_get(jm_ctx); 835 xa_unlock(&pfile->jm_ctxs); 836 show_panfrost_jm_ctx(jm_ctx, i, m); 837 panfrost_jm_ctx_put(jm_ctx); 838 xa_lock(&pfile->jm_ctxs); 839 } 840 xa_unlock(&pfile->jm_ctxs); 841 842 return 0; 843 } 844 845 static struct drm_info_list panthor_debugfs_list[] = { 846 {"gems", 847 panthor_gems_show, 0, NULL}, 848 }; 849 850 static int panthor_gems_debugfs_init(struct drm_minor *minor) 851 { 852 drm_debugfs_create_files(panthor_debugfs_list, 853 ARRAY_SIZE(panthor_debugfs_list), 854 minor->debugfs_root, minor); 855 856 return 0; 857 } 858 859 static int show_each_file(struct seq_file *m, void *arg) 860 { 861 struct drm_info_node *node = (struct drm_info_node *)m->private; 862 struct drm_device *ddev = node->minor->dev; 863 int (*show)(struct panfrost_file_priv *, struct seq_file *) = 864 node->info_ent->data; 865 struct drm_file *file; 866 int ret; 867 868 ret = mutex_lock_interruptible(&ddev->filelist_mutex); 869 if (ret) 870 return ret; 871 872 list_for_each_entry(file, &ddev->filelist, lhead) { 873 struct task_struct *task; 874 struct panfrost_file_priv *pfile = file->driver_priv; 875 struct pid *pid; 876 877 /* 878 * Although we have a valid reference on file->pid, that does 879 * not guarantee that the task_struct who called get_pid() is 880 * still alive (e.g. get_pid(current) => fork() => exit()). 881 * Therefore, we need to protect this ->comm access using RCU. 882 */ 883 rcu_read_lock(); 884 pid = rcu_dereference(file->pid); 885 task = pid_task(pid, PIDTYPE_TGID); 886 seq_printf(m, "client_id %8llu pid %8d command %s:\n", 887 file->client_id, pid_nr(pid), 888 task ? task->comm : "<unknown>"); 889 rcu_read_unlock(); 890 891 ret = show(pfile, m); 892 if (ret < 0) 893 break; 894 895 seq_puts(m, "\n"); 896 } 897 898 mutex_unlock(&ddev->filelist_mutex); 899 return ret; 900 } 901 902 static struct drm_info_list panfrost_sched_debugfs_list[] = { 903 { "sched_ctxs", show_each_file, 0, show_file_jm_ctxs }, 904 }; 905 906 static void panfrost_sched_debugfs_init(struct drm_minor *minor) 907 { 908 drm_debugfs_create_files(panfrost_sched_debugfs_list, 909 ARRAY_SIZE(panfrost_sched_debugfs_list), 910 minor->debugfs_root, minor); 911 } 912 913 static void panfrost_debugfs_init(struct drm_minor *minor) 914 { 915 panthor_gems_debugfs_init(minor); 916 panfrost_sched_debugfs_init(minor); 917 } 918 #endif 919 920 /* 921 * Panfrost driver version: 922 * - 1.0 - initial interface 923 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO 924 * - 1.2 - adds AFBC_FEATURES query 925 * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT 926 * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries 927 * - 1.4 - adds SET_LABEL_BO 928 * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow 929 * context creation with configurable priorities/affinity 930 * - 1.6 - adds PANFROST_BO_MAP_WB, PANFROST_IOCTL_SYNC_BO, 931 * PANFROST_IOCTL_QUERY_BO_INFO and 932 * DRM_PANFROST_PARAM_SELECTED_COHERENCY 933 */ 934 static const struct drm_driver panfrost_drm_driver = { 935 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 936 .open = panfrost_open, 937 .postclose = panfrost_postclose, 938 .show_fdinfo = panfrost_show_fdinfo, 939 .ioctls = panfrost_drm_driver_ioctls, 940 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 941 .fops = &panfrost_drm_driver_fops, 942 .name = "panfrost", 943 .desc = "panfrost DRM", 944 .major = 1, 945 .minor = 6, 946 947 .gem_create_object = panfrost_gem_create_object, 948 .gem_prime_import = panfrost_gem_prime_import, 949 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 950 #ifdef CONFIG_DEBUG_FS 951 .debugfs_init = panfrost_debugfs_init, 952 #endif 953 }; 954 955 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 956 bool panfrost_transparent_hugepage = true; 957 module_param_named(transparent_hugepage, panfrost_transparent_hugepage, bool, 0400); 958 MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)"); 959 #endif 960 961 static int panfrost_probe(struct platform_device *pdev) 962 { 963 struct panfrost_device *pfdev; 964 int err; 965 966 pfdev = devm_drm_dev_alloc(&pdev->dev, &panfrost_drm_driver, 967 struct panfrost_device, base); 968 if (IS_ERR(pfdev)) 969 return PTR_ERR(pfdev); 970 971 platform_set_drvdata(pdev, pfdev); 972 973 pfdev->comp = of_device_get_match_data(&pdev->dev); 974 if (!pfdev->comp) 975 return -ENODEV; 976 977 pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT; 978 979 mutex_init(&pfdev->shrinker_lock); 980 INIT_LIST_HEAD(&pfdev->shrinker_list); 981 982 err = panfrost_device_init(pfdev); 983 if (err) { 984 if (err != -EPROBE_DEFER) 985 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 986 goto err_out0; 987 } 988 989 pm_runtime_set_active(pfdev->base.dev); 990 pm_runtime_mark_last_busy(pfdev->base.dev); 991 pm_runtime_enable(pfdev->base.dev); 992 pm_runtime_set_autosuspend_delay(pfdev->base.dev, 50); /* ~3 frames */ 993 pm_runtime_use_autosuspend(pfdev->base.dev); 994 995 /* 996 * Register the DRM device with the core and the connectors with 997 * sysfs 998 */ 999 err = drm_dev_register(&pfdev->base, 0); 1000 if (err < 0) 1001 goto err_out1; 1002 1003 err = panfrost_gem_shrinker_init(&pfdev->base); 1004 if (err) 1005 goto err_out2; 1006 1007 return 0; 1008 1009 err_out2: 1010 drm_dev_unregister(&pfdev->base); 1011 err_out1: 1012 pm_runtime_disable(pfdev->base.dev); 1013 panfrost_device_fini(pfdev); 1014 pm_runtime_set_suspended(pfdev->base.dev); 1015 err_out0: 1016 return err; 1017 } 1018 1019 static void panfrost_remove(struct platform_device *pdev) 1020 { 1021 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 1022 1023 drm_dev_unregister(&pfdev->base); 1024 panfrost_gem_shrinker_cleanup(&pfdev->base); 1025 1026 pm_runtime_get_sync(pfdev->base.dev); 1027 pm_runtime_disable(pfdev->base.dev); 1028 panfrost_device_fini(pfdev); 1029 pm_runtime_set_suspended(pfdev->base.dev); 1030 } 1031 1032 static ssize_t profiling_show(struct device *dev, 1033 struct device_attribute *attr, char *buf) 1034 { 1035 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1036 1037 return sysfs_emit(buf, "%d\n", pfdev->profile_mode); 1038 } 1039 1040 static ssize_t profiling_store(struct device *dev, 1041 struct device_attribute *attr, 1042 const char *buf, size_t len) 1043 { 1044 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1045 bool value; 1046 int err; 1047 1048 err = kstrtobool(buf, &value); 1049 if (err) 1050 return err; 1051 1052 pfdev->profile_mode = value; 1053 1054 return len; 1055 } 1056 1057 static DEVICE_ATTR_RW(profiling); 1058 1059 static struct attribute *panfrost_attrs[] = { 1060 &dev_attr_profiling.attr, 1061 NULL, 1062 }; 1063 1064 ATTRIBUTE_GROUPS(panfrost); 1065 1066 /* 1067 * The OPP core wants the supply names to be NULL terminated, but we need the 1068 * correct num_supplies value for regulator core. Hence, we NULL terminate here 1069 * and then initialize num_supplies with ARRAY_SIZE - 1. 1070 */ 1071 static const char * const default_supplies[] = { "mali", NULL }; 1072 static const struct panfrost_compatible default_data = { 1073 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1074 .supply_names = default_supplies, 1075 .num_pm_domains = 1, /* optional */ 1076 .pm_domain_names = NULL, 1077 }; 1078 1079 static const struct panfrost_compatible default_pm_rt_data = { 1080 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1081 .supply_names = default_supplies, 1082 .num_pm_domains = 1, 1083 .pm_features = BIT(GPU_PM_RT), 1084 }; 1085 1086 static const struct panfrost_compatible amlogic_data = { 1087 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1088 .supply_names = default_supplies, 1089 .vendor_quirk = panfrost_gpu_amlogic_quirk, 1090 }; 1091 1092 static const char * const mediatek_pm_domains[] = { "core0", "core1", "core2", 1093 "core3", "core4" }; 1094 /* 1095 * The old data with two power supplies for MT8183 is here only to 1096 * keep retro-compatibility with older devicetrees, as DVFS will 1097 * not work with this one. 1098 * 1099 * On new devicetrees please use the _b variant with a single and 1100 * coupled regulators instead. 1101 */ 1102 static const char * const legacy_supplies[] = { "mali", "sram", NULL }; 1103 static const struct panfrost_compatible mediatek_mt8183_data = { 1104 .num_supplies = ARRAY_SIZE(legacy_supplies) - 1, 1105 .supply_names = legacy_supplies, 1106 .num_pm_domains = 3, 1107 .pm_domain_names = mediatek_pm_domains, 1108 }; 1109 1110 static const struct panfrost_compatible mediatek_mt8183_b_data = { 1111 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1112 .supply_names = default_supplies, 1113 .num_pm_domains = 3, 1114 .pm_domain_names = mediatek_pm_domains, 1115 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1116 }; 1117 1118 static const struct panfrost_compatible mediatek_mt8186_data = { 1119 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1120 .supply_names = default_supplies, 1121 .num_pm_domains = 2, 1122 .pm_domain_names = mediatek_pm_domains, 1123 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1124 }; 1125 1126 static const struct panfrost_compatible mediatek_mt8188_data = { 1127 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1128 .supply_names = default_supplies, 1129 .num_pm_domains = 3, 1130 .pm_domain_names = mediatek_pm_domains, 1131 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1132 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1133 }; 1134 1135 static const struct panfrost_compatible mediatek_mt8192_data = { 1136 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1137 .supply_names = default_supplies, 1138 .num_pm_domains = 5, 1139 .pm_domain_names = mediatek_pm_domains, 1140 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1141 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1142 }; 1143 1144 static const struct panfrost_compatible mediatek_mt8370_data = { 1145 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1146 .supply_names = default_supplies, 1147 .num_pm_domains = 2, 1148 .pm_domain_names = mediatek_pm_domains, 1149 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1150 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1151 }; 1152 1153 static const struct of_device_id dt_match[] = { 1154 /* Set first to probe before the generic compatibles */ 1155 { .compatible = "amlogic,meson-gxm-mali", 1156 .data = &amlogic_data, }, 1157 { .compatible = "amlogic,meson-g12a-mali", 1158 .data = &amlogic_data, }, 1159 { .compatible = "renesas,r9a09g047-mali", .data = &default_pm_rt_data }, 1160 { .compatible = "arm,mali-t604", .data = &default_data, }, 1161 { .compatible = "arm,mali-t624", .data = &default_data, }, 1162 { .compatible = "arm,mali-t628", .data = &default_data, }, 1163 { .compatible = "arm,mali-t720", .data = &default_data, }, 1164 { .compatible = "arm,mali-t760", .data = &default_data, }, 1165 { .compatible = "arm,mali-t820", .data = &default_data, }, 1166 { .compatible = "arm,mali-t830", .data = &default_data, }, 1167 { .compatible = "arm,mali-t860", .data = &default_data, }, 1168 { .compatible = "arm,mali-t880", .data = &default_data, }, 1169 { .compatible = "arm,mali-bifrost", .data = &default_data, }, 1170 { .compatible = "arm,mali-valhall-jm", .data = &default_data, }, 1171 { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data }, 1172 { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data }, 1173 { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data }, 1174 { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data }, 1175 { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data }, 1176 { .compatible = "mediatek,mt8370-mali", .data = &mediatek_mt8370_data }, 1177 { .compatible = "allwinner,sun50i-h616-mali", .data = &default_pm_rt_data }, 1178 {} 1179 }; 1180 MODULE_DEVICE_TABLE(of, dt_match); 1181 1182 static struct platform_driver panfrost_driver = { 1183 .probe = panfrost_probe, 1184 .remove = panfrost_remove, 1185 .driver = { 1186 .name = "panfrost", 1187 .pm = pm_ptr(&panfrost_pm_ops), 1188 .of_match_table = dt_match, 1189 .dev_groups = panfrost_groups, 1190 }, 1191 }; 1192 module_platform_driver(panfrost_driver); 1193 1194 MODULE_AUTHOR("Panfrost Project Developers"); 1195 MODULE_DESCRIPTION("Panfrost DRM Driver"); 1196 MODULE_LICENSE("GPL v2"); 1197 MODULE_SOFTDEP("pre: governor_simpleondemand"); 1198