1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #ifdef CONFIG_ARM_ARCH_TIMER 7 #include <asm/arch_timer.h> 8 #endif 9 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/pagemap.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <drm/panfrost_drm.h> 16 #include <drm/drm_debugfs.h> 17 #include <drm/drm_drv.h> 18 #include <drm/drm_ioctl.h> 19 #include <drm/drm_print.h> 20 #include <drm/drm_syncobj.h> 21 #include <drm/drm_utils.h> 22 23 #include "panfrost_device.h" 24 #include "panfrost_drv.h" 25 #include "panfrost_gem.h" 26 #include "panfrost_mmu.h" 27 #include "panfrost_job.h" 28 #include "panfrost_gpu.h" 29 #include "panfrost_perfcnt.h" 30 31 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT) 32 33 static bool unstable_ioctls; 34 module_param_unsafe(unstable_ioctls, bool, 0600); 35 36 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, 37 u64 *arg) 38 { 39 int ret; 40 41 ret = pm_runtime_resume_and_get(pfdev->base.dev); 42 if (ret) 43 return ret; 44 45 panfrost_cycle_counter_get(pfdev); 46 *arg = panfrost_timestamp_read(pfdev); 47 panfrost_cycle_counter_put(pfdev); 48 49 pm_runtime_put(pfdev->base.dev); 50 return 0; 51 } 52 53 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 54 { 55 struct drm_panfrost_get_param *param = data; 56 struct panfrost_device *pfdev = to_panfrost_device(ddev); 57 int ret; 58 59 if (param->pad != 0) 60 return -EINVAL; 61 62 #define PANFROST_FEATURE(name, member) \ 63 case DRM_PANFROST_PARAM_ ## name: \ 64 param->value = pfdev->features.member; \ 65 break 66 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 67 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 68 DRM_PANFROST_PARAM_ ## name ## max: \ 69 param->value = pfdev->features.member[param->param - \ 70 DRM_PANFROST_PARAM_ ## name ## 0]; \ 71 break 72 73 switch (param->param) { 74 PANFROST_FEATURE(GPU_PROD_ID, id); 75 PANFROST_FEATURE(GPU_REVISION, revision); 76 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 77 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 78 PANFROST_FEATURE(L2_PRESENT, l2_present); 79 PANFROST_FEATURE(STACK_PRESENT, stack_present); 80 PANFROST_FEATURE(AS_PRESENT, as_present); 81 PANFROST_FEATURE(JS_PRESENT, js_present); 82 PANFROST_FEATURE(L2_FEATURES, l2_features); 83 PANFROST_FEATURE(CORE_FEATURES, core_features); 84 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 85 PANFROST_FEATURE(MEM_FEATURES, mem_features); 86 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 87 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 88 PANFROST_FEATURE(MAX_THREADS, max_threads); 89 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 90 thread_max_workgroup_sz); 91 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 92 thread_max_barrier_sz); 93 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 94 PANFROST_FEATURE(AFBC_FEATURES, afbc_features); 95 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 96 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 97 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 98 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 99 PANFROST_FEATURE(SELECTED_COHERENCY, selected_coherency); 100 101 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP: 102 ret = panfrost_ioctl_query_timestamp(pfdev, ¶m->value); 103 if (ret) 104 return ret; 105 break; 106 107 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY: 108 #ifdef CONFIG_ARM_ARCH_TIMER 109 param->value = arch_timer_get_cntfrq(); 110 #else 111 param->value = 0; 112 #endif 113 break; 114 115 case DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES: 116 param->value = BIT(PANFROST_JM_CTX_PRIORITY_LOW) | 117 BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM); 118 119 if (panfrost_high_prio_allowed(file)) 120 param->value |= BIT(PANFROST_JM_CTX_PRIORITY_HIGH); 121 break; 122 123 default: 124 return -EINVAL; 125 } 126 127 return 0; 128 } 129 130 #define PANFROST_BO_FLAGS (PANFROST_BO_NOEXEC | \ 131 PANFROST_BO_HEAP | \ 132 PANFROST_BO_WB_MMAP) 133 134 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 135 struct drm_file *file) 136 { 137 struct panfrost_file_priv *priv = file->driver_priv; 138 struct panfrost_gem_object *bo; 139 struct drm_panfrost_create_bo *args = data; 140 struct panfrost_gem_mapping *mapping; 141 int ret; 142 143 if (!args->size || args->pad || (args->flags & ~PANFROST_BO_FLAGS)) 144 return -EINVAL; 145 146 /* Heaps should never be executable */ 147 if ((args->flags & PANFROST_BO_HEAP) && 148 !(args->flags & PANFROST_BO_NOEXEC)) 149 return -EINVAL; 150 151 bo = panfrost_gem_create(dev, args->size, args->flags); 152 if (IS_ERR(bo)) 153 return PTR_ERR(bo); 154 155 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 156 if (ret) 157 goto out; 158 159 mapping = panfrost_gem_mapping_get(bo, priv); 160 if (mapping) { 161 args->offset = mapping->mmnode.start << PAGE_SHIFT; 162 panfrost_gem_mapping_put(mapping); 163 } else { 164 /* This can only happen if the handle from 165 * drm_gem_handle_create() has already been guessed and freed 166 * by user space 167 */ 168 ret = -EINVAL; 169 } 170 171 out: 172 drm_gem_object_put(&bo->base.base); 173 return ret; 174 } 175 176 /** 177 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 178 * referenced by the job. 179 * @dev: DRM device 180 * @file_priv: DRM file for this fd 181 * @args: IOCTL args 182 * @job: job being set up 183 * 184 * Resolve handles from userspace to BOs and attach them to job. 185 * 186 * Note that this function doesn't need to unreference the BOs on 187 * failure, because that will happen at panfrost_job_cleanup() time. 188 */ 189 static int 190 panfrost_lookup_bos(struct drm_device *dev, 191 struct drm_file *file_priv, 192 struct drm_panfrost_submit *args, 193 struct panfrost_job *job) 194 { 195 struct panfrost_file_priv *priv = file_priv->driver_priv; 196 struct panfrost_gem_object *bo; 197 unsigned int i; 198 int ret; 199 200 job->bo_count = args->bo_handle_count; 201 202 if (!job->bo_count) 203 return 0; 204 205 ret = drm_gem_objects_lookup(file_priv, 206 (void __user *)(uintptr_t)args->bo_handles, 207 job->bo_count, &job->bos); 208 if (ret) 209 return ret; 210 211 job->mappings = kvmalloc_objs(struct panfrost_gem_mapping *, 212 job->bo_count, GFP_KERNEL | __GFP_ZERO); 213 if (!job->mappings) 214 return -ENOMEM; 215 216 for (i = 0; i < job->bo_count; i++) { 217 struct panfrost_gem_mapping *mapping; 218 219 bo = to_panfrost_bo(job->bos[i]); 220 mapping = panfrost_gem_mapping_get(bo, priv); 221 if (!mapping) { 222 ret = -EINVAL; 223 break; 224 } 225 226 atomic_inc(&bo->gpu_usecount); 227 job->mappings[i] = mapping; 228 } 229 230 return ret; 231 } 232 233 /** 234 * panfrost_copy_in_sync() - Sets up job->deps with the sync objects 235 * referenced by the job. 236 * @dev: DRM device 237 * @file_priv: DRM file for this fd 238 * @args: IOCTL args 239 * @job: job being set up 240 * 241 * Resolve syncobjs from userspace to fences and attach them to job. 242 * 243 * Note that this function doesn't need to unreference the fences on 244 * failure, because that will happen at panfrost_job_cleanup() time. 245 */ 246 static int 247 panfrost_copy_in_sync(struct drm_device *dev, 248 struct drm_file *file_priv, 249 struct drm_panfrost_submit *args, 250 struct panfrost_job *job) 251 { 252 u32 *handles; 253 int ret = 0; 254 int i, in_fence_count; 255 256 in_fence_count = args->in_sync_count; 257 258 if (!in_fence_count) 259 return 0; 260 261 handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL); 262 if (!handles) { 263 ret = -ENOMEM; 264 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 265 goto fail; 266 } 267 268 if (copy_from_user(handles, 269 (void __user *)(uintptr_t)args->in_syncs, 270 in_fence_count * sizeof(u32))) { 271 ret = -EFAULT; 272 DRM_DEBUG("Failed to copy in syncobj handles\n"); 273 goto fail; 274 } 275 276 for (i = 0; i < in_fence_count; i++) { 277 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, 278 handles[i], 0); 279 if (ret) 280 goto fail; 281 } 282 283 fail: 284 kvfree(handles); 285 return ret; 286 } 287 288 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 289 struct drm_file *file) 290 { 291 struct panfrost_device *pfdev = to_panfrost_device(dev); 292 struct panfrost_file_priv *file_priv = file->driver_priv; 293 struct drm_panfrost_submit *args = data; 294 struct drm_syncobj *sync_out = NULL; 295 struct panfrost_jm_ctx *jm_ctx; 296 struct panfrost_job *job; 297 int ret = 0, slot; 298 299 if (args->pad) 300 return -EINVAL; 301 302 if (!args->jc) 303 return -EINVAL; 304 305 if (args->requirements & ~JOB_REQUIREMENTS) 306 return -EINVAL; 307 308 if (args->out_sync > 0) { 309 sync_out = drm_syncobj_find(file, args->out_sync); 310 if (!sync_out) 311 return -ENODEV; 312 } 313 314 jm_ctx = panfrost_jm_ctx_from_handle(file, args->jm_ctx_handle); 315 if (!jm_ctx) { 316 ret = -EINVAL; 317 goto out_put_syncout; 318 } 319 320 job = kzalloc_obj(*job); 321 if (!job) { 322 ret = -ENOMEM; 323 goto out_put_jm_ctx; 324 } 325 326 kref_init(&job->refcount); 327 328 job->pfdev = pfdev; 329 job->jc = args->jc; 330 job->requirements = args->requirements; 331 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 332 job->mmu = file_priv->mmu; 333 job->ctx = panfrost_jm_ctx_get(jm_ctx); 334 job->engine_usage = &file_priv->engine_usage; 335 336 slot = panfrost_job_get_slot(job); 337 338 ret = drm_sched_job_init(&job->base, 339 &jm_ctx->slot_entity[slot], 340 1, NULL, file->client_id); 341 if (ret) 342 goto out_put_job; 343 344 ret = panfrost_copy_in_sync(dev, file, args, job); 345 if (ret) 346 goto out_cleanup_job; 347 348 ret = panfrost_lookup_bos(dev, file, args, job); 349 if (ret) 350 goto out_cleanup_job; 351 352 ret = panfrost_job_push(job); 353 if (ret) 354 goto out_cleanup_job; 355 356 /* Update the return sync object for the job */ 357 if (sync_out) 358 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 359 360 out_cleanup_job: 361 if (ret) 362 drm_sched_job_cleanup(&job->base); 363 out_put_job: 364 panfrost_job_put(job); 365 out_put_jm_ctx: 366 panfrost_jm_ctx_put(jm_ctx); 367 out_put_syncout: 368 if (sync_out) 369 drm_syncobj_put(sync_out); 370 371 return ret; 372 } 373 374 static int 375 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 376 struct drm_file *file_priv) 377 { 378 long ret; 379 struct drm_panfrost_wait_bo *args = data; 380 struct drm_gem_object *gem_obj; 381 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 382 383 if (args->pad) 384 return -EINVAL; 385 386 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 387 if (!gem_obj) 388 return -ENOENT; 389 390 ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ, 391 true, timeout); 392 if (!ret) 393 ret = timeout ? -ETIMEDOUT : -EBUSY; 394 else if (ret > 0) 395 ret = 0; 396 397 drm_gem_object_put(gem_obj); 398 399 return ret; 400 } 401 402 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 403 struct drm_file *file_priv) 404 { 405 struct drm_panfrost_mmap_bo *args = data; 406 struct drm_gem_object *gem_obj; 407 int ret; 408 409 if (args->flags != 0) { 410 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 411 return -EINVAL; 412 } 413 414 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 415 if (!gem_obj) { 416 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 417 return -ENOENT; 418 } 419 420 /* Don't allow mmapping of heap objects as pages are not pinned. */ 421 if (to_panfrost_bo(gem_obj)->is_heap) { 422 ret = -EINVAL; 423 goto out; 424 } 425 426 ret = drm_gem_create_mmap_offset(gem_obj); 427 if (ret == 0) 428 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 429 430 out: 431 drm_gem_object_put(gem_obj); 432 return ret; 433 } 434 435 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 436 struct drm_file *file_priv) 437 { 438 struct panfrost_file_priv *priv = file_priv->driver_priv; 439 struct drm_panfrost_get_bo_offset *args = data; 440 struct panfrost_gem_mapping *mapping; 441 struct drm_gem_object *gem_obj; 442 struct panfrost_gem_object *bo; 443 444 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 445 if (!gem_obj) { 446 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 447 return -ENOENT; 448 } 449 bo = to_panfrost_bo(gem_obj); 450 451 mapping = panfrost_gem_mapping_get(bo, priv); 452 drm_gem_object_put(gem_obj); 453 454 if (!mapping) 455 return -EINVAL; 456 457 args->offset = mapping->mmnode.start << PAGE_SHIFT; 458 panfrost_gem_mapping_put(mapping); 459 return 0; 460 } 461 462 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, 463 struct drm_file *file_priv) 464 { 465 struct panfrost_file_priv *priv = file_priv->driver_priv; 466 struct drm_panfrost_madvise *args = data; 467 struct panfrost_device *pfdev = to_panfrost_device(dev); 468 struct drm_gem_object *gem_obj; 469 struct panfrost_gem_object *bo; 470 int ret = 0; 471 472 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 473 if (!gem_obj) { 474 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 475 return -ENOENT; 476 } 477 478 bo = to_panfrost_bo(gem_obj); 479 480 ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); 481 if (ret) 482 goto out_put_object; 483 484 mutex_lock(&pfdev->shrinker_lock); 485 mutex_lock(&bo->mappings.lock); 486 if (args->madv == PANFROST_MADV_DONTNEED) { 487 struct panfrost_gem_mapping *first; 488 489 first = list_first_entry(&bo->mappings.list, 490 struct panfrost_gem_mapping, 491 node); 492 493 /* 494 * If we want to mark the BO purgeable, there must be only one 495 * user: the caller FD. 496 * We could do something smarter and mark the BO purgeable only 497 * when all its users have marked it purgeable, but globally 498 * visible/shared BOs are likely to never be marked purgeable 499 * anyway, so let's not bother. 500 */ 501 if (!list_is_singular(&bo->mappings.list) || 502 WARN_ON_ONCE(first->mmu != priv->mmu)) { 503 ret = -EINVAL; 504 goto out_unlock_mappings; 505 } 506 } 507 508 args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv); 509 510 if (args->retained) { 511 if (args->madv == PANFROST_MADV_DONTNEED) 512 list_move_tail(&bo->base.madv_list, 513 &pfdev->shrinker_list); 514 else if (args->madv == PANFROST_MADV_WILLNEED) 515 list_del_init(&bo->base.madv_list); 516 } 517 518 out_unlock_mappings: 519 mutex_unlock(&bo->mappings.lock); 520 mutex_unlock(&pfdev->shrinker_lock); 521 dma_resv_unlock(bo->base.base.resv); 522 out_put_object: 523 drm_gem_object_put(gem_obj); 524 return ret; 525 } 526 527 static int panfrost_ioctl_set_label_bo(struct drm_device *ddev, void *data, 528 struct drm_file *file) 529 { 530 struct drm_panfrost_set_label_bo *args = data; 531 struct drm_gem_object *obj; 532 const char *label = NULL; 533 int ret = 0; 534 535 if (args->pad) 536 return -EINVAL; 537 538 obj = drm_gem_object_lookup(file, args->handle); 539 if (!obj) 540 return -ENOENT; 541 542 if (args->label) { 543 label = strndup_user(u64_to_user_ptr(args->label), 544 PANFROST_BO_LABEL_MAXLEN); 545 if (IS_ERR(label)) { 546 ret = PTR_ERR(label); 547 if (ret == -EINVAL) 548 ret = -E2BIG; 549 goto err_put_obj; 550 } 551 } 552 553 /* 554 * We treat passing a label of length 0 and passing a NULL label 555 * differently, because even though they might seem conceptually 556 * similar, future uses of the BO label might expect a different 557 * behaviour in each case. 558 */ 559 panfrost_gem_set_label(obj, label); 560 561 err_put_obj: 562 drm_gem_object_put(obj); 563 564 return ret; 565 } 566 567 static int panfrost_ioctl_jm_ctx_create(struct drm_device *dev, void *data, 568 struct drm_file *file) 569 { 570 return panfrost_jm_ctx_create(file, data); 571 } 572 573 static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data, 574 struct drm_file *file) 575 { 576 const struct drm_panfrost_jm_ctx_destroy *args = data; 577 578 if (args->pad) 579 return -EINVAL; 580 581 /* We can't destroy the default context created when the file is opened. */ 582 if (!args->handle) 583 return -EINVAL; 584 585 return panfrost_jm_ctx_destroy(file, args->handle); 586 } 587 588 static int panfrost_ioctl_sync_bo(struct drm_device *ddev, void *data, 589 struct drm_file *file) 590 { 591 struct drm_panfrost_sync_bo *args = data; 592 struct drm_panfrost_bo_sync_op *ops; 593 struct drm_gem_object *obj; 594 int ret; 595 u32 i; 596 597 if (args->pad) 598 return -EINVAL; 599 600 if (!args->op_count) 601 return 0; 602 603 ops = kvmalloc_objs(*ops, args->op_count); 604 if (!ops) { 605 DRM_DEBUG("Failed to allocate incoming BO sync ops array\n"); 606 return -ENOMEM; 607 } 608 609 if (copy_from_user(ops, (void __user *)(uintptr_t)args->ops, 610 args->op_count * sizeof(*ops))) { 611 DRM_DEBUG("Failed to copy in BO sync ops\n"); 612 ret = -EFAULT; 613 goto err_ops; 614 } 615 616 for (i = 0; i < args->op_count; i++) { 617 obj = drm_gem_object_lookup(file, ops[i].handle); 618 if (!obj) { 619 ret = -ENOENT; 620 goto err_ops; 621 } 622 623 ret = panfrost_gem_sync(obj, ops[i].type, 624 ops[i].offset, ops[i].size); 625 626 drm_gem_object_put(obj); 627 628 if (ret) 629 goto err_ops; 630 } 631 632 err_ops: 633 kvfree(ops); 634 635 return ret; 636 } 637 638 static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data, 639 struct drm_file *file_priv) 640 { 641 struct drm_panfrost_query_bo_info *args = data; 642 struct drm_gem_object *gem_obj; 643 struct panfrost_gem_object *bo; 644 645 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 646 if (!gem_obj) { 647 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 648 return -ENOENT; 649 } 650 651 bo = to_panfrost_bo(gem_obj); 652 args->pad = 0; 653 args->create_flags = 0; 654 args->extra_flags = 0; 655 656 if (drm_gem_is_imported(gem_obj)) { 657 args->extra_flags |= DRM_PANFROST_BO_IS_IMPORTED; 658 } else { 659 if (bo->noexec) 660 args->create_flags |= PANFROST_BO_NOEXEC; 661 662 if (bo->is_heap) 663 args->create_flags |= PANFROST_BO_HEAP; 664 665 if (!bo->base.map_wc) 666 args->create_flags |= PANFROST_BO_WB_MMAP; 667 } 668 669 drm_gem_object_put(gem_obj); 670 return 0; 671 } 672 673 int panfrost_unstable_ioctl_check(void) 674 { 675 if (!unstable_ioctls) 676 return -ENOSYS; 677 678 return 0; 679 } 680 681 static int 682 panfrost_open(struct drm_device *dev, struct drm_file *file) 683 { 684 int ret; 685 struct panfrost_device *pfdev = to_panfrost_device(dev); 686 struct panfrost_file_priv *panfrost_priv; 687 688 panfrost_priv = kzalloc_obj(*panfrost_priv); 689 if (!panfrost_priv) 690 return -ENOMEM; 691 692 panfrost_priv->pfdev = pfdev; 693 file->driver_priv = panfrost_priv; 694 695 panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev); 696 if (IS_ERR(panfrost_priv->mmu)) { 697 ret = PTR_ERR(panfrost_priv->mmu); 698 goto err_free; 699 } 700 701 ret = panfrost_jm_open(file); 702 if (ret) 703 goto err_job; 704 705 return 0; 706 707 err_job: 708 panfrost_mmu_ctx_put(panfrost_priv->mmu); 709 err_free: 710 kfree(panfrost_priv); 711 return ret; 712 } 713 714 static void 715 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 716 { 717 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 718 719 panfrost_perfcnt_close(file); 720 panfrost_jm_close(file); 721 722 panfrost_mmu_ctx_put(panfrost_priv->mmu); 723 kfree(panfrost_priv); 724 } 725 726 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 727 #define PANFROST_IOCTL(n, func, flags) \ 728 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 729 730 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW), 731 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 732 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 733 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 734 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 735 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 736 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 737 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 738 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), 739 PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW), 740 PANFROST_IOCTL(JM_CTX_CREATE, jm_ctx_create, DRM_RENDER_ALLOW), 741 PANFROST_IOCTL(JM_CTX_DESTROY, jm_ctx_destroy, DRM_RENDER_ALLOW), 742 PANFROST_IOCTL(SYNC_BO, sync_bo, DRM_RENDER_ALLOW), 743 PANFROST_IOCTL(QUERY_BO_INFO, query_bo_info, DRM_RENDER_ALLOW), 744 }; 745 746 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, 747 struct panfrost_file_priv *panfrost_priv, 748 struct drm_printer *p) 749 { 750 int i; 751 752 /* 753 * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not 754 * accurate, as they only provide a rough estimation of the number of 755 * GPU cycles and CPU time spent in a given context. This is due to two 756 * different factors: 757 * - Firstly, we must consider the time the CPU and then the kernel 758 * takes to process the GPU interrupt, which means additional time and 759 * GPU cycles will be added in excess to the real figure. 760 * - Secondly, the pipelining done by the Job Manager (2 job slots per 761 * engine) implies there is no way to know exactly how much time each 762 * job spent on the GPU. 763 */ 764 765 for (i = 0; i < NUM_JOB_SLOTS - 1; i++) { 766 if (pfdev->profile_mode) { 767 drm_printf(p, "drm-engine-%s:\t%llu ns\n", 768 panfrost_engine_names[i], 769 panfrost_priv->engine_usage.elapsed_ns[i]); 770 drm_printf(p, "drm-cycles-%s:\t%llu\n", 771 panfrost_engine_names[i], 772 panfrost_priv->engine_usage.cycles[i]); 773 } 774 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n", 775 panfrost_engine_names[i], pfdev->pfdevfreq.fast_rate); 776 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n", 777 panfrost_engine_names[i], pfdev->pfdevfreq.current_frequency); 778 } 779 } 780 781 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file) 782 { 783 struct panfrost_device *pfdev = to_panfrost_device(file->minor->dev); 784 785 panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p); 786 787 drm_show_memory_stats(p, file); 788 } 789 790 static const struct file_operations panfrost_drm_driver_fops = { 791 .owner = THIS_MODULE, 792 DRM_GEM_FOPS, 793 .show_fdinfo = drm_show_fdinfo, 794 }; 795 796 #ifdef CONFIG_DEBUG_FS 797 static int panthor_gems_show(struct seq_file *m, void *data) 798 { 799 struct drm_info_node *node = m->private; 800 struct panfrost_device *pfdev = to_panfrost_device(node->minor->dev); 801 802 panfrost_gem_debugfs_print_bos(pfdev, m); 803 804 return 0; 805 } 806 807 static void show_panfrost_jm_ctx(struct panfrost_jm_ctx *jm_ctx, u32 handle, 808 struct seq_file *m) 809 { 810 struct drm_device *ddev = ((struct drm_info_node *)m->private)->minor->dev; 811 const char *prio = "UNKNOWN"; 812 813 static const char * const prios[] = { 814 [DRM_SCHED_PRIORITY_HIGH] = "HIGH", 815 [DRM_SCHED_PRIORITY_NORMAL] = "NORMAL", 816 [DRM_SCHED_PRIORITY_LOW] = "LOW", 817 }; 818 819 if (jm_ctx->slot_entity[0].priority != 820 jm_ctx->slot_entity[1].priority) 821 drm_warn(ddev, "Slot priorities should be the same in a single context"); 822 823 if (jm_ctx->slot_entity[0].priority < ARRAY_SIZE(prios)) 824 prio = prios[jm_ctx->slot_entity[0].priority]; 825 826 seq_printf(m, " JM context %u: priority %s\n", handle, prio); 827 } 828 829 static int show_file_jm_ctxs(struct panfrost_file_priv *pfile, 830 struct seq_file *m) 831 { 832 struct panfrost_jm_ctx *jm_ctx; 833 unsigned long i; 834 835 xa_lock(&pfile->jm_ctxs); 836 xa_for_each(&pfile->jm_ctxs, i, jm_ctx) { 837 jm_ctx = panfrost_jm_ctx_get(jm_ctx); 838 xa_unlock(&pfile->jm_ctxs); 839 show_panfrost_jm_ctx(jm_ctx, i, m); 840 panfrost_jm_ctx_put(jm_ctx); 841 xa_lock(&pfile->jm_ctxs); 842 } 843 xa_unlock(&pfile->jm_ctxs); 844 845 return 0; 846 } 847 848 static struct drm_info_list panthor_debugfs_list[] = { 849 {"gems", 850 panthor_gems_show, 0, NULL}, 851 }; 852 853 static int panthor_gems_debugfs_init(struct drm_minor *minor) 854 { 855 drm_debugfs_create_files(panthor_debugfs_list, 856 ARRAY_SIZE(panthor_debugfs_list), 857 minor->debugfs_root, minor); 858 859 return 0; 860 } 861 862 static int show_each_file(struct seq_file *m, void *arg) 863 { 864 struct drm_info_node *node = (struct drm_info_node *)m->private; 865 struct drm_device *ddev = node->minor->dev; 866 int (*show)(struct panfrost_file_priv *, struct seq_file *) = 867 node->info_ent->data; 868 struct drm_file *file; 869 int ret; 870 871 ret = mutex_lock_interruptible(&ddev->filelist_mutex); 872 if (ret) 873 return ret; 874 875 list_for_each_entry(file, &ddev->filelist, lhead) { 876 struct task_struct *task; 877 struct panfrost_file_priv *pfile = file->driver_priv; 878 struct pid *pid; 879 880 /* 881 * Although we have a valid reference on file->pid, that does 882 * not guarantee that the task_struct who called get_pid() is 883 * still alive (e.g. get_pid(current) => fork() => exit()). 884 * Therefore, we need to protect this ->comm access using RCU. 885 */ 886 rcu_read_lock(); 887 pid = rcu_dereference(file->pid); 888 task = pid_task(pid, PIDTYPE_TGID); 889 seq_printf(m, "client_id %8llu pid %8d command %s:\n", 890 file->client_id, pid_nr(pid), 891 task ? task->comm : "<unknown>"); 892 rcu_read_unlock(); 893 894 ret = show(pfile, m); 895 if (ret < 0) 896 break; 897 898 seq_puts(m, "\n"); 899 } 900 901 mutex_unlock(&ddev->filelist_mutex); 902 return ret; 903 } 904 905 static struct drm_info_list panfrost_sched_debugfs_list[] = { 906 { "sched_ctxs", show_each_file, 0, show_file_jm_ctxs }, 907 }; 908 909 static void panfrost_sched_debugfs_init(struct drm_minor *minor) 910 { 911 drm_debugfs_create_files(panfrost_sched_debugfs_list, 912 ARRAY_SIZE(panfrost_sched_debugfs_list), 913 minor->debugfs_root, minor); 914 } 915 916 static void panfrost_debugfs_init(struct drm_minor *minor) 917 { 918 panthor_gems_debugfs_init(minor); 919 panfrost_sched_debugfs_init(minor); 920 } 921 #endif 922 923 /* 924 * Panfrost driver version: 925 * - 1.0 - initial interface 926 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO 927 * - 1.2 - adds AFBC_FEATURES query 928 * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT 929 * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries 930 * - 1.4 - adds SET_LABEL_BO 931 * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow 932 * context creation with configurable priorities/affinity 933 * - 1.6 - adds PANFROST_BO_MAP_WB, PANFROST_IOCTL_SYNC_BO, 934 * PANFROST_IOCTL_QUERY_BO_INFO and 935 * DRM_PANFROST_PARAM_SELECTED_COHERENCY 936 */ 937 static const struct drm_driver panfrost_drm_driver = { 938 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 939 .open = panfrost_open, 940 .postclose = panfrost_postclose, 941 .show_fdinfo = panfrost_show_fdinfo, 942 .ioctls = panfrost_drm_driver_ioctls, 943 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 944 .fops = &panfrost_drm_driver_fops, 945 .name = "panfrost", 946 .desc = "panfrost DRM", 947 .major = 1, 948 .minor = 6, 949 950 .gem_create_object = panfrost_gem_create_object, 951 .gem_prime_import = panfrost_gem_prime_import, 952 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 953 #ifdef CONFIG_DEBUG_FS 954 .debugfs_init = panfrost_debugfs_init, 955 #endif 956 }; 957 958 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 959 bool panfrost_transparent_hugepage = true; 960 module_param_named(transparent_hugepage, panfrost_transparent_hugepage, bool, 0400); 961 MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)"); 962 #endif 963 964 static int panfrost_probe(struct platform_device *pdev) 965 { 966 struct panfrost_device *pfdev; 967 int err; 968 969 pfdev = devm_drm_dev_alloc(&pdev->dev, &panfrost_drm_driver, 970 struct panfrost_device, base); 971 if (IS_ERR(pfdev)) 972 return PTR_ERR(pfdev); 973 974 platform_set_drvdata(pdev, pfdev); 975 976 pfdev->comp = of_device_get_match_data(&pdev->dev); 977 if (!pfdev->comp) 978 return -ENODEV; 979 980 pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT; 981 982 mutex_init(&pfdev->shrinker_lock); 983 INIT_LIST_HEAD(&pfdev->shrinker_list); 984 985 err = panfrost_device_init(pfdev); 986 if (err) { 987 if (err != -EPROBE_DEFER) 988 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 989 goto err_out0; 990 } 991 992 pm_runtime_set_active(pfdev->base.dev); 993 pm_runtime_mark_last_busy(pfdev->base.dev); 994 pm_runtime_enable(pfdev->base.dev); 995 pm_runtime_set_autosuspend_delay(pfdev->base.dev, 50); /* ~3 frames */ 996 pm_runtime_use_autosuspend(pfdev->base.dev); 997 998 /* 999 * Register the DRM device with the core and the connectors with 1000 * sysfs 1001 */ 1002 err = drm_dev_register(&pfdev->base, 0); 1003 if (err < 0) 1004 goto err_out1; 1005 1006 err = panfrost_gem_shrinker_init(&pfdev->base); 1007 if (err) 1008 goto err_out2; 1009 1010 return 0; 1011 1012 err_out2: 1013 drm_dev_unregister(&pfdev->base); 1014 err_out1: 1015 pm_runtime_disable(pfdev->base.dev); 1016 panfrost_device_fini(pfdev); 1017 pm_runtime_set_suspended(pfdev->base.dev); 1018 err_out0: 1019 return err; 1020 } 1021 1022 static void panfrost_remove(struct platform_device *pdev) 1023 { 1024 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 1025 1026 drm_dev_unregister(&pfdev->base); 1027 panfrost_gem_shrinker_cleanup(&pfdev->base); 1028 1029 pm_runtime_get_sync(pfdev->base.dev); 1030 pm_runtime_disable(pfdev->base.dev); 1031 panfrost_device_fini(pfdev); 1032 pm_runtime_set_suspended(pfdev->base.dev); 1033 } 1034 1035 static ssize_t profiling_show(struct device *dev, 1036 struct device_attribute *attr, char *buf) 1037 { 1038 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1039 1040 return sysfs_emit(buf, "%d\n", pfdev->profile_mode); 1041 } 1042 1043 static ssize_t profiling_store(struct device *dev, 1044 struct device_attribute *attr, 1045 const char *buf, size_t len) 1046 { 1047 struct panfrost_device *pfdev = dev_get_drvdata(dev); 1048 bool value; 1049 int err; 1050 1051 err = kstrtobool(buf, &value); 1052 if (err) 1053 return err; 1054 1055 pfdev->profile_mode = value; 1056 1057 return len; 1058 } 1059 1060 static DEVICE_ATTR_RW(profiling); 1061 1062 static struct attribute *panfrost_attrs[] = { 1063 &dev_attr_profiling.attr, 1064 NULL, 1065 }; 1066 1067 ATTRIBUTE_GROUPS(panfrost); 1068 1069 /* 1070 * The OPP core wants the supply names to be NULL terminated, but we need the 1071 * correct num_supplies value for regulator core. Hence, we NULL terminate here 1072 * and then initialize num_supplies with ARRAY_SIZE - 1. 1073 */ 1074 static const char * const default_supplies[] = { "mali", NULL }; 1075 static const struct panfrost_compatible default_data = { 1076 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1077 .supply_names = default_supplies, 1078 .num_pm_domains = 1, /* optional */ 1079 .pm_domain_names = NULL, 1080 }; 1081 1082 static const struct panfrost_compatible default_pm_rt_data = { 1083 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1084 .supply_names = default_supplies, 1085 .num_pm_domains = 1, 1086 .pm_features = BIT(GPU_PM_RT), 1087 }; 1088 1089 static const struct panfrost_compatible amlogic_data = { 1090 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1091 .supply_names = default_supplies, 1092 .vendor_quirk = panfrost_gpu_amlogic_quirk, 1093 }; 1094 1095 static const char * const mediatek_pm_domains[] = { "core0", "core1", "core2", 1096 "core3", "core4" }; 1097 /* 1098 * The old data with two power supplies for MT8183 is here only to 1099 * keep retro-compatibility with older devicetrees, as DVFS will 1100 * not work with this one. 1101 * 1102 * On new devicetrees please use the _b variant with a single and 1103 * coupled regulators instead. 1104 */ 1105 static const char * const legacy_supplies[] = { "mali", "sram", NULL }; 1106 static const struct panfrost_compatible mediatek_mt8183_data = { 1107 .num_supplies = ARRAY_SIZE(legacy_supplies) - 1, 1108 .supply_names = legacy_supplies, 1109 .num_pm_domains = 3, 1110 .pm_domain_names = mediatek_pm_domains, 1111 }; 1112 1113 static const struct panfrost_compatible mediatek_mt8183_b_data = { 1114 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1115 .supply_names = default_supplies, 1116 .num_pm_domains = 3, 1117 .pm_domain_names = mediatek_pm_domains, 1118 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1119 }; 1120 1121 static const struct panfrost_compatible mediatek_mt8186_data = { 1122 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1123 .supply_names = default_supplies, 1124 .num_pm_domains = 2, 1125 .pm_domain_names = mediatek_pm_domains, 1126 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1127 }; 1128 1129 static const struct panfrost_compatible mediatek_mt8188_data = { 1130 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1131 .supply_names = default_supplies, 1132 .num_pm_domains = 3, 1133 .pm_domain_names = mediatek_pm_domains, 1134 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1135 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1136 }; 1137 1138 static const struct panfrost_compatible mediatek_mt8192_data = { 1139 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1140 .supply_names = default_supplies, 1141 .num_pm_domains = 5, 1142 .pm_domain_names = mediatek_pm_domains, 1143 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1144 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1145 }; 1146 1147 static const struct panfrost_compatible mediatek_mt8370_data = { 1148 .num_supplies = ARRAY_SIZE(default_supplies) - 1, 1149 .supply_names = default_supplies, 1150 .num_pm_domains = 2, 1151 .pm_domain_names = mediatek_pm_domains, 1152 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF), 1153 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE), 1154 }; 1155 1156 static const struct of_device_id dt_match[] = { 1157 /* Set first to probe before the generic compatibles */ 1158 { .compatible = "amlogic,meson-gxm-mali", 1159 .data = &amlogic_data, }, 1160 { .compatible = "amlogic,meson-g12a-mali", 1161 .data = &amlogic_data, }, 1162 { .compatible = "renesas,r9a08g046-mali", .data = &default_pm_rt_data }, 1163 { .compatible = "renesas,r9a09g047-mali", .data = &default_pm_rt_data }, 1164 { .compatible = "arm,mali-t604", .data = &default_data, }, 1165 { .compatible = "arm,mali-t624", .data = &default_data, }, 1166 { .compatible = "arm,mali-t628", .data = &default_data, }, 1167 { .compatible = "arm,mali-t720", .data = &default_data, }, 1168 { .compatible = "arm,mali-t760", .data = &default_data, }, 1169 { .compatible = "arm,mali-t820", .data = &default_data, }, 1170 { .compatible = "arm,mali-t830", .data = &default_data, }, 1171 { .compatible = "arm,mali-t860", .data = &default_data, }, 1172 { .compatible = "arm,mali-t880", .data = &default_data, }, 1173 { .compatible = "arm,mali-bifrost", .data = &default_data, }, 1174 { .compatible = "arm,mali-valhall-jm", .data = &default_data, }, 1175 { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data }, 1176 { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data }, 1177 { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data }, 1178 { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data }, 1179 { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data }, 1180 { .compatible = "mediatek,mt8370-mali", .data = &mediatek_mt8370_data }, 1181 { .compatible = "allwinner,sun50i-h616-mali", .data = &default_pm_rt_data }, 1182 {} 1183 }; 1184 MODULE_DEVICE_TABLE(of, dt_match); 1185 1186 static struct platform_driver panfrost_driver = { 1187 .probe = panfrost_probe, 1188 .remove = panfrost_remove, 1189 .driver = { 1190 .name = "panfrost", 1191 .pm = pm_ptr(&panfrost_pm_ops), 1192 .of_match_table = dt_match, 1193 .dev_groups = panfrost_groups, 1194 }, 1195 }; 1196 module_platform_driver(panfrost_driver); 1197 1198 MODULE_AUTHOR("Panfrost Project Developers"); 1199 MODULE_DESCRIPTION("Panfrost DRM Driver"); 1200 MODULE_LICENSE("GPL v2"); 1201 MODULE_SOFTDEP("pre: governor_simpleondemand"); 1202