1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 3 /* Copyright 2023 Collabora ltd. */ 4 /* Copyright 2025 Amazon.com, Inc. or its affiliates */ 5 /* Copyright 2025 ARM Limited. All rights reserved. */ 6 7 #include <linux/cleanup.h> 8 #include <linux/debugfs.h> 9 #include <linux/dma-buf.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/slab.h> 13 #include <linux/vmalloc.h> 14 15 #include <drm/drm_debugfs.h> 16 #include <drm/drm_file.h> 17 #include <drm/drm_gpuvm.h> 18 #include <drm/drm_managed.h> 19 #include <drm/drm_prime.h> 20 #include <drm/drm_print.h> 21 #include <drm/panthor_drm.h> 22 23 #include "panthor_device.h" 24 #include "panthor_drv.h" 25 #include "panthor_fw.h" 26 #include "panthor_gem.h" 27 #include "panthor_mmu.h" 28 29 void panthor_gem_init(struct panthor_device *ptdev) 30 { 31 int err; 32 33 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 34 !panthor_transparent_hugepage) 35 return; 36 37 err = drm_gem_huge_mnt_create(&ptdev->base, "within_size"); 38 if (drm_gem_get_huge_mnt(&ptdev->base)) 39 drm_info(&ptdev->base, "Using Transparent Hugepage\n"); 40 else if (err) 41 drm_warn(&ptdev->base, "Can't use Transparent Hugepage (%d)\n", 42 err); 43 } 44 45 #ifdef CONFIG_DEBUG_FS 46 static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo) 47 { 48 INIT_LIST_HEAD(&bo->debugfs.node); 49 } 50 51 static void panthor_gem_debugfs_bo_add(struct panthor_gem_object *bo) 52 { 53 struct panthor_device *ptdev = container_of(bo->base.dev, 54 struct panthor_device, base); 55 56 bo->debugfs.creator.tgid = current->tgid; 57 get_task_comm(bo->debugfs.creator.process_name, current->group_leader); 58 59 mutex_lock(&ptdev->gems.lock); 60 list_add_tail(&bo->debugfs.node, &ptdev->gems.node); 61 mutex_unlock(&ptdev->gems.lock); 62 } 63 64 static void panthor_gem_debugfs_bo_rm(struct panthor_gem_object *bo) 65 { 66 struct panthor_device *ptdev = container_of(bo->base.dev, 67 struct panthor_device, base); 68 69 if (list_empty(&bo->debugfs.node)) 70 return; 71 72 mutex_lock(&ptdev->gems.lock); 73 list_del_init(&bo->debugfs.node); 74 mutex_unlock(&ptdev->gems.lock); 75 } 76 77 static void panthor_gem_debugfs_set_usage_flags(struct panthor_gem_object *bo, u32 usage_flags) 78 { 79 bo->debugfs.flags = usage_flags; 80 panthor_gem_debugfs_bo_add(bo); 81 } 82 #else 83 static void panthor_gem_debugfs_bo_rm(struct panthor_gem_object *bo) {} 84 static void panthor_gem_debugfs_set_usage_flags(struct panthor_gem_object *bo, u32 usage_flags) {} 85 static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo) {} 86 #endif 87 88 static bool 89 should_map_wc(struct panthor_gem_object *bo) 90 { 91 struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base); 92 93 /* We can't do uncached mappings if the device is coherent, 94 * because the zeroing done by the shmem layer at page allocation 95 * time happens on a cached mapping which isn't CPU-flushed (at least 96 * not on Arm64 where the flush is deferred to PTE setup time, and 97 * only done conditionally based on the mapping permissions). We can't 98 * rely on dma_map_sgtable()/dma_sync_sgtable_for_xxx() either to flush 99 * those, because they are NOPed if dma_dev_coherent() returns true. 100 * 101 * FIXME: Note that this problem is going to pop up again when we 102 * decide to support mapping buffers with the NO_MMAP flag as 103 * non-shareable (AKA buffers accessed only by the GPU), because we 104 * need the same CPU flush to happen after page allocation, otherwise 105 * there's a risk of data leak or late corruption caused by a dirty 106 * cacheline being evicted. At this point we'll need a way to force 107 * CPU cache maintenance regardless of whether the device is coherent 108 * or not. 109 */ 110 if (ptdev->coherent) 111 return false; 112 113 /* Cached mappings are explicitly requested, so no write-combine. */ 114 if (bo->flags & DRM_PANTHOR_BO_WB_MMAP) 115 return false; 116 117 /* The default is write-combine. */ 118 return true; 119 } 120 121 static bool is_gpu_mapped(struct panthor_gem_object *bo, 122 enum panthor_gem_reclaim_state *state) 123 { 124 struct drm_gpuvm_bo *vm_bo; 125 u32 vm_count = 0; 126 127 drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) { 128 /* Skip evicted GPU mappings. */ 129 if (vm_bo->evicted) 130 continue; 131 132 if (vm_count++) { 133 *state = PANTHOR_GEM_GPU_MAPPED_MULTI_VM; 134 break; 135 } 136 137 *state = PANTHOR_GEM_GPU_MAPPED_SINGLE_VM; 138 } 139 140 return vm_count > 0; 141 } 142 143 static enum panthor_gem_reclaim_state 144 panthor_gem_evaluate_reclaim_state_locked(struct panthor_gem_object *bo) 145 { 146 enum panthor_gem_reclaim_state gpu_mapped_state; 147 148 dma_resv_assert_held(bo->base.resv); 149 lockdep_assert_held(&bo->base.gpuva.lock); 150 151 /* If pages have not been allocated, there's nothing to reclaim. */ 152 if (!bo->backing.pages) 153 return PANTHOR_GEM_UNRECLAIMABLE; 154 155 /* If memory is pinned, we prevent reclaim. */ 156 if (refcount_read(&bo->backing.pin_count)) 157 return PANTHOR_GEM_UNRECLAIMABLE; 158 159 if (is_gpu_mapped(bo, &gpu_mapped_state)) 160 return gpu_mapped_state; 161 162 if (refcount_read(&bo->cmap.mmap_count)) 163 return PANTHOR_GEM_MMAPPED; 164 165 return PANTHOR_GEM_UNUSED; 166 } 167 168 void panthor_gem_update_reclaim_state_locked(struct panthor_gem_object *bo, 169 enum panthor_gem_reclaim_state *old_statep) 170 { 171 struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base); 172 enum panthor_gem_reclaim_state old_state = bo->reclaim_state; 173 enum panthor_gem_reclaim_state new_state; 174 bool was_gpu_mapped, is_gpu_mapped; 175 176 if (old_statep) 177 *old_statep = old_state; 178 179 new_state = panthor_gem_evaluate_reclaim_state_locked(bo); 180 if (new_state == old_state) 181 return; 182 183 was_gpu_mapped = old_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM || 184 old_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM; 185 is_gpu_mapped = new_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM || 186 new_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM; 187 188 if (is_gpu_mapped && !was_gpu_mapped) 189 ptdev->reclaim.gpu_mapped_count += bo->base.size >> PAGE_SHIFT; 190 else if (!is_gpu_mapped && was_gpu_mapped) 191 ptdev->reclaim.gpu_mapped_count -= bo->base.size >> PAGE_SHIFT; 192 193 switch (new_state) { 194 case PANTHOR_GEM_UNUSED: 195 drm_gem_lru_move_tail(&ptdev->reclaim.unused, &bo->base); 196 break; 197 case PANTHOR_GEM_MMAPPED: 198 drm_gem_lru_move_tail(&ptdev->reclaim.mmapped, &bo->base); 199 break; 200 case PANTHOR_GEM_GPU_MAPPED_SINGLE_VM: 201 panthor_vm_update_bo_reclaim_lru_locked(bo); 202 break; 203 case PANTHOR_GEM_GPU_MAPPED_MULTI_VM: 204 drm_gem_lru_move_tail(&ptdev->reclaim.gpu_mapped_shared, &bo->base); 205 break; 206 case PANTHOR_GEM_UNRECLAIMABLE: 207 drm_gem_lru_remove(&bo->base); 208 break; 209 default: 210 drm_WARN(&ptdev->base, true, "invalid GEM reclaim state (%d)\n", new_state); 211 break; 212 } 213 214 bo->reclaim_state = new_state; 215 } 216 217 static void 218 bo_assert_locked_or_gone(struct panthor_gem_object *bo) 219 { 220 /* If the refcount is zero, the BO is being freed, and we 221 * allow the lock to not be held in that particular case. 222 */ 223 if (kref_read(&bo->base.refcount)) 224 dma_resv_assert_held(bo->base.resv); 225 } 226 227 static void 228 panthor_gem_backing_cleanup_locked(struct panthor_gem_object *bo) 229 { 230 bo_assert_locked_or_gone(bo); 231 232 if (!bo->backing.pages) 233 return; 234 235 drm_gem_put_pages(&bo->base, bo->backing.pages, true, false); 236 bo->backing.pages = NULL; 237 } 238 239 static int 240 panthor_gem_backing_get_pages_locked(struct panthor_gem_object *bo) 241 { 242 struct page **pages; 243 244 dma_resv_assert_held(bo->base.resv); 245 246 if (bo->backing.pages) 247 return 0; 248 249 pages = drm_gem_get_pages(&bo->base); 250 if (IS_ERR(pages)) { 251 drm_dbg_driver(bo->base.dev, "Failed to get pages (%pe)\n", pages); 252 return PTR_ERR(pages); 253 } 254 255 bo->backing.pages = pages; 256 return 0; 257 } 258 259 static int panthor_gem_backing_pin_locked(struct panthor_gem_object *bo) 260 { 261 int ret; 262 263 dma_resv_assert_held(bo->base.resv); 264 drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)); 265 266 if (refcount_inc_not_zero(&bo->backing.pin_count)) 267 return 0; 268 269 ret = panthor_gem_backing_get_pages_locked(bo); 270 if (!ret) { 271 refcount_set(&bo->backing.pin_count, 1); 272 mutex_lock(&bo->base.gpuva.lock); 273 panthor_gem_update_reclaim_state_locked(bo, NULL); 274 mutex_unlock(&bo->base.gpuva.lock); 275 } 276 277 return ret; 278 } 279 280 static void panthor_gem_backing_unpin_locked(struct panthor_gem_object *bo) 281 { 282 bo_assert_locked_or_gone(bo); 283 drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)); 284 285 if (refcount_dec_and_test(&bo->backing.pin_count)) { 286 /* We don't release anything when pin_count drops to zero. 287 * Pages stay there until an explicit cleanup is requested. 288 */ 289 mutex_lock(&bo->base.gpuva.lock); 290 panthor_gem_update_reclaim_state_locked(bo, NULL); 291 mutex_unlock(&bo->base.gpuva.lock); 292 } 293 } 294 295 static void 296 panthor_gem_dev_map_cleanup_locked(struct panthor_gem_object *bo) 297 { 298 bo_assert_locked_or_gone(bo); 299 300 if (!bo->dmap.sgt) 301 return; 302 303 dma_unmap_sgtable(drm_dev_dma_dev(bo->base.dev), bo->dmap.sgt, DMA_BIDIRECTIONAL, 0); 304 sg_free_table(bo->dmap.sgt); 305 kfree(bo->dmap.sgt); 306 bo->dmap.sgt = NULL; 307 } 308 309 static struct sg_table * 310 panthor_gem_dev_map_get_sgt_locked(struct panthor_gem_object *bo) 311 { 312 struct sg_table *sgt; 313 int ret; 314 315 dma_resv_assert_held(bo->base.resv); 316 317 if (bo->dmap.sgt) 318 return bo->dmap.sgt; 319 320 ret = panthor_gem_backing_get_pages_locked(bo); 321 if (ret) 322 return ERR_PTR(ret); 323 324 sgt = drm_prime_pages_to_sg(bo->base.dev, bo->backing.pages, 325 bo->base.size >> PAGE_SHIFT); 326 if (IS_ERR(sgt)) 327 return sgt; 328 329 /* Map the pages for use by the h/w. */ 330 ret = dma_map_sgtable(drm_dev_dma_dev(bo->base.dev), sgt, DMA_BIDIRECTIONAL, 0); 331 if (ret) 332 goto err_free_sgt; 333 334 bo->dmap.sgt = sgt; 335 return sgt; 336 337 err_free_sgt: 338 sg_free_table(sgt); 339 kfree(sgt); 340 return ERR_PTR(ret); 341 } 342 343 struct sg_table * 344 panthor_gem_get_dev_sgt(struct panthor_gem_object *bo) 345 { 346 struct sg_table *sgt; 347 348 dma_resv_lock(bo->base.resv, NULL); 349 sgt = panthor_gem_dev_map_get_sgt_locked(bo); 350 dma_resv_unlock(bo->base.resv); 351 352 return sgt; 353 } 354 355 static void 356 panthor_gem_vmap_cleanup_locked(struct panthor_gem_object *bo) 357 { 358 if (!bo->cmap.vaddr) 359 return; 360 361 vunmap(bo->cmap.vaddr); 362 bo->cmap.vaddr = NULL; 363 panthor_gem_backing_unpin_locked(bo); 364 } 365 366 static int 367 panthor_gem_prep_for_cpu_map_locked(struct panthor_gem_object *bo) 368 { 369 if (should_map_wc(bo)) { 370 struct sg_table *sgt; 371 372 sgt = panthor_gem_dev_map_get_sgt_locked(bo); 373 if (IS_ERR(sgt)) 374 return PTR_ERR(sgt); 375 } 376 377 return 0; 378 } 379 380 static void * 381 panthor_gem_vmap_get_locked(struct panthor_gem_object *bo) 382 { 383 pgprot_t prot = PAGE_KERNEL; 384 void *vaddr; 385 int ret; 386 387 dma_resv_assert_held(bo->base.resv); 388 389 if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base))) 390 return ERR_PTR(-EINVAL); 391 392 if (refcount_inc_not_zero(&bo->cmap.vaddr_use_count)) { 393 drm_WARN_ON_ONCE(bo->base.dev, !bo->cmap.vaddr); 394 return bo->cmap.vaddr; 395 } 396 397 ret = panthor_gem_backing_pin_locked(bo); 398 if (ret) 399 return ERR_PTR(ret); 400 401 ret = panthor_gem_prep_for_cpu_map_locked(bo); 402 if (ret) 403 goto err_unpin; 404 405 if (should_map_wc(bo)) 406 prot = pgprot_writecombine(prot); 407 408 vaddr = vmap(bo->backing.pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot); 409 if (!vaddr) { 410 ret = -ENOMEM; 411 goto err_unpin; 412 } 413 414 bo->cmap.vaddr = vaddr; 415 refcount_set(&bo->cmap.vaddr_use_count, 1); 416 return vaddr; 417 418 err_unpin: 419 panthor_gem_backing_unpin_locked(bo); 420 return ERR_PTR(ret); 421 } 422 423 static void 424 panthor_gem_vmap_put_locked(struct panthor_gem_object *bo) 425 { 426 dma_resv_assert_held(bo->base.resv); 427 428 if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base))) 429 return; 430 431 if (refcount_dec_and_test(&bo->cmap.vaddr_use_count)) 432 panthor_gem_vmap_cleanup_locked(bo); 433 } 434 435 static void panthor_gem_free_object(struct drm_gem_object *obj) 436 { 437 struct panthor_gem_object *bo = to_panthor_bo(obj); 438 struct drm_gem_object *vm_root_gem = bo->exclusive_vm_root_gem; 439 440 panthor_gem_debugfs_bo_rm(bo); 441 442 /* 443 * Label might have been allocated with kstrdup_const(), 444 * we need to take that into account when freeing the memory 445 */ 446 kfree_const(bo->label.str); 447 448 mutex_destroy(&bo->label.lock); 449 450 if (drm_gem_is_imported(obj)) { 451 drm_prime_gem_destroy(obj, bo->dmap.sgt); 452 } else { 453 /* The last ref on the GEM object can be released 454 * by the shrinker, which can't block on the resv 455 * lock acquisition. In practice, even if we were 456 * taking the lock, it wouldn't block because we're 457 * the last piece of code having a visibility on 458 * this GEM, but lockdep can't see that, so we've 459 * just tought the _cleanup_locked() helpers about 460 * this "being freed" exception, and we call those 461 * without the lock held here. 462 */ 463 panthor_gem_vmap_cleanup_locked(bo); 464 panthor_gem_dev_map_cleanup_locked(bo); 465 panthor_gem_backing_cleanup_locked(bo); 466 } 467 468 drm_gem_object_release(obj); 469 470 kfree(bo); 471 drm_gem_object_put(vm_root_gem); 472 } 473 474 static struct sg_table * 475 panthor_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, 476 enum dma_data_direction dir) 477 { 478 struct sg_table *sgt = drm_gem_map_dma_buf(attach, dir); 479 480 if (!IS_ERR(sgt)) 481 attach->priv = sgt; 482 483 return sgt; 484 } 485 486 static void 487 panthor_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, 488 struct sg_table *sgt, 489 enum dma_data_direction dir) 490 { 491 attach->priv = NULL; 492 drm_gem_unmap_dma_buf(attach, sgt, dir); 493 } 494 495 static int 496 panthor_gem_prime_begin_cpu_access(struct dma_buf *dma_buf, 497 enum dma_data_direction dir) 498 { 499 struct drm_gem_object *obj = dma_buf->priv; 500 struct drm_device *dev = obj->dev; 501 struct panthor_gem_object *bo = to_panthor_bo(obj); 502 struct dma_buf_attachment *attach; 503 504 dma_resv_lock(obj->resv, NULL); 505 if (bo->dmap.sgt) 506 dma_sync_sgtable_for_cpu(drm_dev_dma_dev(dev), bo->dmap.sgt, dir); 507 508 if (bo->cmap.vaddr) 509 invalidate_kernel_vmap_range(bo->cmap.vaddr, bo->base.size); 510 511 list_for_each_entry(attach, &dma_buf->attachments, node) { 512 struct sg_table *sgt = attach->priv; 513 514 if (sgt) 515 dma_sync_sgtable_for_cpu(attach->dev, sgt, dir); 516 } 517 dma_resv_unlock(obj->resv); 518 519 return 0; 520 } 521 522 static int 523 panthor_gem_prime_end_cpu_access(struct dma_buf *dma_buf, 524 enum dma_data_direction dir) 525 { 526 struct drm_gem_object *obj = dma_buf->priv; 527 struct drm_device *dev = obj->dev; 528 struct panthor_gem_object *bo = to_panthor_bo(obj); 529 struct dma_buf_attachment *attach; 530 531 dma_resv_lock(obj->resv, NULL); 532 list_for_each_entry(attach, &dma_buf->attachments, node) { 533 struct sg_table *sgt = attach->priv; 534 535 if (sgt) 536 dma_sync_sgtable_for_device(attach->dev, sgt, dir); 537 } 538 539 if (bo->cmap.vaddr) 540 flush_kernel_vmap_range(bo->cmap.vaddr, bo->base.size); 541 542 if (bo->dmap.sgt) 543 dma_sync_sgtable_for_device(drm_dev_dma_dev(dev), bo->dmap.sgt, dir); 544 545 dma_resv_unlock(obj->resv); 546 return 0; 547 } 548 549 static const struct dma_buf_ops panthor_dma_buf_ops = { 550 .attach = drm_gem_map_attach, 551 .detach = drm_gem_map_detach, 552 .map_dma_buf = panthor_gem_prime_map_dma_buf, 553 .unmap_dma_buf = panthor_gem_prime_unmap_dma_buf, 554 .release = drm_gem_dmabuf_release, 555 .mmap = drm_gem_dmabuf_mmap, 556 .vmap = drm_gem_dmabuf_vmap, 557 .vunmap = drm_gem_dmabuf_vunmap, 558 .begin_cpu_access = panthor_gem_prime_begin_cpu_access, 559 .end_cpu_access = panthor_gem_prime_end_cpu_access, 560 }; 561 562 static struct dma_buf * 563 panthor_gem_prime_export(struct drm_gem_object *obj, int flags) 564 { 565 struct drm_device *dev = obj->dev; 566 struct dma_buf_export_info exp_info = { 567 .exp_name = KBUILD_MODNAME, 568 .owner = THIS_MODULE, 569 .ops = &panthor_dma_buf_ops, 570 .size = obj->size, 571 .flags = flags, 572 .priv = obj, 573 .resv = obj->resv, 574 }; 575 576 /* We can't export GEMs that have an exclusive VM. */ 577 if (to_panthor_bo(obj)->exclusive_vm_root_gem) 578 return ERR_PTR(-EINVAL); 579 580 return drm_gem_dmabuf_export(dev, &exp_info); 581 } 582 583 struct drm_gem_object * 584 panthor_gem_prime_import(struct drm_device *dev, 585 struct dma_buf *dma_buf) 586 { 587 struct drm_gem_object *obj = dma_buf->priv; 588 589 if (dma_buf->ops == &panthor_dma_buf_ops && obj->dev == dev) { 590 /* Importing dmabuf exported from our own gem increases 591 * refcount on gem itself instead of f_count of dmabuf. 592 */ 593 drm_gem_object_get(obj); 594 return obj; 595 } 596 597 return drm_gem_prime_import(dev, dma_buf); 598 } 599 600 static void panthor_gem_print_info(struct drm_printer *p, unsigned int indent, 601 const struct drm_gem_object *obj) 602 { 603 const struct panthor_gem_object *bo = to_panthor_bo(obj); 604 605 if (drm_gem_is_imported(&bo->base)) 606 return; 607 608 drm_printf_indent(p, indent, "resident=%s\n", str_true_false(bo->backing.pages)); 609 drm_printf_indent(p, indent, "pages_pin_count=%u\n", refcount_read(&bo->backing.pin_count)); 610 drm_printf_indent(p, indent, "vmap_use_count=%u\n", 611 refcount_read(&bo->cmap.vaddr_use_count)); 612 drm_printf_indent(p, indent, "vaddr=%p\n", bo->cmap.vaddr); 613 drm_printf_indent(p, indent, "mmap_count=%u\n", refcount_read(&bo->cmap.mmap_count)); 614 } 615 616 static int panthor_gem_pin_locked(struct drm_gem_object *obj) 617 { 618 if (!drm_gem_is_imported(obj)) 619 return panthor_gem_backing_pin_locked(to_panthor_bo(obj)); 620 621 return 0; 622 } 623 624 static void panthor_gem_unpin_locked(struct drm_gem_object *obj) 625 { 626 if (!drm_gem_is_imported(obj)) 627 panthor_gem_backing_unpin_locked(to_panthor_bo(obj)); 628 } 629 630 int panthor_gem_pin(struct panthor_gem_object *bo) 631 { 632 int ret = 0; 633 634 if (drm_gem_is_imported(&bo->base)) 635 return 0; 636 637 if (refcount_inc_not_zero(&bo->backing.pin_count)) 638 return 0; 639 640 dma_resv_lock(bo->base.resv, NULL); 641 ret = panthor_gem_backing_pin_locked(bo); 642 dma_resv_unlock(bo->base.resv); 643 644 return ret; 645 } 646 647 void panthor_gem_unpin(struct panthor_gem_object *bo) 648 { 649 if (drm_gem_is_imported(&bo->base)) 650 return; 651 652 if (refcount_dec_not_one(&bo->backing.pin_count)) 653 return; 654 655 dma_resv_lock(bo->base.resv, NULL); 656 panthor_gem_backing_unpin_locked(bo); 657 dma_resv_unlock(bo->base.resv); 658 } 659 660 int panthor_gem_swapin_locked(struct panthor_gem_object *bo) 661 { 662 struct sg_table *sgt; 663 664 dma_resv_assert_held(bo->base.resv); 665 666 if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base))) 667 return -EINVAL; 668 669 sgt = panthor_gem_dev_map_get_sgt_locked(bo); 670 if (IS_ERR(sgt)) 671 return PTR_ERR(sgt); 672 673 return 0; 674 } 675 676 static void panthor_gem_evict_locked(struct panthor_gem_object *bo) 677 { 678 dma_resv_assert_held(bo->base.resv); 679 lockdep_assert_held(&bo->base.gpuva.lock); 680 681 if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base))) 682 return; 683 684 if (drm_WARN_ON_ONCE(bo->base.dev, refcount_read(&bo->backing.pin_count))) 685 return; 686 687 if (drm_WARN_ON_ONCE(bo->base.dev, !bo->backing.pages)) 688 return; 689 690 atomic_add_unless(&bo->reclaimed_count, 1, INT_MAX); 691 692 panthor_gem_dev_map_cleanup_locked(bo); 693 panthor_gem_backing_cleanup_locked(bo); 694 panthor_gem_update_reclaim_state_locked(bo, NULL); 695 } 696 697 static struct sg_table *panthor_gem_get_sg_table(struct drm_gem_object *obj) 698 { 699 struct panthor_gem_object *bo = to_panthor_bo(obj); 700 701 drm_WARN_ON_ONCE(obj->dev, drm_gem_is_imported(obj)); 702 drm_WARN_ON_ONCE(obj->dev, !bo->backing.pages); 703 drm_WARN_ON_ONCE(obj->dev, !refcount_read(&bo->backing.pin_count)); 704 705 return drm_prime_pages_to_sg(obj->dev, bo->backing.pages, obj->size >> PAGE_SHIFT); 706 } 707 708 static int panthor_gem_vmap_locked(struct drm_gem_object *obj, 709 struct iosys_map *map) 710 { 711 struct panthor_gem_object *bo = to_panthor_bo(obj); 712 void *vaddr; 713 714 dma_resv_assert_held(obj->resv); 715 716 if (drm_gem_is_imported(obj)) 717 return dma_buf_vmap(obj->import_attach->dmabuf, map); 718 719 vaddr = panthor_gem_vmap_get_locked(bo); 720 if (IS_ERR(vaddr)) 721 return PTR_ERR(vaddr); 722 723 iosys_map_set_vaddr(map, vaddr); 724 return 0; 725 } 726 727 static void panthor_gem_vunmap_locked(struct drm_gem_object *obj, 728 struct iosys_map *map) 729 { 730 struct panthor_gem_object *bo = to_panthor_bo(obj); 731 732 dma_resv_assert_held(obj->resv); 733 734 if (drm_gem_is_imported(obj)) { 735 dma_buf_vunmap(obj->import_attach->dmabuf, map); 736 } else { 737 drm_WARN_ON_ONCE(obj->dev, bo->cmap.vaddr != map->vaddr); 738 panthor_gem_vmap_put_locked(bo); 739 } 740 } 741 742 static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 743 { 744 struct panthor_gem_object *bo = to_panthor_bo(obj); 745 int ret; 746 747 if (bo->flags & DRM_PANTHOR_BO_NO_MMAP) 748 return -EINVAL; 749 750 if (drm_gem_is_imported(obj)) { 751 /* Reset both vm_ops and vm_private_data, so we don't end up with 752 * vm_ops pointing to our implementation if the dma-buf backend 753 * doesn't set those fields. 754 */ 755 vma->vm_private_data = NULL; 756 vma->vm_ops = NULL; 757 758 ret = dma_buf_mmap(obj->dma_buf, vma, 0); 759 760 /* Drop the reference drm_gem_mmap_obj() acquired.*/ 761 if (!ret) 762 drm_gem_object_put(obj); 763 764 return ret; 765 } 766 767 if (vma_is_cow_mapping(vma)) 768 return -EINVAL; 769 770 if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) { 771 dma_resv_lock(obj->resv, NULL); 772 if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) { 773 refcount_set(&bo->cmap.mmap_count, 1); 774 mutex_lock(&bo->base.gpuva.lock); 775 panthor_gem_update_reclaim_state_locked(bo, NULL); 776 mutex_unlock(&bo->base.gpuva.lock); 777 } 778 dma_resv_unlock(obj->resv); 779 } 780 781 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 782 vma->vm_page_prot = vma_get_page_prot(vma); 783 if (should_map_wc(bo)) 784 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 785 786 return 0; 787 } 788 789 static enum drm_gem_object_status panthor_gem_status(struct drm_gem_object *obj) 790 { 791 struct panthor_gem_object *bo = to_panthor_bo(obj); 792 enum drm_gem_object_status res = 0; 793 794 if (drm_gem_is_imported(&bo->base) || bo->backing.pages) 795 res |= DRM_GEM_OBJECT_RESIDENT; 796 797 return res; 798 } 799 800 static vm_fault_t insert_page(struct vm_fault *vmf, unsigned int order, struct page *page) 801 { 802 if (!order) { 803 return vmf_insert_pfn(vmf->vma, vmf->address, page_to_pfn(page)); 804 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP 805 } else if (order == PMD_ORDER) { 806 unsigned long pfn = page_to_pfn(page); 807 unsigned long paddr = pfn << PAGE_SHIFT; 808 struct vm_area_struct *vma = vmf->vma; 809 unsigned long start = ALIGN_DOWN(vmf->address, PMD_SIZE); 810 unsigned long end = start + PMD_SIZE; 811 bool in_range = vma->vm_start <= start && end <= vma->vm_end; 812 bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK); 813 814 if (aligned && in_range && 815 folio_test_pmd_mappable(page_folio(page))) { 816 pfn &= PMD_MASK >> PAGE_SHIFT; 817 return vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); 818 } 819 #endif 820 } 821 822 return VM_FAULT_FALLBACK; 823 } 824 825 static vm_fault_t nonblocking_page_setup(struct vm_fault *vmf, 826 unsigned int order, 827 pgoff_t page_offset) 828 { 829 struct vm_area_struct *vma = vmf->vma; 830 struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data); 831 vm_fault_t ret; 832 833 if (!dma_resv_trylock(bo->base.resv)) 834 return VM_FAULT_RETRY; 835 836 if (bo->backing.pages) 837 ret = insert_page(vmf, order, bo->backing.pages[page_offset]); 838 else 839 ret = VM_FAULT_RETRY; 840 841 dma_resv_unlock(bo->base.resv); 842 return ret; 843 } 844 845 static vm_fault_t blocking_page_setup(struct vm_fault *vmf, unsigned int order, 846 struct panthor_gem_object *bo, 847 pgoff_t page_offset, bool mmap_lock_held) 848 { 849 vm_fault_t ret; 850 int err; 851 852 err = dma_resv_lock_interruptible(bo->base.resv, NULL); 853 if (err) 854 return mmap_lock_held ? VM_FAULT_NOPAGE : VM_FAULT_RETRY; 855 856 err = panthor_gem_backing_get_pages_locked(bo); 857 if (!err) 858 err = panthor_gem_prep_for_cpu_map_locked(bo); 859 860 if (err) { 861 ret = mmap_lock_held ? VM_FAULT_SIGBUS : VM_FAULT_RETRY; 862 } else { 863 struct page *page = bo->backing.pages[page_offset]; 864 865 mutex_lock(&bo->base.gpuva.lock); 866 panthor_gem_update_reclaim_state_locked(bo, NULL); 867 mutex_unlock(&bo->base.gpuva.lock); 868 869 if (mmap_lock_held) 870 ret = insert_page(vmf, order, page); 871 else 872 ret = VM_FAULT_RETRY; 873 } 874 875 dma_resv_unlock(bo->base.resv); 876 877 return ret; 878 } 879 880 static vm_fault_t panthor_gem_any_fault(struct vm_fault *vmf, unsigned int order) 881 { 882 struct vm_area_struct *vma = vmf->vma; 883 struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data); 884 loff_t num_pages = bo->base.size >> PAGE_SHIFT; 885 pgoff_t page_offset; 886 vm_fault_t ret; 887 888 if (order && order != PMD_ORDER) 889 return VM_FAULT_FALLBACK; 890 891 /* Offset to faulty address in the VMA. */ 892 page_offset = vmf->pgoff - vma->vm_pgoff; 893 if (page_offset >= num_pages) 894 return VM_FAULT_SIGBUS; 895 896 ret = nonblocking_page_setup(vmf, order, page_offset); 897 if (ret != VM_FAULT_RETRY) 898 return ret; 899 900 /* Check if we're allowed to retry. */ 901 if (fault_flag_allow_retry_first(vmf->flags)) { 902 /* If we're allowed to retry but not wait here, return 903 * immediately, the wait will be done when the fault 904 * handler is called again, with the mmap_lock held. 905 */ 906 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 907 return VM_FAULT_RETRY; 908 909 /* Wait with the mmap lock released, if we're allowed to. */ 910 drm_gem_object_get(&bo->base); 911 912 if (vmf->flags & FAULT_FLAG_VMA_LOCK) 913 vma_end_read(vmf->vma); 914 else 915 mmap_read_unlock(vmf->vma->vm_mm); 916 917 ret = blocking_page_setup(vmf, order, bo, page_offset, false); 918 drm_gem_object_put(&bo->base); 919 return ret; 920 } 921 922 return blocking_page_setup(vmf, order, bo, page_offset, true); 923 } 924 925 static vm_fault_t panthor_gem_fault(struct vm_fault *vmf) 926 { 927 return panthor_gem_any_fault(vmf, 0); 928 } 929 930 static void panthor_gem_vm_open(struct vm_area_struct *vma) 931 { 932 struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data); 933 934 drm_WARN_ON(bo->base.dev, drm_gem_is_imported(&bo->base)); 935 drm_WARN_ON(bo->base.dev, !refcount_inc_not_zero(&bo->cmap.mmap_count)); 936 937 drm_gem_vm_open(vma); 938 } 939 940 static void panthor_gem_vm_close(struct vm_area_struct *vma) 941 { 942 struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data); 943 944 if (drm_gem_is_imported(&bo->base)) 945 goto out; 946 947 if (refcount_dec_not_one(&bo->cmap.mmap_count)) 948 goto out; 949 950 dma_resv_lock(bo->base.resv, NULL); 951 if (refcount_dec_and_test(&bo->cmap.mmap_count)) { 952 mutex_lock(&bo->base.gpuva.lock); 953 panthor_gem_update_reclaim_state_locked(bo, NULL); 954 mutex_unlock(&bo->base.gpuva.lock); 955 } 956 dma_resv_unlock(bo->base.resv); 957 958 out: 959 drm_gem_object_put(&bo->base); 960 } 961 962 static const struct vm_operations_struct panthor_gem_vm_ops = { 963 .fault = panthor_gem_fault, 964 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP 965 .huge_fault = panthor_gem_any_fault, 966 #endif 967 .open = panthor_gem_vm_open, 968 .close = panthor_gem_vm_close, 969 }; 970 971 static const struct drm_gem_object_funcs panthor_gem_funcs = { 972 .free = panthor_gem_free_object, 973 .print_info = panthor_gem_print_info, 974 .pin = panthor_gem_pin_locked, 975 .unpin = panthor_gem_unpin_locked, 976 .get_sg_table = panthor_gem_get_sg_table, 977 .vmap = panthor_gem_vmap_locked, 978 .vunmap = panthor_gem_vunmap_locked, 979 .mmap = panthor_gem_mmap, 980 .status = panthor_gem_status, 981 .export = panthor_gem_prime_export, 982 .vm_ops = &panthor_gem_vm_ops, 983 }; 984 985 static struct panthor_gem_object * 986 panthor_gem_alloc_object(u32 flags) 987 { 988 struct panthor_gem_object *bo; 989 990 bo = kzalloc_obj(*bo); 991 if (!bo) 992 return ERR_PTR(-ENOMEM); 993 994 bo->reclaim_state = PANTHOR_GEM_UNRECLAIMABLE; 995 bo->base.funcs = &panthor_gem_funcs; 996 bo->flags = flags; 997 mutex_init(&bo->label.lock); 998 panthor_gem_debugfs_bo_init(bo); 999 return bo; 1000 } 1001 1002 static struct panthor_gem_object * 1003 panthor_gem_create(struct drm_device *dev, size_t size, uint32_t flags, 1004 struct panthor_vm *exclusive_vm, u32 usage_flags) 1005 { 1006 struct panthor_gem_object *bo; 1007 int ret; 1008 1009 bo = panthor_gem_alloc_object(flags); 1010 if (IS_ERR(bo)) 1011 return bo; 1012 1013 size = PAGE_ALIGN(size); 1014 ret = drm_gem_object_init(dev, &bo->base, size); 1015 if (ret) 1016 goto err_put; 1017 1018 /* Our buffers are kept pinned, so allocating them 1019 * from the MOVABLE zone is a really bad idea, and 1020 * conflicts with CMA. See comments above new_inode() 1021 * why this is required _and_ expected if you're 1022 * going to pin these pages. 1023 */ 1024 mapping_set_gfp_mask(bo->base.filp->f_mapping, 1025 GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 1026 1027 ret = drm_gem_create_mmap_offset(&bo->base); 1028 if (ret) 1029 goto err_put; 1030 1031 if (exclusive_vm) { 1032 bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm); 1033 drm_gem_object_get(bo->exclusive_vm_root_gem); 1034 bo->base.resv = bo->exclusive_vm_root_gem->resv; 1035 } 1036 1037 panthor_gem_debugfs_set_usage_flags(bo, usage_flags); 1038 return bo; 1039 1040 err_put: 1041 drm_gem_object_put(&bo->base); 1042 return ERR_PTR(ret); 1043 } 1044 1045 struct drm_gem_object * 1046 panthor_gem_prime_import_sg_table(struct drm_device *dev, 1047 struct dma_buf_attachment *attach, 1048 struct sg_table *sgt) 1049 { 1050 struct panthor_gem_object *bo; 1051 int ret; 1052 1053 bo = panthor_gem_alloc_object(0); 1054 if (IS_ERR(bo)) 1055 return ERR_CAST(bo); 1056 1057 drm_gem_private_object_init(dev, &bo->base, attach->dmabuf->size); 1058 1059 ret = drm_gem_create_mmap_offset(&bo->base); 1060 if (ret) 1061 goto err_put; 1062 1063 bo->dmap.sgt = sgt; 1064 return &bo->base; 1065 1066 err_put: 1067 drm_gem_object_put(&bo->base); 1068 return ERR_PTR(ret); 1069 } 1070 1071 /** 1072 * panthor_gem_create_with_handle() - Create a GEM object and attach it to a handle. 1073 * @file: DRM file. 1074 * @ddev: DRM device. 1075 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. 1076 * @size: Size of the GEM object to allocate. 1077 * @flags: Combination of drm_panthor_bo_flags flags. 1078 * @handle: Pointer holding the handle pointing to the new GEM object. 1079 * 1080 * Return: Zero on success 1081 */ 1082 int 1083 panthor_gem_create_with_handle(struct drm_file *file, 1084 struct drm_device *ddev, 1085 struct panthor_vm *exclusive_vm, 1086 u64 *size, u32 flags, u32 *handle) 1087 { 1088 int ret; 1089 struct panthor_gem_object *bo; 1090 1091 bo = panthor_gem_create(ddev, *size, flags, exclusive_vm, 0); 1092 if (IS_ERR(bo)) 1093 return PTR_ERR(bo); 1094 1095 /* 1096 * Allocate an id of idr table where the obj is registered 1097 * and handle has the id what user can see. 1098 */ 1099 ret = drm_gem_handle_create(file, &bo->base, handle); 1100 if (!ret) 1101 *size = bo->base.size; 1102 1103 /* drop reference from allocate - handle holds it now. */ 1104 drm_gem_object_put(&bo->base); 1105 return ret; 1106 } 1107 1108 void 1109 panthor_gem_bo_set_label(struct drm_gem_object *obj, const char *label) 1110 { 1111 struct panthor_gem_object *bo = to_panthor_bo(obj); 1112 const char *old_label; 1113 1114 scoped_guard(mutex, &bo->label.lock) { 1115 old_label = bo->label.str; 1116 bo->label.str = label; 1117 } 1118 1119 kfree_const(old_label); 1120 } 1121 1122 void 1123 panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *label) 1124 { 1125 const char *str; 1126 1127 /* We should never attempt labelling a UM-exposed GEM object */ 1128 if (drm_WARN_ON(bo->obj->dev, bo->obj->handle_count > 0)) 1129 return; 1130 1131 if (!label) 1132 return; 1133 1134 str = kstrdup_const(label, GFP_KERNEL); 1135 if (!str) { 1136 /* Failing to allocate memory for a label isn't a fatal condition */ 1137 drm_warn(bo->obj->dev, "Not enough memory to allocate BO label"); 1138 return; 1139 } 1140 1141 panthor_gem_bo_set_label(bo->obj, str); 1142 } 1143 1144 int 1145 panthor_gem_sync(struct drm_gem_object *obj, u32 type, 1146 u64 offset, u64 size) 1147 { 1148 struct panthor_gem_object *bo = to_panthor_bo(obj); 1149 struct device *dma_dev = drm_dev_dma_dev(bo->base.dev); 1150 struct sg_table *sgt; 1151 struct scatterlist *sgl; 1152 unsigned int count; 1153 int ret; 1154 1155 /* Make sure the range is in bounds. */ 1156 if (offset + size < offset || offset + size > bo->base.size) 1157 return -EINVAL; 1158 1159 /* Disallow CPU-cache maintenance on imported buffers. */ 1160 if (drm_gem_is_imported(&bo->base)) 1161 return -EINVAL; 1162 1163 switch (type) { 1164 case DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH: 1165 case DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE: 1166 break; 1167 1168 default: 1169 return -EINVAL; 1170 } 1171 1172 /* Don't bother if it's WC-mapped */ 1173 if (should_map_wc(bo)) 1174 return 0; 1175 1176 /* Nothing to do if the size is zero. */ 1177 if (size == 0) 1178 return 0; 1179 1180 ret = dma_resv_lock_interruptible(bo->base.resv, NULL); 1181 if (ret) 1182 return ret; 1183 1184 /* If there's no pages, there's no point pulling those back, bail out early. */ 1185 if (!bo->backing.pages) { 1186 ret = 0; 1187 goto out_unlock; 1188 } 1189 1190 sgt = panthor_gem_dev_map_get_sgt_locked(bo); 1191 if (IS_ERR(sgt)) { 1192 ret = PTR_ERR(sgt); 1193 goto out_unlock; 1194 } 1195 1196 for_each_sgtable_dma_sg(sgt, sgl, count) { 1197 if (size == 0) 1198 break; 1199 1200 dma_addr_t paddr = sg_dma_address(sgl); 1201 size_t len = sg_dma_len(sgl); 1202 1203 if (len <= offset) { 1204 offset -= len; 1205 continue; 1206 } 1207 1208 paddr += offset; 1209 len -= offset; 1210 len = min_t(size_t, len, size); 1211 size -= len; 1212 offset = 0; 1213 1214 /* It's unclear whether dma_sync_xxx() is the right API to do CPU 1215 * cache maintenance given an IOMMU can register their own 1216 * implementation doing more than just CPU cache flushes/invalidation, 1217 * and what we really care about here is CPU caches only, but that's 1218 * the best we have that is both arch-agnostic and does at least the 1219 * CPU cache maintenance on a <page,offset,size> tuple. 1220 * 1221 * Also, I wish we could do a single 1222 * 1223 * dma_sync_single_for_device(BIDIR) 1224 * 1225 * and get a flush+invalidate, but that's not how it's implemented 1226 * in practice (at least on arm64), so we have to make it 1227 * 1228 * dma_sync_single_for_device(TO_DEVICE) 1229 * dma_sync_single_for_cpu(FROM_DEVICE) 1230 * 1231 * for the flush+invalidate case. 1232 */ 1233 dma_sync_single_for_device(dma_dev, paddr, len, DMA_TO_DEVICE); 1234 if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE) 1235 dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE); 1236 } 1237 1238 ret = 0; 1239 1240 out_unlock: 1241 dma_resv_unlock(bo->base.resv); 1242 return ret; 1243 } 1244 1245 /** 1246 * panthor_kernel_bo_destroy() - Destroy a kernel buffer object 1247 * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction 1248 * is skipped. 1249 */ 1250 void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo) 1251 { 1252 struct panthor_device *ptdev; 1253 struct panthor_vm *vm; 1254 1255 if (IS_ERR_OR_NULL(bo)) 1256 return; 1257 1258 ptdev = container_of(bo->obj->dev, struct panthor_device, base); 1259 vm = bo->vm; 1260 panthor_kernel_bo_vunmap(bo); 1261 1262 drm_WARN_ON(bo->obj->dev, 1263 to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm)); 1264 panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size); 1265 panthor_vm_free_va(vm, &bo->va_node); 1266 if (vm == panthor_fw_vm(ptdev)) 1267 panthor_gem_unpin(to_panthor_bo(bo->obj)); 1268 drm_gem_object_put(bo->obj); 1269 panthor_vm_put(vm); 1270 kfree(bo); 1271 } 1272 1273 /** 1274 * panthor_kernel_bo_create() - Create and map a GEM object to a VM 1275 * @ptdev: Device. 1276 * @vm: VM to map the GEM to. 1277 * @size: Size of the buffer object. 1278 * @bo_flags: Combination of drm_panthor_bo_flags flags. 1279 * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those 1280 * that are related to map operations). 1281 * @gpu_va: GPU address assigned when mapping to the VM. 1282 * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be 1283 * automatically allocated. 1284 * @name: Descriptive label of the BO's contents 1285 * 1286 * Return: A valid pointer in case of success, an ERR_PTR() otherwise. 1287 */ 1288 struct panthor_kernel_bo * 1289 panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, 1290 size_t size, u32 bo_flags, u32 vm_map_flags, 1291 u64 gpu_va, const char *name) 1292 { 1293 struct panthor_kernel_bo *kbo; 1294 struct panthor_gem_object *bo; 1295 u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL; 1296 int ret; 1297 1298 if (drm_WARN_ON(&ptdev->base, !vm)) 1299 return ERR_PTR(-EINVAL); 1300 1301 kbo = kzalloc_obj(*kbo); 1302 if (!kbo) 1303 return ERR_PTR(-ENOMEM); 1304 1305 if (vm == panthor_fw_vm(ptdev)) 1306 debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED; 1307 1308 bo = panthor_gem_create(&ptdev->base, size, bo_flags, vm, debug_flags); 1309 if (IS_ERR(bo)) { 1310 ret = PTR_ERR(bo); 1311 goto err_free_kbo; 1312 } 1313 1314 kbo->obj = &bo->base; 1315 1316 if (vm == panthor_fw_vm(ptdev)) { 1317 ret = panthor_gem_pin(bo); 1318 if (ret) 1319 goto err_put_obj; 1320 } 1321 1322 panthor_gem_kernel_bo_set_label(kbo, name); 1323 1324 /* The system and GPU MMU page size might differ, which becomes a 1325 * problem for FW sections that need to be mapped at explicit address 1326 * since our PAGE_SIZE alignment might cover a VA range that's 1327 * expected to be used for another section. 1328 * Make sure we never map more than we need. 1329 */ 1330 size = ALIGN(size, panthor_vm_page_size(vm)); 1331 ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node); 1332 if (ret) 1333 goto err_unpin; 1334 1335 ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags); 1336 if (ret) 1337 goto err_free_va; 1338 1339 kbo->vm = panthor_vm_get(vm); 1340 return kbo; 1341 1342 err_free_va: 1343 panthor_vm_free_va(vm, &kbo->va_node); 1344 1345 err_unpin: 1346 if (vm == panthor_fw_vm(ptdev)) 1347 panthor_gem_unpin(bo); 1348 1349 err_put_obj: 1350 drm_gem_object_put(&bo->base); 1351 1352 err_free_kbo: 1353 kfree(kbo); 1354 return ERR_PTR(ret); 1355 } 1356 1357 /** 1358 * panthor_dummy_bo_create() - Create a Panthor BO meant to back sparse bindings. 1359 * @ptdev: Device. 1360 * 1361 * Return: A valid pointer in case of success, an ERR_PTR() otherwise. 1362 */ 1363 struct panthor_gem_object * 1364 panthor_dummy_bo_create(struct panthor_device *ptdev) 1365 { 1366 /* Since even when the DRM device's mount point has enabled THP we have no guarantee 1367 * that drm_gem_get_pages() will return a single 2MiB PMD, and also we cannot be sure 1368 * that the 2MiB won't be reclaimed and re-allocated later on as 4KiB chunks, it doesn't 1369 * make sense to pre-populate this object's page array, nor to fall back on a BO size 1370 * of 4KiB. Sticking to a dummy object size of 2MiB lets us keep things simple for now. 1371 */ 1372 return panthor_gem_create(&ptdev->base, SZ_2M, DRM_PANTHOR_BO_NO_MMAP, NULL, 0); 1373 } 1374 1375 static bool can_swap(void) 1376 { 1377 return get_nr_swap_pages() > 0; 1378 } 1379 1380 static bool can_block(struct shrink_control *sc) 1381 { 1382 /* If direct reclaim is allowed, we can always block. 1383 * If kswapd reclaim is allowed, we can block, but only if we're called 1384 * by the kswapd thread. 1385 */ 1386 return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) || 1387 ((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd()); 1388 } 1389 1390 static unsigned long 1391 panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 1392 { 1393 struct panthor_device *ptdev = shrinker->private_data; 1394 unsigned long count; 1395 1396 /* We currently don't have a flag to tell when the content of a 1397 * BO can be discarded. 1398 */ 1399 if (!can_swap()) 1400 return 0; 1401 1402 /* This is racy, but that's okay because the returned count is just a 1403 * hint. That's also what MSM is doing (no atomic var, it's relying on 1404 * the fact unsigned long access is usually atomic), so if it's good 1405 * enough for them, it's good enough for us too. 1406 */ 1407 count = ptdev->reclaim.unused.count; 1408 count += ptdev->reclaim.mmapped.count; 1409 1410 if (can_block(sc)) 1411 count += ptdev->reclaim.gpu_mapped_count; 1412 1413 return count ? count : SHRINK_EMPTY; 1414 } 1415 1416 static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj) 1417 { 1418 /* 1419 * Track last locked entry for unwinding locks in error and 1420 * success paths 1421 */ 1422 struct panthor_gem_object *bo = to_panthor_bo(obj); 1423 struct drm_gpuvm_bo *vm_bo, *last_locked = NULL; 1424 enum panthor_gem_reclaim_state old_state; 1425 int ret = 0; 1426 1427 /* To avoid potential lock ordering issue between bo_gpuva and 1428 * mapping->i_mmap_rwsem, unmap the pages from CPU side before 1429 * acquring the bo_gpuva lock. As the bo_resv lock is held, CPU 1430 * page fault handler won't be able to map in the pages whilst 1431 * eviction is in progress. 1432 */ 1433 drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping); 1434 1435 /* We take this lock when walking the list to prevent 1436 * insertion/deletion. 1437 */ 1438 /* We can only trylock in that path, because 1439 * - allocation might happen while some of these locks are held 1440 * - lock ordering is different in other paths 1441 * vm_resv -> bo_resv -> bo_gpuva 1442 * vs 1443 * bo_resv -> bo_gpuva -> vm_resv 1444 * 1445 * If we fail to lock that's fine, we back off and will get 1446 * back to it later. 1447 */ 1448 if (!mutex_trylock(&bo->base.gpuva.lock)) 1449 return false; 1450 1451 drm_gem_for_each_gpuvm_bo(vm_bo, obj) { 1452 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 1453 1454 if (resv == obj->resv) 1455 continue; 1456 1457 if (!dma_resv_trylock(resv)) { 1458 ret = -EDEADLK; 1459 goto out_unlock; 1460 } 1461 1462 last_locked = vm_bo; 1463 } 1464 1465 /* Update the state before trying to evict the buffer, if the state was 1466 * updated to something that's harder to reclaim (higher value in the 1467 * enum), skip it (will be processed when the relevant LRU is). 1468 */ 1469 panthor_gem_update_reclaim_state_locked(bo, &old_state); 1470 if (old_state < bo->reclaim_state) { 1471 ret = -EAGAIN; 1472 goto out_unlock; 1473 } 1474 1475 /* Couldn't teardown the GPU mappings? Skip. */ 1476 ret = panthor_vm_evict_bo_mappings_locked(bo); 1477 if (ret) 1478 goto out_unlock; 1479 1480 /* If everything went fine, evict the object. */ 1481 panthor_gem_evict_locked(bo); 1482 1483 out_unlock: 1484 if (last_locked) { 1485 drm_gem_for_each_gpuvm_bo(vm_bo, obj) { 1486 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 1487 1488 if (resv == obj->resv) 1489 continue; 1490 1491 dma_resv_unlock(resv); 1492 1493 if (last_locked == vm_bo) 1494 break; 1495 } 1496 } 1497 mutex_unlock(&bo->base.gpuva.lock); 1498 1499 return ret == 0; 1500 } 1501 1502 static bool panthor_gem_try_evict(struct drm_gem_object *obj) 1503 { 1504 struct panthor_gem_object *bo = to_panthor_bo(obj); 1505 1506 /* Wait was too long, skip. */ 1507 if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0) 1508 return false; 1509 1510 return panthor_gem_try_evict_no_resv_wait(&bo->base); 1511 } 1512 1513 static unsigned long 1514 panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 1515 { 1516 struct panthor_device *ptdev = shrinker->private_data; 1517 unsigned long remaining = 0; 1518 unsigned long freed = 0; 1519 1520 if (!can_swap()) 1521 goto out; 1522 1523 freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.unused, 1524 sc->nr_to_scan - freed, &remaining, 1525 panthor_gem_try_evict_no_resv_wait); 1526 if (freed >= sc->nr_to_scan) 1527 goto out; 1528 1529 freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.mmapped, 1530 sc->nr_to_scan - freed, &remaining, 1531 panthor_gem_try_evict_no_resv_wait); 1532 if (freed >= sc->nr_to_scan) 1533 goto out; 1534 1535 if (!can_block(sc)) 1536 goto out; 1537 1538 freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed, 1539 &remaining, panthor_gem_try_evict); 1540 if (freed >= sc->nr_to_scan) 1541 goto out; 1542 1543 freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.gpu_mapped_shared, 1544 sc->nr_to_scan - freed, &remaining, 1545 panthor_gem_try_evict); 1546 1547 out: 1548 #ifdef CONFIG_DEBUG_FS 1549 /* This is racy, but that's okay, because this is just debugfs 1550 * reporting and doesn't need to be accurate. 1551 */ 1552 ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed; 1553 #endif 1554 1555 /* If there are things to reclaim, try a couple times before giving up. */ 1556 if (!freed && remaining > 0 && 1557 atomic_inc_return(&ptdev->reclaim.retry_count) < 2) 1558 return 0; 1559 1560 atomic_set(&ptdev->reclaim.retry_count, 0); 1561 1562 if (freed) 1563 return freed; 1564 1565 /* There's nothing left to reclaim, or the resources are contended. Give up now. */ 1566 return SHRINK_STOP; 1567 } 1568 1569 int panthor_gem_shrinker_init(struct panthor_device *ptdev) 1570 { 1571 struct shrinker *shrinker; 1572 1573 INIT_LIST_HEAD(&ptdev->reclaim.vms); 1574 drm_gem_lru_init(&ptdev->reclaim.unused); 1575 drm_gem_lru_init(&ptdev->reclaim.mmapped); 1576 drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared); 1577 ptdev->reclaim.gpu_mapped_count = 0; 1578 1579 /* Teach lockdep about lock ordering wrt. shrinker: */ 1580 fs_reclaim_acquire(GFP_KERNEL); 1581 might_lock(&ptdev->base.gem_lru_mutex); 1582 fs_reclaim_release(GFP_KERNEL); 1583 1584 shrinker = shrinker_alloc(0, "drm-panthor-gem"); 1585 if (!shrinker) 1586 return -ENOMEM; 1587 1588 shrinker->count_objects = panthor_gem_shrinker_count; 1589 shrinker->scan_objects = panthor_gem_shrinker_scan; 1590 shrinker->private_data = ptdev; 1591 ptdev->reclaim.shrinker = shrinker; 1592 1593 shrinker_register(shrinker); 1594 return 0; 1595 } 1596 1597 void panthor_gem_shrinker_unplug(struct panthor_device *ptdev) 1598 { 1599 if (ptdev->reclaim.shrinker) 1600 shrinker_free(ptdev->reclaim.shrinker); 1601 } 1602 1603 #ifdef CONFIG_DEBUG_FS 1604 struct gem_size_totals { 1605 size_t size; 1606 size_t resident; 1607 size_t reclaimable; 1608 }; 1609 1610 static void panthor_gem_debugfs_print_flag_names(struct seq_file *m) 1611 { 1612 int len; 1613 int i; 1614 1615 static const char * const gem_state_flags_names[] = { 1616 [PANTHOR_DEBUGFS_GEM_STATE_IMPORTED_BIT] = "imported", 1617 [PANTHOR_DEBUGFS_GEM_STATE_EXPORTED_BIT] = "exported", 1618 [PANTHOR_DEBUGFS_GEM_STATE_EVICTED_BIT] = "evicted", 1619 }; 1620 1621 static const char * const gem_usage_flags_names[] = { 1622 [PANTHOR_DEBUGFS_GEM_USAGE_KERNEL_BIT] = "kernel", 1623 [PANTHOR_DEBUGFS_GEM_USAGE_FW_MAPPED_BIT] = "fw-mapped", 1624 }; 1625 1626 seq_puts(m, "GEM state flags: "); 1627 for (i = 0, len = ARRAY_SIZE(gem_state_flags_names); i < len; i++) { 1628 if (!gem_state_flags_names[i]) 1629 continue; 1630 seq_printf(m, "%s (0x%x)%s", gem_state_flags_names[i], 1631 (u32)BIT(i), (i < len - 1) ? ", " : "\n"); 1632 } 1633 1634 seq_puts(m, "GEM usage flags: "); 1635 for (i = 0, len = ARRAY_SIZE(gem_usage_flags_names); i < len; i++) { 1636 if (!gem_usage_flags_names[i]) 1637 continue; 1638 seq_printf(m, "%s (0x%x)%s", gem_usage_flags_names[i], 1639 (u32)BIT(i), (i < len - 1) ? ", " : "\n\n"); 1640 } 1641 } 1642 1643 static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo, 1644 struct seq_file *m, 1645 struct gem_size_totals *totals) 1646 { 1647 enum panthor_gem_reclaim_state reclaim_state = bo->reclaim_state; 1648 unsigned int refcount = kref_read(&bo->base.refcount); 1649 int reclaimed_count = atomic_read(&bo->reclaimed_count); 1650 char creator_info[32] = {}; 1651 size_t resident_size; 1652 u32 gem_usage_flags = bo->debugfs.flags; 1653 u32 gem_state_flags = 0; 1654 1655 /* Skip BOs being destroyed. */ 1656 if (!refcount) 1657 return; 1658 1659 resident_size = bo->backing.pages ? bo->base.size : 0; 1660 1661 snprintf(creator_info, sizeof(creator_info), 1662 "%s/%d", bo->debugfs.creator.process_name, bo->debugfs.creator.tgid); 1663 seq_printf(m, "%-32s%-16d%-11d%-11d%-16zd%-16zd0x%-16lx", 1664 creator_info, 1665 bo->base.name, 1666 refcount, 1667 reclaimed_count, 1668 bo->base.size, 1669 resident_size, 1670 drm_vma_node_start(&bo->base.vma_node)); 1671 1672 if (drm_gem_is_imported(&bo->base)) 1673 gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_IMPORTED; 1674 else if (!resident_size && reclaimed_count) 1675 gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_EVICTED; 1676 1677 if (bo->base.dma_buf) 1678 gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_EXPORTED; 1679 1680 seq_printf(m, "0x%-8x 0x%-10x", gem_state_flags, gem_usage_flags); 1681 1682 scoped_guard(mutex, &bo->label.lock) { 1683 seq_printf(m, "%s\n", bo->label.str ? : ""); 1684 } 1685 1686 totals->size += bo->base.size; 1687 totals->resident += resident_size; 1688 if (reclaim_state != PANTHOR_GEM_UNRECLAIMABLE) 1689 totals->reclaimable += resident_size; 1690 } 1691 1692 static void panthor_gem_debugfs_print_bos(struct panthor_device *ptdev, 1693 struct seq_file *m) 1694 { 1695 struct gem_size_totals totals = {0}; 1696 struct panthor_gem_object *bo; 1697 1698 panthor_gem_debugfs_print_flag_names(m); 1699 1700 seq_puts(m, "created-by global-name refcount evictions size resident-size file-offset state usage label\n"); 1701 seq_puts(m, "----------------------------------------------------------------------------------------------------------------------------------------------------\n"); 1702 1703 scoped_guard(mutex, &ptdev->gems.lock) { 1704 list_for_each_entry(bo, &ptdev->gems.node, debugfs.node) { 1705 panthor_gem_debugfs_bo_print(bo, m, &totals); 1706 } 1707 } 1708 1709 seq_puts(m, "====================================================================================================================================================\n"); 1710 seq_printf(m, "Total size: %zd, Total resident: %zd, Total reclaimable: %zd\n", 1711 totals.size, totals.resident, totals.reclaimable); 1712 } 1713 1714 static int panthor_gem_show_bos(struct seq_file *m, void *data) 1715 { 1716 struct drm_info_node *node = m->private; 1717 struct drm_device *dev = node->minor->dev; 1718 struct panthor_device *ptdev = 1719 container_of(dev, struct panthor_device, base); 1720 1721 panthor_gem_debugfs_print_bos(ptdev, m); 1722 1723 return 0; 1724 } 1725 1726 static struct drm_info_list panthor_gem_debugfs_list[] = { 1727 { "gems", panthor_gem_show_bos, 0, NULL }, 1728 }; 1729 1730 static int shrink_get(void *data, u64 *val) 1731 { 1732 struct panthor_device *ptdev = 1733 container_of(data, struct panthor_device, base); 1734 1735 *val = ptdev->reclaim.nr_pages_reclaimed_on_last_scan; 1736 return 0; 1737 } 1738 1739 static int shrink_set(void *data, u64 val) 1740 { 1741 struct panthor_device *ptdev = 1742 container_of(data, struct panthor_device, base); 1743 struct shrink_control sc = { 1744 .gfp_mask = GFP_KERNEL, 1745 .nr_to_scan = val, 1746 }; 1747 1748 fs_reclaim_acquire(GFP_KERNEL); 1749 if (ptdev->reclaim.shrinker) 1750 panthor_gem_shrinker_scan(ptdev->reclaim.shrinker, &sc); 1751 fs_reclaim_release(GFP_KERNEL); 1752 1753 return 0; 1754 } 1755 1756 DEFINE_DEBUGFS_ATTRIBUTE(panthor_gem_debugfs_shrink_fops, 1757 shrink_get, shrink_set, 1758 "0x%08llx\n"); 1759 1760 void panthor_gem_debugfs_init(struct drm_minor *minor) 1761 { 1762 drm_debugfs_create_files(panthor_gem_debugfs_list, 1763 ARRAY_SIZE(panthor_gem_debugfs_list), 1764 minor->debugfs_root, minor); 1765 debugfs_create_file("shrink", 0600, minor->debugfs_root, 1766 minor->dev, &panthor_gem_debugfs_shrink_fops); 1767 } 1768 #endif 1769