1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #define pr_fmt(fmt) "[TTM] " fmt 33 34 #include <drm/ttm/ttm_module.h> 35 #include <drm/ttm/ttm_bo_driver.h> 36 #include <drm/ttm/ttm_placement.h> 37 #include <linux/jiffies.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 #include <linux/mm.h> 41 #include <linux/file.h> 42 #include <linux/module.h> 43 #include <linux/atomic.h> 44 #include <linux/dma-resv.h> 45 46 static void ttm_bo_global_kobj_release(struct kobject *kobj); 47 48 /** 49 * ttm_global_mutex - protecting the global BO state 50 */ 51 DEFINE_MUTEX(ttm_global_mutex); 52 unsigned ttm_bo_glob_use_count; 53 struct ttm_bo_global ttm_bo_glob; 54 EXPORT_SYMBOL(ttm_bo_glob); 55 56 static struct attribute ttm_bo_count = { 57 .name = "bo_count", 58 .mode = S_IRUGO 59 }; 60 61 /* default destructor */ 62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) 63 { 64 kfree(bo); 65 } 66 67 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 68 uint32_t *mem_type) 69 { 70 int pos; 71 72 pos = ffs(place->flags & TTM_PL_MASK_MEM); 73 if (unlikely(!pos)) 74 return -EINVAL; 75 76 *mem_type = pos - 1; 77 return 0; 78 } 79 80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p, 81 int mem_type) 82 { 83 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 84 85 drm_printf(p, " has_type: %d\n", man->has_type); 86 drm_printf(p, " use_type: %d\n", man->use_type); 87 drm_printf(p, " flags: 0x%08X\n", man->flags); 88 drm_printf(p, " gpu_offset: 0x%08llX\n", man->gpu_offset); 89 drm_printf(p, " size: %llu\n", man->size); 90 drm_printf(p, " available_caching: 0x%08X\n", man->available_caching); 91 drm_printf(p, " default_caching: 0x%08X\n", man->default_caching); 92 if (mem_type != TTM_PL_SYSTEM) 93 (*man->func->debug)(man, p); 94 } 95 96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 97 struct ttm_placement *placement) 98 { 99 struct drm_printer p = drm_debug_printer(TTM_PFX); 100 int i, ret, mem_type; 101 102 drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n", 103 bo, bo->mem.num_pages, bo->mem.size >> 10, 104 bo->mem.size >> 20); 105 for (i = 0; i < placement->num_placement; i++) { 106 ret = ttm_mem_type_from_place(&placement->placement[i], 107 &mem_type); 108 if (ret) 109 return; 110 drm_printf(&p, " placement[%d]=0x%08X (%d)\n", 111 i, placement->placement[i].flags, mem_type); 112 ttm_mem_type_debug(bo->bdev, &p, mem_type); 113 } 114 } 115 116 static ssize_t ttm_bo_global_show(struct kobject *kobj, 117 struct attribute *attr, 118 char *buffer) 119 { 120 struct ttm_bo_global *glob = 121 container_of(kobj, struct ttm_bo_global, kobj); 122 123 return snprintf(buffer, PAGE_SIZE, "%d\n", 124 atomic_read(&glob->bo_count)); 125 } 126 127 static struct attribute *ttm_bo_global_attrs[] = { 128 &ttm_bo_count, 129 NULL 130 }; 131 132 static const struct sysfs_ops ttm_bo_global_ops = { 133 .show = &ttm_bo_global_show 134 }; 135 136 static struct kobj_type ttm_bo_glob_kobj_type = { 137 .release = &ttm_bo_global_kobj_release, 138 .sysfs_ops = &ttm_bo_global_ops, 139 .default_attrs = ttm_bo_global_attrs 140 }; 141 142 143 static inline uint32_t ttm_bo_type_flags(unsigned type) 144 { 145 return 1 << (type); 146 } 147 148 static void ttm_bo_release_list(struct kref *list_kref) 149 { 150 struct ttm_buffer_object *bo = 151 container_of(list_kref, struct ttm_buffer_object, list_kref); 152 size_t acc_size = bo->acc_size; 153 154 BUG_ON(kref_read(&bo->list_kref)); 155 BUG_ON(kref_read(&bo->kref)); 156 BUG_ON(bo->mem.mm_node != NULL); 157 BUG_ON(!list_empty(&bo->lru)); 158 BUG_ON(!list_empty(&bo->ddestroy)); 159 ttm_tt_destroy(bo->ttm); 160 atomic_dec(&ttm_bo_glob.bo_count); 161 dma_fence_put(bo->moving); 162 if (!ttm_bo_uses_embedded_gem_object(bo)) 163 dma_resv_fini(&bo->base._resv); 164 bo->destroy(bo); 165 ttm_mem_global_free(&ttm_mem_glob, acc_size); 166 } 167 168 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, 169 struct ttm_mem_reg *mem) 170 { 171 struct ttm_bo_device *bdev = bo->bdev; 172 struct ttm_mem_type_manager *man; 173 174 dma_resv_assert_held(bo->base.resv); 175 176 if (!list_empty(&bo->lru)) 177 return; 178 179 if (mem->placement & TTM_PL_FLAG_NO_EVICT) 180 return; 181 182 man = &bdev->man[mem->mem_type]; 183 list_add_tail(&bo->lru, &man->lru[bo->priority]); 184 kref_get(&bo->list_kref); 185 186 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm && 187 !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | 188 TTM_PAGE_FLAG_SWAPPED))) { 189 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]); 190 kref_get(&bo->list_kref); 191 } 192 } 193 194 static void ttm_bo_ref_bug(struct kref *list_kref) 195 { 196 BUG(); 197 } 198 199 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 200 { 201 struct ttm_bo_device *bdev = bo->bdev; 202 bool notify = false; 203 204 if (!list_empty(&bo->swap)) { 205 list_del_init(&bo->swap); 206 kref_put(&bo->list_kref, ttm_bo_ref_bug); 207 notify = true; 208 } 209 if (!list_empty(&bo->lru)) { 210 list_del_init(&bo->lru); 211 kref_put(&bo->list_kref, ttm_bo_ref_bug); 212 notify = true; 213 } 214 215 if (notify && bdev->driver->del_from_lru_notify) 216 bdev->driver->del_from_lru_notify(bo); 217 } 218 219 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, 220 struct ttm_buffer_object *bo) 221 { 222 if (!pos->first) 223 pos->first = bo; 224 pos->last = bo; 225 } 226 227 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, 228 struct ttm_lru_bulk_move *bulk) 229 { 230 dma_resv_assert_held(bo->base.resv); 231 232 ttm_bo_del_from_lru(bo); 233 ttm_bo_add_mem_to_lru(bo, &bo->mem); 234 235 if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 236 switch (bo->mem.mem_type) { 237 case TTM_PL_TT: 238 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); 239 break; 240 241 case TTM_PL_VRAM: 242 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); 243 break; 244 } 245 if (bo->ttm && !(bo->ttm->page_flags & 246 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) 247 ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo); 248 } 249 } 250 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 251 252 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) 253 { 254 unsigned i; 255 256 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 257 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; 258 struct ttm_mem_type_manager *man; 259 260 if (!pos->first) 261 continue; 262 263 dma_resv_assert_held(pos->first->base.resv); 264 dma_resv_assert_held(pos->last->base.resv); 265 266 man = &pos->first->bdev->man[TTM_PL_TT]; 267 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 268 &pos->last->lru); 269 } 270 271 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 272 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; 273 struct ttm_mem_type_manager *man; 274 275 if (!pos->first) 276 continue; 277 278 dma_resv_assert_held(pos->first->base.resv); 279 dma_resv_assert_held(pos->last->base.resv); 280 281 man = &pos->first->bdev->man[TTM_PL_VRAM]; 282 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 283 &pos->last->lru); 284 } 285 286 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 287 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i]; 288 struct list_head *lru; 289 290 if (!pos->first) 291 continue; 292 293 dma_resv_assert_held(pos->first->base.resv); 294 dma_resv_assert_held(pos->last->base.resv); 295 296 lru = &ttm_bo_glob.swap_lru[i]; 297 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); 298 } 299 } 300 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); 301 302 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 303 struct ttm_mem_reg *mem, bool evict, 304 struct ttm_operation_ctx *ctx) 305 { 306 struct ttm_bo_device *bdev = bo->bdev; 307 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 308 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 309 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 310 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 311 int ret = 0; 312 313 if (old_is_pci || new_is_pci || 314 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 315 ret = ttm_mem_io_lock(old_man, true); 316 if (unlikely(ret != 0)) 317 goto out_err; 318 ttm_bo_unmap_virtual_locked(bo); 319 ttm_mem_io_unlock(old_man); 320 } 321 322 /* 323 * Create and bind a ttm if required. 324 */ 325 326 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 327 if (bo->ttm == NULL) { 328 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 329 ret = ttm_tt_create(bo, zero); 330 if (ret) 331 goto out_err; 332 } 333 334 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 335 if (ret) 336 goto out_err; 337 338 if (mem->mem_type != TTM_PL_SYSTEM) { 339 ret = ttm_tt_bind(bo->ttm, mem, ctx); 340 if (ret) 341 goto out_err; 342 } 343 344 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 345 if (bdev->driver->move_notify) 346 bdev->driver->move_notify(bo, evict, mem); 347 bo->mem = *mem; 348 mem->mm_node = NULL; 349 goto moved; 350 } 351 } 352 353 if (bdev->driver->move_notify) 354 bdev->driver->move_notify(bo, evict, mem); 355 356 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 357 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 358 ret = ttm_bo_move_ttm(bo, ctx, mem); 359 else if (bdev->driver->move) 360 ret = bdev->driver->move(bo, evict, ctx, mem); 361 else 362 ret = ttm_bo_move_memcpy(bo, ctx, mem); 363 364 if (ret) { 365 if (bdev->driver->move_notify) { 366 swap(*mem, bo->mem); 367 bdev->driver->move_notify(bo, false, mem); 368 swap(*mem, bo->mem); 369 } 370 371 goto out_err; 372 } 373 374 moved: 375 if (bo->evicted) { 376 if (bdev->driver->invalidate_caches) { 377 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 378 if (ret) 379 pr_err("Can not flush read caches\n"); 380 } 381 bo->evicted = false; 382 } 383 384 if (bo->mem.mm_node) 385 bo->offset = (bo->mem.start << PAGE_SHIFT) + 386 bdev->man[bo->mem.mem_type].gpu_offset; 387 else 388 bo->offset = 0; 389 390 ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; 391 return 0; 392 393 out_err: 394 new_man = &bdev->man[bo->mem.mem_type]; 395 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 396 ttm_tt_destroy(bo->ttm); 397 bo->ttm = NULL; 398 } 399 400 return ret; 401 } 402 403 /** 404 * Call bo::reserved. 405 * Will release GPU memory type usage on destruction. 406 * This is the place to put in driver specific hooks to release 407 * driver private resources. 408 * Will release the bo::reserved lock. 409 */ 410 411 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 412 { 413 if (bo->bdev->driver->move_notify) 414 bo->bdev->driver->move_notify(bo, false, NULL); 415 416 ttm_tt_destroy(bo->ttm); 417 bo->ttm = NULL; 418 ttm_bo_mem_put(bo, &bo->mem); 419 } 420 421 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 422 { 423 int r; 424 425 if (bo->base.resv == &bo->base._resv) 426 return 0; 427 428 BUG_ON(!dma_resv_trylock(&bo->base._resv)); 429 430 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); 431 if (r) 432 dma_resv_unlock(&bo->base._resv); 433 434 return r; 435 } 436 437 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 438 { 439 struct dma_resv_list *fobj; 440 struct dma_fence *fence; 441 int i; 442 443 fobj = dma_resv_get_list(&bo->base._resv); 444 fence = dma_resv_get_excl(&bo->base._resv); 445 if (fence && !fence->ops->signaled) 446 dma_fence_enable_sw_signaling(fence); 447 448 for (i = 0; fobj && i < fobj->shared_count; ++i) { 449 fence = rcu_dereference_protected(fobj->shared[i], 450 dma_resv_held(bo->base.resv)); 451 452 if (!fence->ops->signaled) 453 dma_fence_enable_sw_signaling(fence); 454 } 455 } 456 457 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 458 { 459 struct ttm_bo_device *bdev = bo->bdev; 460 int ret; 461 462 ret = ttm_bo_individualize_resv(bo); 463 if (ret) { 464 /* Last resort, if we fail to allocate memory for the 465 * fences block for the BO to become idle 466 */ 467 dma_resv_wait_timeout_rcu(bo->base.resv, true, false, 468 30 * HZ); 469 spin_lock(&ttm_bo_glob.lru_lock); 470 goto error; 471 } 472 473 spin_lock(&ttm_bo_glob.lru_lock); 474 ret = dma_resv_trylock(bo->base.resv) ? 0 : -EBUSY; 475 if (!ret) { 476 if (dma_resv_test_signaled_rcu(&bo->base._resv, true)) { 477 ttm_bo_del_from_lru(bo); 478 spin_unlock(&ttm_bo_glob.lru_lock); 479 if (bo->base.resv != &bo->base._resv) 480 dma_resv_unlock(&bo->base._resv); 481 482 ttm_bo_cleanup_memtype_use(bo); 483 dma_resv_unlock(bo->base.resv); 484 return; 485 } 486 487 ttm_bo_flush_all_fences(bo); 488 489 /* 490 * Make NO_EVICT bos immediately available to 491 * shrinkers, now that they are queued for 492 * destruction. 493 */ 494 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 495 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 496 ttm_bo_move_to_lru_tail(bo, NULL); 497 } 498 499 dma_resv_unlock(bo->base.resv); 500 } 501 if (bo->base.resv != &bo->base._resv) 502 dma_resv_unlock(&bo->base._resv); 503 504 error: 505 kref_get(&bo->list_kref); 506 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 507 spin_unlock(&ttm_bo_glob.lru_lock); 508 509 schedule_delayed_work(&bdev->wq, 510 ((HZ / 100) < 1) ? 1 : HZ / 100); 511 } 512 513 /** 514 * function ttm_bo_cleanup_refs 515 * If bo idle, remove from delayed- and lru lists, and unref. 516 * If not idle, do nothing. 517 * 518 * Must be called with lru_lock and reservation held, this function 519 * will drop the lru lock and optionally the reservation lock before returning. 520 * 521 * @interruptible Any sleeps should occur interruptibly. 522 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 523 * @unlock_resv Unlock the reservation lock as well. 524 */ 525 526 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 527 bool interruptible, bool no_wait_gpu, 528 bool unlock_resv) 529 { 530 struct dma_resv *resv; 531 int ret; 532 533 if (unlikely(list_empty(&bo->ddestroy))) 534 resv = bo->base.resv; 535 else 536 resv = &bo->base._resv; 537 538 if (dma_resv_test_signaled_rcu(resv, true)) 539 ret = 0; 540 else 541 ret = -EBUSY; 542 543 if (ret && !no_wait_gpu) { 544 long lret; 545 546 if (unlock_resv) 547 dma_resv_unlock(bo->base.resv); 548 spin_unlock(&ttm_bo_glob.lru_lock); 549 550 lret = dma_resv_wait_timeout_rcu(resv, true, 551 interruptible, 552 30 * HZ); 553 554 if (lret < 0) 555 return lret; 556 else if (lret == 0) 557 return -EBUSY; 558 559 spin_lock(&ttm_bo_glob.lru_lock); 560 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { 561 /* 562 * We raced, and lost, someone else holds the reservation now, 563 * and is probably busy in ttm_bo_cleanup_memtype_use. 564 * 565 * Even if it's not the case, because we finished waiting any 566 * delayed destruction would succeed, so just return success 567 * here. 568 */ 569 spin_unlock(&ttm_bo_glob.lru_lock); 570 return 0; 571 } 572 ret = 0; 573 } 574 575 if (ret || unlikely(list_empty(&bo->ddestroy))) { 576 if (unlock_resv) 577 dma_resv_unlock(bo->base.resv); 578 spin_unlock(&ttm_bo_glob.lru_lock); 579 return ret; 580 } 581 582 ttm_bo_del_from_lru(bo); 583 list_del_init(&bo->ddestroy); 584 kref_put(&bo->list_kref, ttm_bo_ref_bug); 585 586 spin_unlock(&ttm_bo_glob.lru_lock); 587 ttm_bo_cleanup_memtype_use(bo); 588 589 if (unlock_resv) 590 dma_resv_unlock(bo->base.resv); 591 592 return 0; 593 } 594 595 /** 596 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 597 * encountered buffers. 598 */ 599 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 600 { 601 struct ttm_bo_global *glob = &ttm_bo_glob; 602 struct list_head removed; 603 bool empty; 604 605 INIT_LIST_HEAD(&removed); 606 607 spin_lock(&glob->lru_lock); 608 while (!list_empty(&bdev->ddestroy)) { 609 struct ttm_buffer_object *bo; 610 611 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, 612 ddestroy); 613 kref_get(&bo->list_kref); 614 list_move_tail(&bo->ddestroy, &removed); 615 616 if (remove_all || bo->base.resv != &bo->base._resv) { 617 spin_unlock(&glob->lru_lock); 618 dma_resv_lock(bo->base.resv, NULL); 619 620 spin_lock(&glob->lru_lock); 621 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 622 623 } else if (dma_resv_trylock(bo->base.resv)) { 624 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 625 } else { 626 spin_unlock(&glob->lru_lock); 627 } 628 629 kref_put(&bo->list_kref, ttm_bo_release_list); 630 spin_lock(&glob->lru_lock); 631 } 632 list_splice_tail(&removed, &bdev->ddestroy); 633 empty = list_empty(&bdev->ddestroy); 634 spin_unlock(&glob->lru_lock); 635 636 return empty; 637 } 638 639 static void ttm_bo_delayed_workqueue(struct work_struct *work) 640 { 641 struct ttm_bo_device *bdev = 642 container_of(work, struct ttm_bo_device, wq.work); 643 644 if (!ttm_bo_delayed_delete(bdev, false)) 645 schedule_delayed_work(&bdev->wq, 646 ((HZ / 100) < 1) ? 1 : HZ / 100); 647 } 648 649 static void ttm_bo_release(struct kref *kref) 650 { 651 struct ttm_buffer_object *bo = 652 container_of(kref, struct ttm_buffer_object, kref); 653 struct ttm_bo_device *bdev = bo->bdev; 654 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 655 656 if (bo->bdev->driver->release_notify) 657 bo->bdev->driver->release_notify(bo); 658 659 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); 660 ttm_mem_io_lock(man, false); 661 ttm_mem_io_free_vm(bo); 662 ttm_mem_io_unlock(man); 663 ttm_bo_cleanup_refs_or_queue(bo); 664 kref_put(&bo->list_kref, ttm_bo_release_list); 665 } 666 667 void ttm_bo_put(struct ttm_buffer_object *bo) 668 { 669 kref_put(&bo->kref, ttm_bo_release); 670 } 671 EXPORT_SYMBOL(ttm_bo_put); 672 673 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 674 { 675 return cancel_delayed_work_sync(&bdev->wq); 676 } 677 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 678 679 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 680 { 681 if (resched) 682 schedule_delayed_work(&bdev->wq, 683 ((HZ / 100) < 1) ? 1 : HZ / 100); 684 } 685 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 686 687 static int ttm_bo_evict(struct ttm_buffer_object *bo, 688 struct ttm_operation_ctx *ctx) 689 { 690 struct ttm_bo_device *bdev = bo->bdev; 691 struct ttm_mem_reg evict_mem; 692 struct ttm_placement placement; 693 int ret = 0; 694 695 dma_resv_assert_held(bo->base.resv); 696 697 placement.num_placement = 0; 698 placement.num_busy_placement = 0; 699 bdev->driver->evict_flags(bo, &placement); 700 701 if (!placement.num_placement && !placement.num_busy_placement) { 702 ret = ttm_bo_pipeline_gutting(bo); 703 if (ret) 704 return ret; 705 706 return ttm_tt_create(bo, false); 707 } 708 709 evict_mem = bo->mem; 710 evict_mem.mm_node = NULL; 711 evict_mem.bus.io_reserved_vm = false; 712 evict_mem.bus.io_reserved_count = 0; 713 714 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 715 if (ret) { 716 if (ret != -ERESTARTSYS) { 717 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 718 bo); 719 ttm_bo_mem_space_debug(bo, &placement); 720 } 721 goto out; 722 } 723 724 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); 725 if (unlikely(ret)) { 726 if (ret != -ERESTARTSYS) 727 pr_err("Buffer eviction failed\n"); 728 ttm_bo_mem_put(bo, &evict_mem); 729 goto out; 730 } 731 bo->evicted = true; 732 out: 733 return ret; 734 } 735 736 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 737 const struct ttm_place *place) 738 { 739 /* Don't evict this BO if it's outside of the 740 * requested placement range 741 */ 742 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 743 (place->lpfn && place->lpfn <= bo->mem.start)) 744 return false; 745 746 return true; 747 } 748 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 749 750 /** 751 * Check the target bo is allowable to be evicted or swapout, including cases: 752 * 753 * a. if share same reservation object with ctx->resv, have assumption 754 * reservation objects should already be locked, so not lock again and 755 * return true directly when either the opreation allow_reserved_eviction 756 * or the target bo already is in delayed free list; 757 * 758 * b. Otherwise, trylock it. 759 */ 760 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 761 struct ttm_operation_ctx *ctx, bool *locked, bool *busy) 762 { 763 bool ret = false; 764 765 if (bo->base.resv == ctx->resv) { 766 dma_resv_assert_held(bo->base.resv); 767 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT 768 || !list_empty(&bo->ddestroy)) 769 ret = true; 770 *locked = false; 771 if (busy) 772 *busy = false; 773 } else { 774 ret = dma_resv_trylock(bo->base.resv); 775 *locked = ret; 776 if (busy) 777 *busy = !ret; 778 } 779 780 return ret; 781 } 782 783 /** 784 * ttm_mem_evict_wait_busy - wait for a busy BO to become available 785 * 786 * @busy_bo: BO which couldn't be locked with trylock 787 * @ctx: operation context 788 * @ticket: acquire ticket 789 * 790 * Try to lock a busy buffer object to avoid failing eviction. 791 */ 792 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, 793 struct ttm_operation_ctx *ctx, 794 struct ww_acquire_ctx *ticket) 795 { 796 int r; 797 798 if (!busy_bo || !ticket) 799 return -EBUSY; 800 801 if (ctx->interruptible) 802 r = dma_resv_lock_interruptible(busy_bo->base.resv, 803 ticket); 804 else 805 r = dma_resv_lock(busy_bo->base.resv, ticket); 806 807 /* 808 * TODO: It would be better to keep the BO locked until allocation is at 809 * least tried one more time, but that would mean a much larger rework 810 * of TTM. 811 */ 812 if (!r) 813 dma_resv_unlock(busy_bo->base.resv); 814 815 return r == -EDEADLK ? -EBUSY : r; 816 } 817 818 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 819 uint32_t mem_type, 820 const struct ttm_place *place, 821 struct ttm_operation_ctx *ctx, 822 struct ww_acquire_ctx *ticket) 823 { 824 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; 825 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 826 bool locked = false; 827 unsigned i; 828 int ret; 829 830 spin_lock(&ttm_bo_glob.lru_lock); 831 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 832 list_for_each_entry(bo, &man->lru[i], lru) { 833 bool busy; 834 835 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, 836 &busy)) { 837 if (busy && !busy_bo && ticket != 838 dma_resv_locking_ctx(bo->base.resv)) 839 busy_bo = bo; 840 continue; 841 } 842 843 if (place && !bdev->driver->eviction_valuable(bo, 844 place)) { 845 if (locked) 846 dma_resv_unlock(bo->base.resv); 847 continue; 848 } 849 break; 850 } 851 852 /* If the inner loop terminated early, we have our candidate */ 853 if (&bo->lru != &man->lru[i]) 854 break; 855 856 bo = NULL; 857 } 858 859 if (!bo) { 860 if (busy_bo) 861 kref_get(&busy_bo->list_kref); 862 spin_unlock(&ttm_bo_glob.lru_lock); 863 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); 864 if (busy_bo) 865 kref_put(&busy_bo->list_kref, ttm_bo_release_list); 866 return ret; 867 } 868 869 kref_get(&bo->list_kref); 870 871 if (!list_empty(&bo->ddestroy)) { 872 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 873 ctx->no_wait_gpu, locked); 874 kref_put(&bo->list_kref, ttm_bo_release_list); 875 return ret; 876 } 877 878 spin_unlock(&ttm_bo_glob.lru_lock); 879 880 ret = ttm_bo_evict(bo, ctx); 881 if (locked) 882 ttm_bo_unreserve(bo); 883 884 kref_put(&bo->list_kref, ttm_bo_release_list); 885 return ret; 886 } 887 888 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 889 { 890 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 891 892 if (mem->mm_node) 893 (*man->func->put_node)(man, mem); 894 } 895 EXPORT_SYMBOL(ttm_bo_mem_put); 896 897 /** 898 * Add the last move fence to the BO and reserve a new shared slot. 899 */ 900 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 901 struct ttm_mem_type_manager *man, 902 struct ttm_mem_reg *mem) 903 { 904 struct dma_fence *fence; 905 int ret; 906 907 spin_lock(&man->move_lock); 908 fence = dma_fence_get(man->move); 909 spin_unlock(&man->move_lock); 910 911 if (fence) { 912 dma_resv_add_shared_fence(bo->base.resv, fence); 913 914 ret = dma_resv_reserve_shared(bo->base.resv, 1); 915 if (unlikely(ret)) { 916 dma_fence_put(fence); 917 return ret; 918 } 919 920 dma_fence_put(bo->moving); 921 bo->moving = fence; 922 } 923 924 return 0; 925 } 926 927 /** 928 * Repeatedly evict memory from the LRU for @mem_type until we create enough 929 * space, or we've evicted everything and there isn't enough space. 930 */ 931 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 932 const struct ttm_place *place, 933 struct ttm_mem_reg *mem, 934 struct ttm_operation_ctx *ctx) 935 { 936 struct ttm_bo_device *bdev = bo->bdev; 937 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 938 struct ww_acquire_ctx *ticket; 939 int ret; 940 941 ticket = dma_resv_locking_ctx(bo->base.resv); 942 do { 943 ret = (*man->func->get_node)(man, bo, place, mem); 944 if (unlikely(ret != 0)) 945 return ret; 946 if (mem->mm_node) 947 break; 948 ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx, 949 ticket); 950 if (unlikely(ret != 0)) 951 return ret; 952 } while (1); 953 954 return ttm_bo_add_move_fence(bo, man, mem); 955 } 956 957 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 958 uint32_t cur_placement, 959 uint32_t proposed_placement) 960 { 961 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 962 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 963 964 /** 965 * Keep current caching if possible. 966 */ 967 968 if ((cur_placement & caching) != 0) 969 result |= (cur_placement & caching); 970 else if ((man->default_caching & caching) != 0) 971 result |= man->default_caching; 972 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 973 result |= TTM_PL_FLAG_CACHED; 974 else if ((TTM_PL_FLAG_WC & caching) != 0) 975 result |= TTM_PL_FLAG_WC; 976 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 977 result |= TTM_PL_FLAG_UNCACHED; 978 979 return result; 980 } 981 982 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 983 uint32_t mem_type, 984 const struct ttm_place *place, 985 uint32_t *masked_placement) 986 { 987 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 988 989 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 990 return false; 991 992 if ((place->flags & man->available_caching) == 0) 993 return false; 994 995 cur_flags |= (place->flags & man->available_caching); 996 997 *masked_placement = cur_flags; 998 return true; 999 } 1000 1001 /** 1002 * ttm_bo_mem_placement - check if placement is compatible 1003 * @bo: BO to find memory for 1004 * @place: where to search 1005 * @mem: the memory object to fill in 1006 * @ctx: operation context 1007 * 1008 * Check if placement is compatible and fill in mem structure. 1009 * Returns -EBUSY if placement won't work or negative error code. 1010 * 0 when placement can be used. 1011 */ 1012 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, 1013 const struct ttm_place *place, 1014 struct ttm_mem_reg *mem, 1015 struct ttm_operation_ctx *ctx) 1016 { 1017 struct ttm_bo_device *bdev = bo->bdev; 1018 uint32_t mem_type = TTM_PL_SYSTEM; 1019 struct ttm_mem_type_manager *man; 1020 uint32_t cur_flags = 0; 1021 int ret; 1022 1023 ret = ttm_mem_type_from_place(place, &mem_type); 1024 if (ret) 1025 return ret; 1026 1027 man = &bdev->man[mem_type]; 1028 if (!man->has_type || !man->use_type) 1029 return -EBUSY; 1030 1031 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 1032 return -EBUSY; 1033 1034 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); 1035 /* 1036 * Use the access and other non-mapping-related flag bits from 1037 * the memory placement flags to the current flags 1038 */ 1039 ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE); 1040 1041 mem->mem_type = mem_type; 1042 mem->placement = cur_flags; 1043 1044 spin_lock(&ttm_bo_glob.lru_lock); 1045 ttm_bo_del_from_lru(bo); 1046 ttm_bo_add_mem_to_lru(bo, mem); 1047 spin_unlock(&ttm_bo_glob.lru_lock); 1048 1049 return 0; 1050 } 1051 1052 /** 1053 * Creates space for memory region @mem according to its type. 1054 * 1055 * This function first searches for free space in compatible memory types in 1056 * the priority order defined by the driver. If free space isn't found, then 1057 * ttm_bo_mem_force_space is attempted in priority order to evict and find 1058 * space. 1059 */ 1060 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 1061 struct ttm_placement *placement, 1062 struct ttm_mem_reg *mem, 1063 struct ttm_operation_ctx *ctx) 1064 { 1065 struct ttm_bo_device *bdev = bo->bdev; 1066 bool type_found = false; 1067 int i, ret; 1068 1069 ret = dma_resv_reserve_shared(bo->base.resv, 1); 1070 if (unlikely(ret)) 1071 return ret; 1072 1073 mem->mm_node = NULL; 1074 for (i = 0; i < placement->num_placement; ++i) { 1075 const struct ttm_place *place = &placement->placement[i]; 1076 struct ttm_mem_type_manager *man; 1077 1078 ret = ttm_bo_mem_placement(bo, place, mem, ctx); 1079 if (ret == -EBUSY) 1080 continue; 1081 if (ret) 1082 goto error; 1083 1084 type_found = true; 1085 mem->mm_node = NULL; 1086 if (mem->mem_type == TTM_PL_SYSTEM) 1087 return 0; 1088 1089 man = &bdev->man[mem->mem_type]; 1090 ret = (*man->func->get_node)(man, bo, place, mem); 1091 if (unlikely(ret)) 1092 goto error; 1093 1094 if (mem->mm_node) { 1095 ret = ttm_bo_add_move_fence(bo, man, mem); 1096 if (unlikely(ret)) { 1097 (*man->func->put_node)(man, mem); 1098 goto error; 1099 } 1100 return 0; 1101 } 1102 } 1103 1104 for (i = 0; i < placement->num_busy_placement; ++i) { 1105 const struct ttm_place *place = &placement->busy_placement[i]; 1106 1107 ret = ttm_bo_mem_placement(bo, place, mem, ctx); 1108 if (ret == -EBUSY) 1109 continue; 1110 if (ret) 1111 goto error; 1112 1113 type_found = true; 1114 mem->mm_node = NULL; 1115 if (mem->mem_type == TTM_PL_SYSTEM) 1116 return 0; 1117 1118 ret = ttm_bo_mem_force_space(bo, place, mem, ctx); 1119 if (ret == 0 && mem->mm_node) 1120 return 0; 1121 1122 if (ret && ret != -EBUSY) 1123 goto error; 1124 } 1125 1126 ret = -ENOMEM; 1127 if (!type_found) { 1128 pr_err(TTM_PFX "No compatible memory type found\n"); 1129 ret = -EINVAL; 1130 } 1131 1132 error: 1133 if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) { 1134 spin_lock(&ttm_bo_glob.lru_lock); 1135 ttm_bo_move_to_lru_tail(bo, NULL); 1136 spin_unlock(&ttm_bo_glob.lru_lock); 1137 } 1138 1139 return ret; 1140 } 1141 EXPORT_SYMBOL(ttm_bo_mem_space); 1142 1143 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1144 struct ttm_placement *placement, 1145 struct ttm_operation_ctx *ctx) 1146 { 1147 int ret = 0; 1148 struct ttm_mem_reg mem; 1149 1150 dma_resv_assert_held(bo->base.resv); 1151 1152 mem.num_pages = bo->num_pages; 1153 mem.size = mem.num_pages << PAGE_SHIFT; 1154 mem.page_alignment = bo->mem.page_alignment; 1155 mem.bus.io_reserved_vm = false; 1156 mem.bus.io_reserved_count = 0; 1157 /* 1158 * Determine where to move the buffer. 1159 */ 1160 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); 1161 if (ret) 1162 goto out_unlock; 1163 ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); 1164 out_unlock: 1165 if (ret && mem.mm_node) 1166 ttm_bo_mem_put(bo, &mem); 1167 return ret; 1168 } 1169 1170 static bool ttm_bo_places_compat(const struct ttm_place *places, 1171 unsigned num_placement, 1172 struct ttm_mem_reg *mem, 1173 uint32_t *new_flags) 1174 { 1175 unsigned i; 1176 1177 for (i = 0; i < num_placement; i++) { 1178 const struct ttm_place *heap = &places[i]; 1179 1180 if (mem->mm_node && (mem->start < heap->fpfn || 1181 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1182 continue; 1183 1184 *new_flags = heap->flags; 1185 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1186 (*new_flags & mem->placement & TTM_PL_MASK_MEM) && 1187 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 1188 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 1189 return true; 1190 } 1191 return false; 1192 } 1193 1194 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1195 struct ttm_mem_reg *mem, 1196 uint32_t *new_flags) 1197 { 1198 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 1199 mem, new_flags)) 1200 return true; 1201 1202 if ((placement->busy_placement != placement->placement || 1203 placement->num_busy_placement > placement->num_placement) && 1204 ttm_bo_places_compat(placement->busy_placement, 1205 placement->num_busy_placement, 1206 mem, new_flags)) 1207 return true; 1208 1209 return false; 1210 } 1211 EXPORT_SYMBOL(ttm_bo_mem_compat); 1212 1213 int ttm_bo_validate(struct ttm_buffer_object *bo, 1214 struct ttm_placement *placement, 1215 struct ttm_operation_ctx *ctx) 1216 { 1217 int ret; 1218 uint32_t new_flags; 1219 1220 dma_resv_assert_held(bo->base.resv); 1221 /* 1222 * Check whether we need to move buffer. 1223 */ 1224 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1225 ret = ttm_bo_move_buffer(bo, placement, ctx); 1226 if (ret) 1227 return ret; 1228 } else { 1229 /* 1230 * Use the access and other non-mapping-related flag bits from 1231 * the compatible memory placement flags to the active flags 1232 */ 1233 ttm_flag_masked(&bo->mem.placement, new_flags, 1234 ~TTM_PL_MASK_MEMTYPE); 1235 } 1236 /* 1237 * We might need to add a TTM. 1238 */ 1239 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1240 ret = ttm_tt_create(bo, true); 1241 if (ret) 1242 return ret; 1243 } 1244 return 0; 1245 } 1246 EXPORT_SYMBOL(ttm_bo_validate); 1247 1248 int ttm_bo_init_reserved(struct ttm_bo_device *bdev, 1249 struct ttm_buffer_object *bo, 1250 unsigned long size, 1251 enum ttm_bo_type type, 1252 struct ttm_placement *placement, 1253 uint32_t page_alignment, 1254 struct ttm_operation_ctx *ctx, 1255 size_t acc_size, 1256 struct sg_table *sg, 1257 struct dma_resv *resv, 1258 void (*destroy) (struct ttm_buffer_object *)) 1259 { 1260 struct ttm_mem_global *mem_glob = &ttm_mem_glob; 1261 int ret = 0; 1262 unsigned long num_pages; 1263 bool locked; 1264 1265 ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); 1266 if (ret) { 1267 pr_err("Out of kernel memory\n"); 1268 if (destroy) 1269 (*destroy)(bo); 1270 else 1271 kfree(bo); 1272 return -ENOMEM; 1273 } 1274 1275 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1276 if (num_pages == 0) { 1277 pr_err("Illegal buffer object size\n"); 1278 if (destroy) 1279 (*destroy)(bo); 1280 else 1281 kfree(bo); 1282 ttm_mem_global_free(mem_glob, acc_size); 1283 return -EINVAL; 1284 } 1285 bo->destroy = destroy ? destroy : ttm_bo_default_destroy; 1286 1287 kref_init(&bo->kref); 1288 kref_init(&bo->list_kref); 1289 INIT_LIST_HEAD(&bo->lru); 1290 INIT_LIST_HEAD(&bo->ddestroy); 1291 INIT_LIST_HEAD(&bo->swap); 1292 INIT_LIST_HEAD(&bo->io_reserve_lru); 1293 bo->bdev = bdev; 1294 bo->type = type; 1295 bo->num_pages = num_pages; 1296 bo->mem.size = num_pages << PAGE_SHIFT; 1297 bo->mem.mem_type = TTM_PL_SYSTEM; 1298 bo->mem.num_pages = bo->num_pages; 1299 bo->mem.mm_node = NULL; 1300 bo->mem.page_alignment = page_alignment; 1301 bo->mem.bus.io_reserved_vm = false; 1302 bo->mem.bus.io_reserved_count = 0; 1303 bo->moving = NULL; 1304 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1305 bo->acc_size = acc_size; 1306 bo->sg = sg; 1307 if (resv) { 1308 bo->base.resv = resv; 1309 dma_resv_assert_held(bo->base.resv); 1310 } else { 1311 bo->base.resv = &bo->base._resv; 1312 } 1313 if (!ttm_bo_uses_embedded_gem_object(bo)) { 1314 /* 1315 * bo.gem is not initialized, so we have to setup the 1316 * struct elements we want use regardless. 1317 */ 1318 dma_resv_init(&bo->base._resv); 1319 drm_vma_node_reset(&bo->base.vma_node); 1320 } 1321 atomic_inc(&ttm_bo_glob.bo_count); 1322 1323 /* 1324 * For ttm_bo_type_device buffers, allocate 1325 * address space from the device. 1326 */ 1327 if (bo->type == ttm_bo_type_device || 1328 bo->type == ttm_bo_type_sg) 1329 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, 1330 bo->mem.num_pages); 1331 1332 /* passed reservation objects should already be locked, 1333 * since otherwise lockdep will be angered in radeon. 1334 */ 1335 if (!resv) { 1336 locked = dma_resv_trylock(bo->base.resv); 1337 WARN_ON(!locked); 1338 } 1339 1340 if (likely(!ret)) 1341 ret = ttm_bo_validate(bo, placement, ctx); 1342 1343 if (unlikely(ret)) { 1344 if (!resv) 1345 ttm_bo_unreserve(bo); 1346 1347 ttm_bo_put(bo); 1348 return ret; 1349 } 1350 1351 spin_lock(&ttm_bo_glob.lru_lock); 1352 ttm_bo_move_to_lru_tail(bo, NULL); 1353 spin_unlock(&ttm_bo_glob.lru_lock); 1354 1355 return ret; 1356 } 1357 EXPORT_SYMBOL(ttm_bo_init_reserved); 1358 1359 int ttm_bo_init(struct ttm_bo_device *bdev, 1360 struct ttm_buffer_object *bo, 1361 unsigned long size, 1362 enum ttm_bo_type type, 1363 struct ttm_placement *placement, 1364 uint32_t page_alignment, 1365 bool interruptible, 1366 size_t acc_size, 1367 struct sg_table *sg, 1368 struct dma_resv *resv, 1369 void (*destroy) (struct ttm_buffer_object *)) 1370 { 1371 struct ttm_operation_ctx ctx = { interruptible, false }; 1372 int ret; 1373 1374 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1375 page_alignment, &ctx, acc_size, 1376 sg, resv, destroy); 1377 if (ret) 1378 return ret; 1379 1380 if (!resv) 1381 ttm_bo_unreserve(bo); 1382 1383 return 0; 1384 } 1385 EXPORT_SYMBOL(ttm_bo_init); 1386 1387 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1388 unsigned long bo_size, 1389 unsigned struct_size) 1390 { 1391 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1392 size_t size = 0; 1393 1394 size += ttm_round_pot(struct_size); 1395 size += ttm_round_pot(npages * sizeof(void *)); 1396 size += ttm_round_pot(sizeof(struct ttm_tt)); 1397 return size; 1398 } 1399 EXPORT_SYMBOL(ttm_bo_acc_size); 1400 1401 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1402 unsigned long bo_size, 1403 unsigned struct_size) 1404 { 1405 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1406 size_t size = 0; 1407 1408 size += ttm_round_pot(struct_size); 1409 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1410 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1411 return size; 1412 } 1413 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1414 1415 int ttm_bo_create(struct ttm_bo_device *bdev, 1416 unsigned long size, 1417 enum ttm_bo_type type, 1418 struct ttm_placement *placement, 1419 uint32_t page_alignment, 1420 bool interruptible, 1421 struct ttm_buffer_object **p_bo) 1422 { 1423 struct ttm_buffer_object *bo; 1424 size_t acc_size; 1425 int ret; 1426 1427 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1428 if (unlikely(bo == NULL)) 1429 return -ENOMEM; 1430 1431 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1432 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1433 interruptible, acc_size, 1434 NULL, NULL, NULL); 1435 if (likely(ret == 0)) 1436 *p_bo = bo; 1437 1438 return ret; 1439 } 1440 EXPORT_SYMBOL(ttm_bo_create); 1441 1442 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1443 unsigned mem_type) 1444 { 1445 struct ttm_operation_ctx ctx = { 1446 .interruptible = false, 1447 .no_wait_gpu = false, 1448 .flags = TTM_OPT_FLAG_FORCE_ALLOC 1449 }; 1450 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1451 struct ttm_bo_global *glob = &ttm_bo_glob; 1452 struct dma_fence *fence; 1453 int ret; 1454 unsigned i; 1455 1456 /* 1457 * Can't use standard list traversal since we're unlocking. 1458 */ 1459 1460 spin_lock(&glob->lru_lock); 1461 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1462 while (!list_empty(&man->lru[i])) { 1463 spin_unlock(&glob->lru_lock); 1464 ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx, 1465 NULL); 1466 if (ret) 1467 return ret; 1468 spin_lock(&glob->lru_lock); 1469 } 1470 } 1471 spin_unlock(&glob->lru_lock); 1472 1473 spin_lock(&man->move_lock); 1474 fence = dma_fence_get(man->move); 1475 spin_unlock(&man->move_lock); 1476 1477 if (fence) { 1478 ret = dma_fence_wait(fence, false); 1479 dma_fence_put(fence); 1480 if (ret) 1481 return ret; 1482 } 1483 1484 return 0; 1485 } 1486 1487 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1488 { 1489 struct ttm_mem_type_manager *man; 1490 int ret = -EINVAL; 1491 1492 if (mem_type >= TTM_NUM_MEM_TYPES) { 1493 pr_err("Illegal memory type %d\n", mem_type); 1494 return ret; 1495 } 1496 man = &bdev->man[mem_type]; 1497 1498 if (!man->has_type) { 1499 pr_err("Trying to take down uninitialized memory manager type %u\n", 1500 mem_type); 1501 return ret; 1502 } 1503 1504 man->use_type = false; 1505 man->has_type = false; 1506 1507 ret = 0; 1508 if (mem_type > 0) { 1509 ret = ttm_bo_force_list_clean(bdev, mem_type); 1510 if (ret) { 1511 pr_err("Cleanup eviction failed\n"); 1512 return ret; 1513 } 1514 1515 ret = (*man->func->takedown)(man); 1516 } 1517 1518 dma_fence_put(man->move); 1519 man->move = NULL; 1520 1521 return ret; 1522 } 1523 EXPORT_SYMBOL(ttm_bo_clean_mm); 1524 1525 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1526 { 1527 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1528 1529 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1530 pr_err("Illegal memory manager memory type %u\n", mem_type); 1531 return -EINVAL; 1532 } 1533 1534 if (!man->has_type) { 1535 pr_err("Memory type %u has not been initialized\n", mem_type); 1536 return 0; 1537 } 1538 1539 return ttm_bo_force_list_clean(bdev, mem_type); 1540 } 1541 EXPORT_SYMBOL(ttm_bo_evict_mm); 1542 1543 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1544 unsigned long p_size) 1545 { 1546 int ret; 1547 struct ttm_mem_type_manager *man; 1548 unsigned i; 1549 1550 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1551 man = &bdev->man[type]; 1552 BUG_ON(man->has_type); 1553 man->io_reserve_fastpath = true; 1554 man->use_io_reserve_lru = false; 1555 mutex_init(&man->io_reserve_mutex); 1556 spin_lock_init(&man->move_lock); 1557 INIT_LIST_HEAD(&man->io_reserve_lru); 1558 1559 ret = bdev->driver->init_mem_type(bdev, type, man); 1560 if (ret) 1561 return ret; 1562 man->bdev = bdev; 1563 1564 if (type != TTM_PL_SYSTEM) { 1565 ret = (*man->func->init)(man, p_size); 1566 if (ret) 1567 return ret; 1568 } 1569 man->has_type = true; 1570 man->use_type = true; 1571 man->size = p_size; 1572 1573 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1574 INIT_LIST_HEAD(&man->lru[i]); 1575 man->move = NULL; 1576 1577 return 0; 1578 } 1579 EXPORT_SYMBOL(ttm_bo_init_mm); 1580 1581 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1582 { 1583 struct ttm_bo_global *glob = 1584 container_of(kobj, struct ttm_bo_global, kobj); 1585 1586 __free_page(glob->dummy_read_page); 1587 } 1588 1589 static void ttm_bo_global_release(void) 1590 { 1591 struct ttm_bo_global *glob = &ttm_bo_glob; 1592 1593 mutex_lock(&ttm_global_mutex); 1594 if (--ttm_bo_glob_use_count > 0) 1595 goto out; 1596 1597 kobject_del(&glob->kobj); 1598 kobject_put(&glob->kobj); 1599 ttm_mem_global_release(&ttm_mem_glob); 1600 memset(glob, 0, sizeof(*glob)); 1601 out: 1602 mutex_unlock(&ttm_global_mutex); 1603 } 1604 1605 static int ttm_bo_global_init(void) 1606 { 1607 struct ttm_bo_global *glob = &ttm_bo_glob; 1608 int ret = 0; 1609 unsigned i; 1610 1611 mutex_lock(&ttm_global_mutex); 1612 if (++ttm_bo_glob_use_count > 1) 1613 goto out; 1614 1615 ret = ttm_mem_global_init(&ttm_mem_glob); 1616 if (ret) 1617 goto out; 1618 1619 spin_lock_init(&glob->lru_lock); 1620 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1621 1622 if (unlikely(glob->dummy_read_page == NULL)) { 1623 ret = -ENOMEM; 1624 goto out; 1625 } 1626 1627 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1628 INIT_LIST_HEAD(&glob->swap_lru[i]); 1629 INIT_LIST_HEAD(&glob->device_list); 1630 atomic_set(&glob->bo_count, 0); 1631 1632 ret = kobject_init_and_add( 1633 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1634 if (unlikely(ret != 0)) 1635 kobject_put(&glob->kobj); 1636 out: 1637 mutex_unlock(&ttm_global_mutex); 1638 return ret; 1639 } 1640 1641 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1642 { 1643 struct ttm_bo_global *glob = &ttm_bo_glob; 1644 int ret = 0; 1645 unsigned i = TTM_NUM_MEM_TYPES; 1646 struct ttm_mem_type_manager *man; 1647 1648 while (i--) { 1649 man = &bdev->man[i]; 1650 if (man->has_type) { 1651 man->use_type = false; 1652 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1653 ret = -EBUSY; 1654 pr_err("DRM memory manager type %d is not clean\n", 1655 i); 1656 } 1657 man->has_type = false; 1658 } 1659 } 1660 1661 mutex_lock(&ttm_global_mutex); 1662 list_del(&bdev->device_list); 1663 mutex_unlock(&ttm_global_mutex); 1664 1665 cancel_delayed_work_sync(&bdev->wq); 1666 1667 if (ttm_bo_delayed_delete(bdev, true)) 1668 pr_debug("Delayed destroy list was clean\n"); 1669 1670 spin_lock(&glob->lru_lock); 1671 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1672 if (list_empty(&bdev->man[0].lru[0])) 1673 pr_debug("Swap list %d was clean\n", i); 1674 spin_unlock(&glob->lru_lock); 1675 1676 if (!ret) 1677 ttm_bo_global_release(); 1678 1679 return ret; 1680 } 1681 EXPORT_SYMBOL(ttm_bo_device_release); 1682 1683 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1684 struct ttm_bo_driver *driver, 1685 struct address_space *mapping, 1686 struct drm_vma_offset_manager *vma_manager, 1687 bool need_dma32) 1688 { 1689 struct ttm_bo_global *glob = &ttm_bo_glob; 1690 int ret; 1691 1692 if (WARN_ON(vma_manager == NULL)) 1693 return -EINVAL; 1694 1695 ret = ttm_bo_global_init(); 1696 if (ret) 1697 return ret; 1698 1699 bdev->driver = driver; 1700 1701 memset(bdev->man, 0, sizeof(bdev->man)); 1702 1703 /* 1704 * Initialize the system memory buffer type. 1705 * Other types need to be driver / IOCTL initialized. 1706 */ 1707 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1708 if (unlikely(ret != 0)) 1709 goto out_no_sys; 1710 1711 bdev->vma_manager = vma_manager; 1712 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1713 INIT_LIST_HEAD(&bdev->ddestroy); 1714 bdev->dev_mapping = mapping; 1715 bdev->need_dma32 = need_dma32; 1716 mutex_lock(&ttm_global_mutex); 1717 list_add_tail(&bdev->device_list, &glob->device_list); 1718 mutex_unlock(&ttm_global_mutex); 1719 1720 return 0; 1721 out_no_sys: 1722 ttm_bo_global_release(); 1723 return ret; 1724 } 1725 EXPORT_SYMBOL(ttm_bo_device_init); 1726 1727 /* 1728 * buffer object vm functions. 1729 */ 1730 1731 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1732 { 1733 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1734 1735 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1736 if (mem->mem_type == TTM_PL_SYSTEM) 1737 return false; 1738 1739 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1740 return false; 1741 1742 if (mem->placement & TTM_PL_FLAG_CACHED) 1743 return false; 1744 } 1745 return true; 1746 } 1747 1748 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1749 { 1750 struct ttm_bo_device *bdev = bo->bdev; 1751 1752 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); 1753 ttm_mem_io_free_vm(bo); 1754 } 1755 1756 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1757 { 1758 struct ttm_bo_device *bdev = bo->bdev; 1759 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1760 1761 ttm_mem_io_lock(man, false); 1762 ttm_bo_unmap_virtual_locked(bo); 1763 ttm_mem_io_unlock(man); 1764 } 1765 1766 1767 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1768 1769 int ttm_bo_wait(struct ttm_buffer_object *bo, 1770 bool interruptible, bool no_wait) 1771 { 1772 long timeout = 15 * HZ; 1773 1774 if (no_wait) { 1775 if (dma_resv_test_signaled_rcu(bo->base.resv, true)) 1776 return 0; 1777 else 1778 return -EBUSY; 1779 } 1780 1781 timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true, 1782 interruptible, timeout); 1783 if (timeout < 0) 1784 return timeout; 1785 1786 if (timeout == 0) 1787 return -EBUSY; 1788 1789 dma_resv_add_excl_fence(bo->base.resv, NULL); 1790 return 0; 1791 } 1792 EXPORT_SYMBOL(ttm_bo_wait); 1793 1794 /** 1795 * A buffer object shrink method that tries to swap out the first 1796 * buffer object on the bo_global::swap_lru list. 1797 */ 1798 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) 1799 { 1800 struct ttm_buffer_object *bo; 1801 int ret = -EBUSY; 1802 bool locked; 1803 unsigned i; 1804 1805 spin_lock(&glob->lru_lock); 1806 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1807 list_for_each_entry(bo, &glob->swap_lru[i], swap) { 1808 if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked, 1809 NULL)) { 1810 ret = 0; 1811 break; 1812 } 1813 } 1814 if (!ret) 1815 break; 1816 } 1817 1818 if (ret) { 1819 spin_unlock(&glob->lru_lock); 1820 return ret; 1821 } 1822 1823 kref_get(&bo->list_kref); 1824 1825 if (!list_empty(&bo->ddestroy)) { 1826 ret = ttm_bo_cleanup_refs(bo, false, false, locked); 1827 kref_put(&bo->list_kref, ttm_bo_release_list); 1828 return ret; 1829 } 1830 1831 ttm_bo_del_from_lru(bo); 1832 spin_unlock(&glob->lru_lock); 1833 1834 /** 1835 * Move to system cached 1836 */ 1837 1838 if (bo->mem.mem_type != TTM_PL_SYSTEM || 1839 bo->ttm->caching_state != tt_cached) { 1840 struct ttm_operation_ctx ctx = { false, false }; 1841 struct ttm_mem_reg evict_mem; 1842 1843 evict_mem = bo->mem; 1844 evict_mem.mm_node = NULL; 1845 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1846 evict_mem.mem_type = TTM_PL_SYSTEM; 1847 1848 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); 1849 if (unlikely(ret != 0)) 1850 goto out; 1851 } 1852 1853 /** 1854 * Make sure BO is idle. 1855 */ 1856 1857 ret = ttm_bo_wait(bo, false, false); 1858 if (unlikely(ret != 0)) 1859 goto out; 1860 1861 ttm_bo_unmap_virtual(bo); 1862 1863 /** 1864 * Swap out. Buffer will be swapped in again as soon as 1865 * anyone tries to access a ttm page. 1866 */ 1867 1868 if (bo->bdev->driver->swap_notify) 1869 bo->bdev->driver->swap_notify(bo); 1870 1871 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1872 out: 1873 1874 /** 1875 * 1876 * Unreserve without putting on LRU to avoid swapping out an 1877 * already swapped buffer. 1878 */ 1879 if (locked) 1880 dma_resv_unlock(bo->base.resv); 1881 kref_put(&bo->list_kref, ttm_bo_release_list); 1882 return ret; 1883 } 1884 EXPORT_SYMBOL(ttm_bo_swapout); 1885 1886 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1887 { 1888 struct ttm_operation_ctx ctx = { 1889 .interruptible = false, 1890 .no_wait_gpu = false 1891 }; 1892 1893 while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0); 1894 } 1895 EXPORT_SYMBOL(ttm_bo_swapout_all); 1896