1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #define pr_fmt(fmt) "[TTM] " fmt 33 34 #include <drm/ttm/ttm_module.h> 35 #include <drm/ttm/ttm_bo_driver.h> 36 #include <drm/ttm/ttm_placement.h> 37 #include <linux/jiffies.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 #include <linux/mm.h> 41 #include <linux/file.h> 42 #include <linux/module.h> 43 #include <linux/atomic.h> 44 #include <linux/reservation.h> 45 46 static void ttm_bo_global_kobj_release(struct kobject *kobj); 47 48 /** 49 * ttm_global_mutex - protecting the global BO state 50 */ 51 DEFINE_MUTEX(ttm_global_mutex); 52 struct ttm_bo_global ttm_bo_glob = { 53 .use_count = 0 54 }; 55 56 static struct attribute ttm_bo_count = { 57 .name = "bo_count", 58 .mode = S_IRUGO 59 }; 60 61 /* default destructor */ 62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) 63 { 64 kfree(bo); 65 } 66 67 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 68 uint32_t *mem_type) 69 { 70 int pos; 71 72 pos = ffs(place->flags & TTM_PL_MASK_MEM); 73 if (unlikely(!pos)) 74 return -EINVAL; 75 76 *mem_type = pos - 1; 77 return 0; 78 } 79 80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 81 { 82 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 83 struct drm_printer p = drm_debug_printer(TTM_PFX); 84 85 pr_err(" has_type: %d\n", man->has_type); 86 pr_err(" use_type: %d\n", man->use_type); 87 pr_err(" flags: 0x%08X\n", man->flags); 88 pr_err(" gpu_offset: 0x%08llX\n", man->gpu_offset); 89 pr_err(" size: %llu\n", man->size); 90 pr_err(" available_caching: 0x%08X\n", man->available_caching); 91 pr_err(" default_caching: 0x%08X\n", man->default_caching); 92 if (mem_type != TTM_PL_SYSTEM) 93 (*man->func->debug)(man, &p); 94 } 95 96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 97 struct ttm_placement *placement) 98 { 99 int i, ret, mem_type; 100 101 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 102 bo, bo->mem.num_pages, bo->mem.size >> 10, 103 bo->mem.size >> 20); 104 for (i = 0; i < placement->num_placement; i++) { 105 ret = ttm_mem_type_from_place(&placement->placement[i], 106 &mem_type); 107 if (ret) 108 return; 109 pr_err(" placement[%d]=0x%08X (%d)\n", 110 i, placement->placement[i].flags, mem_type); 111 ttm_mem_type_debug(bo->bdev, mem_type); 112 } 113 } 114 115 static ssize_t ttm_bo_global_show(struct kobject *kobj, 116 struct attribute *attr, 117 char *buffer) 118 { 119 struct ttm_bo_global *glob = 120 container_of(kobj, struct ttm_bo_global, kobj); 121 122 return snprintf(buffer, PAGE_SIZE, "%d\n", 123 atomic_read(&glob->bo_count)); 124 } 125 126 static struct attribute *ttm_bo_global_attrs[] = { 127 &ttm_bo_count, 128 NULL 129 }; 130 131 static const struct sysfs_ops ttm_bo_global_ops = { 132 .show = &ttm_bo_global_show 133 }; 134 135 static struct kobj_type ttm_bo_glob_kobj_type = { 136 .release = &ttm_bo_global_kobj_release, 137 .sysfs_ops = &ttm_bo_global_ops, 138 .default_attrs = ttm_bo_global_attrs 139 }; 140 141 142 static inline uint32_t ttm_bo_type_flags(unsigned type) 143 { 144 return 1 << (type); 145 } 146 147 static void ttm_bo_release_list(struct kref *list_kref) 148 { 149 struct ttm_buffer_object *bo = 150 container_of(list_kref, struct ttm_buffer_object, list_kref); 151 struct ttm_bo_device *bdev = bo->bdev; 152 size_t acc_size = bo->acc_size; 153 154 BUG_ON(kref_read(&bo->list_kref)); 155 BUG_ON(kref_read(&bo->kref)); 156 BUG_ON(atomic_read(&bo->cpu_writers)); 157 BUG_ON(bo->mem.mm_node != NULL); 158 BUG_ON(!list_empty(&bo->lru)); 159 BUG_ON(!list_empty(&bo->ddestroy)); 160 ttm_tt_destroy(bo->ttm); 161 atomic_dec(&bo->bdev->glob->bo_count); 162 dma_fence_put(bo->moving); 163 reservation_object_fini(&bo->ttm_resv); 164 mutex_destroy(&bo->wu_mutex); 165 bo->destroy(bo); 166 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 167 } 168 169 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 170 { 171 struct ttm_bo_device *bdev = bo->bdev; 172 struct ttm_mem_type_manager *man; 173 174 reservation_object_assert_held(bo->resv); 175 176 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 177 BUG_ON(!list_empty(&bo->lru)); 178 179 man = &bdev->man[bo->mem.mem_type]; 180 list_add_tail(&bo->lru, &man->lru[bo->priority]); 181 kref_get(&bo->list_kref); 182 183 if (bo->ttm && !(bo->ttm->page_flags & 184 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { 185 list_add_tail(&bo->swap, 186 &bdev->glob->swap_lru[bo->priority]); 187 kref_get(&bo->list_kref); 188 } 189 } 190 } 191 EXPORT_SYMBOL(ttm_bo_add_to_lru); 192 193 static void ttm_bo_ref_bug(struct kref *list_kref) 194 { 195 BUG(); 196 } 197 198 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 199 { 200 if (!list_empty(&bo->swap)) { 201 list_del_init(&bo->swap); 202 kref_put(&bo->list_kref, ttm_bo_ref_bug); 203 } 204 if (!list_empty(&bo->lru)) { 205 list_del_init(&bo->lru); 206 kref_put(&bo->list_kref, ttm_bo_ref_bug); 207 } 208 209 /* 210 * TODO: Add a driver hook to delete from 211 * driver-specific LRU's here. 212 */ 213 } 214 215 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 216 { 217 struct ttm_bo_global *glob = bo->bdev->glob; 218 219 spin_lock(&glob->lru_lock); 220 ttm_bo_del_from_lru(bo); 221 spin_unlock(&glob->lru_lock); 222 } 223 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 224 225 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, 226 struct ttm_buffer_object *bo) 227 { 228 if (!pos->first) 229 pos->first = bo; 230 pos->last = bo; 231 } 232 233 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, 234 struct ttm_lru_bulk_move *bulk) 235 { 236 reservation_object_assert_held(bo->resv); 237 238 ttm_bo_del_from_lru(bo); 239 ttm_bo_add_to_lru(bo); 240 241 if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 242 switch (bo->mem.mem_type) { 243 case TTM_PL_TT: 244 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); 245 break; 246 247 case TTM_PL_VRAM: 248 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); 249 break; 250 } 251 if (bo->ttm && !(bo->ttm->page_flags & 252 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) 253 ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo); 254 } 255 } 256 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 257 258 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) 259 { 260 unsigned i; 261 262 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 263 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; 264 struct ttm_mem_type_manager *man; 265 266 if (!pos->first) 267 continue; 268 269 reservation_object_assert_held(pos->first->resv); 270 reservation_object_assert_held(pos->last->resv); 271 272 man = &pos->first->bdev->man[TTM_PL_TT]; 273 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 274 &pos->last->lru); 275 } 276 277 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 278 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; 279 struct ttm_mem_type_manager *man; 280 281 if (!pos->first) 282 continue; 283 284 reservation_object_assert_held(pos->first->resv); 285 reservation_object_assert_held(pos->last->resv); 286 287 man = &pos->first->bdev->man[TTM_PL_VRAM]; 288 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 289 &pos->last->lru); 290 } 291 292 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 293 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i]; 294 struct list_head *lru; 295 296 if (!pos->first) 297 continue; 298 299 reservation_object_assert_held(pos->first->resv); 300 reservation_object_assert_held(pos->last->resv); 301 302 lru = &pos->first->bdev->glob->swap_lru[i]; 303 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); 304 } 305 } 306 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); 307 308 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 309 struct ttm_mem_reg *mem, bool evict, 310 struct ttm_operation_ctx *ctx) 311 { 312 struct ttm_bo_device *bdev = bo->bdev; 313 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 314 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 315 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 316 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 317 int ret = 0; 318 319 if (old_is_pci || new_is_pci || 320 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 321 ret = ttm_mem_io_lock(old_man, true); 322 if (unlikely(ret != 0)) 323 goto out_err; 324 ttm_bo_unmap_virtual_locked(bo); 325 ttm_mem_io_unlock(old_man); 326 } 327 328 /* 329 * Create and bind a ttm if required. 330 */ 331 332 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 333 if (bo->ttm == NULL) { 334 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 335 ret = ttm_tt_create(bo, zero); 336 if (ret) 337 goto out_err; 338 } 339 340 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 341 if (ret) 342 goto out_err; 343 344 if (mem->mem_type != TTM_PL_SYSTEM) { 345 ret = ttm_tt_bind(bo->ttm, mem, ctx); 346 if (ret) 347 goto out_err; 348 } 349 350 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 351 if (bdev->driver->move_notify) 352 bdev->driver->move_notify(bo, evict, mem); 353 bo->mem = *mem; 354 mem->mm_node = NULL; 355 goto moved; 356 } 357 } 358 359 if (bdev->driver->move_notify) 360 bdev->driver->move_notify(bo, evict, mem); 361 362 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 363 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 364 ret = ttm_bo_move_ttm(bo, ctx, mem); 365 else if (bdev->driver->move) 366 ret = bdev->driver->move(bo, evict, ctx, mem); 367 else 368 ret = ttm_bo_move_memcpy(bo, ctx, mem); 369 370 if (ret) { 371 if (bdev->driver->move_notify) { 372 swap(*mem, bo->mem); 373 bdev->driver->move_notify(bo, false, mem); 374 swap(*mem, bo->mem); 375 } 376 377 goto out_err; 378 } 379 380 moved: 381 if (bo->evicted) { 382 if (bdev->driver->invalidate_caches) { 383 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 384 if (ret) 385 pr_err("Can not flush read caches\n"); 386 } 387 bo->evicted = false; 388 } 389 390 if (bo->mem.mm_node) 391 bo->offset = (bo->mem.start << PAGE_SHIFT) + 392 bdev->man[bo->mem.mem_type].gpu_offset; 393 else 394 bo->offset = 0; 395 396 ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; 397 return 0; 398 399 out_err: 400 new_man = &bdev->man[bo->mem.mem_type]; 401 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 402 ttm_tt_destroy(bo->ttm); 403 bo->ttm = NULL; 404 } 405 406 return ret; 407 } 408 409 /** 410 * Call bo::reserved. 411 * Will release GPU memory type usage on destruction. 412 * This is the place to put in driver specific hooks to release 413 * driver private resources. 414 * Will release the bo::reserved lock. 415 */ 416 417 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 418 { 419 if (bo->bdev->driver->move_notify) 420 bo->bdev->driver->move_notify(bo, false, NULL); 421 422 ttm_tt_destroy(bo->ttm); 423 bo->ttm = NULL; 424 ttm_bo_mem_put(bo, &bo->mem); 425 } 426 427 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 428 { 429 int r; 430 431 if (bo->resv == &bo->ttm_resv) 432 return 0; 433 434 BUG_ON(!reservation_object_trylock(&bo->ttm_resv)); 435 436 r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv); 437 if (r) 438 reservation_object_unlock(&bo->ttm_resv); 439 440 return r; 441 } 442 443 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 444 { 445 struct reservation_object_list *fobj; 446 struct dma_fence *fence; 447 int i; 448 449 fobj = reservation_object_get_list(&bo->ttm_resv); 450 fence = reservation_object_get_excl(&bo->ttm_resv); 451 if (fence && !fence->ops->signaled) 452 dma_fence_enable_sw_signaling(fence); 453 454 for (i = 0; fobj && i < fobj->shared_count; ++i) { 455 fence = rcu_dereference_protected(fobj->shared[i], 456 reservation_object_held(bo->resv)); 457 458 if (!fence->ops->signaled) 459 dma_fence_enable_sw_signaling(fence); 460 } 461 } 462 463 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 464 { 465 struct ttm_bo_device *bdev = bo->bdev; 466 struct ttm_bo_global *glob = bdev->glob; 467 int ret; 468 469 ret = ttm_bo_individualize_resv(bo); 470 if (ret) { 471 /* Last resort, if we fail to allocate memory for the 472 * fences block for the BO to become idle 473 */ 474 reservation_object_wait_timeout_rcu(bo->resv, true, false, 475 30 * HZ); 476 spin_lock(&glob->lru_lock); 477 goto error; 478 } 479 480 spin_lock(&glob->lru_lock); 481 ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; 482 if (!ret) { 483 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { 484 ttm_bo_del_from_lru(bo); 485 spin_unlock(&glob->lru_lock); 486 if (bo->resv != &bo->ttm_resv) 487 reservation_object_unlock(&bo->ttm_resv); 488 489 ttm_bo_cleanup_memtype_use(bo); 490 reservation_object_unlock(bo->resv); 491 return; 492 } 493 494 ttm_bo_flush_all_fences(bo); 495 496 /* 497 * Make NO_EVICT bos immediately available to 498 * shrinkers, now that they are queued for 499 * destruction. 500 */ 501 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 502 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 503 ttm_bo_add_to_lru(bo); 504 } 505 506 reservation_object_unlock(bo->resv); 507 } 508 if (bo->resv != &bo->ttm_resv) 509 reservation_object_unlock(&bo->ttm_resv); 510 511 error: 512 kref_get(&bo->list_kref); 513 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 514 spin_unlock(&glob->lru_lock); 515 516 schedule_delayed_work(&bdev->wq, 517 ((HZ / 100) < 1) ? 1 : HZ / 100); 518 } 519 520 /** 521 * function ttm_bo_cleanup_refs 522 * If bo idle, remove from delayed- and lru lists, and unref. 523 * If not idle, do nothing. 524 * 525 * Must be called with lru_lock and reservation held, this function 526 * will drop the lru lock and optionally the reservation lock before returning. 527 * 528 * @interruptible Any sleeps should occur interruptibly. 529 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 530 * @unlock_resv Unlock the reservation lock as well. 531 */ 532 533 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 534 bool interruptible, bool no_wait_gpu, 535 bool unlock_resv) 536 { 537 struct ttm_bo_global *glob = bo->bdev->glob; 538 struct reservation_object *resv; 539 int ret; 540 541 if (unlikely(list_empty(&bo->ddestroy))) 542 resv = bo->resv; 543 else 544 resv = &bo->ttm_resv; 545 546 if (reservation_object_test_signaled_rcu(resv, true)) 547 ret = 0; 548 else 549 ret = -EBUSY; 550 551 if (ret && !no_wait_gpu) { 552 long lret; 553 554 if (unlock_resv) 555 reservation_object_unlock(bo->resv); 556 spin_unlock(&glob->lru_lock); 557 558 lret = reservation_object_wait_timeout_rcu(resv, true, 559 interruptible, 560 30 * HZ); 561 562 if (lret < 0) 563 return lret; 564 else if (lret == 0) 565 return -EBUSY; 566 567 spin_lock(&glob->lru_lock); 568 if (unlock_resv && !reservation_object_trylock(bo->resv)) { 569 /* 570 * We raced, and lost, someone else holds the reservation now, 571 * and is probably busy in ttm_bo_cleanup_memtype_use. 572 * 573 * Even if it's not the case, because we finished waiting any 574 * delayed destruction would succeed, so just return success 575 * here. 576 */ 577 spin_unlock(&glob->lru_lock); 578 return 0; 579 } 580 ret = 0; 581 } 582 583 if (ret || unlikely(list_empty(&bo->ddestroy))) { 584 if (unlock_resv) 585 reservation_object_unlock(bo->resv); 586 spin_unlock(&glob->lru_lock); 587 return ret; 588 } 589 590 ttm_bo_del_from_lru(bo); 591 list_del_init(&bo->ddestroy); 592 kref_put(&bo->list_kref, ttm_bo_ref_bug); 593 594 spin_unlock(&glob->lru_lock); 595 ttm_bo_cleanup_memtype_use(bo); 596 597 if (unlock_resv) 598 reservation_object_unlock(bo->resv); 599 600 return 0; 601 } 602 603 /** 604 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 605 * encountered buffers. 606 */ 607 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 608 { 609 struct ttm_bo_global *glob = bdev->glob; 610 struct list_head removed; 611 bool empty; 612 613 INIT_LIST_HEAD(&removed); 614 615 spin_lock(&glob->lru_lock); 616 while (!list_empty(&bdev->ddestroy)) { 617 struct ttm_buffer_object *bo; 618 619 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, 620 ddestroy); 621 kref_get(&bo->list_kref); 622 list_move_tail(&bo->ddestroy, &removed); 623 624 if (remove_all || bo->resv != &bo->ttm_resv) { 625 spin_unlock(&glob->lru_lock); 626 reservation_object_lock(bo->resv, NULL); 627 628 spin_lock(&glob->lru_lock); 629 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 630 631 } else if (reservation_object_trylock(bo->resv)) { 632 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 633 } else { 634 spin_unlock(&glob->lru_lock); 635 } 636 637 kref_put(&bo->list_kref, ttm_bo_release_list); 638 spin_lock(&glob->lru_lock); 639 } 640 list_splice_tail(&removed, &bdev->ddestroy); 641 empty = list_empty(&bdev->ddestroy); 642 spin_unlock(&glob->lru_lock); 643 644 return empty; 645 } 646 647 static void ttm_bo_delayed_workqueue(struct work_struct *work) 648 { 649 struct ttm_bo_device *bdev = 650 container_of(work, struct ttm_bo_device, wq.work); 651 652 if (!ttm_bo_delayed_delete(bdev, false)) 653 schedule_delayed_work(&bdev->wq, 654 ((HZ / 100) < 1) ? 1 : HZ / 100); 655 } 656 657 static void ttm_bo_release(struct kref *kref) 658 { 659 struct ttm_buffer_object *bo = 660 container_of(kref, struct ttm_buffer_object, kref); 661 struct ttm_bo_device *bdev = bo->bdev; 662 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 663 664 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 665 ttm_mem_io_lock(man, false); 666 ttm_mem_io_free_vm(bo); 667 ttm_mem_io_unlock(man); 668 ttm_bo_cleanup_refs_or_queue(bo); 669 kref_put(&bo->list_kref, ttm_bo_release_list); 670 } 671 672 void ttm_bo_put(struct ttm_buffer_object *bo) 673 { 674 kref_put(&bo->kref, ttm_bo_release); 675 } 676 EXPORT_SYMBOL(ttm_bo_put); 677 678 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 679 { 680 struct ttm_buffer_object *bo = *p_bo; 681 682 *p_bo = NULL; 683 ttm_bo_put(bo); 684 } 685 EXPORT_SYMBOL(ttm_bo_unref); 686 687 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 688 { 689 return cancel_delayed_work_sync(&bdev->wq); 690 } 691 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 692 693 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 694 { 695 if (resched) 696 schedule_delayed_work(&bdev->wq, 697 ((HZ / 100) < 1) ? 1 : HZ / 100); 698 } 699 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 700 701 static int ttm_bo_evict(struct ttm_buffer_object *bo, 702 struct ttm_operation_ctx *ctx) 703 { 704 struct ttm_bo_device *bdev = bo->bdev; 705 struct ttm_mem_reg evict_mem; 706 struct ttm_placement placement; 707 int ret = 0; 708 709 reservation_object_assert_held(bo->resv); 710 711 placement.num_placement = 0; 712 placement.num_busy_placement = 0; 713 bdev->driver->evict_flags(bo, &placement); 714 715 if (!placement.num_placement && !placement.num_busy_placement) { 716 ret = ttm_bo_pipeline_gutting(bo); 717 if (ret) 718 return ret; 719 720 return ttm_tt_create(bo, false); 721 } 722 723 evict_mem = bo->mem; 724 evict_mem.mm_node = NULL; 725 evict_mem.bus.io_reserved_vm = false; 726 evict_mem.bus.io_reserved_count = 0; 727 728 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 729 if (ret) { 730 if (ret != -ERESTARTSYS) { 731 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 732 bo); 733 ttm_bo_mem_space_debug(bo, &placement); 734 } 735 goto out; 736 } 737 738 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); 739 if (unlikely(ret)) { 740 if (ret != -ERESTARTSYS) 741 pr_err("Buffer eviction failed\n"); 742 ttm_bo_mem_put(bo, &evict_mem); 743 goto out; 744 } 745 bo->evicted = true; 746 out: 747 return ret; 748 } 749 750 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 751 const struct ttm_place *place) 752 { 753 /* Don't evict this BO if it's outside of the 754 * requested placement range 755 */ 756 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 757 (place->lpfn && place->lpfn <= bo->mem.start)) 758 return false; 759 760 return true; 761 } 762 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 763 764 /** 765 * Check the target bo is allowable to be evicted or swapout, including cases: 766 * 767 * a. if share same reservation object with ctx->resv, have assumption 768 * reservation objects should already be locked, so not lock again and 769 * return true directly when either the opreation allow_reserved_eviction 770 * or the target bo already is in delayed free list; 771 * 772 * b. Otherwise, trylock it. 773 */ 774 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 775 struct ttm_operation_ctx *ctx, bool *locked) 776 { 777 bool ret = false; 778 779 *locked = false; 780 if (bo->resv == ctx->resv) { 781 reservation_object_assert_held(bo->resv); 782 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT 783 || !list_empty(&bo->ddestroy)) 784 ret = true; 785 } else { 786 *locked = reservation_object_trylock(bo->resv); 787 ret = *locked; 788 } 789 790 return ret; 791 } 792 793 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 794 uint32_t mem_type, 795 const struct ttm_place *place, 796 struct ttm_operation_ctx *ctx) 797 { 798 struct ttm_bo_global *glob = bdev->glob; 799 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 800 struct ttm_buffer_object *bo = NULL; 801 bool locked = false; 802 unsigned i; 803 int ret; 804 805 spin_lock(&glob->lru_lock); 806 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 807 list_for_each_entry(bo, &man->lru[i], lru) { 808 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) 809 continue; 810 811 if (place && !bdev->driver->eviction_valuable(bo, 812 place)) { 813 if (locked) 814 reservation_object_unlock(bo->resv); 815 continue; 816 } 817 break; 818 } 819 820 /* If the inner loop terminated early, we have our candidate */ 821 if (&bo->lru != &man->lru[i]) 822 break; 823 824 bo = NULL; 825 } 826 827 if (!bo) { 828 spin_unlock(&glob->lru_lock); 829 return -EBUSY; 830 } 831 832 kref_get(&bo->list_kref); 833 834 if (!list_empty(&bo->ddestroy)) { 835 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 836 ctx->no_wait_gpu, locked); 837 kref_put(&bo->list_kref, ttm_bo_release_list); 838 return ret; 839 } 840 841 ttm_bo_del_from_lru(bo); 842 spin_unlock(&glob->lru_lock); 843 844 ret = ttm_bo_evict(bo, ctx); 845 if (locked) { 846 ttm_bo_unreserve(bo); 847 } else { 848 spin_lock(&glob->lru_lock); 849 ttm_bo_add_to_lru(bo); 850 spin_unlock(&glob->lru_lock); 851 } 852 853 kref_put(&bo->list_kref, ttm_bo_release_list); 854 return ret; 855 } 856 857 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 858 { 859 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 860 861 if (mem->mm_node) 862 (*man->func->put_node)(man, mem); 863 } 864 EXPORT_SYMBOL(ttm_bo_mem_put); 865 866 /** 867 * Add the last move fence to the BO and reserve a new shared slot. 868 */ 869 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 870 struct ttm_mem_type_manager *man, 871 struct ttm_mem_reg *mem) 872 { 873 struct dma_fence *fence; 874 int ret; 875 876 spin_lock(&man->move_lock); 877 fence = dma_fence_get(man->move); 878 spin_unlock(&man->move_lock); 879 880 if (fence) { 881 reservation_object_add_shared_fence(bo->resv, fence); 882 883 ret = reservation_object_reserve_shared(bo->resv, 1); 884 if (unlikely(ret)) 885 return ret; 886 887 dma_fence_put(bo->moving); 888 bo->moving = fence; 889 } 890 891 return 0; 892 } 893 894 /** 895 * Repeatedly evict memory from the LRU for @mem_type until we create enough 896 * space, or we've evicted everything and there isn't enough space. 897 */ 898 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 899 uint32_t mem_type, 900 const struct ttm_place *place, 901 struct ttm_mem_reg *mem, 902 struct ttm_operation_ctx *ctx) 903 { 904 struct ttm_bo_device *bdev = bo->bdev; 905 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 906 int ret; 907 908 do { 909 ret = (*man->func->get_node)(man, bo, place, mem); 910 if (unlikely(ret != 0)) 911 return ret; 912 if (mem->mm_node) 913 break; 914 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); 915 if (unlikely(ret != 0)) 916 return ret; 917 } while (1); 918 mem->mem_type = mem_type; 919 return ttm_bo_add_move_fence(bo, man, mem); 920 } 921 922 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 923 uint32_t cur_placement, 924 uint32_t proposed_placement) 925 { 926 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 927 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 928 929 /** 930 * Keep current caching if possible. 931 */ 932 933 if ((cur_placement & caching) != 0) 934 result |= (cur_placement & caching); 935 else if ((man->default_caching & caching) != 0) 936 result |= man->default_caching; 937 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 938 result |= TTM_PL_FLAG_CACHED; 939 else if ((TTM_PL_FLAG_WC & caching) != 0) 940 result |= TTM_PL_FLAG_WC; 941 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 942 result |= TTM_PL_FLAG_UNCACHED; 943 944 return result; 945 } 946 947 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 948 uint32_t mem_type, 949 const struct ttm_place *place, 950 uint32_t *masked_placement) 951 { 952 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 953 954 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 955 return false; 956 957 if ((place->flags & man->available_caching) == 0) 958 return false; 959 960 cur_flags |= (place->flags & man->available_caching); 961 962 *masked_placement = cur_flags; 963 return true; 964 } 965 966 /** 967 * Creates space for memory region @mem according to its type. 968 * 969 * This function first searches for free space in compatible memory types in 970 * the priority order defined by the driver. If free space isn't found, then 971 * ttm_bo_mem_force_space is attempted in priority order to evict and find 972 * space. 973 */ 974 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 975 struct ttm_placement *placement, 976 struct ttm_mem_reg *mem, 977 struct ttm_operation_ctx *ctx) 978 { 979 struct ttm_bo_device *bdev = bo->bdev; 980 struct ttm_mem_type_manager *man; 981 uint32_t mem_type = TTM_PL_SYSTEM; 982 uint32_t cur_flags = 0; 983 bool type_found = false; 984 bool type_ok = false; 985 bool has_erestartsys = false; 986 int i, ret; 987 988 ret = reservation_object_reserve_shared(bo->resv, 1); 989 if (unlikely(ret)) 990 return ret; 991 992 mem->mm_node = NULL; 993 for (i = 0; i < placement->num_placement; ++i) { 994 const struct ttm_place *place = &placement->placement[i]; 995 996 ret = ttm_mem_type_from_place(place, &mem_type); 997 if (ret) 998 return ret; 999 man = &bdev->man[mem_type]; 1000 if (!man->has_type || !man->use_type) 1001 continue; 1002 1003 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 1004 &cur_flags); 1005 1006 if (!type_ok) 1007 continue; 1008 1009 type_found = true; 1010 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1011 cur_flags); 1012 /* 1013 * Use the access and other non-mapping-related flag bits from 1014 * the memory placement flags to the current flags 1015 */ 1016 ttm_flag_masked(&cur_flags, place->flags, 1017 ~TTM_PL_MASK_MEMTYPE); 1018 1019 if (mem_type == TTM_PL_SYSTEM) 1020 break; 1021 1022 ret = (*man->func->get_node)(man, bo, place, mem); 1023 if (unlikely(ret)) 1024 return ret; 1025 1026 if (mem->mm_node) { 1027 ret = ttm_bo_add_move_fence(bo, man, mem); 1028 if (unlikely(ret)) { 1029 (*man->func->put_node)(man, mem); 1030 return ret; 1031 } 1032 break; 1033 } 1034 } 1035 1036 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 1037 mem->mem_type = mem_type; 1038 mem->placement = cur_flags; 1039 return 0; 1040 } 1041 1042 for (i = 0; i < placement->num_busy_placement; ++i) { 1043 const struct ttm_place *place = &placement->busy_placement[i]; 1044 1045 ret = ttm_mem_type_from_place(place, &mem_type); 1046 if (ret) 1047 return ret; 1048 man = &bdev->man[mem_type]; 1049 if (!man->has_type || !man->use_type) 1050 continue; 1051 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 1052 continue; 1053 1054 type_found = true; 1055 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1056 cur_flags); 1057 /* 1058 * Use the access and other non-mapping-related flag bits from 1059 * the memory placement flags to the current flags 1060 */ 1061 ttm_flag_masked(&cur_flags, place->flags, 1062 ~TTM_PL_MASK_MEMTYPE); 1063 1064 if (mem_type == TTM_PL_SYSTEM) { 1065 mem->mem_type = mem_type; 1066 mem->placement = cur_flags; 1067 mem->mm_node = NULL; 1068 return 0; 1069 } 1070 1071 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx); 1072 if (ret == 0 && mem->mm_node) { 1073 mem->placement = cur_flags; 1074 return 0; 1075 } 1076 if (ret == -ERESTARTSYS) 1077 has_erestartsys = true; 1078 } 1079 1080 if (!type_found) { 1081 pr_err(TTM_PFX "No compatible memory type found\n"); 1082 return -EINVAL; 1083 } 1084 1085 return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1086 } 1087 EXPORT_SYMBOL(ttm_bo_mem_space); 1088 1089 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1090 struct ttm_placement *placement, 1091 struct ttm_operation_ctx *ctx) 1092 { 1093 int ret = 0; 1094 struct ttm_mem_reg mem; 1095 1096 reservation_object_assert_held(bo->resv); 1097 1098 mem.num_pages = bo->num_pages; 1099 mem.size = mem.num_pages << PAGE_SHIFT; 1100 mem.page_alignment = bo->mem.page_alignment; 1101 mem.bus.io_reserved_vm = false; 1102 mem.bus.io_reserved_count = 0; 1103 /* 1104 * Determine where to move the buffer. 1105 */ 1106 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); 1107 if (ret) 1108 goto out_unlock; 1109 ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); 1110 out_unlock: 1111 if (ret && mem.mm_node) 1112 ttm_bo_mem_put(bo, &mem); 1113 return ret; 1114 } 1115 1116 static bool ttm_bo_places_compat(const struct ttm_place *places, 1117 unsigned num_placement, 1118 struct ttm_mem_reg *mem, 1119 uint32_t *new_flags) 1120 { 1121 unsigned i; 1122 1123 for (i = 0; i < num_placement; i++) { 1124 const struct ttm_place *heap = &places[i]; 1125 1126 if (mem->mm_node && (mem->start < heap->fpfn || 1127 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1128 continue; 1129 1130 *new_flags = heap->flags; 1131 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1132 (*new_flags & mem->placement & TTM_PL_MASK_MEM) && 1133 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 1134 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 1135 return true; 1136 } 1137 return false; 1138 } 1139 1140 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1141 struct ttm_mem_reg *mem, 1142 uint32_t *new_flags) 1143 { 1144 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 1145 mem, new_flags)) 1146 return true; 1147 1148 if ((placement->busy_placement != placement->placement || 1149 placement->num_busy_placement > placement->num_placement) && 1150 ttm_bo_places_compat(placement->busy_placement, 1151 placement->num_busy_placement, 1152 mem, new_flags)) 1153 return true; 1154 1155 return false; 1156 } 1157 EXPORT_SYMBOL(ttm_bo_mem_compat); 1158 1159 int ttm_bo_validate(struct ttm_buffer_object *bo, 1160 struct ttm_placement *placement, 1161 struct ttm_operation_ctx *ctx) 1162 { 1163 int ret; 1164 uint32_t new_flags; 1165 1166 reservation_object_assert_held(bo->resv); 1167 /* 1168 * Check whether we need to move buffer. 1169 */ 1170 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1171 ret = ttm_bo_move_buffer(bo, placement, ctx); 1172 if (ret) 1173 return ret; 1174 } else { 1175 /* 1176 * Use the access and other non-mapping-related flag bits from 1177 * the compatible memory placement flags to the active flags 1178 */ 1179 ttm_flag_masked(&bo->mem.placement, new_flags, 1180 ~TTM_PL_MASK_MEMTYPE); 1181 } 1182 /* 1183 * We might need to add a TTM. 1184 */ 1185 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1186 ret = ttm_tt_create(bo, true); 1187 if (ret) 1188 return ret; 1189 } 1190 return 0; 1191 } 1192 EXPORT_SYMBOL(ttm_bo_validate); 1193 1194 int ttm_bo_init_reserved(struct ttm_bo_device *bdev, 1195 struct ttm_buffer_object *bo, 1196 unsigned long size, 1197 enum ttm_bo_type type, 1198 struct ttm_placement *placement, 1199 uint32_t page_alignment, 1200 struct ttm_operation_ctx *ctx, 1201 size_t acc_size, 1202 struct sg_table *sg, 1203 struct reservation_object *resv, 1204 void (*destroy) (struct ttm_buffer_object *)) 1205 { 1206 int ret = 0; 1207 unsigned long num_pages; 1208 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1209 bool locked; 1210 1211 ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); 1212 if (ret) { 1213 pr_err("Out of kernel memory\n"); 1214 if (destroy) 1215 (*destroy)(bo); 1216 else 1217 kfree(bo); 1218 return -ENOMEM; 1219 } 1220 1221 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1222 if (num_pages == 0) { 1223 pr_err("Illegal buffer object size\n"); 1224 if (destroy) 1225 (*destroy)(bo); 1226 else 1227 kfree(bo); 1228 ttm_mem_global_free(mem_glob, acc_size); 1229 return -EINVAL; 1230 } 1231 bo->destroy = destroy ? destroy : ttm_bo_default_destroy; 1232 1233 kref_init(&bo->kref); 1234 kref_init(&bo->list_kref); 1235 atomic_set(&bo->cpu_writers, 0); 1236 INIT_LIST_HEAD(&bo->lru); 1237 INIT_LIST_HEAD(&bo->ddestroy); 1238 INIT_LIST_HEAD(&bo->swap); 1239 INIT_LIST_HEAD(&bo->io_reserve_lru); 1240 mutex_init(&bo->wu_mutex); 1241 bo->bdev = bdev; 1242 bo->type = type; 1243 bo->num_pages = num_pages; 1244 bo->mem.size = num_pages << PAGE_SHIFT; 1245 bo->mem.mem_type = TTM_PL_SYSTEM; 1246 bo->mem.num_pages = bo->num_pages; 1247 bo->mem.mm_node = NULL; 1248 bo->mem.page_alignment = page_alignment; 1249 bo->mem.bus.io_reserved_vm = false; 1250 bo->mem.bus.io_reserved_count = 0; 1251 bo->moving = NULL; 1252 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1253 bo->acc_size = acc_size; 1254 bo->sg = sg; 1255 if (resv) { 1256 bo->resv = resv; 1257 reservation_object_assert_held(bo->resv); 1258 } else { 1259 bo->resv = &bo->ttm_resv; 1260 } 1261 reservation_object_init(&bo->ttm_resv); 1262 atomic_inc(&bo->bdev->glob->bo_count); 1263 drm_vma_node_reset(&bo->vma_node); 1264 1265 /* 1266 * For ttm_bo_type_device buffers, allocate 1267 * address space from the device. 1268 */ 1269 if (bo->type == ttm_bo_type_device || 1270 bo->type == ttm_bo_type_sg) 1271 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1272 bo->mem.num_pages); 1273 1274 /* passed reservation objects should already be locked, 1275 * since otherwise lockdep will be angered in radeon. 1276 */ 1277 if (!resv) { 1278 locked = reservation_object_trylock(bo->resv); 1279 WARN_ON(!locked); 1280 } 1281 1282 if (likely(!ret)) 1283 ret = ttm_bo_validate(bo, placement, ctx); 1284 1285 if (unlikely(ret)) { 1286 if (!resv) 1287 ttm_bo_unreserve(bo); 1288 1289 ttm_bo_put(bo); 1290 return ret; 1291 } 1292 1293 if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 1294 spin_lock(&bdev->glob->lru_lock); 1295 ttm_bo_add_to_lru(bo); 1296 spin_unlock(&bdev->glob->lru_lock); 1297 } 1298 1299 return ret; 1300 } 1301 EXPORT_SYMBOL(ttm_bo_init_reserved); 1302 1303 int ttm_bo_init(struct ttm_bo_device *bdev, 1304 struct ttm_buffer_object *bo, 1305 unsigned long size, 1306 enum ttm_bo_type type, 1307 struct ttm_placement *placement, 1308 uint32_t page_alignment, 1309 bool interruptible, 1310 size_t acc_size, 1311 struct sg_table *sg, 1312 struct reservation_object *resv, 1313 void (*destroy) (struct ttm_buffer_object *)) 1314 { 1315 struct ttm_operation_ctx ctx = { interruptible, false }; 1316 int ret; 1317 1318 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1319 page_alignment, &ctx, acc_size, 1320 sg, resv, destroy); 1321 if (ret) 1322 return ret; 1323 1324 if (!resv) 1325 ttm_bo_unreserve(bo); 1326 1327 return 0; 1328 } 1329 EXPORT_SYMBOL(ttm_bo_init); 1330 1331 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1332 unsigned long bo_size, 1333 unsigned struct_size) 1334 { 1335 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1336 size_t size = 0; 1337 1338 size += ttm_round_pot(struct_size); 1339 size += ttm_round_pot(npages * sizeof(void *)); 1340 size += ttm_round_pot(sizeof(struct ttm_tt)); 1341 return size; 1342 } 1343 EXPORT_SYMBOL(ttm_bo_acc_size); 1344 1345 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1346 unsigned long bo_size, 1347 unsigned struct_size) 1348 { 1349 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1350 size_t size = 0; 1351 1352 size += ttm_round_pot(struct_size); 1353 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1354 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1355 return size; 1356 } 1357 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1358 1359 int ttm_bo_create(struct ttm_bo_device *bdev, 1360 unsigned long size, 1361 enum ttm_bo_type type, 1362 struct ttm_placement *placement, 1363 uint32_t page_alignment, 1364 bool interruptible, 1365 struct ttm_buffer_object **p_bo) 1366 { 1367 struct ttm_buffer_object *bo; 1368 size_t acc_size; 1369 int ret; 1370 1371 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1372 if (unlikely(bo == NULL)) 1373 return -ENOMEM; 1374 1375 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1376 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1377 interruptible, acc_size, 1378 NULL, NULL, NULL); 1379 if (likely(ret == 0)) 1380 *p_bo = bo; 1381 1382 return ret; 1383 } 1384 EXPORT_SYMBOL(ttm_bo_create); 1385 1386 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1387 unsigned mem_type) 1388 { 1389 struct ttm_operation_ctx ctx = { 1390 .interruptible = false, 1391 .no_wait_gpu = false, 1392 .flags = TTM_OPT_FLAG_FORCE_ALLOC 1393 }; 1394 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1395 struct ttm_bo_global *glob = bdev->glob; 1396 struct dma_fence *fence; 1397 int ret; 1398 unsigned i; 1399 1400 /* 1401 * Can't use standard list traversal since we're unlocking. 1402 */ 1403 1404 spin_lock(&glob->lru_lock); 1405 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1406 while (!list_empty(&man->lru[i])) { 1407 spin_unlock(&glob->lru_lock); 1408 ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); 1409 if (ret) 1410 return ret; 1411 spin_lock(&glob->lru_lock); 1412 } 1413 } 1414 spin_unlock(&glob->lru_lock); 1415 1416 spin_lock(&man->move_lock); 1417 fence = dma_fence_get(man->move); 1418 spin_unlock(&man->move_lock); 1419 1420 if (fence) { 1421 ret = dma_fence_wait(fence, false); 1422 dma_fence_put(fence); 1423 if (ret) 1424 return ret; 1425 } 1426 1427 return 0; 1428 } 1429 1430 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1431 { 1432 struct ttm_mem_type_manager *man; 1433 int ret = -EINVAL; 1434 1435 if (mem_type >= TTM_NUM_MEM_TYPES) { 1436 pr_err("Illegal memory type %d\n", mem_type); 1437 return ret; 1438 } 1439 man = &bdev->man[mem_type]; 1440 1441 if (!man->has_type) { 1442 pr_err("Trying to take down uninitialized memory manager type %u\n", 1443 mem_type); 1444 return ret; 1445 } 1446 1447 man->use_type = false; 1448 man->has_type = false; 1449 1450 ret = 0; 1451 if (mem_type > 0) { 1452 ret = ttm_bo_force_list_clean(bdev, mem_type); 1453 if (ret) { 1454 pr_err("Cleanup eviction failed\n"); 1455 return ret; 1456 } 1457 1458 ret = (*man->func->takedown)(man); 1459 } 1460 1461 dma_fence_put(man->move); 1462 man->move = NULL; 1463 1464 return ret; 1465 } 1466 EXPORT_SYMBOL(ttm_bo_clean_mm); 1467 1468 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1469 { 1470 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1471 1472 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1473 pr_err("Illegal memory manager memory type %u\n", mem_type); 1474 return -EINVAL; 1475 } 1476 1477 if (!man->has_type) { 1478 pr_err("Memory type %u has not been initialized\n", mem_type); 1479 return 0; 1480 } 1481 1482 return ttm_bo_force_list_clean(bdev, mem_type); 1483 } 1484 EXPORT_SYMBOL(ttm_bo_evict_mm); 1485 1486 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1487 unsigned long p_size) 1488 { 1489 int ret; 1490 struct ttm_mem_type_manager *man; 1491 unsigned i; 1492 1493 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1494 man = &bdev->man[type]; 1495 BUG_ON(man->has_type); 1496 man->io_reserve_fastpath = true; 1497 man->use_io_reserve_lru = false; 1498 mutex_init(&man->io_reserve_mutex); 1499 spin_lock_init(&man->move_lock); 1500 INIT_LIST_HEAD(&man->io_reserve_lru); 1501 1502 ret = bdev->driver->init_mem_type(bdev, type, man); 1503 if (ret) 1504 return ret; 1505 man->bdev = bdev; 1506 1507 if (type != TTM_PL_SYSTEM) { 1508 ret = (*man->func->init)(man, p_size); 1509 if (ret) 1510 return ret; 1511 } 1512 man->has_type = true; 1513 man->use_type = true; 1514 man->size = p_size; 1515 1516 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1517 INIT_LIST_HEAD(&man->lru[i]); 1518 man->move = NULL; 1519 1520 return 0; 1521 } 1522 EXPORT_SYMBOL(ttm_bo_init_mm); 1523 1524 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1525 { 1526 struct ttm_bo_global *glob = 1527 container_of(kobj, struct ttm_bo_global, kobj); 1528 1529 __free_page(glob->dummy_read_page); 1530 } 1531 1532 static void ttm_bo_global_release(void) 1533 { 1534 struct ttm_bo_global *glob = &ttm_bo_glob; 1535 1536 mutex_lock(&ttm_global_mutex); 1537 if (--glob->use_count > 0) 1538 goto out; 1539 1540 kobject_del(&glob->kobj); 1541 kobject_put(&glob->kobj); 1542 ttm_mem_global_release(&ttm_mem_glob); 1543 out: 1544 mutex_unlock(&ttm_global_mutex); 1545 } 1546 1547 static int ttm_bo_global_init(void) 1548 { 1549 struct ttm_bo_global *glob = &ttm_bo_glob; 1550 int ret = 0; 1551 unsigned i; 1552 1553 mutex_lock(&ttm_global_mutex); 1554 if (++glob->use_count > 1) 1555 goto out; 1556 1557 ret = ttm_mem_global_init(&ttm_mem_glob); 1558 if (ret) 1559 goto out; 1560 1561 spin_lock_init(&glob->lru_lock); 1562 glob->mem_glob = &ttm_mem_glob; 1563 glob->mem_glob->bo_glob = glob; 1564 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1565 1566 if (unlikely(glob->dummy_read_page == NULL)) { 1567 ret = -ENOMEM; 1568 goto out; 1569 } 1570 1571 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1572 INIT_LIST_HEAD(&glob->swap_lru[i]); 1573 INIT_LIST_HEAD(&glob->device_list); 1574 atomic_set(&glob->bo_count, 0); 1575 1576 ret = kobject_init_and_add( 1577 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1578 if (unlikely(ret != 0)) 1579 kobject_put(&glob->kobj); 1580 out: 1581 mutex_unlock(&ttm_global_mutex); 1582 return ret; 1583 } 1584 1585 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1586 { 1587 int ret = 0; 1588 unsigned i = TTM_NUM_MEM_TYPES; 1589 struct ttm_mem_type_manager *man; 1590 struct ttm_bo_global *glob = bdev->glob; 1591 1592 while (i--) { 1593 man = &bdev->man[i]; 1594 if (man->has_type) { 1595 man->use_type = false; 1596 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1597 ret = -EBUSY; 1598 pr_err("DRM memory manager type %d is not clean\n", 1599 i); 1600 } 1601 man->has_type = false; 1602 } 1603 } 1604 1605 mutex_lock(&ttm_global_mutex); 1606 list_del(&bdev->device_list); 1607 mutex_unlock(&ttm_global_mutex); 1608 1609 cancel_delayed_work_sync(&bdev->wq); 1610 1611 if (ttm_bo_delayed_delete(bdev, true)) 1612 pr_debug("Delayed destroy list was clean\n"); 1613 1614 spin_lock(&glob->lru_lock); 1615 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1616 if (list_empty(&bdev->man[0].lru[0])) 1617 pr_debug("Swap list %d was clean\n", i); 1618 spin_unlock(&glob->lru_lock); 1619 1620 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1621 1622 if (!ret) 1623 ttm_bo_global_release(); 1624 1625 return ret; 1626 } 1627 EXPORT_SYMBOL(ttm_bo_device_release); 1628 1629 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1630 struct ttm_bo_driver *driver, 1631 struct address_space *mapping, 1632 uint64_t file_page_offset, 1633 bool need_dma32) 1634 { 1635 struct ttm_bo_global *glob = &ttm_bo_glob; 1636 int ret; 1637 1638 ret = ttm_bo_global_init(); 1639 if (ret) 1640 return ret; 1641 1642 bdev->driver = driver; 1643 1644 memset(bdev->man, 0, sizeof(bdev->man)); 1645 1646 /* 1647 * Initialize the system memory buffer type. 1648 * Other types need to be driver / IOCTL initialized. 1649 */ 1650 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1651 if (unlikely(ret != 0)) 1652 goto out_no_sys; 1653 1654 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1655 0x10000000); 1656 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1657 INIT_LIST_HEAD(&bdev->ddestroy); 1658 bdev->dev_mapping = mapping; 1659 bdev->glob = glob; 1660 bdev->need_dma32 = need_dma32; 1661 mutex_lock(&ttm_global_mutex); 1662 list_add_tail(&bdev->device_list, &glob->device_list); 1663 mutex_unlock(&ttm_global_mutex); 1664 1665 return 0; 1666 out_no_sys: 1667 ttm_bo_global_release(); 1668 return ret; 1669 } 1670 EXPORT_SYMBOL(ttm_bo_device_init); 1671 1672 /* 1673 * buffer object vm functions. 1674 */ 1675 1676 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1677 { 1678 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1679 1680 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1681 if (mem->mem_type == TTM_PL_SYSTEM) 1682 return false; 1683 1684 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1685 return false; 1686 1687 if (mem->placement & TTM_PL_FLAG_CACHED) 1688 return false; 1689 } 1690 return true; 1691 } 1692 1693 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1694 { 1695 struct ttm_bo_device *bdev = bo->bdev; 1696 1697 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1698 ttm_mem_io_free_vm(bo); 1699 } 1700 1701 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1702 { 1703 struct ttm_bo_device *bdev = bo->bdev; 1704 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1705 1706 ttm_mem_io_lock(man, false); 1707 ttm_bo_unmap_virtual_locked(bo); 1708 ttm_mem_io_unlock(man); 1709 } 1710 1711 1712 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1713 1714 int ttm_bo_wait(struct ttm_buffer_object *bo, 1715 bool interruptible, bool no_wait) 1716 { 1717 long timeout = 15 * HZ; 1718 1719 if (no_wait) { 1720 if (reservation_object_test_signaled_rcu(bo->resv, true)) 1721 return 0; 1722 else 1723 return -EBUSY; 1724 } 1725 1726 timeout = reservation_object_wait_timeout_rcu(bo->resv, true, 1727 interruptible, timeout); 1728 if (timeout < 0) 1729 return timeout; 1730 1731 if (timeout == 0) 1732 return -EBUSY; 1733 1734 reservation_object_add_excl_fence(bo->resv, NULL); 1735 return 0; 1736 } 1737 EXPORT_SYMBOL(ttm_bo_wait); 1738 1739 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1740 { 1741 int ret = 0; 1742 1743 /* 1744 * Using ttm_bo_reserve makes sure the lru lists are updated. 1745 */ 1746 1747 ret = ttm_bo_reserve(bo, true, no_wait, NULL); 1748 if (unlikely(ret != 0)) 1749 return ret; 1750 ret = ttm_bo_wait(bo, true, no_wait); 1751 if (likely(ret == 0)) 1752 atomic_inc(&bo->cpu_writers); 1753 ttm_bo_unreserve(bo); 1754 return ret; 1755 } 1756 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1757 1758 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1759 { 1760 atomic_dec(&bo->cpu_writers); 1761 } 1762 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1763 1764 /** 1765 * A buffer object shrink method that tries to swap out the first 1766 * buffer object on the bo_global::swap_lru list. 1767 */ 1768 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) 1769 { 1770 struct ttm_buffer_object *bo; 1771 int ret = -EBUSY; 1772 bool locked; 1773 unsigned i; 1774 1775 spin_lock(&glob->lru_lock); 1776 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1777 list_for_each_entry(bo, &glob->swap_lru[i], swap) { 1778 if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { 1779 ret = 0; 1780 break; 1781 } 1782 } 1783 if (!ret) 1784 break; 1785 } 1786 1787 if (ret) { 1788 spin_unlock(&glob->lru_lock); 1789 return ret; 1790 } 1791 1792 kref_get(&bo->list_kref); 1793 1794 if (!list_empty(&bo->ddestroy)) { 1795 ret = ttm_bo_cleanup_refs(bo, false, false, locked); 1796 kref_put(&bo->list_kref, ttm_bo_release_list); 1797 return ret; 1798 } 1799 1800 ttm_bo_del_from_lru(bo); 1801 spin_unlock(&glob->lru_lock); 1802 1803 /** 1804 * Move to system cached 1805 */ 1806 1807 if (bo->mem.mem_type != TTM_PL_SYSTEM || 1808 bo->ttm->caching_state != tt_cached) { 1809 struct ttm_operation_ctx ctx = { false, false }; 1810 struct ttm_mem_reg evict_mem; 1811 1812 evict_mem = bo->mem; 1813 evict_mem.mm_node = NULL; 1814 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1815 evict_mem.mem_type = TTM_PL_SYSTEM; 1816 1817 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); 1818 if (unlikely(ret != 0)) 1819 goto out; 1820 } 1821 1822 /** 1823 * Make sure BO is idle. 1824 */ 1825 1826 ret = ttm_bo_wait(bo, false, false); 1827 if (unlikely(ret != 0)) 1828 goto out; 1829 1830 ttm_bo_unmap_virtual(bo); 1831 1832 /** 1833 * Swap out. Buffer will be swapped in again as soon as 1834 * anyone tries to access a ttm page. 1835 */ 1836 1837 if (bo->bdev->driver->swap_notify) 1838 bo->bdev->driver->swap_notify(bo); 1839 1840 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1841 out: 1842 1843 /** 1844 * 1845 * Unreserve without putting on LRU to avoid swapping out an 1846 * already swapped buffer. 1847 */ 1848 if (locked) 1849 reservation_object_unlock(bo->resv); 1850 kref_put(&bo->list_kref, ttm_bo_release_list); 1851 return ret; 1852 } 1853 EXPORT_SYMBOL(ttm_bo_swapout); 1854 1855 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1856 { 1857 struct ttm_operation_ctx ctx = { 1858 .interruptible = false, 1859 .no_wait_gpu = false 1860 }; 1861 1862 while (ttm_bo_swapout(bdev->glob, &ctx) == 0) 1863 ; 1864 } 1865 EXPORT_SYMBOL(ttm_bo_swapout_all); 1866 1867 /** 1868 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1869 * unreserved 1870 * 1871 * @bo: Pointer to buffer 1872 */ 1873 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1874 { 1875 int ret; 1876 1877 /* 1878 * In the absense of a wait_unlocked API, 1879 * Use the bo::wu_mutex to avoid triggering livelocks due to 1880 * concurrent use of this function. Note that this use of 1881 * bo::wu_mutex can go away if we change locking order to 1882 * mmap_sem -> bo::reserve. 1883 */ 1884 ret = mutex_lock_interruptible(&bo->wu_mutex); 1885 if (unlikely(ret != 0)) 1886 return -ERESTARTSYS; 1887 if (!ww_mutex_is_locked(&bo->resv->lock)) 1888 goto out_unlock; 1889 ret = reservation_object_lock_interruptible(bo->resv, NULL); 1890 if (ret == -EINTR) 1891 ret = -ERESTARTSYS; 1892 if (unlikely(ret != 0)) 1893 goto out_unlock; 1894 reservation_object_unlock(bo->resv); 1895 1896 out_unlock: 1897 mutex_unlock(&bo->wu_mutex); 1898 return ret; 1899 } 1900