11297bf2eSDirk Hohndel /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2ba4e7d97SThomas Hellstrom /************************************************************************** 3ba4e7d97SThomas Hellstrom * 4ba4e7d97SThomas Hellstrom * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5ba4e7d97SThomas Hellstrom * All Rights Reserved. 6ba4e7d97SThomas Hellstrom * 7ba4e7d97SThomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a 8ba4e7d97SThomas Hellstrom * copy of this software and associated documentation files (the 9ba4e7d97SThomas Hellstrom * "Software"), to deal in the Software without restriction, including 10ba4e7d97SThomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish, 11ba4e7d97SThomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to 12ba4e7d97SThomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to 13ba4e7d97SThomas Hellstrom * the following conditions: 14ba4e7d97SThomas Hellstrom * 15ba4e7d97SThomas Hellstrom * The above copyright notice and this permission notice (including the 16ba4e7d97SThomas Hellstrom * next paragraph) shall be included in all copies or substantial portions 17ba4e7d97SThomas Hellstrom * of the Software. 18ba4e7d97SThomas Hellstrom * 19ba4e7d97SThomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20ba4e7d97SThomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21ba4e7d97SThomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22ba4e7d97SThomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23ba4e7d97SThomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24ba4e7d97SThomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25ba4e7d97SThomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE. 26ba4e7d97SThomas Hellstrom * 27ba4e7d97SThomas Hellstrom **************************************************************************/ 28ba4e7d97SThomas Hellstrom /* 29ba4e7d97SThomas Hellstrom * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30ba4e7d97SThomas Hellstrom */ 31ba4e7d97SThomas Hellstrom 3225d0479aSJoe Perches #define pr_fmt(fmt) "[TTM] " fmt 3325d0479aSJoe Perches 34a3185f91SChristian König #include <drm/ttm/ttm_bo.h> 35760285e7SDavid Howells #include <drm/ttm/ttm_placement.h> 36a3185f91SChristian König #include <drm/ttm/ttm_tt.h> 37a3185f91SChristian König 38ba4e7d97SThomas Hellstrom #include <linux/jiffies.h> 39ba4e7d97SThomas Hellstrom #include <linux/slab.h> 40ba4e7d97SThomas Hellstrom #include <linux/sched.h> 41ba4e7d97SThomas Hellstrom #include <linux/mm.h> 42ba4e7d97SThomas Hellstrom #include <linux/file.h> 43ba4e7d97SThomas Hellstrom #include <linux/module.h> 4460063497SArun Sharma #include <linux/atomic.h> 4552791eeeSChristian König #include <linux/dma-resv.h> 46ba4e7d97SThomas Hellstrom 475cf82904SChristian König #include "ttm_module.h" 485cf82904SChristian König 49fb53f862SJerome Glisse static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 50fb53f862SJerome Glisse struct ttm_placement *placement) 51fb53f862SJerome Glisse { 52e154c4fcSJani Nikula struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); 539de59bc2SDave Airlie struct ttm_resource_manager *man; 5448e07c23SChristian König int i, mem_type; 55fb53f862SJerome Glisse 56fb53f862SJerome Glisse for (i = 0; i < placement->num_placement; i++) { 5748e07c23SChristian König mem_type = placement->placement[i].mem_type; 580f6be2c0SMichel Dänzer drm_printf(&p, " placement[%d]=0x%08X (%d)\n", 59f1217ed0SChristian König i, placement->placement[i].flags, mem_type); 609eca33f4SDave Airlie man = ttm_manager_type(bo->bdev, mem_type); 619de59bc2SDave Airlie ttm_resource_manager_debug(man, &p); 62fb53f862SJerome Glisse } 63fb53f862SJerome Glisse } 64fb53f862SJerome Glisse 65fee2ede1SChristian König /** 66fee2ede1SChristian König * ttm_bo_move_to_lru_tail 67fee2ede1SChristian König * 68fee2ede1SChristian König * @bo: The buffer object. 69fee2ede1SChristian König * 70fee2ede1SChristian König * Move this BO to the tail of all lru lists used to lookup and reserve an 71fee2ede1SChristian König * object. This function must be called with struct ttm_global::lru_lock 72fee2ede1SChristian König * held, and is used to make a BO less likely to be considered for eviction. 73fee2ede1SChristian König */ 74fee2ede1SChristian König void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 75ab749618SChristian König { 7652791eeeSChristian König dma_resv_assert_held(bo->base.resv); 77ab749618SChristian König 786a9b0289SChristian König if (bo->resource) 79fee2ede1SChristian König ttm_resource_move_to_lru_tail(bo->resource); 80ab749618SChristian König } 81ab749618SChristian König EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 82ab749618SChristian König 83fee2ede1SChristian König /** 84fee2ede1SChristian König * ttm_bo_set_bulk_move - update BOs bulk move object 85fee2ede1SChristian König * 86fee2ede1SChristian König * @bo: The buffer object. 87862643c7SLee Jones * @bulk: bulk move structure 88fee2ede1SChristian König * 89fee2ede1SChristian König * Update the BOs bulk move object, making sure that resources are added/removed 90fee2ede1SChristian König * as well. A bulk move allows to move many resource on the LRU at once, 91fee2ede1SChristian König * resulting in much less overhead of maintaining the LRU. 92fee2ede1SChristian König * The only requirement is that the resources stay together on the LRU and are 93fee2ede1SChristian König * never separated. This is enforces by setting the bulk_move structure on a BO. 94fee2ede1SChristian König * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of 95fee2ede1SChristian König * their LRU list. 96fee2ede1SChristian König */ 97fee2ede1SChristian König void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, 98fee2ede1SChristian König struct ttm_lru_bulk_move *bulk) 99fee2ede1SChristian König { 100fee2ede1SChristian König dma_resv_assert_held(bo->base.resv); 101fee2ede1SChristian König 102fee2ede1SChristian König if (bo->bulk_move == bulk) 103fee2ede1SChristian König return; 104fee2ede1SChristian König 105fee2ede1SChristian König spin_lock(&bo->bdev->lru_lock); 1060f9cd1eaSChristian König if (bo->resource) 1070f9cd1eaSChristian König ttm_resource_del_bulk_move(bo->resource, bo); 108fee2ede1SChristian König bo->bulk_move = bulk; 1090f9cd1eaSChristian König if (bo->resource) 1100f9cd1eaSChristian König ttm_resource_add_bulk_move(bo->resource, bo); 111fee2ede1SChristian König spin_unlock(&bo->bdev->lru_lock); 112fee2ede1SChristian König } 113fee2ede1SChristian König EXPORT_SYMBOL(ttm_bo_set_bulk_move); 114fee2ede1SChristian König 115ba4e7d97SThomas Hellstrom static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 1162966141aSDave Airlie struct ttm_resource *mem, bool evict, 117ebdf5651SDave Airlie struct ttm_operation_ctx *ctx, 118ebdf5651SDave Airlie struct ttm_place *hop) 119ba4e7d97SThomas Hellstrom { 1208af8a109SChristian König struct ttm_device *bdev = bo->bdev; 1214d8f6854SChristian König bool old_use_tt, new_use_tt; 122f5a9a938SChristian König int ret; 123ba4e7d97SThomas Hellstrom 12424243212SMatthew Auld old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; 1254d8f6854SChristian König new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; 126d3116756SChristian König 127fe662d84SChristian König ttm_bo_unmap_virtual(bo); 128ba4e7d97SThomas Hellstrom 129ba4e7d97SThomas Hellstrom /* 130ba4e7d97SThomas Hellstrom * Create and bind a ttm if required. 131ba4e7d97SThomas Hellstrom */ 132ba4e7d97SThomas Hellstrom 1334d8f6854SChristian König if (new_use_tt) { 134be1213a3SChristian König /* Zero init the new TTM structure if the old location should 135be1213a3SChristian König * have used one as well. 136be1213a3SChristian König */ 1374d8f6854SChristian König ret = ttm_tt_create(bo, old_use_tt); 138ba4e7d97SThomas Hellstrom if (ret) 139ba4e7d97SThomas Hellstrom goto out_err; 140ba4e7d97SThomas Hellstrom 14128ee846eSDave Airlie if (mem->mem_type != TTM_PL_SYSTEM) { 14228ee846eSDave Airlie ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); 1432040ec97SDave Airlie if (ret) 1442040ec97SDave Airlie goto out_err; 14528ee846eSDave Airlie } 1466236d953SDave Airlie } 147ba4e7d97SThomas Hellstrom 148c8d4c18bSChristian König ret = dma_resv_reserve_fences(bo->base.resv, 1); 149c8d4c18bSChristian König if (ret) 150c8d4c18bSChristian König goto out_err; 151c8d4c18bSChristian König 1528af8a109SChristian König ret = bdev->funcs->move(bo, evict, ctx, mem, hop); 153ebdf5651SDave Airlie if (ret) { 154ebdf5651SDave Airlie if (ret == -EMULTIHOP) 155ebdf5651SDave Airlie return ret; 1569f1feed2SBen Skeggs goto out_err; 157ebdf5651SDave Airlie } 158dc97b340SJerome Glisse 159e11bfb99SChristian König ctx->bytes_moved += bo->base.size; 160ba4e7d97SThomas Hellstrom return 0; 161ba4e7d97SThomas Hellstrom 162ba4e7d97SThomas Hellstrom out_err: 1634d8f6854SChristian König if (!old_use_tt) 1642ff6e69cSDave Airlie ttm_bo_tt_destroy(bo); 165ba4e7d97SThomas Hellstrom 166ba4e7d97SThomas Hellstrom return ret; 167ba4e7d97SThomas Hellstrom } 168ba4e7d97SThomas Hellstrom 169108cfddfSLee Jones /* 17040d857bbSThomas Hellstrom * Call bo::reserved. 1711df6a2ebSThomas Hellstrom * Will release GPU memory type usage on destruction. 17240d857bbSThomas Hellstrom * This is the place to put in driver specific hooks to release 17340d857bbSThomas Hellstrom * driver private resources. 17440d857bbSThomas Hellstrom * Will release the bo::reserved lock. 1751df6a2ebSThomas Hellstrom */ 1761df6a2ebSThomas Hellstrom 1771df6a2ebSThomas Hellstrom static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 1781df6a2ebSThomas Hellstrom { 1798af8a109SChristian König if (bo->bdev->funcs->delete_mem_notify) 1808af8a109SChristian König bo->bdev->funcs->delete_mem_notify(bo); 181dc97b340SJerome Glisse 1822ff6e69cSDave Airlie ttm_bo_tt_destroy(bo); 183bfa3357eSChristian König ttm_resource_free(bo, &bo->resource); 1841df6a2ebSThomas Hellstrom } 1851df6a2ebSThomas Hellstrom 186841e763bSChristian König static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 187841e763bSChristian König { 188841e763bSChristian König int r; 189841e763bSChristian König 190e532a135SGerd Hoffmann if (bo->base.resv == &bo->base._resv) 191841e763bSChristian König return 0; 192841e763bSChristian König 19352791eeeSChristian König BUG_ON(!dma_resv_trylock(&bo->base._resv)); 194841e763bSChristian König 19552791eeeSChristian König r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); 19652791eeeSChristian König dma_resv_unlock(&bo->base._resv); 197519c2de0SChristian König if (r) 198519c2de0SChristian König return r; 199519c2de0SChristian König 200519c2de0SChristian König if (bo->type != ttm_bo_type_sg) { 201519c2de0SChristian König /* This works because the BO is about to be destroyed and nobody 202519c2de0SChristian König * reference it any more. The only tricky case is the trylock on 203519c2de0SChristian König * the resv object while holding the lru_lock. 204519c2de0SChristian König */ 205a1f091f8SChristian König spin_lock(&bo->bdev->lru_lock); 206519c2de0SChristian König bo->base.resv = &bo->base._resv; 207a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 208519c2de0SChristian König } 209841e763bSChristian König 210841e763bSChristian König return r; 211841e763bSChristian König } 212841e763bSChristian König 213f2c24b83SMaarten Lankhorst static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 214f2c24b83SMaarten Lankhorst { 215e81a2557SChristian König struct dma_resv *resv = &bo->base._resv; 216dbcae3bfSChristian König struct dma_resv_iter cursor; 217f54d1867SChris Wilson struct dma_fence *fence; 218f2c24b83SMaarten Lankhorst 2190cc848a7SChristian König dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); 220dbcae3bfSChristian König dma_resv_for_each_fence_unlocked(&cursor, fence) { 221f2c24b83SMaarten Lankhorst if (!fence->ops->signaled) 222f54d1867SChris Wilson dma_fence_enable_sw_signaling(fence); 223f2c24b83SMaarten Lankhorst } 224dbcae3bfSChristian König dma_resv_iter_end(&cursor); 225f2c24b83SMaarten Lankhorst } 226f2c24b83SMaarten Lankhorst 227e1efc9b6SThomas Hellstrom /** 228089fae1eSLee Jones * ttm_bo_cleanup_refs 2291ec39923SChristian König * If bo idle, remove from lru lists, and unref. 2301ec39923SChristian König * If not idle, block if possible. 231e1efc9b6SThomas Hellstrom * 23285b144f8SMaarten Lankhorst * Must be called with lru_lock and reservation held, this function 2336c41f302SChristian König * will drop the lru lock and optionally the reservation lock before returning. 23485b144f8SMaarten Lankhorst * 235108cfddfSLee Jones * @bo: The buffer object to clean-up 236108cfddfSLee Jones * @interruptible: Any sleeps should occur interruptibly. 237108cfddfSLee Jones * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. 238108cfddfSLee Jones * @unlock_resv: Unlock the reservation lock as well. 239e1efc9b6SThomas Hellstrom */ 240e1efc9b6SThomas Hellstrom 2416c41f302SChristian König static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 2426c41f302SChristian König bool interruptible, bool no_wait_gpu, 2436c41f302SChristian König bool unlock_resv) 244e1efc9b6SThomas Hellstrom { 245832c90dfSChristian König struct dma_resv *resv = &bo->base._resv; 24685b144f8SMaarten Lankhorst int ret; 247e1efc9b6SThomas Hellstrom 2480cc848a7SChristian König if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP)) 249841e763bSChristian König ret = 0; 250841e763bSChristian König else 251841e763bSChristian König ret = -EBUSY; 25285b144f8SMaarten Lankhorst 25385b144f8SMaarten Lankhorst if (ret && !no_wait_gpu) { 254472db7abSMaarten Lankhorst long lret; 25536a0680aSChristian König 2566c41f302SChristian König if (unlock_resv) 25752791eeeSChristian König dma_resv_unlock(bo->base.resv); 258a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 259472db7abSMaarten Lankhorst 2600cc848a7SChristian König lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, 2617bc80a54SChristian König interruptible, 262472db7abSMaarten Lankhorst 30 * HZ); 263472db7abSMaarten Lankhorst 264472db7abSMaarten Lankhorst if (lret < 0) 265472db7abSMaarten Lankhorst return lret; 266472db7abSMaarten Lankhorst else if (lret == 0) 267472db7abSMaarten Lankhorst return -EBUSY; 26885b144f8SMaarten Lankhorst 269a1f091f8SChristian König spin_lock(&bo->bdev->lru_lock); 27052791eeeSChristian König if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { 27185b144f8SMaarten Lankhorst /* 27285b144f8SMaarten Lankhorst * We raced, and lost, someone else holds the reservation now, 27385b144f8SMaarten Lankhorst * and is probably busy in ttm_bo_cleanup_memtype_use. 27485b144f8SMaarten Lankhorst * 27585b144f8SMaarten Lankhorst * Even if it's not the case, because we finished waiting any 27685b144f8SMaarten Lankhorst * delayed destruction would succeed, so just return success 27785b144f8SMaarten Lankhorst * here. 27885b144f8SMaarten Lankhorst */ 279a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 28085b144f8SMaarten Lankhorst return 0; 28185b144f8SMaarten Lankhorst } 2826c41f302SChristian König ret = 0; 2837040138fSMaarten Lankhorst } 28485b144f8SMaarten Lankhorst 2859bff18d1SChristian König if (ret) { 2866c41f302SChristian König if (unlock_resv) 28752791eeeSChristian König dma_resv_unlock(bo->base.resv); 288a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 28985b144f8SMaarten Lankhorst return ret; 290e1efc9b6SThomas Hellstrom } 291e1efc9b6SThomas Hellstrom 292a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 293e1efc9b6SThomas Hellstrom ttm_bo_cleanup_memtype_use(bo); 2946c41f302SChristian König 2956c41f302SChristian König if (unlock_resv) 29652791eeeSChristian König dma_resv_unlock(bo->base.resv); 297e1efc9b6SThomas Hellstrom 298e1efc9b6SThomas Hellstrom return 0; 299ba4e7d97SThomas Hellstrom } 300ba4e7d97SThomas Hellstrom 301108cfddfSLee Jones /* 3029bff18d1SChristian König * Block for the dma_resv object to become idle, lock the buffer and clean up 3039bff18d1SChristian König * the resource and tt object. 304ba4e7d97SThomas Hellstrom */ 3059bff18d1SChristian König static void ttm_bo_delayed_delete(struct work_struct *work) 306ba4e7d97SThomas Hellstrom { 307c0c2c3bfSChristian König struct ttm_buffer_object *bo; 308ba4e7d97SThomas Hellstrom 3099bff18d1SChristian König bo = container_of(work, typeof(*bo), delayed_delete); 31085b144f8SMaarten Lankhorst 3119bff18d1SChristian König dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false, 3129bff18d1SChristian König MAX_SCHEDULE_TIMEOUT); 31352791eeeSChristian König dma_resv_lock(bo->base.resv, NULL); 3149bff18d1SChristian König ttm_bo_cleanup_memtype_use(bo); 3159bff18d1SChristian König dma_resv_unlock(bo->base.resv); 3161ec39923SChristian König ttm_bo_put(bo); 317ba4e7d97SThomas Hellstrom } 318ba4e7d97SThomas Hellstrom 319ba4e7d97SThomas Hellstrom static void ttm_bo_release(struct kref *kref) 320ba4e7d97SThomas Hellstrom { 321ba4e7d97SThomas Hellstrom struct ttm_buffer_object *bo = 322ba4e7d97SThomas Hellstrom container_of(kref, struct ttm_buffer_object, kref); 3238af8a109SChristian König struct ttm_device *bdev = bo->bdev; 3241ec39923SChristian König int ret; 325ba4e7d97SThomas Hellstrom 32644292a0fSChristian König WARN_ON_ONCE(bo->pin_count); 327fee2ede1SChristian König WARN_ON_ONCE(bo->bulk_move); 32844292a0fSChristian König 3291ec39923SChristian König if (!bo->deleted) { 3301ec39923SChristian König ret = ttm_bo_individualize_resv(bo); 3311ec39923SChristian König if (ret) { 3321ec39923SChristian König /* Last resort, if we fail to allocate memory for the 3331ec39923SChristian König * fences block for the BO to become idle 3341ec39923SChristian König */ 3357bc80a54SChristian König dma_resv_wait_timeout(bo->base.resv, 3360cc848a7SChristian König DMA_RESV_USAGE_BOOKKEEP, false, 3371ec39923SChristian König 30 * HZ); 3381ec39923SChristian König } 339f704ff7cSChristian König 3408af8a109SChristian König if (bo->bdev->funcs->release_notify) 3418af8a109SChristian König bo->bdev->funcs->release_notify(bo); 342f704ff7cSChristian König 343f704ff7cSChristian König drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); 344d3116756SChristian König ttm_mem_io_free(bdev, bo->resource); 3451ec39923SChristian König 3469bff18d1SChristian König if (!dma_resv_test_signaled(bo->base.resv, 3479bff18d1SChristian König DMA_RESV_USAGE_BOOKKEEP) || 34835d67ee3SRajneesh Bhardwaj (want_init_on_free() && (bo->ttm != NULL)) || 349*d99fbd9aSThomas Hellström bo->type == ttm_bo_type_sg || 350efa557bcSChristian König !dma_resv_trylock(bo->base.resv)) { 3511ec39923SChristian König /* The BO is not idle, resurrect it for delayed destroy */ 3521ec39923SChristian König ttm_bo_flush_all_fences(bo); 3531ec39923SChristian König bo->deleted = true; 3541ec39923SChristian König 355a1f091f8SChristian König spin_lock(&bo->bdev->lru_lock); 3561ec39923SChristian König 3571ec39923SChristian König /* 358e8c93e0cSChristian König * Make pinned bos immediately available to 3591ec39923SChristian König * shrinkers, now that they are queued for 3601ec39923SChristian König * destruction. 361d228f8d8SChristian König * 362d228f8d8SChristian König * FIXME: QXL is triggering this. Can be removed when the 363d228f8d8SChristian König * driver is fixed. 3641ec39923SChristian König */ 36544292a0fSChristian König if (bo->pin_count) { 366deb0814bSChristian König bo->pin_count = 0; 367fee2ede1SChristian König ttm_resource_move_to_lru_tail(bo->resource); 3681ec39923SChristian König } 3691ec39923SChristian König 3701ec39923SChristian König kref_init(&bo->kref); 371a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 3721ec39923SChristian König 3739bff18d1SChristian König INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); 374b0a7ce53SRajneesh Bhardwaj 375b0a7ce53SRajneesh Bhardwaj /* Schedule the worker on the closest NUMA node. This 376b0a7ce53SRajneesh Bhardwaj * improves performance since system memory might be 377b0a7ce53SRajneesh Bhardwaj * cleared on free and that is best done on a CPU core 378b0a7ce53SRajneesh Bhardwaj * close to it. 379b0a7ce53SRajneesh Bhardwaj */ 380b0a7ce53SRajneesh Bhardwaj queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); 3811ec39923SChristian König return; 3821ec39923SChristian König } 3831ec39923SChristian König 3841ec39923SChristian König ttm_bo_cleanup_memtype_use(bo); 385efa557bcSChristian König dma_resv_unlock(bo->base.resv); 3869bff18d1SChristian König } 3871ec39923SChristian König 3888af8a109SChristian König atomic_dec(&ttm_glob.bo_count); 3891ec39923SChristian König bo->destroy(bo); 390ba4e7d97SThomas Hellstrom } 391ba4e7d97SThomas Hellstrom 392a3185f91SChristian König /** 393a3185f91SChristian König * ttm_bo_put 394a3185f91SChristian König * 395a3185f91SChristian König * @bo: The buffer object. 396a3185f91SChristian König * 397a3185f91SChristian König * Unreference a buffer object. 398a3185f91SChristian König */ 39989c815efSThomas Zimmermann void ttm_bo_put(struct ttm_buffer_object *bo) 40089c815efSThomas Zimmermann { 40189c815efSThomas Zimmermann kref_put(&bo->kref, ttm_bo_release); 40289c815efSThomas Zimmermann } 40389c815efSThomas Zimmermann EXPORT_SYMBOL(ttm_bo_put); 40489c815efSThomas Zimmermann 40540379792SAndrey Grodzovsky static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, 40640379792SAndrey Grodzovsky struct ttm_operation_ctx *ctx, 40740379792SAndrey Grodzovsky struct ttm_place *hop) 40840379792SAndrey Grodzovsky { 40940379792SAndrey Grodzovsky struct ttm_placement hop_placement; 41040379792SAndrey Grodzovsky struct ttm_resource *hop_mem; 41140379792SAndrey Grodzovsky int ret; 41240379792SAndrey Grodzovsky 413a78a8da5SSomalapuram Amaranath hop_placement.num_placement = 1; 414a78a8da5SSomalapuram Amaranath hop_placement.placement = hop; 41540379792SAndrey Grodzovsky 41640379792SAndrey Grodzovsky /* find space in the bounce domain */ 41740379792SAndrey Grodzovsky ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); 41840379792SAndrey Grodzovsky if (ret) 41940379792SAndrey Grodzovsky return ret; 42040379792SAndrey Grodzovsky /* move to the bounce domain */ 42140379792SAndrey Grodzovsky ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); 42240379792SAndrey Grodzovsky if (ret) { 42340379792SAndrey Grodzovsky ttm_resource_free(bo, &hop_mem); 42440379792SAndrey Grodzovsky return ret; 42540379792SAndrey Grodzovsky } 42640379792SAndrey Grodzovsky return 0; 42740379792SAndrey Grodzovsky } 42840379792SAndrey Grodzovsky 42983876c1bSChristian König static int ttm_bo_evict(struct ttm_buffer_object *bo, 43083876c1bSChristian König struct ttm_operation_ctx *ctx) 431ba4e7d97SThomas Hellstrom { 4328af8a109SChristian König struct ttm_device *bdev = bo->bdev; 433bfa3357eSChristian König struct ttm_resource *evict_mem; 434ca262a99SJerome Glisse struct ttm_placement placement; 435ebdf5651SDave Airlie struct ttm_place hop; 436ca262a99SJerome Glisse int ret = 0; 437ba4e7d97SThomas Hellstrom 438ebdf5651SDave Airlie memset(&hop, 0, sizeof(hop)); 439ebdf5651SDave Airlie 44052791eeeSChristian König dma_resv_assert_held(bo->base.resv); 441ba4e7d97SThomas Hellstrom 4425d951098SChristian König placement.num_placement = 0; 4438af8a109SChristian König bdev->funcs->evict_flags(bo, &placement); 4445d951098SChristian König 445a78a8da5SSomalapuram Amaranath if (!placement.num_placement) { 44642523924SChristian König ret = ttm_bo_wait_ctx(bo, ctx); 447a3be8cd7SThomas Hellström if (ret) 448a3be8cd7SThomas Hellström return ret; 449d59bc632SChristian König 450a3be8cd7SThomas Hellström /* 451a3be8cd7SThomas Hellström * Since we've already synced, this frees backing store 452a3be8cd7SThomas Hellström * immediately. 453a3be8cd7SThomas Hellström */ 454a3be8cd7SThomas Hellström return ttm_bo_pipeline_gutting(bo); 455d59bc632SChristian König } 4565d951098SChristian König 45783876c1bSChristian König ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 458ba4e7d97SThomas Hellstrom if (ret) { 459fb53f862SJerome Glisse if (ret != -ERESTARTSYS) { 46025d0479aSJoe Perches pr_err("Failed to find memory space for buffer 0x%p eviction\n", 46125d0479aSJoe Perches bo); 462fb53f862SJerome Glisse ttm_bo_mem_space_debug(bo, &placement); 463fb53f862SJerome Glisse } 464ba4e7d97SThomas Hellstrom goto out; 465ba4e7d97SThomas Hellstrom } 466ba4e7d97SThomas Hellstrom 467e8188c46SThomas Hellström do { 468bfa3357eSChristian König ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); 469e8188c46SThomas Hellström if (ret != -EMULTIHOP) 470e8188c46SThomas Hellström break; 471e8188c46SThomas Hellström 472ba42ecb5SJesse Zhang ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); 473e8188c46SThomas Hellström } while (!ret); 474e8188c46SThomas Hellström 47540379792SAndrey Grodzovsky if (ret) { 476e8188c46SThomas Hellström ttm_resource_free(bo, &evict_mem); 4778ab3b066SThomas Hellström if (ret != -ERESTARTSYS && ret != -EINTR) 47825d0479aSJoe Perches pr_err("Buffer eviction failed\n"); 479ca262a99SJerome Glisse } 480ba4e7d97SThomas Hellstrom out: 481ba4e7d97SThomas Hellstrom return ret; 482ba4e7d97SThomas Hellstrom } 483ba4e7d97SThomas Hellstrom 484a3185f91SChristian König /** 485a3185f91SChristian König * ttm_bo_eviction_valuable 486a3185f91SChristian König * 487a3185f91SChristian König * @bo: The buffer object to evict 488a3185f91SChristian König * @place: the placement we need to make room for 489a3185f91SChristian König * 490a3185f91SChristian König * Check if it is valuable to evict the BO to make room for the given placement. 491a3185f91SChristian König */ 492a2ab19feSChristian König bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 493a2ab19feSChristian König const struct ttm_place *place) 494a2ab19feSChristian König { 49554443270SArunpravin Paneer Selvam struct ttm_resource *res = bo->resource; 49654443270SArunpravin Paneer Selvam struct ttm_device *bdev = bo->bdev; 49754443270SArunpravin Paneer Selvam 498abb50d67SThomas Hellström dma_resv_assert_held(bo->base.resv); 499abb50d67SThomas Hellström if (bo->resource->mem_type == TTM_PL_SYSTEM) 500abb50d67SThomas Hellström return true; 501abb50d67SThomas Hellström 502a2ab19feSChristian König /* Don't evict this BO if it's outside of the 503a2ab19feSChristian König * requested placement range 504a2ab19feSChristian König */ 50554443270SArunpravin Paneer Selvam return ttm_resource_intersects(bdev, res, place, bo->base.size); 506a2ab19feSChristian König } 507a2ab19feSChristian König EXPORT_SYMBOL(ttm_bo_eviction_valuable); 508a2ab19feSChristian König 509108cfddfSLee Jones /* 510d5769ba3SRoger He * Check the target bo is allowable to be evicted or swapout, including cases: 511d5769ba3SRoger He * 512d5769ba3SRoger He * a. if share same reservation object with ctx->resv, have assumption 513d5769ba3SRoger He * reservation objects should already be locked, so not lock again and 514d5769ba3SRoger He * return true directly when either the opreation allow_reserved_eviction 515d5769ba3SRoger He * or the target bo already is in delayed free list; 516d5769ba3SRoger He * 517d5769ba3SRoger He * b. Otherwise, trylock it. 518d5769ba3SRoger He */ 519d5769ba3SRoger He static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 520abb50d67SThomas Hellström struct ttm_operation_ctx *ctx, 521abb50d67SThomas Hellström const struct ttm_place *place, 522abb50d67SThomas Hellström bool *locked, bool *busy) 523d5769ba3SRoger He { 524d5769ba3SRoger He bool ret = false; 525d5769ba3SRoger He 526a2848d08SChristian König if (bo->pin_count) { 527a2848d08SChristian König *locked = false; 5282dedcf41SGuchun Chen if (busy) 529a2848d08SChristian König *busy = false; 530a2848d08SChristian König return false; 531a2848d08SChristian König } 532a2848d08SChristian König 533e532a135SGerd Hoffmann if (bo->base.resv == ctx->resv) { 53452791eeeSChristian König dma_resv_assert_held(bo->base.resv); 535c44dfe4dSChristian König if (ctx->allow_res_evict) 536d5769ba3SRoger He ret = true; 537d367bd2aSChristian König *locked = false; 538d367bd2aSChristian König if (busy) 539d367bd2aSChristian König *busy = false; 540d5769ba3SRoger He } else { 54152791eeeSChristian König ret = dma_resv_trylock(bo->base.resv); 542d367bd2aSChristian König *locked = ret; 543d367bd2aSChristian König if (busy) 544d367bd2aSChristian König *busy = !ret; 545d5769ba3SRoger He } 546d5769ba3SRoger He 5477120a447Sxinhui pan if (ret && place && (bo->resource->mem_type != place->mem_type || 5487120a447Sxinhui pan !bo->bdev->funcs->eviction_valuable(bo, place))) { 549abb50d67SThomas Hellström ret = false; 550abb50d67SThomas Hellström if (*locked) { 551abb50d67SThomas Hellström dma_resv_unlock(bo->base.resv); 552abb50d67SThomas Hellström *locked = false; 553abb50d67SThomas Hellström } 554abb50d67SThomas Hellström } 555abb50d67SThomas Hellström 556d5769ba3SRoger He return ret; 557d5769ba3SRoger He } 558d5769ba3SRoger He 559d367bd2aSChristian König /** 560d367bd2aSChristian König * ttm_mem_evict_wait_busy - wait for a busy BO to become available 561d367bd2aSChristian König * 562d367bd2aSChristian König * @busy_bo: BO which couldn't be locked with trylock 563d367bd2aSChristian König * @ctx: operation context 564d367bd2aSChristian König * @ticket: acquire ticket 565d367bd2aSChristian König * 566d367bd2aSChristian König * Try to lock a busy buffer object to avoid failing eviction. 567d367bd2aSChristian König */ 568d367bd2aSChristian König static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, 569d367bd2aSChristian König struct ttm_operation_ctx *ctx, 570d367bd2aSChristian König struct ww_acquire_ctx *ticket) 571d367bd2aSChristian König { 572d367bd2aSChristian König int r; 573d367bd2aSChristian König 574d367bd2aSChristian König if (!busy_bo || !ticket) 575d367bd2aSChristian König return -EBUSY; 576d367bd2aSChristian König 577d367bd2aSChristian König if (ctx->interruptible) 57852791eeeSChristian König r = dma_resv_lock_interruptible(busy_bo->base.resv, 579d367bd2aSChristian König ticket); 580d367bd2aSChristian König else 58152791eeeSChristian König r = dma_resv_lock(busy_bo->base.resv, ticket); 582d367bd2aSChristian König 583d367bd2aSChristian König /* 584d367bd2aSChristian König * TODO: It would be better to keep the BO locked until allocation is at 585d367bd2aSChristian König * least tried one more time, but that would mean a much larger rework 586d367bd2aSChristian König * of TTM. 587d367bd2aSChristian König */ 588d367bd2aSChristian König if (!r) 58952791eeeSChristian König dma_resv_unlock(busy_bo->base.resv); 590d367bd2aSChristian König 591f6319596SFelix Kuehling return r == -EDEADLK ? -EBUSY : r; 592d367bd2aSChristian König } 593d367bd2aSChristian König 5948af8a109SChristian König int ttm_mem_evict_first(struct ttm_device *bdev, 5959de59bc2SDave Airlie struct ttm_resource_manager *man, 596e300180fSMichel Dänzer const struct ttm_place *place, 597d367bd2aSChristian König struct ttm_operation_ctx *ctx, 598d367bd2aSChristian König struct ww_acquire_ctx *ticket) 599ca262a99SJerome Glisse { 600d367bd2aSChristian König struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; 6015d05b988SChristian König struct ttm_resource_cursor cursor; 6026a9b0289SChristian König struct ttm_resource *res; 6036ba43581SChristian König bool locked = false; 6046ba43581SChristian König int ret; 605ca262a99SJerome Glisse 60658442f0dSChristian König spin_lock(&bdev->lru_lock); 6075d05b988SChristian König ttm_resource_manager_for_each_res(man, &cursor, res) { 608d367bd2aSChristian König bool busy; 609d367bd2aSChristian König 6105d05b988SChristian König if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place, 611abb50d67SThomas Hellström &locked, &busy)) { 6120dbd555aSChristian König if (busy && !busy_bo && ticket != 6135d05b988SChristian König dma_resv_locking_ctx(res->bo->base.resv)) 6145d05b988SChristian König busy_bo = res->bo; 615a2ab19feSChristian König continue; 616d367bd2aSChristian König } 617a2ab19feSChristian König 6185d05b988SChristian König if (ttm_bo_get_unless_zero(res->bo)) { 6195d05b988SChristian König bo = res->bo; 6205d05b988SChristian König break; 6215d05b988SChristian König } 6221ec39923SChristian König if (locked) 6235d05b988SChristian König dma_resv_unlock(res->bo->base.resv); 624cf6c467dSChristian König } 625cf6c467dSChristian König 6266ba43581SChristian König if (!bo) { 6271ec39923SChristian König if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) 6281ec39923SChristian König busy_bo = NULL; 62958442f0dSChristian König spin_unlock(&bdev->lru_lock); 630d367bd2aSChristian König ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); 631d367bd2aSChristian König if (busy_bo) 6321ec39923SChristian König ttm_bo_put(busy_bo); 633d367bd2aSChristian König return ret; 634e7ab2019SMaarten Lankhorst } 635e7ab2019SMaarten Lankhorst 6361ec39923SChristian König if (bo->deleted) { 63783876c1bSChristian König ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 63883876c1bSChristian König ctx->no_wait_gpu, locked); 6391ec39923SChristian König ttm_bo_put(bo); 640e1efc9b6SThomas Hellstrom return ret; 641e1efc9b6SThomas Hellstrom } 642e1efc9b6SThomas Hellstrom 64358442f0dSChristian König spin_unlock(&bdev->lru_lock); 6449c51ba1dSThomas Hellstrom 64583876c1bSChristian König ret = ttm_bo_evict(bo, ctx); 6469165fb87SChristian König if (locked) 647ca262a99SJerome Glisse ttm_bo_unreserve(bo); 648781050b0Sxinhui pan else 649781050b0Sxinhui pan ttm_bo_move_to_lru_tail_unlocked(bo); 6509c51ba1dSThomas Hellstrom 6511ec39923SChristian König ttm_bo_put(bo); 652ca262a99SJerome Glisse return ret; 653ca262a99SJerome Glisse } 654ca262a99SJerome Glisse 6557842cf65SChristian König /** 6567842cf65SChristian König * ttm_bo_pin - Pin the buffer object. 6577842cf65SChristian König * @bo: The buffer object to pin 6587842cf65SChristian König * 6597842cf65SChristian König * Make sure the buffer is not evicted any more during memory pressure. 6607842cf65SChristian König * @bo must be unpinned again by calling ttm_bo_unpin(). 6617842cf65SChristian König */ 6627842cf65SChristian König void ttm_bo_pin(struct ttm_buffer_object *bo) 6637842cf65SChristian König { 6647842cf65SChristian König dma_resv_assert_held(bo->base.resv); 6657842cf65SChristian König WARN_ON_ONCE(!kref_read(&bo->kref)); 6660f9cd1eaSChristian König spin_lock(&bo->bdev->lru_lock); 6670f9cd1eaSChristian König if (bo->resource) 6680f9cd1eaSChristian König ttm_resource_del_bulk_move(bo->resource, bo); 6690f9cd1eaSChristian König ++bo->pin_count; 6700f9cd1eaSChristian König spin_unlock(&bo->bdev->lru_lock); 6717842cf65SChristian König } 6727842cf65SChristian König EXPORT_SYMBOL(ttm_bo_pin); 6737842cf65SChristian König 6747842cf65SChristian König /** 6757842cf65SChristian König * ttm_bo_unpin - Unpin the buffer object. 6767842cf65SChristian König * @bo: The buffer object to unpin 6777842cf65SChristian König * 6787842cf65SChristian König * Allows the buffer object to be evicted again during memory pressure. 6797842cf65SChristian König */ 6807842cf65SChristian König void ttm_bo_unpin(struct ttm_buffer_object *bo) 6817842cf65SChristian König { 6827842cf65SChristian König dma_resv_assert_held(bo->base.resv); 6837842cf65SChristian König WARN_ON_ONCE(!kref_read(&bo->kref)); 684fee2ede1SChristian König if (WARN_ON_ONCE(!bo->pin_count)) 685fee2ede1SChristian König return; 686fee2ede1SChristian König 6870f9cd1eaSChristian König spin_lock(&bo->bdev->lru_lock); 6880f9cd1eaSChristian König --bo->pin_count; 6890f9cd1eaSChristian König if (bo->resource) 6900f9cd1eaSChristian König ttm_resource_add_bulk_move(bo->resource, bo); 6910f9cd1eaSChristian König spin_unlock(&bo->bdev->lru_lock); 6927842cf65SChristian König } 6937842cf65SChristian König EXPORT_SYMBOL(ttm_bo_unpin); 6947842cf65SChristian König 695108cfddfSLee Jones /* 6968bb31587SChristian König * Add the last move fence to the BO as kernel dependency and reserve a new 6978bb31587SChristian König * fence slot. 6983ddf4ad9SChristian König */ 6993ddf4ad9SChristian König static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 7009de59bc2SDave Airlie struct ttm_resource_manager *man, 7013084cf46SChristian König bool no_wait_gpu) 7023ddf4ad9SChristian König { 703f54d1867SChris Wilson struct dma_fence *fence; 7043ddf4ad9SChristian König int ret; 7053ddf4ad9SChristian König 7063ddf4ad9SChristian König spin_lock(&man->move_lock); 707f54d1867SChris Wilson fence = dma_fence_get(man->move); 7083ddf4ad9SChristian König spin_unlock(&man->move_lock); 7093ddf4ad9SChristian König 7103084cf46SChristian König if (!fence) 7113084cf46SChristian König return 0; 7123084cf46SChristian König 71311425c45SXiyu Yang if (no_wait_gpu) { 714e92b0ff6SFelix Kuehling ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; 71511425c45SXiyu Yang dma_fence_put(fence); 716e92b0ff6SFelix Kuehling return ret; 71711425c45SXiyu Yang } 7183084cf46SChristian König 719b29895e1SChristian König dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); 7203ddf4ad9SChristian König 721c8d4c18bSChristian König ret = dma_resv_reserve_fences(bo->base.resv, 1); 722543c364dSLin Yi dma_fence_put(fence); 7233ddf4ad9SChristian König return ret; 724543c364dSLin Yi } 7253ddf4ad9SChristian König 726a3185f91SChristian König /** 727cc941c70SChristian König * ttm_bo_alloc_resource - Allocate backing store for a BO 728ba4e7d97SThomas Hellstrom * 729cc941c70SChristian König * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for 730cc941c70SChristian König * @placement: Proposed new placement for the buffer object 731a3185f91SChristian König * @ctx: if and how to sleep, lock buffers and alloc memory 732cc941c70SChristian König * @force_space: If we should evict buffers to force space 733cc941c70SChristian König * @res: The resulting struct ttm_resource. 734a3185f91SChristian König * 735cc941c70SChristian König * Allocates a resource for the buffer object pointed to by @bo, using the 736cc941c70SChristian König * placement flags in @placement, potentially evicting other buffer objects when 737cc941c70SChristian König * @force_space is true. 738cc941c70SChristian König * This function may sleep while waiting for resources to become available. 739a3185f91SChristian König * Returns: 740cc941c70SChristian König * -EBUSY: No space available (only if no_wait == true). 74128e51267SChristian König * -ENOSPC: Could not allocate space for the buffer object, either due to 742a3185f91SChristian König * fragmentation or concurrent allocators. 743a3185f91SChristian König * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. 744ba4e7d97SThomas Hellstrom */ 745cc941c70SChristian König static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, 746ca262a99SJerome Glisse struct ttm_placement *placement, 747cc941c70SChristian König struct ttm_operation_ctx *ctx, 748cc941c70SChristian König bool force_space, 749cc941c70SChristian König struct ttm_resource **res) 750ba4e7d97SThomas Hellstrom { 7518af8a109SChristian König struct ttm_device *bdev = bo->bdev; 752cc941c70SChristian König struct ww_acquire_ctx *ticket; 753ca262a99SJerome Glisse int i, ret; 754ba4e7d97SThomas Hellstrom 755cc941c70SChristian König ticket = dma_resv_locking_ctx(bo->base.resv); 756c8d4c18bSChristian König ret = dma_resv_reserve_fences(bo->base.resv, 1); 7573ddf4ad9SChristian König if (unlikely(ret)) 7583ddf4ad9SChristian König return ret; 7593ddf4ad9SChristian König 760b6637526SDave Airlie for (i = 0; i < placement->num_placement; ++i) { 761f1217ed0SChristian König const struct ttm_place *place = &placement->placement[i]; 7629de59bc2SDave Airlie struct ttm_resource_manager *man; 763f1217ed0SChristian König 764bfa3357eSChristian König man = ttm_manager_type(bdev, place->mem_type); 765bfa3357eSChristian König if (!man || !ttm_resource_manager_used(man)) 76648e07c23SChristian König continue; 767ba4e7d97SThomas Hellstrom 768cc941c70SChristian König if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : 769cc941c70SChristian König TTM_PL_FLAG_FALLBACK)) 77058e4d686SChristian König continue; 771e30f3963SThomas Hellstrom 772cc941c70SChristian König do { 773cc941c70SChristian König ret = ttm_resource_alloc(bo, place, res); 774cc941c70SChristian König if (unlikely(ret && ret != -ENOSPC)) 775cc941c70SChristian König return ret; 776cc941c70SChristian König if (likely(!ret) || !force_space) 777cc941c70SChristian König break; 778cc941c70SChristian König 779cc941c70SChristian König ret = ttm_mem_evict_first(bdev, man, place, ctx, 780cc941c70SChristian König ticket); 781cc941c70SChristian König if (unlikely(ret == -EBUSY)) 782cc941c70SChristian König break; 783cc941c70SChristian König if (unlikely(ret)) 784cc941c70SChristian König return ret; 785cc941c70SChristian König } while (1); 786cc941c70SChristian König if (ret) 787cc941c70SChristian König continue; 788cc941c70SChristian König 789ba42ecb5SJesse Zhang ret = ttm_bo_add_move_fence(bo, man, ctx->no_wait_gpu); 7903ddf4ad9SChristian König if (unlikely(ret)) { 791cc941c70SChristian König ttm_resource_free(bo, res); 7923084cf46SChristian König if (ret == -EBUSY) 7933084cf46SChristian König continue; 7943084cf46SChristian König 795cc941c70SChristian König return ret; 7963ddf4ad9SChristian König } 797ba4e7d97SThomas Hellstrom return 0; 798ba4e7d97SThomas Hellstrom } 799ba4e7d97SThomas Hellstrom 800cc941c70SChristian König return -ENOSPC; 801ba4e7d97SThomas Hellstrom } 802e30f3963SThomas Hellstrom 803cc941c70SChristian König /* 804cc941c70SChristian König * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource 805cc941c70SChristian König * 806cc941c70SChristian König * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for 807cc941c70SChristian König * @placement: Proposed new placement for the buffer object 808cc941c70SChristian König * @res: The resulting struct ttm_resource. 809cc941c70SChristian König * @ctx: if and how to sleep, lock buffers and alloc memory 810cc941c70SChristian König * 811cc941c70SChristian König * Tries both idle allocation and forcefully eviction of buffers. See 812cc941c70SChristian König * ttm_bo_alloc_resource for details. 813cc941c70SChristian König */ 814cc941c70SChristian König int ttm_bo_mem_space(struct ttm_buffer_object *bo, 815cc941c70SChristian König struct ttm_placement *placement, 816cc941c70SChristian König struct ttm_resource **res, 817cc941c70SChristian König struct ttm_operation_ctx *ctx) 818cc941c70SChristian König { 819cc941c70SChristian König bool force_space = false; 820cc941c70SChristian König int ret; 821e30f3963SThomas Hellstrom 822cc941c70SChristian König do { 823cc941c70SChristian König ret = ttm_bo_alloc_resource(bo, placement, ctx, 824cc941c70SChristian König force_space, res); 825cc941c70SChristian König force_space = !force_space; 826cc941c70SChristian König } while (ret == -ENOSPC && force_space); 827cc941c70SChristian König 828224ee02aSChristian König return ret; 829ba4e7d97SThomas Hellstrom } 830ba4e7d97SThomas Hellstrom EXPORT_SYMBOL(ttm_bo_mem_space); 831ba4e7d97SThomas Hellstrom 832a3185f91SChristian König /** 833a3185f91SChristian König * ttm_bo_validate 834a3185f91SChristian König * 835a3185f91SChristian König * @bo: The buffer object. 836a3185f91SChristian König * @placement: Proposed placement for the buffer object. 837a3185f91SChristian König * @ctx: validation parameters. 838a3185f91SChristian König * 839a3185f91SChristian König * Changes placement and caching policy of the buffer object 840a3185f91SChristian König * according proposed placement. 841a3185f91SChristian König * Returns 842a3185f91SChristian König * -EINVAL on invalid proposed placement. 843a3185f91SChristian König * -ENOMEM on out-of-memory condition. 844a3185f91SChristian König * -EBUSY if no_wait is true and buffer busy. 845a3185f91SChristian König * -ERESTARTSYS if interrupted by a signal. 846a3185f91SChristian König */ 84709855acbSJerome Glisse int ttm_bo_validate(struct ttm_buffer_object *bo, 848ca262a99SJerome Glisse struct ttm_placement *placement, 84919be5570SChristian König struct ttm_operation_ctx *ctx) 850ba4e7d97SThomas Hellstrom { 851cc941c70SChristian König struct ttm_resource *res; 852cc941c70SChristian König struct ttm_place hop; 853cc941c70SChristian König bool force_space; 854ba4e7d97SThomas Hellstrom int ret; 855ba4e7d97SThomas Hellstrom 85652791eeeSChristian König dma_resv_assert_held(bo->base.resv); 857d2588d2dSChristian König 858d2588d2dSChristian König /* 859d2588d2dSChristian König * Remove the backing store if no placement is given. 860d2588d2dSChristian König */ 861a78a8da5SSomalapuram Amaranath if (!placement->num_placement) 862a3be8cd7SThomas Hellström return ttm_bo_pipeline_gutting(bo); 863d2588d2dSChristian König 864cc941c70SChristian König force_space = false; 865cc941c70SChristian König do { 866f87c1f0bSChristian König /* Check whether we need to move buffer. */ 867cc941c70SChristian König if (bo->resource && 868cc941c70SChristian König ttm_resource_compatible(bo->resource, placement, 869cc941c70SChristian König force_space)) 870f87c1f0bSChristian König return 0; 871f87c1f0bSChristian König 872f87c1f0bSChristian König /* Moving of pinned BOs is forbidden */ 873f87c1f0bSChristian König if (bo->pin_count) 874f87c1f0bSChristian König return -EINVAL; 875f87c1f0bSChristian König 876cc941c70SChristian König /* 877cc941c70SChristian König * Determine where to move the buffer. 878cc941c70SChristian König * 879cc941c70SChristian König * If driver determines move is going to need 880cc941c70SChristian König * an extra step then it will return -EMULTIHOP 881cc941c70SChristian König * and the buffer will be moved to the temporary 882cc941c70SChristian König * stop and the driver will be called to make 883cc941c70SChristian König * the second hop. 884cc941c70SChristian König */ 885cc941c70SChristian König ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, 886cc941c70SChristian König &res); 887cc941c70SChristian König force_space = !force_space; 888cc941c70SChristian König if (ret == -ENOSPC) 889cc941c70SChristian König continue; 890cc941c70SChristian König if (ret) 891cc941c70SChristian König return ret; 892cc941c70SChristian König 893cc941c70SChristian König bounce: 894cc941c70SChristian König ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); 895cc941c70SChristian König if (ret == -EMULTIHOP) { 896ba42ecb5SJesse Zhang ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); 897cc941c70SChristian König /* try and move to final place now. */ 898cc941c70SChristian König if (!ret) 899cc941c70SChristian König goto bounce; 900cc941c70SChristian König } 901cc941c70SChristian König if (ret) { 902cc941c70SChristian König ttm_resource_free(bo, &res); 903cc941c70SChristian König return ret; 904cc941c70SChristian König } 905cc941c70SChristian König 906cc941c70SChristian König } while (ret && force_space); 907cc941c70SChristian König 90828e51267SChristian König /* For backward compatibility with userspace */ 90928e51267SChristian König if (ret == -ENOSPC) 91028e51267SChristian König return -ENOMEM; 911f87c1f0bSChristian König 91262975d27SChristian König /* 91362975d27SChristian König * We might need to add a TTM. 91462975d27SChristian König */ 915cf4b7387SArunpravin Paneer Selvam if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { 91662975d27SChristian König ret = ttm_tt_create(bo, true); 91762975d27SChristian König if (ret) 91862975d27SChristian König return ret; 91962975d27SChristian König } 920ba4e7d97SThomas Hellstrom return 0; 921ba4e7d97SThomas Hellstrom } 92209855acbSJerome Glisse EXPORT_SYMBOL(ttm_bo_validate); 923ba4e7d97SThomas Hellstrom 924347987a2SChristian König /** 925347987a2SChristian König * ttm_bo_init_reserved 926347987a2SChristian König * 927347987a2SChristian König * @bdev: Pointer to a ttm_device struct. 928347987a2SChristian König * @bo: Pointer to a ttm_buffer_object to be initialized. 929347987a2SChristian König * @type: Requested type of buffer object. 930347987a2SChristian König * @placement: Initial placement for buffer object. 931347987a2SChristian König * @alignment: Data alignment in pages. 932347987a2SChristian König * @ctx: TTM operation context for memory allocation. 933347987a2SChristian König * @sg: Scatter-gather table. 934347987a2SChristian König * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. 935347987a2SChristian König * @destroy: Destroy function. Use NULL for kfree(). 936347987a2SChristian König * 937347987a2SChristian König * This function initializes a pre-allocated struct ttm_buffer_object. 938347987a2SChristian König * As this object may be part of a larger structure, this function, 939347987a2SChristian König * together with the @destroy function, enables driver-specific objects 940347987a2SChristian König * derived from a ttm_buffer_object. 941347987a2SChristian König * 942347987a2SChristian König * On successful return, the caller owns an object kref to @bo. The kref and 943347987a2SChristian König * list_kref are usually set to 1, but note that in some situations, other 944347987a2SChristian König * tasks may already be holding references to @bo as well. 945347987a2SChristian König * Furthermore, if resv == NULL, the buffer's reservation lock will be held, 946347987a2SChristian König * and it is the caller's responsibility to call ttm_bo_unreserve. 947347987a2SChristian König * 948347987a2SChristian König * If a failure occurs, the function will call the @destroy function. Thus, 949347987a2SChristian König * after a failure, dereferencing @bo is illegal and will likely cause memory 950347987a2SChristian König * corruption. 951347987a2SChristian König * 952347987a2SChristian König * Returns 953347987a2SChristian König * -ENOMEM: Out of memory. 954347987a2SChristian König * -EINVAL: Invalid placement flags. 955347987a2SChristian König * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. 956347987a2SChristian König */ 957347987a2SChristian König int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, 958347987a2SChristian König enum ttm_bo_type type, struct ttm_placement *placement, 959347987a2SChristian König uint32_t alignment, struct ttm_operation_ctx *ctx, 960347987a2SChristian König struct sg_table *sg, struct dma_resv *resv, 961ba4e7d97SThomas Hellstrom void (*destroy) (struct ttm_buffer_object *)) 962ba4e7d97SThomas Hellstrom { 963bfa3357eSChristian König int ret; 96457de4ba9SJerome Glisse 965ba4e7d97SThomas Hellstrom kref_init(&bo->kref); 966ba4e7d97SThomas Hellstrom bo->bdev = bdev; 967ba4e7d97SThomas Hellstrom bo->type = type; 968347987a2SChristian König bo->page_alignment = alignment; 969347987a2SChristian König bo->destroy = destroy; 970deb0814bSChristian König bo->pin_count = 0; 971129b78bfSDave Airlie bo->sg = sg; 972fee2ede1SChristian König bo->bulk_move = NULL; 973347987a2SChristian König if (resv) 9742e3c9ec4SGerd Hoffmann bo->base.resv = resv; 975347987a2SChristian König else 9762e3c9ec4SGerd Hoffmann bo->base.resv = &bo->base._resv; 9778af8a109SChristian König atomic_inc(&ttm_glob.bo_count); 978ba4e7d97SThomas Hellstrom 979ba4e7d97SThomas Hellstrom /* 980ba4e7d97SThomas Hellstrom * For ttm_bo_type_device buffers, allocate 981ba4e7d97SThomas Hellstrom * address space from the device. 982ba4e7d97SThomas Hellstrom */ 983347987a2SChristian König if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { 9849d6f4484SGerd Hoffmann ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, 985347987a2SChristian König PFN_UP(bo->base.size)); 986347987a2SChristian König if (ret) 987347987a2SChristian König goto err_put; 988347987a2SChristian König } 989ba4e7d97SThomas Hellstrom 990f4f4e3e3SMaarten Lankhorst /* passed reservation objects should already be locked, 991f4f4e3e3SMaarten Lankhorst * since otherwise lockdep will be angered in radeon. 992f4f4e3e3SMaarten Lankhorst */ 993c2c139cfSNicolai Hähnle if (!resv) 994347987a2SChristian König WARN_ON(!dma_resv_trylock(bo->base.resv)); 995347987a2SChristian König else 996347987a2SChristian König dma_resv_assert_held(resv); 997ba4e7d97SThomas Hellstrom 998347987a2SChristian König ret = ttm_bo_validate(bo, placement, ctx); 999347987a2SChristian König if (unlikely(ret)) 1000347987a2SChristian König goto err_unlock; 1001347987a2SChristian König 1002347987a2SChristian König return 0; 1003347987a2SChristian König 1004347987a2SChristian König err_unlock: 1005347987a2SChristian König if (!resv) 1006347987a2SChristian König dma_resv_unlock(bo->base.resv); 1007347987a2SChristian König 1008347987a2SChristian König err_put: 1009f4490759SThomas Zimmermann ttm_bo_put(bo); 1010c2c139cfSNicolai Hähnle return ret; 1011c2c139cfSNicolai Hähnle } 1012ca9cf68dSNicolai Hähnle EXPORT_SYMBOL(ttm_bo_init_reserved); 1013ca9cf68dSNicolai Hähnle 1014347987a2SChristian König /** 1015347987a2SChristian König * ttm_bo_init_validate 1016347987a2SChristian König * 1017347987a2SChristian König * @bdev: Pointer to a ttm_device struct. 1018347987a2SChristian König * @bo: Pointer to a ttm_buffer_object to be initialized. 1019347987a2SChristian König * @type: Requested type of buffer object. 1020347987a2SChristian König * @placement: Initial placement for buffer object. 1021347987a2SChristian König * @alignment: Data alignment in pages. 1022347987a2SChristian König * @interruptible: If needing to sleep to wait for GPU resources, 1023347987a2SChristian König * sleep interruptible. 1024347987a2SChristian König * pinned in physical memory. If this behaviour is not desired, this member 1025347987a2SChristian König * holds a pointer to a persistent shmem object. Typically, this would 1026347987a2SChristian König * point to the shmem object backing a GEM object if TTM is used to back a 1027347987a2SChristian König * GEM user interface. 1028347987a2SChristian König * @sg: Scatter-gather table. 1029347987a2SChristian König * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. 1030347987a2SChristian König * @destroy: Destroy function. Use NULL for kfree(). 1031347987a2SChristian König * 1032347987a2SChristian König * This function initializes a pre-allocated struct ttm_buffer_object. 1033347987a2SChristian König * As this object may be part of a larger structure, this function, 1034347987a2SChristian König * together with the @destroy function, 1035347987a2SChristian König * enables driver-specific objects derived from a ttm_buffer_object. 1036347987a2SChristian König * 1037347987a2SChristian König * On successful return, the caller owns an object kref to @bo. The kref and 1038347987a2SChristian König * list_kref are usually set to 1, but note that in some situations, other 1039347987a2SChristian König * tasks may already be holding references to @bo as well. 1040347987a2SChristian König * 1041347987a2SChristian König * If a failure occurs, the function will call the @destroy function, Thus, 1042347987a2SChristian König * after a failure, dereferencing @bo is illegal and will likely cause memory 1043347987a2SChristian König * corruption. 1044347987a2SChristian König * 1045347987a2SChristian König * Returns 1046347987a2SChristian König * -ENOMEM: Out of memory. 1047347987a2SChristian König * -EINVAL: Invalid placement flags. 1048347987a2SChristian König * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. 1049347987a2SChristian König */ 1050347987a2SChristian König int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, 1051347987a2SChristian König enum ttm_bo_type type, struct ttm_placement *placement, 1052347987a2SChristian König uint32_t alignment, bool interruptible, 1053347987a2SChristian König struct sg_table *sg, struct dma_resv *resv, 1054ca9cf68dSNicolai Hähnle void (*destroy) (struct ttm_buffer_object *)) 1055ca9cf68dSNicolai Hähnle { 10566fead44aSChristian König struct ttm_operation_ctx ctx = { interruptible, false }; 1057ca9cf68dSNicolai Hähnle int ret; 1058ca9cf68dSNicolai Hähnle 1059347987a2SChristian König ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, 1060347987a2SChristian König sg, resv, destroy); 1061ca9cf68dSNicolai Hähnle if (ret) 1062ca9cf68dSNicolai Hähnle return ret; 1063ca9cf68dSNicolai Hähnle 1064ca9cf68dSNicolai Hähnle if (!resv) 1065ca9cf68dSNicolai Hähnle ttm_bo_unreserve(bo); 1066ca9cf68dSNicolai Hähnle 1067ca9cf68dSNicolai Hähnle return 0; 1068ca9cf68dSNicolai Hähnle } 1069347987a2SChristian König EXPORT_SYMBOL(ttm_bo_init_validate); 1070ba4e7d97SThomas Hellstrom 1071ba4e7d97SThomas Hellstrom /* 1072ba4e7d97SThomas Hellstrom * buffer object vm functions. 1073ba4e7d97SThomas Hellstrom */ 1074ba4e7d97SThomas Hellstrom 1075a3185f91SChristian König /** 1076a3185f91SChristian König * ttm_bo_unmap_virtual 1077a3185f91SChristian König * 1078a3185f91SChristian König * @bo: tear down the virtual mappings for this BO 1079a3185f91SChristian König */ 1080fe662d84SChristian König void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1081ba4e7d97SThomas Hellstrom { 10828af8a109SChristian König struct ttm_device *bdev = bo->bdev; 1083ba4e7d97SThomas Hellstrom 1084b96f3e7cSGerd Hoffmann drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); 1085d3116756SChristian König ttm_mem_io_free(bdev, bo->resource); 1086ba4e7d97SThomas Hellstrom } 1087e024e110SDave Airlie EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1088ba4e7d97SThomas Hellstrom 1089a3185f91SChristian König /** 109013acb368SChristian König * ttm_bo_wait_ctx - wait for buffer idle. 1091a3185f91SChristian König * 1092a3185f91SChristian König * @bo: The buffer object. 109313acb368SChristian König * @ctx: defines how to wait 1094a3185f91SChristian König * 109513acb368SChristian König * Waits for the buffer to be idle. Used timeout depends on the context. 109613acb368SChristian König * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or 109713acb368SChristian König * zero on success. 1098a3185f91SChristian König */ 109913acb368SChristian König int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) 1100ba4e7d97SThomas Hellstrom { 110113acb368SChristian König long ret; 110298a6dd90SChristian König 110313acb368SChristian König if (ctx->no_wait_gpu) { 110413acb368SChristian König if (dma_resv_test_signaled(bo->base.resv, 110513acb368SChristian König DMA_RESV_USAGE_BOOKKEEP)) 110698a6dd90SChristian König return 0; 110798a6dd90SChristian König else 110898a6dd90SChristian König return -EBUSY; 110998a6dd90SChristian König } 1110ba4e7d97SThomas Hellstrom 111113acb368SChristian König ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, 111213acb368SChristian König ctx->interruptible, 15 * HZ); 111313acb368SChristian König if (unlikely(ret < 0)) 111413acb368SChristian König return ret; 111513acb368SChristian König if (unlikely(ret == 0)) 1116f2c24b83SMaarten Lankhorst return -EBUSY; 1117f2c24b83SMaarten Lankhorst return 0; 1118ba4e7d97SThomas Hellstrom } 1119a3185f91SChristian König EXPORT_SYMBOL(ttm_bo_wait_ctx); 1120a3185f91SChristian König 1121ebd59851SChristian König int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, 1122ebd59851SChristian König gfp_t gfp_flags) 1123ba4e7d97SThomas Hellstrom { 1124abb50d67SThomas Hellström struct ttm_place place; 1125dc947770SRoger He bool locked; 112613acb368SChristian König long ret; 1127ba4e7d97SThomas Hellstrom 1128abb50d67SThomas Hellström /* 1129abb50d67SThomas Hellström * While the bo may already reside in SYSTEM placement, set 1130abb50d67SThomas Hellström * SYSTEM as new placement to cover also the move further below. 1131abb50d67SThomas Hellström * The driver may use the fact that we're moving from SYSTEM 1132abb50d67SThomas Hellström * as an indication that we're about to swap out. 1133abb50d67SThomas Hellström */ 1134abb50d67SThomas Hellström memset(&place, 0, sizeof(place)); 1135e485382eSChristian König place.mem_type = bo->resource->mem_type; 1136abb50d67SThomas Hellström if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) 1137ebd59851SChristian König return -EBUSY; 11381ec39923SChristian König 11392d2ddb58SChristian König if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || 114043d46f0bSMatthew Auld bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || 114143d46f0bSMatthew Auld bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED || 11422d2ddb58SChristian König !ttm_bo_get_unless_zero(bo)) { 11431ec39923SChristian König if (locked) 11441ec39923SChristian König dma_resv_unlock(bo->base.resv); 1145ebd59851SChristian König return -EBUSY; 11462b7b3ad2SMaarten Lankhorst } 11472b7b3ad2SMaarten Lankhorst 11481ec39923SChristian König if (bo->deleted) { 114970982eefSxinhui pan ret = ttm_bo_cleanup_refs(bo, false, false, locked); 11501ec39923SChristian König ttm_bo_put(bo); 115170982eefSxinhui pan return ret == -EBUSY ? -ENOSPC : ret; 1152e1efc9b6SThomas Hellstrom } 1153e1efc9b6SThomas Hellstrom 1154ebd59851SChristian König /* TODO: Cleanup the locking */ 1155a1f091f8SChristian König spin_unlock(&bo->bdev->lru_lock); 1156ba4e7d97SThomas Hellstrom 1157ebd59851SChristian König /* 115861ede070SChristian König * Move to system cached 1159ba4e7d97SThomas Hellstrom */ 1160d3116756SChristian König if (bo->resource->mem_type != TTM_PL_SYSTEM) { 1161bfa3357eSChristian König struct ttm_resource *evict_mem; 1162abb50d67SThomas Hellström struct ttm_place hop; 1163ebdf5651SDave Airlie 1164ebdf5651SDave Airlie memset(&hop, 0, sizeof(hop)); 1165e485382eSChristian König place.mem_type = TTM_PL_SYSTEM; 11665109d297SChristian König ret = ttm_resource_alloc(bo, &place, &evict_mem); 11675109d297SChristian König if (unlikely(ret)) 11685109d297SChristian König goto out; 1169ba4e7d97SThomas Hellstrom 1170731f4ab5SThomas Hellström ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); 1171ebdf5651SDave Airlie if (unlikely(ret != 0)) { 1172ebdf5651SDave Airlie WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); 1173a590f03dSThomas Hellström ttm_resource_free(bo, &evict_mem); 1174ba4e7d97SThomas Hellstrom goto out; 1175ba4e7d97SThomas Hellstrom } 1176ebdf5651SDave Airlie } 1177ba4e7d97SThomas Hellstrom 1178ebd59851SChristian König /* 117961ede070SChristian König * Make sure BO is idle. 118061ede070SChristian König */ 118142523924SChristian König ret = ttm_bo_wait_ctx(bo, ctx); 118261ede070SChristian König if (unlikely(ret != 0)) 118361ede070SChristian König goto out; 118461ede070SChristian König 1185ba4e7d97SThomas Hellstrom ttm_bo_unmap_virtual(bo); 1186ba4e7d97SThomas Hellstrom 1187ebd59851SChristian König /* 1188ba4e7d97SThomas Hellstrom * Swap out. Buffer will be swapped in again as soon as 1189ba4e7d97SThomas Hellstrom * anyone tries to access a ttm page. 1190ba4e7d97SThomas Hellstrom */ 11918af8a109SChristian König if (bo->bdev->funcs->swap_notify) 11928af8a109SChristian König bo->bdev->funcs->swap_notify(bo); 11933f09ea4eSThomas Hellstrom 1194abb50d67SThomas Hellström if (ttm_tt_is_populated(bo->ttm)) 1195d4bd7776SChristian König ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); 1196ba4e7d97SThomas Hellstrom out: 1197ba4e7d97SThomas Hellstrom 1198ebd59851SChristian König /* 1199ba4e7d97SThomas Hellstrom * Unreserve without putting on LRU to avoid swapping out an 1200ba4e7d97SThomas Hellstrom * already swapped buffer. 1201ba4e7d97SThomas Hellstrom */ 1202ad76c65eSFelix Kuehling if (locked) 120352791eeeSChristian König dma_resv_unlock(bo->base.resv); 12041ec39923SChristian König ttm_bo_put(bo); 120570982eefSxinhui pan return ret == -EBUSY ? -ENOSPC : ret; 1206ba4e7d97SThomas Hellstrom } 1207ba4e7d97SThomas Hellstrom 12082ff6e69cSDave Airlie void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) 12092ff6e69cSDave Airlie { 1210395a73f8SDave Airlie if (bo->ttm == NULL) 1211395a73f8SDave Airlie return; 12129e9a153bSDave Airlie 1213d5f45d1eSChristian König ttm_tt_unpopulate(bo->bdev, bo->ttm); 12142ff6e69cSDave Airlie ttm_tt_destroy(bo->bdev, bo->ttm); 12152ff6e69cSDave Airlie bo->ttm = NULL; 12162ff6e69cSDave Airlie } 1217