1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_bo_evict.h" 7 8 #include "xe_bo.h" 9 #include "xe_device.h" 10 #include "xe_ggtt.h" 11 #include "xe_tile.h" 12 13 /** 14 * xe_bo_evict_all - evict all BOs from VRAM 15 * 16 * @xe: xe device 17 * 18 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next 19 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. 20 * All eviction magic done via TTM calls. 21 * 22 * Evict == move VRAM BOs to temporary (typically system) memory. 23 * 24 * This function should be called before the device goes into a suspend state 25 * where the VRAM loses power. 26 */ 27 int xe_bo_evict_all(struct xe_device *xe) 28 { 29 struct ttm_device *bdev = &xe->ttm; 30 struct xe_bo *bo; 31 struct xe_tile *tile; 32 struct list_head still_in_list; 33 u32 mem_type; 34 u8 id; 35 int ret; 36 37 /* User memory */ 38 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) { 39 struct ttm_resource_manager *man = 40 ttm_manager_type(bdev, mem_type); 41 42 /* 43 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS 44 * state since this state lives inside graphics stolen memory which doesn't survive 45 * hibernation. 46 * 47 * This can be further improved by only evicting objects that we know have actually 48 * used a compression enabled PAT index. 49 */ 50 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe))) 51 continue; 52 53 if (man) { 54 ret = ttm_resource_manager_evict_all(bdev, man); 55 if (ret) 56 return ret; 57 } 58 } 59 60 /* Pinned user memory in VRAM */ 61 INIT_LIST_HEAD(&still_in_list); 62 spin_lock(&xe->pinned.lock); 63 for (;;) { 64 bo = list_first_entry_or_null(&xe->pinned.external_vram, 65 typeof(*bo), pinned_link); 66 if (!bo) 67 break; 68 xe_bo_get(bo); 69 list_move_tail(&bo->pinned_link, &still_in_list); 70 spin_unlock(&xe->pinned.lock); 71 72 xe_bo_lock(bo, false); 73 ret = xe_bo_evict_pinned(bo); 74 xe_bo_unlock(bo); 75 xe_bo_put(bo); 76 if (ret) { 77 spin_lock(&xe->pinned.lock); 78 list_splice_tail(&still_in_list, 79 &xe->pinned.external_vram); 80 spin_unlock(&xe->pinned.lock); 81 return ret; 82 } 83 84 spin_lock(&xe->pinned.lock); 85 } 86 list_splice_tail(&still_in_list, &xe->pinned.external_vram); 87 spin_unlock(&xe->pinned.lock); 88 89 /* 90 * Wait for all user BO to be evicted as those evictions depend on the 91 * memory moved below. 92 */ 93 for_each_tile(tile, xe, id) 94 xe_tile_migrate_wait(tile); 95 96 spin_lock(&xe->pinned.lock); 97 for (;;) { 98 bo = list_first_entry_or_null(&xe->pinned.kernel_bo_present, 99 typeof(*bo), pinned_link); 100 if (!bo) 101 break; 102 xe_bo_get(bo); 103 list_move_tail(&bo->pinned_link, &xe->pinned.evicted); 104 spin_unlock(&xe->pinned.lock); 105 106 xe_bo_lock(bo, false); 107 ret = xe_bo_evict_pinned(bo); 108 xe_bo_unlock(bo); 109 xe_bo_put(bo); 110 if (ret) 111 return ret; 112 113 spin_lock(&xe->pinned.lock); 114 } 115 spin_unlock(&xe->pinned.lock); 116 117 return 0; 118 } 119 120 /** 121 * xe_bo_restore_kernel - restore kernel BOs to VRAM 122 * 123 * @xe: xe device 124 * 125 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All 126 * moves done via TTM calls. 127 * 128 * This function should be called early, before trying to init the GT, on device 129 * resume. 130 */ 131 int xe_bo_restore_kernel(struct xe_device *xe) 132 { 133 struct xe_bo *bo; 134 int ret; 135 136 spin_lock(&xe->pinned.lock); 137 for (;;) { 138 bo = list_first_entry_or_null(&xe->pinned.evicted, 139 typeof(*bo), pinned_link); 140 if (!bo) 141 break; 142 xe_bo_get(bo); 143 list_move_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present); 144 spin_unlock(&xe->pinned.lock); 145 146 xe_bo_lock(bo, false); 147 ret = xe_bo_restore_pinned(bo); 148 xe_bo_unlock(bo); 149 if (ret) { 150 xe_bo_put(bo); 151 return ret; 152 } 153 154 if (bo->flags & XE_BO_FLAG_GGTT) { 155 struct xe_tile *tile = bo->tile; 156 157 mutex_lock(&tile->mem.ggtt->lock); 158 xe_ggtt_map_bo(tile->mem.ggtt, bo); 159 mutex_unlock(&tile->mem.ggtt->lock); 160 } 161 162 /* 163 * We expect validate to trigger a move VRAM and our move code 164 * should setup the iosys map. 165 */ 166 xe_assert(xe, !iosys_map_is_null(&bo->vmap)); 167 168 xe_bo_put(bo); 169 170 spin_lock(&xe->pinned.lock); 171 } 172 spin_unlock(&xe->pinned.lock); 173 174 return 0; 175 } 176 177 /** 178 * xe_bo_restore_user - restore pinned user BOs to VRAM 179 * 180 * @xe: xe device 181 * 182 * Move pinned user BOs from temporary (typically system) memory to VRAM via 183 * CPU. All moves done via TTM calls. 184 * 185 * This function should be called late, after GT init, on device resume. 186 */ 187 int xe_bo_restore_user(struct xe_device *xe) 188 { 189 struct xe_bo *bo; 190 struct xe_tile *tile; 191 struct list_head still_in_list; 192 u8 id; 193 int ret; 194 195 if (!IS_DGFX(xe)) 196 return 0; 197 198 /* Pinned user memory in VRAM should be validated on resume */ 199 INIT_LIST_HEAD(&still_in_list); 200 spin_lock(&xe->pinned.lock); 201 for (;;) { 202 bo = list_first_entry_or_null(&xe->pinned.external_vram, 203 typeof(*bo), pinned_link); 204 if (!bo) 205 break; 206 list_move_tail(&bo->pinned_link, &still_in_list); 207 xe_bo_get(bo); 208 spin_unlock(&xe->pinned.lock); 209 210 xe_bo_lock(bo, false); 211 ret = xe_bo_restore_pinned(bo); 212 xe_bo_unlock(bo); 213 xe_bo_put(bo); 214 if (ret) { 215 spin_lock(&xe->pinned.lock); 216 list_splice_tail(&still_in_list, 217 &xe->pinned.external_vram); 218 spin_unlock(&xe->pinned.lock); 219 return ret; 220 } 221 222 spin_lock(&xe->pinned.lock); 223 } 224 list_splice_tail(&still_in_list, &xe->pinned.external_vram); 225 spin_unlock(&xe->pinned.lock); 226 227 /* Wait for restore to complete */ 228 for_each_tile(tile, xe, id) 229 xe_tile_migrate_wait(tile); 230 231 return 0; 232 } 233