1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_bo_evict.h" 7 8 #include "xe_bo.h" 9 #include "xe_device.h" 10 #include "xe_ggtt.h" 11 #include "xe_tile.h" 12 13 typedef int (*xe_pinned_fn)(struct xe_bo *bo); 14 15 static int xe_bo_apply_to_pinned(struct xe_device *xe, 16 struct list_head *pinned_list, 17 struct list_head *new_list, 18 const xe_pinned_fn pinned_fn) 19 { 20 LIST_HEAD(still_in_list); 21 struct xe_bo *bo; 22 int ret = 0; 23 24 spin_lock(&xe->pinned.lock); 25 while (!ret) { 26 bo = list_first_entry_or_null(pinned_list, typeof(*bo), 27 pinned_link); 28 if (!bo) 29 break; 30 xe_bo_get(bo); 31 list_move_tail(&bo->pinned_link, &still_in_list); 32 spin_unlock(&xe->pinned.lock); 33 34 ret = pinned_fn(bo); 35 if (ret && pinned_list != new_list) { 36 spin_lock(&xe->pinned.lock); 37 /* 38 * We might no longer be pinned, since PM notifier can 39 * call this. If the pinned link is now empty, keep it 40 * that way. 41 */ 42 if (!list_empty(&bo->pinned_link)) 43 list_move(&bo->pinned_link, pinned_list); 44 spin_unlock(&xe->pinned.lock); 45 } 46 xe_bo_put(bo); 47 spin_lock(&xe->pinned.lock); 48 } 49 list_splice_tail(&still_in_list, new_list); 50 spin_unlock(&xe->pinned.lock); 51 52 return ret; 53 } 54 55 /** 56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all 57 * pinned VRAM objects which need to be saved. 58 * @xe: xe device 59 * 60 * Should be called from PM notifier when preparing for s3/s4. 61 * 62 * Return: 0 on success, negative error code on error. 63 */ 64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe) 65 { 66 int ret; 67 68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 69 &xe->pinned.early.kernel_bo_present, 70 xe_bo_notifier_prepare_pinned); 71 if (!ret) 72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 73 &xe->pinned.late.kernel_bo_present, 74 xe_bo_notifier_prepare_pinned); 75 76 return ret; 77 } 78 79 /** 80 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all 81 * pinned VRAM objects which have been restored. 82 * @xe: xe device 83 * 84 * Should be called from PM notifier after exiting s3/s4 (either on success or 85 * failure). 86 */ 87 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe) 88 { 89 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 90 &xe->pinned.early.kernel_bo_present, 91 xe_bo_notifier_unprepare_pinned); 92 93 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 94 &xe->pinned.late.kernel_bo_present, 95 xe_bo_notifier_unprepare_pinned); 96 } 97 98 /** 99 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM 100 * @xe: xe device 101 * 102 * Evict non-pinned user BOs (via GPU). 103 * 104 * Evict == move VRAM BOs to temporary (typically system) memory. 105 */ 106 int xe_bo_evict_all_user(struct xe_device *xe) 107 { 108 struct ttm_device *bdev = &xe->ttm; 109 u32 mem_type; 110 int ret; 111 112 /* User memory */ 113 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) { 114 struct ttm_resource_manager *man = 115 ttm_manager_type(bdev, mem_type); 116 117 /* 118 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS 119 * state since this state lives inside graphics stolen memory which doesn't survive 120 * hibernation. 121 * 122 * This can be further improved by only evicting objects that we know have actually 123 * used a compression enabled PAT index. 124 */ 125 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe))) 126 continue; 127 128 if (man) { 129 ret = ttm_resource_manager_evict_all(bdev, man); 130 if (ret) 131 return ret; 132 } 133 } 134 135 return 0; 136 } 137 138 /** 139 * xe_bo_evict_all - evict all BOs from VRAM 140 * @xe: xe device 141 * 142 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next 143 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. 144 * All eviction magic done via TTM calls. 145 * 146 * Evict == move VRAM BOs to temporary (typically system) memory. 147 * 148 * This function should be called before the device goes into a suspend state 149 * where the VRAM loses power. 150 */ 151 int xe_bo_evict_all(struct xe_device *xe) 152 { 153 struct xe_tile *tile; 154 u8 id; 155 int ret; 156 157 ret = xe_bo_evict_all_user(xe); 158 if (ret) 159 return ret; 160 161 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 162 &xe->pinned.late.external, xe_bo_evict_pinned); 163 164 if (!ret) 165 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 166 &xe->pinned.late.evicted, xe_bo_evict_pinned); 167 168 /* 169 * Wait for all user BO to be evicted as those evictions depend on the 170 * memory moved below. 171 */ 172 for_each_tile(tile, xe, id) 173 xe_tile_migrate_wait(tile); 174 175 if (ret) 176 return ret; 177 178 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 179 &xe->pinned.early.evicted, 180 xe_bo_evict_pinned); 181 } 182 183 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) 184 { 185 struct xe_device *xe = xe_bo_device(bo); 186 int ret; 187 188 ret = xe_bo_restore_pinned(bo); 189 if (ret) 190 return ret; 191 192 if (bo->flags & XE_BO_FLAG_GGTT) { 193 struct xe_tile *tile; 194 u8 id; 195 196 for_each_tile(tile, xe_bo_device(bo), id) { 197 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile))) 198 continue; 199 200 xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo); 201 } 202 } 203 204 /* 205 * We expect validate to trigger a move VRAM and our move code 206 * should setup the iosys map. 207 */ 208 xe_assert(xe, !(bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) || 209 !iosys_map_is_null(&bo->vmap)); 210 211 return 0; 212 } 213 214 /** 215 * xe_bo_restore_early - restore early phase kernel BOs to VRAM 216 * 217 * @xe: xe device 218 * 219 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All 220 * moves done via TTM calls. 221 * 222 * This function should be called early, before trying to init the GT, on device 223 * resume. 224 */ 225 int xe_bo_restore_early(struct xe_device *xe) 226 { 227 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted, 228 &xe->pinned.early.kernel_bo_present, 229 xe_bo_restore_and_map_ggtt); 230 } 231 232 /** 233 * xe_bo_restore_late - restore pinned late phase BOs 234 * 235 * @xe: xe device 236 * 237 * Move pinned user and kernel BOs which can use blitter from temporary 238 * (typically system) memory to VRAM. All moves done via TTM calls. 239 * 240 * This function should be called late, after GT init, on device resume. 241 */ 242 int xe_bo_restore_late(struct xe_device *xe) 243 { 244 struct xe_tile *tile; 245 int ret, id; 246 247 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted, 248 &xe->pinned.late.kernel_bo_present, 249 xe_bo_restore_and_map_ggtt); 250 251 for_each_tile(tile, xe, id) 252 xe_tile_migrate_wait(tile); 253 254 if (ret) 255 return ret; 256 257 if (!IS_DGFX(xe)) 258 return 0; 259 260 /* Pinned user memory in VRAM should be validated on resume */ 261 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 262 &xe->pinned.late.external, 263 xe_bo_restore_pinned); 264 265 /* Wait for restore to complete */ 266 for_each_tile(tile, xe, id) 267 xe_tile_migrate_wait(tile); 268 269 return ret; 270 } 271 272 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe) 273 { 274 struct xe_tile *tile; 275 unsigned int id; 276 277 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 278 &xe->pinned.late.external, 279 xe_bo_dma_unmap_pinned); 280 for_each_tile(tile, xe, id) 281 xe_tile_migrate_wait(tile); 282 } 283 284 /** 285 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed 286 * @xe: The xe device. 287 * 288 * On pci_device removal we need to drop all dma mappings and move 289 * the data of exported bos out to system. This includes SVM bos and 290 * exported dma-buf bos. This is done by evicting all bos, but 291 * the evict placement in xe_evict_flags() is chosen such that all 292 * bos except those mentioned are purged, and thus their memory 293 * is released. 294 * 295 * For pinned bos, we're unmapping dma. 296 */ 297 void xe_bo_pci_dev_remove_all(struct xe_device *xe) 298 { 299 unsigned int mem_type; 300 301 /* 302 * Move pagemap bos and exported dma-buf to system, and 303 * purge everything else. 304 */ 305 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) { 306 struct ttm_resource_manager *man = 307 ttm_manager_type(&xe->ttm, mem_type); 308 309 if (man) { 310 int ret = ttm_resource_manager_evict_all(&xe->ttm, man); 311 312 drm_WARN_ON(&xe->drm, ret); 313 } 314 } 315 316 xe_bo_pci_dev_remove_pinned(xe); 317 } 318 319 static void xe_bo_pinned_fini(void *arg) 320 { 321 struct xe_device *xe = arg; 322 323 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 324 &xe->pinned.late.kernel_bo_present, 325 xe_bo_dma_unmap_pinned); 326 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 327 &xe->pinned.early.kernel_bo_present, 328 xe_bo_dma_unmap_pinned); 329 } 330 331 /** 332 * xe_bo_pinned_init() - Initialize pinned bo tracking 333 * @xe: The xe device. 334 * 335 * Initializes the lists and locks required for pinned bo 336 * tracking and registers a callback to dma-unmap 337 * any remaining pinned bos on pci device removal. 338 * 339 * Return: %0 on success, negative error code on error. 340 */ 341 int xe_bo_pinned_init(struct xe_device *xe) 342 { 343 spin_lock_init(&xe->pinned.lock); 344 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present); 345 INIT_LIST_HEAD(&xe->pinned.early.evicted); 346 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present); 347 INIT_LIST_HEAD(&xe->pinned.late.evicted); 348 INIT_LIST_HEAD(&xe->pinned.late.external); 349 350 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe); 351 } 352