1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_bo_evict.h" 7 8 #include "xe_bo.h" 9 #include "xe_device.h" 10 #include "xe_ggtt.h" 11 #include "xe_tile.h" 12 13 typedef int (*xe_pinned_fn)(struct xe_bo *bo); 14 15 static int xe_bo_apply_to_pinned(struct xe_device *xe, 16 struct list_head *pinned_list, 17 struct list_head *new_list, 18 const xe_pinned_fn pinned_fn) 19 { 20 LIST_HEAD(still_in_list); 21 struct xe_bo *bo; 22 int ret = 0; 23 24 spin_lock(&xe->pinned.lock); 25 while (!ret) { 26 bo = list_first_entry_or_null(pinned_list, typeof(*bo), 27 pinned_link); 28 if (!bo) 29 break; 30 xe_bo_get(bo); 31 list_move_tail(&bo->pinned_link, &still_in_list); 32 spin_unlock(&xe->pinned.lock); 33 34 ret = pinned_fn(bo); 35 if (ret && pinned_list != new_list) { 36 spin_lock(&xe->pinned.lock); 37 /* 38 * We might no longer be pinned, since PM notifier can 39 * call this. If the pinned link is now empty, keep it 40 * that way. 41 */ 42 if (!list_empty(&bo->pinned_link)) 43 list_move(&bo->pinned_link, pinned_list); 44 spin_unlock(&xe->pinned.lock); 45 } 46 xe_bo_put(bo); 47 spin_lock(&xe->pinned.lock); 48 } 49 list_splice_tail(&still_in_list, new_list); 50 spin_unlock(&xe->pinned.lock); 51 52 return ret; 53 } 54 55 /** 56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all 57 * pinned VRAM objects which need to be saved. 58 * @xe: xe device 59 * 60 * Should be called from PM notifier when preparing for s3/s4. 61 * 62 * Return: 0 on success, negative error code on error. 63 */ 64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe) 65 { 66 int ret; 67 68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 69 &xe->pinned.early.kernel_bo_present, 70 xe_bo_notifier_prepare_pinned); 71 if (!ret) 72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 73 &xe->pinned.late.kernel_bo_present, 74 xe_bo_notifier_prepare_pinned); 75 76 if (!ret) 77 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 78 &xe->pinned.late.external, 79 xe_bo_notifier_prepare_pinned); 80 81 return ret; 82 } 83 84 /** 85 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all 86 * pinned VRAM objects which have been restored. 87 * @xe: xe device 88 * 89 * Should be called from PM notifier after exiting s3/s4 (either on success or 90 * failure). 91 */ 92 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe) 93 { 94 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 95 &xe->pinned.early.kernel_bo_present, 96 xe_bo_notifier_unprepare_pinned); 97 98 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 99 &xe->pinned.late.kernel_bo_present, 100 xe_bo_notifier_unprepare_pinned); 101 102 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 103 &xe->pinned.late.external, 104 xe_bo_notifier_unprepare_pinned); 105 } 106 107 /** 108 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM 109 * @xe: xe device 110 * 111 * Evict non-pinned user BOs (via GPU). 112 * 113 * Evict == move VRAM BOs to temporary (typically system) memory. 114 */ 115 int xe_bo_evict_all_user(struct xe_device *xe) 116 { 117 struct ttm_device *bdev = &xe->ttm; 118 u32 mem_type; 119 int ret; 120 121 /* User memory */ 122 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) { 123 struct ttm_resource_manager *man = 124 ttm_manager_type(bdev, mem_type); 125 126 /* 127 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS 128 * state since this state lives inside graphics stolen memory which doesn't survive 129 * hibernation. 130 * 131 * This can be further improved by only evicting objects that we know have actually 132 * used a compression enabled PAT index. 133 */ 134 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe))) 135 continue; 136 137 if (man) { 138 ret = ttm_resource_manager_evict_all(bdev, man); 139 if (ret) 140 return ret; 141 } 142 } 143 144 return 0; 145 } 146 147 /** 148 * xe_bo_evict_all - evict all BOs from VRAM 149 * @xe: xe device 150 * 151 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next 152 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. 153 * All eviction magic done via TTM calls. 154 * 155 * Evict == move VRAM BOs to temporary (typically system) memory. 156 * 157 * This function should be called before the device goes into a suspend state 158 * where the VRAM loses power. 159 */ 160 int xe_bo_evict_all(struct xe_device *xe) 161 { 162 struct xe_tile *tile; 163 u8 id; 164 int ret; 165 166 ret = xe_bo_evict_all_user(xe); 167 if (ret) 168 return ret; 169 170 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 171 &xe->pinned.late.external, xe_bo_evict_pinned); 172 173 if (!ret) 174 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 175 &xe->pinned.late.evicted, xe_bo_evict_pinned); 176 177 /* 178 * Wait for all user BO to be evicted as those evictions depend on the 179 * memory moved below. 180 */ 181 for_each_tile(tile, xe, id) 182 xe_tile_migrate_wait(tile); 183 184 if (ret) 185 return ret; 186 187 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 188 &xe->pinned.early.evicted, 189 xe_bo_evict_pinned); 190 } 191 192 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) 193 { 194 int ret; 195 196 ret = xe_bo_restore_pinned(bo); 197 if (ret) 198 return ret; 199 200 if (bo->flags & XE_BO_FLAG_GGTT) { 201 struct xe_tile *tile; 202 u8 id; 203 204 for_each_tile(tile, xe_bo_device(bo), id) { 205 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile))) 206 continue; 207 208 xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo); 209 } 210 } 211 212 return 0; 213 } 214 215 /** 216 * xe_bo_restore_early - restore early phase kernel BOs to VRAM 217 * 218 * @xe: xe device 219 * 220 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All 221 * moves done via TTM calls. 222 * 223 * This function should be called early, before trying to init the GT, on device 224 * resume. 225 */ 226 int xe_bo_restore_early(struct xe_device *xe) 227 { 228 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted, 229 &xe->pinned.early.kernel_bo_present, 230 xe_bo_restore_and_map_ggtt); 231 } 232 233 /** 234 * xe_bo_restore_late - restore pinned late phase BOs 235 * 236 * @xe: xe device 237 * 238 * Move pinned user and kernel BOs which can use blitter from temporary 239 * (typically system) memory to VRAM. All moves done via TTM calls. 240 * 241 * This function should be called late, after GT init, on device resume. 242 */ 243 int xe_bo_restore_late(struct xe_device *xe) 244 { 245 struct xe_tile *tile; 246 int ret, id; 247 248 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted, 249 &xe->pinned.late.kernel_bo_present, 250 xe_bo_restore_and_map_ggtt); 251 252 for_each_tile(tile, xe, id) 253 xe_tile_migrate_wait(tile); 254 255 if (ret) 256 return ret; 257 258 if (!IS_DGFX(xe)) 259 return 0; 260 261 /* Pinned user memory in VRAM should be validated on resume */ 262 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 263 &xe->pinned.late.external, 264 xe_bo_restore_pinned); 265 266 /* Wait for restore to complete */ 267 for_each_tile(tile, xe, id) 268 xe_tile_migrate_wait(tile); 269 270 return ret; 271 } 272 273 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe) 274 { 275 struct xe_tile *tile; 276 unsigned int id; 277 278 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 279 &xe->pinned.late.external, 280 xe_bo_dma_unmap_pinned); 281 for_each_tile(tile, xe, id) 282 xe_tile_migrate_wait(tile); 283 } 284 285 /** 286 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed 287 * @xe: The xe device. 288 * 289 * On pci_device removal we need to drop all dma mappings and move 290 * the data of exported bos out to system. This includes SVM bos and 291 * exported dma-buf bos. This is done by evicting all bos, but 292 * the evict placement in xe_evict_flags() is chosen such that all 293 * bos except those mentioned are purged, and thus their memory 294 * is released. 295 * 296 * For pinned bos, we're unmapping dma. 297 */ 298 void xe_bo_pci_dev_remove_all(struct xe_device *xe) 299 { 300 unsigned int mem_type; 301 302 /* 303 * Move pagemap bos and exported dma-buf to system, and 304 * purge everything else. 305 */ 306 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) { 307 struct ttm_resource_manager *man = 308 ttm_manager_type(&xe->ttm, mem_type); 309 310 if (man) { 311 int ret = ttm_resource_manager_evict_all(&xe->ttm, man); 312 313 drm_WARN_ON(&xe->drm, ret); 314 } 315 } 316 317 xe_bo_pci_dev_remove_pinned(xe); 318 } 319 320 static void xe_bo_pinned_fini(void *arg) 321 { 322 struct xe_device *xe = arg; 323 324 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 325 &xe->pinned.late.kernel_bo_present, 326 xe_bo_dma_unmap_pinned); 327 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 328 &xe->pinned.early.kernel_bo_present, 329 xe_bo_dma_unmap_pinned); 330 } 331 332 /** 333 * xe_bo_pinned_init() - Initialize pinned bo tracking 334 * @xe: The xe device. 335 * 336 * Initializes the lists and locks required for pinned bo 337 * tracking and registers a callback to dma-unmap 338 * any remaining pinned bos on pci device removal. 339 * 340 * Return: %0 on success, negative error code on error. 341 */ 342 int xe_bo_pinned_init(struct xe_device *xe) 343 { 344 spin_lock_init(&xe->pinned.lock); 345 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present); 346 INIT_LIST_HEAD(&xe->pinned.early.evicted); 347 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present); 348 INIT_LIST_HEAD(&xe->pinned.late.evicted); 349 INIT_LIST_HEAD(&xe->pinned.late.external); 350 351 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe); 352 } 353