1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_bo_evict.h"
7
8 #include "xe_bo.h"
9 #include "xe_device.h"
10 #include "xe_ggtt.h"
11 #include "xe_tile.h"
12
13 typedef int (*xe_pinned_fn)(struct xe_bo *bo);
14
xe_bo_apply_to_pinned(struct xe_device * xe,struct list_head * pinned_list,struct list_head * new_list,const xe_pinned_fn pinned_fn)15 static int xe_bo_apply_to_pinned(struct xe_device *xe,
16 struct list_head *pinned_list,
17 struct list_head *new_list,
18 const xe_pinned_fn pinned_fn)
19 {
20 LIST_HEAD(still_in_list);
21 struct xe_bo *bo;
22 int ret = 0;
23
24 spin_lock(&xe->pinned.lock);
25 while (!ret) {
26 bo = list_first_entry_or_null(pinned_list, typeof(*bo),
27 pinned_link);
28 if (!bo)
29 break;
30 xe_bo_get(bo);
31 list_move_tail(&bo->pinned_link, &still_in_list);
32 spin_unlock(&xe->pinned.lock);
33
34 ret = pinned_fn(bo);
35 if (ret && pinned_list != new_list) {
36 spin_lock(&xe->pinned.lock);
37 /*
38 * We might no longer be pinned, since PM notifier can
39 * call this. If the pinned link is now empty, keep it
40 * that way.
41 */
42 if (!list_empty(&bo->pinned_link))
43 list_move(&bo->pinned_link, pinned_list);
44 spin_unlock(&xe->pinned.lock);
45 }
46 xe_bo_put(bo);
47 spin_lock(&xe->pinned.lock);
48 }
49 list_splice_tail(&still_in_list, new_list);
50 spin_unlock(&xe->pinned.lock);
51
52 return ret;
53 }
54
55 /**
56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all
57 * pinned VRAM objects which need to be saved.
58 * @xe: xe device
59 *
60 * Should be called from PM notifier when preparing for s3/s4.
61 *
62 * Return: 0 on success, negative error code on error.
63 */
xe_bo_notifier_prepare_all_pinned(struct xe_device * xe)64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe)
65 {
66 int ret;
67
68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
69 &xe->pinned.early.kernel_bo_present,
70 xe_bo_notifier_prepare_pinned);
71 if (!ret)
72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
73 &xe->pinned.late.kernel_bo_present,
74 xe_bo_notifier_prepare_pinned);
75
76 return ret;
77 }
78
79 /**
80 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all
81 * pinned VRAM objects which have been restored.
82 * @xe: xe device
83 *
84 * Should be called from PM notifier after exiting s3/s4 (either on success or
85 * failure).
86 */
xe_bo_notifier_unprepare_all_pinned(struct xe_device * xe)87 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe)
88 {
89 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
90 &xe->pinned.early.kernel_bo_present,
91 xe_bo_notifier_unprepare_pinned);
92
93 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
94 &xe->pinned.late.kernel_bo_present,
95 xe_bo_notifier_unprepare_pinned);
96 }
97
98 /**
99 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM
100 * @xe: xe device
101 *
102 * Evict non-pinned user BOs (via GPU).
103 *
104 * Evict == move VRAM BOs to temporary (typically system) memory.
105 */
xe_bo_evict_all_user(struct xe_device * xe)106 int xe_bo_evict_all_user(struct xe_device *xe)
107 {
108 struct ttm_device *bdev = &xe->ttm;
109 u32 mem_type;
110 int ret;
111
112 /* User memory */
113 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) {
114 struct ttm_resource_manager *man =
115 ttm_manager_type(bdev, mem_type);
116
117 /*
118 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS
119 * state since this state lives inside graphics stolen memory which doesn't survive
120 * hibernation.
121 *
122 * This can be further improved by only evicting objects that we know have actually
123 * used a compression enabled PAT index.
124 */
125 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)))
126 continue;
127
128 if (man) {
129 ret = ttm_resource_manager_evict_all(bdev, man);
130 if (ret)
131 return ret;
132 }
133 }
134
135 return 0;
136 }
137
138 /**
139 * xe_bo_evict_all - evict all BOs from VRAM
140 * @xe: xe device
141 *
142 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next
143 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.
144 * All eviction magic done via TTM calls.
145 *
146 * Evict == move VRAM BOs to temporary (typically system) memory.
147 *
148 * This function should be called before the device goes into a suspend state
149 * where the VRAM loses power.
150 */
xe_bo_evict_all(struct xe_device * xe)151 int xe_bo_evict_all(struct xe_device *xe)
152 {
153 struct xe_tile *tile;
154 u8 id;
155 int ret;
156
157 ret = xe_bo_evict_all_user(xe);
158 if (ret)
159 return ret;
160
161 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
162 &xe->pinned.late.external, xe_bo_evict_pinned);
163
164 if (!ret)
165 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
166 &xe->pinned.late.evicted, xe_bo_evict_pinned);
167
168 /*
169 * Wait for all user BO to be evicted as those evictions depend on the
170 * memory moved below.
171 */
172 for_each_tile(tile, xe, id)
173 xe_tile_migrate_wait(tile);
174
175 if (ret)
176 return ret;
177
178 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
179 &xe->pinned.early.evicted,
180 xe_bo_evict_pinned);
181 }
182
xe_bo_restore_and_map_ggtt(struct xe_bo * bo)183 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
184 {
185 struct xe_device *xe = xe_bo_device(bo);
186 int ret;
187
188 ret = xe_bo_restore_pinned(bo);
189 if (ret)
190 return ret;
191
192 if (bo->flags & XE_BO_FLAG_GGTT) {
193 struct xe_tile *tile;
194 u8 id;
195
196 for_each_tile(tile, xe_bo_device(bo), id) {
197 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile)))
198 continue;
199
200 xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo);
201 }
202 }
203
204 /*
205 * We expect validate to trigger a move VRAM and our move code
206 * should setup the iosys map.
207 */
208 xe_assert(xe, !(bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) ||
209 !iosys_map_is_null(&bo->vmap));
210
211 return 0;
212 }
213
214 /**
215 * xe_bo_restore_early - restore early phase kernel BOs to VRAM
216 *
217 * @xe: xe device
218 *
219 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All
220 * moves done via TTM calls.
221 *
222 * This function should be called early, before trying to init the GT, on device
223 * resume.
224 */
xe_bo_restore_early(struct xe_device * xe)225 int xe_bo_restore_early(struct xe_device *xe)
226 {
227 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,
228 &xe->pinned.early.kernel_bo_present,
229 xe_bo_restore_and_map_ggtt);
230 }
231
232 /**
233 * xe_bo_restore_late - restore pinned late phase BOs
234 *
235 * @xe: xe device
236 *
237 * Move pinned user and kernel BOs which can use blitter from temporary
238 * (typically system) memory to VRAM. All moves done via TTM calls.
239 *
240 * This function should be called late, after GT init, on device resume.
241 */
xe_bo_restore_late(struct xe_device * xe)242 int xe_bo_restore_late(struct xe_device *xe)
243 {
244 struct xe_tile *tile;
245 int ret, id;
246
247 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,
248 &xe->pinned.late.kernel_bo_present,
249 xe_bo_restore_and_map_ggtt);
250
251 for_each_tile(tile, xe, id)
252 xe_tile_migrate_wait(tile);
253
254 if (ret)
255 return ret;
256
257 if (!IS_DGFX(xe))
258 return 0;
259
260 /* Pinned user memory in VRAM should be validated on resume */
261 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
262 &xe->pinned.late.external,
263 xe_bo_restore_pinned);
264
265 /* Wait for restore to complete */
266 for_each_tile(tile, xe, id)
267 xe_tile_migrate_wait(tile);
268
269 return ret;
270 }
271
xe_bo_pci_dev_remove_pinned(struct xe_device * xe)272 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe)
273 {
274 struct xe_tile *tile;
275 unsigned int id;
276
277 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
278 &xe->pinned.late.external,
279 xe_bo_dma_unmap_pinned);
280 for_each_tile(tile, xe, id)
281 xe_tile_migrate_wait(tile);
282 }
283
284 /**
285 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed
286 * @xe: The xe device.
287 *
288 * On pci_device removal we need to drop all dma mappings and move
289 * the data of exported bos out to system. This includes SVM bos and
290 * exported dma-buf bos. This is done by evicting all bos, but
291 * the evict placement in xe_evict_flags() is chosen such that all
292 * bos except those mentioned are purged, and thus their memory
293 * is released.
294 *
295 * For pinned bos, we're unmapping dma.
296 */
xe_bo_pci_dev_remove_all(struct xe_device * xe)297 void xe_bo_pci_dev_remove_all(struct xe_device *xe)
298 {
299 unsigned int mem_type;
300
301 /*
302 * Move pagemap bos and exported dma-buf to system, and
303 * purge everything else.
304 */
305 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) {
306 struct ttm_resource_manager *man =
307 ttm_manager_type(&xe->ttm, mem_type);
308
309 if (man) {
310 int ret = ttm_resource_manager_evict_all(&xe->ttm, man);
311
312 drm_WARN_ON(&xe->drm, ret);
313 }
314 }
315
316 xe_bo_pci_dev_remove_pinned(xe);
317 }
318
xe_bo_pinned_fini(void * arg)319 static void xe_bo_pinned_fini(void *arg)
320 {
321 struct xe_device *xe = arg;
322
323 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
324 &xe->pinned.late.kernel_bo_present,
325 xe_bo_dma_unmap_pinned);
326 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
327 &xe->pinned.early.kernel_bo_present,
328 xe_bo_dma_unmap_pinned);
329 }
330
331 /**
332 * xe_bo_pinned_init() - Initialize pinned bo tracking
333 * @xe: The xe device.
334 *
335 * Initializes the lists and locks required for pinned bo
336 * tracking and registers a callback to dma-unmap
337 * any remaining pinned bos on pci device removal.
338 *
339 * Return: %0 on success, negative error code on error.
340 */
xe_bo_pinned_init(struct xe_device * xe)341 int xe_bo_pinned_init(struct xe_device *xe)
342 {
343 spin_lock_init(&xe->pinned.lock);
344 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);
345 INIT_LIST_HEAD(&xe->pinned.early.evicted);
346 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);
347 INIT_LIST_HEAD(&xe->pinned.late.evicted);
348 INIT_LIST_HEAD(&xe->pinned.late.external);
349
350 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);
351 }
352