1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_bo_evict.h"
7
8 #include "xe_bo.h"
9 #include "xe_device.h"
10 #include "xe_ggtt.h"
11 #include "xe_tile.h"
12
13 typedef int (*xe_pinned_fn)(struct xe_bo *bo);
14
xe_bo_apply_to_pinned(struct xe_device * xe,struct list_head * pinned_list,struct list_head * new_list,const xe_pinned_fn pinned_fn)15 static int xe_bo_apply_to_pinned(struct xe_device *xe,
16 struct list_head *pinned_list,
17 struct list_head *new_list,
18 const xe_pinned_fn pinned_fn)
19 {
20 LIST_HEAD(still_in_list);
21 struct xe_bo *bo;
22 int ret = 0;
23
24 spin_lock(&xe->pinned.lock);
25 while (!ret) {
26 bo = list_first_entry_or_null(pinned_list, typeof(*bo),
27 pinned_link);
28 if (!bo)
29 break;
30 xe_bo_get(bo);
31 list_move_tail(&bo->pinned_link, &still_in_list);
32 spin_unlock(&xe->pinned.lock);
33
34 ret = pinned_fn(bo);
35 if (ret && pinned_list != new_list) {
36 spin_lock(&xe->pinned.lock);
37 /*
38 * We might no longer be pinned, since PM notifier can
39 * call this. If the pinned link is now empty, keep it
40 * that way.
41 */
42 if (!list_empty(&bo->pinned_link))
43 list_move(&bo->pinned_link, pinned_list);
44 spin_unlock(&xe->pinned.lock);
45 }
46 xe_bo_put(bo);
47 spin_lock(&xe->pinned.lock);
48 }
49 list_splice_tail(&still_in_list, new_list);
50 spin_unlock(&xe->pinned.lock);
51
52 return ret;
53 }
54
55 /**
56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all
57 * pinned VRAM objects which need to be saved.
58 * @xe: xe device
59 *
60 * Should be called from PM notifier when preparing for s3/s4.
61 *
62 * Return: 0 on success, negative error code on error.
63 */
xe_bo_notifier_prepare_all_pinned(struct xe_device * xe)64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe)
65 {
66 int ret;
67
68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
69 &xe->pinned.early.kernel_bo_present,
70 xe_bo_notifier_prepare_pinned);
71 if (!ret)
72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
73 &xe->pinned.late.kernel_bo_present,
74 xe_bo_notifier_prepare_pinned);
75
76 if (!ret)
77 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
78 &xe->pinned.late.external,
79 xe_bo_notifier_prepare_pinned);
80
81 return ret;
82 }
83
84 /**
85 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all
86 * pinned VRAM objects which have been restored.
87 * @xe: xe device
88 *
89 * Should be called from PM notifier after exiting s3/s4 (either on success or
90 * failure).
91 */
xe_bo_notifier_unprepare_all_pinned(struct xe_device * xe)92 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe)
93 {
94 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
95 &xe->pinned.early.kernel_bo_present,
96 xe_bo_notifier_unprepare_pinned);
97
98 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
99 &xe->pinned.late.kernel_bo_present,
100 xe_bo_notifier_unprepare_pinned);
101
102 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
103 &xe->pinned.late.external,
104 xe_bo_notifier_unprepare_pinned);
105 }
106
107 /**
108 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM
109 * @xe: xe device
110 *
111 * Evict non-pinned user BOs (via GPU).
112 *
113 * Evict == move VRAM BOs to temporary (typically system) memory.
114 */
xe_bo_evict_all_user(struct xe_device * xe)115 int xe_bo_evict_all_user(struct xe_device *xe)
116 {
117 struct ttm_device *bdev = &xe->ttm;
118 u32 mem_type;
119 int ret;
120
121 /* User memory */
122 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) {
123 struct ttm_resource_manager *man =
124 ttm_manager_type(bdev, mem_type);
125
126 /*
127 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS
128 * state since this state lives inside graphics stolen memory which doesn't survive
129 * hibernation.
130 *
131 * This can be further improved by only evicting objects that we know have actually
132 * used a compression enabled PAT index.
133 */
134 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)))
135 continue;
136
137 if (man) {
138 ret = ttm_resource_manager_evict_all(bdev, man);
139 if (ret)
140 return ret;
141 }
142 }
143
144 return 0;
145 }
146
147 /**
148 * xe_bo_evict_all - evict all BOs from VRAM
149 * @xe: xe device
150 *
151 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next
152 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.
153 * All eviction magic done via TTM calls.
154 *
155 * Evict == move VRAM BOs to temporary (typically system) memory.
156 *
157 * This function should be called before the device goes into a suspend state
158 * where the VRAM loses power.
159 */
xe_bo_evict_all(struct xe_device * xe)160 int xe_bo_evict_all(struct xe_device *xe)
161 {
162 struct xe_tile *tile;
163 u8 id;
164 int ret;
165
166 ret = xe_bo_evict_all_user(xe);
167 if (ret)
168 return ret;
169
170 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
171 &xe->pinned.late.external, xe_bo_evict_pinned);
172
173 if (!ret)
174 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
175 &xe->pinned.late.evicted, xe_bo_evict_pinned);
176
177 /*
178 * Wait for all user BO to be evicted as those evictions depend on the
179 * memory moved below.
180 */
181 for_each_tile(tile, xe, id)
182 xe_tile_migrate_wait(tile);
183
184 if (ret)
185 return ret;
186
187 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
188 &xe->pinned.early.evicted,
189 xe_bo_evict_pinned);
190 }
191
xe_bo_restore_and_map_ggtt(struct xe_bo * bo)192 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
193 {
194 int ret;
195
196 ret = xe_bo_restore_pinned(bo);
197 if (ret)
198 return ret;
199
200 if (bo->flags & XE_BO_FLAG_GGTT) {
201 struct xe_tile *tile;
202 u8 id;
203
204 for_each_tile(tile, xe_bo_device(bo), id) {
205 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile)))
206 continue;
207
208 xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo);
209 }
210 }
211
212 return 0;
213 }
214
215 /**
216 * xe_bo_restore_early - restore early phase kernel BOs to VRAM
217 *
218 * @xe: xe device
219 *
220 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All
221 * moves done via TTM calls.
222 *
223 * This function should be called early, before trying to init the GT, on device
224 * resume.
225 */
xe_bo_restore_early(struct xe_device * xe)226 int xe_bo_restore_early(struct xe_device *xe)
227 {
228 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,
229 &xe->pinned.early.kernel_bo_present,
230 xe_bo_restore_and_map_ggtt);
231 }
232
233 /**
234 * xe_bo_restore_late - restore pinned late phase BOs
235 *
236 * @xe: xe device
237 *
238 * Move pinned user and kernel BOs which can use blitter from temporary
239 * (typically system) memory to VRAM. All moves done via TTM calls.
240 *
241 * This function should be called late, after GT init, on device resume.
242 */
xe_bo_restore_late(struct xe_device * xe)243 int xe_bo_restore_late(struct xe_device *xe)
244 {
245 struct xe_tile *tile;
246 int ret, id;
247
248 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,
249 &xe->pinned.late.kernel_bo_present,
250 xe_bo_restore_and_map_ggtt);
251
252 for_each_tile(tile, xe, id)
253 xe_tile_migrate_wait(tile);
254
255 if (ret)
256 return ret;
257
258 if (!IS_DGFX(xe))
259 return 0;
260
261 /* Pinned user memory in VRAM should be validated on resume */
262 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
263 &xe->pinned.late.external,
264 xe_bo_restore_pinned);
265
266 /* Wait for restore to complete */
267 for_each_tile(tile, xe, id)
268 xe_tile_migrate_wait(tile);
269
270 return ret;
271 }
272
xe_bo_pci_dev_remove_pinned(struct xe_device * xe)273 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe)
274 {
275 struct xe_tile *tile;
276 unsigned int id;
277
278 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
279 &xe->pinned.late.external,
280 xe_bo_dma_unmap_pinned);
281 for_each_tile(tile, xe, id)
282 xe_tile_migrate_wait(tile);
283 }
284
285 /**
286 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed
287 * @xe: The xe device.
288 *
289 * On pci_device removal we need to drop all dma mappings and move
290 * the data of exported bos out to system. This includes SVM bos and
291 * exported dma-buf bos. This is done by evicting all bos, but
292 * the evict placement in xe_evict_flags() is chosen such that all
293 * bos except those mentioned are purged, and thus their memory
294 * is released.
295 *
296 * For pinned bos, we're unmapping dma.
297 */
xe_bo_pci_dev_remove_all(struct xe_device * xe)298 void xe_bo_pci_dev_remove_all(struct xe_device *xe)
299 {
300 unsigned int mem_type;
301
302 /*
303 * Move pagemap bos and exported dma-buf to system, and
304 * purge everything else.
305 */
306 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) {
307 struct ttm_resource_manager *man =
308 ttm_manager_type(&xe->ttm, mem_type);
309
310 if (man) {
311 int ret = ttm_resource_manager_evict_all(&xe->ttm, man);
312
313 drm_WARN_ON(&xe->drm, ret);
314 }
315 }
316
317 xe_bo_pci_dev_remove_pinned(xe);
318 }
319
xe_bo_pinned_fini(void * arg)320 static void xe_bo_pinned_fini(void *arg)
321 {
322 struct xe_device *xe = arg;
323
324 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
325 &xe->pinned.late.kernel_bo_present,
326 xe_bo_dma_unmap_pinned);
327 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
328 &xe->pinned.early.kernel_bo_present,
329 xe_bo_dma_unmap_pinned);
330 }
331
332 /**
333 * xe_bo_pinned_init() - Initialize pinned bo tracking
334 * @xe: The xe device.
335 *
336 * Initializes the lists and locks required for pinned bo
337 * tracking and registers a callback to dma-unmap
338 * any remaining pinned bos on pci device removal.
339 *
340 * Return: %0 on success, negative error code on error.
341 */
xe_bo_pinned_init(struct xe_device * xe)342 int xe_bo_pinned_init(struct xe_device *xe)
343 {
344 spin_lock_init(&xe->pinned.lock);
345 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);
346 INIT_LIST_HEAD(&xe->pinned.early.evicted);
347 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);
348 INIT_LIST_HEAD(&xe->pinned.late.evicted);
349 INIT_LIST_HEAD(&xe->pinned.late.external);
350
351 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);
352 }
353