1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_bo_evict.h"
7
8 #include "xe_bo.h"
9 #include "xe_device.h"
10 #include "xe_ggtt.h"
11 #include "xe_tile.h"
12
13 typedef int (*xe_pinned_fn)(struct xe_bo *bo);
14
xe_bo_apply_to_pinned(struct xe_device * xe,struct list_head * pinned_list,struct list_head * new_list,const xe_pinned_fn pinned_fn)15 static int xe_bo_apply_to_pinned(struct xe_device *xe,
16 struct list_head *pinned_list,
17 struct list_head *new_list,
18 const xe_pinned_fn pinned_fn)
19 {
20 LIST_HEAD(still_in_list);
21 struct xe_bo *bo;
22 int ret = 0;
23
24 spin_lock(&xe->pinned.lock);
25 while (!ret) {
26 bo = list_first_entry_or_null(pinned_list, typeof(*bo),
27 pinned_link);
28 if (!bo)
29 break;
30 xe_bo_get(bo);
31 list_move_tail(&bo->pinned_link, &still_in_list);
32 spin_unlock(&xe->pinned.lock);
33
34 ret = pinned_fn(bo);
35 if (ret && pinned_list != new_list) {
36 spin_lock(&xe->pinned.lock);
37 /*
38 * We might no longer be pinned, since PM notifier can
39 * call this. If the pinned link is now empty, keep it
40 * that way.
41 */
42 if (!list_empty(&bo->pinned_link))
43 list_move(&bo->pinned_link, pinned_list);
44 spin_unlock(&xe->pinned.lock);
45 }
46 xe_bo_put(bo);
47 spin_lock(&xe->pinned.lock);
48 }
49 list_splice_tail(&still_in_list, new_list);
50 spin_unlock(&xe->pinned.lock);
51
52 return ret;
53 }
54
55 /**
56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all
57 * pinned VRAM objects which need to be saved.
58 * @xe: xe device
59 *
60 * Should be called from PM notifier when preparing for s3/s4.
61 *
62 * Return: 0 on success, negative error code on error.
63 */
xe_bo_notifier_prepare_all_pinned(struct xe_device * xe)64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe)
65 {
66 int ret;
67
68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
69 &xe->pinned.early.kernel_bo_present,
70 xe_bo_notifier_prepare_pinned);
71 if (!ret)
72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
73 &xe->pinned.late.kernel_bo_present,
74 xe_bo_notifier_prepare_pinned);
75
76 return ret;
77 }
78
79 /**
80 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all
81 * pinned VRAM objects which have been restored.
82 * @xe: xe device
83 *
84 * Should be called from PM notifier after exiting s3/s4 (either on success or
85 * failure).
86 */
xe_bo_notifier_unprepare_all_pinned(struct xe_device * xe)87 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe)
88 {
89 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
90 &xe->pinned.early.kernel_bo_present,
91 xe_bo_notifier_unprepare_pinned);
92
93 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
94 &xe->pinned.late.kernel_bo_present,
95 xe_bo_notifier_unprepare_pinned);
96 }
97
98 /**
99 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM
100 * @xe: xe device
101 *
102 * Evict non-pinned user BOs (via GPU).
103 *
104 * Evict == move VRAM BOs to temporary (typically system) memory.
105 */
xe_bo_evict_all_user(struct xe_device * xe)106 int xe_bo_evict_all_user(struct xe_device *xe)
107 {
108 struct ttm_device *bdev = &xe->ttm;
109 u32 mem_type;
110 int ret;
111
112 /* User memory */
113 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) {
114 struct ttm_resource_manager *man =
115 ttm_manager_type(bdev, mem_type);
116
117 /*
118 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS
119 * state since this state lives inside graphics stolen memory which doesn't survive
120 * hibernation.
121 *
122 * This can be further improved by only evicting objects that we know have actually
123 * used a compression enabled PAT index.
124 */
125 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)))
126 continue;
127
128 if (man) {
129 ret = ttm_resource_manager_evict_all(bdev, man);
130 if (ret)
131 return ret;
132 }
133 }
134
135 return 0;
136 }
137
138 /**
139 * xe_bo_evict_all - evict all BOs from VRAM
140 * @xe: xe device
141 *
142 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next
143 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.
144 * All eviction magic done via TTM calls.
145 *
146 * Evict == move VRAM BOs to temporary (typically system) memory.
147 *
148 * This function should be called before the device goes into a suspend state
149 * where the VRAM loses power.
150 */
xe_bo_evict_all(struct xe_device * xe)151 int xe_bo_evict_all(struct xe_device *xe)
152 {
153 struct xe_tile *tile;
154 u8 id;
155 int ret;
156
157 ret = xe_bo_evict_all_user(xe);
158 if (ret)
159 return ret;
160
161 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
162 &xe->pinned.late.external, xe_bo_evict_pinned);
163
164 if (!ret)
165 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
166 &xe->pinned.late.evicted, xe_bo_evict_pinned);
167
168 /*
169 * Wait for all user BO to be evicted as those evictions depend on the
170 * memory moved below.
171 */
172 for_each_tile(tile, xe, id)
173 xe_tile_migrate_wait(tile);
174
175 if (ret)
176 return ret;
177
178 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
179 &xe->pinned.early.evicted,
180 xe_bo_evict_pinned);
181 }
182
xe_bo_restore_and_map_ggtt(struct xe_bo * bo)183 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
184 {
185 int ret;
186
187 ret = xe_bo_restore_pinned(bo);
188 if (ret)
189 return ret;
190
191 if (bo->flags & XE_BO_FLAG_GGTT) {
192 struct xe_tile *tile;
193 u8 id;
194
195 for_each_tile(tile, xe_bo_device(bo), id) {
196 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile)))
197 continue;
198
199 xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo);
200 }
201 }
202
203 return 0;
204 }
205
206 /**
207 * xe_bo_restore_early - restore early phase kernel BOs to VRAM
208 *
209 * @xe: xe device
210 *
211 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All
212 * moves done via TTM calls.
213 *
214 * This function should be called early, before trying to init the GT, on device
215 * resume.
216 */
xe_bo_restore_early(struct xe_device * xe)217 int xe_bo_restore_early(struct xe_device *xe)
218 {
219 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,
220 &xe->pinned.early.kernel_bo_present,
221 xe_bo_restore_and_map_ggtt);
222 }
223
224 /**
225 * xe_bo_restore_late - restore pinned late phase BOs
226 *
227 * @xe: xe device
228 *
229 * Move pinned user and kernel BOs which can use blitter from temporary
230 * (typically system) memory to VRAM. All moves done via TTM calls.
231 *
232 * This function should be called late, after GT init, on device resume.
233 */
xe_bo_restore_late(struct xe_device * xe)234 int xe_bo_restore_late(struct xe_device *xe)
235 {
236 struct xe_tile *tile;
237 int ret, id;
238
239 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,
240 &xe->pinned.late.kernel_bo_present,
241 xe_bo_restore_and_map_ggtt);
242
243 for_each_tile(tile, xe, id)
244 xe_tile_migrate_wait(tile);
245
246 if (ret)
247 return ret;
248
249 if (!IS_DGFX(xe))
250 return 0;
251
252 /* Pinned user memory in VRAM should be validated on resume */
253 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
254 &xe->pinned.late.external,
255 xe_bo_restore_pinned);
256
257 /* Wait for restore to complete */
258 for_each_tile(tile, xe, id)
259 xe_tile_migrate_wait(tile);
260
261 return ret;
262 }
263
xe_bo_pci_dev_remove_pinned(struct xe_device * xe)264 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe)
265 {
266 struct xe_tile *tile;
267 unsigned int id;
268
269 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
270 &xe->pinned.late.external,
271 xe_bo_dma_unmap_pinned);
272 for_each_tile(tile, xe, id)
273 xe_tile_migrate_wait(tile);
274 }
275
276 /**
277 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed
278 * @xe: The xe device.
279 *
280 * On pci_device removal we need to drop all dma mappings and move
281 * the data of exported bos out to system. This includes SVM bos and
282 * exported dma-buf bos. This is done by evicting all bos, but
283 * the evict placement in xe_evict_flags() is chosen such that all
284 * bos except those mentioned are purged, and thus their memory
285 * is released.
286 *
287 * For pinned bos, we're unmapping dma.
288 */
xe_bo_pci_dev_remove_all(struct xe_device * xe)289 void xe_bo_pci_dev_remove_all(struct xe_device *xe)
290 {
291 unsigned int mem_type;
292
293 /*
294 * Move pagemap bos and exported dma-buf to system, and
295 * purge everything else.
296 */
297 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) {
298 struct ttm_resource_manager *man =
299 ttm_manager_type(&xe->ttm, mem_type);
300
301 if (man) {
302 int ret = ttm_resource_manager_evict_all(&xe->ttm, man);
303
304 drm_WARN_ON(&xe->drm, ret);
305 }
306 }
307
308 xe_bo_pci_dev_remove_pinned(xe);
309 }
310
xe_bo_pinned_fini(void * arg)311 static void xe_bo_pinned_fini(void *arg)
312 {
313 struct xe_device *xe = arg;
314
315 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
316 &xe->pinned.late.kernel_bo_present,
317 xe_bo_dma_unmap_pinned);
318 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
319 &xe->pinned.early.kernel_bo_present,
320 xe_bo_dma_unmap_pinned);
321 }
322
323 /**
324 * xe_bo_pinned_init() - Initialize pinned bo tracking
325 * @xe: The xe device.
326 *
327 * Initializes the lists and locks required for pinned bo
328 * tracking and registers a callback to dma-unmap
329 * any remaining pinned bos on pci device removal.
330 *
331 * Return: %0 on success, negative error code on error.
332 */
xe_bo_pinned_init(struct xe_device * xe)333 int xe_bo_pinned_init(struct xe_device *xe)
334 {
335 spin_lock_init(&xe->pinned.lock);
336 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);
337 INIT_LIST_HEAD(&xe->pinned.early.evicted);
338 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);
339 INIT_LIST_HEAD(&xe->pinned.late.evicted);
340 INIT_LIST_HEAD(&xe->pinned.late.external);
341
342 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);
343 }
344