xref: /linux/drivers/gpu/drm/xe/xe_shrinker.c (revision 9de430b6fbbdfeee1a5eec33edf53f52c13c869b)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2024 Intel Corporation
4  */
5 
6 #include <linux/shrinker.h>
7 
8 #include <drm/drm_managed.h>
9 #include <drm/ttm/ttm_backup.h>
10 #include <drm/ttm/ttm_bo.h>
11 #include <drm/ttm/ttm_tt.h>
12 
13 #include "xe_bo.h"
14 #include "xe_pm.h"
15 #include "xe_shrinker.h"
16 
17 /**
18  * struct xe_shrinker - per-device shrinker
19  * @xe: Back pointer to the device.
20  * @lock: Lock protecting accounting.
21  * @shrinkable_pages: Number of pages that are currently shrinkable.
22  * @purgeable_pages: Number of pages that are currently purgeable.
23  * @shrink: Pointer to the mm shrinker.
24  * @pm_worker: Worker to wake up the device if required.
25  */
26 struct xe_shrinker {
27 	struct xe_device *xe;
28 	rwlock_t lock;
29 	long shrinkable_pages;
30 	long purgeable_pages;
31 	struct shrinker *shrink;
32 	struct work_struct pm_worker;
33 };
34 
35 static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink)
36 {
37 	return shrink->private_data;
38 }
39 
40 /**
41  * xe_shrinker_mod_pages() - Modify shrinker page accounting
42  * @shrinker: Pointer to the struct xe_shrinker.
43  * @shrinkable: Shrinkable pages delta. May be negative.
44  * @purgeable: Purgeable page delta. May be negative.
45  *
46  * Modifies the shrinkable and purgeable pages accounting.
47  */
48 void
49 xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable)
50 {
51 	write_lock(&shrinker->lock);
52 	shrinker->shrinkable_pages += shrinkable;
53 	shrinker->purgeable_pages += purgeable;
54 	write_unlock(&shrinker->lock);
55 }
56 
57 static bool __xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker)
58 {
59 	struct xe_device *xe = shrinker->xe;
60 
61 	if (xe_pm_runtime_get_if_active(xe))
62 		return true;
63 
64 	if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) {
65 		xe_pm_runtime_get(xe);
66 		return true;
67 	}
68 
69 	queue_work(xe->unordered_wq, &shrinker->pm_worker);
70 
71 	return false;
72 }
73 
74 static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm)
75 {
76 	if (runtime_pm)
77 		xe_pm_runtime_put(shrinker->xe);
78 }
79 
80 static int __xe_shrinker_walk(struct xe_shrinker *shrinker,
81 			      struct ttm_operation_ctx *ctx,
82 			      const struct xe_bo_shrink_flags flags,
83 			      unsigned long to_scan, unsigned long *scanned,
84 			      unsigned long *freed)
85 {
86 	struct xe_device *xe = shrinker->xe;
87 	unsigned int mem_type;
88 	bool rpm = false;
89 	int ret = 0;
90 	s64 lret;
91 
92 	for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) {
93 		struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type);
94 		struct ttm_bo_lru_cursor curs;
95 		struct ttm_buffer_object *ttm_bo;
96 		struct ttm_lru_walk_arg arg = {
97 			.ctx = ctx,
98 			.trylock_only = true,
99 		};
100 
101 		if (!man || !man->use_tt)
102 			continue;
103 
104 		if (mem_type != XE_PL_SYSTEM && !rpm &&
105 		    xe_device_is_l2_flush_optimized(xe)) {
106 			if (!__xe_shrinker_runtime_pm_get(shrinker))
107 				break;
108 			rpm = true;
109 		}
110 
111 		ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) {
112 			if (!ttm_bo_shrink_suitable(ttm_bo, ctx))
113 				continue;
114 
115 			lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned);
116 			/* Like the TTM LRU walk, skip bos we can't shrink now. */
117 			if (lret == -EBUSY)
118 				continue;
119 			if (lret < 0) {
120 				ret = lret;
121 				goto out;
122 			}
123 
124 			*freed += lret;
125 			if (*scanned >= to_scan)
126 				goto out;
127 		}
128 		/* Trylocks should never error, just fail. */
129 		xe_assert(xe, !IS_ERR(ttm_bo));
130 	}
131 
132 out:
133 	xe_shrinker_runtime_pm_put(shrinker, rpm);
134 
135 	return ret;
136 }
137 
138 /*
139  * Try shrinking idle objects without writeback first, then if not sufficient,
140  * try also non-idle objects and finally if that's not sufficient either,
141  * add writeback. This avoids stalls and explicit writebacks with light or
142  * moderate memory pressure.
143  */
144 static int xe_shrinker_walk(struct xe_shrinker *shrinker,
145 			    struct ttm_operation_ctx *ctx,
146 			    const struct xe_bo_shrink_flags flags,
147 			    unsigned long to_scan, unsigned long *scanned,
148 			    unsigned long *freed)
149 {
150 	bool no_wait_gpu = true;
151 	struct xe_bo_shrink_flags save_flags = flags;
152 	int ret;
153 
154 	swap(no_wait_gpu, ctx->no_wait_gpu);
155 	save_flags.writeback = false;
156 	ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned,
157 				 freed);
158 	swap(no_wait_gpu, ctx->no_wait_gpu);
159 	if (ret || *scanned >= to_scan)
160 		return ret;
161 
162 	if (!ctx->no_wait_gpu) {
163 		ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned,
164 					 freed);
165 		if (ret || *scanned >= to_scan)
166 			return ret;
167 	}
168 
169 	if (flags.writeback)
170 		ret = __xe_shrinker_walk(shrinker, ctx, flags, to_scan, scanned,
171 					 freed);
172 
173 	return ret;
174 }
175 
176 static unsigned long
177 xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
178 {
179 	struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
180 	unsigned long num_pages;
181 	bool can_backup = !!(sc->gfp_mask & __GFP_FS);
182 
183 	num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT;
184 	read_lock(&shrinker->lock);
185 
186 	if (can_backup)
187 		num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages);
188 	else
189 		num_pages = 0;
190 
191 	num_pages += shrinker->purgeable_pages;
192 	read_unlock(&shrinker->lock);
193 
194 	return num_pages ? num_pages : SHRINK_EMPTY;
195 }
196 
197 /*
198  * Check if we need runtime pm, and if so try to grab a reference if
199  * already active. If grabbing a reference fails, queue a worker that
200  * does it for us outside of reclaim, but don't wait for it to complete.
201  * If bo shrinking needs an rpm reference and we don't have it (yet),
202  * that bo will be skipped anyway.
203  */
204 static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force,
205 				       unsigned long nr_to_scan, bool can_backup)
206 {
207 	struct xe_device *xe = shrinker->xe;
208 
209 	if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) ||
210 	    !ttm_backup_bytes_avail())
211 		return false;
212 
213 	if (!force) {
214 		read_lock(&shrinker->lock);
215 		force = (nr_to_scan > shrinker->purgeable_pages && can_backup);
216 		read_unlock(&shrinker->lock);
217 		if (!force)
218 			return false;
219 	}
220 
221 	return __xe_shrinker_runtime_pm_get(shrinker);
222 }
223 
224 static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
225 {
226 	struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
227 	struct ttm_operation_ctx ctx = {
228 		.interruptible = false,
229 		.no_wait_gpu = ttm_bo_shrink_avoid_wait(),
230 	};
231 	unsigned long nr_to_scan, nr_scanned = 0, freed = 0;
232 	struct xe_bo_shrink_flags shrink_flags = {
233 		.purge = true,
234 		/* Don't request writeback without __GFP_IO. */
235 		.writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO),
236 	};
237 	bool runtime_pm;
238 	bool purgeable;
239 	bool can_backup = !!(sc->gfp_mask & __GFP_FS);
240 
241 	nr_to_scan = sc->nr_to_scan;
242 
243 	read_lock(&shrinker->lock);
244 	purgeable = !!shrinker->purgeable_pages;
245 	read_unlock(&shrinker->lock);
246 
247 	/* Might need runtime PM. Try to wake early if it looks like it. */
248 	runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup);
249 
250 	if (purgeable && nr_scanned < nr_to_scan)
251 		xe_shrinker_walk(shrinker, &ctx, shrink_flags,
252 				 nr_to_scan, &nr_scanned, &freed);
253 
254 	sc->nr_scanned = nr_scanned;
255 	if (nr_scanned >= nr_to_scan || !can_backup)
256 		goto out;
257 
258 	/* If we didn't wake before, try to do it now if needed. */
259 	if (!runtime_pm)
260 		runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup);
261 
262 	shrink_flags.purge = false;
263 
264 	xe_shrinker_walk(shrinker, &ctx, shrink_flags,
265 			 nr_to_scan, &nr_scanned, &freed);
266 
267 	sc->nr_scanned = nr_scanned;
268 out:
269 	xe_shrinker_runtime_pm_put(shrinker, runtime_pm);
270 	return nr_scanned ? freed : SHRINK_STOP;
271 }
272 
273 /* Wake up the device for shrinking. */
274 static void xe_shrinker_pm(struct work_struct *work)
275 {
276 	struct xe_shrinker *shrinker =
277 		container_of(work, typeof(*shrinker), pm_worker);
278 
279 	xe_pm_runtime_get(shrinker->xe);
280 	xe_pm_runtime_put(shrinker->xe);
281 }
282 
283 static void xe_shrinker_fini(struct drm_device *drm, void *arg)
284 {
285 	struct xe_shrinker *shrinker = arg;
286 
287 	xe_assert(shrinker->xe, !shrinker->shrinkable_pages);
288 	xe_assert(shrinker->xe, !shrinker->purgeable_pages);
289 	shrinker_free(shrinker->shrink);
290 	flush_work(&shrinker->pm_worker);
291 	kfree(shrinker);
292 }
293 
294 /**
295  * xe_shrinker_create() - Create an xe per-device shrinker
296  * @xe: Pointer to the xe device.
297  *
298  * Return: %0 on success. Negative error code on failure.
299  */
300 int xe_shrinker_create(struct xe_device *xe)
301 {
302 	struct xe_shrinker *shrinker = kzalloc_obj(*shrinker);
303 
304 	if (!shrinker)
305 		return -ENOMEM;
306 
307 	shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique);
308 	if (!shrinker->shrink) {
309 		kfree(shrinker);
310 		return -ENOMEM;
311 	}
312 
313 	INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm);
314 	shrinker->xe = xe;
315 	rwlock_init(&shrinker->lock);
316 	shrinker->shrink->count_objects = xe_shrinker_count;
317 	shrinker->shrink->scan_objects = xe_shrinker_scan;
318 	shrinker->shrink->private_data = shrinker;
319 	shrinker_register(shrinker->shrink);
320 	xe->mem.shrinker = shrinker;
321 
322 	return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker);
323 }
324