1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <linux/shrinker.h> 7 8 #include <drm/drm_managed.h> 9 #include <drm/ttm/ttm_backup.h> 10 #include <drm/ttm/ttm_bo.h> 11 #include <drm/ttm/ttm_tt.h> 12 13 #include "xe_bo.h" 14 #include "xe_pm.h" 15 #include "xe_shrinker.h" 16 17 /** 18 * struct xe_shrinker - per-device shrinker 19 * @xe: Back pointer to the device. 20 * @lock: Lock protecting accounting. 21 * @shrinkable_pages: Number of pages that are currently shrinkable. 22 * @purgeable_pages: Number of pages that are currently purgeable. 23 * @shrink: Pointer to the mm shrinker. 24 * @pm_worker: Worker to wake up the device if required. 25 */ 26 struct xe_shrinker { 27 struct xe_device *xe; 28 rwlock_t lock; 29 long shrinkable_pages; 30 long purgeable_pages; 31 struct shrinker *shrink; 32 struct work_struct pm_worker; 33 }; 34 35 static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink) 36 { 37 return shrink->private_data; 38 } 39 40 /** 41 * xe_shrinker_mod_pages() - Modify shrinker page accounting 42 * @shrinker: Pointer to the struct xe_shrinker. 43 * @shrinkable: Shrinkable pages delta. May be negative. 44 * @purgeable: Purgeable page delta. May be negative. 45 * 46 * Modifies the shrinkable and purgeable pages accounting. 47 */ 48 void 49 xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable) 50 { 51 write_lock(&shrinker->lock); 52 shrinker->shrinkable_pages += shrinkable; 53 shrinker->purgeable_pages += purgeable; 54 write_unlock(&shrinker->lock); 55 } 56 57 static bool __xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker) 58 { 59 struct xe_device *xe = shrinker->xe; 60 61 if (xe_pm_runtime_get_if_active(xe)) 62 return true; 63 64 if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { 65 xe_pm_runtime_get(xe); 66 return true; 67 } 68 69 queue_work(xe->unordered_wq, &shrinker->pm_worker); 70 71 return false; 72 } 73 74 static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) 75 { 76 if (runtime_pm) 77 xe_pm_runtime_put(shrinker->xe); 78 } 79 80 static int __xe_shrinker_walk(struct xe_shrinker *shrinker, 81 struct ttm_operation_ctx *ctx, 82 const struct xe_bo_shrink_flags flags, 83 unsigned long to_scan, unsigned long *scanned, 84 unsigned long *freed) 85 { 86 struct xe_device *xe = shrinker->xe; 87 unsigned int mem_type; 88 bool rpm = false; 89 int ret = 0; 90 s64 lret; 91 92 for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { 93 struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); 94 struct ttm_bo_lru_cursor curs; 95 struct ttm_buffer_object *ttm_bo; 96 struct ttm_lru_walk_arg arg = { 97 .ctx = ctx, 98 .trylock_only = true, 99 }; 100 101 if (!man || !man->use_tt) 102 continue; 103 104 if (mem_type != XE_PL_SYSTEM && !rpm && 105 xe_device_is_l2_flush_optimized(xe)) { 106 if (!__xe_shrinker_runtime_pm_get(shrinker)) 107 break; 108 rpm = true; 109 } 110 111 ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) { 112 if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) 113 continue; 114 115 lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); 116 /* Like the TTM LRU walk, skip bos we can't shrink now. */ 117 if (lret == -EBUSY) 118 continue; 119 if (lret < 0) { 120 ret = lret; 121 goto out; 122 } 123 124 *freed += lret; 125 if (*scanned >= to_scan) 126 goto out; 127 } 128 /* Trylocks should never error, just fail. */ 129 xe_assert(xe, !IS_ERR(ttm_bo)); 130 } 131 132 out: 133 xe_shrinker_runtime_pm_put(shrinker, rpm); 134 135 return ret; 136 } 137 138 /* 139 * Try shrinking idle objects without writeback first, then if not sufficient, 140 * try also non-idle objects and finally if that's not sufficient either, 141 * add writeback. This avoids stalls and explicit writebacks with light or 142 * moderate memory pressure. 143 */ 144 static int xe_shrinker_walk(struct xe_shrinker *shrinker, 145 struct ttm_operation_ctx *ctx, 146 const struct xe_bo_shrink_flags flags, 147 unsigned long to_scan, unsigned long *scanned, 148 unsigned long *freed) 149 { 150 bool no_wait_gpu = true; 151 struct xe_bo_shrink_flags save_flags = flags; 152 int ret; 153 154 swap(no_wait_gpu, ctx->no_wait_gpu); 155 save_flags.writeback = false; 156 ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, 157 freed); 158 swap(no_wait_gpu, ctx->no_wait_gpu); 159 if (ret || *scanned >= to_scan) 160 return ret; 161 162 if (!ctx->no_wait_gpu) { 163 ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, 164 freed); 165 if (ret || *scanned >= to_scan) 166 return ret; 167 } 168 169 if (flags.writeback) 170 ret = __xe_shrinker_walk(shrinker, ctx, flags, to_scan, scanned, 171 freed); 172 173 return ret; 174 } 175 176 static unsigned long 177 xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 178 { 179 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 180 unsigned long num_pages; 181 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 182 183 num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT; 184 read_lock(&shrinker->lock); 185 186 if (can_backup) 187 num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages); 188 else 189 num_pages = 0; 190 191 num_pages += shrinker->purgeable_pages; 192 read_unlock(&shrinker->lock); 193 194 return num_pages ? num_pages : SHRINK_EMPTY; 195 } 196 197 /* 198 * Check if we need runtime pm, and if so try to grab a reference if 199 * already active. If grabbing a reference fails, queue a worker that 200 * does it for us outside of reclaim, but don't wait for it to complete. 201 * If bo shrinking needs an rpm reference and we don't have it (yet), 202 * that bo will be skipped anyway. 203 */ 204 static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, 205 unsigned long nr_to_scan, bool can_backup) 206 { 207 struct xe_device *xe = shrinker->xe; 208 209 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) || 210 !ttm_backup_bytes_avail()) 211 return false; 212 213 if (!force) { 214 read_lock(&shrinker->lock); 215 force = (nr_to_scan > shrinker->purgeable_pages && can_backup); 216 read_unlock(&shrinker->lock); 217 if (!force) 218 return false; 219 } 220 221 return __xe_shrinker_runtime_pm_get(shrinker); 222 } 223 224 static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 225 { 226 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 227 struct ttm_operation_ctx ctx = { 228 .interruptible = false, 229 .no_wait_gpu = ttm_bo_shrink_avoid_wait(), 230 }; 231 unsigned long nr_to_scan, nr_scanned = 0, freed = 0; 232 struct xe_bo_shrink_flags shrink_flags = { 233 .purge = true, 234 /* Don't request writeback without __GFP_IO. */ 235 .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO), 236 }; 237 bool runtime_pm; 238 bool purgeable; 239 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 240 241 nr_to_scan = sc->nr_to_scan; 242 243 read_lock(&shrinker->lock); 244 purgeable = !!shrinker->purgeable_pages; 245 read_unlock(&shrinker->lock); 246 247 /* Might need runtime PM. Try to wake early if it looks like it. */ 248 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); 249 250 if (purgeable && nr_scanned < nr_to_scan) 251 xe_shrinker_walk(shrinker, &ctx, shrink_flags, 252 nr_to_scan, &nr_scanned, &freed); 253 254 sc->nr_scanned = nr_scanned; 255 if (nr_scanned >= nr_to_scan || !can_backup) 256 goto out; 257 258 /* If we didn't wake before, try to do it now if needed. */ 259 if (!runtime_pm) 260 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup); 261 262 shrink_flags.purge = false; 263 264 xe_shrinker_walk(shrinker, &ctx, shrink_flags, 265 nr_to_scan, &nr_scanned, &freed); 266 267 sc->nr_scanned = nr_scanned; 268 out: 269 xe_shrinker_runtime_pm_put(shrinker, runtime_pm); 270 return nr_scanned ? freed : SHRINK_STOP; 271 } 272 273 /* Wake up the device for shrinking. */ 274 static void xe_shrinker_pm(struct work_struct *work) 275 { 276 struct xe_shrinker *shrinker = 277 container_of(work, typeof(*shrinker), pm_worker); 278 279 xe_pm_runtime_get(shrinker->xe); 280 xe_pm_runtime_put(shrinker->xe); 281 } 282 283 static void xe_shrinker_fini(struct drm_device *drm, void *arg) 284 { 285 struct xe_shrinker *shrinker = arg; 286 287 xe_assert(shrinker->xe, !shrinker->shrinkable_pages); 288 xe_assert(shrinker->xe, !shrinker->purgeable_pages); 289 shrinker_free(shrinker->shrink); 290 flush_work(&shrinker->pm_worker); 291 kfree(shrinker); 292 } 293 294 /** 295 * xe_shrinker_create() - Create an xe per-device shrinker 296 * @xe: Pointer to the xe device. 297 * 298 * Return: %0 on success. Negative error code on failure. 299 */ 300 int xe_shrinker_create(struct xe_device *xe) 301 { 302 struct xe_shrinker *shrinker = kzalloc_obj(*shrinker); 303 304 if (!shrinker) 305 return -ENOMEM; 306 307 shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique); 308 if (!shrinker->shrink) { 309 kfree(shrinker); 310 return -ENOMEM; 311 } 312 313 INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm); 314 shrinker->xe = xe; 315 rwlock_init(&shrinker->lock); 316 shrinker->shrink->count_objects = xe_shrinker_count; 317 shrinker->shrink->scan_objects = xe_shrinker_scan; 318 shrinker->shrink->private_data = shrinker; 319 shrinker_register(shrinker->shrink); 320 xe->mem.shrinker = shrinker; 321 322 return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker); 323 } 324