1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <linux/shrinker.h> 7 8 #include <drm/ttm/ttm_backup.h> 9 #include <drm/ttm/ttm_bo.h> 10 #include <drm/ttm/ttm_tt.h> 11 12 #include "xe_bo.h" 13 #include "xe_pm.h" 14 #include "xe_shrinker.h" 15 16 /** 17 * struct xe_shrinker - per-device shrinker 18 * @xe: Back pointer to the device. 19 * @lock: Lock protecting accounting. 20 * @shrinkable_pages: Number of pages that are currently shrinkable. 21 * @purgeable_pages: Number of pages that are currently purgeable. 22 * @shrink: Pointer to the mm shrinker. 23 * @pm_worker: Worker to wake up the device if required. 24 */ 25 struct xe_shrinker { 26 struct xe_device *xe; 27 rwlock_t lock; 28 long shrinkable_pages; 29 long purgeable_pages; 30 struct shrinker *shrink; 31 struct work_struct pm_worker; 32 }; 33 34 static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink) 35 { 36 return shrink->private_data; 37 } 38 39 /** 40 * xe_shrinker_mod_pages() - Modify shrinker page accounting 41 * @shrinker: Pointer to the struct xe_shrinker. 42 * @shrinkable: Shrinkable pages delta. May be negative. 43 * @purgeable: Purgeable page delta. May be negative. 44 * 45 * Modifies the shrinkable and purgeable pages accounting. 46 */ 47 void 48 xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable) 49 { 50 write_lock(&shrinker->lock); 51 shrinker->shrinkable_pages += shrinkable; 52 shrinker->purgeable_pages += purgeable; 53 write_unlock(&shrinker->lock); 54 } 55 56 static s64 xe_shrinker_walk(struct xe_device *xe, 57 struct ttm_operation_ctx *ctx, 58 const struct xe_bo_shrink_flags flags, 59 unsigned long to_scan, unsigned long *scanned) 60 { 61 unsigned int mem_type; 62 s64 freed = 0, lret; 63 64 for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { 65 struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); 66 struct ttm_bo_lru_cursor curs; 67 struct ttm_buffer_object *ttm_bo; 68 struct ttm_lru_walk_arg arg = { 69 .ctx = ctx, 70 .trylock_only = true, 71 }; 72 73 if (!man || !man->use_tt) 74 continue; 75 76 ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) { 77 if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) 78 continue; 79 80 lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); 81 if (lret < 0) 82 return lret; 83 84 freed += lret; 85 if (*scanned >= to_scan) 86 break; 87 } 88 /* Trylocks should never error, just fail. */ 89 xe_assert(xe, !IS_ERR(ttm_bo)); 90 } 91 92 return freed; 93 } 94 95 static unsigned long 96 xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 97 { 98 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 99 unsigned long num_pages; 100 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 101 102 num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT; 103 read_lock(&shrinker->lock); 104 105 if (can_backup) 106 num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages); 107 else 108 num_pages = 0; 109 110 num_pages += shrinker->purgeable_pages; 111 read_unlock(&shrinker->lock); 112 113 return num_pages ? num_pages : SHRINK_EMPTY; 114 } 115 116 /* 117 * Check if we need runtime pm, and if so try to grab a reference if 118 * already active. If grabbing a reference fails, queue a worker that 119 * does it for us outside of reclaim, but don't wait for it to complete. 120 * If bo shrinking needs an rpm reference and we don't have it (yet), 121 * that bo will be skipped anyway. 122 */ 123 static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, 124 unsigned long nr_to_scan, bool can_backup) 125 { 126 struct xe_device *xe = shrinker->xe; 127 128 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) || 129 !ttm_backup_bytes_avail()) 130 return false; 131 132 if (!force) { 133 read_lock(&shrinker->lock); 134 force = (nr_to_scan > shrinker->purgeable_pages && can_backup); 135 read_unlock(&shrinker->lock); 136 if (!force) 137 return false; 138 } 139 140 if (!xe_pm_runtime_get_if_active(xe)) { 141 if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { 142 xe_pm_runtime_get(xe); 143 return true; 144 } 145 queue_work(xe->unordered_wq, &shrinker->pm_worker); 146 return false; 147 } 148 149 return true; 150 } 151 152 static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) 153 { 154 if (runtime_pm) 155 xe_pm_runtime_put(shrinker->xe); 156 } 157 158 static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 159 { 160 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 161 struct ttm_operation_ctx ctx = { 162 .interruptible = false, 163 .no_wait_gpu = ttm_bo_shrink_avoid_wait(), 164 }; 165 unsigned long nr_to_scan, nr_scanned = 0, freed = 0; 166 struct xe_bo_shrink_flags shrink_flags = { 167 .purge = true, 168 /* Don't request writeback without __GFP_IO. */ 169 .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO), 170 }; 171 bool runtime_pm; 172 bool purgeable; 173 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 174 s64 lret; 175 176 nr_to_scan = sc->nr_to_scan; 177 178 read_lock(&shrinker->lock); 179 purgeable = !!shrinker->purgeable_pages; 180 read_unlock(&shrinker->lock); 181 182 /* Might need runtime PM. Try to wake early if it looks like it. */ 183 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); 184 185 if (purgeable && nr_scanned < nr_to_scan) { 186 lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, 187 nr_to_scan, &nr_scanned); 188 if (lret >= 0) 189 freed += lret; 190 } 191 192 sc->nr_scanned = nr_scanned; 193 if (nr_scanned >= nr_to_scan || !can_backup) 194 goto out; 195 196 /* If we didn't wake before, try to do it now if needed. */ 197 if (!runtime_pm) 198 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup); 199 200 shrink_flags.purge = false; 201 lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, 202 nr_to_scan, &nr_scanned); 203 if (lret >= 0) 204 freed += lret; 205 206 sc->nr_scanned = nr_scanned; 207 out: 208 xe_shrinker_runtime_pm_put(shrinker, runtime_pm); 209 return nr_scanned ? freed : SHRINK_STOP; 210 } 211 212 /* Wake up the device for shrinking. */ 213 static void xe_shrinker_pm(struct work_struct *work) 214 { 215 struct xe_shrinker *shrinker = 216 container_of(work, typeof(*shrinker), pm_worker); 217 218 xe_pm_runtime_get(shrinker->xe); 219 xe_pm_runtime_put(shrinker->xe); 220 } 221 222 /** 223 * xe_shrinker_create() - Create an xe per-device shrinker 224 * @xe: Pointer to the xe device. 225 * 226 * Returns: A pointer to the created shrinker on success, 227 * Negative error code on failure. 228 */ 229 struct xe_shrinker *xe_shrinker_create(struct xe_device *xe) 230 { 231 struct xe_shrinker *shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL); 232 233 if (!shrinker) 234 return ERR_PTR(-ENOMEM); 235 236 shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique); 237 if (!shrinker->shrink) { 238 kfree(shrinker); 239 return ERR_PTR(-ENOMEM); 240 } 241 242 INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm); 243 shrinker->xe = xe; 244 rwlock_init(&shrinker->lock); 245 shrinker->shrink->count_objects = xe_shrinker_count; 246 shrinker->shrink->scan_objects = xe_shrinker_scan; 247 shrinker->shrink->private_data = shrinker; 248 shrinker_register(shrinker->shrink); 249 250 return shrinker; 251 } 252 253 /** 254 * xe_shrinker_destroy() - Destroy an xe per-device shrinker 255 * @shrinker: Pointer to the shrinker to destroy. 256 */ 257 void xe_shrinker_destroy(struct xe_shrinker *shrinker) 258 { 259 xe_assert(shrinker->xe, !shrinker->shrinkable_pages); 260 xe_assert(shrinker->xe, !shrinker->purgeable_pages); 261 shrinker_free(shrinker->shrink); 262 flush_work(&shrinker->pm_worker); 263 kfree(shrinker); 264 } 265