1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <linux/shrinker.h> 7 8 #include <drm/ttm/ttm_backup.h> 9 #include <drm/ttm/ttm_bo.h> 10 #include <drm/ttm/ttm_tt.h> 11 12 #include "xe_bo.h" 13 #include "xe_pm.h" 14 #include "xe_shrinker.h" 15 16 /** 17 * struct xe_shrinker - per-device shrinker 18 * @xe: Back pointer to the device. 19 * @lock: Lock protecting accounting. 20 * @shrinkable_pages: Number of pages that are currently shrinkable. 21 * @purgeable_pages: Number of pages that are currently purgeable. 22 * @shrink: Pointer to the mm shrinker. 23 * @pm_worker: Worker to wake up the device if required. 24 */ 25 struct xe_shrinker { 26 struct xe_device *xe; 27 rwlock_t lock; 28 long shrinkable_pages; 29 long purgeable_pages; 30 struct shrinker *shrink; 31 struct work_struct pm_worker; 32 }; 33 34 static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink) 35 { 36 return shrink->private_data; 37 } 38 39 /** 40 * xe_shrinker_mod_pages() - Modify shrinker page accounting 41 * @shrinker: Pointer to the struct xe_shrinker. 42 * @shrinkable: Shrinkable pages delta. May be negative. 43 * @purgeable: Purgeable page delta. May be negative. 44 * 45 * Modifies the shrinkable and purgeable pages accounting. 46 */ 47 void 48 xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable) 49 { 50 write_lock(&shrinker->lock); 51 shrinker->shrinkable_pages += shrinkable; 52 shrinker->purgeable_pages += purgeable; 53 write_unlock(&shrinker->lock); 54 } 55 56 static s64 xe_shrinker_walk(struct xe_device *xe, 57 struct ttm_operation_ctx *ctx, 58 const struct xe_bo_shrink_flags flags, 59 unsigned long to_scan, unsigned long *scanned) 60 { 61 unsigned int mem_type; 62 s64 freed = 0, lret; 63 64 for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { 65 struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); 66 struct ttm_bo_lru_cursor curs; 67 struct ttm_buffer_object *ttm_bo; 68 69 if (!man || !man->use_tt) 70 continue; 71 72 ttm_bo_lru_for_each_reserved_guarded(&curs, man, ctx, ttm_bo) { 73 if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) 74 continue; 75 76 lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); 77 if (lret < 0) 78 return lret; 79 80 freed += lret; 81 if (*scanned >= to_scan) 82 break; 83 } 84 } 85 86 return freed; 87 } 88 89 static unsigned long 90 xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 91 { 92 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 93 unsigned long num_pages; 94 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 95 96 num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT; 97 read_lock(&shrinker->lock); 98 99 if (can_backup) 100 num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages); 101 else 102 num_pages = 0; 103 104 num_pages += shrinker->purgeable_pages; 105 read_unlock(&shrinker->lock); 106 107 return num_pages ? num_pages : SHRINK_EMPTY; 108 } 109 110 /* 111 * Check if we need runtime pm, and if so try to grab a reference if 112 * already active. If grabbing a reference fails, queue a worker that 113 * does it for us outside of reclaim, but don't wait for it to complete. 114 * If bo shrinking needs an rpm reference and we don't have it (yet), 115 * that bo will be skipped anyway. 116 */ 117 static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, 118 unsigned long nr_to_scan, bool can_backup) 119 { 120 struct xe_device *xe = shrinker->xe; 121 122 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) || 123 !ttm_backup_bytes_avail()) 124 return false; 125 126 if (!force) { 127 read_lock(&shrinker->lock); 128 force = (nr_to_scan > shrinker->purgeable_pages && can_backup); 129 read_unlock(&shrinker->lock); 130 if (!force) 131 return false; 132 } 133 134 if (!xe_pm_runtime_get_if_active(xe)) { 135 if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { 136 xe_pm_runtime_get(xe); 137 return true; 138 } 139 queue_work(xe->unordered_wq, &shrinker->pm_worker); 140 return false; 141 } 142 143 return true; 144 } 145 146 static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) 147 { 148 if (runtime_pm) 149 xe_pm_runtime_put(shrinker->xe); 150 } 151 152 static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 153 { 154 struct xe_shrinker *shrinker = to_xe_shrinker(shrink); 155 struct ttm_operation_ctx ctx = { 156 .interruptible = false, 157 .no_wait_gpu = ttm_bo_shrink_avoid_wait(), 158 }; 159 unsigned long nr_to_scan, nr_scanned = 0, freed = 0; 160 struct xe_bo_shrink_flags shrink_flags = { 161 .purge = true, 162 /* Don't request writeback without __GFP_IO. */ 163 .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO), 164 }; 165 bool runtime_pm; 166 bool purgeable; 167 bool can_backup = !!(sc->gfp_mask & __GFP_FS); 168 s64 lret; 169 170 nr_to_scan = sc->nr_to_scan; 171 172 read_lock(&shrinker->lock); 173 purgeable = !!shrinker->purgeable_pages; 174 read_unlock(&shrinker->lock); 175 176 /* Might need runtime PM. Try to wake early if it looks like it. */ 177 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); 178 179 if (purgeable && nr_scanned < nr_to_scan) { 180 lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, 181 nr_to_scan, &nr_scanned); 182 if (lret >= 0) 183 freed += lret; 184 } 185 186 sc->nr_scanned = nr_scanned; 187 if (nr_scanned >= nr_to_scan || !can_backup) 188 goto out; 189 190 /* If we didn't wake before, try to do it now if needed. */ 191 if (!runtime_pm) 192 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup); 193 194 shrink_flags.purge = false; 195 lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, 196 nr_to_scan, &nr_scanned); 197 if (lret >= 0) 198 freed += lret; 199 200 sc->nr_scanned = nr_scanned; 201 out: 202 xe_shrinker_runtime_pm_put(shrinker, runtime_pm); 203 return nr_scanned ? freed : SHRINK_STOP; 204 } 205 206 /* Wake up the device for shrinking. */ 207 static void xe_shrinker_pm(struct work_struct *work) 208 { 209 struct xe_shrinker *shrinker = 210 container_of(work, typeof(*shrinker), pm_worker); 211 212 xe_pm_runtime_get(shrinker->xe); 213 xe_pm_runtime_put(shrinker->xe); 214 } 215 216 /** 217 * xe_shrinker_create() - Create an xe per-device shrinker 218 * @xe: Pointer to the xe device. 219 * 220 * Returns: A pointer to the created shrinker on success, 221 * Negative error code on failure. 222 */ 223 struct xe_shrinker *xe_shrinker_create(struct xe_device *xe) 224 { 225 struct xe_shrinker *shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL); 226 227 if (!shrinker) 228 return ERR_PTR(-ENOMEM); 229 230 shrinker->shrink = shrinker_alloc(0, "xe system shrinker"); 231 if (!shrinker->shrink) { 232 kfree(shrinker); 233 return ERR_PTR(-ENOMEM); 234 } 235 236 INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm); 237 shrinker->xe = xe; 238 rwlock_init(&shrinker->lock); 239 shrinker->shrink->count_objects = xe_shrinker_count; 240 shrinker->shrink->scan_objects = xe_shrinker_scan; 241 shrinker->shrink->private_data = shrinker; 242 shrinker_register(shrinker->shrink); 243 244 return shrinker; 245 } 246 247 /** 248 * xe_shrinker_destroy() - Destroy an xe per-device shrinker 249 * @shrinker: Pointer to the shrinker to destroy. 250 */ 251 void xe_shrinker_destroy(struct xe_shrinker *shrinker) 252 { 253 xe_assert(shrinker->xe, !shrinker->shrinkable_pages); 254 xe_assert(shrinker->xe, !shrinker->purgeable_pages); 255 shrinker_free(shrinker->shrink); 256 flush_work(&shrinker->pm_worker); 257 kfree(shrinker); 258 } 259