1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/vmalloc.h> 8 #include <linux/sched/mm.h> 9 10 #include "msm_drv.h" 11 #include "msm_gem.h" 12 #include "msm_gpu.h" 13 #include "msm_gpu_trace.h" 14 15 /* Default disabled for now until it has some more testing on the different 16 * iommu combinations that can be paired with the driver: 17 */ 18 static bool enable_eviction = true; 19 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers"); 20 module_param(enable_eviction, bool, 0600); 21 22 static bool can_swap(void) 23 { 24 return enable_eviction && get_nr_swap_pages() > 0; 25 } 26 27 static bool can_block(struct shrink_control *sc) 28 { 29 return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) || 30 (current_is_kswapd() && (sc->gfp_mask & __GFP_KSWAPD_RECLAIM)); 31 } 32 33 static unsigned long 34 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 35 { 36 struct msm_drm_private *priv = shrinker->private_data; 37 unsigned count = priv->lru.dontneed.count; 38 39 if (can_swap()) 40 count += priv->lru.willneed.count; 41 42 return count; 43 } 44 45 static bool 46 with_vm_locks(void (*fn)(struct drm_gem_object *obj), 47 struct drm_gem_object *obj) 48 { 49 /* 50 * Track last locked entry for for unwinding locks in error and 51 * success paths 52 */ 53 struct drm_gpuvm_bo *vm_bo, *last_locked = NULL; 54 bool locked = true; 55 56 drm_gem_for_each_gpuvm_bo (vm_bo, obj) { 57 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 58 59 if (resv == obj->resv) 60 continue; 61 62 /* 63 * dma_resv_lock can't be used due to acquiring 'ticket' before the 64 * fs_reclaim lock, which is held in shrinker context 65 */ 66 if (!dma_resv_trylock(resv)) { 67 locked = false; 68 goto out_unlock; 69 } 70 71 /* 72 * Hold a ref to prevent the vm_bo from being freed 73 * and removed from the obj's gpuva list, as that would 74 * would result in missing the unlock below 75 */ 76 drm_gpuvm_bo_get(vm_bo); 77 78 last_locked = vm_bo; 79 } 80 81 fn(obj); 82 83 out_unlock: 84 if (last_locked) { 85 drm_gem_for_each_gpuvm_bo (vm_bo, obj) { 86 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 87 88 if (resv == obj->resv) 89 continue; 90 91 dma_resv_unlock(resv); 92 93 /* Drop the ref taken while locking: */ 94 drm_gpuvm_bo_put(vm_bo); 95 96 if (last_locked == vm_bo) 97 break; 98 } 99 } 100 101 return locked; 102 } 103 104 static bool 105 purge(struct drm_gem_object *obj, struct ww_acquire_ctx *unused) 106 { 107 if (!is_purgeable(to_msm_bo(obj))) 108 return false; 109 110 if (msm_gem_active(obj)) 111 return false; 112 113 return with_vm_locks(msm_gem_purge, obj); 114 } 115 116 static bool 117 evict(struct drm_gem_object *obj, struct ww_acquire_ctx *unused) 118 { 119 if (is_unevictable(to_msm_bo(obj))) 120 return false; 121 122 if (msm_gem_active(obj)) 123 return false; 124 125 return with_vm_locks(msm_gem_evict, obj); 126 } 127 128 static bool 129 wait_for_idle(struct drm_gem_object *obj) 130 { 131 enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP; 132 return dma_resv_wait_timeout(obj->resv, usage, false, 10) > 0; 133 } 134 135 static bool 136 active_purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket) 137 { 138 if (!wait_for_idle(obj)) 139 return false; 140 141 return purge(obj, ticket); 142 } 143 144 static bool 145 active_evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket) 146 { 147 if (!wait_for_idle(obj)) 148 return false; 149 150 return evict(obj, ticket); 151 } 152 153 static unsigned long 154 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 155 { 156 struct msm_drm_private *priv = shrinker->private_data; 157 struct { 158 struct drm_gem_lru *lru; 159 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket); 160 bool cond; 161 unsigned long freed; 162 unsigned long remaining; 163 } stages[] = { 164 /* Stages of progressively more aggressive/expensive reclaim: */ 165 { &priv->lru.dontneed, purge, true }, 166 { &priv->lru.willneed, evict, can_swap() }, 167 { &priv->lru.dontneed, active_purge, can_block(sc) }, 168 { &priv->lru.willneed, active_evict, can_swap() && can_block(sc) }, 169 }; 170 long nr = sc->nr_to_scan; 171 unsigned long freed = 0; 172 unsigned long remaining = 0; 173 174 for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) { 175 if (!stages[i].cond) 176 continue; 177 /* 178 * 'ticket' not needed on trylock paths 179 */ 180 stages[i].freed = 181 drm_gem_lru_scan(priv->dev, stages[i].lru, nr, 182 &stages[i].remaining, 183 stages[i].shrink, 184 NULL); 185 nr -= stages[i].freed; 186 freed += stages[i].freed; 187 remaining += stages[i].remaining; 188 } 189 190 if (freed) { 191 trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed, 192 stages[1].freed, stages[2].freed, 193 stages[3].freed); 194 } 195 196 return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP; 197 } 198 199 #ifdef CONFIG_DEBUG_FS 200 unsigned long 201 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan) 202 { 203 struct msm_drm_private *priv = dev->dev_private; 204 struct shrink_control sc = { 205 .nr_to_scan = nr_to_scan, 206 }; 207 unsigned long ret = SHRINK_STOP; 208 209 fs_reclaim_acquire(GFP_KERNEL); 210 if (priv->shrinker) 211 ret = msm_gem_shrinker_scan(priv->shrinker, &sc); 212 fs_reclaim_release(GFP_KERNEL); 213 214 return ret; 215 } 216 #endif 217 218 /* since we don't know any better, lets bail after a few 219 * and if necessary the shrinker will be invoked again. 220 * Seems better than unmapping *everything* 221 */ 222 static const int vmap_shrink_limit = 15; 223 224 static bool 225 vmap_shrink(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket) 226 { 227 if (!is_vunmapable(to_msm_bo(obj))) 228 return false; 229 230 msm_gem_vunmap(obj); 231 232 return true; 233 } 234 235 static int 236 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 237 { 238 struct msm_drm_private *priv = 239 container_of(nb, struct msm_drm_private, vmap_notifier); 240 struct drm_gem_lru *lrus[] = { 241 &priv->lru.dontneed, 242 &priv->lru.willneed, 243 &priv->lru.pinned, 244 NULL, 245 }; 246 unsigned idx, unmapped = 0; 247 unsigned long remaining = 0; 248 249 for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) { 250 unmapped += drm_gem_lru_scan(priv->dev, lrus[idx], 251 vmap_shrink_limit - unmapped, 252 &remaining, 253 vmap_shrink, 254 NULL); 255 } 256 257 *(unsigned long *)ptr += unmapped; 258 259 if (unmapped > 0) 260 trace_msm_gem_purge_vmaps(unmapped); 261 262 return NOTIFY_DONE; 263 } 264 265 /** 266 * msm_gem_shrinker_init - Initialize msm shrinker 267 * @dev: drm device 268 * 269 * This function registers and sets up the msm shrinker. 270 */ 271 int msm_gem_shrinker_init(struct drm_device *dev) 272 { 273 struct msm_drm_private *priv = dev->dev_private; 274 275 priv->shrinker = shrinker_alloc(0, "drm-msm_gem"); 276 if (!priv->shrinker) 277 return -ENOMEM; 278 279 priv->shrinker->count_objects = msm_gem_shrinker_count; 280 priv->shrinker->scan_objects = msm_gem_shrinker_scan; 281 priv->shrinker->private_data = priv; 282 283 shrinker_register(priv->shrinker); 284 285 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap; 286 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier)); 287 288 return 0; 289 } 290 291 /** 292 * msm_gem_shrinker_cleanup - Clean up msm shrinker 293 * @dev: drm device 294 * 295 * This function unregisters the msm shrinker. 296 */ 297 void msm_gem_shrinker_cleanup(struct drm_device *dev) 298 { 299 struct msm_drm_private *priv = dev->dev_private; 300 301 if (priv->shrinker) { 302 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier)); 303 shrinker_free(priv->shrinker); 304 } 305 } 306