1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/vmalloc.h> 8 #include <linux/sched/mm.h> 9 10 #include "msm_drv.h" 11 #include "msm_gem.h" 12 #include "msm_gpu.h" 13 #include "msm_gpu_trace.h" 14 15 /* Default disabled for now until it has some more testing on the different 16 * iommu combinations that can be paired with the driver: 17 */ 18 static bool enable_eviction = true; 19 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers"); 20 module_param(enable_eviction, bool, 0600); 21 22 static bool eviction_can_block = true; 23 MODULE_PARM_DESC(eviction_can_block, "Enable blocking for GEM buffer to become idle for eviction"); 24 module_param(eviction_can_block, bool, 0600); 25 26 static bool can_swap(void) 27 { 28 return enable_eviction && get_nr_swap_pages() > 0; 29 } 30 31 static bool can_block(struct shrink_control *sc) 32 { 33 if (!eviction_can_block) 34 return false; 35 return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) || 36 (current_is_kswapd() && (sc->gfp_mask & __GFP_KSWAPD_RECLAIM)); 37 } 38 39 static unsigned long 40 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 41 { 42 struct msm_drm_private *priv = shrinker->private_data; 43 unsigned count = priv->lru.dontneed.count; 44 45 if (can_swap()) 46 count += priv->lru.willneed.count; 47 48 return count; 49 } 50 51 static bool 52 with_vm_locks(void (*fn)(struct drm_gem_object *obj), 53 struct drm_gem_object *obj) 54 { 55 /* 56 * Track last locked entry for for unwinding locks in error and 57 * success paths 58 */ 59 struct drm_gpuvm_bo *vm_bo, *last_locked = NULL; 60 struct drm_gpuvm_bo *next; 61 bool locked = true; 62 63 drm_gem_for_each_gpuvm_bo (vm_bo, obj) { 64 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 65 66 if (resv == obj->resv) 67 continue; 68 69 /* 70 * dma_resv_lock can't be used due to acquiring 'ticket' before the 71 * fs_reclaim lock, which is held in shrinker context 72 */ 73 if (!dma_resv_trylock(resv)) { 74 locked = false; 75 goto out_unlock; 76 } 77 78 /* 79 * Hold a ref to prevent the vm_bo from being freed 80 * and removed from the obj's gpuva list, as that would 81 * would result in missing the unlock below 82 */ 83 drm_gpuvm_bo_get(vm_bo); 84 85 last_locked = vm_bo; 86 } 87 88 fn(obj); 89 90 out_unlock: 91 if (last_locked) { 92 drm_gem_for_each_gpuvm_bo_safe(vm_bo, next, obj) { 93 struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); 94 95 if (resv == obj->resv) 96 continue; 97 98 dma_resv_unlock(resv); 99 100 /* Drop the ref taken while locking: */ 101 drm_gpuvm_bo_put(vm_bo); 102 103 if (last_locked == vm_bo) 104 break; 105 } 106 } 107 108 return locked; 109 } 110 111 static bool 112 purge(struct drm_gem_object *obj) 113 { 114 if (!is_purgeable(to_msm_bo(obj))) 115 return false; 116 117 if (msm_gem_active(obj)) 118 return false; 119 120 return with_vm_locks(msm_gem_purge, obj); 121 } 122 123 static bool 124 evict(struct drm_gem_object *obj) 125 { 126 if (is_unevictable(to_msm_bo(obj))) 127 return false; 128 129 if (msm_gem_active(obj)) 130 return false; 131 132 return with_vm_locks(msm_gem_evict, obj); 133 } 134 135 static bool 136 wait_for_idle(struct drm_gem_object *obj) 137 { 138 enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP; 139 return dma_resv_wait_timeout(obj->resv, usage, false, 10) > 0; 140 } 141 142 static bool 143 active_purge(struct drm_gem_object *obj) 144 { 145 if (!wait_for_idle(obj)) 146 return false; 147 148 return purge(obj); 149 } 150 151 static bool 152 active_evict(struct drm_gem_object *obj) 153 { 154 if (!wait_for_idle(obj)) 155 return false; 156 157 return evict(obj); 158 } 159 160 static unsigned long 161 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 162 { 163 struct msm_drm_private *priv = shrinker->private_data; 164 struct { 165 struct drm_gem_lru *lru; 166 bool (*shrink)(struct drm_gem_object *obj); 167 bool cond; 168 unsigned long freed; 169 unsigned long remaining; 170 } stages[] = { 171 /* Stages of progressively more aggressive/expensive reclaim: */ 172 { &priv->lru.dontneed, purge, true }, 173 { &priv->lru.willneed, evict, can_swap() }, 174 { &priv->lru.dontneed, active_purge, can_block(sc) }, 175 { &priv->lru.willneed, active_evict, can_swap() && can_block(sc) }, 176 }; 177 long nr = sc->nr_to_scan; 178 unsigned long freed = 0; 179 unsigned long remaining = 0; 180 181 for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) { 182 if (!stages[i].cond) 183 continue; 184 /* 185 * 'ticket' not needed on trylock paths 186 */ 187 stages[i].freed = 188 drm_gem_lru_scan(priv->dev, stages[i].lru, nr, 189 &stages[i].remaining, 190 stages[i].shrink); 191 nr -= stages[i].freed; 192 freed += stages[i].freed; 193 remaining += stages[i].remaining; 194 } 195 196 if (freed) { 197 trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed, 198 stages[1].freed, stages[2].freed, 199 stages[3].freed); 200 } 201 202 return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP; 203 } 204 205 #ifdef CONFIG_DEBUG_FS 206 unsigned long 207 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan) 208 { 209 struct msm_drm_private *priv = dev->dev_private; 210 struct shrink_control sc = { 211 .nr_to_scan = nr_to_scan, 212 }; 213 unsigned long ret = SHRINK_STOP; 214 215 fs_reclaim_acquire(GFP_KERNEL); 216 if (priv->shrinker) 217 ret = msm_gem_shrinker_scan(priv->shrinker, &sc); 218 fs_reclaim_release(GFP_KERNEL); 219 220 return ret; 221 } 222 #endif 223 224 /* since we don't know any better, lets bail after a few 225 * and if necessary the shrinker will be invoked again. 226 * Seems better than unmapping *everything* 227 */ 228 static const int vmap_shrink_limit = 15; 229 230 static bool 231 vmap_shrink(struct drm_gem_object *obj) 232 { 233 if (!is_vunmapable(to_msm_bo(obj))) 234 return false; 235 236 msm_gem_vunmap(obj); 237 238 return true; 239 } 240 241 static int 242 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 243 { 244 struct msm_drm_private *priv = 245 container_of(nb, struct msm_drm_private, vmap_notifier); 246 struct drm_gem_lru *lrus[] = { 247 &priv->lru.dontneed, 248 &priv->lru.willneed, 249 &priv->lru.pinned, 250 NULL, 251 }; 252 unsigned idx, unmapped = 0; 253 unsigned long remaining = 0; 254 255 for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) { 256 unmapped += drm_gem_lru_scan(priv->dev, lrus[idx], 257 vmap_shrink_limit - unmapped, 258 &remaining, 259 vmap_shrink); 260 } 261 262 *(unsigned long *)ptr += unmapped; 263 264 if (unmapped > 0) 265 trace_msm_gem_purge_vmaps(unmapped); 266 267 return NOTIFY_DONE; 268 } 269 270 /** 271 * msm_gem_shrinker_init - Initialize msm shrinker 272 * @dev: drm device 273 * 274 * This function registers and sets up the msm shrinker. 275 */ 276 int msm_gem_shrinker_init(struct drm_device *dev) 277 { 278 struct msm_drm_private *priv = dev->dev_private; 279 280 priv->shrinker = shrinker_alloc(0, "drm-msm_gem"); 281 if (!priv->shrinker) 282 return -ENOMEM; 283 284 priv->shrinker->count_objects = msm_gem_shrinker_count; 285 priv->shrinker->scan_objects = msm_gem_shrinker_scan; 286 priv->shrinker->private_data = priv; 287 288 shrinker_register(priv->shrinker); 289 290 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap; 291 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier)); 292 293 return 0; 294 } 295 296 /** 297 * msm_gem_shrinker_cleanup - Clean up msm shrinker 298 * @dev: drm device 299 * 300 * This function unregisters the msm shrinker. 301 */ 302 void msm_gem_shrinker_cleanup(struct drm_device *dev) 303 { 304 struct msm_drm_private *priv = dev->dev_private; 305 306 if (priv->shrinker) { 307 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier)); 308 shrinker_free(priv->shrinker); 309 } 310 } 311