1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/vmalloc.h> 8 #include <linux/sched/mm.h> 9 10 #include "msm_drv.h" 11 #include "msm_gem.h" 12 #include "msm_gpu.h" 13 #include "msm_gpu_trace.h" 14 15 /* Default disabled for now until it has some more testing on the different 16 * iommu combinations that can be paired with the driver: 17 */ 18 bool enable_eviction = false; 19 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers"); 20 module_param(enable_eviction, bool, 0600); 21 22 static bool can_swap(void) 23 { 24 return enable_eviction && get_nr_swap_pages() > 0; 25 } 26 27 static unsigned long 28 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 29 { 30 struct msm_drm_private *priv = 31 container_of(shrinker, struct msm_drm_private, shrinker); 32 unsigned count = priv->shrinkable_count; 33 34 if (can_swap()) 35 count += priv->evictable_count; 36 37 return count; 38 } 39 40 static bool 41 purge(struct msm_gem_object *msm_obj) 42 { 43 if (!is_purgeable(msm_obj)) 44 return false; 45 46 /* 47 * This will move the obj out of still_in_list to 48 * the purged list 49 */ 50 msm_gem_purge(&msm_obj->base); 51 52 return true; 53 } 54 55 static bool 56 evict(struct msm_gem_object *msm_obj) 57 { 58 if (is_unevictable(msm_obj)) 59 return false; 60 61 msm_gem_evict(&msm_obj->base); 62 63 return true; 64 } 65 66 static unsigned long 67 scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list, 68 bool (*shrink)(struct msm_gem_object *msm_obj)) 69 { 70 unsigned freed = 0; 71 struct list_head still_in_list; 72 73 INIT_LIST_HEAD(&still_in_list); 74 75 mutex_lock(&priv->mm_lock); 76 77 while (freed < nr_to_scan) { 78 struct msm_gem_object *msm_obj = list_first_entry_or_null( 79 list, typeof(*msm_obj), mm_list); 80 81 if (!msm_obj) 82 break; 83 84 list_move_tail(&msm_obj->mm_list, &still_in_list); 85 86 /* 87 * If it is in the process of being freed, msm_gem_free_object 88 * can be blocked on mm_lock waiting to remove it. So just 89 * skip it. 90 */ 91 if (!kref_get_unless_zero(&msm_obj->base.refcount)) 92 continue; 93 94 /* 95 * Now that we own a reference, we can drop mm_lock for the 96 * rest of the loop body, to reduce contention with the 97 * retire_submit path (which could make more objects purgeable) 98 */ 99 100 mutex_unlock(&priv->mm_lock); 101 102 /* 103 * Note that this still needs to be trylock, since we can 104 * hit shrinker in response to trying to get backing pages 105 * for this obj (ie. while it's lock is already held) 106 */ 107 if (!msm_gem_trylock(&msm_obj->base)) 108 goto tail; 109 110 if (shrink(msm_obj)) 111 freed += msm_obj->base.size >> PAGE_SHIFT; 112 113 msm_gem_unlock(&msm_obj->base); 114 115 tail: 116 drm_gem_object_put(&msm_obj->base); 117 mutex_lock(&priv->mm_lock); 118 } 119 120 list_splice_tail(&still_in_list, list); 121 mutex_unlock(&priv->mm_lock); 122 123 return freed; 124 } 125 126 static unsigned long 127 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 128 { 129 struct msm_drm_private *priv = 130 container_of(shrinker, struct msm_drm_private, shrinker); 131 unsigned long freed; 132 133 freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge); 134 135 if (freed > 0) 136 trace_msm_gem_purge(freed << PAGE_SHIFT); 137 138 if (can_swap() && freed < sc->nr_to_scan) { 139 int evicted = scan(priv, sc->nr_to_scan - freed, 140 &priv->inactive_willneed, evict); 141 142 if (evicted > 0) 143 trace_msm_gem_evict(evicted << PAGE_SHIFT); 144 145 freed += evicted; 146 } 147 148 return (freed > 0) ? freed : SHRINK_STOP; 149 } 150 151 #ifdef CONFIG_DEBUG_FS 152 unsigned long 153 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan) 154 { 155 struct msm_drm_private *priv = dev->dev_private; 156 struct shrink_control sc = { 157 .nr_to_scan = nr_to_scan, 158 }; 159 int ret; 160 161 fs_reclaim_acquire(GFP_KERNEL); 162 ret = msm_gem_shrinker_scan(&priv->shrinker, &sc); 163 fs_reclaim_release(GFP_KERNEL); 164 165 return ret; 166 } 167 #endif 168 169 /* since we don't know any better, lets bail after a few 170 * and if necessary the shrinker will be invoked again. 171 * Seems better than unmapping *everything* 172 */ 173 static const int vmap_shrink_limit = 15; 174 175 static bool 176 vmap_shrink(struct msm_gem_object *msm_obj) 177 { 178 if (!is_vunmapable(msm_obj)) 179 return false; 180 181 msm_gem_vunmap(&msm_obj->base); 182 183 return true; 184 } 185 186 static int 187 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 188 { 189 struct msm_drm_private *priv = 190 container_of(nb, struct msm_drm_private, vmap_notifier); 191 struct list_head *mm_lists[] = { 192 &priv->inactive_dontneed, 193 &priv->inactive_willneed, 194 priv->gpu ? &priv->gpu->active_list : NULL, 195 NULL, 196 }; 197 unsigned idx, unmapped = 0; 198 199 for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) { 200 unmapped += scan(priv, vmap_shrink_limit - unmapped, 201 mm_lists[idx], vmap_shrink); 202 } 203 204 *(unsigned long *)ptr += unmapped; 205 206 if (unmapped > 0) 207 trace_msm_gem_purge_vmaps(unmapped); 208 209 return NOTIFY_DONE; 210 } 211 212 /** 213 * msm_gem_shrinker_init - Initialize msm shrinker 214 * @dev: drm device 215 * 216 * This function registers and sets up the msm shrinker. 217 */ 218 void msm_gem_shrinker_init(struct drm_device *dev) 219 { 220 struct msm_drm_private *priv = dev->dev_private; 221 priv->shrinker.count_objects = msm_gem_shrinker_count; 222 priv->shrinker.scan_objects = msm_gem_shrinker_scan; 223 priv->shrinker.seeks = DEFAULT_SEEKS; 224 WARN_ON(register_shrinker(&priv->shrinker)); 225 226 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap; 227 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier)); 228 } 229 230 /** 231 * msm_gem_shrinker_cleanup - Clean up msm shrinker 232 * @dev: drm device 233 * 234 * This function unregisters the msm shrinker. 235 */ 236 void msm_gem_shrinker_cleanup(struct drm_device *dev) 237 { 238 struct msm_drm_private *priv = dev->dev_private; 239 240 if (priv->shrinker.nr_deferred) { 241 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier)); 242 unregister_shrinker(&priv->shrinker); 243 } 244 } 245