1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_internal.h" 7 #include "gem/i915_gem_object.h" 8 9 #include "i915_drv.h" 10 #include "i915_list_util.h" 11 #include "intel_engine_pm.h" 12 #include "intel_gt_buffer_pool.h" 13 14 static struct list_head * 15 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz) 16 { 17 int n; 18 19 /* 20 * Compute a power-of-two bucket, but throw everything greater than 21 * 16KiB into the same bucket: i.e. the buckets hold objects of 22 * (1 page, 2 pages, 4 pages, 8+ pages). 23 */ 24 n = fls(sz >> PAGE_SHIFT) - 1; 25 if (n >= ARRAY_SIZE(pool->cache_list)) 26 n = ARRAY_SIZE(pool->cache_list) - 1; 27 28 return &pool->cache_list[n]; 29 } 30 31 static void node_free(struct intel_gt_buffer_pool_node *node) 32 { 33 i915_gem_object_put(node->obj); 34 i915_active_fini(&node->active); 35 kfree_rcu(node, rcu); 36 } 37 38 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep) 39 { 40 struct intel_gt_buffer_pool_node *node, *stale = NULL; 41 bool active = false; 42 int n; 43 44 /* Free buffers that have not been used in the past second */ 45 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { 46 struct list_head *list = &pool->cache_list[n]; 47 48 if (list_empty(list)) 49 continue; 50 51 if (spin_trylock_irq(&pool->lock)) { 52 struct list_head *pos; 53 54 /* Most recent at head; oldest at tail */ 55 list_for_each_prev(pos, list) { 56 unsigned long age; 57 58 node = list_entry(pos, typeof(*node), link); 59 60 age = READ_ONCE(node->age); 61 if (!age || jiffies - age < keep) 62 break; 63 64 /* Check we are the first to claim this node */ 65 if (!xchg(&node->age, 0)) 66 break; 67 68 node->free = stale; 69 stale = node; 70 } 71 if (!list_is_last(pos, list)) 72 __list_del_many(pos, list); 73 74 spin_unlock_irq(&pool->lock); 75 } 76 77 active |= !list_empty(list); 78 } 79 80 while ((node = stale)) { 81 stale = stale->free; 82 node_free(node); 83 } 84 85 return active; 86 } 87 88 static void pool_free_work(struct work_struct *wrk) 89 { 90 struct intel_gt_buffer_pool *pool = 91 container_of(wrk, typeof(*pool), work.work); 92 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); 93 94 if (pool_free_older_than(pool, HZ)) 95 queue_delayed_work(gt->i915->unordered_wq, &pool->work, 96 round_jiffies_up_relative(HZ)); 97 } 98 99 static void pool_retire(struct i915_active *ref) 100 { 101 struct intel_gt_buffer_pool_node *node = 102 container_of(ref, typeof(*node), active); 103 struct intel_gt_buffer_pool *pool = node->pool; 104 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); 105 struct list_head *list = bucket_for_size(pool, node->obj->base.size); 106 unsigned long flags; 107 108 if (node->pinned) { 109 i915_gem_object_unpin_pages(node->obj); 110 111 /* Return this object to the shrinker pool */ 112 i915_gem_object_make_purgeable(node->obj); 113 node->pinned = false; 114 } 115 116 GEM_BUG_ON(node->age); 117 spin_lock_irqsave(&pool->lock, flags); 118 list_add_rcu(&node->link, list); 119 WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */ 120 spin_unlock_irqrestore(&pool->lock, flags); 121 122 queue_delayed_work(gt->i915->unordered_wq, &pool->work, 123 round_jiffies_up_relative(HZ)); 124 } 125 126 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node) 127 { 128 assert_object_held(node->obj); 129 130 if (node->pinned) 131 return; 132 133 __i915_gem_object_pin_pages(node->obj); 134 /* Hide this pinned object from the shrinker until retired */ 135 i915_gem_object_make_unshrinkable(node->obj); 136 node->pinned = true; 137 } 138 139 static struct intel_gt_buffer_pool_node * 140 node_create(struct intel_gt_buffer_pool *pool, size_t sz, 141 enum i915_map_type type) 142 { 143 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); 144 struct intel_gt_buffer_pool_node *node; 145 struct drm_i915_gem_object *obj; 146 147 node = kmalloc(sizeof(*node), 148 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 149 if (!node) 150 return ERR_PTR(-ENOMEM); 151 152 node->age = 0; 153 node->pool = pool; 154 node->pinned = false; 155 i915_active_init(&node->active, NULL, pool_retire, 0); 156 157 obj = i915_gem_object_create_internal(gt->i915, sz); 158 if (IS_ERR(obj)) { 159 i915_active_fini(&node->active); 160 kfree(node); 161 return ERR_CAST(obj); 162 } 163 164 i915_gem_object_set_readonly(obj); 165 166 node->type = type; 167 node->obj = obj; 168 return node; 169 } 170 171 struct intel_gt_buffer_pool_node * 172 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size, 173 enum i915_map_type type) 174 { 175 struct intel_gt_buffer_pool *pool = >->buffer_pool; 176 struct intel_gt_buffer_pool_node *node; 177 struct list_head *list; 178 int ret; 179 180 size = PAGE_ALIGN(size); 181 list = bucket_for_size(pool, size); 182 183 rcu_read_lock(); 184 list_for_each_entry_rcu(node, list, link) { 185 unsigned long age; 186 187 if (node->obj->base.size < size) 188 continue; 189 190 if (node->type != type) 191 continue; 192 193 age = READ_ONCE(node->age); 194 if (!age) 195 continue; 196 197 if (cmpxchg(&node->age, age, 0) == age) { 198 spin_lock_irq(&pool->lock); 199 list_del_rcu(&node->link); 200 spin_unlock_irq(&pool->lock); 201 break; 202 } 203 } 204 rcu_read_unlock(); 205 206 if (&node->link == list) { 207 node = node_create(pool, size, type); 208 if (IS_ERR(node)) 209 return node; 210 } 211 212 ret = i915_active_acquire(&node->active); 213 if (ret) { 214 node_free(node); 215 return ERR_PTR(ret); 216 } 217 218 return node; 219 } 220 221 void intel_gt_init_buffer_pool(struct intel_gt *gt) 222 { 223 struct intel_gt_buffer_pool *pool = >->buffer_pool; 224 int n; 225 226 spin_lock_init(&pool->lock); 227 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 228 INIT_LIST_HEAD(&pool->cache_list[n]); 229 INIT_DELAYED_WORK(&pool->work, pool_free_work); 230 } 231 232 void intel_gt_flush_buffer_pool(struct intel_gt *gt) 233 { 234 struct intel_gt_buffer_pool *pool = >->buffer_pool; 235 236 do { 237 while (pool_free_older_than(pool, 0)) 238 ; 239 } while (cancel_delayed_work_sync(&pool->work)); 240 } 241 242 void intel_gt_fini_buffer_pool(struct intel_gt *gt) 243 { 244 struct intel_gt_buffer_pool *pool = >->buffer_pool; 245 int n; 246 247 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 248 GEM_BUG_ON(!list_empty(&pool->cache_list[n])); 249 } 250