1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_object.h" 7 8 #include "i915_drv.h" 9 #include "intel_engine_pm.h" 10 #include "intel_gt_buffer_pool.h" 11 12 static struct intel_gt *to_gt(struct intel_gt_buffer_pool *pool) 13 { 14 return container_of(pool, struct intel_gt, buffer_pool); 15 } 16 17 static struct list_head * 18 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz) 19 { 20 int n; 21 22 /* 23 * Compute a power-of-two bucket, but throw everything greater than 24 * 16KiB into the same bucket: i.e. the buckets hold objects of 25 * (1 page, 2 pages, 4 pages, 8+ pages). 26 */ 27 n = fls(sz >> PAGE_SHIFT) - 1; 28 if (n >= ARRAY_SIZE(pool->cache_list)) 29 n = ARRAY_SIZE(pool->cache_list) - 1; 30 31 return &pool->cache_list[n]; 32 } 33 34 static void node_free(struct intel_gt_buffer_pool_node *node) 35 { 36 i915_gem_object_put(node->obj); 37 i915_active_fini(&node->active); 38 kfree_rcu(node, rcu); 39 } 40 41 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep) 42 { 43 struct intel_gt_buffer_pool_node *node, *stale = NULL; 44 bool active = false; 45 int n; 46 47 /* Free buffers that have not been used in the past second */ 48 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { 49 struct list_head *list = &pool->cache_list[n]; 50 51 if (list_empty(list)) 52 continue; 53 54 if (spin_trylock_irq(&pool->lock)) { 55 struct list_head *pos; 56 57 /* Most recent at head; oldest at tail */ 58 list_for_each_prev(pos, list) { 59 unsigned long age; 60 61 node = list_entry(pos, typeof(*node), link); 62 63 age = READ_ONCE(node->age); 64 if (!age || jiffies - age < keep) 65 break; 66 67 /* Check we are the first to claim this node */ 68 if (!xchg(&node->age, 0)) 69 break; 70 71 node->free = stale; 72 stale = node; 73 } 74 if (!list_is_last(pos, list)) 75 __list_del_many(pos, list); 76 77 spin_unlock_irq(&pool->lock); 78 } 79 80 active |= !list_empty(list); 81 } 82 83 while ((node = stale)) { 84 stale = stale->free; 85 node_free(node); 86 } 87 88 return active; 89 } 90 91 static void pool_free_work(struct work_struct *wrk) 92 { 93 struct intel_gt_buffer_pool *pool = 94 container_of(wrk, typeof(*pool), work.work); 95 96 if (pool_free_older_than(pool, HZ)) 97 schedule_delayed_work(&pool->work, 98 round_jiffies_up_relative(HZ)); 99 } 100 101 __i915_active_call 102 static void pool_retire(struct i915_active *ref) 103 { 104 struct intel_gt_buffer_pool_node *node = 105 container_of(ref, typeof(*node), active); 106 struct intel_gt_buffer_pool *pool = node->pool; 107 struct list_head *list = bucket_for_size(pool, node->obj->base.size); 108 unsigned long flags; 109 110 if (node->pinned) { 111 i915_gem_object_unpin_pages(node->obj); 112 113 /* Return this object to the shrinker pool */ 114 i915_gem_object_make_purgeable(node->obj); 115 node->pinned = false; 116 } 117 118 GEM_BUG_ON(node->age); 119 spin_lock_irqsave(&pool->lock, flags); 120 list_add_rcu(&node->link, list); 121 WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */ 122 spin_unlock_irqrestore(&pool->lock, flags); 123 124 schedule_delayed_work(&pool->work, 125 round_jiffies_up_relative(HZ)); 126 } 127 128 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node) 129 { 130 assert_object_held(node->obj); 131 132 if (node->pinned) 133 return; 134 135 __i915_gem_object_pin_pages(node->obj); 136 /* Hide this pinned object from the shrinker until retired */ 137 i915_gem_object_make_unshrinkable(node->obj); 138 node->pinned = true; 139 } 140 141 static struct intel_gt_buffer_pool_node * 142 node_create(struct intel_gt_buffer_pool *pool, size_t sz, 143 enum i915_map_type type) 144 { 145 struct intel_gt *gt = to_gt(pool); 146 struct intel_gt_buffer_pool_node *node; 147 struct drm_i915_gem_object *obj; 148 149 node = kmalloc(sizeof(*node), 150 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 151 if (!node) 152 return ERR_PTR(-ENOMEM); 153 154 node->age = 0; 155 node->pool = pool; 156 node->pinned = false; 157 i915_active_init(&node->active, NULL, pool_retire); 158 159 obj = i915_gem_object_create_internal(gt->i915, sz); 160 if (IS_ERR(obj)) { 161 i915_active_fini(&node->active); 162 kfree(node); 163 return ERR_CAST(obj); 164 } 165 166 i915_gem_object_set_readonly(obj); 167 168 node->type = type; 169 node->obj = obj; 170 return node; 171 } 172 173 struct intel_gt_buffer_pool_node * 174 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size, 175 enum i915_map_type type) 176 { 177 struct intel_gt_buffer_pool *pool = >->buffer_pool; 178 struct intel_gt_buffer_pool_node *node; 179 struct list_head *list; 180 int ret; 181 182 size = PAGE_ALIGN(size); 183 list = bucket_for_size(pool, size); 184 185 rcu_read_lock(); 186 list_for_each_entry_rcu(node, list, link) { 187 unsigned long age; 188 189 if (node->obj->base.size < size) 190 continue; 191 192 if (node->type != type) 193 continue; 194 195 age = READ_ONCE(node->age); 196 if (!age) 197 continue; 198 199 if (cmpxchg(&node->age, age, 0) == age) { 200 spin_lock_irq(&pool->lock); 201 list_del_rcu(&node->link); 202 spin_unlock_irq(&pool->lock); 203 break; 204 } 205 } 206 rcu_read_unlock(); 207 208 if (&node->link == list) { 209 node = node_create(pool, size, type); 210 if (IS_ERR(node)) 211 return node; 212 } 213 214 ret = i915_active_acquire(&node->active); 215 if (ret) { 216 node_free(node); 217 return ERR_PTR(ret); 218 } 219 220 return node; 221 } 222 223 void intel_gt_init_buffer_pool(struct intel_gt *gt) 224 { 225 struct intel_gt_buffer_pool *pool = >->buffer_pool; 226 int n; 227 228 spin_lock_init(&pool->lock); 229 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 230 INIT_LIST_HEAD(&pool->cache_list[n]); 231 INIT_DELAYED_WORK(&pool->work, pool_free_work); 232 } 233 234 void intel_gt_flush_buffer_pool(struct intel_gt *gt) 235 { 236 struct intel_gt_buffer_pool *pool = >->buffer_pool; 237 238 do { 239 while (pool_free_older_than(pool, 0)) 240 ; 241 } while (cancel_delayed_work_sync(&pool->work)); 242 } 243 244 void intel_gt_fini_buffer_pool(struct intel_gt *gt) 245 { 246 struct intel_gt_buffer_pool *pool = >->buffer_pool; 247 int n; 248 249 intel_gt_flush_buffer_pool(gt); 250 251 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 252 GEM_BUG_ON(!list_empty(&pool->cache_list[n])); 253 } 254