1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2014-2018 Intel Corporation
4 */
5
6 #include "gem/i915_gem_internal.h"
7 #include "gem/i915_gem_object.h"
8
9 #include "i915_drv.h"
10 #include "i915_list_util.h"
11 #include "intel_engine_pm.h"
12 #include "intel_gt_buffer_pool.h"
13
14 static struct list_head *
bucket_for_size(struct intel_gt_buffer_pool * pool,size_t sz)15 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz)
16 {
17 int n;
18
19 /*
20 * Compute a power-of-two bucket, but throw everything greater than
21 * 16KiB into the same bucket: i.e. the buckets hold objects of
22 * (1 page, 2 pages, 4 pages, 8+ pages).
23 */
24 n = fls(sz >> PAGE_SHIFT) - 1;
25 if (n >= ARRAY_SIZE(pool->cache_list))
26 n = ARRAY_SIZE(pool->cache_list) - 1;
27
28 return &pool->cache_list[n];
29 }
30
node_free(struct intel_gt_buffer_pool_node * node)31 static void node_free(struct intel_gt_buffer_pool_node *node)
32 {
33 i915_gem_object_put(node->obj);
34 i915_active_fini(&node->active);
35 kfree_rcu(node, rcu);
36 }
37
pool_free_older_than(struct intel_gt_buffer_pool * pool,long keep)38 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep)
39 {
40 struct intel_gt_buffer_pool_node *node, *stale = NULL;
41 bool active = false;
42 int n;
43
44 /* Free buffers that have not been used in the past second */
45 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
46 struct list_head *list = &pool->cache_list[n];
47
48 if (list_empty(list))
49 continue;
50
51 if (spin_trylock_irq(&pool->lock)) {
52 struct list_head *pos;
53
54 /* Most recent at head; oldest at tail */
55 list_for_each_prev(pos, list) {
56 unsigned long age;
57
58 node = list_entry(pos, typeof(*node), link);
59
60 age = READ_ONCE(node->age);
61 if (!age || jiffies - age < keep)
62 break;
63
64 /* Check we are the first to claim this node */
65 if (!xchg(&node->age, 0))
66 break;
67
68 node->free = stale;
69 stale = node;
70 }
71 if (!list_is_last(pos, list))
72 __list_del_many(pos, list);
73
74 spin_unlock_irq(&pool->lock);
75 }
76
77 active |= !list_empty(list);
78 }
79
80 while ((node = stale)) {
81 stale = stale->free;
82 node_free(node);
83 }
84
85 return active;
86 }
87
pool_free_work(struct work_struct * wrk)88 static void pool_free_work(struct work_struct *wrk)
89 {
90 struct intel_gt_buffer_pool *pool =
91 container_of(wrk, typeof(*pool), work.work);
92 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
93
94 if (pool_free_older_than(pool, HZ))
95 queue_delayed_work(gt->i915->unordered_wq, &pool->work,
96 round_jiffies_up_relative(HZ));
97 }
98
pool_retire(struct i915_active * ref)99 static void pool_retire(struct i915_active *ref)
100 {
101 struct intel_gt_buffer_pool_node *node =
102 container_of(ref, typeof(*node), active);
103 struct intel_gt_buffer_pool *pool = node->pool;
104 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
105 struct list_head *list = bucket_for_size(pool, node->obj->base.size);
106 unsigned long flags;
107
108 if (node->pinned) {
109 i915_gem_object_unpin_pages(node->obj);
110
111 /* Return this object to the shrinker pool */
112 i915_gem_object_make_purgeable(node->obj);
113 node->pinned = false;
114 }
115
116 GEM_BUG_ON(node->age);
117 spin_lock_irqsave(&pool->lock, flags);
118 list_add_rcu(&node->link, list);
119 WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */
120 spin_unlock_irqrestore(&pool->lock, flags);
121
122 queue_delayed_work(gt->i915->unordered_wq, &pool->work,
123 round_jiffies_up_relative(HZ));
124 }
125
intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node * node)126 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
127 {
128 assert_object_held(node->obj);
129
130 if (node->pinned)
131 return;
132
133 __i915_gem_object_pin_pages(node->obj);
134 /* Hide this pinned object from the shrinker until retired */
135 i915_gem_object_make_unshrinkable(node->obj);
136 node->pinned = true;
137 }
138
139 static struct intel_gt_buffer_pool_node *
node_create(struct intel_gt_buffer_pool * pool,size_t sz,enum i915_map_type type)140 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
141 enum i915_map_type type)
142 {
143 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
144 struct intel_gt_buffer_pool_node *node;
145 struct drm_i915_gem_object *obj;
146
147 node = kmalloc_obj(*node,
148 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
149 if (!node)
150 return ERR_PTR(-ENOMEM);
151
152 node->age = 0;
153 node->pool = pool;
154 node->pinned = false;
155 i915_active_init(&node->active, NULL, pool_retire, 0);
156
157 obj = i915_gem_object_create_internal(gt->i915, sz);
158 if (IS_ERR(obj)) {
159 i915_active_fini(&node->active);
160 kfree(node);
161 return ERR_CAST(obj);
162 }
163
164 i915_gem_object_set_readonly(obj);
165
166 node->type = type;
167 node->obj = obj;
168 return node;
169 }
170
171 struct intel_gt_buffer_pool_node *
intel_gt_get_buffer_pool(struct intel_gt * gt,size_t size,enum i915_map_type type)172 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
173 enum i915_map_type type)
174 {
175 struct intel_gt_buffer_pool *pool = >->buffer_pool;
176 struct intel_gt_buffer_pool_node *node;
177 struct list_head *list;
178 int ret;
179
180 size = PAGE_ALIGN(size);
181 list = bucket_for_size(pool, size);
182
183 rcu_read_lock();
184 list_for_each_entry_rcu(node, list, link) {
185 unsigned long age;
186
187 if (node->obj->base.size < size)
188 continue;
189
190 if (node->type != type)
191 continue;
192
193 age = READ_ONCE(node->age);
194 if (!age)
195 continue;
196
197 if (cmpxchg(&node->age, age, 0) == age) {
198 spin_lock_irq(&pool->lock);
199 list_del_rcu(&node->link);
200 spin_unlock_irq(&pool->lock);
201 break;
202 }
203 }
204 rcu_read_unlock();
205
206 if (&node->link == list) {
207 node = node_create(pool, size, type);
208 if (IS_ERR(node))
209 return node;
210 }
211
212 ret = i915_active_acquire(&node->active);
213 if (ret) {
214 node_free(node);
215 return ERR_PTR(ret);
216 }
217
218 return node;
219 }
220
intel_gt_init_buffer_pool(struct intel_gt * gt)221 void intel_gt_init_buffer_pool(struct intel_gt *gt)
222 {
223 struct intel_gt_buffer_pool *pool = >->buffer_pool;
224 int n;
225
226 spin_lock_init(&pool->lock);
227 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
228 INIT_LIST_HEAD(&pool->cache_list[n]);
229 INIT_DELAYED_WORK(&pool->work, pool_free_work);
230 }
231
intel_gt_flush_buffer_pool(struct intel_gt * gt)232 void intel_gt_flush_buffer_pool(struct intel_gt *gt)
233 {
234 struct intel_gt_buffer_pool *pool = >->buffer_pool;
235
236 do {
237 while (pool_free_older_than(pool, 0))
238 ;
239 } while (cancel_delayed_work_sync(&pool->work));
240 }
241
intel_gt_fini_buffer_pool(struct intel_gt * gt)242 void intel_gt_fini_buffer_pool(struct intel_gt *gt)
243 {
244 struct intel_gt_buffer_pool *pool = >->buffer_pool;
245 int n;
246
247 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
248 GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
249 }
250