13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only
2b29d4986SJoe Thornber /*
3b29d4986SJoe Thornber * Copyright (C) 2017 Red Hat. All rights reserved.
4b29d4986SJoe Thornber *
5b29d4986SJoe Thornber * This file is released under the GPL.
6b29d4986SJoe Thornber */
7b29d4986SJoe Thornber
8b29d4986SJoe Thornber #include "dm-cache-background-tracker.h"
9b29d4986SJoe Thornber
10b29d4986SJoe Thornber /*----------------------------------------------------------------*/
11b29d4986SJoe Thornber
12b29d4986SJoe Thornber #define DM_MSG_PREFIX "dm-background-tracker"
13b29d4986SJoe Thornber
14b29d4986SJoe Thornber struct bt_work {
15b29d4986SJoe Thornber struct list_head list;
16b29d4986SJoe Thornber struct rb_node node;
17b29d4986SJoe Thornber struct policy_work work;
18b29d4986SJoe Thornber };
19b29d4986SJoe Thornber
20b29d4986SJoe Thornber struct background_tracker {
21*86a3238cSHeinz Mauelshagen unsigned int max_work;
22b29d4986SJoe Thornber atomic_t pending_promotes;
23b29d4986SJoe Thornber atomic_t pending_writebacks;
24b29d4986SJoe Thornber atomic_t pending_demotes;
25b29d4986SJoe Thornber
26b29d4986SJoe Thornber struct list_head issued;
27b29d4986SJoe Thornber struct list_head queued;
28b29d4986SJoe Thornber struct rb_root pending;
29b29d4986SJoe Thornber
30b29d4986SJoe Thornber struct kmem_cache *work_cache;
31b29d4986SJoe Thornber };
32b29d4986SJoe Thornber
btracker_create(unsigned int max_work)33*86a3238cSHeinz Mauelshagen struct background_tracker *btracker_create(unsigned int max_work)
34b29d4986SJoe Thornber {
35b29d4986SJoe Thornber struct background_tracker *b = kmalloc(sizeof(*b), GFP_KERNEL);
36b29d4986SJoe Thornber
377e1b9521SColin Ian King if (!b) {
387e1b9521SColin Ian King DMERR("couldn't create background_tracker");
397e1b9521SColin Ian King return NULL;
407e1b9521SColin Ian King }
417e1b9521SColin Ian King
42b29d4986SJoe Thornber b->max_work = max_work;
43b29d4986SJoe Thornber atomic_set(&b->pending_promotes, 0);
44b29d4986SJoe Thornber atomic_set(&b->pending_writebacks, 0);
45b29d4986SJoe Thornber atomic_set(&b->pending_demotes, 0);
46b29d4986SJoe Thornber
47b29d4986SJoe Thornber INIT_LIST_HEAD(&b->issued);
48b29d4986SJoe Thornber INIT_LIST_HEAD(&b->queued);
49b29d4986SJoe Thornber
50b29d4986SJoe Thornber b->pending = RB_ROOT;
51b29d4986SJoe Thornber b->work_cache = KMEM_CACHE(bt_work, 0);
52b29d4986SJoe Thornber if (!b->work_cache) {
53b29d4986SJoe Thornber DMERR("couldn't create mempool for background work items");
54b29d4986SJoe Thornber kfree(b);
55b29d4986SJoe Thornber b = NULL;
56b29d4986SJoe Thornber }
57b29d4986SJoe Thornber
58b29d4986SJoe Thornber return b;
59b29d4986SJoe Thornber }
60b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_create);
61b29d4986SJoe Thornber
btracker_destroy(struct background_tracker * b)62b29d4986SJoe Thornber void btracker_destroy(struct background_tracker *b)
63b29d4986SJoe Thornber {
6495ab80a8SJoe Thornber struct bt_work *w, *tmp;
6595ab80a8SJoe Thornber
6695ab80a8SJoe Thornber BUG_ON(!list_empty(&b->issued));
6795ab80a8SJoe Thornber list_for_each_entry_safe (w, tmp, &b->queued, list) {
6895ab80a8SJoe Thornber list_del(&w->list);
6995ab80a8SJoe Thornber kmem_cache_free(b->work_cache, w);
7095ab80a8SJoe Thornber }
7195ab80a8SJoe Thornber
72b29d4986SJoe Thornber kmem_cache_destroy(b->work_cache);
73b29d4986SJoe Thornber kfree(b);
74b29d4986SJoe Thornber }
75b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_destroy);
76b29d4986SJoe Thornber
cmp_oblock(dm_oblock_t lhs,dm_oblock_t rhs)77b29d4986SJoe Thornber static int cmp_oblock(dm_oblock_t lhs, dm_oblock_t rhs)
78b29d4986SJoe Thornber {
79b29d4986SJoe Thornber if (from_oblock(lhs) < from_oblock(rhs))
80b29d4986SJoe Thornber return -1;
81b29d4986SJoe Thornber
82b29d4986SJoe Thornber if (from_oblock(rhs) < from_oblock(lhs))
83b29d4986SJoe Thornber return 1;
84b29d4986SJoe Thornber
85b29d4986SJoe Thornber return 0;
86b29d4986SJoe Thornber }
87b29d4986SJoe Thornber
__insert_pending(struct background_tracker * b,struct bt_work * nw)88b29d4986SJoe Thornber static bool __insert_pending(struct background_tracker *b,
89b29d4986SJoe Thornber struct bt_work *nw)
90b29d4986SJoe Thornber {
91b29d4986SJoe Thornber int cmp;
92b29d4986SJoe Thornber struct bt_work *w;
93b29d4986SJoe Thornber struct rb_node **new = &b->pending.rb_node, *parent = NULL;
94b29d4986SJoe Thornber
95b29d4986SJoe Thornber while (*new) {
96b29d4986SJoe Thornber w = container_of(*new, struct bt_work, node);
97b29d4986SJoe Thornber
98b29d4986SJoe Thornber parent = *new;
99b29d4986SJoe Thornber cmp = cmp_oblock(w->work.oblock, nw->work.oblock);
100b29d4986SJoe Thornber if (cmp < 0)
101b29d4986SJoe Thornber new = &((*new)->rb_left);
102b29d4986SJoe Thornber
103b29d4986SJoe Thornber else if (cmp > 0)
104b29d4986SJoe Thornber new = &((*new)->rb_right);
105b29d4986SJoe Thornber
106b29d4986SJoe Thornber else
107b29d4986SJoe Thornber /* already present */
108b29d4986SJoe Thornber return false;
109b29d4986SJoe Thornber }
110b29d4986SJoe Thornber
111b29d4986SJoe Thornber rb_link_node(&nw->node, parent, new);
112b29d4986SJoe Thornber rb_insert_color(&nw->node, &b->pending);
113b29d4986SJoe Thornber
114b29d4986SJoe Thornber return true;
115b29d4986SJoe Thornber }
116b29d4986SJoe Thornber
__find_pending(struct background_tracker * b,dm_oblock_t oblock)117b29d4986SJoe Thornber static struct bt_work *__find_pending(struct background_tracker *b,
118b29d4986SJoe Thornber dm_oblock_t oblock)
119b29d4986SJoe Thornber {
120b29d4986SJoe Thornber int cmp;
121b29d4986SJoe Thornber struct bt_work *w;
122b29d4986SJoe Thornber struct rb_node **new = &b->pending.rb_node;
123b29d4986SJoe Thornber
124b29d4986SJoe Thornber while (*new) {
125b29d4986SJoe Thornber w = container_of(*new, struct bt_work, node);
126b29d4986SJoe Thornber
127b29d4986SJoe Thornber cmp = cmp_oblock(w->work.oblock, oblock);
128b29d4986SJoe Thornber if (cmp < 0)
129b29d4986SJoe Thornber new = &((*new)->rb_left);
130b29d4986SJoe Thornber
131b29d4986SJoe Thornber else if (cmp > 0)
132b29d4986SJoe Thornber new = &((*new)->rb_right);
133b29d4986SJoe Thornber
134b29d4986SJoe Thornber else
135b29d4986SJoe Thornber break;
136b29d4986SJoe Thornber }
137b29d4986SJoe Thornber
138b29d4986SJoe Thornber return *new ? w : NULL;
139b29d4986SJoe Thornber }
140b29d4986SJoe Thornber
141b29d4986SJoe Thornber
update_stats(struct background_tracker * b,struct policy_work * w,int delta)142b29d4986SJoe Thornber static void update_stats(struct background_tracker *b, struct policy_work *w, int delta)
143b29d4986SJoe Thornber {
144b29d4986SJoe Thornber switch (w->op) {
145b29d4986SJoe Thornber case POLICY_PROMOTE:
146b29d4986SJoe Thornber atomic_add(delta, &b->pending_promotes);
147b29d4986SJoe Thornber break;
148b29d4986SJoe Thornber
149b29d4986SJoe Thornber case POLICY_DEMOTE:
150b29d4986SJoe Thornber atomic_add(delta, &b->pending_demotes);
151b29d4986SJoe Thornber break;
152b29d4986SJoe Thornber
153b29d4986SJoe Thornber case POLICY_WRITEBACK:
154b29d4986SJoe Thornber atomic_add(delta, &b->pending_writebacks);
155b29d4986SJoe Thornber break;
156b29d4986SJoe Thornber }
157b29d4986SJoe Thornber }
158b29d4986SJoe Thornber
btracker_nr_writebacks_queued(struct background_tracker * b)159*86a3238cSHeinz Mauelshagen unsigned int btracker_nr_writebacks_queued(struct background_tracker *b)
160b29d4986SJoe Thornber {
161b29d4986SJoe Thornber return atomic_read(&b->pending_writebacks);
162b29d4986SJoe Thornber }
163b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_nr_writebacks_queued);
164b29d4986SJoe Thornber
btracker_nr_demotions_queued(struct background_tracker * b)165*86a3238cSHeinz Mauelshagen unsigned int btracker_nr_demotions_queued(struct background_tracker *b)
166b29d4986SJoe Thornber {
167b29d4986SJoe Thornber return atomic_read(&b->pending_demotes);
168b29d4986SJoe Thornber }
169b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_nr_demotions_queued);
170b29d4986SJoe Thornber
max_work_reached(struct background_tracker * b)171b29d4986SJoe Thornber static bool max_work_reached(struct background_tracker *b)
172b29d4986SJoe Thornber {
17364748b16SJoe Thornber return atomic_read(&b->pending_promotes) +
17464748b16SJoe Thornber atomic_read(&b->pending_writebacks) +
17564748b16SJoe Thornber atomic_read(&b->pending_demotes) >= b->max_work;
17664748b16SJoe Thornber }
17764748b16SJoe Thornber
alloc_work(struct background_tracker * b)178280884faSMike Snitzer static struct bt_work *alloc_work(struct background_tracker *b)
17964748b16SJoe Thornber {
18064748b16SJoe Thornber if (max_work_reached(b))
18164748b16SJoe Thornber return NULL;
18264748b16SJoe Thornber
18364748b16SJoe Thornber return kmem_cache_alloc(b->work_cache, GFP_NOWAIT);
184b29d4986SJoe Thornber }
185b29d4986SJoe Thornber
btracker_queue(struct background_tracker * b,struct policy_work * work,struct policy_work ** pwork)186b29d4986SJoe Thornber int btracker_queue(struct background_tracker *b,
187b29d4986SJoe Thornber struct policy_work *work,
188b29d4986SJoe Thornber struct policy_work **pwork)
189b29d4986SJoe Thornber {
190b29d4986SJoe Thornber struct bt_work *w;
191b29d4986SJoe Thornber
192b29d4986SJoe Thornber if (pwork)
193b29d4986SJoe Thornber *pwork = NULL;
194b29d4986SJoe Thornber
19564748b16SJoe Thornber w = alloc_work(b);
196b29d4986SJoe Thornber if (!w)
197b29d4986SJoe Thornber return -ENOMEM;
198b29d4986SJoe Thornber
199b29d4986SJoe Thornber memcpy(&w->work, work, sizeof(*work));
200b29d4986SJoe Thornber
201b29d4986SJoe Thornber if (!__insert_pending(b, w)) {
202b29d4986SJoe Thornber /*
203b29d4986SJoe Thornber * There was a race, we'll just ignore this second
204b29d4986SJoe Thornber * bit of work for the same oblock.
205b29d4986SJoe Thornber */
206b29d4986SJoe Thornber kmem_cache_free(b->work_cache, w);
207b29d4986SJoe Thornber return -EINVAL;
208b29d4986SJoe Thornber }
209b29d4986SJoe Thornber
210b29d4986SJoe Thornber if (pwork) {
211b29d4986SJoe Thornber *pwork = &w->work;
212b29d4986SJoe Thornber list_add(&w->list, &b->issued);
213b29d4986SJoe Thornber } else
214b29d4986SJoe Thornber list_add(&w->list, &b->queued);
215b29d4986SJoe Thornber update_stats(b, &w->work, 1);
216b29d4986SJoe Thornber
217b29d4986SJoe Thornber return 0;
218b29d4986SJoe Thornber }
219b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_queue);
220b29d4986SJoe Thornber
221b29d4986SJoe Thornber /*
222b29d4986SJoe Thornber * Returns -ENODATA if there's no work.
223b29d4986SJoe Thornber */
btracker_issue(struct background_tracker * b,struct policy_work ** work)224b29d4986SJoe Thornber int btracker_issue(struct background_tracker *b, struct policy_work **work)
225b29d4986SJoe Thornber {
226b29d4986SJoe Thornber struct bt_work *w;
227b29d4986SJoe Thornber
228b29d4986SJoe Thornber if (list_empty(&b->queued))
229b29d4986SJoe Thornber return -ENODATA;
230b29d4986SJoe Thornber
231b29d4986SJoe Thornber w = list_first_entry(&b->queued, struct bt_work, list);
232b29d4986SJoe Thornber list_move(&w->list, &b->issued);
233b29d4986SJoe Thornber *work = &w->work;
234b29d4986SJoe Thornber
235b29d4986SJoe Thornber return 0;
236b29d4986SJoe Thornber }
237b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_issue);
238b29d4986SJoe Thornber
btracker_complete(struct background_tracker * b,struct policy_work * op)239b29d4986SJoe Thornber void btracker_complete(struct background_tracker *b,
240b29d4986SJoe Thornber struct policy_work *op)
241b29d4986SJoe Thornber {
242b29d4986SJoe Thornber struct bt_work *w = container_of(op, struct bt_work, work);
243b29d4986SJoe Thornber
244b29d4986SJoe Thornber update_stats(b, &w->work, -1);
245b29d4986SJoe Thornber rb_erase(&w->node, &b->pending);
246b29d4986SJoe Thornber list_del(&w->list);
247b29d4986SJoe Thornber kmem_cache_free(b->work_cache, w);
248b29d4986SJoe Thornber }
249b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_complete);
250b29d4986SJoe Thornber
btracker_promotion_already_present(struct background_tracker * b,dm_oblock_t oblock)251b29d4986SJoe Thornber bool btracker_promotion_already_present(struct background_tracker *b,
252b29d4986SJoe Thornber dm_oblock_t oblock)
253b29d4986SJoe Thornber {
254b29d4986SJoe Thornber return __find_pending(b, oblock) != NULL;
255b29d4986SJoe Thornber }
256b29d4986SJoe Thornber EXPORT_SYMBOL_GPL(btracker_promotion_already_present);
257b29d4986SJoe Thornber
258b29d4986SJoe Thornber /*----------------------------------------------------------------*/
259