xref: /linux/drivers/md/dm-pcache/cache.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/blk_types.h>
3 
4 #include "cache.h"
5 #include "cache_dev.h"
6 #include "backing_dev.h"
7 #include "dm_pcache.h"
8 
9 struct kmem_cache *key_cache;
10 
11 static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache)
12 {
13 	return (struct pcache_cache_info *)((char *)cache->cache_info_addr +
14 						(size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE);
15 }
16 
17 static void cache_info_write(struct pcache_cache *cache)
18 {
19 	struct pcache_cache_info *cache_info = &cache->cache_info;
20 
21 	cache_info->header.seq++;
22 	cache_info->header.crc = pcache_meta_crc(&cache_info->header,
23 						sizeof(struct pcache_cache_info));
24 
25 	cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX;
26 	memcpy_flushcache(get_cache_info_addr(cache), cache_info,
27 			sizeof(struct pcache_cache_info));
28 	pmem_wmb();
29 }
30 
31 static void cache_info_init_default(struct pcache_cache *cache);
32 static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts)
33 {
34 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
35 	struct pcache_cache_info *cache_info_addr;
36 
37 	cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header,
38 						sizeof(struct pcache_cache_info),
39 						PCACHE_CACHE_INFO_SIZE,
40 						&cache->cache_info);
41 	if (IS_ERR(cache_info_addr))
42 		return PTR_ERR(cache_info_addr);
43 
44 	if (cache_info_addr) {
45 		if (opts->data_crc !=
46 				(cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) {
47 			pcache_dev_err(pcache, "invalid option for data_crc: %s, expected: %s",
48 					opts->data_crc ? "true" : "false",
49 					cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false");
50 			return -EINVAL;
51 		}
52 
53 		cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE;
54 
55 		return 0;
56 	}
57 
58 	/* init cache_info for new cache */
59 	cache_info_init_default(cache);
60 	cache_mode_set(cache, opts->cache_mode);
61 	if (opts->data_crc)
62 		cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC;
63 
64 	return 0;
65 }
66 
67 static void cache_info_set_gc_percent(struct pcache_cache_info *cache_info, u8 percent)
68 {
69 	cache_info->flags &= ~PCACHE_CACHE_FLAGS_GC_PERCENT_MASK;
70 	cache_info->flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_GC_PERCENT_MASK, percent);
71 }
72 
73 int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent)
74 {
75 	if (percent > PCACHE_CACHE_GC_PERCENT_MAX || percent < PCACHE_CACHE_GC_PERCENT_MIN)
76 		return -EINVAL;
77 
78 	mutex_lock(&cache->cache_info_lock);
79 	cache_info_set_gc_percent(&cache->cache_info, percent);
80 
81 	cache_info_write(cache);
82 	mutex_unlock(&cache->cache_info_lock);
83 
84 	return 0;
85 }
86 
87 void cache_pos_encode(struct pcache_cache *cache,
88 			     struct pcache_cache_pos_onmedia *pos_onmedia_base,
89 			     struct pcache_cache_pos *pos, u64 seq, u32 *index)
90 {
91 	struct pcache_cache_pos_onmedia pos_onmedia;
92 	struct pcache_cache_pos_onmedia *pos_onmedia_addr = pos_onmedia_base + *index;
93 
94 	pos_onmedia.cache_seg_id = pos->cache_seg->cache_seg_id;
95 	pos_onmedia.seg_off = pos->seg_off;
96 	pos_onmedia.header.seq = seq;
97 	pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia);
98 
99 	*index = (*index + 1) % PCACHE_META_INDEX_MAX;
100 
101 	memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia));
102 	pmem_wmb();
103 }
104 
105 int cache_pos_decode(struct pcache_cache *cache,
106 			    struct pcache_cache_pos_onmedia *pos_onmedia,
107 			    struct pcache_cache_pos *pos, u64 *seq, u32 *index)
108 {
109 	struct pcache_cache_pos_onmedia latest, *latest_addr;
110 
111 	latest_addr = pcache_meta_find_latest(&pos_onmedia->header,
112 					sizeof(struct pcache_cache_pos_onmedia),
113 					sizeof(struct pcache_cache_pos_onmedia),
114 					&latest);
115 	if (IS_ERR(latest_addr))
116 		return PTR_ERR(latest_addr);
117 
118 	if (!latest_addr)
119 		return -EIO;
120 
121 	if (!cache_seg_id_valid(cache, latest.cache_seg_id))
122 		return -EIO;
123 
124 	pos->cache_seg = &cache->segments[latest.cache_seg_id];
125 
126 	if (latest.seg_off >= pos->cache_seg->segment.data_size)
127 		return -EIO;
128 
129 	pos->seg_off = latest.seg_off;
130 	*seq = latest.header.seq;
131 	*index = (latest_addr - pos_onmedia);
132 
133 	return 0;
134 }
135 
136 static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id)
137 {
138 	cache->cache_info.seg_id = seg_id;
139 }
140 
141 static int cache_init(struct dm_pcache *pcache)
142 {
143 	struct pcache_cache *cache = &pcache->cache;
144 	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
145 	struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
146 	int ret;
147 
148 	cache->segments = kvzalloc_objs(struct pcache_cache_segment,
149 					cache_dev->seg_num);
150 	if (!cache->segments) {
151 		ret = -ENOMEM;
152 		goto err;
153 	}
154 
155 	cache->seg_map = kvzalloc_objs(unsigned long,
156 				       BITS_TO_LONGS(cache_dev->seg_num));
157 	if (!cache->seg_map) {
158 		ret = -ENOMEM;
159 		goto free_segments;
160 	}
161 
162 	cache->backing_dev = backing_dev;
163 	cache->cache_dev = &pcache->cache_dev;
164 	cache->n_segs = cache_dev->seg_num;
165 	atomic_set(&cache->gc_errors, 0);
166 	atomic_set(&cache->writeback_errors, 0);
167 	spin_lock_init(&cache->seg_map_lock);
168 	spin_lock_init(&cache->key_head_lock);
169 
170 	mutex_init(&cache->cache_info_lock);
171 	mutex_init(&cache->key_tail_lock);
172 	mutex_init(&cache->dirty_tail_lock);
173 	mutex_init(&cache->writeback_lock);
174 
175 	INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn);
176 	INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn);
177 	INIT_WORK(&cache->clean_work, clean_fn);
178 
179 	return 0;
180 
181 free_segments:
182 	kvfree(cache->segments);
183 err:
184 	return ret;
185 }
186 
187 static void cache_exit(struct pcache_cache *cache)
188 {
189 	kvfree(cache->seg_map);
190 	kvfree(cache->segments);
191 }
192 
193 static void cache_info_init_default(struct pcache_cache *cache)
194 {
195 	struct pcache_cache_info *cache_info = &cache->cache_info;
196 
197 	memset(cache_info, 0, sizeof(*cache_info));
198 	cache_info->n_segs = cache->cache_dev->seg_num;
199 	cache_info_set_gc_percent(cache_info, PCACHE_CACHE_GC_PERCENT_DEFAULT);
200 }
201 
202 static int cache_tail_init(struct pcache_cache *cache)
203 {
204 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
205 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
206 	int ret;
207 
208 	if (new_cache) {
209 		__set_bit(0, cache->seg_map);
210 
211 		cache->key_head.cache_seg = &cache->segments[0];
212 		cache->key_head.seg_off = 0;
213 		cache_pos_copy(&cache->key_tail, &cache->key_head);
214 		cache_pos_copy(&cache->dirty_tail, &cache->key_head);
215 
216 		cache_encode_dirty_tail(cache);
217 		cache_encode_key_tail(cache);
218 	} else {
219 		if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) {
220 			pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n");
221 			return -EIO;
222 		}
223 
224 		ret = cache_verify_dirty_tail(cache);
225 		if (ret) {
226 			pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n");
227 			return ret;
228 		}
229 	}
230 
231 	return 0;
232 }
233 
234 static int get_seg_id(struct pcache_cache *cache,
235 		      struct pcache_cache_segment *prev_cache_seg,
236 		      bool new_cache, u32 *seg_id)
237 {
238 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
239 	struct pcache_cache_dev *cache_dev = cache->cache_dev;
240 	int ret;
241 
242 	if (new_cache) {
243 		ret = cache_dev_get_empty_segment_id(cache_dev, seg_id);
244 		if (ret) {
245 			pcache_dev_err(pcache, "no available segment\n");
246 			goto err;
247 		}
248 
249 		if (prev_cache_seg)
250 			cache_seg_set_next_seg(prev_cache_seg, *seg_id);
251 		else
252 			cache_info_set_seg_id(cache, *seg_id);
253 	} else {
254 		if (prev_cache_seg) {
255 			struct pcache_segment_info *prev_seg_info;
256 
257 			prev_seg_info = &prev_cache_seg->cache_seg_info;
258 			if (!segment_info_has_next(prev_seg_info)) {
259 				ret = -EFAULT;
260 				goto err;
261 			}
262 			*seg_id = prev_cache_seg->cache_seg_info.next_seg;
263 		} else {
264 			*seg_id = cache->cache_info.seg_id;
265 		}
266 
267 		if (*seg_id >= cache_dev->seg_num) {
268 			pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n",
269 				       *seg_id, cache_dev->seg_num);
270 			ret = -EIO;
271 			goto err;
272 		}
273 	}
274 	return 0;
275 err:
276 	return ret;
277 }
278 
279 static int cache_segs_init(struct pcache_cache *cache)
280 {
281 	struct pcache_cache_segment *prev_cache_seg = NULL;
282 	struct pcache_cache_info *cache_info = &cache->cache_info;
283 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
284 	u32 seg_id;
285 	int ret;
286 	u32 i;
287 
288 	if (cache_info->n_segs > cache->cache_dev->seg_num) {
289 		pcache_dev_err(CACHE_TO_PCACHE(cache),
290 			       "cache_info n_segs %u exceeds cache device segments %u\n",
291 			       cache_info->n_segs, cache->cache_dev->seg_num);
292 		return -EIO;
293 	}
294 
295 	for (i = 0; i < cache_info->n_segs; i++) {
296 		ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);
297 		if (ret)
298 			goto err;
299 
300 		ret = cache_seg_init(cache, seg_id, i, new_cache);
301 		if (ret)
302 			goto err;
303 
304 		prev_cache_seg = &cache->segments[i];
305 	}
306 	return 0;
307 err:
308 	return ret;
309 }
310 
311 static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral)
312 {
313 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
314 	u32 n_subtrees;
315 	int ret;
316 	u32 i, cpu;
317 
318 	/* Calculate number of cache trees based on the device size */
319 	n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE);
320 	ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees);
321 	if (ret)
322 		goto err;
323 
324 	cache->n_ksets = n_paral;
325 	cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL);
326 	if (!cache->ksets) {
327 		ret = -ENOMEM;
328 		goto req_tree_exit;
329 	}
330 
331 	/*
332 	 * Initialize each kset with a spinlock and delayed work for flushing.
333 	 * Each kset is associated with one queue to ensure independent handling
334 	 * of cache keys across multiple queues, maximizing multiqueue concurrency.
335 	 */
336 	for (i = 0; i < cache->n_ksets; i++) {
337 		struct pcache_cache_kset *kset = get_kset(cache, i);
338 
339 		kset->cache = cache;
340 		spin_lock_init(&kset->kset_lock);
341 		INIT_DELAYED_WORK(&kset->flush_work, kset_flush_fn);
342 	}
343 
344 	cache->data_heads = alloc_percpu(struct pcache_cache_data_head);
345 	if (!cache->data_heads) {
346 		ret = -ENOMEM;
347 		goto free_kset;
348 	}
349 
350 	for_each_possible_cpu(cpu) {
351 		struct pcache_cache_data_head *h =
352 			per_cpu_ptr(cache->data_heads, cpu);
353 		h->head_pos.cache_seg = NULL;
354 	}
355 
356 	/*
357 	 * Replay persisted cache keys using cache_replay.
358 	 * This function loads and replays cache keys from previously stored
359 	 * ksets, allowing the cache to restore its state after a restart.
360 	 */
361 	ret = cache_replay(cache);
362 	if (ret) {
363 		pcache_dev_err(pcache, "failed to replay keys\n");
364 		goto free_heads;
365 	}
366 
367 	return 0;
368 
369 free_heads:
370 	free_percpu(cache->data_heads);
371 free_kset:
372 	kvfree(cache->ksets);
373 req_tree_exit:
374 	cache_tree_exit(&cache->req_key_tree);
375 err:
376 	return ret;
377 }
378 
379 static void cache_destroy_req_keys(struct pcache_cache *cache)
380 {
381 	u32 i;
382 
383 	for (i = 0; i < cache->n_ksets; i++) {
384 		struct pcache_cache_kset *kset = get_kset(cache, i);
385 
386 		cancel_delayed_work_sync(&kset->flush_work);
387 	}
388 
389 	free_percpu(cache->data_heads);
390 	kvfree(cache->ksets);
391 	cache_tree_exit(&cache->req_key_tree);
392 }
393 
394 int pcache_cache_start(struct dm_pcache *pcache)
395 {
396 	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
397 	struct pcache_cache *cache = &pcache->cache;
398 	struct pcache_cache_options *opts = &pcache->opts;
399 	int ret;
400 
401 	ret = cache_init(pcache);
402 	if (ret)
403 		return ret;
404 
405 	cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev);
406 	cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev);
407 	backing_dev->cache = cache;
408 	cache->dev_size = backing_dev->dev_size;
409 
410 	ret = cache_info_init(cache, opts);
411 	if (ret)
412 		goto cache_exit;
413 
414 	ret = cache_segs_init(cache);
415 	if (ret)
416 		goto cache_exit;
417 
418 	ret = cache_tail_init(cache);
419 	if (ret)
420 		goto cache_exit;
421 
422 	ret = cache_init_req_keys(cache, num_online_cpus());
423 	if (ret)
424 		goto cache_exit;
425 
426 	ret = cache_writeback_init(cache);
427 	if (ret)
428 		goto destroy_keys;
429 
430 	cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE;
431 	cache_info_write(cache);
432 	queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0);
433 
434 	return 0;
435 
436 destroy_keys:
437 	cache_destroy_req_keys(cache);
438 cache_exit:
439 	cache_exit(cache);
440 
441 	return ret;
442 }
443 
444 void pcache_cache_stop(struct dm_pcache *pcache)
445 {
446 	struct pcache_cache *cache = &pcache->cache;
447 
448 	pcache_cache_flush(cache);
449 
450 	cancel_delayed_work_sync(&cache->gc_work);
451 	flush_work(&cache->clean_work);
452 	cache_writeback_exit(cache);
453 
454 	if (cache->req_key_tree.n_subtrees)
455 		cache_destroy_req_keys(cache);
456 
457 	cache_exit(cache);
458 }
459 
460 struct workqueue_struct *cache_get_wq(struct pcache_cache *cache)
461 {
462 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
463 
464 	return pcache->task_wq;
465 }
466 
467 int pcache_cache_init(void)
468 {
469 	key_cache = KMEM_CACHE(pcache_cache_key, 0);
470 	if (!key_cache)
471 		return -ENOMEM;
472 
473 	return 0;
474 }
475 
476 void pcache_cache_exit(void)
477 {
478 	kmem_cache_destroy(key_cache);
479 }
480