4 #include "cache.h"
11 static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache)  in get_cache_info_addr()  argument
13 	return cache->cache_info_addr + cache->info_index;  in get_cache_info_addr()
16 static void cache_info_write(struct pcache_cache *cache)  in cache_info_write()  argument
18 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_info_write()
24 	memcpy_flushcache(get_cache_info_addr(cache), cache_info,  in cache_info_write()
27 	cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX;  in cache_info_write()
30 static void cache_info_init_default(struct pcache_cache *cache);
31 static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts)  in cache_info_init()  argument
33 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_info_init()
36 	cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header,  in cache_info_init()
39 						&cache->cache_info);  in cache_info_init()
45 				(cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) {  in cache_info_init()
48 					cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false");  in cache_info_init()
55 	/* init cache_info for new cache */  in cache_info_init()
56 	cache_info_init_default(cache);  in cache_info_init()
57 	cache_mode_set(cache, opts->cache_mode);  in cache_info_init()
59 		cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC;  in cache_info_init()
70 int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent)  in pcache_cache_set_gc_percent()  argument
75 	mutex_lock(&cache->cache_info_lock);  in pcache_cache_set_gc_percent()
76 	cache_info_set_gc_percent(&cache->cache_info, percent);  in pcache_cache_set_gc_percent()
78 	cache_info_write(cache);  in pcache_cache_set_gc_percent()
79 	mutex_unlock(&cache->cache_info_lock);  in pcache_cache_set_gc_percent()
84 void cache_pos_encode(struct pcache_cache *cache,  in cache_pos_encode()  argument
102 int cache_pos_decode(struct pcache_cache *cache,  in cache_pos_decode()  argument
118 	pos->cache_seg = &cache->segments[latest.cache_seg_id];  in cache_pos_decode()
126 static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id)  in cache_info_set_seg_id()  argument
128 	cache->cache_info.seg_id = seg_id;  in cache_info_set_seg_id()
133 	struct pcache_cache *cache = &pcache->cache;  in cache_init()  local
138 	cache->segments = kvcalloc(cache_dev->seg_num, sizeof(struct pcache_cache_segment), GFP_KERNEL);  in cache_init()
139 	if (!cache->segments) {  in cache_init()
144 	cache->seg_map = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);  in cache_init()
145 	if (!cache->seg_map) {  in cache_init()
150 	cache->backing_dev = backing_dev;  in cache_init()
151 	cache->cache_dev = &pcache->cache_dev;  in cache_init()
152 	cache->n_segs = cache_dev->seg_num;  in cache_init()
153 	atomic_set(&cache->gc_errors, 0);  in cache_init()
154 	spin_lock_init(&cache->seg_map_lock);  in cache_init()
155 	spin_lock_init(&cache->key_head_lock);  in cache_init()
157 	mutex_init(&cache->cache_info_lock);  in cache_init()
158 	mutex_init(&cache->key_tail_lock);  in cache_init()
159 	mutex_init(&cache->dirty_tail_lock);  in cache_init()
160 	mutex_init(&cache->writeback_lock);  in cache_init()
162 	INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn);  in cache_init()
163 	INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn);  in cache_init()
164 	INIT_WORK(&cache->clean_work, clean_fn);  in cache_init()
169 	kvfree(cache->segments);  in cache_init()
174 static void cache_exit(struct pcache_cache *cache)  in cache_exit()  argument
176 	kvfree(cache->seg_map);  in cache_exit()
177 	kvfree(cache->segments);  in cache_exit()
180 static void cache_info_init_default(struct pcache_cache *cache)  in cache_info_init_default()  argument
182 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_info_init_default()
185 	cache_info->n_segs = cache->cache_dev->seg_num;  in cache_info_init_default()
189 static int cache_tail_init(struct pcache_cache *cache)  in cache_tail_init()  argument
191 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_tail_init()
192 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);  in cache_tail_init()
195 		__set_bit(0, cache->seg_map);  in cache_tail_init()
197 		cache->key_head.cache_seg = &cache->segments[0];  in cache_tail_init()
198 		cache->key_head.seg_off = 0;  in cache_tail_init()
199 		cache_pos_copy(&cache->key_tail, &cache->key_head);  in cache_tail_init()
200 		cache_pos_copy(&cache->dirty_tail, &cache->key_head);  in cache_tail_init()
202 		cache_encode_dirty_tail(cache);  in cache_tail_init()
203 		cache_encode_key_tail(cache);  in cache_tail_init()
205 		if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) {  in cache_tail_init()
214 static int get_seg_id(struct pcache_cache *cache,  in get_seg_id()  argument
218 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in get_seg_id()
219 	struct pcache_cache_dev *cache_dev = cache->cache_dev;  in get_seg_id()
232 			cache_info_set_seg_id(cache, *seg_id);  in get_seg_id()
244 			*seg_id = cache->cache_info.seg_id;  in get_seg_id()
252 static int cache_segs_init(struct pcache_cache *cache)  in cache_segs_init()  argument
255 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_segs_init()
256 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);  in cache_segs_init()
262 		ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);  in cache_segs_init()
266 		ret = cache_seg_init(cache, seg_id, i, new_cache);  in cache_segs_init()
270 		prev_cache_seg = &cache->segments[i];  in cache_segs_init()
277 static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral)  in cache_init_req_keys()  argument
279 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_init_req_keys()
284 	/* Calculate number of cache trees based on the device size */  in cache_init_req_keys()
285 	n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE);  in cache_init_req_keys()
286 	ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees);  in cache_init_req_keys()
290 	cache->n_ksets = n_paral;  in cache_init_req_keys()
291 	cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL);  in cache_init_req_keys()
292 	if (!cache->ksets) {  in cache_init_req_keys()
300 	 * of cache keys across multiple queues, maximizing multiqueue concurrency.  in cache_init_req_keys()
302 	for (i = 0; i < cache->n_ksets; i++) {  in cache_init_req_keys()
303 		struct pcache_cache_kset *kset = get_kset(cache, i);  in cache_init_req_keys()
305 		kset->cache = cache;  in cache_init_req_keys()
310 	cache->data_heads = alloc_percpu(struct pcache_cache_data_head);  in cache_init_req_keys()
311 	if (!cache->data_heads) {  in cache_init_req_keys()
318 			per_cpu_ptr(cache->data_heads, cpu);  in cache_init_req_keys()
323 	 * Replay persisted cache keys using cache_replay.  in cache_init_req_keys()
324 	 * This function loads and replays cache keys from previously stored  in cache_init_req_keys()
325 	 * ksets, allowing the cache to restore its state after a restart.  in cache_init_req_keys()
327 	ret = cache_replay(cache);  in cache_init_req_keys()
336 	free_percpu(cache->data_heads);  in cache_init_req_keys()
338 	kvfree(cache->ksets);  in cache_init_req_keys()
340 	cache_tree_exit(&cache->req_key_tree);  in cache_init_req_keys()
345 static void cache_destroy_req_keys(struct pcache_cache *cache)  in cache_destroy_req_keys()  argument
349 	for (i = 0; i < cache->n_ksets; i++) {  in cache_destroy_req_keys()
350 		struct pcache_cache_kset *kset = get_kset(cache, i);  in cache_destroy_req_keys()
355 	free_percpu(cache->data_heads);  in cache_destroy_req_keys()
356 	kvfree(cache->ksets);  in cache_destroy_req_keys()
357 	cache_tree_exit(&cache->req_key_tree);  in cache_destroy_req_keys()
363 	struct pcache_cache *cache = &pcache->cache;  in pcache_cache_start()  local
371 	cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev);  in pcache_cache_start()
372 	cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev);  in pcache_cache_start()
373 	backing_dev->cache = cache;  in pcache_cache_start()
374 	cache->dev_size = backing_dev->dev_size;  in pcache_cache_start()
376 	ret = cache_info_init(cache, opts);  in pcache_cache_start()
380 	ret = cache_segs_init(cache);  in pcache_cache_start()
384 	ret = cache_tail_init(cache);  in pcache_cache_start()
388 	ret = cache_init_req_keys(cache, num_online_cpus());  in pcache_cache_start()
392 	ret = cache_writeback_init(cache);  in pcache_cache_start()
396 	cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE;  in pcache_cache_start()
397 	cache_info_write(cache);  in pcache_cache_start()
398 	queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0);  in pcache_cache_start()
403 	cache_destroy_req_keys(cache);  in pcache_cache_start()
405 	cache_exit(cache);  in pcache_cache_start()
412 	struct pcache_cache *cache = &pcache->cache;  in pcache_cache_stop()  local
414 	pcache_cache_flush(cache);  in pcache_cache_stop()
416 	cancel_delayed_work_sync(&cache->gc_work);  in pcache_cache_stop()
417 	flush_work(&cache->clean_work);  in pcache_cache_stop()
418 	cache_writeback_exit(cache);  in pcache_cache_stop()
420 	if (cache->req_key_tree.n_subtrees)  in pcache_cache_stop()
421 		cache_destroy_req_keys(cache);  in pcache_cache_stop()
423 	cache_exit(cache);  in pcache_cache_stop()
426 struct workqueue_struct *cache_get_wq(struct pcache_cache *cache)  in cache_get_wq()  argument
428 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_get_wq()