11 static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache)  in get_cache_info_addr()  argument
13 	return (struct pcache_cache_info *)((char *)cache->cache_info_addr +  in get_cache_info_addr()
14 						(size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE);  in get_cache_info_addr()
17 static void cache_info_write(struct pcache_cache *cache)  in cache_info_write()  argument
19 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_info_write()
25 	cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX;  in cache_info_write()
26 	memcpy_flushcache(get_cache_info_addr(cache), cache_info,  in cache_info_write()
31 static void cache_info_init_default(struct pcache_cache *cache);
32 static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts)  in cache_info_init()  argument
34 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_info_init()
37 	cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header,  in cache_info_init()
40 						&cache->cache_info);  in cache_info_init()
46 				(cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) {  in cache_info_init()
49 					cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false");  in cache_info_init()
53 …cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO…  in cache_info_init()
59 	cache_info_init_default(cache);  in cache_info_init()
60 	cache_mode_set(cache, opts->cache_mode);  in cache_info_init()
62 		cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC;  in cache_info_init()
73 int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent)  in pcache_cache_set_gc_percent()  argument
78 	mutex_lock(&cache->cache_info_lock);  in pcache_cache_set_gc_percent()
79 	cache_info_set_gc_percent(&cache->cache_info, percent);  in pcache_cache_set_gc_percent()
81 	cache_info_write(cache);  in pcache_cache_set_gc_percent()
82 	mutex_unlock(&cache->cache_info_lock);  in pcache_cache_set_gc_percent()
87 void cache_pos_encode(struct pcache_cache *cache,  in cache_pos_encode()  argument
105 int cache_pos_decode(struct pcache_cache *cache,  in cache_pos_decode()  argument
121 	pos->cache_seg = &cache->segments[latest.cache_seg_id];  in cache_pos_decode()
129 static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id)  in cache_info_set_seg_id()  argument
131 	cache->cache_info.seg_id = seg_id;  in cache_info_set_seg_id()
136 	struct pcache_cache *cache = &pcache->cache;  in cache_init()  local
141 	cache->segments = kvcalloc(cache_dev->seg_num, sizeof(struct pcache_cache_segment), GFP_KERNEL);  in cache_init()
142 	if (!cache->segments) {  in cache_init()
147 	cache->seg_map = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);  in cache_init()
148 	if (!cache->seg_map) {  in cache_init()
153 	cache->backing_dev = backing_dev;  in cache_init()
154 	cache->cache_dev = &pcache->cache_dev;  in cache_init()
155 	cache->n_segs = cache_dev->seg_num;  in cache_init()
156 	atomic_set(&cache->gc_errors, 0);  in cache_init()
157 	spin_lock_init(&cache->seg_map_lock);  in cache_init()
158 	spin_lock_init(&cache->key_head_lock);  in cache_init()
160 	mutex_init(&cache->cache_info_lock);  in cache_init()
161 	mutex_init(&cache->key_tail_lock);  in cache_init()
162 	mutex_init(&cache->dirty_tail_lock);  in cache_init()
163 	mutex_init(&cache->writeback_lock);  in cache_init()
165 	INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn);  in cache_init()
166 	INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn);  in cache_init()
167 	INIT_WORK(&cache->clean_work, clean_fn);  in cache_init()
172 	kvfree(cache->segments);  in cache_init()
177 static void cache_exit(struct pcache_cache *cache)  in cache_exit()  argument
179 	kvfree(cache->seg_map);  in cache_exit()
180 	kvfree(cache->segments);  in cache_exit()
183 static void cache_info_init_default(struct pcache_cache *cache)  in cache_info_init_default()  argument
185 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_info_init_default()
188 	cache_info->n_segs = cache->cache_dev->seg_num;  in cache_info_init_default()
192 static int cache_tail_init(struct pcache_cache *cache)  in cache_tail_init()  argument
194 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_tail_init()
195 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);  in cache_tail_init()
198 		__set_bit(0, cache->seg_map);  in cache_tail_init()
200 		cache->key_head.cache_seg = &cache->segments[0];  in cache_tail_init()
201 		cache->key_head.seg_off = 0;  in cache_tail_init()
202 		cache_pos_copy(&cache->key_tail, &cache->key_head);  in cache_tail_init()
203 		cache_pos_copy(&cache->dirty_tail, &cache->key_head);  in cache_tail_init()
205 		cache_encode_dirty_tail(cache);  in cache_tail_init()
206 		cache_encode_key_tail(cache);  in cache_tail_init()
208 		if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) {  in cache_tail_init()
217 static int get_seg_id(struct pcache_cache *cache,  in get_seg_id()  argument
221 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in get_seg_id()
222 	struct pcache_cache_dev *cache_dev = cache->cache_dev;  in get_seg_id()
235 			cache_info_set_seg_id(cache, *seg_id);  in get_seg_id()
247 			*seg_id = cache->cache_info.seg_id;  in get_seg_id()
255 static int cache_segs_init(struct pcache_cache *cache)  in cache_segs_init()  argument
258 	struct pcache_cache_info *cache_info = &cache->cache_info;  in cache_segs_init()
259 	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);  in cache_segs_init()
265 		ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);  in cache_segs_init()
269 		ret = cache_seg_init(cache, seg_id, i, new_cache);  in cache_segs_init()
273 		prev_cache_seg = &cache->segments[i];  in cache_segs_init()
280 static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral)  in cache_init_req_keys()  argument
282 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_init_req_keys()
288 	n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE);  in cache_init_req_keys()
289 	ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees);  in cache_init_req_keys()
293 	cache->n_ksets = n_paral;  in cache_init_req_keys()
294 	cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL);  in cache_init_req_keys()
295 	if (!cache->ksets) {  in cache_init_req_keys()
305 	for (i = 0; i < cache->n_ksets; i++) {  in cache_init_req_keys()
306 		struct pcache_cache_kset *kset = get_kset(cache, i);  in cache_init_req_keys()
308 		kset->cache = cache;  in cache_init_req_keys()
313 	cache->data_heads = alloc_percpu(struct pcache_cache_data_head);  in cache_init_req_keys()
314 	if (!cache->data_heads) {  in cache_init_req_keys()
321 			per_cpu_ptr(cache->data_heads, cpu);  in cache_init_req_keys()
330 	ret = cache_replay(cache);  in cache_init_req_keys()
339 	free_percpu(cache->data_heads);  in cache_init_req_keys()
341 	kvfree(cache->ksets);  in cache_init_req_keys()
343 	cache_tree_exit(&cache->req_key_tree);  in cache_init_req_keys()
348 static void cache_destroy_req_keys(struct pcache_cache *cache)  in cache_destroy_req_keys()  argument
352 	for (i = 0; i < cache->n_ksets; i++) {  in cache_destroy_req_keys()
353 		struct pcache_cache_kset *kset = get_kset(cache, i);  in cache_destroy_req_keys()
358 	free_percpu(cache->data_heads);  in cache_destroy_req_keys()
359 	kvfree(cache->ksets);  in cache_destroy_req_keys()
360 	cache_tree_exit(&cache->req_key_tree);  in cache_destroy_req_keys()
366 	struct pcache_cache *cache = &pcache->cache;  in pcache_cache_start()  local
374 	cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev);  in pcache_cache_start()
375 	cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev);  in pcache_cache_start()
376 	backing_dev->cache = cache;  in pcache_cache_start()
377 	cache->dev_size = backing_dev->dev_size;  in pcache_cache_start()
379 	ret = cache_info_init(cache, opts);  in pcache_cache_start()
383 	ret = cache_segs_init(cache);  in pcache_cache_start()
387 	ret = cache_tail_init(cache);  in pcache_cache_start()
391 	ret = cache_init_req_keys(cache, num_online_cpus());  in pcache_cache_start()
395 	ret = cache_writeback_init(cache);  in pcache_cache_start()
399 	cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE;  in pcache_cache_start()
400 	cache_info_write(cache);  in pcache_cache_start()
401 	queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0);  in pcache_cache_start()
406 	cache_destroy_req_keys(cache);  in pcache_cache_start()
408 	cache_exit(cache);  in pcache_cache_start()
415 	struct pcache_cache *cache = &pcache->cache;  in pcache_cache_stop()  local
417 	pcache_cache_flush(cache);  in pcache_cache_stop()
419 	cancel_delayed_work_sync(&cache->gc_work);  in pcache_cache_stop()
420 	flush_work(&cache->clean_work);  in pcache_cache_stop()
421 	cache_writeback_exit(cache);  in pcache_cache_stop()
423 	if (cache->req_key_tree.n_subtrees)  in pcache_cache_stop()
424 		cache_destroy_req_keys(cache);  in pcache_cache_stop()
426 	cache_exit(cache);  in pcache_cache_stop()
429 struct workqueue_struct *cache_get_wq(struct pcache_cache *cache)  in cache_get_wq()  argument
431 	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);  in cache_get_wq()