1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/bio.h> 4 5 #include "cache.h" 6 #include "backing_dev.h" 7 #include "cache_dev.h" 8 #include "dm_pcache.h" 9 10 static void writeback_ctx_end(struct pcache_cache *cache, int ret) 11 { 12 if (ret && !cache->writeback_ctx.ret) { 13 pcache_dev_err(CACHE_TO_PCACHE(cache), "writeback error: %d", ret); 14 cache->writeback_ctx.ret = ret; 15 } 16 17 if (!atomic_dec_and_test(&cache->writeback_ctx.pending)) 18 return; 19 20 if (!cache->writeback_ctx.ret) { 21 backing_dev_flush(cache->backing_dev); 22 23 mutex_lock(&cache->dirty_tail_lock); 24 cache_pos_advance(&cache->dirty_tail, cache->writeback_ctx.advance); 25 cache_encode_dirty_tail(cache); 26 mutex_unlock(&cache->dirty_tail_lock); 27 } 28 queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0); 29 } 30 31 static void writeback_end_req(struct pcache_backing_dev_req *backing_req, int ret) 32 { 33 struct pcache_cache *cache = backing_req->priv_data; 34 35 mutex_lock(&cache->writeback_lock); 36 writeback_ctx_end(cache, ret); 37 mutex_unlock(&cache->writeback_lock); 38 } 39 40 static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cache_pos *dirty_tail) 41 { 42 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 43 struct pcache_cache_kset_onmedia *kset_onmedia; 44 u32 to_copy; 45 void *addr; 46 int ret; 47 48 addr = cache_pos_addr(dirty_tail); 49 kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; 50 51 to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail)); 52 ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy); 53 if (ret) { 54 pcache_dev_err(pcache, "error to read kset: %d", ret); 55 return true; 56 } 57 58 /* Reject a corrupted or out-of-bounds kset before reading its keys */ 59 if (!kset_onmedia_valid(kset_onmedia)) { 60 pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n", 61 dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, 62 kset_onmedia->magic, kset_onmedia->key_num); 63 return true; 64 } 65 66 /* Verify the CRC checksum for data integrity */ 67 if (kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { 68 pcache_dev_debug(pcache, "dirty_tail: %u:%u crc: %x, not expected: %x\n", 69 dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, 70 cache_kset_crc(kset_onmedia), kset_onmedia->crc); 71 return true; 72 } 73 74 return false; 75 } 76 77 void cache_writeback_exit(struct pcache_cache *cache) 78 { 79 cancel_delayed_work_sync(&cache->writeback_work); 80 backing_dev_flush(cache->backing_dev); 81 cache_tree_exit(&cache->writeback_key_tree); 82 } 83 84 int cache_writeback_init(struct pcache_cache *cache) 85 { 86 int ret; 87 88 ret = cache_tree_init(cache, &cache->writeback_key_tree, 1); 89 if (ret) 90 goto err; 91 92 atomic_set(&cache->writeback_ctx.pending, 0); 93 94 /* Queue delayed work to start writeback handling */ 95 queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0); 96 97 return 0; 98 err: 99 return ret; 100 } 101 102 static void cache_key_writeback(struct pcache_cache *cache, struct pcache_cache_key *key) 103 { 104 struct pcache_backing_dev_req *writeback_req; 105 struct pcache_backing_dev_req_opts writeback_req_opts = { 0 }; 106 struct pcache_cache_pos *pos; 107 void *addr; 108 u32 seg_remain, req_len, done = 0; 109 110 if (cache_key_clean(key)) 111 return; 112 113 pos = &key->cache_pos; 114 115 seg_remain = cache_seg_remain(pos); 116 BUG_ON(seg_remain < key->len); 117 next_req: 118 addr = cache_pos_addr(pos) + done; 119 req_len = backing_dev_req_coalesced_max_len(addr, key->len - done); 120 121 writeback_req_opts.type = BACKING_DEV_REQ_TYPE_KMEM; 122 writeback_req_opts.gfp_mask = GFP_NOIO; 123 writeback_req_opts.end_fn = writeback_end_req; 124 writeback_req_opts.priv_data = cache; 125 126 writeback_req_opts.kmem.data = addr; 127 writeback_req_opts.kmem.opf = REQ_OP_WRITE; 128 writeback_req_opts.kmem.len = req_len; 129 writeback_req_opts.kmem.backing_off = key->off + done; 130 131 writeback_req = backing_dev_req_create(cache->backing_dev, &writeback_req_opts); 132 133 atomic_inc(&cache->writeback_ctx.pending); 134 backing_dev_req_submit(writeback_req, true); 135 136 done += req_len; 137 if (done < key->len) 138 goto next_req; 139 } 140 141 static void cache_wb_tree_writeback(struct pcache_cache *cache, u32 advance) 142 { 143 struct pcache_cache_tree *cache_tree = &cache->writeback_key_tree; 144 struct pcache_cache_subtree *cache_subtree; 145 struct rb_node *node; 146 struct pcache_cache_key *key; 147 u32 i; 148 149 cache->writeback_ctx.ret = 0; 150 cache->writeback_ctx.advance = advance; 151 atomic_set(&cache->writeback_ctx.pending, 1); 152 153 for (i = 0; i < cache_tree->n_subtrees; i++) { 154 cache_subtree = &cache_tree->subtrees[i]; 155 156 node = rb_first(&cache_subtree->root); 157 while (node) { 158 key = CACHE_KEY(node); 159 node = rb_next(node); 160 161 cache_key_writeback(cache, key); 162 cache_key_delete(key); 163 } 164 } 165 writeback_ctx_end(cache, 0); 166 } 167 168 static int cache_kset_insert_tree(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) 169 { 170 struct pcache_cache_key_onmedia *key_onmedia; 171 struct pcache_cache_subtree *cache_subtree; 172 struct pcache_cache_key *key; 173 int ret; 174 u32 i; 175 176 /* Iterate through all keys in the kset and write each back to storage */ 177 for (i = 0; i < kset_onmedia->key_num; i++) { 178 key_onmedia = &kset_onmedia->data[i]; 179 180 key = cache_key_alloc(&cache->writeback_key_tree, GFP_NOIO); 181 ret = cache_key_decode(cache, key_onmedia, key); 182 if (ret) { 183 cache_key_put(key); 184 goto clear_tree; 185 } 186 187 cache_subtree = get_subtree(&cache->writeback_key_tree, key->off); 188 spin_lock(&cache_subtree->tree_lock); 189 cache_key_insert(&cache->writeback_key_tree, key, true); 190 spin_unlock(&cache_subtree->tree_lock); 191 } 192 193 return 0; 194 clear_tree: 195 cache_tree_clear(&cache->writeback_key_tree); 196 return ret; 197 } 198 199 static int last_kset_writeback(struct pcache_cache *cache, 200 struct pcache_cache_kset_onmedia *last_kset_onmedia) 201 { 202 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 203 struct pcache_cache_segment *next_seg; 204 205 if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) { 206 pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n", 207 last_kset_onmedia->next_cache_seg_id, cache->n_segs); 208 return -EIO; 209 } 210 211 pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id); 212 213 next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id]; 214 215 mutex_lock(&cache->dirty_tail_lock); 216 cache->dirty_tail.cache_seg = next_seg; 217 cache->dirty_tail.seg_off = 0; 218 cache_encode_dirty_tail(cache); 219 mutex_unlock(&cache->dirty_tail_lock); 220 221 return 0; 222 } 223 224 void cache_writeback_fn(struct work_struct *work) 225 { 226 struct pcache_cache *cache = container_of(work, struct pcache_cache, writeback_work.work); 227 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 228 struct pcache_cache_pos dirty_tail; 229 struct pcache_cache_kset_onmedia *kset_onmedia; 230 u32 delay; 231 int ret; 232 233 mutex_lock(&cache->writeback_lock); 234 if (atomic_read(&cache->writeback_ctx.pending)) 235 goto unlock; 236 237 if (pcache_is_stopping(pcache)) 238 goto unlock; 239 240 if (atomic_read(&cache->writeback_errors)) 241 goto unlock; 242 243 kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; 244 245 mutex_lock(&cache->dirty_tail_lock); 246 cache_pos_copy(&dirty_tail, &cache->dirty_tail); 247 mutex_unlock(&cache->dirty_tail_lock); 248 249 if (is_cache_clean(cache, &dirty_tail)) { 250 delay = PCACHE_CACHE_WRITEBACK_INTERVAL; 251 goto queue_work; 252 } 253 254 if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { 255 ret = last_kset_writeback(cache, kset_onmedia); 256 if (ret) { 257 atomic_inc(&cache->writeback_errors); 258 goto unlock; 259 } 260 delay = 0; 261 goto queue_work; 262 } 263 264 if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) { 265 atomic_inc(&cache->writeback_errors); 266 goto unlock; 267 } 268 269 ret = cache_kset_insert_tree(cache, kset_onmedia); 270 if (ret) { 271 atomic_inc(&cache->writeback_errors); 272 goto unlock; 273 } 274 275 cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia)); 276 delay = 0; 277 queue_work: 278 queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, delay); 279 unlock: 280 mutex_unlock(&cache->writeback_lock); 281 } 282