1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/blk_types.h> 3 4 #include "cache.h" 5 #include "cache_dev.h" 6 #include "backing_dev.h" 7 #include "dm_pcache.h" 8 9 struct kmem_cache *key_cache; 10 11 static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache) 12 { 13 return (struct pcache_cache_info *)((char *)cache->cache_info_addr + 14 (size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE); 15 } 16 17 static void cache_info_write(struct pcache_cache *cache) 18 { 19 struct pcache_cache_info *cache_info = &cache->cache_info; 20 21 cache_info->header.seq++; 22 cache_info->header.crc = pcache_meta_crc(&cache_info->header, 23 sizeof(struct pcache_cache_info)); 24 25 cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX; 26 memcpy_flushcache(get_cache_info_addr(cache), cache_info, 27 sizeof(struct pcache_cache_info)); 28 pmem_wmb(); 29 } 30 31 static void cache_info_init_default(struct pcache_cache *cache); 32 static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts) 33 { 34 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 35 struct pcache_cache_info *cache_info_addr; 36 37 cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header, 38 sizeof(struct pcache_cache_info), 39 PCACHE_CACHE_INFO_SIZE, 40 &cache->cache_info); 41 if (IS_ERR(cache_info_addr)) 42 return PTR_ERR(cache_info_addr); 43 44 if (cache_info_addr) { 45 if (opts->data_crc != 46 (cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) { 47 pcache_dev_err(pcache, "invalid option for data_crc: %s, expected: %s", 48 opts->data_crc ? "true" : "false", 49 cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false"); 50 return -EINVAL; 51 } 52 53 cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE; 54 55 return 0; 56 } 57 58 /* init cache_info for new cache */ 59 cache_info_init_default(cache); 60 cache_mode_set(cache, opts->cache_mode); 61 if (opts->data_crc) 62 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC; 63 64 return 0; 65 } 66 67 static void cache_info_set_gc_percent(struct pcache_cache_info *cache_info, u8 percent) 68 { 69 cache_info->flags &= ~PCACHE_CACHE_FLAGS_GC_PERCENT_MASK; 70 cache_info->flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_GC_PERCENT_MASK, percent); 71 } 72 73 int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent) 74 { 75 if (percent > PCACHE_CACHE_GC_PERCENT_MAX || percent < PCACHE_CACHE_GC_PERCENT_MIN) 76 return -EINVAL; 77 78 mutex_lock(&cache->cache_info_lock); 79 cache_info_set_gc_percent(&cache->cache_info, percent); 80 81 cache_info_write(cache); 82 mutex_unlock(&cache->cache_info_lock); 83 84 return 0; 85 } 86 87 void cache_pos_encode(struct pcache_cache *cache, 88 struct pcache_cache_pos_onmedia *pos_onmedia_base, 89 struct pcache_cache_pos *pos, u64 seq, u32 *index) 90 { 91 struct pcache_cache_pos_onmedia pos_onmedia; 92 struct pcache_cache_pos_onmedia *pos_onmedia_addr = pos_onmedia_base + *index; 93 94 pos_onmedia.cache_seg_id = pos->cache_seg->cache_seg_id; 95 pos_onmedia.seg_off = pos->seg_off; 96 pos_onmedia.header.seq = seq; 97 pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia); 98 99 *index = (*index + 1) % PCACHE_META_INDEX_MAX; 100 101 memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia)); 102 pmem_wmb(); 103 } 104 105 int cache_pos_decode(struct pcache_cache *cache, 106 struct pcache_cache_pos_onmedia *pos_onmedia, 107 struct pcache_cache_pos *pos, u64 *seq, u32 *index) 108 { 109 struct pcache_cache_pos_onmedia latest, *latest_addr; 110 111 latest_addr = pcache_meta_find_latest(&pos_onmedia->header, 112 sizeof(struct pcache_cache_pos_onmedia), 113 sizeof(struct pcache_cache_pos_onmedia), 114 &latest); 115 if (IS_ERR(latest_addr)) 116 return PTR_ERR(latest_addr); 117 118 if (!latest_addr) 119 return -EIO; 120 121 if (!cache_seg_id_valid(cache, latest.cache_seg_id)) 122 return -EIO; 123 124 pos->cache_seg = &cache->segments[latest.cache_seg_id]; 125 126 if (latest.seg_off >= pos->cache_seg->segment.data_size) 127 return -EIO; 128 129 pos->seg_off = latest.seg_off; 130 *seq = latest.header.seq; 131 *index = (latest_addr - pos_onmedia); 132 133 return 0; 134 } 135 136 static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id) 137 { 138 cache->cache_info.seg_id = seg_id; 139 } 140 141 static int cache_init(struct dm_pcache *pcache) 142 { 143 struct pcache_cache *cache = &pcache->cache; 144 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 145 struct pcache_cache_dev *cache_dev = &pcache->cache_dev; 146 int ret; 147 148 cache->segments = kvzalloc_objs(struct pcache_cache_segment, 149 cache_dev->seg_num); 150 if (!cache->segments) { 151 ret = -ENOMEM; 152 goto err; 153 } 154 155 cache->seg_map = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL); 156 if (!cache->seg_map) { 157 ret = -ENOMEM; 158 goto free_segments; 159 } 160 161 cache->backing_dev = backing_dev; 162 cache->cache_dev = &pcache->cache_dev; 163 cache->n_segs = cache_dev->seg_num; 164 atomic_set(&cache->gc_errors, 0); 165 atomic_set(&cache->writeback_errors, 0); 166 spin_lock_init(&cache->seg_map_lock); 167 spin_lock_init(&cache->key_head_lock); 168 169 mutex_init(&cache->cache_info_lock); 170 mutex_init(&cache->key_tail_lock); 171 mutex_init(&cache->dirty_tail_lock); 172 mutex_init(&cache->writeback_lock); 173 174 INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn); 175 INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn); 176 INIT_WORK(&cache->clean_work, clean_fn); 177 178 return 0; 179 180 free_segments: 181 kvfree(cache->segments); 182 err: 183 return ret; 184 } 185 186 static void cache_exit(struct pcache_cache *cache) 187 { 188 kvfree(cache->seg_map); 189 kvfree(cache->segments); 190 } 191 192 static void cache_info_init_default(struct pcache_cache *cache) 193 { 194 struct pcache_cache_info *cache_info = &cache->cache_info; 195 196 memset(cache_info, 0, sizeof(*cache_info)); 197 cache_info->n_segs = cache->cache_dev->seg_num; 198 cache_info_set_gc_percent(cache_info, PCACHE_CACHE_GC_PERCENT_DEFAULT); 199 } 200 201 static int cache_tail_init(struct pcache_cache *cache) 202 { 203 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 204 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); 205 int ret; 206 207 if (new_cache) { 208 __set_bit(0, cache->seg_map); 209 210 cache->key_head.cache_seg = &cache->segments[0]; 211 cache->key_head.seg_off = 0; 212 cache_pos_copy(&cache->key_tail, &cache->key_head); 213 cache_pos_copy(&cache->dirty_tail, &cache->key_head); 214 215 cache_encode_dirty_tail(cache); 216 cache_encode_key_tail(cache); 217 } else { 218 if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) { 219 pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n"); 220 return -EIO; 221 } 222 223 ret = cache_verify_dirty_tail(cache); 224 if (ret) { 225 pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n"); 226 return ret; 227 } 228 } 229 230 return 0; 231 } 232 233 static int get_seg_id(struct pcache_cache *cache, 234 struct pcache_cache_segment *prev_cache_seg, 235 bool new_cache, u32 *seg_id) 236 { 237 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 238 struct pcache_cache_dev *cache_dev = cache->cache_dev; 239 int ret; 240 241 if (new_cache) { 242 ret = cache_dev_get_empty_segment_id(cache_dev, seg_id); 243 if (ret) { 244 pcache_dev_err(pcache, "no available segment\n"); 245 goto err; 246 } 247 248 if (prev_cache_seg) 249 cache_seg_set_next_seg(prev_cache_seg, *seg_id); 250 else 251 cache_info_set_seg_id(cache, *seg_id); 252 } else { 253 if (prev_cache_seg) { 254 struct pcache_segment_info *prev_seg_info; 255 256 prev_seg_info = &prev_cache_seg->cache_seg_info; 257 if (!segment_info_has_next(prev_seg_info)) { 258 ret = -EFAULT; 259 goto err; 260 } 261 *seg_id = prev_cache_seg->cache_seg_info.next_seg; 262 } else { 263 *seg_id = cache->cache_info.seg_id; 264 } 265 266 if (*seg_id >= cache_dev->seg_num) { 267 pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n", 268 *seg_id, cache_dev->seg_num); 269 ret = -EIO; 270 goto err; 271 } 272 } 273 return 0; 274 err: 275 return ret; 276 } 277 278 static int cache_segs_init(struct pcache_cache *cache) 279 { 280 struct pcache_cache_segment *prev_cache_seg = NULL; 281 struct pcache_cache_info *cache_info = &cache->cache_info; 282 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); 283 u32 seg_id; 284 int ret; 285 u32 i; 286 287 if (cache_info->n_segs > cache->cache_dev->seg_num) { 288 pcache_dev_err(CACHE_TO_PCACHE(cache), 289 "cache_info n_segs %u exceeds cache device segments %u\n", 290 cache_info->n_segs, cache->cache_dev->seg_num); 291 return -EIO; 292 } 293 294 for (i = 0; i < cache_info->n_segs; i++) { 295 ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id); 296 if (ret) 297 goto err; 298 299 ret = cache_seg_init(cache, seg_id, i, new_cache); 300 if (ret) 301 goto err; 302 303 prev_cache_seg = &cache->segments[i]; 304 } 305 return 0; 306 err: 307 return ret; 308 } 309 310 static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral) 311 { 312 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 313 u32 n_subtrees; 314 int ret; 315 u32 i, cpu; 316 317 /* Calculate number of cache trees based on the device size */ 318 n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE); 319 ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees); 320 if (ret) 321 goto err; 322 323 cache->n_ksets = n_paral; 324 cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL); 325 if (!cache->ksets) { 326 ret = -ENOMEM; 327 goto req_tree_exit; 328 } 329 330 /* 331 * Initialize each kset with a spinlock and delayed work for flushing. 332 * Each kset is associated with one queue to ensure independent handling 333 * of cache keys across multiple queues, maximizing multiqueue concurrency. 334 */ 335 for (i = 0; i < cache->n_ksets; i++) { 336 struct pcache_cache_kset *kset = get_kset(cache, i); 337 338 kset->cache = cache; 339 spin_lock_init(&kset->kset_lock); 340 INIT_DELAYED_WORK(&kset->flush_work, kset_flush_fn); 341 } 342 343 cache->data_heads = alloc_percpu(struct pcache_cache_data_head); 344 if (!cache->data_heads) { 345 ret = -ENOMEM; 346 goto free_kset; 347 } 348 349 for_each_possible_cpu(cpu) { 350 struct pcache_cache_data_head *h = 351 per_cpu_ptr(cache->data_heads, cpu); 352 h->head_pos.cache_seg = NULL; 353 } 354 355 /* 356 * Replay persisted cache keys using cache_replay. 357 * This function loads and replays cache keys from previously stored 358 * ksets, allowing the cache to restore its state after a restart. 359 */ 360 ret = cache_replay(cache); 361 if (ret) { 362 pcache_dev_err(pcache, "failed to replay keys\n"); 363 goto free_heads; 364 } 365 366 return 0; 367 368 free_heads: 369 free_percpu(cache->data_heads); 370 free_kset: 371 kvfree(cache->ksets); 372 req_tree_exit: 373 cache_tree_exit(&cache->req_key_tree); 374 err: 375 return ret; 376 } 377 378 static void cache_destroy_req_keys(struct pcache_cache *cache) 379 { 380 u32 i; 381 382 for (i = 0; i < cache->n_ksets; i++) { 383 struct pcache_cache_kset *kset = get_kset(cache, i); 384 385 cancel_delayed_work_sync(&kset->flush_work); 386 } 387 388 free_percpu(cache->data_heads); 389 kvfree(cache->ksets); 390 cache_tree_exit(&cache->req_key_tree); 391 } 392 393 int pcache_cache_start(struct dm_pcache *pcache) 394 { 395 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 396 struct pcache_cache *cache = &pcache->cache; 397 struct pcache_cache_options *opts = &pcache->opts; 398 int ret; 399 400 ret = cache_init(pcache); 401 if (ret) 402 return ret; 403 404 cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev); 405 cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev); 406 backing_dev->cache = cache; 407 cache->dev_size = backing_dev->dev_size; 408 409 ret = cache_info_init(cache, opts); 410 if (ret) 411 goto cache_exit; 412 413 ret = cache_segs_init(cache); 414 if (ret) 415 goto cache_exit; 416 417 ret = cache_tail_init(cache); 418 if (ret) 419 goto cache_exit; 420 421 ret = cache_init_req_keys(cache, num_online_cpus()); 422 if (ret) 423 goto cache_exit; 424 425 ret = cache_writeback_init(cache); 426 if (ret) 427 goto destroy_keys; 428 429 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE; 430 cache_info_write(cache); 431 queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0); 432 433 return 0; 434 435 destroy_keys: 436 cache_destroy_req_keys(cache); 437 cache_exit: 438 cache_exit(cache); 439 440 return ret; 441 } 442 443 void pcache_cache_stop(struct dm_pcache *pcache) 444 { 445 struct pcache_cache *cache = &pcache->cache; 446 447 pcache_cache_flush(cache); 448 449 cancel_delayed_work_sync(&cache->gc_work); 450 flush_work(&cache->clean_work); 451 cache_writeback_exit(cache); 452 453 if (cache->req_key_tree.n_subtrees) 454 cache_destroy_req_keys(cache); 455 456 cache_exit(cache); 457 } 458 459 struct workqueue_struct *cache_get_wq(struct pcache_cache *cache) 460 { 461 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 462 463 return pcache->task_wq; 464 } 465 466 int pcache_cache_init(void) 467 { 468 key_cache = KMEM_CACHE(pcache_cache_key, 0); 469 if (!key_cache) 470 return -ENOMEM; 471 472 return 0; 473 } 474 475 void pcache_cache_exit(void) 476 { 477 kmem_cache_destroy(key_cache); 478 } 479