1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/blk_types.h> 3 4 #include "cache.h" 5 #include "cache_dev.h" 6 #include "backing_dev.h" 7 #include "dm_pcache.h" 8 9 struct kmem_cache *key_cache; 10 11 static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache) 12 { 13 return (struct pcache_cache_info *)((char *)cache->cache_info_addr + 14 (size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE); 15 } 16 17 static void cache_info_write(struct pcache_cache *cache) 18 { 19 struct pcache_cache_info *cache_info = &cache->cache_info; 20 21 cache_info->header.seq++; 22 cache_info->header.crc = pcache_meta_crc(&cache_info->header, 23 sizeof(struct pcache_cache_info)); 24 25 cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX; 26 memcpy_flushcache(get_cache_info_addr(cache), cache_info, 27 sizeof(struct pcache_cache_info)); 28 pmem_wmb(); 29 } 30 31 static void cache_info_init_default(struct pcache_cache *cache); 32 static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts) 33 { 34 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 35 struct pcache_cache_info *cache_info_addr; 36 37 cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header, 38 sizeof(struct pcache_cache_info), 39 PCACHE_CACHE_INFO_SIZE, 40 &cache->cache_info); 41 if (IS_ERR(cache_info_addr)) 42 return PTR_ERR(cache_info_addr); 43 44 if (cache_info_addr) { 45 if (opts->data_crc != 46 (cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) { 47 pcache_dev_err(pcache, "invalid option for data_crc: %s, expected: %s", 48 opts->data_crc ? "true" : "false", 49 cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false"); 50 return -EINVAL; 51 } 52 53 cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE; 54 55 return 0; 56 } 57 58 /* init cache_info for new cache */ 59 cache_info_init_default(cache); 60 cache_mode_set(cache, opts->cache_mode); 61 if (opts->data_crc) 62 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC; 63 64 return 0; 65 } 66 67 static void cache_info_set_gc_percent(struct pcache_cache_info *cache_info, u8 percent) 68 { 69 cache_info->flags &= ~PCACHE_CACHE_FLAGS_GC_PERCENT_MASK; 70 cache_info->flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_GC_PERCENT_MASK, percent); 71 } 72 73 int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent) 74 { 75 if (percent > PCACHE_CACHE_GC_PERCENT_MAX || percent < PCACHE_CACHE_GC_PERCENT_MIN) 76 return -EINVAL; 77 78 mutex_lock(&cache->cache_info_lock); 79 cache_info_set_gc_percent(&cache->cache_info, percent); 80 81 cache_info_write(cache); 82 mutex_unlock(&cache->cache_info_lock); 83 84 return 0; 85 } 86 87 void cache_pos_encode(struct pcache_cache *cache, 88 struct pcache_cache_pos_onmedia *pos_onmedia_base, 89 struct pcache_cache_pos *pos, u64 seq, u32 *index) 90 { 91 struct pcache_cache_pos_onmedia pos_onmedia; 92 struct pcache_cache_pos_onmedia *pos_onmedia_addr = pos_onmedia_base + *index; 93 94 pos_onmedia.cache_seg_id = pos->cache_seg->cache_seg_id; 95 pos_onmedia.seg_off = pos->seg_off; 96 pos_onmedia.header.seq = seq; 97 pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia); 98 99 *index = (*index + 1) % PCACHE_META_INDEX_MAX; 100 101 memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia)); 102 pmem_wmb(); 103 } 104 105 int cache_pos_decode(struct pcache_cache *cache, 106 struct pcache_cache_pos_onmedia *pos_onmedia, 107 struct pcache_cache_pos *pos, u64 *seq, u32 *index) 108 { 109 struct pcache_cache_pos_onmedia latest, *latest_addr; 110 111 latest_addr = pcache_meta_find_latest(&pos_onmedia->header, 112 sizeof(struct pcache_cache_pos_onmedia), 113 sizeof(struct pcache_cache_pos_onmedia), 114 &latest); 115 if (IS_ERR(latest_addr)) 116 return PTR_ERR(latest_addr); 117 118 if (!latest_addr) 119 return -EIO; 120 121 if (!cache_seg_id_valid(cache, latest.cache_seg_id)) 122 return -EIO; 123 124 pos->cache_seg = &cache->segments[latest.cache_seg_id]; 125 126 if (latest.seg_off >= pos->cache_seg->segment.data_size) 127 return -EIO; 128 129 pos->seg_off = latest.seg_off; 130 *seq = latest.header.seq; 131 *index = (latest_addr - pos_onmedia); 132 133 return 0; 134 } 135 136 static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id) 137 { 138 cache->cache_info.seg_id = seg_id; 139 } 140 141 static int cache_init(struct dm_pcache *pcache) 142 { 143 struct pcache_cache *cache = &pcache->cache; 144 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 145 struct pcache_cache_dev *cache_dev = &pcache->cache_dev; 146 int ret; 147 148 cache->segments = kvzalloc_objs(struct pcache_cache_segment, 149 cache_dev->seg_num); 150 if (!cache->segments) { 151 ret = -ENOMEM; 152 goto err; 153 } 154 155 cache->seg_map = kvzalloc_objs(unsigned long, 156 BITS_TO_LONGS(cache_dev->seg_num)); 157 if (!cache->seg_map) { 158 ret = -ENOMEM; 159 goto free_segments; 160 } 161 162 cache->backing_dev = backing_dev; 163 cache->cache_dev = &pcache->cache_dev; 164 cache->n_segs = cache_dev->seg_num; 165 atomic_set(&cache->gc_errors, 0); 166 atomic_set(&cache->writeback_errors, 0); 167 spin_lock_init(&cache->seg_map_lock); 168 spin_lock_init(&cache->key_head_lock); 169 170 mutex_init(&cache->cache_info_lock); 171 mutex_init(&cache->key_tail_lock); 172 mutex_init(&cache->dirty_tail_lock); 173 mutex_init(&cache->writeback_lock); 174 175 INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn); 176 INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn); 177 INIT_WORK(&cache->clean_work, clean_fn); 178 179 return 0; 180 181 free_segments: 182 kvfree(cache->segments); 183 err: 184 return ret; 185 } 186 187 static void cache_exit(struct pcache_cache *cache) 188 { 189 kvfree(cache->seg_map); 190 kvfree(cache->segments); 191 } 192 193 static void cache_info_init_default(struct pcache_cache *cache) 194 { 195 struct pcache_cache_info *cache_info = &cache->cache_info; 196 197 memset(cache_info, 0, sizeof(*cache_info)); 198 cache_info->n_segs = cache->cache_dev->seg_num; 199 cache_info_set_gc_percent(cache_info, PCACHE_CACHE_GC_PERCENT_DEFAULT); 200 } 201 202 static int cache_tail_init(struct pcache_cache *cache) 203 { 204 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 205 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); 206 int ret; 207 208 if (new_cache) { 209 __set_bit(0, cache->seg_map); 210 211 cache->key_head.cache_seg = &cache->segments[0]; 212 cache->key_head.seg_off = 0; 213 cache_pos_copy(&cache->key_tail, &cache->key_head); 214 cache_pos_copy(&cache->dirty_tail, &cache->key_head); 215 216 cache_encode_dirty_tail(cache); 217 cache_encode_key_tail(cache); 218 } else { 219 if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) { 220 pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n"); 221 return -EIO; 222 } 223 224 ret = cache_verify_dirty_tail(cache); 225 if (ret) { 226 pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n"); 227 return ret; 228 } 229 } 230 231 return 0; 232 } 233 234 static int get_seg_id(struct pcache_cache *cache, 235 struct pcache_cache_segment *prev_cache_seg, 236 bool new_cache, u32 *seg_id) 237 { 238 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 239 struct pcache_cache_dev *cache_dev = cache->cache_dev; 240 int ret; 241 242 if (new_cache) { 243 ret = cache_dev_get_empty_segment_id(cache_dev, seg_id); 244 if (ret) { 245 pcache_dev_err(pcache, "no available segment\n"); 246 goto err; 247 } 248 249 if (prev_cache_seg) 250 cache_seg_set_next_seg(prev_cache_seg, *seg_id); 251 else 252 cache_info_set_seg_id(cache, *seg_id); 253 } else { 254 if (prev_cache_seg) { 255 struct pcache_segment_info *prev_seg_info; 256 257 prev_seg_info = &prev_cache_seg->cache_seg_info; 258 if (!segment_info_has_next(prev_seg_info)) { 259 ret = -EFAULT; 260 goto err; 261 } 262 *seg_id = prev_cache_seg->cache_seg_info.next_seg; 263 } else { 264 *seg_id = cache->cache_info.seg_id; 265 } 266 267 if (*seg_id >= cache_dev->seg_num) { 268 pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n", 269 *seg_id, cache_dev->seg_num); 270 ret = -EIO; 271 goto err; 272 } 273 } 274 return 0; 275 err: 276 return ret; 277 } 278 279 static int cache_segs_init(struct pcache_cache *cache) 280 { 281 struct pcache_cache_segment *prev_cache_seg = NULL; 282 struct pcache_cache_info *cache_info = &cache->cache_info; 283 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); 284 u32 seg_id; 285 int ret; 286 u32 i; 287 288 if (cache_info->n_segs > cache->cache_dev->seg_num) { 289 pcache_dev_err(CACHE_TO_PCACHE(cache), 290 "cache_info n_segs %u exceeds cache device segments %u\n", 291 cache_info->n_segs, cache->cache_dev->seg_num); 292 return -EIO; 293 } 294 295 for (i = 0; i < cache_info->n_segs; i++) { 296 ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id); 297 if (ret) 298 goto err; 299 300 ret = cache_seg_init(cache, seg_id, i, new_cache); 301 if (ret) 302 goto err; 303 304 prev_cache_seg = &cache->segments[i]; 305 } 306 return 0; 307 err: 308 return ret; 309 } 310 311 static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral) 312 { 313 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 314 u32 n_subtrees; 315 int ret; 316 u32 i, cpu; 317 318 /* Calculate number of cache trees based on the device size */ 319 n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE); 320 ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees); 321 if (ret) 322 goto err; 323 324 cache->n_ksets = n_paral; 325 cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL); 326 if (!cache->ksets) { 327 ret = -ENOMEM; 328 goto req_tree_exit; 329 } 330 331 /* 332 * Initialize each kset with a spinlock and delayed work for flushing. 333 * Each kset is associated with one queue to ensure independent handling 334 * of cache keys across multiple queues, maximizing multiqueue concurrency. 335 */ 336 for (i = 0; i < cache->n_ksets; i++) { 337 struct pcache_cache_kset *kset = get_kset(cache, i); 338 339 kset->cache = cache; 340 spin_lock_init(&kset->kset_lock); 341 INIT_DELAYED_WORK(&kset->flush_work, kset_flush_fn); 342 } 343 344 cache->data_heads = alloc_percpu(struct pcache_cache_data_head); 345 if (!cache->data_heads) { 346 ret = -ENOMEM; 347 goto free_kset; 348 } 349 350 for_each_possible_cpu(cpu) { 351 struct pcache_cache_data_head *h = 352 per_cpu_ptr(cache->data_heads, cpu); 353 h->head_pos.cache_seg = NULL; 354 } 355 356 /* 357 * Replay persisted cache keys using cache_replay. 358 * This function loads and replays cache keys from previously stored 359 * ksets, allowing the cache to restore its state after a restart. 360 */ 361 ret = cache_replay(cache); 362 if (ret) { 363 pcache_dev_err(pcache, "failed to replay keys\n"); 364 goto free_heads; 365 } 366 367 return 0; 368 369 free_heads: 370 free_percpu(cache->data_heads); 371 free_kset: 372 kvfree(cache->ksets); 373 req_tree_exit: 374 cache_tree_exit(&cache->req_key_tree); 375 err: 376 return ret; 377 } 378 379 static void cache_destroy_req_keys(struct pcache_cache *cache) 380 { 381 u32 i; 382 383 for (i = 0; i < cache->n_ksets; i++) { 384 struct pcache_cache_kset *kset = get_kset(cache, i); 385 386 cancel_delayed_work_sync(&kset->flush_work); 387 } 388 389 free_percpu(cache->data_heads); 390 kvfree(cache->ksets); 391 cache_tree_exit(&cache->req_key_tree); 392 } 393 394 int pcache_cache_start(struct dm_pcache *pcache) 395 { 396 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 397 struct pcache_cache *cache = &pcache->cache; 398 struct pcache_cache_options *opts = &pcache->opts; 399 int ret; 400 401 ret = cache_init(pcache); 402 if (ret) 403 return ret; 404 405 cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev); 406 cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev); 407 backing_dev->cache = cache; 408 cache->dev_size = backing_dev->dev_size; 409 410 ret = cache_info_init(cache, opts); 411 if (ret) 412 goto cache_exit; 413 414 ret = cache_segs_init(cache); 415 if (ret) 416 goto cache_exit; 417 418 ret = cache_tail_init(cache); 419 if (ret) 420 goto cache_exit; 421 422 ret = cache_init_req_keys(cache, num_online_cpus()); 423 if (ret) 424 goto cache_exit; 425 426 ret = cache_writeback_init(cache); 427 if (ret) 428 goto destroy_keys; 429 430 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE; 431 cache_info_write(cache); 432 queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0); 433 434 return 0; 435 436 destroy_keys: 437 cache_destroy_req_keys(cache); 438 cache_exit: 439 cache_exit(cache); 440 441 return ret; 442 } 443 444 void pcache_cache_stop(struct dm_pcache *pcache) 445 { 446 struct pcache_cache *cache = &pcache->cache; 447 448 pcache_cache_flush(cache); 449 450 cancel_delayed_work_sync(&cache->gc_work); 451 flush_work(&cache->clean_work); 452 cache_writeback_exit(cache); 453 454 if (cache->req_key_tree.n_subtrees) 455 cache_destroy_req_keys(cache); 456 457 cache_exit(cache); 458 } 459 460 struct workqueue_struct *cache_get_wq(struct pcache_cache *cache) 461 { 462 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 463 464 return pcache->task_wq; 465 } 466 467 int pcache_cache_init(void) 468 { 469 key_cache = KMEM_CACHE(pcache_cache_key, 0); 470 if (!key_cache) 471 return -ENOMEM; 472 473 return 0; 474 } 475 476 void pcache_cache_exit(void) 477 { 478 kmem_cache_destroy(key_cache); 479 } 480