1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/module.h> 3 #include <linux/blkdev.h> 4 #include <linux/bio.h> 5 6 #include "../dm-core.h" 7 #include "cache_dev.h" 8 #include "backing_dev.h" 9 #include "cache.h" 10 #include "dm_pcache.h" 11 12 void pcache_defer_reqs_kick(struct dm_pcache *pcache) 13 { 14 struct pcache_cache *cache = &pcache->cache; 15 16 spin_lock(&cache->seg_map_lock); 17 if (!cache->cache_full) 18 queue_work(pcache->task_wq, &pcache->defered_req_work); 19 spin_unlock(&cache->seg_map_lock); 20 } 21 22 static void defer_req(struct pcache_request *pcache_req) 23 { 24 struct dm_pcache *pcache = pcache_req->pcache; 25 26 BUG_ON(!list_empty(&pcache_req->list_node)); 27 28 spin_lock(&pcache->defered_req_list_lock); 29 list_add(&pcache_req->list_node, &pcache->defered_req_list); 30 pcache_defer_reqs_kick(pcache); 31 spin_unlock(&pcache->defered_req_list_lock); 32 } 33 34 static void defered_req_fn(struct work_struct *work) 35 { 36 struct dm_pcache *pcache = container_of(work, struct dm_pcache, defered_req_work); 37 struct pcache_request *pcache_req; 38 LIST_HEAD(tmp_list); 39 int ret; 40 41 if (pcache_is_stopping(pcache)) 42 return; 43 44 spin_lock(&pcache->defered_req_list_lock); 45 list_splice_init(&pcache->defered_req_list, &tmp_list); 46 spin_unlock(&pcache->defered_req_list_lock); 47 48 while (!list_empty(&tmp_list)) { 49 pcache_req = list_first_entry(&tmp_list, 50 struct pcache_request, list_node); 51 list_del_init(&pcache_req->list_node); 52 pcache_req->ret = 0; 53 ret = pcache_cache_handle_req(&pcache->cache, pcache_req); 54 if (ret == -EBUSY) 55 defer_req(pcache_req); 56 else 57 pcache_req_put(pcache_req, ret); 58 } 59 } 60 61 void pcache_req_get(struct pcache_request *pcache_req) 62 { 63 kref_get(&pcache_req->ref); 64 } 65 66 static void end_req(struct kref *ref) 67 { 68 struct pcache_request *pcache_req = container_of(ref, struct pcache_request, ref); 69 struct dm_pcache *pcache = pcache_req->pcache; 70 struct bio *bio = pcache_req->bio; 71 int ret = pcache_req->ret; 72 73 if (ret == -EBUSY) { 74 pcache_req_get(pcache_req); 75 defer_req(pcache_req); 76 } else { 77 bio->bi_status = errno_to_blk_status(ret); 78 bio_endio(bio); 79 80 if (atomic_dec_and_test(&pcache->inflight_reqs)) 81 wake_up(&pcache->inflight_wq); 82 } 83 } 84 85 void pcache_req_put(struct pcache_request *pcache_req, int ret) 86 { 87 /* Set the return status if it is not already set */ 88 if (ret && !pcache_req->ret) 89 pcache_req->ret = ret; 90 91 kref_put(&pcache_req->ref, end_req); 92 } 93 94 static bool at_least_one_arg(struct dm_arg_set *as, char **error) 95 { 96 if (!as->argc) { 97 *error = "Insufficient args"; 98 return false; 99 } 100 101 return true; 102 } 103 104 static int parse_cache_dev(struct dm_pcache *pcache, struct dm_arg_set *as, 105 char **error) 106 { 107 int ret; 108 109 if (!at_least_one_arg(as, error)) 110 return -EINVAL; 111 ret = dm_get_device(pcache->ti, dm_shift_arg(as), 112 BLK_OPEN_READ | BLK_OPEN_WRITE, 113 &pcache->cache_dev.dm_dev); 114 if (ret) { 115 *error = "Error opening cache device"; 116 return ret; 117 } 118 119 return 0; 120 } 121 122 static int parse_backing_dev(struct dm_pcache *pcache, struct dm_arg_set *as, 123 char **error) 124 { 125 int ret; 126 127 if (!at_least_one_arg(as, error)) 128 return -EINVAL; 129 130 ret = dm_get_device(pcache->ti, dm_shift_arg(as), 131 BLK_OPEN_READ | BLK_OPEN_WRITE, 132 &pcache->backing_dev.dm_dev); 133 if (ret) { 134 *error = "Error opening backing device"; 135 return ret; 136 } 137 138 return 0; 139 } 140 141 static void pcache_init_opts(struct pcache_cache_options *opts) 142 { 143 opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK; 144 opts->data_crc = false; 145 } 146 147 static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, 148 char **error) 149 { 150 struct pcache_cache_options *opts = &pcache->opts; 151 static const struct dm_arg _args[] = { 152 {0, 4, "Invalid number of cache option arguments"}, 153 }; 154 unsigned int argc; 155 const char *arg; 156 int ret; 157 158 pcache_init_opts(opts); 159 if (!as->argc) 160 return 0; 161 162 ret = dm_read_arg_group(_args, as, &argc, error); 163 if (ret) 164 return -EINVAL; 165 166 while (argc) { 167 arg = dm_shift_arg(as); 168 argc--; 169 170 if (!strcmp(arg, "cache_mode")) { 171 if (!argc) { 172 *error = "Missing value for cache_mode"; 173 return -EINVAL; 174 } 175 arg = dm_shift_arg(as); 176 if (!strcmp(arg, "writeback")) { 177 opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK; 178 } else { 179 *error = "Invalid cache mode parameter"; 180 return -EINVAL; 181 } 182 argc--; 183 } else if (!strcmp(arg, "data_crc")) { 184 if (!argc) { 185 *error = "Missing value for data_crc"; 186 return -EINVAL; 187 } 188 arg = dm_shift_arg(as); 189 if (!strcmp(arg, "true")) { 190 opts->data_crc = true; 191 } else if (!strcmp(arg, "false")) { 192 opts->data_crc = false; 193 } else { 194 *error = "Invalid data crc parameter"; 195 return -EINVAL; 196 } 197 argc--; 198 } else { 199 *error = "Unrecognised cache option requested"; 200 return -EINVAL; 201 } 202 } 203 204 return 0; 205 } 206 207 static int pcache_start(struct dm_pcache *pcache, char **error) 208 { 209 int ret; 210 211 ret = cache_dev_start(pcache); 212 if (ret) { 213 *error = "Failed to start cache dev"; 214 return ret; 215 } 216 217 ret = backing_dev_start(pcache); 218 if (ret) { 219 *error = "Failed to start backing dev"; 220 goto stop_cache; 221 } 222 223 ret = pcache_cache_start(pcache); 224 if (ret) { 225 *error = "Failed to start pcache"; 226 goto stop_backing; 227 } 228 229 return 0; 230 stop_backing: 231 backing_dev_stop(pcache); 232 stop_cache: 233 cache_dev_stop(pcache); 234 235 return ret; 236 } 237 238 static void pcache_destroy_args(struct dm_pcache *pcache) 239 { 240 if (pcache->cache_dev.dm_dev) 241 dm_put_device(pcache->ti, pcache->cache_dev.dm_dev); 242 if (pcache->backing_dev.dm_dev) 243 dm_put_device(pcache->ti, pcache->backing_dev.dm_dev); 244 } 245 246 static int pcache_parse_args(struct dm_pcache *pcache, unsigned int argc, char **argv, 247 char **error) 248 { 249 struct dm_arg_set as; 250 int ret; 251 252 as.argc = argc; 253 as.argv = argv; 254 255 /* 256 * Parse cache device 257 */ 258 ret = parse_cache_dev(pcache, &as, error); 259 if (ret) 260 return ret; 261 /* 262 * Parse backing device 263 */ 264 ret = parse_backing_dev(pcache, &as, error); 265 if (ret) 266 goto out; 267 /* 268 * Parse optional arguments 269 */ 270 ret = parse_cache_opts(pcache, &as, error); 271 if (ret) 272 goto out; 273 274 return 0; 275 out: 276 pcache_destroy_args(pcache); 277 return ret; 278 } 279 280 static int dm_pcache_ctr(struct dm_target *ti, unsigned int argc, char **argv) 281 { 282 struct mapped_device *md = ti->table->md; 283 struct dm_pcache *pcache; 284 int ret; 285 286 if (md->map) { 287 ti->error = "Don't support table loading for live md"; 288 return -EOPNOTSUPP; 289 } 290 291 /* Allocate memory for the cache structure */ 292 pcache = kzalloc_obj(struct dm_pcache); 293 if (!pcache) 294 return -ENOMEM; 295 296 pcache->task_wq = alloc_workqueue("pcache-%s-wq", WQ_UNBOUND | WQ_MEM_RECLAIM, 297 0, md->name); 298 if (!pcache->task_wq) { 299 ret = -ENOMEM; 300 goto free_pcache; 301 } 302 303 spin_lock_init(&pcache->defered_req_list_lock); 304 INIT_LIST_HEAD(&pcache->defered_req_list); 305 INIT_WORK(&pcache->defered_req_work, defered_req_fn); 306 pcache->ti = ti; 307 308 ret = pcache_parse_args(pcache, argc, argv, &ti->error); 309 if (ret) 310 goto destroy_wq; 311 312 ret = pcache_start(pcache, &ti->error); 313 if (ret) 314 goto destroy_args; 315 316 ti->num_flush_bios = 1; 317 ti->flush_supported = true; 318 ti->per_io_data_size = sizeof(struct pcache_request); 319 ti->private = pcache; 320 atomic_set(&pcache->inflight_reqs, 0); 321 atomic_set(&pcache->state, PCACHE_STATE_RUNNING); 322 init_waitqueue_head(&pcache->inflight_wq); 323 324 return 0; 325 destroy_args: 326 pcache_destroy_args(pcache); 327 destroy_wq: 328 destroy_workqueue(pcache->task_wq); 329 free_pcache: 330 kfree(pcache); 331 332 return ret; 333 } 334 335 static void defer_req_stop(struct dm_pcache *pcache) 336 { 337 struct pcache_request *pcache_req; 338 LIST_HEAD(tmp_list); 339 340 flush_work(&pcache->defered_req_work); 341 342 spin_lock(&pcache->defered_req_list_lock); 343 list_splice_init(&pcache->defered_req_list, &tmp_list); 344 spin_unlock(&pcache->defered_req_list_lock); 345 346 while (!list_empty(&tmp_list)) { 347 pcache_req = list_first_entry(&tmp_list, 348 struct pcache_request, list_node); 349 list_del_init(&pcache_req->list_node); 350 pcache_req_put(pcache_req, -EIO); 351 } 352 } 353 354 static void dm_pcache_dtr(struct dm_target *ti) 355 { 356 struct dm_pcache *pcache; 357 358 pcache = ti->private; 359 atomic_set(&pcache->state, PCACHE_STATE_STOPPING); 360 defer_req_stop(pcache); 361 362 wait_event(pcache->inflight_wq, 363 atomic_read(&pcache->inflight_reqs) == 0); 364 365 pcache_cache_stop(pcache); 366 backing_dev_stop(pcache); 367 cache_dev_stop(pcache); 368 369 pcache_destroy_args(pcache); 370 drain_workqueue(pcache->task_wq); 371 destroy_workqueue(pcache->task_wq); 372 373 kfree(pcache); 374 } 375 376 static int dm_pcache_map_bio(struct dm_target *ti, struct bio *bio) 377 { 378 struct pcache_request *pcache_req = dm_per_bio_data(bio, sizeof(struct pcache_request)); 379 struct dm_pcache *pcache = ti->private; 380 int ret; 381 382 pcache_req->pcache = pcache; 383 kref_init(&pcache_req->ref); 384 pcache_req->ret = 0; 385 pcache_req->bio = bio; 386 pcache_req->off = (u64)bio->bi_iter.bi_sector << SECTOR_SHIFT; 387 pcache_req->data_len = bio->bi_iter.bi_size; 388 INIT_LIST_HEAD(&pcache_req->list_node); 389 atomic_inc(&pcache->inflight_reqs); 390 391 ret = pcache_cache_handle_req(&pcache->cache, pcache_req); 392 if (ret == -EBUSY) 393 defer_req(pcache_req); 394 else 395 pcache_req_put(pcache_req, ret); 396 397 return DM_MAPIO_SUBMITTED; 398 } 399 400 static void dm_pcache_status(struct dm_target *ti, status_type_t type, 401 unsigned int status_flags, char *result, 402 unsigned int maxlen) 403 { 404 struct dm_pcache *pcache = ti->private; 405 struct pcache_cache_dev *cache_dev = &pcache->cache_dev; 406 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 407 struct pcache_cache *cache = &pcache->cache; 408 unsigned int sz = 0; 409 410 switch (type) { 411 case STATUSTYPE_INFO: 412 DMEMIT("%x %u %u %u %u %x %u:%u %u:%u %u:%u", 413 cache_dev->sb_flags, 414 cache_dev->seg_num, 415 cache->n_segs, 416 bitmap_weight(cache->seg_map, cache->n_segs), 417 pcache_cache_get_gc_percent(cache), 418 cache->cache_info.flags, 419 cache->key_head.cache_seg->cache_seg_id, 420 cache->key_head.seg_off, 421 cache->dirty_tail.cache_seg->cache_seg_id, 422 cache->dirty_tail.seg_off, 423 cache->key_tail.cache_seg->cache_seg_id, 424 cache->key_tail.seg_off); 425 break; 426 case STATUSTYPE_TABLE: 427 DMEMIT("%s %s 4 cache_mode writeback crc %s", 428 cache_dev->dm_dev->name, 429 backing_dev->dm_dev->name, 430 cache_data_crc_on(cache) ? "true" : "false"); 431 break; 432 case STATUSTYPE_IMA: 433 *result = '\0'; 434 break; 435 } 436 } 437 438 static int dm_pcache_message(struct dm_target *ti, unsigned int argc, 439 char **argv, char *result, unsigned int maxlen) 440 { 441 struct dm_pcache *pcache = ti->private; 442 unsigned long val; 443 444 if (argc != 2) 445 goto err; 446 447 if (!strcasecmp(argv[0], "gc_percent")) { 448 if (kstrtoul(argv[1], 10, &val)) 449 goto err; 450 451 return pcache_cache_set_gc_percent(&pcache->cache, val); 452 } 453 err: 454 return -EINVAL; 455 } 456 457 static struct target_type dm_pcache_target = { 458 .name = "pcache", 459 .version = {0, 1, 0}, 460 .module = THIS_MODULE, 461 .features = DM_TARGET_SINGLETON, 462 .ctr = dm_pcache_ctr, 463 .dtr = dm_pcache_dtr, 464 .map = dm_pcache_map_bio, 465 .status = dm_pcache_status, 466 .message = dm_pcache_message, 467 }; 468 469 static int __init dm_pcache_init(void) 470 { 471 int ret; 472 473 ret = pcache_backing_init(); 474 if (ret) 475 goto err; 476 477 ret = pcache_cache_init(); 478 if (ret) 479 goto backing_exit; 480 481 ret = dm_register_target(&dm_pcache_target); 482 if (ret) 483 goto cache_exit; 484 return 0; 485 486 cache_exit: 487 pcache_cache_exit(); 488 backing_exit: 489 pcache_backing_exit(); 490 err: 491 return ret; 492 } 493 module_init(dm_pcache_init); 494 495 static void __exit dm_pcache_exit(void) 496 { 497 dm_unregister_target(&dm_pcache_target); 498 pcache_cache_exit(); 499 pcache_backing_exit(); 500 } 501 module_exit(dm_pcache_exit); 502 503 MODULE_DESCRIPTION("dm-pcache Persistent Cache for block device"); 504 MODULE_AUTHOR("Dongsheng Yang <dongsheng.yang@linux.dev>"); 505 MODULE_LICENSE("GPL"); 506