1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/blkdev.h> 3 4 #include "../dm-core.h" 5 #include "pcache_internal.h" 6 #include "cache_dev.h" 7 #include "backing_dev.h" 8 #include "cache.h" 9 #include "dm_pcache.h" 10 11 static struct kmem_cache *backing_req_cache; 12 static struct kmem_cache *backing_bvec_cache; 13 14 static void backing_dev_exit(struct pcache_backing_dev *backing_dev) 15 { 16 mempool_exit(&backing_dev->req_pool); 17 mempool_exit(&backing_dev->bvec_pool); 18 } 19 20 static void req_submit_fn(struct work_struct *work); 21 static void req_complete_fn(struct work_struct *work); 22 static int backing_dev_init(struct dm_pcache *pcache) 23 { 24 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 25 int ret; 26 27 ret = mempool_init_slab_pool(&backing_dev->req_pool, 128, backing_req_cache); 28 if (ret) 29 goto err; 30 31 ret = mempool_init_slab_pool(&backing_dev->bvec_pool, 128, backing_bvec_cache); 32 if (ret) 33 goto req_pool_exit; 34 35 INIT_LIST_HEAD(&backing_dev->submit_list); 36 INIT_LIST_HEAD(&backing_dev->complete_list); 37 spin_lock_init(&backing_dev->submit_lock); 38 spin_lock_init(&backing_dev->complete_lock); 39 INIT_WORK(&backing_dev->req_submit_work, req_submit_fn); 40 INIT_WORK(&backing_dev->req_complete_work, req_complete_fn); 41 atomic_set(&backing_dev->inflight_reqs, 0); 42 init_waitqueue_head(&backing_dev->inflight_wq); 43 44 return 0; 45 46 req_pool_exit: 47 mempool_exit(&backing_dev->req_pool); 48 err: 49 return ret; 50 } 51 52 int backing_dev_start(struct dm_pcache *pcache) 53 { 54 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 55 int ret; 56 57 ret = backing_dev_init(pcache); 58 if (ret) 59 return ret; 60 61 backing_dev->dev_size = bdev_nr_sectors(backing_dev->dm_dev->bdev); 62 63 return 0; 64 } 65 66 void backing_dev_stop(struct dm_pcache *pcache) 67 { 68 struct pcache_backing_dev *backing_dev = &pcache->backing_dev; 69 70 /* 71 * There should not be any new request comming, just wait 72 * inflight requests done. 73 */ 74 wait_event(backing_dev->inflight_wq, 75 atomic_read(&backing_dev->inflight_reqs) == 0); 76 77 flush_work(&backing_dev->req_submit_work); 78 flush_work(&backing_dev->req_complete_work); 79 80 backing_dev_exit(backing_dev); 81 } 82 83 /* pcache_backing_dev_req functions */ 84 void backing_dev_req_end(struct pcache_backing_dev_req *backing_req) 85 { 86 struct pcache_backing_dev *backing_dev = backing_req->backing_dev; 87 88 if (backing_req->end_req) 89 backing_req->end_req(backing_req, backing_req->ret); 90 91 switch (backing_req->type) { 92 case BACKING_DEV_REQ_TYPE_REQ: 93 if (backing_req->req.upper_req) 94 pcache_req_put(backing_req->req.upper_req, backing_req->ret); 95 break; 96 case BACKING_DEV_REQ_TYPE_KMEM: 97 if (backing_req->kmem.bvecs != backing_req->kmem.inline_bvecs) 98 mempool_free(backing_req->kmem.bvecs, &backing_dev->bvec_pool); 99 break; 100 default: 101 BUG(); 102 } 103 104 mempool_free(backing_req, &backing_dev->req_pool); 105 106 if (atomic_dec_and_test(&backing_dev->inflight_reqs)) 107 wake_up(&backing_dev->inflight_wq); 108 } 109 110 static void req_complete_fn(struct work_struct *work) 111 { 112 struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_complete_work); 113 struct pcache_backing_dev_req *backing_req; 114 LIST_HEAD(tmp_list); 115 116 spin_lock_irq(&backing_dev->complete_lock); 117 list_splice_init(&backing_dev->complete_list, &tmp_list); 118 spin_unlock_irq(&backing_dev->complete_lock); 119 120 while (!list_empty(&tmp_list)) { 121 backing_req = list_first_entry(&tmp_list, 122 struct pcache_backing_dev_req, node); 123 list_del_init(&backing_req->node); 124 backing_dev_req_end(backing_req); 125 } 126 } 127 128 static void backing_dev_bio_end(struct bio *bio) 129 { 130 struct pcache_backing_dev_req *backing_req = bio->bi_private; 131 struct pcache_backing_dev *backing_dev = backing_req->backing_dev; 132 unsigned long flags; 133 134 backing_req->ret = blk_status_to_errno(bio->bi_status); 135 136 spin_lock_irqsave(&backing_dev->complete_lock, flags); 137 list_move_tail(&backing_req->node, &backing_dev->complete_list); 138 queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_complete_work); 139 spin_unlock_irqrestore(&backing_dev->complete_lock, flags); 140 } 141 142 static void req_submit_fn(struct work_struct *work) 143 { 144 struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_submit_work); 145 struct pcache_backing_dev_req *backing_req; 146 LIST_HEAD(tmp_list); 147 148 spin_lock(&backing_dev->submit_lock); 149 list_splice_init(&backing_dev->submit_list, &tmp_list); 150 spin_unlock(&backing_dev->submit_lock); 151 152 while (!list_empty(&tmp_list)) { 153 backing_req = list_first_entry(&tmp_list, 154 struct pcache_backing_dev_req, node); 155 list_del_init(&backing_req->node); 156 submit_bio_noacct(&backing_req->bio); 157 } 158 } 159 160 void backing_dev_req_submit(struct pcache_backing_dev_req *backing_req, bool direct) 161 { 162 struct pcache_backing_dev *backing_dev = backing_req->backing_dev; 163 164 if (direct) { 165 submit_bio_noacct(&backing_req->bio); 166 return; 167 } 168 169 spin_lock(&backing_dev->submit_lock); 170 list_add_tail(&backing_req->node, &backing_dev->submit_list); 171 queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_submit_work); 172 spin_unlock(&backing_dev->submit_lock); 173 } 174 175 static void bio_map(struct bio *bio, void *base, size_t size) 176 { 177 struct page *page; 178 unsigned int offset; 179 unsigned int len; 180 181 if (!is_vmalloc_addr(base)) { 182 page = virt_to_page(base); 183 offset = offset_in_page(base); 184 185 BUG_ON(!bio_add_page(bio, page, size, offset)); 186 return; 187 } 188 189 flush_kernel_vmap_range(base, size); 190 while (size) { 191 page = vmalloc_to_page(base); 192 offset = offset_in_page(base); 193 len = min_t(size_t, PAGE_SIZE - offset, size); 194 195 BUG_ON(!bio_add_page(bio, page, len, offset)); 196 size -= len; 197 base += len; 198 } 199 } 200 201 static struct pcache_backing_dev_req *req_type_req_alloc(struct pcache_backing_dev *backing_dev, 202 struct pcache_backing_dev_req_opts *opts) 203 { 204 struct pcache_request *pcache_req = opts->req.upper_req; 205 struct pcache_backing_dev_req *backing_req; 206 struct bio *orig = pcache_req->bio; 207 208 backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask); 209 if (!backing_req) 210 return NULL; 211 212 memset(backing_req, 0, sizeof(struct pcache_backing_dev_req)); 213 214 bio_init_clone(backing_dev->dm_dev->bdev, &backing_req->bio, orig, opts->gfp_mask); 215 216 backing_req->type = BACKING_DEV_REQ_TYPE_REQ; 217 backing_req->backing_dev = backing_dev; 218 atomic_inc(&backing_dev->inflight_reqs); 219 220 return backing_req; 221 } 222 223 static struct pcache_backing_dev_req *kmem_type_req_alloc(struct pcache_backing_dev *backing_dev, 224 struct pcache_backing_dev_req_opts *opts) 225 { 226 struct pcache_backing_dev_req *backing_req; 227 u32 n_vecs = bio_add_max_vecs(opts->kmem.data, opts->kmem.len); 228 229 backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask); 230 if (!backing_req) 231 return NULL; 232 233 memset(backing_req, 0, sizeof(struct pcache_backing_dev_req)); 234 235 if (n_vecs > BACKING_DEV_REQ_INLINE_BVECS) { 236 backing_req->kmem.bvecs = mempool_alloc(&backing_dev->bvec_pool, opts->gfp_mask); 237 if (!backing_req->kmem.bvecs) 238 goto free_backing_req; 239 } else { 240 backing_req->kmem.bvecs = backing_req->kmem.inline_bvecs; 241 } 242 243 backing_req->kmem.n_vecs = n_vecs; 244 backing_req->type = BACKING_DEV_REQ_TYPE_KMEM; 245 backing_req->backing_dev = backing_dev; 246 atomic_inc(&backing_dev->inflight_reqs); 247 248 return backing_req; 249 250 free_backing_req: 251 mempool_free(backing_req, &backing_dev->req_pool); 252 return NULL; 253 } 254 255 struct pcache_backing_dev_req *backing_dev_req_alloc(struct pcache_backing_dev *backing_dev, 256 struct pcache_backing_dev_req_opts *opts) 257 { 258 if (opts->type == BACKING_DEV_REQ_TYPE_REQ) 259 return req_type_req_alloc(backing_dev, opts); 260 261 if (opts->type == BACKING_DEV_REQ_TYPE_KMEM) 262 return kmem_type_req_alloc(backing_dev, opts); 263 264 BUG(); 265 } 266 267 static void req_type_req_init(struct pcache_backing_dev_req *backing_req, 268 struct pcache_backing_dev_req_opts *opts) 269 { 270 struct pcache_request *pcache_req = opts->req.upper_req; 271 struct bio *clone; 272 u32 off = opts->req.req_off; 273 u32 len = opts->req.len; 274 275 clone = &backing_req->bio; 276 BUG_ON(off & SECTOR_MASK); 277 BUG_ON(len & SECTOR_MASK); 278 bio_trim(clone, off >> SECTOR_SHIFT, len >> SECTOR_SHIFT); 279 280 clone->bi_iter.bi_sector = (pcache_req->off + off) >> SECTOR_SHIFT; 281 clone->bi_private = backing_req; 282 clone->bi_end_io = backing_dev_bio_end; 283 284 INIT_LIST_HEAD(&backing_req->node); 285 backing_req->end_req = opts->end_fn; 286 287 pcache_req_get(pcache_req); 288 backing_req->req.upper_req = pcache_req; 289 backing_req->req.bio_off = off; 290 } 291 292 static void kmem_type_req_init(struct pcache_backing_dev_req *backing_req, 293 struct pcache_backing_dev_req_opts *opts) 294 { 295 struct pcache_backing_dev *backing_dev = backing_req->backing_dev; 296 struct bio *backing_bio; 297 298 bio_init(&backing_req->bio, backing_dev->dm_dev->bdev, backing_req->kmem.bvecs, 299 backing_req->kmem.n_vecs, opts->kmem.opf); 300 301 backing_bio = &backing_req->bio; 302 bio_map(backing_bio, opts->kmem.data, opts->kmem.len); 303 304 backing_bio->bi_iter.bi_sector = (opts->kmem.backing_off) >> SECTOR_SHIFT; 305 backing_bio->bi_private = backing_req; 306 backing_bio->bi_end_io = backing_dev_bio_end; 307 308 INIT_LIST_HEAD(&backing_req->node); 309 backing_req->end_req = opts->end_fn; 310 backing_req->priv_data = opts->priv_data; 311 } 312 313 void backing_dev_req_init(struct pcache_backing_dev_req *backing_req, 314 struct pcache_backing_dev_req_opts *opts) 315 { 316 if (opts->type == BACKING_DEV_REQ_TYPE_REQ) 317 return req_type_req_init(backing_req, opts); 318 319 if (opts->type == BACKING_DEV_REQ_TYPE_KMEM) 320 return kmem_type_req_init(backing_req, opts); 321 322 BUG(); 323 } 324 325 struct pcache_backing_dev_req *backing_dev_req_create(struct pcache_backing_dev *backing_dev, 326 struct pcache_backing_dev_req_opts *opts) 327 { 328 struct pcache_backing_dev_req *backing_req; 329 330 backing_req = backing_dev_req_alloc(backing_dev, opts); 331 if (!backing_req) 332 return NULL; 333 334 backing_dev_req_init(backing_req, opts); 335 336 return backing_req; 337 } 338 339 void backing_dev_flush(struct pcache_backing_dev *backing_dev) 340 { 341 blkdev_issue_flush(backing_dev->dm_dev->bdev); 342 } 343 344 int pcache_backing_init(void) 345 { 346 u32 max_bvecs = (PCACHE_CACHE_SUBTREE_SIZE >> PAGE_SHIFT) + 1; 347 int ret; 348 349 backing_req_cache = KMEM_CACHE(pcache_backing_dev_req, 0); 350 if (!backing_req_cache) { 351 ret = -ENOMEM; 352 goto err; 353 } 354 355 backing_bvec_cache = kmem_cache_create("pcache-bvec-slab", 356 max_bvecs * sizeof(struct bio_vec), 357 0, 0, NULL); 358 if (!backing_bvec_cache) { 359 ret = -ENOMEM; 360 goto destroy_req_cache; 361 } 362 363 return 0; 364 destroy_req_cache: 365 kmem_cache_destroy(backing_req_cache); 366 err: 367 return ret; 368 } 369 370 void pcache_backing_exit(void) 371 { 372 kmem_cache_destroy(backing_bvec_cache); 373 kmem_cache_destroy(backing_req_cache); 374 } 375