Lines Matching +full:cache +full:- +full:op +full:- +full:block +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2009-2011 Red Hat, Inc.
10 #include <linux/dm-bufio.h>
12 #include <linux/device-mapper.h>
13 #include <linux/dm-io.h>
65 * dm_buffer->list_mode
71 /*--------------------------------------------------------------*/
99 /*--------------*/
103 lru->cursor = NULL; in lru_init()
104 lru->count = 0; in lru_init()
105 INIT_LIST_HEAD(&lru->iterators); in lru_init()
110 WARN_ON_ONCE(lru->cursor); in lru_destroy()
111 WARN_ON_ONCE(!list_empty(&lru->iterators)); in lru_destroy()
123 atomic_set(&le->referenced, 0); in lru_insert()
125 if (lru->cursor) { in lru_insert()
126 list_add_tail(&le->list, lru->cursor); in lru_insert()
128 INIT_LIST_HEAD(&le->list); in lru_insert()
129 lru->cursor = &le->list; in lru_insert()
131 lru->count++; in lru_insert()
134 /*--------------*/
149 it->lru = lru; in lru_iter_begin()
150 it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL; in lru_iter_begin()
151 it->e = lru->cursor ? to_le(lru->cursor) : NULL; in lru_iter_begin()
152 list_add(&it->list, &lru->iterators); in lru_iter_begin()
160 list_del(&it->list); in lru_iter_end()
176 while (it->e) { in lru_iter_next()
177 e = it->e; in lru_iter_next()
180 if (it->e == it->stop) in lru_iter_next()
181 it->e = NULL; in lru_iter_next()
183 it->e = to_le(it->e->list.next); in lru_iter_next()
200 list_for_each_entry(it, &lru->iterators, list) { in lru_iter_invalidate()
201 /* Move c->e forwards if necc. */ in lru_iter_invalidate()
202 if (it->e == e) { in lru_iter_invalidate()
203 it->e = to_le(it->e->list.next); in lru_iter_invalidate()
204 if (it->e == e) in lru_iter_invalidate()
205 it->e = NULL; in lru_iter_invalidate()
208 /* Move it->stop backwards if necc. */ in lru_iter_invalidate()
209 if (it->stop == e) { in lru_iter_invalidate()
210 it->stop = to_le(it->stop->list.prev); in lru_iter_invalidate()
211 if (it->stop == e) in lru_iter_invalidate()
212 it->stop = NULL; in lru_iter_invalidate()
217 /*--------------*/
225 if (lru->count == 1) { in lru_remove()
226 lru->cursor = NULL; in lru_remove()
228 if (lru->cursor == &le->list) in lru_remove()
229 lru->cursor = lru->cursor->next; in lru_remove()
230 list_del(&le->list); in lru_remove()
232 lru->count--; in lru_remove()
240 atomic_set(&le->referenced, 1); in lru_reference()
243 /*--------------*/
260 struct list_head *h = lru->cursor; in lru_evict()
270 while (tested < lru->count) { in lru_evict()
273 if (atomic_read(&le->referenced)) { in lru_evict()
274 atomic_set(&le->referenced, 0); in lru_evict()
283 lru->cursor = le->list.next; in lru_evict()
291 lru->cursor = le->list.next; in lru_evict()
296 h = h->next; in lru_evict()
305 /*--------------------------------------------------------------*/
315 * Describes how the block was allocated:
331 sector_t block; member
365 /*--------------------------------------------------------------*/
368 * The buffer cache manages buffers, particularly:
369 * - inc/dec of holder count
370 * - setting the last_accessed field
371 * - maintains clean/dirty state along with lru
372 * - selecting buffers that match predicates
375 * - allocation/freeing of buffers.
376 * - IO
377 * - Eviction or cache sizing.
406 static inline unsigned int cache_index(sector_t block, unsigned int num_locks) in cache_index() argument
408 return dm_hash_locks_index(block, num_locks); in cache_index()
411 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) in cache_read_lock() argument
413 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_lock()
414 read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_lock()
416 down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_lock()
419 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_read_unlock() argument
421 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_unlock()
422 read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_unlock()
424 up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_unlock()
427 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) in cache_write_lock() argument
429 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_lock()
430 write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_lock()
432 down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_lock()
435 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_write_unlock() argument
437 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_unlock()
438 write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_unlock()
440 up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_unlock()
448 struct dm_buffer_cache *cache; member
454 static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write) in lh_init() argument
456 lh->cache = cache; in lh_init()
457 lh->write = write; in lh_init()
458 lh->no_previous = cache->num_locks; in lh_init()
459 lh->previous = lh->no_previous; in lh_init()
464 if (lh->write) { in __lh_lock()
465 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
466 write_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
468 down_write(&lh->cache->trees[index].u.lock); in __lh_lock()
470 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
471 read_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
473 down_read(&lh->cache->trees[index].u.lock); in __lh_lock()
479 if (lh->write) { in __lh_unlock()
480 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
481 write_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
483 up_write(&lh->cache->trees[index].u.lock); in __lh_unlock()
485 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
486 read_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
488 up_read(&lh->cache->trees[index].u.lock); in __lh_unlock()
497 if (lh->previous != lh->no_previous) { in lh_exit()
498 __lh_unlock(lh, lh->previous); in lh_exit()
499 lh->previous = lh->no_previous; in lh_exit()
509 unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */ in lh_next()
511 if (lh->previous != lh->no_previous) { in lh_next()
512 if (lh->previous != index) { in lh_next()
513 __lh_unlock(lh, lh->previous); in lh_next()
515 lh->previous = index; in lh_next()
519 lh->previous = index; in lh_next()
539 bc->num_locks = num_locks; in cache_init()
540 bc->no_sleep = no_sleep; in cache_init()
542 for (i = 0; i < bc->num_locks; i++) { in cache_init()
544 rwlock_init(&bc->trees[i].u.spinlock); in cache_init()
546 init_rwsem(&bc->trees[i].u.lock); in cache_init()
547 bc->trees[i].root = RB_ROOT; in cache_init()
550 lru_init(&bc->lru[LIST_CLEAN]); in cache_init()
551 lru_init(&bc->lru[LIST_DIRTY]); in cache_init()
558 for (i = 0; i < bc->num_locks; i++) in cache_destroy()
559 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root)); in cache_destroy()
561 lru_destroy(&bc->lru[LIST_CLEAN]); in cache_destroy()
562 lru_destroy(&bc->lru[LIST_DIRTY]); in cache_destroy()
565 /*--------------*/
572 return bc->lru[list_mode].count; in cache_count()
580 /*--------------*/
583 * Gets a specific buffer, indexed by block.
589 static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block) in __cache_get() argument
591 struct rb_node *n = root->rb_node; in __cache_get()
597 if (b->block == block) in __cache_get()
600 n = block < b->block ? n->rb_left : n->rb_right; in __cache_get()
608 atomic_inc(&b->hold_count); in __cache_inc_buffer()
609 WRITE_ONCE(b->last_accessed, jiffies); in __cache_inc_buffer()
612 static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) in cache_get() argument
616 cache_read_lock(bc, block); in cache_get()
617 b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); in cache_get()
619 lru_reference(&b->lru); in cache_get()
622 cache_read_unlock(bc, block); in cache_get()
627 /*--------------*/
637 cache_read_lock(bc, b->block); in cache_put()
638 BUG_ON(!atomic_read(&b->hold_count)); in cache_put()
639 r = atomic_dec_and_test(&b->hold_count); in cache_put()
640 cache_read_unlock(bc, b->block); in cache_put()
645 /*--------------*/
669 lh_next(w->lh, b->block); in __evict_pred()
671 if (atomic_read(&b->hold_count)) in __evict_pred()
674 return w->pred(b, w->context); in __evict_pred()
685 le = lru_evict(&bc->lru[list_mode], __evict_pred, &w, bc->no_sleep); in __cache_evict()
691 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in __cache_evict()
709 /*--------------*/
716 cache_write_lock(bc, b->block); in cache_mark()
717 if (list_mode != b->list_mode) { in cache_mark()
718 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_mark()
719 b->list_mode = list_mode; in cache_mark()
720 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_mark()
722 cache_write_unlock(bc, b->block); in cache_mark()
725 /*--------------*/
739 le = lru_evict(&bc->lru[old_mode], __evict_pred, &w, bc->no_sleep); in __cache_mark_many()
744 b->list_mode = new_mode; in __cache_mark_many()
745 lru_insert(&bc->lru[b->list_mode], &b->lru); in __cache_mark_many()
759 /*--------------*/
780 struct lru *lru = &bc->lru[list_mode]; in __cache_iterate()
783 if (!lru->cursor) in __cache_iterate()
786 first = le = to_le(lru->cursor); in __cache_iterate()
790 lh_next(lh, b->block); in __cache_iterate()
801 le = to_le(le->list.next); in __cache_iterate()
815 /*--------------*/
818 * Passes ownership of the buffer to the cache. Returns false if the
828 struct rb_node **new = &root->rb_node, *parent = NULL; in __cache_insert()
834 if (found->block == b->block) in __cache_insert()
838 new = b->block < found->block ? in __cache_insert()
839 &found->node.rb_left : &found->node.rb_right; in __cache_insert()
842 rb_link_node(&b->node, parent, new); in __cache_insert()
843 rb_insert_color(&b->node, root); in __cache_insert()
852 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) in cache_insert()
855 cache_write_lock(bc, b->block); in cache_insert()
856 BUG_ON(atomic_read(&b->hold_count) != 1); in cache_insert()
857 r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); in cache_insert()
859 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_insert()
860 cache_write_unlock(bc, b->block); in cache_insert()
865 /*--------------*/
868 * Removes buffer from cache, ownership of the buffer passes back to the caller.
877 cache_write_lock(bc, b->block); in cache_remove()
879 if (atomic_read(&b->hold_count) != 1) { in cache_remove()
883 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in cache_remove()
884 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_remove()
887 cache_write_unlock(bc, b->block); in cache_remove()
892 /*--------------*/
896 static struct dm_buffer *__find_next(struct rb_root *root, sector_t block) in __find_next() argument
898 struct rb_node *n = root->rb_node; in __find_next()
905 if (b->block == block) in __find_next()
908 if (block <= b->block) { in __find_next()
909 n = n->rb_left; in __find_next()
912 n = n->rb_right; in __find_next()
930 if (!b || (b->block >= end)) in __remove_range()
933 begin = b->block + 1; in __remove_range()
935 if (atomic_read(&b->hold_count)) in __remove_range()
939 rb_erase(&b->node, root); in __remove_range()
940 lru_remove(&bc->lru[b->list_mode], &b->lru); in __remove_range()
952 BUG_ON(bc->no_sleep); in cache_remove_range()
953 for (i = 0; i < bc->num_locks; i++) { in cache_remove_range()
954 down_write(&bc->trees[i].u.lock); in cache_remove_range()
955 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release); in cache_remove_range()
956 up_write(&bc->trees[i].u.lock); in cache_remove_range()
960 /*----------------------------------------------------------------*/
973 * context), so some clean-not-writing buffers can be held on
1014 struct dm_buffer_cache cache; /* must be last member */ member
1017 /*----------------------------------------------------------------*/
1019 #define dm_bufio_in_request() (!!current->bio_list)
1023 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_lock()
1024 spin_lock_bh(&c->spinlock); in dm_bufio_lock()
1026 mutex_lock_nested(&c->lock, dm_bufio_in_request()); in dm_bufio_lock()
1031 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_unlock()
1032 spin_unlock_bh(&c->spinlock); in dm_bufio_unlock()
1034 mutex_unlock(&c->lock); in dm_bufio_unlock()
1037 /*----------------------------------------------------------------*/
1040 * Default cache size: available memory divided by the ratio.
1045 * Total cache size set by the user.
1051 * at any time. If it disagrees, the user has changed cache size.
1069 /*----------------------------------------------------------------*/
1094 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); in buffer_record_stack()
1098 /*----------------------------------------------------------------*/
1111 data_mode = b->data_mode; in adjust_total_allocated()
1112 diff = (long)b->c->block_size; in adjust_total_allocated()
1114 diff = -diff; in adjust_total_allocated()
1134 * Change the number of clients and recalculate per-client limit.
1146 * Use default if set to 0 and report the actual cache size used. in __cache_size_refresh()
1179 if (unlikely(c->slab_cache != NULL)) { in alloc_buffer_data()
1181 return kmem_cache_alloc(c->slab_cache, gfp_mask); in alloc_buffer_data()
1184 if (c->block_size <= KMALLOC_MAX_SIZE && in alloc_buffer_data()
1188 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in alloc_buffer_data()
1193 return __vmalloc(c->block_size, gfp_mask); in alloc_buffer_data()
1204 kmem_cache_free(c->slab_cache, data); in free_buffer_data()
1209 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in free_buffer_data()
1228 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); in alloc_buffer()
1233 b->c = c; in alloc_buffer()
1235 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); in alloc_buffer()
1236 if (!b->data) { in alloc_buffer()
1237 kmem_cache_free(c->slab_buffer, b); in alloc_buffer()
1243 b->stack_len = 0; in alloc_buffer()
1253 struct dm_bufio_client *c = b->c; in free_buffer()
1256 free_buffer_data(c, b->data, b->data_mode); in free_buffer()
1257 kmem_cache_free(c->slab_buffer, b); in free_buffer()
1261 *--------------------------------------------------------------------------
1266 * memory-consumption per buffer, so it is not viable);
1268 * the memory must be direct-mapped, not vmalloced;
1274 * rejects the bio because it is too large, use dm-io layer to do the I/O.
1275 * The dm-io layer splits the I/O into multiple requests, avoiding the above
1277 *--------------------------------------------------------------------------
1281 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
1288 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); in dmio_complete()
1291 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, in use_dmio() argument
1297 .bi_opf = op, in use_dmio()
1300 .client = b->c->dm_io, in use_dmio()
1303 .bdev = b->c->bdev, in use_dmio()
1308 if (b->data_mode != DATA_MODE_VMALLOC) { in use_dmio()
1310 io_req.mem.ptr.addr = (char *)b->data + offset; in use_dmio()
1313 io_req.mem.ptr.vma = (char *)b->data + offset; in use_dmio()
1318 b->end_io(b, errno_to_blk_status(r)); in use_dmio()
1323 struct dm_buffer *b = bio->bi_private; in bio_complete()
1324 blk_status_t status = bio->bi_status; in bio_complete()
1328 b->end_io(b, status); in bio_complete()
1331 static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, in use_bio() argument
1341 use_dmio(b, op, sector, n_sectors, offset, ioprio); in use_bio()
1344 bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); in use_bio()
1345 bio->bi_iter.bi_sector = sector; in use_bio()
1346 bio->bi_end_io = bio_complete; in use_bio()
1347 bio->bi_private = b; in use_bio()
1348 bio->bi_ioprio = ioprio; in use_bio()
1350 ptr = (char *)b->data + offset; in use_bio()
1358 static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block) in block_to_sector() argument
1362 if (likely(c->sectors_per_block_bits >= 0)) in block_to_sector()
1363 sector = block << c->sectors_per_block_bits; in block_to_sector()
1365 sector = block * (c->block_size >> SECTOR_SHIFT); in block_to_sector()
1366 sector += c->start; in block_to_sector()
1371 static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio, in submit_io() argument
1378 b->end_io = end_io; in submit_io()
1380 sector = block_to_sector(b->c, b->block); in submit_io()
1382 if (op != REQ_OP_WRITE) { in submit_io()
1383 n_sectors = b->c->block_size >> SECTOR_SHIFT; in submit_io()
1386 if (b->c->write_callback) in submit_io()
1387 b->c->write_callback(b); in submit_io()
1388 offset = b->write_start; in submit_io()
1389 end = b->write_end; in submit_io()
1390 offset &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1391 end += DM_BUFIO_WRITE_ALIGN - 1; in submit_io()
1392 end &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1393 if (unlikely(end > b->c->block_size)) in submit_io()
1394 end = b->c->block_size; in submit_io()
1397 n_sectors = (end - offset) >> SECTOR_SHIFT; in submit_io()
1400 if (b->data_mode != DATA_MODE_VMALLOC) in submit_io()
1401 use_bio(b, op, sector, n_sectors, offset, ioprio); in submit_io()
1403 use_dmio(b, op, sector, n_sectors, offset, ioprio); in submit_io()
1407 *--------------------------------------------------------------
1409 *--------------------------------------------------------------
1420 b->write_error = status; in write_endio()
1422 struct dm_bufio_client *c = b->c; in write_endio()
1424 (void)cmpxchg(&c->async_write_error, 0, in write_endio()
1428 BUG_ON(!test_bit(B_WRITING, &b->state)); in write_endio()
1431 clear_bit(B_WRITING, &b->state); in write_endio()
1434 wake_up_bit(&b->state, B_WRITING); in write_endio()
1440 * - If the buffer is not dirty, exit.
1441 * - If there some previous write going on, wait for it to finish (we can't
1443 * - Submit our write and don't wait on it. We set B_WRITING indicating
1449 if (!test_bit(B_DIRTY, &b->state)) in __write_dirty_buffer()
1452 clear_bit(B_DIRTY, &b->state); in __write_dirty_buffer()
1453 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __write_dirty_buffer()
1455 b->write_start = b->dirty_start; in __write_dirty_buffer()
1456 b->write_end = b->dirty_end; in __write_dirty_buffer()
1461 list_add_tail(&b->write_list, write_list); in __write_dirty_buffer()
1471 list_entry(write_list->next, struct dm_buffer, write_list); in __flush_write_list()
1472 list_del(&b->write_list); in __flush_write_list()
1486 BUG_ON(atomic_read(&b->hold_count)); in __make_buffer_clean()
1489 if (!smp_load_acquire(&b->state)) /* fast case */ in __make_buffer_clean()
1492 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1494 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1502 if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state))) in is_clean()
1504 if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state))) in is_clean()
1506 if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN)) in is_clean()
1509 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep && in is_clean()
1510 unlikely(test_bit(B_READING, &b->state))) in is_clean()
1519 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in is_dirty()
1521 if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY)) in is_dirty()
1535 b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c); in __get_unclaimed_buffer()
1542 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in __get_unclaimed_buffer()
1545 b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL); in __get_unclaimed_buffer()
1558 * This function is entered with c->lock held, drops it and regains it
1565 add_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1571 * hold c->lock when wake_up is called. So we have a timeout here, in __wait_for_free_buffer()
1576 remove_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1600 * dm-bufio is resistant to allocation failures (it just keeps in __alloc_buffer_wait_no_callback()
1609 * For debugging, if we set the cache size to 1, no new buffers will in __alloc_buffer_wait_no_callback()
1631 if (!list_empty(&c->reserved_buffers)) { in __alloc_buffer_wait_no_callback()
1632 b = list_to_buffer(c->reserved_buffers.next); in __alloc_buffer_wait_no_callback()
1633 list_del(&b->lru.list); in __alloc_buffer_wait_no_callback()
1634 c->need_reserved_buffers++; in __alloc_buffer_wait_no_callback()
1654 if (c->alloc_callback) in __alloc_buffer_wait()
1655 c->alloc_callback(b); in __alloc_buffer_wait()
1665 struct dm_bufio_client *c = b->c; in __free_buffer_wake()
1667 b->block = -1; in __free_buffer_wake()
1668 if (!c->need_reserved_buffers) in __free_buffer_wake()
1671 list_add(&b->lru.list, &c->reserved_buffers); in __free_buffer_wake()
1672 c->need_reserved_buffers--; in __free_buffer_wake()
1679 if (unlikely(waitqueue_active(&c->free_buffer_wait))) in __free_buffer_wake()
1680 wake_up(&c->free_buffer_wait); in __free_buffer_wake()
1685 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in cleaned()
1688 if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state)) in cleaned()
1696 cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL); in __move_clean_buffers()
1708 if (wc->no_wait && test_bit(B_WRITING, &b->state)) in write_one()
1711 __write_dirty_buffer(b, wc->write_list); in write_one()
1721 cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc); in __write_dirty_buffers_async()
1727 * If we're over "limit_buffers", block until we get under the limit.
1732 if (cache_count(&c->cache, LIST_DIRTY) > in __check_watermark()
1733 cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO) in __check_watermark()
1738 *--------------------------------------------------------------
1740 *--------------------------------------------------------------
1749 if (cache_put(&c->cache, b) && in cache_put_and_wake()
1750 unlikely(waitqueue_active(&c->free_buffer_wait))) in cache_put_and_wake()
1751 wake_up(&c->free_buffer_wait); in cache_put_and_wake()
1755 * This assumes you have already checked the cache to see if the buffer
1758 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, in __bufio_new() argument
1778 b = cache_get(&c->cache, block); in __bufio_new()
1787 atomic_set(&b->hold_count, 1); in __bufio_new()
1788 WRITE_ONCE(b->last_accessed, jiffies); in __bufio_new()
1789 b->block = block; in __bufio_new()
1790 b->read_error = 0; in __bufio_new()
1791 b->write_error = 0; in __bufio_new()
1792 b->list_mode = LIST_CLEAN; in __bufio_new()
1795 b->state = 0; in __bufio_new()
1797 b->state = 1 << B_READING; in __bufio_new()
1802 * We mustn't insert into the cache until the B_READING state in __bufio_new()
1806 cache_insert(&c->cache, b); in __bufio_new()
1823 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in __bufio_new()
1837 b->read_error = status; in read_endio()
1839 BUG_ON(!test_bit(B_READING, &b->state)); in read_endio()
1842 clear_bit(B_READING, &b->state); in read_endio()
1845 wake_up_bit(&b->state, B_READING); in read_endio()
1854 static void *new_read(struct dm_bufio_client *c, sector_t block, in new_read() argument
1866 * Fast path, hopefully the block is already in the cache. No need in new_read()
1869 b = cache_get(&c->cache, block); in new_read()
1883 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in new_read()
1894 b = __bufio_new(c, block, nf, &need_submit, &write_list); in new_read()
1899 if (b && (atomic_read(&b->hold_count) == 1)) in new_read()
1912 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in new_read()
1914 if (b->read_error) { in new_read()
1915 int error = blk_status_to_errno(b->read_error); in new_read()
1924 return b->data; in new_read()
1927 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, in dm_bufio_get() argument
1930 return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); in dm_bufio_get()
1934 static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, in __dm_bufio_read() argument
1938 return ERR_PTR(-EINVAL); in __dm_bufio_read()
1940 return new_read(c, block, NF_READ, bp, ioprio); in __dm_bufio_read()
1943 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, in dm_bufio_read() argument
1946 return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); in dm_bufio_read()
1950 void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, in dm_bufio_read_with_ioprio() argument
1953 return __dm_bufio_read(c, block, bp, ioprio); in dm_bufio_read_with_ioprio()
1957 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, in dm_bufio_new() argument
1961 return ERR_PTR(-EINVAL); in dm_bufio_new()
1963 return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); in dm_bufio_new()
1968 sector_t block, unsigned int n_blocks, in __dm_bufio_prefetch() argument
1980 for (; n_blocks--; block++) { in __dm_bufio_prefetch()
1984 b = cache_get(&c->cache, block); in __dm_bufio_prefetch()
1986 /* already in cache */ in __dm_bufio_prefetch()
1992 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, in __dm_bufio_prefetch()
2021 void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) in dm_bufio_prefetch() argument
2023 return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); in dm_bufio_prefetch()
2027 void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, in dm_bufio_prefetch_with_ioprio() argument
2030 return __dm_bufio_prefetch(c, block, n_blocks, ioprio); in dm_bufio_prefetch_with_ioprio()
2036 struct dm_bufio_client *c = b->c; in dm_bufio_release()
2043 if ((b->read_error || b->write_error) && in dm_bufio_release()
2044 !test_bit_acquire(B_READING, &b->state) && in dm_bufio_release()
2045 !test_bit(B_WRITING, &b->state) && in dm_bufio_release()
2046 !test_bit(B_DIRTY, &b->state)) { in dm_bufio_release()
2049 /* cache remove can fail if there are other holders */ in dm_bufio_release()
2050 if (cache_remove(&c->cache, b)) { in dm_bufio_release()
2066 struct dm_bufio_client *c = b->c; in dm_bufio_mark_partial_buffer_dirty()
2069 BUG_ON(end > b->c->block_size); in dm_bufio_mark_partial_buffer_dirty()
2073 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_mark_partial_buffer_dirty()
2075 if (!test_and_set_bit(B_DIRTY, &b->state)) { in dm_bufio_mark_partial_buffer_dirty()
2076 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2077 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2078 cache_mark(&c->cache, b, LIST_DIRTY); in dm_bufio_mark_partial_buffer_dirty()
2080 if (start < b->dirty_start) in dm_bufio_mark_partial_buffer_dirty()
2081 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2082 if (end > b->dirty_end) in dm_bufio_mark_partial_buffer_dirty()
2083 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2092 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); in dm_bufio_mark_buffer_dirty()
2112 * and simultaneously (so that the block layer can merge the writes) and then
2115 * Finally, we flush hardware disk cache.
2121 return test_bit(B_WRITING, &b->state); in is_writing()
2139 nr_buffers = cache_count(&c->cache, LIST_DIRTY); in dm_bufio_write_dirty_buffers()
2140 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); in dm_bufio_write_dirty_buffers()
2145 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_write_dirty_buffers()
2148 nr_buffers--; in dm_bufio_write_dirty_buffers()
2150 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2153 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2156 if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) in dm_bufio_write_dirty_buffers()
2157 cache_mark(&c->cache, b, LIST_CLEAN); in dm_bufio_write_dirty_buffers()
2165 wake_up(&c->free_buffer_wait); in dm_bufio_write_dirty_buffers()
2168 a = xchg(&c->async_write_error, 0); in dm_bufio_write_dirty_buffers()
2178 * Use dm-io to send an empty barrier to flush the device.
2186 .client = c->dm_io, in dm_bufio_issue_flush()
2189 .bdev = c->bdev, in dm_bufio_issue_flush()
2195 return -EINVAL; in dm_bufio_issue_flush()
2202 * Use dm-io to send a discard request to flush the device.
2204 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count) in dm_bufio_issue_discard() argument
2210 .client = c->dm_io, in dm_bufio_issue_discard()
2213 .bdev = c->bdev, in dm_bufio_issue_discard()
2214 .sector = block_to_sector(c, block), in dm_bufio_issue_discard()
2219 return -EINVAL; /* discards are optional */ in dm_bufio_issue_discard()
2225 static bool forget_buffer(struct dm_bufio_client *c, sector_t block) in forget_buffer() argument
2229 b = cache_get(&c->cache, block); in forget_buffer()
2231 if (likely(!smp_load_acquire(&b->state))) { in forget_buffer()
2232 if (cache_remove(&c->cache, b)) in forget_buffer()
2250 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block) in dm_bufio_forget() argument
2253 forget_buffer(c, block); in dm_bufio_forget()
2260 return b->state ? ER_DONT_EVICT : ER_EVICT; in idle()
2263 void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks) in dm_bufio_forget_buffers() argument
2266 cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake); in dm_bufio_forget_buffers()
2273 c->minimum_buffers = n; in dm_bufio_set_minimum_buffers()
2279 return c->block_size; in dm_bufio_get_block_size()
2285 sector_t s = bdev_nr_sectors(c->bdev); in dm_bufio_get_device_size()
2287 if (s >= c->start) in dm_bufio_get_device_size()
2288 s -= c->start; in dm_bufio_get_device_size()
2291 if (likely(c->sectors_per_block_bits >= 0)) in dm_bufio_get_device_size()
2292 s >>= c->sectors_per_block_bits; in dm_bufio_get_device_size()
2294 sector_div(s, c->block_size >> SECTOR_SHIFT); in dm_bufio_get_device_size()
2301 return c->dm_io; in dm_bufio_get_dm_io_client()
2307 return b->block; in dm_bufio_get_block_number()
2313 return b->data; in dm_bufio_get_block_data()
2325 return b->c; in dm_bufio_get_client()
2336 (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode); in warn_leak()
2338 stack_trace_print(b->stack_entries, b->stack_len, 1); in warn_leak()
2340 atomic_set(&b->hold_count, 0); in warn_leak()
2354 * An optimization so that the buffers are not written one-by-one. in drop_buffers()
2366 cache_iterate(&c->cache, i, warn_leak, &warned); in drop_buffers()
2375 WARN_ON(cache_count(&c->cache, i)); in drop_buffers()
2384 if (likely(c->sectors_per_block_bits >= 0)) in get_retain_buffers()
2385 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; in get_retain_buffers()
2387 retain_bytes /= c->block_size; in get_retain_buffers()
2398 unsigned long count = cache_total(&c->cache); in __scan()
2402 if (count - freed <= retain_target) in __scan()
2403 atomic_long_set(&c->need_shrink, 0); in __scan()
2404 if (!atomic_long_read(&c->need_shrink)) in __scan()
2407 b = cache_evict(&c->cache, l, in __scan()
2415 atomic_long_dec(&c->need_shrink); in __scan()
2435 c = shrink->private_data; in dm_bufio_shrink_scan()
2436 atomic_long_add(sc->nr_to_scan, &c->need_shrink); in dm_bufio_shrink_scan()
2437 queue_work(dm_bufio_wq, &c->shrink_work); in dm_bufio_shrink_scan()
2439 return sc->nr_to_scan; in dm_bufio_shrink_scan()
2444 struct dm_bufio_client *c = shrink->private_data; in dm_bufio_shrink_count()
2445 unsigned long count = cache_total(&c->cache); in dm_bufio_shrink_count()
2447 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink); in dm_bufio_shrink_count()
2452 count -= retain_target; in dm_bufio_shrink_count()
2457 count -= queued_for_cleanup; in dm_bufio_shrink_count()
2477 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { in dm_bufio_client_create()
2478 DMERR("%s: block size not specified or is not multiple of 512b", __func__); in dm_bufio_client_create()
2479 r = -EINVAL; in dm_bufio_client_create()
2486 r = -ENOMEM; in dm_bufio_client_create()
2489 cache_init(&c->cache, num_locks, (flags & DM_BUFIO_CLIENT_NO_SLEEP) != 0); in dm_bufio_client_create()
2491 c->bdev = bdev; in dm_bufio_client_create()
2492 c->block_size = block_size; in dm_bufio_client_create()
2494 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; in dm_bufio_client_create()
2496 c->sectors_per_block_bits = -1; in dm_bufio_client_create()
2498 c->alloc_callback = alloc_callback; in dm_bufio_client_create()
2499 c->write_callback = write_callback; in dm_bufio_client_create()
2502 c->no_sleep = true; in dm_bufio_client_create()
2506 mutex_init(&c->lock); in dm_bufio_client_create()
2507 spin_lock_init(&c->spinlock); in dm_bufio_client_create()
2508 INIT_LIST_HEAD(&c->reserved_buffers); in dm_bufio_client_create()
2509 c->need_reserved_buffers = reserved_buffers; in dm_bufio_client_create()
2513 init_waitqueue_head(&c->free_buffer_wait); in dm_bufio_client_create()
2514 c->async_write_error = 0; in dm_bufio_client_create()
2516 c->dm_io = dm_io_client_create(); in dm_bufio_client_create()
2517 if (IS_ERR(c->dm_io)) { in dm_bufio_client_create()
2518 r = PTR_ERR(c->dm_io); in dm_bufio_client_create()
2526 snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u-%u", in dm_bufio_client_create()
2528 c->slab_cache = kmem_cache_create(slab_name, block_size, align, in dm_bufio_client_create()
2530 if (!c->slab_cache) { in dm_bufio_client_create()
2531 r = -ENOMEM; in dm_bufio_client_create()
2536 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u-%u", in dm_bufio_client_create()
2539 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", in dm_bufio_client_create()
2541 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, in dm_bufio_client_create()
2543 if (!c->slab_buffer) { in dm_bufio_client_create()
2544 r = -ENOMEM; in dm_bufio_client_create()
2548 while (c->need_reserved_buffers) { in dm_bufio_client_create()
2552 r = -ENOMEM; in dm_bufio_client_create()
2558 INIT_WORK(&c->shrink_work, shrink_work); in dm_bufio_client_create()
2559 atomic_long_set(&c->need_shrink, 0); in dm_bufio_client_create()
2561 c->shrinker = shrinker_alloc(0, "dm-bufio:(%u:%u)", in dm_bufio_client_create()
2562 MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); in dm_bufio_client_create()
2563 if (!c->shrinker) { in dm_bufio_client_create()
2564 r = -ENOMEM; in dm_bufio_client_create()
2568 c->shrinker->count_objects = dm_bufio_shrink_count; in dm_bufio_client_create()
2569 c->shrinker->scan_objects = dm_bufio_shrink_scan; in dm_bufio_client_create()
2570 c->shrinker->seeks = 1; in dm_bufio_client_create()
2571 c->shrinker->batch = 0; in dm_bufio_client_create()
2572 c->shrinker->private_data = c; in dm_bufio_client_create()
2574 shrinker_register(c->shrinker); in dm_bufio_client_create()
2578 list_add(&c->client_list, &dm_bufio_all_clients); in dm_bufio_client_create()
2585 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_create()
2586 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_create()
2588 list_del(&b->lru.list); in dm_bufio_client_create()
2591 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_create()
2592 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_create()
2593 dm_io_client_destroy(c->dm_io); in dm_bufio_client_create()
2595 mutex_destroy(&c->lock); in dm_bufio_client_create()
2596 if (c->no_sleep) in dm_bufio_client_create()
2614 shrinker_free(c->shrinker); in dm_bufio_client_destroy()
2615 flush_work(&c->shrink_work); in dm_bufio_client_destroy()
2619 list_del(&c->client_list); in dm_bufio_client_destroy()
2620 dm_bufio_client_count--; in dm_bufio_client_destroy()
2625 WARN_ON(c->need_reserved_buffers); in dm_bufio_client_destroy()
2627 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_destroy()
2628 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_destroy()
2630 list_del(&b->lru.list); in dm_bufio_client_destroy()
2635 if (cache_count(&c->cache, i)) in dm_bufio_client_destroy()
2636 DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2639 WARN_ON(cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2641 cache_destroy(&c->cache); in dm_bufio_client_destroy()
2642 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_destroy()
2643 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_destroy()
2644 dm_io_client_destroy(c->dm_io); in dm_bufio_client_destroy()
2645 mutex_destroy(&c->lock); in dm_bufio_client_destroy()
2646 if (c->no_sleep) in dm_bufio_client_destroy()
2655 flush_work(&c->shrink_work); in dm_bufio_client_reset()
2661 c->start = start; in dm_bufio_set_sector_offset()
2665 /*--------------------------------------------------------------*/
2679 return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz); in older_than()
2706 if (!(params->gfp & __GFP_FS) || in select_for_evict()
2707 (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) { in select_for_evict()
2708 if (test_bit_acquire(B_READING, &b->state) || in select_for_evict()
2709 test_bit(B_WRITING, &b->state) || in select_for_evict()
2710 test_bit(B_DIRTY, &b->state)) in select_for_evict()
2714 return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP; in select_for_evict()
2726 b = cache_evict(&c->cache, list_mode, select_for_evict, params); in __evict_many()
2730 last_accessed = READ_ONCE(b->last_accessed); in __evict_many()
2731 if (time_after_eq(params->last_accessed, last_accessed)) in __evict_many()
2732 params->last_accessed = last_accessed; in __evict_many()
2759 count = cache_total(&c->cache); in evict_old_buffers()
2761 __evict_many(c, ¶ms, LIST_CLEAN, count - retain); in evict_old_buffers()
2789 /*--------------------------------------------------------------*/
2820 if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer)) in __insert_client()
2822 h = h->next; in __insert_client()
2825 list_add_tail(&new_client->client_list, h); in __insert_client()
2848 c->oldest_buffer = params.last_accessed; in __evict_a_few()
2872 unsigned long threshold = dm_bufio_cache_size - in evict_old()
2891 *--------------------------------------------------------------
2893 *--------------------------------------------------------------
2909 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), in dm_bufio_init()
2928 return -ENOMEM; in dm_bufio_init()
2979 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
3000 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
3002 MODULE_AUTHOR("Mikulas Patocka <dm-devel@lists.linux.dev>");