1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include "cache.h" 3 #include "backing_dev.h" 4 #include "cache_dev.h" 5 #include "dm_pcache.h" 6 7 /** 8 * cache_key_gc - Releases the reference of a cache key segment. 9 * @key: Pointer to the cache key to be garbage collected. 10 * 11 * This function decrements the reference count of the cache segment 12 * associated with the given key. If the reference count drops to zero, 13 * the segment may be invalidated and reused. 14 */ 15 static void cache_key_gc(struct pcache_cache_key *key) 16 { 17 cache_seg_put(key->cache_pos.cache_seg); 18 } 19 20 static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_tail, struct pcache_cache_pos *key_tail) 21 { 22 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 23 struct pcache_cache_kset_onmedia *kset_onmedia; 24 void *dirty_addr, *key_addr; 25 u32 segs_used, segs_gc_threshold, to_copy; 26 int ret; 27 28 dirty_addr = cache_pos_addr(dirty_tail); 29 key_addr = cache_pos_addr(key_tail); 30 if (dirty_addr == key_addr) { 31 pcache_dev_debug(pcache, "key tail is equal to dirty tail: %u:%u\n", 32 dirty_tail->cache_seg->cache_seg_id, 33 dirty_tail->seg_off); 34 return false; 35 } 36 37 kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->gc_kset_onmedia_buf; 38 39 to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(key_tail)); 40 ret = copy_mc_to_kernel(kset_onmedia, key_addr, to_copy); 41 if (ret) { 42 pcache_dev_err(pcache, "error to read kset: %d", ret); 43 return false; 44 } 45 46 /* Reject a corrupted or out-of-bounds kset before reading its keys */ 47 if (!kset_onmedia_valid(kset_onmedia)) { 48 pcache_dev_debug(pcache, "gc error: invalid kset. key_tail: %u:%u magic: %llx, key_num: %u\n", 49 key_tail->cache_seg->cache_seg_id, key_tail->seg_off, 50 kset_onmedia->magic, kset_onmedia->key_num); 51 return false; 52 } 53 54 /* Verify the CRC of the kset_onmedia */ 55 if (kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { 56 pcache_dev_debug(pcache, "gc error: crc is not as expected. crc: %x, expected: %x\n", 57 cache_kset_crc(kset_onmedia), kset_onmedia->crc); 58 return false; 59 } 60 61 segs_used = bitmap_weight(cache->seg_map, cache->n_segs); 62 segs_gc_threshold = cache->n_segs * pcache_cache_get_gc_percent(cache) / 100; 63 if (segs_used < segs_gc_threshold) { 64 pcache_dev_debug(pcache, "segs_used: %u, segs_gc_threshold: %u\n", segs_used, segs_gc_threshold); 65 return false; 66 } 67 68 return true; 69 } 70 71 /** 72 * last_kset_gc - Advances the garbage collection for the last kset. 73 * @cache: Pointer to the pcache_cache structure. 74 * @kset_onmedia: Pointer to the kset_onmedia structure for the last kset. 75 */ 76 static int last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) 77 { 78 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 79 struct pcache_cache_segment *cur_seg, *next_seg; 80 81 if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) { 82 pcache_dev_err(pcache, "invalid next_cache_seg_id %u in gc (n_segs %u)\n", 83 kset_onmedia->next_cache_seg_id, cache->n_segs); 84 return -EIO; 85 } 86 87 cur_seg = cache->key_tail.cache_seg; 88 89 next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; 90 91 mutex_lock(&cache->key_tail_lock); 92 cache->key_tail.cache_seg = next_seg; 93 cache->key_tail.seg_off = 0; 94 cache_encode_key_tail(cache); 95 mutex_unlock(&cache->key_tail_lock); 96 97 pcache_dev_debug(pcache, "gc advance kset seg: %u\n", cur_seg->cache_seg_id); 98 99 spin_lock(&cache->seg_map_lock); 100 __clear_bit(cur_seg->cache_seg_id, cache->seg_map); 101 spin_unlock(&cache->seg_map_lock); 102 103 return 0; 104 } 105 106 void pcache_cache_gc_fn(struct work_struct *work) 107 { 108 struct pcache_cache *cache = container_of(work, struct pcache_cache, gc_work.work); 109 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); 110 struct pcache_cache_pos dirty_tail, key_tail; 111 struct pcache_cache_kset_onmedia *kset_onmedia; 112 struct pcache_cache_key_onmedia *key_onmedia; 113 struct pcache_cache_key *key; 114 int ret; 115 int i; 116 117 kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->gc_kset_onmedia_buf; 118 119 while (true) { 120 if (pcache_is_stopping(pcache) || atomic_read(&cache->gc_errors)) 121 return; 122 123 /* Get new tail positions */ 124 mutex_lock(&cache->dirty_tail_lock); 125 cache_pos_copy(&dirty_tail, &cache->dirty_tail); 126 mutex_unlock(&cache->dirty_tail_lock); 127 128 mutex_lock(&cache->key_tail_lock); 129 cache_pos_copy(&key_tail, &cache->key_tail); 130 mutex_unlock(&cache->key_tail_lock); 131 132 if (!need_gc(cache, &dirty_tail, &key_tail)) 133 break; 134 135 if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { 136 /* Don't move to the next segment if dirty_tail has not moved */ 137 if (dirty_tail.cache_seg == key_tail.cache_seg) 138 break; 139 140 ret = last_kset_gc(cache, kset_onmedia); 141 if (ret) { 142 atomic_inc(&cache->gc_errors); 143 return; 144 } 145 continue; 146 } 147 148 if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&key_tail)) { 149 atomic_inc(&cache->gc_errors); 150 return; 151 } 152 153 for (i = 0; i < kset_onmedia->key_num; i++) { 154 struct pcache_cache_key key_tmp = { 0 }; 155 156 key_onmedia = &kset_onmedia->data[i]; 157 158 key = &key_tmp; 159 cache_key_init(&cache->req_key_tree, key); 160 161 ret = cache_key_decode(cache, key_onmedia, key); 162 if (ret) { 163 /* return without re-arm gc work, and prevent future 164 * gc, because we can't retry the partial-gc-ed kset 165 */ 166 atomic_inc(&cache->gc_errors); 167 pcache_dev_err(pcache, "failed to decode cache key in gc\n"); 168 return; 169 } 170 171 cache_key_gc(key); 172 } 173 174 pcache_dev_debug(pcache, "gc advance: %u:%u %u\n", 175 key_tail.cache_seg->cache_seg_id, 176 key_tail.seg_off, 177 get_kset_onmedia_size(kset_onmedia)); 178 179 mutex_lock(&cache->key_tail_lock); 180 cache_pos_advance(&cache->key_tail, get_kset_onmedia_size(kset_onmedia)); 181 cache_encode_key_tail(cache); 182 mutex_unlock(&cache->key_tail_lock); 183 } 184 185 queue_delayed_work(cache_get_wq(cache), &cache->gc_work, PCACHE_CACHE_GC_INTERVAL); 186 } 187