1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/blkdev.h> 4 #include <linux/dax.h> 5 #include <linux/vmalloc.h> 6 #include <linux/parser.h> 7 8 #include "cache_dev.h" 9 #include "backing_dev.h" 10 #include "cache.h" 11 #include "dm_pcache.h" 12 13 static void cache_dev_dax_exit(struct pcache_cache_dev *cache_dev) 14 { 15 if (cache_dev->use_vmap) 16 vunmap(cache_dev->mapping); 17 } 18 19 static int build_vmap(struct dax_device *dax_dev, long total_pages, void **vaddr) 20 { 21 struct page **pages; 22 long i = 0, chunk; 23 unsigned long pfn; 24 int ret; 25 26 pages = vmalloc_array(total_pages, sizeof(struct page *)); 27 if (!pages) 28 return -ENOMEM; 29 30 do { 31 chunk = dax_direct_access(dax_dev, i, total_pages - i, 32 DAX_ACCESS, NULL, &pfn); 33 if (chunk <= 0) { 34 ret = chunk ? chunk : -EINVAL; 35 goto out_free; 36 } 37 38 if (!pfn_valid(pfn)) { 39 ret = -EOPNOTSUPP; 40 goto out_free; 41 } 42 43 while (chunk-- && i < total_pages) { 44 pages[i++] = pfn_to_page(pfn); 45 pfn++; 46 if (!(i & 15)) 47 cond_resched(); 48 } 49 } while (i < total_pages); 50 51 *vaddr = vmap(pages, total_pages, VM_MAP, PAGE_KERNEL); 52 if (!*vaddr) { 53 ret = -ENOMEM; 54 goto out_free; 55 } 56 57 ret = 0; 58 59 out_free: 60 vfree(pages); 61 return ret; 62 } 63 64 static int cache_dev_dax_init(struct pcache_cache_dev *cache_dev) 65 { 66 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); 67 struct dax_device *dax_dev; 68 long total_pages, mapped_pages; 69 u64 bdev_size; 70 void *vaddr; 71 int ret; 72 int id; 73 unsigned long pfn; 74 75 dax_dev = cache_dev->dm_dev->dax_dev; 76 /* total size check */ 77 bdev_size = bdev_nr_bytes(cache_dev->dm_dev->bdev); 78 if (bdev_size < PCACHE_CACHE_DEV_SIZE_MIN) { 79 pcache_dev_err(pcache, "dax device is too small, required at least %llu", 80 PCACHE_CACHE_DEV_SIZE_MIN); 81 ret = -ENOSPC; 82 goto out; 83 } 84 85 total_pages = bdev_size >> PAGE_SHIFT; 86 /* attempt: direct-map the whole range */ 87 id = dax_read_lock(); 88 mapped_pages = dax_direct_access(dax_dev, 0, total_pages, 89 DAX_ACCESS, &vaddr, &pfn); 90 if (mapped_pages < 0) { 91 pcache_dev_err(pcache, "dax_direct_access failed: %ld\n", mapped_pages); 92 ret = mapped_pages; 93 goto unlock; 94 } 95 96 if (!pfn_valid(pfn)) { 97 ret = -EOPNOTSUPP; 98 goto unlock; 99 } 100 101 if (mapped_pages == total_pages) { 102 /* success: contiguous direct mapping */ 103 cache_dev->mapping = vaddr; 104 } else { 105 /* need vmap fallback */ 106 ret = build_vmap(dax_dev, total_pages, &vaddr); 107 if (ret) { 108 pcache_dev_err(pcache, "vmap fallback failed: %d\n", ret); 109 goto unlock; 110 } 111 112 cache_dev->mapping = vaddr; 113 cache_dev->use_vmap = true; 114 } 115 dax_read_unlock(id); 116 117 return 0; 118 unlock: 119 dax_read_unlock(id); 120 out: 121 return ret; 122 } 123 124 void cache_dev_zero_range(struct pcache_cache_dev *cache_dev, void *pos, u32 size) 125 { 126 memset(pos, 0, size); 127 dax_flush(cache_dev->dm_dev->dax_dev, pos, size); 128 } 129 130 static int sb_read(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb) 131 { 132 struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev); 133 134 if (copy_mc_to_kernel(sb, sb_addr, sizeof(struct pcache_sb))) 135 return -EIO; 136 137 return 0; 138 } 139 140 static void sb_write(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb) 141 { 142 struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev); 143 144 memcpy_flushcache(sb_addr, sb, sizeof(struct pcache_sb)); 145 pmem_wmb(); 146 } 147 148 static int sb_init(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb) 149 { 150 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); 151 u64 nr_segs; 152 u64 cache_dev_size; 153 u64 magic; 154 u32 flags = 0; 155 156 magic = le64_to_cpu(sb->magic); 157 if (magic) 158 return -EEXIST; 159 160 cache_dev_size = bdev_nr_bytes(file_bdev(cache_dev->dm_dev->bdev_file)); 161 if (cache_dev_size < PCACHE_CACHE_DEV_SIZE_MIN) { 162 pcache_dev_err(pcache, "dax device is too small, required at least %llu", 163 PCACHE_CACHE_DEV_SIZE_MIN); 164 return -ENOSPC; 165 } 166 167 nr_segs = (cache_dev_size - PCACHE_SEGMENTS_OFF) / ((PCACHE_SEG_SIZE)); 168 169 #if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN) 170 flags |= PCACHE_SB_F_BIGENDIAN; 171 #endif 172 sb->flags = cpu_to_le32(flags); 173 sb->magic = cpu_to_le64(PCACHE_MAGIC); 174 sb->seg_num = cpu_to_le32(nr_segs); 175 sb->crc = cpu_to_le32(crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4)); 176 177 cache_dev_zero_range(cache_dev, CACHE_DEV_CACHE_INFO(cache_dev), 178 PCACHE_CACHE_INFO_SIZE * PCACHE_META_INDEX_MAX + 179 PCACHE_CACHE_CTRL_SIZE); 180 181 return 0; 182 } 183 184 static int sb_validate(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb) 185 { 186 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev); 187 u32 flags; 188 u32 crc; 189 190 if (le64_to_cpu(sb->magic) != PCACHE_MAGIC) { 191 pcache_dev_err(pcache, "unexpected magic: %llx\n", 192 le64_to_cpu(sb->magic)); 193 return -EINVAL; 194 } 195 196 crc = crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4); 197 if (crc != le32_to_cpu(sb->crc)) { 198 pcache_dev_err(pcache, "corrupted sb: %u, expected: %u\n", crc, le32_to_cpu(sb->crc)); 199 return -EINVAL; 200 } 201 202 flags = le32_to_cpu(sb->flags); 203 #if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN) 204 if (!(flags & PCACHE_SB_F_BIGENDIAN)) { 205 pcache_dev_err(pcache, "cache_dev is not big endian\n"); 206 return -EINVAL; 207 } 208 #else 209 if (flags & PCACHE_SB_F_BIGENDIAN) { 210 pcache_dev_err(pcache, "cache_dev is big endian\n"); 211 return -EINVAL; 212 } 213 #endif 214 return 0; 215 } 216 217 static int cache_dev_init(struct pcache_cache_dev *cache_dev, u32 seg_num) 218 { 219 cache_dev->seg_num = seg_num; 220 cache_dev->seg_bitmap = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL); 221 if (!cache_dev->seg_bitmap) 222 return -ENOMEM; 223 224 return 0; 225 } 226 227 static void cache_dev_exit(struct pcache_cache_dev *cache_dev) 228 { 229 kvfree(cache_dev->seg_bitmap); 230 } 231 232 void cache_dev_stop(struct dm_pcache *pcache) 233 { 234 struct pcache_cache_dev *cache_dev = &pcache->cache_dev; 235 236 cache_dev_exit(cache_dev); 237 cache_dev_dax_exit(cache_dev); 238 } 239 240 int cache_dev_start(struct dm_pcache *pcache) 241 { 242 struct pcache_cache_dev *cache_dev = &pcache->cache_dev; 243 struct pcache_sb sb; 244 bool format = false; 245 int ret; 246 247 mutex_init(&cache_dev->seg_lock); 248 249 ret = cache_dev_dax_init(cache_dev); 250 if (ret) { 251 pcache_dev_err(pcache, "failed to init cache_dev %s via dax way: %d.", 252 cache_dev->dm_dev->name, ret); 253 goto err; 254 } 255 256 ret = sb_read(cache_dev, &sb); 257 if (ret) 258 goto dax_release; 259 260 if (le64_to_cpu(sb.magic) == 0) { 261 format = true; 262 ret = sb_init(cache_dev, &sb); 263 if (ret < 0) 264 goto dax_release; 265 } 266 267 ret = sb_validate(cache_dev, &sb); 268 if (ret) 269 goto dax_release; 270 271 cache_dev->sb_flags = le32_to_cpu(sb.flags); 272 ret = cache_dev_init(cache_dev, le32_to_cpu(sb.seg_num)); 273 if (ret) 274 goto dax_release; 275 276 if (format) 277 sb_write(cache_dev, &sb); 278 279 return 0; 280 281 dax_release: 282 cache_dev_dax_exit(cache_dev); 283 err: 284 return ret; 285 } 286 287 int cache_dev_get_empty_segment_id(struct pcache_cache_dev *cache_dev, u32 *seg_id) 288 { 289 int ret; 290 291 mutex_lock(&cache_dev->seg_lock); 292 *seg_id = find_next_zero_bit(cache_dev->seg_bitmap, cache_dev->seg_num, 0); 293 if (*seg_id == cache_dev->seg_num) { 294 ret = -ENOSPC; 295 goto unlock; 296 } 297 298 __set_bit(*seg_id, cache_dev->seg_bitmap); 299 ret = 0; 300 unlock: 301 mutex_unlock(&cache_dev->seg_lock); 302 return ret; 303 } 304