1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/spa.h> 30 #include <sys/vdev_impl.h> 31 #include <sys/zio.h> 32 33 /* 34 * Virtual device read-ahead caching. 35 * 36 * This file implements a simple LRU read-ahead cache. When the DMU reads 37 * a given block, it will often want other, nearby blocks soon thereafter. 38 * We take advantage of this by reading a larger disk region and caching 39 * the result. In the best case, this can turn 256 back-to-back 512-byte 40 * reads into a single 128k read followed by 255 cache hits; this reduces 41 * latency dramatically. In the worst case, it can turn an isolated 512-byte 42 * read into a 128k read, which doesn't affect latency all that much but is 43 * terribly wasteful of bandwidth. A more intelligent version of the cache 44 * could keep track of access patterns and not do read-ahead unless it sees 45 * at least two temporally close I/Os to the same region. It could also 46 * take advantage of semantic information about the I/O. And it could use 47 * something faster than an AVL tree; that was chosen solely for convenience. 48 * 49 * There are five cache operations: allocate, fill, read, write, evict. 50 * 51 * (1) Allocate. This reserves a cache entry for the specified region. 52 * We separate the allocate and fill operations so that multiple threads 53 * don't generate I/O for the same cache miss. 54 * 55 * (2) Fill. When the I/O for a cache miss completes, the fill routine 56 * places the data in the previously allocated cache entry. 57 * 58 * (3) Read. Read data from the cache. 59 * 60 * (4) Write. Update cache contents after write completion. 61 * 62 * (5) Evict. When allocating a new entry, we evict the oldest (LRU) entry 63 * if the total cache size exceeds zfs_vdev_cache_size. 64 */ 65 66 /* 67 * These tunables are for performance analysis. 68 */ 69 /* 70 * All i/os smaller than zfs_vdev_cache_max will be turned into 71 * 1<<zfs_vdev_cache_bshift byte reads by the vdev_cache (aka software 72 * track buffer. At most zfs_vdev_cache_size bytes will be kept in each 73 * vdev's vdev_cache. 74 */ 75 int zfs_vdev_cache_max = 1<<14; 76 int zfs_vdev_cache_size = 10ULL << 20; 77 int zfs_vdev_cache_bshift = 16; 78 79 #define VCBS (1 << zfs_vdev_cache_bshift) 80 81 static int 82 vdev_cache_offset_compare(const void *a1, const void *a2) 83 { 84 const vdev_cache_entry_t *ve1 = a1; 85 const vdev_cache_entry_t *ve2 = a2; 86 87 if (ve1->ve_offset < ve2->ve_offset) 88 return (-1); 89 if (ve1->ve_offset > ve2->ve_offset) 90 return (1); 91 return (0); 92 } 93 94 static int 95 vdev_cache_lastused_compare(const void *a1, const void *a2) 96 { 97 const vdev_cache_entry_t *ve1 = a1; 98 const vdev_cache_entry_t *ve2 = a2; 99 100 if (ve1->ve_lastused < ve2->ve_lastused) 101 return (-1); 102 if (ve1->ve_lastused > ve2->ve_lastused) 103 return (1); 104 105 /* 106 * Among equally old entries, sort by offset to ensure uniqueness. 107 */ 108 return (vdev_cache_offset_compare(a1, a2)); 109 } 110 111 /* 112 * Evict the specified entry from the cache. 113 */ 114 static void 115 vdev_cache_evict(vdev_cache_t *vc, vdev_cache_entry_t *ve) 116 { 117 ASSERT(MUTEX_HELD(&vc->vc_lock)); 118 ASSERT(ve->ve_fill_io == NULL); 119 ASSERT(ve->ve_data != NULL); 120 121 dprintf("evicting %p, off %llx, LRU %llu, age %lu, hits %u, stale %u\n", 122 vc, ve->ve_offset, ve->ve_lastused, lbolt - ve->ve_lastused, 123 ve->ve_hits, ve->ve_missed_update); 124 125 avl_remove(&vc->vc_lastused_tree, ve); 126 avl_remove(&vc->vc_offset_tree, ve); 127 zio_buf_free(ve->ve_data, VCBS); 128 kmem_free(ve, sizeof (vdev_cache_entry_t)); 129 } 130 131 /* 132 * Allocate an entry in the cache. At the point we don't have the data, 133 * we're just creating a placeholder so that multiple threads don't all 134 * go off and read the same blocks. 135 */ 136 static vdev_cache_entry_t * 137 vdev_cache_allocate(zio_t *zio) 138 { 139 vdev_cache_t *vc = &zio->io_vd->vdev_cache; 140 uint64_t offset = P2ALIGN(zio->io_offset, VCBS); 141 vdev_cache_entry_t *ve; 142 143 ASSERT(MUTEX_HELD(&vc->vc_lock)); 144 145 if (zfs_vdev_cache_size == 0) 146 return (NULL); 147 148 /* 149 * If adding a new entry would exceed the cache size, 150 * evict the oldest entry (LRU). 151 */ 152 if ((avl_numnodes(&vc->vc_lastused_tree) << zfs_vdev_cache_bshift) > 153 zfs_vdev_cache_size) { 154 ve = avl_first(&vc->vc_lastused_tree); 155 if (ve->ve_fill_io != NULL) { 156 dprintf("can't evict in %p, still filling\n", vc); 157 return (NULL); 158 } 159 ASSERT(ve->ve_hits != 0); 160 vdev_cache_evict(vc, ve); 161 } 162 163 ve = kmem_zalloc(sizeof (vdev_cache_entry_t), KM_SLEEP); 164 ve->ve_offset = offset; 165 ve->ve_lastused = lbolt; 166 ve->ve_data = zio_buf_alloc(VCBS); 167 168 avl_add(&vc->vc_offset_tree, ve); 169 avl_add(&vc->vc_lastused_tree, ve); 170 171 return (ve); 172 } 173 174 static void 175 vdev_cache_hit(vdev_cache_t *vc, vdev_cache_entry_t *ve, zio_t *zio) 176 { 177 uint64_t cache_phase = P2PHASE(zio->io_offset, VCBS); 178 179 ASSERT(MUTEX_HELD(&vc->vc_lock)); 180 ASSERT(ve->ve_fill_io == NULL); 181 182 if (ve->ve_lastused != lbolt) { 183 avl_remove(&vc->vc_lastused_tree, ve); 184 ve->ve_lastused = lbolt; 185 avl_add(&vc->vc_lastused_tree, ve); 186 } 187 188 ve->ve_hits++; 189 bcopy(ve->ve_data + cache_phase, zio->io_data, zio->io_size); 190 } 191 192 /* 193 * Fill a previously allocated cache entry with data. 194 */ 195 static void 196 vdev_cache_fill(zio_t *zio) 197 { 198 vdev_t *vd = zio->io_vd; 199 vdev_cache_t *vc = &vd->vdev_cache; 200 vdev_cache_entry_t *ve = zio->io_private; 201 zio_t *dio; 202 203 ASSERT(zio->io_size == VCBS); 204 205 /* 206 * Add data to the cache. 207 */ 208 mutex_enter(&vc->vc_lock); 209 210 ASSERT(ve->ve_fill_io == zio); 211 ASSERT(ve->ve_offset == zio->io_offset); 212 ASSERT(ve->ve_data == zio->io_data); 213 214 ve->ve_fill_io = NULL; 215 216 /* 217 * Even if this cache line was invalidated by a missed write update, 218 * any reads that were queued up before the missed update are still 219 * valid, so we can satisfy them from this line before we evict it. 220 */ 221 for (dio = zio->io_delegate_list; dio; dio = dio->io_delegate_next) 222 vdev_cache_hit(vc, ve, dio); 223 224 if (zio->io_error || ve->ve_missed_update) 225 vdev_cache_evict(vc, ve); 226 227 mutex_exit(&vc->vc_lock); 228 229 while ((dio = zio->io_delegate_list) != NULL) { 230 zio->io_delegate_list = dio->io_delegate_next; 231 dio->io_delegate_next = NULL; 232 dio->io_error = zio->io_error; 233 zio_next_stage(dio); 234 } 235 } 236 237 /* 238 * Read data from the cache. Returns 0 on cache hit, errno on a miss. 239 */ 240 int 241 vdev_cache_read(zio_t *zio) 242 { 243 vdev_cache_t *vc = &zio->io_vd->vdev_cache; 244 vdev_cache_entry_t *ve, ve_search; 245 uint64_t cache_offset = P2ALIGN(zio->io_offset, VCBS); 246 uint64_t cache_phase = P2PHASE(zio->io_offset, VCBS); 247 zio_t *fio; 248 249 ASSERT(zio->io_type == ZIO_TYPE_READ); 250 251 if (zio->io_flags & ZIO_FLAG_DONT_CACHE) 252 return (EINVAL); 253 254 if (zio->io_size > zfs_vdev_cache_max) 255 return (EOVERFLOW); 256 257 /* 258 * If the I/O straddles two or more cache blocks, don't cache it. 259 */ 260 if (P2CROSS(zio->io_offset, zio->io_offset + zio->io_size - 1, VCBS)) 261 return (EXDEV); 262 263 ASSERT(cache_phase + zio->io_size <= VCBS); 264 265 mutex_enter(&vc->vc_lock); 266 267 ve_search.ve_offset = cache_offset; 268 ve = avl_find(&vc->vc_offset_tree, &ve_search, NULL); 269 270 if (ve != NULL) { 271 if (ve->ve_missed_update) { 272 mutex_exit(&vc->vc_lock); 273 return (ESTALE); 274 } 275 276 if ((fio = ve->ve_fill_io) != NULL) { 277 zio->io_delegate_next = fio->io_delegate_list; 278 fio->io_delegate_list = zio; 279 zio_vdev_io_bypass(zio); 280 mutex_exit(&vc->vc_lock); 281 return (0); 282 } 283 284 vdev_cache_hit(vc, ve, zio); 285 zio_vdev_io_bypass(zio); 286 287 mutex_exit(&vc->vc_lock); 288 zio_next_stage(zio); 289 return (0); 290 } 291 292 ve = vdev_cache_allocate(zio); 293 294 if (ve == NULL) { 295 mutex_exit(&vc->vc_lock); 296 return (ENOMEM); 297 } 298 299 fio = zio_vdev_child_io(zio, NULL, zio->io_vd, cache_offset, 300 ve->ve_data, VCBS, ZIO_TYPE_READ, ZIO_PRIORITY_CACHE_FILL, 301 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_PROPAGATE | 302 ZIO_FLAG_DONT_RETRY | ZIO_FLAG_NOBOOKMARK, 303 vdev_cache_fill, ve); 304 305 ve->ve_fill_io = fio; 306 fio->io_delegate_list = zio; 307 zio_vdev_io_bypass(zio); 308 309 mutex_exit(&vc->vc_lock); 310 zio_nowait(fio); 311 312 return (0); 313 } 314 315 /* 316 * Update cache contents upon write completion. 317 */ 318 void 319 vdev_cache_write(zio_t *zio) 320 { 321 vdev_cache_t *vc = &zio->io_vd->vdev_cache; 322 vdev_cache_entry_t *ve, ve_search; 323 uint64_t io_start = zio->io_offset; 324 uint64_t io_end = io_start + zio->io_size; 325 uint64_t min_offset = P2ALIGN(io_start, VCBS); 326 uint64_t max_offset = P2ROUNDUP(io_end, VCBS); 327 avl_index_t where; 328 329 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 330 331 mutex_enter(&vc->vc_lock); 332 333 ve_search.ve_offset = min_offset; 334 ve = avl_find(&vc->vc_offset_tree, &ve_search, &where); 335 336 if (ve == NULL) 337 ve = avl_nearest(&vc->vc_offset_tree, where, AVL_AFTER); 338 339 while (ve != NULL && ve->ve_offset < max_offset) { 340 uint64_t start = MAX(ve->ve_offset, io_start); 341 uint64_t end = MIN(ve->ve_offset + VCBS, io_end); 342 343 if (ve->ve_fill_io != NULL) { 344 ve->ve_missed_update = 1; 345 } else { 346 bcopy((char *)zio->io_data + start - io_start, 347 ve->ve_data + start - ve->ve_offset, end - start); 348 } 349 ve = AVL_NEXT(&vc->vc_offset_tree, ve); 350 } 351 mutex_exit(&vc->vc_lock); 352 } 353 354 void 355 vdev_cache_init(vdev_t *vd) 356 { 357 vdev_cache_t *vc = &vd->vdev_cache; 358 359 mutex_init(&vc->vc_lock, NULL, MUTEX_DEFAULT, NULL); 360 361 avl_create(&vc->vc_offset_tree, vdev_cache_offset_compare, 362 sizeof (vdev_cache_entry_t), 363 offsetof(struct vdev_cache_entry, ve_offset_node)); 364 365 avl_create(&vc->vc_lastused_tree, vdev_cache_lastused_compare, 366 sizeof (vdev_cache_entry_t), 367 offsetof(struct vdev_cache_entry, ve_lastused_node)); 368 } 369 370 void 371 vdev_cache_fini(vdev_t *vd) 372 { 373 vdev_cache_t *vc = &vd->vdev_cache; 374 vdev_cache_entry_t *ve; 375 376 mutex_enter(&vc->vc_lock); 377 while ((ve = avl_first(&vc->vc_offset_tree)) != NULL) 378 vdev_cache_evict(vc, ve); 379 mutex_exit(&vc->vc_lock); 380 381 avl_destroy(&vc->vc_offset_tree); 382 avl_destroy(&vc->vc_lastused_tree); 383 384 mutex_destroy(&vc->vc_lock); 385 } 386