xref: /illumos-gate/usr/src/uts/common/fs/zfs/dmu_zfetch.c (revision 8be869c0c2fb6677f65f293a53e36156ea86c9a7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
28  */
29 
30 #include <sys/zfs_context.h>
31 #include <sys/dnode.h>
32 #include <sys/dmu_objset.h>
33 #include <sys/dmu_zfetch.h>
34 #include <sys/dmu.h>
35 #include <sys/dbuf.h>
36 #include <sys/kstat.h>
37 
38 /*
39  * This tunable disables predictive prefetch.  Note that it leaves "prescient"
40  * prefetch (e.g. prefetch for zfs send) intact.  Unlike predictive prefetch,
41  * prescient prefetch never issues i/os that end up not being needed,
42  * so it can't hurt performance.
43  */
44 boolean_t zfs_prefetch_disable = B_FALSE;
45 
46 /* max # of streams per zfetch */
47 uint32_t	zfetch_max_streams = 8;
48 /* min time before stream reclaim */
49 uint32_t	zfetch_min_sec_reap = 2;
50 /* max bytes to prefetch per stream (default 8MB) */
51 uint32_t	zfetch_max_distance = 8 * 1024 * 1024;
52 /* max bytes to prefetch indirects for per stream (default 64MB) */
53 uint32_t	zfetch_max_idistance = 64 * 1024 * 1024;
54 /* max number of bytes in an array_read in which we allow prefetching (1MB) */
55 uint64_t	zfetch_array_rd_sz = 1024 * 1024;
56 
57 typedef struct zfetch_stats {
58 	kstat_named_t zfetchstat_hits;
59 	kstat_named_t zfetchstat_misses;
60 	kstat_named_t zfetchstat_max_streams;
61 	kstat_named_t zfetchstat_max_completion_us;
62 	kstat_named_t zfetchstat_last_completion_us;
63 	kstat_named_t zfetchstat_io_issued;
64 } zfetch_stats_t;
65 
66 static zfetch_stats_t zfetch_stats = {
67 	{ "hits",			KSTAT_DATA_UINT64 },
68 	{ "misses",			KSTAT_DATA_UINT64 },
69 	{ "max_streams",		KSTAT_DATA_UINT64 },
70 	{ "max_completion_us",		KSTAT_DATA_UINT64 },
71 	{ "last_completion_us",		KSTAT_DATA_UINT64 },
72 	{ "io_issued",		KSTAT_DATA_UINT64 },
73 };
74 
75 #define	ZFETCHSTAT_BUMP(stat) \
76 	atomic_inc_64(&zfetch_stats.stat.value.ui64)
77 #define	ZFETCHSTAT_ADD(stat, val)				\
78 	atomic_add_64(&zfetch_stats.stat.value.ui64, val)
79 #define	ZFETCHSTAT_SET(stat, val)				\
80 	zfetch_stats.stat.value.ui64 = val
81 #define	ZFETCHSTAT_GET(stat)					\
82 	zfetch_stats.stat.value.ui64
83 
84 
85 kstat_t		*zfetch_ksp;
86 
87 void
zfetch_init(void)88 zfetch_init(void)
89 {
90 	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
91 	    KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
92 	    KSTAT_FLAG_VIRTUAL);
93 
94 	if (zfetch_ksp != NULL) {
95 		zfetch_ksp->ks_data = &zfetch_stats;
96 		kstat_install(zfetch_ksp);
97 	}
98 }
99 
100 void
zfetch_fini(void)101 zfetch_fini(void)
102 {
103 	if (zfetch_ksp != NULL) {
104 		kstat_delete(zfetch_ksp);
105 		zfetch_ksp = NULL;
106 	}
107 }
108 
109 /*
110  * This takes a pointer to a zfetch structure and a dnode.  It performs the
111  * necessary setup for the zfetch structure, grokking data from the
112  * associated dnode.
113  */
114 void
dmu_zfetch_init(zfetch_t * zf,dnode_t * dno)115 dmu_zfetch_init(zfetch_t *zf, dnode_t *dno)
116 {
117 	if (zf == NULL)
118 		return;
119 	zf->zf_dnode = dno;
120 	zf->zf_numstreams = 0;
121 
122 	list_create(&zf->zf_stream, sizeof (zstream_t),
123 	    offsetof(zstream_t, zs_node));
124 
125 	rw_init(&zf->zf_rwlock, NULL, RW_DEFAULT, NULL);
126 }
127 
128 static void
dmu_zfetch_stream_fini(zstream_t * zs)129 dmu_zfetch_stream_fini(zstream_t *zs)
130 {
131 	mutex_destroy(&zs->zs_lock);
132 	zfs_refcount_destroy(&zs->zs_blocks);
133 	kmem_free(zs, sizeof (*zs));
134 }
135 
136 static void
dmu_zfetch_stream_remove(zfetch_t * zf,zstream_t * zs)137 dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)
138 {
139 	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
140 	list_remove(&zf->zf_stream, zs);
141 	dmu_zfetch_stream_fini(zs);
142 	zf->zf_numstreams--;
143 }
144 
145 static void
dmu_zfetch_stream_orphan(zfetch_t * zf,zstream_t * zs)146 dmu_zfetch_stream_orphan(zfetch_t *zf, zstream_t *zs)
147 {
148 	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
149 	list_remove(&zf->zf_stream, zs);
150 	zs->zs_fetch = NULL;
151 	zf->zf_numstreams--;
152 }
153 
154 /*
155  * Clean-up state associated with a zfetch structure (e.g. destroy the
156  * streams).  This doesn't free the zfetch_t itself, that's left to the caller.
157  */
158 void
dmu_zfetch_fini(zfetch_t * zf)159 dmu_zfetch_fini(zfetch_t *zf)
160 {
161 	zstream_t *zs;
162 
163 	ASSERT(!RW_LOCK_HELD(&zf->zf_rwlock));
164 
165 	rw_enter(&zf->zf_rwlock, RW_WRITER);
166 	while ((zs = list_head(&zf->zf_stream)) != NULL) {
167 		if (zfs_refcount_count(&zs->zs_blocks) != 0)
168 			dmu_zfetch_stream_orphan(zf, zs);
169 		else
170 			dmu_zfetch_stream_remove(zf, zs);
171 	}
172 	rw_exit(&zf->zf_rwlock);
173 	list_destroy(&zf->zf_stream);
174 	rw_destroy(&zf->zf_rwlock);
175 
176 	zf->zf_dnode = NULL;
177 }
178 
179 /*
180  * If there aren't too many streams already, create a new stream.
181  * The "blkid" argument is the next block that we expect this stream to access.
182  * While we're here, clean up old streams (which haven't been
183  * accessed for at least zfetch_min_sec_reap seconds).
184  */
185 static void
dmu_zfetch_stream_create(zfetch_t * zf,uint64_t blkid)186 dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid)
187 {
188 	zstream_t *zs_next;
189 	hrtime_t now = gethrtime();
190 
191 	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
192 
193 	/*
194 	 * Clean up old streams.
195 	 */
196 	for (zstream_t *zs = list_head(&zf->zf_stream);
197 	    zs != NULL; zs = zs_next) {
198 		zs_next = list_next(&zf->zf_stream, zs);
199 		/*
200 		 * Skip gethrtime() call if there are still references
201 		 */
202 		if (zfs_refcount_count(&zs->zs_blocks) != 0)
203 			continue;
204 		if (((now - zs->zs_atime) / NANOSEC) >
205 		    zfetch_min_sec_reap)
206 			dmu_zfetch_stream_remove(zf, zs);
207 	}
208 
209 	/*
210 	 * The maximum number of streams is normally zfetch_max_streams,
211 	 * but for small files we lower it such that it's at least possible
212 	 * for all the streams to be non-overlapping.
213 	 *
214 	 * If we are already at the maximum number of streams for this file,
215 	 * even after removing old streams, then don't create this stream.
216 	 */
217 	uint32_t max_streams = MAX(1, MIN(zfetch_max_streams,
218 	    zf->zf_dnode->dn_maxblkid * zf->zf_dnode->dn_datablksz /
219 	    zfetch_max_distance));
220 	if (zf->zf_numstreams >= max_streams) {
221 		ZFETCHSTAT_BUMP(zfetchstat_max_streams);
222 		return;
223 	}
224 
225 	zstream_t *zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
226 	zs->zs_blkid = blkid;
227 	zs->zs_pf_blkid = blkid;
228 	zs->zs_ipf_blkid = blkid;
229 	zs->zs_atime = now;
230 	zs->zs_fetch = zf;
231 	zfs_refcount_create(&zs->zs_blocks);
232 	mutex_init(&zs->zs_lock, NULL, MUTEX_DEFAULT, NULL);
233 	zf->zf_numstreams++;
234 	list_insert_head(&zf->zf_stream, zs);
235 }
236 
237 static void
dmu_zfetch_stream_done(void * arg,boolean_t io_issued)238 dmu_zfetch_stream_done(void *arg, boolean_t io_issued)
239 {
240 	zstream_t *zs = arg;
241 
242 	if (zs->zs_start_time && io_issued) {
243 		hrtime_t now = gethrtime();
244 		hrtime_t delta = NSEC2USEC(now - zs->zs_start_time);
245 
246 		zs->zs_start_time = 0;
247 		ZFETCHSTAT_SET(zfetchstat_last_completion_us, delta);
248 		if (delta > ZFETCHSTAT_GET(zfetchstat_max_completion_us))
249 			ZFETCHSTAT_SET(zfetchstat_max_completion_us, delta);
250 	}
251 
252 	if (zfs_refcount_remove(&zs->zs_blocks, NULL) != 0)
253 		return;
254 
255 	/*
256 	 * The parent fetch structure has gone away
257 	 */
258 	if (zs->zs_fetch == NULL)
259 		dmu_zfetch_stream_fini(zs);
260 }
261 
262 /*
263  * This is the predictive prefetch entry point.  It associates dnode access
264  * specified with blkid and nblks arguments with prefetch stream, predicts
265  * further accesses based on that stats and initiates speculative prefetch.
266  * fetch_data argument specifies whether actual data blocks should be fetched:
267  *   FALSE -- prefetch only indirect blocks for predicted data blocks;
268  *   TRUE -- prefetch predicted data blocks plus following indirect blocks.
269  */
270 void
dmu_zfetch(zfetch_t * zf,uint64_t blkid,uint64_t nblks,boolean_t fetch_data,boolean_t have_lock)271 dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data,
272     boolean_t have_lock)
273 {
274 	zstream_t *zs;
275 	int64_t pf_start, ipf_start, ipf_istart, ipf_iend;
276 	int64_t pf_ahead_blks, max_blks;
277 	int epbs, max_dist_blks, pf_nblks, ipf_nblks, issued;
278 	uint64_t end_of_access_blkid = blkid + nblks;
279 	spa_t *spa = zf->zf_dnode->dn_objset->os_spa;
280 
281 	if (zfs_prefetch_disable)
282 		return;
283 
284 	/*
285 	 * If we haven't yet loaded the indirect vdevs' mappings, we
286 	 * can only read from blocks that we carefully ensure are on
287 	 * concrete vdevs (or previously-loaded indirect vdevs).  So we
288 	 * can't allow the predictive prefetcher to attempt reads of other
289 	 * blocks (e.g. of the MOS's dnode obejct).
290 	 */
291 	if (!spa_indirect_vdevs_loaded(spa))
292 		return;
293 
294 	/*
295 	 * As a fast path for small (single-block) files, ignore access
296 	 * to the first block.
297 	 */
298 	if (!have_lock && blkid == 0)
299 		return;
300 
301 	if (!have_lock)
302 		rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER);
303 
304 
305 	/*
306 	 * A fast path for small files for which no prefetch will
307 	 * happen.
308 	 */
309 	if (zf->zf_dnode->dn_maxblkid < 2) {
310 		if (!have_lock)
311 			rw_exit(&zf->zf_dnode->dn_struct_rwlock);
312 		return;
313 	}
314 	rw_enter(&zf->zf_rwlock, RW_READER);
315 
316 	/*
317 	 * Find matching prefetch stream.  Depending on whether the accesses
318 	 * are block-aligned, first block of the new access may either follow
319 	 * the last block of the previous access, or be equal to it.
320 	 */
321 	for (zs = list_head(&zf->zf_stream); zs != NULL;
322 	    zs = list_next(&zf->zf_stream, zs)) {
323 		if (blkid == zs->zs_blkid || blkid + 1 == zs->zs_blkid) {
324 			mutex_enter(&zs->zs_lock);
325 			/*
326 			 * zs_blkid could have changed before we
327 			 * acquired zs_lock; re-check them here.
328 			 */
329 			if (blkid == zs->zs_blkid) {
330 				break;
331 			} else if (blkid + 1 == zs->zs_blkid) {
332 				blkid++;
333 				nblks--;
334 				if (nblks == 0) {
335 					/* Already prefetched this before. */
336 					mutex_exit(&zs->zs_lock);
337 					rw_exit(&zf->zf_rwlock);
338 					if (!have_lock) {
339 						rw_exit(&zf->zf_dnode->
340 						    dn_struct_rwlock);
341 					}
342 					return;
343 				}
344 				break;
345 			}
346 			mutex_exit(&zs->zs_lock);
347 		}
348 	}
349 
350 	if (zs == NULL) {
351 		/*
352 		 * This access is not part of any existing stream.  Create
353 		 * a new stream for it.
354 		 */
355 		ZFETCHSTAT_BUMP(zfetchstat_misses);
356 		if (rw_tryupgrade(&zf->zf_rwlock))
357 			dmu_zfetch_stream_create(zf, end_of_access_blkid);
358 		rw_exit(&zf->zf_rwlock);
359 		if (!have_lock)
360 			rw_exit(&zf->zf_dnode->dn_struct_rwlock);
361 		return;
362 	}
363 
364 	/*
365 	 * This access was to a block that we issued a prefetch for on
366 	 * behalf of this stream. Issue further prefetches for this stream.
367 	 *
368 	 * Normally, we start prefetching where we stopped
369 	 * prefetching last (zs_pf_blkid).  But when we get our first
370 	 * hit on this stream, zs_pf_blkid == zs_blkid, we don't
371 	 * want to prefetch the block we just accessed.  In this case,
372 	 * start just after the block we just accessed.
373 	 */
374 	pf_start = MAX(zs->zs_pf_blkid, end_of_access_blkid);
375 
376 	/*
377 	 * Double our amount of prefetched data, but don't let the
378 	 * prefetch get further ahead than zfetch_max_distance.
379 	 */
380 	if (fetch_data) {
381 		max_dist_blks =
382 		    zfetch_max_distance >> zf->zf_dnode->dn_datablkshift;
383 		/*
384 		 * Previously, we were (zs_pf_blkid - blkid) ahead.  We
385 		 * want to now be double that, so read that amount again,
386 		 * plus the amount we are catching up by (i.e. the amount
387 		 * read just now).
388 		 */
389 		pf_ahead_blks = zs->zs_pf_blkid - blkid + nblks;
390 		max_blks = max_dist_blks - (pf_start - end_of_access_blkid);
391 		pf_nblks = MIN(pf_ahead_blks, max_blks);
392 	} else {
393 		pf_nblks = 0;
394 	}
395 
396 	zs->zs_pf_blkid = pf_start + pf_nblks;
397 
398 	/*
399 	 * Do the same for indirects, starting from where we stopped last,
400 	 * or where we will stop reading data blocks (and the indirects
401 	 * that point to them).
402 	 */
403 	ipf_start = MAX(zs->zs_ipf_blkid, zs->zs_pf_blkid);
404 	max_dist_blks = zfetch_max_idistance >> zf->zf_dnode->dn_datablkshift;
405 	/*
406 	 * We want to double our distance ahead of the data prefetch
407 	 * (or reader, if we are not prefetching data).  Previously, we
408 	 * were (zs_ipf_blkid - blkid) ahead.  To double that, we read
409 	 * that amount again, plus the amount we are catching up by
410 	 * (i.e. the amount read now + the amount of data prefetched now).
411 	 */
412 	pf_ahead_blks = zs->zs_ipf_blkid - blkid + nblks + pf_nblks;
413 	max_blks = max_dist_blks - (ipf_start - end_of_access_blkid);
414 	ipf_nblks = MIN(pf_ahead_blks, max_blks);
415 	zs->zs_ipf_blkid = ipf_start + ipf_nblks;
416 
417 	epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
418 	ipf_istart = P2ROUNDUP(ipf_start, 1 << epbs) >> epbs;
419 	ipf_iend = P2ROUNDUP(zs->zs_ipf_blkid, 1 << epbs) >> epbs;
420 
421 	zs->zs_atime = gethrtime();
422 	/* no prior reads in progress */
423 	if (zfs_refcount_count(&zs->zs_blocks) == 0)
424 		zs->zs_start_time = zs->zs_atime;
425 	zs->zs_blkid = end_of_access_blkid;
426 	zfs_refcount_add_few(&zs->zs_blocks, pf_nblks + ipf_iend - ipf_istart,
427 	    NULL);
428 	mutex_exit(&zs->zs_lock);
429 	rw_exit(&zf->zf_rwlock);
430 	issued = 0;
431 
432 	/*
433 	 * dbuf_prefetch() is asynchronous (even when it needs to read
434 	 * indirect blocks), but we still prefer to drop our locks before
435 	 * calling it to reduce the time we hold them.
436 	 */
437 
438 	for (int i = 0; i < pf_nblks; i++) {
439 		issued += dbuf_prefetch_impl(zf->zf_dnode, 0, pf_start + i,
440 		    ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH,
441 		    dmu_zfetch_stream_done, zs);
442 	}
443 	for (int64_t iblk = ipf_istart; iblk < ipf_iend; iblk++) {
444 		issued += dbuf_prefetch_impl(zf->zf_dnode, 1, iblk,
445 		    ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH,
446 		    dmu_zfetch_stream_done, zs);
447 	}
448 	if (!have_lock)
449 		rw_exit(&zf->zf_dnode->dn_struct_rwlock);
450 	ZFETCHSTAT_BUMP(zfetchstat_hits);
451 
452 	if (issued)
453 		ZFETCHSTAT_ADD(zfetchstat_io_issued, issued);
454 }
455