Lines Matching +full:per +full:- +full:stream
1 // SPDX-License-Identifier: CDDL-1.0
10 * or https://opensource.org/licenses/CDDL-1.0.
50 /* max # of streams per zfetch */
52 /* min time before stream reclaim */
54 /* max time before stream delete */
57 /* min bytes to prefetch per stream (default 2MB) */
59 /* max bytes to prefetch per stream (default 8MB) */
62 /* min bytes to prefetch per stream (default 4MB) */
64 /* max bytes to prefetch per stream (default 64MB) */
67 /* max bytes to prefetch indirects for per stream (default 128MB) */
69 /* max request reorder distance within a stream (default 16MB) */
71 /* Max log2 fraction of holes in a stream */
118 zfetch_stats_t *zs = ksp->ks_data; in zfetch_kstats_update()
122 zs->zfetchstat_hits.value.ui64 = in zfetch_kstats_update()
124 zs->zfetchstat_future.value.ui64 = in zfetch_kstats_update()
126 zs->zfetchstat_stride.value.ui64 = in zfetch_kstats_update()
128 zs->zfetchstat_past.value.ui64 = in zfetch_kstats_update()
130 zs->zfetchstat_misses.value.ui64 = in zfetch_kstats_update()
132 zs->zfetchstat_max_streams.value.ui64 = in zfetch_kstats_update()
134 zs->zfetchstat_io_issued.value.ui64 = in zfetch_kstats_update()
136 zs->zfetchstat_io_active.value.ui64 = in zfetch_kstats_update()
158 zfetch_ksp->ks_data = &zfetch_stats; in zfetch_init()
159 zfetch_ksp->ks_update = zfetch_kstats_update; in zfetch_init()
193 zf->zf_dnode = dno; in dmu_zfetch_init()
194 zf->zf_numstreams = 0; in dmu_zfetch_init()
196 list_create(&zf->zf_stream, sizeof (zstream_t), in dmu_zfetch_init()
199 mutex_init(&zf->zf_lock, NULL, MUTEX_DEFAULT, NULL); in dmu_zfetch_init()
205 ASSERT(!list_link_active(&zs->zs_node)); in dmu_zfetch_stream_fini()
206 zfs_refcount_destroy(&zs->zs_callers); in dmu_zfetch_stream_fini()
207 zfs_refcount_destroy(&zs->zs_refs); in dmu_zfetch_stream_fini()
214 ASSERT(MUTEX_HELD(&zf->zf_lock)); in dmu_zfetch_stream_remove()
215 list_remove(&zf->zf_stream, zs); in dmu_zfetch_stream_remove()
216 zf->zf_numstreams--; in dmu_zfetch_stream_remove()
218 if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0) in dmu_zfetch_stream_remove()
223 * Clean-up state associated with a zfetch structure (e.g. destroy the
231 mutex_enter(&zf->zf_lock); in dmu_zfetch_fini()
232 while ((zs = list_head(&zf->zf_stream)) != NULL) in dmu_zfetch_fini()
234 mutex_exit(&zf->zf_lock); in dmu_zfetch_fini()
235 list_destroy(&zf->zf_stream); in dmu_zfetch_fini()
236 mutex_destroy(&zf->zf_lock); in dmu_zfetch_fini()
238 zf->zf_dnode = NULL; in dmu_zfetch_fini()
244 * If needed, reuse oldest stream without hits for zfetch_min_sec_reap or ever.
245 * The "blkid" argument is the next block that we expect this stream to access.
253 ASSERT(MUTEX_HELD(&zf->zf_lock)); in dmu_zfetch_stream_create()
258 t = now - zfetch_max_sec_reap; in dmu_zfetch_stream_create()
259 for (zs = list_head(&zf->zf_stream); zs != NULL; zs = zs_next) { in dmu_zfetch_stream_create()
260 zs_next = list_next(&zf->zf_stream, zs); in dmu_zfetch_stream_create()
262 * Skip if still active. 1 -- zf_stream reference. in dmu_zfetch_stream_create()
264 if ((int)(zs->zs_atime - t) >= 0) in dmu_zfetch_stream_create()
266 if (zfs_refcount_count(&zs->zs_refs) != 1) in dmu_zfetch_stream_create()
275 list_remove(&zf->zf_stream, zs); in dmu_zfetch_stream_create()
282 * for all the streams to be non-overlapping. in dmu_zfetch_stream_create()
285 (zf->zf_dnode->dn_maxblkid << zf->zf_dnode->dn_datablkshift) / in dmu_zfetch_stream_create()
287 if (zf->zf_numstreams >= max_streams) { in dmu_zfetch_stream_create()
288 t = now - zfetch_min_sec_reap; in dmu_zfetch_stream_create()
289 for (zs = list_head(&zf->zf_stream); zs != NULL; in dmu_zfetch_stream_create()
290 zs = list_next(&zf->zf_stream, zs)) { in dmu_zfetch_stream_create()
291 if ((int)(zs->zs_atime - t) >= 0) in dmu_zfetch_stream_create()
293 if (zfs_refcount_count(&zs->zs_refs) != 1) in dmu_zfetch_stream_create()
296 (int)(zs_old->zs_atime - zs->zs_atime) >= 0) in dmu_zfetch_stream_create()
301 list_remove(&zf->zf_stream, zs); in dmu_zfetch_stream_create()
309 zfs_refcount_create(&zs->zs_callers); in dmu_zfetch_stream_create()
310 zfs_refcount_create(&zs->zs_refs); in dmu_zfetch_stream_create()
312 zfs_refcount_add(&zs->zs_refs, NULL); in dmu_zfetch_stream_create()
313 zf->zf_numstreams++; in dmu_zfetch_stream_create()
316 list_insert_head(&zf->zf_stream, zs); in dmu_zfetch_stream_create()
317 zs->zs_blkid = blkid; in dmu_zfetch_stream_create()
318 /* Allow immediate stream reuse until first hit. */ in dmu_zfetch_stream_create()
319 zs->zs_atime = now - zfetch_min_sec_reap; in dmu_zfetch_stream_create()
320 memset(zs->zs_ranges, 0, sizeof (zs->zs_ranges)); in dmu_zfetch_stream_create()
321 zs->zs_pf_dist = 0; in dmu_zfetch_stream_create()
322 zs->zs_ipf_dist = 0; in dmu_zfetch_stream_create()
323 zs->zs_pf_start = blkid; in dmu_zfetch_stream_create()
324 zs->zs_pf_end = blkid; in dmu_zfetch_stream_create()
325 zs->zs_ipf_start = blkid; in dmu_zfetch_stream_create()
326 zs->zs_ipf_end = blkid; in dmu_zfetch_stream_create()
327 zs->zs_missed = B_FALSE; in dmu_zfetch_stream_create()
328 zs->zs_more = B_FALSE; in dmu_zfetch_stream_create()
336 if (io_issued && level == 0 && blkid < zs->zs_blkid) in dmu_zfetch_done()
337 zs->zs_more = B_TRUE; in dmu_zfetch_done()
338 if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0) in dmu_zfetch_done()
340 aggsum_add(&zfetch_sums.zfetchstat_io_active, -1); in dmu_zfetch_done()
344 * Process stream hit access for nblks blocks starting at zs_blkid. Return
353 if (zs->zs_ranges[0].start == 0) in dmu_zfetch_hit()
358 zsrange_t *r = &zs->zs_ranges[i]; in dmu_zfetch_hit()
359 if (r->start == 0 || r->start > nblks) in dmu_zfetch_hit()
361 if (r->end >= nblks) { in dmu_zfetch_hit()
362 nblks = r->end; in dmu_zfetch_hit()
370 if (zs->zs_ranges[i].start == 0) in dmu_zfetch_hit()
372 ASSERT3U(zs->zs_ranges[i].start, >, nblks); in dmu_zfetch_hit()
373 ASSERT3U(zs->zs_ranges[i].end, >, nblks); in dmu_zfetch_hit()
374 zs->zs_ranges[j].start = zs->zs_ranges[i].start - nblks; in dmu_zfetch_hit()
375 zs->zs_ranges[j].end = zs->zs_ranges[i].end - nblks; in dmu_zfetch_hit()
378 zs->zs_ranges[j].start = 0; in dmu_zfetch_hit()
379 zs->zs_ranges[j].end = 0; in dmu_zfetch_hit()
383 zs->zs_blkid += nblks; in dmu_zfetch_hit()
388 * Process future stream access for nblks blocks starting at blkid. Return
394 ASSERT3U(blkid, >, zs->zs_blkid); in dmu_zfetch_future()
395 blkid -= zs->zs_blkid; in dmu_zfetch_future()
401 zsrange_t *r = &zs->zs_ranges[i]; in dmu_zfetch_future()
402 if (r->start == 0 || r->start > blkid + nblks) in dmu_zfetch_future()
404 if (r->end < blkid) in dmu_zfetch_future()
413 if (zs->zs_ranges[f].start > blkid) in dmu_zfetch_future()
414 zs->zs_ranges[f].start = blkid; in dmu_zfetch_future()
415 zs->zs_ranges[f].end = MAX(zs->zs_ranges[l].end, blkid + nblks); in dmu_zfetch_future()
419 zs->zs_ranges[f].start = zs->zs_ranges[l].start; in dmu_zfetch_future()
420 zs->zs_ranges[f].end = zs->zs_ranges[l].end; in dmu_zfetch_future()
422 zs->zs_ranges[f].start = 0; in dmu_zfetch_future()
423 zs->zs_ranges[f].end = 0; in dmu_zfetch_future()
427 for (l = ZFETCH_RANGES - 1; l > i; l--) { in dmu_zfetch_future()
428 zs->zs_ranges[l].start = zs->zs_ranges[l - 1].start; in dmu_zfetch_future()
429 zs->zs_ranges[l].end = zs->zs_ranges[l - 1].end; in dmu_zfetch_future()
431 zs->zs_ranges[i].start = blkid; in dmu_zfetch_future()
432 zs->zs_ranges[i].end = blkid + nblks; in dmu_zfetch_future()
443 zsrange_t *r = &zs->zs_ranges[i]; in dmu_zfetch_future()
444 if (r->start == 0) in dmu_zfetch_future()
446 hole += r->start - f; in dmu_zfetch_future()
447 f = r->end; in dmu_zfetch_future()
448 if (hole <= r->end >> zfetch_hole_shift) in dmu_zfetch_future()
449 l = r->end; in dmu_zfetch_future()
460 * prefetch stream, predicts further accesses based on that stats and returns
461 * the stream pointer on success. That pointer must later be passed to
462 * dmu_zfetch_run() to initiate the speculative prefetch for the stream and
466 * FALSE -- prefetch only indirect blocks for predicted data blocks;
467 * TRUE -- prefetch predicted data blocks plus following indirect blocks.
474 spa_t *spa = zf->zf_dnode->dn_objset->os_spa; in dmu_zfetch_prepare()
475 zfs_prefetch_type_t os_prefetch = zf->zf_dnode->dn_objset->os_prefetch; in dmu_zfetch_prepare()
487 * concrete vdevs (or previously-loaded indirect vdevs). So we in dmu_zfetch_prepare()
495 * As a fast path for small (single-block) files, ignore access in dmu_zfetch_prepare()
502 rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER); in dmu_zfetch_prepare()
508 uint64_t maxblkid = zf->zf_dnode->dn_maxblkid; in dmu_zfetch_prepare()
511 rw_exit(&zf->zf_dnode->dn_struct_rwlock); in dmu_zfetch_prepare()
514 mutex_enter(&zf->zf_lock); in dmu_zfetch_prepare()
517 * Find perfect prefetch stream. Depending on whether the accesses in dmu_zfetch_prepare()
518 * are block-aligned, first block of the new access may either follow in dmu_zfetch_prepare()
521 unsigned int dbs = zf->zf_dnode->dn_datablkshift; in dmu_zfetch_prepare()
523 for (zs = list_head(&zf->zf_stream); zs != NULL; in dmu_zfetch_prepare()
524 zs = list_next(&zf->zf_stream, zs)) { in dmu_zfetch_prepare()
525 if (blkid == zs->zs_blkid) { in dmu_zfetch_prepare()
527 } else if (blkid + 1 == zs->zs_blkid) { in dmu_zfetch_prepare()
529 nblks--; in dmu_zfetch_prepare()
535 * Find close enough prefetch stream. Access crossing stream position in dmu_zfetch_prepare()
536 * is a hit in its new part. Access ahead of stream position considered in dmu_zfetch_prepare()
538 * or stored for future otherwise. Access behind stream position is in dmu_zfetch_prepare()
542 uint_t t = gethrestime_sec() - zfetch_max_sec_reap; in dmu_zfetch_prepare()
543 for (zs = list_head(&zf->zf_stream); zs != NULL; in dmu_zfetch_prepare()
544 zs = list_next(&zf->zf_stream, zs)) { in dmu_zfetch_prepare()
545 if (blkid > zs->zs_blkid) { in dmu_zfetch_prepare()
546 if (end_blkid <= zs->zs_blkid + max_reorder) { in dmu_zfetch_prepare()
549 end_blkid - zs->zs_blkid); in dmu_zfetch_prepare()
560 } else if (end_blkid >= zs->zs_blkid) { in dmu_zfetch_prepare()
561 nblks -= zs->zs_blkid - blkid; in dmu_zfetch_prepare()
562 blkid += zs->zs_blkid - blkid; in dmu_zfetch_prepare()
564 } else if (end_blkid + max_reorder > zs->zs_blkid && in dmu_zfetch_prepare()
565 (int)(zs->zs_atime - t) >= 0) { in dmu_zfetch_prepare()
567 zs->zs_atime = gethrestime_sec(); in dmu_zfetch_prepare()
573 * This access is not part of any existing stream. Create a new in dmu_zfetch_prepare()
574 * stream for it unless we are at the end of file. in dmu_zfetch_prepare()
579 mutex_exit(&zf->zf_lock); in dmu_zfetch_prepare()
589 zs->zs_atime = gethrestime_sec(); in dmu_zfetch_prepare()
595 /* If the file is ending, remove the stream. */ in dmu_zfetch_prepare()
596 end_blkid = zs->zs_blkid; in dmu_zfetch_prepare()
600 mutex_exit(&zf->zf_lock); in dmu_zfetch_prepare()
602 rw_exit(&zf->zf_dnode->dn_struct_rwlock); in dmu_zfetch_prepare()
608 * behalf of this stream. Calculate further prefetch distances. in dmu_zfetch_prepare()
621 if (unlikely(zs->zs_pf_dist < nbytes)) in dmu_zfetch_prepare()
622 zs->zs_pf_dist = nbytes; in dmu_zfetch_prepare()
623 else if (zs->zs_pf_dist < zfetch_min_distance && in dmu_zfetch_prepare()
624 (zs->zs_pf_dist < (1 << dbs) || in dmu_zfetch_prepare()
627 zs->zs_pf_dist *= 2; in dmu_zfetch_prepare()
628 else if (zs->zs_more) in dmu_zfetch_prepare()
629 zs->zs_pf_dist += zs->zs_pf_dist / 8; in dmu_zfetch_prepare()
630 zs->zs_more = B_FALSE; in dmu_zfetch_prepare()
631 if (zs->zs_pf_dist > zfetch_max_distance) in dmu_zfetch_prepare()
632 zs->zs_pf_dist = zfetch_max_distance; in dmu_zfetch_prepare()
633 pf_nblks = zs->zs_pf_dist >> dbs; in dmu_zfetch_prepare()
637 if (zs->zs_pf_start < end_blkid) in dmu_zfetch_prepare()
638 zs->zs_pf_start = end_blkid; in dmu_zfetch_prepare()
639 if (zs->zs_pf_end < end_blkid + pf_nblks) in dmu_zfetch_prepare()
640 zs->zs_pf_end = end_blkid + pf_nblks; in dmu_zfetch_prepare()
646 if (unlikely(zs->zs_ipf_dist < nbytes)) in dmu_zfetch_prepare()
647 zs->zs_ipf_dist = nbytes; in dmu_zfetch_prepare()
649 zs->zs_ipf_dist *= 2; in dmu_zfetch_prepare()
650 if (zs->zs_ipf_dist > zfetch_max_idistance) in dmu_zfetch_prepare()
651 zs->zs_ipf_dist = zfetch_max_idistance; in dmu_zfetch_prepare()
652 pf_nblks = zs->zs_ipf_dist >> dbs; in dmu_zfetch_prepare()
653 if (zs->zs_ipf_start < zs->zs_pf_end) in dmu_zfetch_prepare()
654 zs->zs_ipf_start = zs->zs_pf_end; in dmu_zfetch_prepare()
655 ipf_start = zs->zs_ipf_end; in dmu_zfetch_prepare()
656 if (zs->zs_ipf_end < zs->zs_pf_end + pf_nblks) in dmu_zfetch_prepare()
657 zs->zs_ipf_end = zs->zs_pf_end + pf_nblks; in dmu_zfetch_prepare()
659 zfs_refcount_add(&zs->zs_refs, NULL); in dmu_zfetch_prepare()
661 zfs_refcount_add(&zs->zs_callers, NULL); in dmu_zfetch_prepare()
662 mutex_exit(&zf->zf_lock); in dmu_zfetch_prepare()
674 int epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT; in dmu_zfetch_prepare()
680 issued += dbuf_prefetch(zf->zf_dnode, 1, iblk, in dmu_zfetch_prepare()
685 rw_exit(&zf->zf_dnode->dn_struct_rwlock); in dmu_zfetch_prepare()
699 zs->zs_missed = missed; in dmu_zfetch_run()
706 if (zfs_refcount_remove(&zs->zs_callers, NULL) != 0) { in dmu_zfetch_run()
708 if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0) in dmu_zfetch_run()
713 mutex_enter(&zf->zf_lock); in dmu_zfetch_run()
714 if (zs->zs_missed) { in dmu_zfetch_run()
715 pf_start = zs->zs_pf_start; in dmu_zfetch_run()
716 pf_end = zs->zs_pf_start = zs->zs_pf_end; in dmu_zfetch_run()
720 ipf_start = zs->zs_ipf_start; in dmu_zfetch_run()
721 ipf_end = zs->zs_ipf_start = zs->zs_ipf_end; in dmu_zfetch_run()
722 mutex_exit(&zf->zf_lock); in dmu_zfetch_run()
726 epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT; in dmu_zfetch_run()
730 issued = pf_end - pf_start + ipf_end - ipf_start; in dmu_zfetch_run()
733 zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL); in dmu_zfetch_run()
736 if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0) in dmu_zfetch_run()
743 rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER); in dmu_zfetch_run()
747 issued += dbuf_prefetch_impl(zf->zf_dnode, 0, blk, in dmu_zfetch_run()
752 issued += dbuf_prefetch_impl(zf->zf_dnode, 1, iblk, in dmu_zfetch_run()
757 rw_exit(&zf->zf_dnode->dn_struct_rwlock); in dmu_zfetch_run()
778 "Max number of streams per zfetch");
781 "Min time before stream reclaim");
784 "Max time before stream delete");
787 "Min bytes to prefetch per stream");
790 "Max bytes to prefetch per stream");
793 "Max bytes to prefetch indirects for per stream");
796 "Max request reorder distance within a stream");
799 "Max log2 fraction of holes in a stream");