118 	zfetch_stats_t *zs = ksp->ks_data;  in zfetch_kstats_update()  local
122 	zs->zfetchstat_hits.value.ui64 =  in zfetch_kstats_update()
124 	zs->zfetchstat_future.value.ui64 =  in zfetch_kstats_update()
126 	zs->zfetchstat_stride.value.ui64 =  in zfetch_kstats_update()
128 	zs->zfetchstat_past.value.ui64 =  in zfetch_kstats_update()
130 	zs->zfetchstat_misses.value.ui64 =  in zfetch_kstats_update()
132 	zs->zfetchstat_max_streams.value.ui64 =  in zfetch_kstats_update()
134 	zs->zfetchstat_io_issued.value.ui64 =  in zfetch_kstats_update()
136 	zs->zfetchstat_io_active.value.ui64 =  in zfetch_kstats_update()
203 dmu_zfetch_stream_fini(zstream_t *zs)  in dmu_zfetch_stream_fini()  argument
205 	ASSERT(!list_link_active(&zs->zs_node));  in dmu_zfetch_stream_fini()
206 	zfs_refcount_destroy(&zs->zs_callers);  in dmu_zfetch_stream_fini()
207 	zfs_refcount_destroy(&zs->zs_refs);  in dmu_zfetch_stream_fini()
208 	kmem_free(zs, sizeof (*zs));  in dmu_zfetch_stream_fini()
212 dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)  in dmu_zfetch_stream_remove()  argument
215 	list_remove(&zf->zf_stream, zs);  in dmu_zfetch_stream_remove()
218 	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)  in dmu_zfetch_stream_remove()
219 		dmu_zfetch_stream_fini(zs);  in dmu_zfetch_stream_remove()
229 	zstream_t *zs;  in dmu_zfetch_fini()  local
232 	while ((zs = list_head(&zf->zf_stream)) != NULL)  in dmu_zfetch_fini()
233 		dmu_zfetch_stream_remove(zf, zs);  in dmu_zfetch_fini()
250 	zstream_t *zs, *zs_next, *zs_old = NULL;  in dmu_zfetch_stream_create()  local
259 	for (zs = list_head(&zf->zf_stream); zs != NULL; zs = zs_next) {  in dmu_zfetch_stream_create()
260 		zs_next = list_next(&zf->zf_stream, zs);  in dmu_zfetch_stream_create()
264 		if ((int)(zs->zs_atime - t) >= 0)  in dmu_zfetch_stream_create()
266 		if (zfs_refcount_count(&zs->zs_refs) != 1)  in dmu_zfetch_stream_create()
269 			dmu_zfetch_stream_remove(zf, zs);  in dmu_zfetch_stream_create()
271 			zs_old = zs;  in dmu_zfetch_stream_create()
274 		zs = zs_old;  in dmu_zfetch_stream_create()
275 		list_remove(&zf->zf_stream, zs);  in dmu_zfetch_stream_create()
289 		for (zs = list_head(&zf->zf_stream); zs != NULL;  in dmu_zfetch_stream_create()
290 		    zs = list_next(&zf->zf_stream, zs)) {  in dmu_zfetch_stream_create()
291 			if ((int)(zs->zs_atime - t) >= 0)  in dmu_zfetch_stream_create()
293 			if (zfs_refcount_count(&zs->zs_refs) != 1)  in dmu_zfetch_stream_create()
296 			    (int)(zs_old->zs_atime - zs->zs_atime) >= 0)  in dmu_zfetch_stream_create()
297 				zs_old = zs;  in dmu_zfetch_stream_create()
300 			zs = zs_old;  in dmu_zfetch_stream_create()
301 			list_remove(&zf->zf_stream, zs);  in dmu_zfetch_stream_create()
308 	zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);  in dmu_zfetch_stream_create()
309 	zfs_refcount_create(&zs->zs_callers);  in dmu_zfetch_stream_create()
310 	zfs_refcount_create(&zs->zs_refs);  in dmu_zfetch_stream_create()
312 	zfs_refcount_add(&zs->zs_refs, NULL);  in dmu_zfetch_stream_create()
316 	list_insert_head(&zf->zf_stream, zs);  in dmu_zfetch_stream_create()
317 	zs->zs_blkid = blkid;  in dmu_zfetch_stream_create()
319 	zs->zs_atime = now - zfetch_min_sec_reap;  in dmu_zfetch_stream_create()
320 	memset(zs->zs_ranges, 0, sizeof (zs->zs_ranges));  in dmu_zfetch_stream_create()
321 	zs->zs_pf_dist = 0;  in dmu_zfetch_stream_create()
322 	zs->zs_ipf_dist = 0;  in dmu_zfetch_stream_create()
323 	zs->zs_pf_start = blkid;  in dmu_zfetch_stream_create()
324 	zs->zs_pf_end = blkid;  in dmu_zfetch_stream_create()
325 	zs->zs_ipf_start = blkid;  in dmu_zfetch_stream_create()
326 	zs->zs_ipf_end = blkid;  in dmu_zfetch_stream_create()
327 	zs->zs_missed = B_FALSE;  in dmu_zfetch_stream_create()
328 	zs->zs_more = B_FALSE;  in dmu_zfetch_stream_create()
334 	zstream_t *zs = arg;  in dmu_zfetch_done()  local
336 	if (io_issued && level == 0 && blkid < zs->zs_blkid)  in dmu_zfetch_done()
337 		zs->zs_more = B_TRUE;  in dmu_zfetch_done()
338 	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)  in dmu_zfetch_done()
339 		dmu_zfetch_stream_fini(zs);  in dmu_zfetch_done()
348 dmu_zfetch_hit(zstream_t *zs, uint64_t nblks)  in dmu_zfetch_hit()  argument
353 	if (zs->zs_ranges[0].start == 0)  in dmu_zfetch_hit()
358 		zsrange_t *r = &zs->zs_ranges[i];  in dmu_zfetch_hit()
370 		if (zs->zs_ranges[i].start == 0)  in dmu_zfetch_hit()
372 		ASSERT3U(zs->zs_ranges[i].start, >, nblks);  in dmu_zfetch_hit()
373 		ASSERT3U(zs->zs_ranges[i].end, >, nblks);  in dmu_zfetch_hit()
374 		zs->zs_ranges[j].start = zs->zs_ranges[i].start - nblks;  in dmu_zfetch_hit()
375 		zs->zs_ranges[j].end = zs->zs_ranges[i].end - nblks;  in dmu_zfetch_hit()
378 		zs->zs_ranges[j].start = 0;  in dmu_zfetch_hit()
379 		zs->zs_ranges[j].end = 0;  in dmu_zfetch_hit()
383 	zs->zs_blkid += nblks;  in dmu_zfetch_hit()
392 dmu_zfetch_future(zstream_t *zs, uint64_t blkid, uint64_t nblks)  in dmu_zfetch_future()  argument
394 	ASSERT3U(blkid, >, zs->zs_blkid);  in dmu_zfetch_future()
395 	blkid -= zs->zs_blkid;  in dmu_zfetch_future()
401 		zsrange_t *r = &zs->zs_ranges[i];  in dmu_zfetch_future()
413 		if (zs->zs_ranges[f].start > blkid)  in dmu_zfetch_future()
414 			zs->zs_ranges[f].start = blkid;  in dmu_zfetch_future()
415 		zs->zs_ranges[f].end = MAX(zs->zs_ranges[l].end, blkid + nblks);  in dmu_zfetch_future()
419 				zs->zs_ranges[f].start = zs->zs_ranges[l].start;  in dmu_zfetch_future()
420 				zs->zs_ranges[f].end = zs->zs_ranges[l].end;  in dmu_zfetch_future()
422 			zs->zs_ranges[f].start = 0;  in dmu_zfetch_future()
423 			zs->zs_ranges[f].end = 0;  in dmu_zfetch_future()
428 			zs->zs_ranges[l].start = zs->zs_ranges[l - 1].start;  in dmu_zfetch_future()
429 			zs->zs_ranges[l].end = zs->zs_ranges[l - 1].end;  in dmu_zfetch_future()
431 		zs->zs_ranges[i].start = blkid;  in dmu_zfetch_future()
432 		zs->zs_ranges[i].end = blkid + nblks;  in dmu_zfetch_future()
443 		zsrange_t *r = &zs->zs_ranges[i];  in dmu_zfetch_future()
452 		return (dmu_zfetch_hit(zs, l));  in dmu_zfetch_future()
473 	zstream_t *zs;  in dmu_zfetch_prepare()  local
523 	for (zs = list_head(&zf->zf_stream); zs != NULL;  in dmu_zfetch_prepare()
524 	    zs = list_next(&zf->zf_stream, zs)) {  in dmu_zfetch_prepare()
525 		if (blkid == zs->zs_blkid) {  in dmu_zfetch_prepare()
527 		} else if (blkid + 1 == zs->zs_blkid) {  in dmu_zfetch_prepare()
543 	for (zs = list_head(&zf->zf_stream); zs != NULL;  in dmu_zfetch_prepare()
544 	    zs = list_next(&zf->zf_stream, zs)) {  in dmu_zfetch_prepare()
545 		if (blkid > zs->zs_blkid) {  in dmu_zfetch_prepare()
546 			if (end_blkid <= zs->zs_blkid + max_reorder) {  in dmu_zfetch_prepare()
548 					nblks = dmu_zfetch_hit(zs,  in dmu_zfetch_prepare()
549 					    end_blkid - zs->zs_blkid);  in dmu_zfetch_prepare()
553 				nblks = dmu_zfetch_future(zs, blkid, nblks);  in dmu_zfetch_prepare()
560 		} else if (end_blkid >= zs->zs_blkid) {  in dmu_zfetch_prepare()
561 			nblks -= zs->zs_blkid - blkid;  in dmu_zfetch_prepare()
562 			blkid += zs->zs_blkid - blkid;  in dmu_zfetch_prepare()
564 		} else if (end_blkid + max_reorder > zs->zs_blkid &&  in dmu_zfetch_prepare()
565 		    (int)(zs->zs_atime - t) >= 0) {  in dmu_zfetch_prepare()
567 			zs->zs_atime = gethrestime_sec();  in dmu_zfetch_prepare()
576 	ASSERT0P(zs);  in dmu_zfetch_prepare()
585 	nblks = dmu_zfetch_hit(zs, nblks);  in dmu_zfetch_prepare()
589 	zs->zs_atime = gethrestime_sec();  in dmu_zfetch_prepare()
596 	end_blkid = zs->zs_blkid;  in dmu_zfetch_prepare()
598 		dmu_zfetch_stream_remove(zf, zs);  in dmu_zfetch_prepare()
621 		if (unlikely(zs->zs_pf_dist < nbytes))  in dmu_zfetch_prepare()
622 			zs->zs_pf_dist = nbytes;  in dmu_zfetch_prepare()
623 		else if (zs->zs_pf_dist < zfetch_min_distance &&  in dmu_zfetch_prepare()
624 		    (zs->zs_pf_dist < (1 << dbs) ||  in dmu_zfetch_prepare()
627 			zs->zs_pf_dist *= 2;  in dmu_zfetch_prepare()
628 		else if (zs->zs_more)  in dmu_zfetch_prepare()
629 			zs->zs_pf_dist += zs->zs_pf_dist / 8;  in dmu_zfetch_prepare()
630 		zs->zs_more = B_FALSE;  in dmu_zfetch_prepare()
631 		if (zs->zs_pf_dist > zfetch_max_distance)  in dmu_zfetch_prepare()
632 			zs->zs_pf_dist = zfetch_max_distance;  in dmu_zfetch_prepare()
633 		pf_nblks = zs->zs_pf_dist >> dbs;  in dmu_zfetch_prepare()
637 	if (zs->zs_pf_start < end_blkid)  in dmu_zfetch_prepare()
638 		zs->zs_pf_start = end_blkid;  in dmu_zfetch_prepare()
639 	if (zs->zs_pf_end < end_blkid + pf_nblks)  in dmu_zfetch_prepare()
640 		zs->zs_pf_end = end_blkid + pf_nblks;  in dmu_zfetch_prepare()
646 	if (unlikely(zs->zs_ipf_dist < nbytes))  in dmu_zfetch_prepare()
647 		zs->zs_ipf_dist = nbytes;  in dmu_zfetch_prepare()
649 		zs->zs_ipf_dist *= 2;  in dmu_zfetch_prepare()
650 	if (zs->zs_ipf_dist > zfetch_max_idistance)  in dmu_zfetch_prepare()
651 		zs->zs_ipf_dist = zfetch_max_idistance;  in dmu_zfetch_prepare()
652 	pf_nblks = zs->zs_ipf_dist >> dbs;  in dmu_zfetch_prepare()
653 	if (zs->zs_ipf_start < zs->zs_pf_end)  in dmu_zfetch_prepare()
654 		zs->zs_ipf_start = zs->zs_pf_end;  in dmu_zfetch_prepare()
655 	ipf_start = zs->zs_ipf_end;  in dmu_zfetch_prepare()
656 	if (zs->zs_ipf_end < zs->zs_pf_end + pf_nblks)  in dmu_zfetch_prepare()
657 		zs->zs_ipf_end = zs->zs_pf_end + pf_nblks;  in dmu_zfetch_prepare()
659 	zfs_refcount_add(&zs->zs_refs, NULL);  in dmu_zfetch_prepare()
661 	zfs_refcount_add(&zs->zs_callers, NULL);  in dmu_zfetch_prepare()
688 	return (zs);  in dmu_zfetch_prepare()
692 dmu_zfetch_run(zfetch_t *zf, zstream_t *zs, boolean_t missed,  in dmu_zfetch_run()  argument
699 		zs->zs_missed = missed;  in dmu_zfetch_run()
706 	if (zfs_refcount_remove(&zs->zs_callers, NULL) != 0) {  in dmu_zfetch_run()
708 		if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)  in dmu_zfetch_run()
709 			dmu_zfetch_stream_fini(zs);  in dmu_zfetch_run()
714 	if (zs->zs_missed) {  in dmu_zfetch_run()
715 		pf_start = zs->zs_pf_start;  in dmu_zfetch_run()
716 		pf_end = zs->zs_pf_start = zs->zs_pf_end;  in dmu_zfetch_run()
720 	ipf_start = zs->zs_ipf_start;  in dmu_zfetch_run()
721 	ipf_end = zs->zs_ipf_start = zs->zs_ipf_end;  in dmu_zfetch_run()
733 		zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL);  in dmu_zfetch_run()
736 		if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)  in dmu_zfetch_run()
737 			dmu_zfetch_stream_fini(zs);  in dmu_zfetch_run()
749 		    ARC_FLAG_UNCACHED : 0, dmu_zfetch_done, zs);  in dmu_zfetch_run()
753 		    ZIO_PRIORITY_ASYNC_READ, 0, dmu_zfetch_done, zs);  in dmu_zfetch_run()
767 	zstream_t *zs;  in dmu_zfetch()  local
769 	zs = dmu_zfetch_prepare(zf, blkid, nblks, fetch_data, have_lock);  in dmu_zfetch()
770 	if (zs)  in dmu_zfetch()
771 		dmu_zfetch_run(zf, zs, missed, have_lock, uncached);  in dmu_zfetch()