xref: /freebsd/sys/dev/virtio/block/virtio_blk.c (revision 36712a94975f5bd0d26c85377283b49a2369c82f)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO block devices. */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bio.h>
38 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/sglist.h>
41 #include <sys/sysctl.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/queue.h>
45 
46 #include <geom/geom.h>
47 #include <geom/geom_disk.h>
48 
49 #include <machine/bus.h>
50 #include <machine/resource.h>
51 #include <sys/bus.h>
52 #include <sys/rman.h>
53 
54 #include <dev/virtio/virtio.h>
55 #include <dev/virtio/virtqueue.h>
56 #include <dev/virtio/block/virtio_blk.h>
57 
58 #include "virtio_if.h"
59 
60 struct vtblk_request {
61 	struct virtio_blk_outhdr	 vbr_hdr;
62 	struct bio			*vbr_bp;
63 	uint8_t				 vbr_ack;
64 	TAILQ_ENTRY(vtblk_request)	 vbr_link;
65 };
66 
67 enum vtblk_cache_mode {
68 	VTBLK_CACHE_WRITETHROUGH,
69 	VTBLK_CACHE_WRITEBACK,
70 	VTBLK_CACHE_MAX
71 };
72 
73 struct vtblk_softc {
74 	device_t		 vtblk_dev;
75 	struct mtx		 vtblk_mtx;
76 	uint64_t		 vtblk_features;
77 	uint32_t		 vtblk_flags;
78 #define VTBLK_FLAG_INDIRECT	0x0001
79 #define VTBLK_FLAG_READONLY	0x0002
80 #define VTBLK_FLAG_DETACH	0x0004
81 #define VTBLK_FLAG_SUSPEND	0x0008
82 #define VTBLK_FLAG_BARRIER	0x0010
83 #define VTBLK_FLAG_WC_CONFIG	0x0020
84 
85 	struct virtqueue	*vtblk_vq;
86 	struct sglist		*vtblk_sglist;
87 	struct disk		*vtblk_disk;
88 
89 	struct bio_queue_head	 vtblk_bioq;
90 	TAILQ_HEAD(, vtblk_request)
91 				 vtblk_req_free;
92 	TAILQ_HEAD(, vtblk_request)
93 				 vtblk_req_ready;
94 	struct vtblk_request	*vtblk_req_ordered;
95 
96 	int			 vtblk_max_nsegs;
97 	int			 vtblk_request_count;
98 	enum vtblk_cache_mode	 vtblk_write_cache;
99 
100 	struct bio_queue	 vtblk_dump_queue;
101 	struct vtblk_request	 vtblk_dump_request;
102 };
103 
104 static struct virtio_feature_desc vtblk_feature_desc[] = {
105 	{ VIRTIO_BLK_F_BARRIER,		"HostBarrier"	},
106 	{ VIRTIO_BLK_F_SIZE_MAX,	"MaxSegSize"	},
107 	{ VIRTIO_BLK_F_SEG_MAX,		"MaxNumSegs"	},
108 	{ VIRTIO_BLK_F_GEOMETRY,	"DiskGeometry"	},
109 	{ VIRTIO_BLK_F_RO,		"ReadOnly"	},
110 	{ VIRTIO_BLK_F_BLK_SIZE,	"BlockSize"	},
111 	{ VIRTIO_BLK_F_SCSI,		"SCSICmds"	},
112 	{ VIRTIO_BLK_F_WCE,		"WriteCache"	},
113 	{ VIRTIO_BLK_F_TOPOLOGY,	"Topology"	},
114 	{ VIRTIO_BLK_F_CONFIG_WCE,	"ConfigWCE"	},
115 
116 	{ 0, NULL }
117 };
118 
119 static int	vtblk_modevent(module_t, int, void *);
120 
121 static int	vtblk_probe(device_t);
122 static int	vtblk_attach(device_t);
123 static int	vtblk_detach(device_t);
124 static int	vtblk_suspend(device_t);
125 static int	vtblk_resume(device_t);
126 static int	vtblk_shutdown(device_t);
127 static int	vtblk_config_change(device_t);
128 
129 static int	vtblk_open(struct disk *);
130 static int	vtblk_close(struct disk *);
131 static int	vtblk_ioctl(struct disk *, u_long, void *, int,
132 		    struct thread *);
133 static int	vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
134 static void	vtblk_strategy(struct bio *);
135 
136 static void	vtblk_negotiate_features(struct vtblk_softc *);
137 static void	vtblk_setup_features(struct vtblk_softc *);
138 static int	vtblk_maximum_segments(struct vtblk_softc *,
139 		    struct virtio_blk_config *);
140 static int	vtblk_alloc_virtqueue(struct vtblk_softc *);
141 static void	vtblk_resize_disk(struct vtblk_softc *, uint64_t);
142 static void	vtblk_alloc_disk(struct vtblk_softc *,
143 		    struct virtio_blk_config *);
144 static void	vtblk_create_disk(struct vtblk_softc *);
145 
146 static int	vtblk_request_prealloc(struct vtblk_softc *);
147 static void	vtblk_request_free(struct vtblk_softc *);
148 static struct vtblk_request *
149 		vtblk_request_dequeue(struct vtblk_softc *);
150 static void	vtblk_request_enqueue(struct vtblk_softc *,
151 		    struct vtblk_request *);
152 static struct vtblk_request *
153 		vtblk_request_next_ready(struct vtblk_softc *);
154 static void	vtblk_request_requeue_ready(struct vtblk_softc *,
155 		    struct vtblk_request *);
156 static struct vtblk_request *
157 		vtblk_request_next(struct vtblk_softc *);
158 static struct vtblk_request *
159 		vtblk_request_bio(struct vtblk_softc *);
160 static int	vtblk_request_execute(struct vtblk_softc *,
161 		    struct vtblk_request *);
162 static int	vtblk_request_error(struct vtblk_request *);
163 
164 static void	vtblk_queue_completed(struct vtblk_softc *,
165 		    struct bio_queue *);
166 static void	vtblk_done_completed(struct vtblk_softc *,
167 		    struct bio_queue *);
168 static void	vtblk_drain_vq(struct vtblk_softc *);
169 static void	vtblk_drain(struct vtblk_softc *);
170 
171 static void	vtblk_startio(struct vtblk_softc *);
172 static void	vtblk_bio_done(struct vtblk_softc *, struct bio *, int);
173 
174 static void	vtblk_read_config(struct vtblk_softc *,
175 		    struct virtio_blk_config *);
176 static void	vtblk_ident(struct vtblk_softc *);
177 static int	vtblk_poll_request(struct vtblk_softc *,
178 		    struct vtblk_request *);
179 static int	vtblk_quiesce(struct vtblk_softc *);
180 static void	vtblk_vq_intr(void *);
181 static void	vtblk_stop(struct vtblk_softc *);
182 
183 static void	vtblk_dump_quiesce(struct vtblk_softc *);
184 static int	vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t);
185 static int	vtblk_dump_flush(struct vtblk_softc *);
186 static void	vtblk_dump_complete(struct vtblk_softc *);
187 
188 static void	vtblk_set_write_cache(struct vtblk_softc *, int);
189 static int	vtblk_write_cache_enabled(struct vtblk_softc *sc,
190 		    struct virtio_blk_config *);
191 static int	vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
192 
193 static void	vtblk_setup_sysctl(struct vtblk_softc *);
194 static int	vtblk_tunable_int(struct vtblk_softc *, const char *, int);
195 
196 /* Tunables. */
197 static int vtblk_no_ident = 0;
198 TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
199 static int vtblk_writecache_mode = -1;
200 TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);
201 
202 /* Features desired/implemented by this driver. */
203 #define VTBLK_FEATURES \
204     (VIRTIO_BLK_F_BARRIER		| \
205      VIRTIO_BLK_F_SIZE_MAX		| \
206      VIRTIO_BLK_F_SEG_MAX		| \
207      VIRTIO_BLK_F_GEOMETRY		| \
208      VIRTIO_BLK_F_RO			| \
209      VIRTIO_BLK_F_BLK_SIZE		| \
210      VIRTIO_BLK_F_WCE			| \
211      VIRTIO_BLK_F_TOPOLOGY		| \
212      VIRTIO_BLK_F_CONFIG_WCE		| \
213      VIRTIO_RING_F_INDIRECT_DESC)
214 
215 #define VTBLK_MTX(_sc)		&(_sc)->vtblk_mtx
216 #define VTBLK_LOCK_INIT(_sc, _name) \
217 				mtx_init(VTBLK_MTX((_sc)), (_name), \
218 				    "VirtIO Block Lock", MTX_DEF)
219 #define VTBLK_LOCK(_sc)		mtx_lock(VTBLK_MTX((_sc)))
220 #define VTBLK_UNLOCK(_sc)	mtx_unlock(VTBLK_MTX((_sc)))
221 #define VTBLK_LOCK_DESTROY(_sc)	mtx_destroy(VTBLK_MTX((_sc)))
222 #define VTBLK_LOCK_ASSERT(_sc)	mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
223 #define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
224 				mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
225 
226 #define VTBLK_DISK_NAME		"vtbd"
227 #define VTBLK_QUIESCE_TIMEOUT	(30 * hz)
228 
229 /*
230  * Each block request uses at least two segments - one for the header
231  * and one for the status.
232  */
233 #define VTBLK_MIN_SEGMENTS	2
234 
235 static device_method_t vtblk_methods[] = {
236 	/* Device methods. */
237 	DEVMETHOD(device_probe,		vtblk_probe),
238 	DEVMETHOD(device_attach,	vtblk_attach),
239 	DEVMETHOD(device_detach,	vtblk_detach),
240 	DEVMETHOD(device_suspend,	vtblk_suspend),
241 	DEVMETHOD(device_resume,	vtblk_resume),
242 	DEVMETHOD(device_shutdown,	vtblk_shutdown),
243 
244 	/* VirtIO methods. */
245 	DEVMETHOD(virtio_config_change,	vtblk_config_change),
246 
247 	DEVMETHOD_END
248 };
249 
250 static driver_t vtblk_driver = {
251 	"vtblk",
252 	vtblk_methods,
253 	sizeof(struct vtblk_softc)
254 };
255 static devclass_t vtblk_devclass;
256 
257 DRIVER_MODULE(virtio_blk, virtio_mmio, vtblk_driver, vtblk_devclass,
258     vtblk_modevent, 0);
259 DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
260     vtblk_modevent, 0);
261 MODULE_VERSION(virtio_blk, 1);
262 MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
263 
264 VIRTIO_SIMPLE_PNPTABLE(virtio_blk, VIRTIO_ID_BLOCK, "VirtIO Block Adapter");
265 VIRTIO_SIMPLE_PNPINFO(virtio_mmio, virtio_blk);
266 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_blk);
267 
268 static int
269 vtblk_modevent(module_t mod, int type, void *unused)
270 {
271 	int error;
272 
273 	error = 0;
274 
275 	switch (type) {
276 	case MOD_LOAD:
277 	case MOD_QUIESCE:
278 	case MOD_UNLOAD:
279 	case MOD_SHUTDOWN:
280 		break;
281 	default:
282 		error = EOPNOTSUPP;
283 		break;
284 	}
285 
286 	return (error);
287 }
288 
289 static int
290 vtblk_probe(device_t dev)
291 {
292 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_blk));
293 }
294 
295 static int
296 vtblk_attach(device_t dev)
297 {
298 	struct vtblk_softc *sc;
299 	struct virtio_blk_config blkcfg;
300 	int error;
301 
302 	virtio_set_feature_desc(dev, vtblk_feature_desc);
303 
304 	sc = device_get_softc(dev);
305 	sc->vtblk_dev = dev;
306 	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
307 	bioq_init(&sc->vtblk_bioq);
308 	TAILQ_INIT(&sc->vtblk_dump_queue);
309 	TAILQ_INIT(&sc->vtblk_req_free);
310 	TAILQ_INIT(&sc->vtblk_req_ready);
311 
312 	vtblk_setup_sysctl(sc);
313 	vtblk_setup_features(sc);
314 
315 	vtblk_read_config(sc, &blkcfg);
316 
317 	/*
318 	 * With the current sglist(9) implementation, it is not easy
319 	 * for us to support a maximum segment size as adjacent
320 	 * segments are coalesced. For now, just make sure it's larger
321 	 * than the maximum supported transfer size.
322 	 */
323 	if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
324 		if (blkcfg.size_max < MAXPHYS) {
325 			error = ENOTSUP;
326 			device_printf(dev, "host requires unsupported "
327 			    "maximum segment size feature\n");
328 			goto fail;
329 		}
330 	}
331 
332 	sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
333 	if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
334 		error = EINVAL;
335 		device_printf(dev, "fewer than minimum number of segments "
336 		    "allowed: %d\n", sc->vtblk_max_nsegs);
337 		goto fail;
338 	}
339 
340 	sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
341 	if (sc->vtblk_sglist == NULL) {
342 		error = ENOMEM;
343 		device_printf(dev, "cannot allocate sglist\n");
344 		goto fail;
345 	}
346 
347 	error = vtblk_alloc_virtqueue(sc);
348 	if (error) {
349 		device_printf(dev, "cannot allocate virtqueue\n");
350 		goto fail;
351 	}
352 
353 	error = vtblk_request_prealloc(sc);
354 	if (error) {
355 		device_printf(dev, "cannot preallocate requests\n");
356 		goto fail;
357 	}
358 
359 	vtblk_alloc_disk(sc, &blkcfg);
360 
361 	error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY);
362 	if (error) {
363 		device_printf(dev, "cannot setup virtqueue interrupt\n");
364 		goto fail;
365 	}
366 
367 	vtblk_create_disk(sc);
368 
369 	virtqueue_enable_intr(sc->vtblk_vq);
370 
371 fail:
372 	if (error)
373 		vtblk_detach(dev);
374 
375 	return (error);
376 }
377 
378 static int
379 vtblk_detach(device_t dev)
380 {
381 	struct vtblk_softc *sc;
382 
383 	sc = device_get_softc(dev);
384 
385 	VTBLK_LOCK(sc);
386 	sc->vtblk_flags |= VTBLK_FLAG_DETACH;
387 	if (device_is_attached(dev))
388 		vtblk_stop(sc);
389 	VTBLK_UNLOCK(sc);
390 
391 	vtblk_drain(sc);
392 
393 	if (sc->vtblk_disk != NULL) {
394 		disk_destroy(sc->vtblk_disk);
395 		sc->vtblk_disk = NULL;
396 	}
397 
398 	if (sc->vtblk_sglist != NULL) {
399 		sglist_free(sc->vtblk_sglist);
400 		sc->vtblk_sglist = NULL;
401 	}
402 
403 	VTBLK_LOCK_DESTROY(sc);
404 
405 	return (0);
406 }
407 
408 static int
409 vtblk_suspend(device_t dev)
410 {
411 	struct vtblk_softc *sc;
412 	int error;
413 
414 	sc = device_get_softc(dev);
415 
416 	VTBLK_LOCK(sc);
417 	sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
418 	/* XXX BMV: virtio_stop(), etc needed here? */
419 	error = vtblk_quiesce(sc);
420 	if (error)
421 		sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
422 	VTBLK_UNLOCK(sc);
423 
424 	return (error);
425 }
426 
427 static int
428 vtblk_resume(device_t dev)
429 {
430 	struct vtblk_softc *sc;
431 
432 	sc = device_get_softc(dev);
433 
434 	VTBLK_LOCK(sc);
435 	/* XXX BMV: virtio_reinit(), etc needed here? */
436 	sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
437 	vtblk_startio(sc);
438 	VTBLK_UNLOCK(sc);
439 
440 	return (0);
441 }
442 
443 static int
444 vtblk_shutdown(device_t dev)
445 {
446 
447 	return (0);
448 }
449 
450 static int
451 vtblk_config_change(device_t dev)
452 {
453 	struct vtblk_softc *sc;
454 	struct virtio_blk_config blkcfg;
455 	uint64_t capacity;
456 
457 	sc = device_get_softc(dev);
458 
459 	vtblk_read_config(sc, &blkcfg);
460 
461 	/* Capacity is always in 512-byte units. */
462 	capacity = blkcfg.capacity * 512;
463 
464 	if (sc->vtblk_disk->d_mediasize != capacity)
465 		vtblk_resize_disk(sc, capacity);
466 
467 	return (0);
468 }
469 
470 static int
471 vtblk_open(struct disk *dp)
472 {
473 	struct vtblk_softc *sc;
474 
475 	if ((sc = dp->d_drv1) == NULL)
476 		return (ENXIO);
477 
478 	return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
479 }
480 
481 static int
482 vtblk_close(struct disk *dp)
483 {
484 	struct vtblk_softc *sc;
485 
486 	if ((sc = dp->d_drv1) == NULL)
487 		return (ENXIO);
488 
489 	return (0);
490 }
491 
492 static int
493 vtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag,
494     struct thread *td)
495 {
496 	struct vtblk_softc *sc;
497 
498 	if ((sc = dp->d_drv1) == NULL)
499 		return (ENXIO);
500 
501 	return (ENOTTY);
502 }
503 
504 static int
505 vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset,
506     size_t length)
507 {
508 	struct disk *dp;
509 	struct vtblk_softc *sc;
510 	int error;
511 
512 	dp = arg;
513 	error = 0;
514 
515 	if ((sc = dp->d_drv1) == NULL)
516 		return (ENXIO);
517 
518 	VTBLK_LOCK(sc);
519 
520 	vtblk_dump_quiesce(sc);
521 
522 	if (length > 0)
523 		error = vtblk_dump_write(sc, virtual, offset, length);
524 	if (error || (virtual == NULL && offset == 0))
525 		vtblk_dump_complete(sc);
526 
527 	VTBLK_UNLOCK(sc);
528 
529 	return (error);
530 }
531 
532 static void
533 vtblk_strategy(struct bio *bp)
534 {
535 	struct vtblk_softc *sc;
536 
537 	if ((sc = bp->bio_disk->d_drv1) == NULL) {
538 		vtblk_bio_done(NULL, bp, EINVAL);
539 		return;
540 	}
541 
542 	/*
543 	 * Fail any write if RO. Unfortunately, there does not seem to
544 	 * be a better way to report our readonly'ness to GEOM above.
545 	 */
546 	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
547 	    (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
548 		vtblk_bio_done(sc, bp, EROFS);
549 		return;
550 	}
551 
552 	VTBLK_LOCK(sc);
553 
554 	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
555 		VTBLK_UNLOCK(sc);
556 		vtblk_bio_done(sc, bp, ENXIO);
557 		return;
558 	}
559 
560 	bioq_insert_tail(&sc->vtblk_bioq, bp);
561 	vtblk_startio(sc);
562 
563 	VTBLK_UNLOCK(sc);
564 }
565 
566 static void
567 vtblk_negotiate_features(struct vtblk_softc *sc)
568 {
569 	device_t dev;
570 	uint64_t features;
571 
572 	dev = sc->vtblk_dev;
573 	features = VTBLK_FEATURES;
574 
575 	sc->vtblk_features = virtio_negotiate_features(dev, features);
576 }
577 
578 static void
579 vtblk_setup_features(struct vtblk_softc *sc)
580 {
581 	device_t dev;
582 
583 	dev = sc->vtblk_dev;
584 
585 	vtblk_negotiate_features(sc);
586 
587 	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
588 		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
589 	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
590 		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
591 	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
592 		sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
593 	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
594 		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
595 }
596 
597 static int
598 vtblk_maximum_segments(struct vtblk_softc *sc,
599     struct virtio_blk_config *blkcfg)
600 {
601 	device_t dev;
602 	int nsegs;
603 
604 	dev = sc->vtblk_dev;
605 	nsegs = VTBLK_MIN_SEGMENTS;
606 
607 	if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
608 		nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1);
609 		if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT)
610 			nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
611 	} else
612 		nsegs += 1;
613 
614 	return (nsegs);
615 }
616 
617 static int
618 vtblk_alloc_virtqueue(struct vtblk_softc *sc)
619 {
620 	device_t dev;
621 	struct vq_alloc_info vq_info;
622 
623 	dev = sc->vtblk_dev;
624 
625 	VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
626 	    vtblk_vq_intr, sc, &sc->vtblk_vq,
627 	    "%s request", device_get_nameunit(dev));
628 
629 	return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
630 }
631 
632 static void
633 vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity)
634 {
635 	device_t dev;
636 	struct disk *dp;
637 	int error;
638 
639 	dev = sc->vtblk_dev;
640 	dp = sc->vtblk_disk;
641 
642 	dp->d_mediasize = new_capacity;
643 	if (bootverbose) {
644 		device_printf(dev, "resized to %juMB (%ju %u byte sectors)\n",
645 		    (uintmax_t) dp->d_mediasize >> 20,
646 		    (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
647 		    dp->d_sectorsize);
648 	}
649 
650 	error = disk_resize(dp, M_NOWAIT);
651 	if (error) {
652 		device_printf(dev,
653 		    "disk_resize(9) failed, error: %d\n", error);
654 	}
655 }
656 
657 static void
658 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
659 {
660 	device_t dev;
661 	struct disk *dp;
662 
663 	dev = sc->vtblk_dev;
664 
665 	sc->vtblk_disk = dp = disk_alloc();
666 	dp->d_open = vtblk_open;
667 	dp->d_close = vtblk_close;
668 	dp->d_ioctl = vtblk_ioctl;
669 	dp->d_strategy = vtblk_strategy;
670 	dp->d_name = VTBLK_DISK_NAME;
671 	dp->d_unit = device_get_unit(dev);
672 	dp->d_drv1 = sc;
673 	dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO |
674 	    DISKFLAG_DIRECT_COMPLETION;
675 	dp->d_hba_vendor = virtio_get_vendor(dev);
676 	dp->d_hba_device = virtio_get_device(dev);
677 	dp->d_hba_subvendor = virtio_get_subvendor(dev);
678 	dp->d_hba_subdevice = virtio_get_subdevice(dev);
679 
680 	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
681 		dp->d_dump = vtblk_dump;
682 
683 	/* Capacity is always in 512-byte units. */
684 	dp->d_mediasize = blkcfg->capacity * 512;
685 
686 	if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE))
687 		dp->d_sectorsize = blkcfg->blk_size;
688 	else
689 		dp->d_sectorsize = 512;
690 
691 	/*
692 	 * The VirtIO maximum I/O size is given in terms of segments.
693 	 * However, FreeBSD limits I/O size by logical buffer size, not
694 	 * by physically contiguous pages. Therefore, we have to assume
695 	 * no pages are contiguous. This may impose an artificially low
696 	 * maximum I/O size. But in practice, since QEMU advertises 128
697 	 * segments, this gives us a maximum IO size of 125 * PAGE_SIZE,
698 	 * which is typically greater than MAXPHYS. Eventually we should
699 	 * just advertise MAXPHYS and split buffers that are too big.
700 	 *
701 	 * Note we must subtract one additional segment in case of non
702 	 * page aligned buffers.
703 	 */
704 	dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS - 1) *
705 	    PAGE_SIZE;
706 	if (dp->d_maxsize < PAGE_SIZE)
707 		dp->d_maxsize = PAGE_SIZE; /* XXX */
708 
709 	if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) {
710 		dp->d_fwsectors = blkcfg->geometry.sectors;
711 		dp->d_fwheads = blkcfg->geometry.heads;
712 	}
713 
714 	if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) &&
715 	    blkcfg->topology.physical_block_exp > 0) {
716 		dp->d_stripesize = dp->d_sectorsize *
717 		    (1 << blkcfg->topology.physical_block_exp);
718 		dp->d_stripeoffset = (dp->d_stripesize -
719 		    blkcfg->topology.alignment_offset * dp->d_sectorsize) %
720 		    dp->d_stripesize;
721 	}
722 
723 	if (vtblk_write_cache_enabled(sc, blkcfg) != 0)
724 		sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK;
725 	else
726 		sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH;
727 }
728 
729 static void
730 vtblk_create_disk(struct vtblk_softc *sc)
731 {
732 	struct disk *dp;
733 
734 	dp = sc->vtblk_disk;
735 
736 	vtblk_ident(sc);
737 
738 	device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
739 	    (uintmax_t) dp->d_mediasize >> 20,
740 	    (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
741 	    dp->d_sectorsize);
742 
743 	disk_create(dp, DISK_VERSION);
744 }
745 
746 static int
747 vtblk_request_prealloc(struct vtblk_softc *sc)
748 {
749 	struct vtblk_request *req;
750 	int i, nreqs;
751 
752 	nreqs = virtqueue_size(sc->vtblk_vq);
753 
754 	/*
755 	 * Preallocate sufficient requests to keep the virtqueue full. Each
756 	 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
757 	 * the number allocated when indirect descriptors are not available.
758 	 */
759 	if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
760 		nreqs /= VTBLK_MIN_SEGMENTS;
761 
762 	for (i = 0; i < nreqs; i++) {
763 		req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
764 		if (req == NULL)
765 			return (ENOMEM);
766 
767 		MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1);
768 		MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1);
769 
770 		sc->vtblk_request_count++;
771 		vtblk_request_enqueue(sc, req);
772 	}
773 
774 	return (0);
775 }
776 
777 static void
778 vtblk_request_free(struct vtblk_softc *sc)
779 {
780 	struct vtblk_request *req;
781 
782 	MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready));
783 
784 	while ((req = vtblk_request_dequeue(sc)) != NULL) {
785 		sc->vtblk_request_count--;
786 		free(req, M_DEVBUF);
787 	}
788 
789 	KASSERT(sc->vtblk_request_count == 0,
790 	    ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
791 }
792 
793 static struct vtblk_request *
794 vtblk_request_dequeue(struct vtblk_softc *sc)
795 {
796 	struct vtblk_request *req;
797 
798 	req = TAILQ_FIRST(&sc->vtblk_req_free);
799 	if (req != NULL) {
800 		TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
801 		bzero(req, sizeof(struct vtblk_request));
802 	}
803 
804 	return (req);
805 }
806 
807 static void
808 vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req)
809 {
810 
811 	TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
812 }
813 
814 static struct vtblk_request *
815 vtblk_request_next_ready(struct vtblk_softc *sc)
816 {
817 	struct vtblk_request *req;
818 
819 	req = TAILQ_FIRST(&sc->vtblk_req_ready);
820 	if (req != NULL)
821 		TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
822 
823 	return (req);
824 }
825 
826 static void
827 vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
828 {
829 
830 	/* NOTE: Currently, there will be at most one request in the queue. */
831 	TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
832 }
833 
834 static struct vtblk_request *
835 vtblk_request_next(struct vtblk_softc *sc)
836 {
837 	struct vtblk_request *req;
838 
839 	req = vtblk_request_next_ready(sc);
840 	if (req != NULL)
841 		return (req);
842 
843 	return (vtblk_request_bio(sc));
844 }
845 
846 static struct vtblk_request *
847 vtblk_request_bio(struct vtblk_softc *sc)
848 {
849 	struct bio_queue_head *bioq;
850 	struct vtblk_request *req;
851 	struct bio *bp;
852 
853 	bioq = &sc->vtblk_bioq;
854 
855 	if (bioq_first(bioq) == NULL)
856 		return (NULL);
857 
858 	req = vtblk_request_dequeue(sc);
859 	if (req == NULL)
860 		return (NULL);
861 
862 	bp = bioq_takefirst(bioq);
863 	req->vbr_bp = bp;
864 	req->vbr_ack = -1;
865 	req->vbr_hdr.ioprio = 1;
866 
867 	switch (bp->bio_cmd) {
868 	case BIO_FLUSH:
869 		req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
870 		break;
871 	case BIO_READ:
872 		req->vbr_hdr.type = VIRTIO_BLK_T_IN;
873 		req->vbr_hdr.sector = bp->bio_offset / 512;
874 		break;
875 	case BIO_WRITE:
876 		req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
877 		req->vbr_hdr.sector = bp->bio_offset / 512;
878 		break;
879 	default:
880 		panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
881 	}
882 
883 	if (bp->bio_flags & BIO_ORDERED)
884 		req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
885 
886 	return (req);
887 }
888 
889 static int
890 vtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req)
891 {
892 	struct virtqueue *vq;
893 	struct sglist *sg;
894 	struct bio *bp;
895 	int ordered, readable, writable, error;
896 
897 	vq = sc->vtblk_vq;
898 	sg = sc->vtblk_sglist;
899 	bp = req->vbr_bp;
900 	ordered = 0;
901 	writable = 0;
902 
903 	/*
904 	 * Some hosts (such as bhyve) do not implement the barrier feature,
905 	 * so we emulate it in the driver by allowing the barrier request
906 	 * to be the only one in flight.
907 	 */
908 	if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
909 		if (sc->vtblk_req_ordered != NULL)
910 			return (EBUSY);
911 		if (bp->bio_flags & BIO_ORDERED) {
912 			if (!virtqueue_empty(vq))
913 				return (EBUSY);
914 			ordered = 1;
915 			req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
916 		}
917 	}
918 
919 	sglist_reset(sg);
920 	sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
921 
922 	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
923 		error = sglist_append_bio(sg, bp);
924 		if (error || sg->sg_nseg == sg->sg_maxseg) {
925 			panic("%s: bio %p data buffer too big %d",
926 			    __func__, bp, error);
927 		}
928 
929 		/* BIO_READ means the host writes into our buffer. */
930 		if (bp->bio_cmd == BIO_READ)
931 			writable = sg->sg_nseg - 1;
932 	}
933 
934 	writable++;
935 	sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
936 	readable = sg->sg_nseg - writable;
937 
938 	error = virtqueue_enqueue(vq, req, sg, readable, writable);
939 	if (error == 0 && ordered)
940 		sc->vtblk_req_ordered = req;
941 
942 	return (error);
943 }
944 
945 static int
946 vtblk_request_error(struct vtblk_request *req)
947 {
948 	int error;
949 
950 	switch (req->vbr_ack) {
951 	case VIRTIO_BLK_S_OK:
952 		error = 0;
953 		break;
954 	case VIRTIO_BLK_S_UNSUPP:
955 		error = ENOTSUP;
956 		break;
957 	default:
958 		error = EIO;
959 		break;
960 	}
961 
962 	return (error);
963 }
964 
965 static void
966 vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue)
967 {
968 	struct vtblk_request *req;
969 	struct bio *bp;
970 
971 	while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
972 		if (sc->vtblk_req_ordered != NULL) {
973 			MPASS(sc->vtblk_req_ordered == req);
974 			sc->vtblk_req_ordered = NULL;
975 		}
976 
977 		bp = req->vbr_bp;
978 		bp->bio_error = vtblk_request_error(req);
979 		TAILQ_INSERT_TAIL(queue, bp, bio_queue);
980 
981 		vtblk_request_enqueue(sc, req);
982 	}
983 }
984 
985 static void
986 vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue)
987 {
988 	struct bio *bp, *tmp;
989 
990 	TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) {
991 		if (bp->bio_error != 0)
992 			disk_err(bp, "hard error", -1, 1);
993 		vtblk_bio_done(sc, bp, bp->bio_error);
994 	}
995 }
996 
997 static void
998 vtblk_drain_vq(struct vtblk_softc *sc)
999 {
1000 	struct virtqueue *vq;
1001 	struct vtblk_request *req;
1002 	int last;
1003 
1004 	vq = sc->vtblk_vq;
1005 	last = 0;
1006 
1007 	while ((req = virtqueue_drain(vq, &last)) != NULL) {
1008 		vtblk_bio_done(sc, req->vbr_bp, ENXIO);
1009 		vtblk_request_enqueue(sc, req);
1010 	}
1011 
1012 	sc->vtblk_req_ordered = NULL;
1013 	KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
1014 }
1015 
1016 static void
1017 vtblk_drain(struct vtblk_softc *sc)
1018 {
1019 	struct bio_queue queue;
1020 	struct bio_queue_head *bioq;
1021 	struct vtblk_request *req;
1022 	struct bio *bp;
1023 
1024 	bioq = &sc->vtblk_bioq;
1025 	TAILQ_INIT(&queue);
1026 
1027 	if (sc->vtblk_vq != NULL) {
1028 		vtblk_queue_completed(sc, &queue);
1029 		vtblk_done_completed(sc, &queue);
1030 
1031 		vtblk_drain_vq(sc);
1032 	}
1033 
1034 	while ((req = vtblk_request_next_ready(sc)) != NULL) {
1035 		vtblk_bio_done(sc, req->vbr_bp, ENXIO);
1036 		vtblk_request_enqueue(sc, req);
1037 	}
1038 
1039 	while (bioq_first(bioq) != NULL) {
1040 		bp = bioq_takefirst(bioq);
1041 		vtblk_bio_done(sc, bp, ENXIO);
1042 	}
1043 
1044 	vtblk_request_free(sc);
1045 }
1046 
1047 static void
1048 vtblk_startio(struct vtblk_softc *sc)
1049 {
1050 	struct virtqueue *vq;
1051 	struct vtblk_request *req;
1052 	int enq;
1053 
1054 	VTBLK_LOCK_ASSERT(sc);
1055 	vq = sc->vtblk_vq;
1056 	enq = 0;
1057 
1058 	if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
1059 		return;
1060 
1061 	while (!virtqueue_full(vq)) {
1062 		req = vtblk_request_next(sc);
1063 		if (req == NULL)
1064 			break;
1065 
1066 		if (vtblk_request_execute(sc, req) != 0) {
1067 			vtblk_request_requeue_ready(sc, req);
1068 			break;
1069 		}
1070 
1071 		enq++;
1072 	}
1073 
1074 	if (enq > 0)
1075 		virtqueue_notify(vq);
1076 }
1077 
1078 static void
1079 vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error)
1080 {
1081 
1082 	/* Because of GEOM direct dispatch, we cannot hold any locks. */
1083 	if (sc != NULL)
1084 		VTBLK_LOCK_ASSERT_NOTOWNED(sc);
1085 
1086 	if (error) {
1087 		bp->bio_resid = bp->bio_bcount;
1088 		bp->bio_error = error;
1089 		bp->bio_flags |= BIO_ERROR;
1090 	}
1091 
1092 	biodone(bp);
1093 }
1094 
1095 #define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg)			\
1096 	if (virtio_with_feature(_dev, _feature)) {			\
1097 		virtio_read_device_config(_dev,				\
1098 		    offsetof(struct virtio_blk_config, _field),		\
1099 		    &(_cfg)->_field, sizeof((_cfg)->_field));		\
1100 	}
1101 
1102 static void
1103 vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
1104 {
1105 	device_t dev;
1106 
1107 	dev = sc->vtblk_dev;
1108 
1109 	bzero(blkcfg, sizeof(struct virtio_blk_config));
1110 
1111 	/* The capacity is always available. */
1112 	virtio_read_device_config(dev, offsetof(struct virtio_blk_config,
1113 	    capacity), &blkcfg->capacity, sizeof(blkcfg->capacity));
1114 
1115 	/* Read the configuration if the feature was negotiated. */
1116 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);
1117 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);
1118 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg);
1119 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);
1120 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg);
1121 	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg);
1122 }
1123 
1124 #undef VTBLK_GET_CONFIG
1125 
1126 static void
1127 vtblk_ident(struct vtblk_softc *sc)
1128 {
1129 	struct bio buf;
1130 	struct disk *dp;
1131 	struct vtblk_request *req;
1132 	int len, error;
1133 
1134 	dp = sc->vtblk_disk;
1135 	len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE);
1136 
1137 	if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0)
1138 		return;
1139 
1140 	req = vtblk_request_dequeue(sc);
1141 	if (req == NULL)
1142 		return;
1143 
1144 	req->vbr_ack = -1;
1145 	req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID;
1146 	req->vbr_hdr.ioprio = 1;
1147 	req->vbr_hdr.sector = 0;
1148 
1149 	req->vbr_bp = &buf;
1150 	g_reset_bio(&buf);
1151 
1152 	buf.bio_cmd = BIO_READ;
1153 	buf.bio_data = dp->d_ident;
1154 	buf.bio_bcount = len;
1155 
1156 	VTBLK_LOCK(sc);
1157 	error = vtblk_poll_request(sc, req);
1158 	VTBLK_UNLOCK(sc);
1159 
1160 	vtblk_request_enqueue(sc, req);
1161 
1162 	if (error) {
1163 		device_printf(sc->vtblk_dev,
1164 		    "error getting device identifier: %d\n", error);
1165 	}
1166 }
1167 
1168 static int
1169 vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
1170 {
1171 	struct virtqueue *vq;
1172 	int error;
1173 
1174 	vq = sc->vtblk_vq;
1175 
1176 	if (!virtqueue_empty(vq))
1177 		return (EBUSY);
1178 
1179 	error = vtblk_request_execute(sc, req);
1180 	if (error)
1181 		return (error);
1182 
1183 	virtqueue_notify(vq);
1184 	virtqueue_poll(vq, NULL);
1185 
1186 	error = vtblk_request_error(req);
1187 	if (error && bootverbose) {
1188 		device_printf(sc->vtblk_dev,
1189 		    "%s: IO error: %d\n", __func__, error);
1190 	}
1191 
1192 	return (error);
1193 }
1194 
1195 static int
1196 vtblk_quiesce(struct vtblk_softc *sc)
1197 {
1198 	int error;
1199 
1200 	VTBLK_LOCK_ASSERT(sc);
1201 	error = 0;
1202 
1203 	while (!virtqueue_empty(sc->vtblk_vq)) {
1204 		if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
1205 		    VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
1206 			error = EBUSY;
1207 			break;
1208 		}
1209 	}
1210 
1211 	return (error);
1212 }
1213 
1214 static void
1215 vtblk_vq_intr(void *xsc)
1216 {
1217 	struct vtblk_softc *sc;
1218 	struct virtqueue *vq;
1219 	struct bio_queue queue;
1220 
1221 	sc = xsc;
1222 	vq = sc->vtblk_vq;
1223 	TAILQ_INIT(&queue);
1224 
1225 	VTBLK_LOCK(sc);
1226 
1227 again:
1228 	if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
1229 		goto out;
1230 
1231 	vtblk_queue_completed(sc, &queue);
1232 	vtblk_startio(sc);
1233 
1234 	if (virtqueue_enable_intr(vq) != 0) {
1235 		virtqueue_disable_intr(vq);
1236 		goto again;
1237 	}
1238 
1239 	if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
1240 		wakeup(&sc->vtblk_vq);
1241 
1242 out:
1243 	VTBLK_UNLOCK(sc);
1244 	vtblk_done_completed(sc, &queue);
1245 }
1246 
1247 static void
1248 vtblk_stop(struct vtblk_softc *sc)
1249 {
1250 
1251 	virtqueue_disable_intr(sc->vtblk_vq);
1252 	virtio_stop(sc->vtblk_dev);
1253 }
1254 
1255 static void
1256 vtblk_dump_quiesce(struct vtblk_softc *sc)
1257 {
1258 
1259 	/*
1260 	 * Spin here until all the requests in-flight at the time of the
1261 	 * dump are completed and queued. The queued requests will be
1262 	 * biodone'd once the dump is finished.
1263 	 */
1264 	while (!virtqueue_empty(sc->vtblk_vq))
1265 		vtblk_queue_completed(sc, &sc->vtblk_dump_queue);
1266 }
1267 
1268 static int
1269 vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset,
1270     size_t length)
1271 {
1272 	struct bio buf;
1273 	struct vtblk_request *req;
1274 
1275 	req = &sc->vtblk_dump_request;
1276 	req->vbr_ack = -1;
1277 	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
1278 	req->vbr_hdr.ioprio = 1;
1279 	req->vbr_hdr.sector = offset / 512;
1280 
1281 	req->vbr_bp = &buf;
1282 	g_reset_bio(&buf);
1283 
1284 	buf.bio_cmd = BIO_WRITE;
1285 	buf.bio_data = virtual;
1286 	buf.bio_bcount = length;
1287 
1288 	return (vtblk_poll_request(sc, req));
1289 }
1290 
1291 static int
1292 vtblk_dump_flush(struct vtblk_softc *sc)
1293 {
1294 	struct bio buf;
1295 	struct vtblk_request *req;
1296 
1297 	req = &sc->vtblk_dump_request;
1298 	req->vbr_ack = -1;
1299 	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
1300 	req->vbr_hdr.ioprio = 1;
1301 	req->vbr_hdr.sector = 0;
1302 
1303 	req->vbr_bp = &buf;
1304 	g_reset_bio(&buf);
1305 
1306 	buf.bio_cmd = BIO_FLUSH;
1307 
1308 	return (vtblk_poll_request(sc, req));
1309 }
1310 
1311 static void
1312 vtblk_dump_complete(struct vtblk_softc *sc)
1313 {
1314 
1315 	vtblk_dump_flush(sc);
1316 
1317 	VTBLK_UNLOCK(sc);
1318 	vtblk_done_completed(sc, &sc->vtblk_dump_queue);
1319 	VTBLK_LOCK(sc);
1320 }
1321 
1322 static void
1323 vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
1324 {
1325 
1326 	/* Set either writeback (1) or writethrough (0) mode. */
1327 	virtio_write_dev_config_1(sc->vtblk_dev,
1328 	    offsetof(struct virtio_blk_config, writeback), wc);
1329 }
1330 
1331 static int
1332 vtblk_write_cache_enabled(struct vtblk_softc *sc,
1333     struct virtio_blk_config *blkcfg)
1334 {
1335 	int wc;
1336 
1337 	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
1338 		wc = vtblk_tunable_int(sc, "writecache_mode",
1339 		    vtblk_writecache_mode);
1340 		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
1341 			vtblk_set_write_cache(sc, wc);
1342 		else
1343 			wc = blkcfg->writeback;
1344 	} else
1345 		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
1346 
1347 	return (wc);
1348 }
1349 
1350 static int
1351 vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
1352 {
1353 	struct vtblk_softc *sc;
1354 	int wc, error;
1355 
1356 	sc = oidp->oid_arg1;
1357 	wc = sc->vtblk_write_cache;
1358 
1359 	error = sysctl_handle_int(oidp, &wc, 0, req);
1360 	if (error || req->newptr == NULL)
1361 		return (error);
1362 	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
1363 		return (EPERM);
1364 	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
1365 		return (EINVAL);
1366 
1367 	VTBLK_LOCK(sc);
1368 	sc->vtblk_write_cache = wc;
1369 	vtblk_set_write_cache(sc, sc->vtblk_write_cache);
1370 	VTBLK_UNLOCK(sc);
1371 
1372 	return (0);
1373 }
1374 
1375 static void
1376 vtblk_setup_sysctl(struct vtblk_softc *sc)
1377 {
1378 	device_t dev;
1379 	struct sysctl_ctx_list *ctx;
1380 	struct sysctl_oid *tree;
1381 	struct sysctl_oid_list *child;
1382 
1383 	dev = sc->vtblk_dev;
1384 	ctx = device_get_sysctl_ctx(dev);
1385 	tree = device_get_sysctl_tree(dev);
1386 	child = SYSCTL_CHILDREN(tree);
1387 
1388 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode",
1389 	    CTLTYPE_INT | CTLFLAG_RW, sc, 0, vtblk_write_cache_sysctl,
1390 	    "I", "Write cache mode (writethrough (0) or writeback (1))");
1391 }
1392 
1393 static int
1394 vtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def)
1395 {
1396 	char path[64];
1397 
1398 	snprintf(path, sizeof(path),
1399 	    "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob);
1400 	TUNABLE_INT_FETCH(path, &def);
1401 
1402 	return (def);
1403 }
1404