148 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
150 typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun,
239 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
241 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
243 static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
245 static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun,
247 static void ctl_be_block_unmap_file(struct ctl_be_block_lun *be_lun,
249 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
251 static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
253 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
255 static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun,
257 static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
259 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
261 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
267 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
269 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
271 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
272 static int ctl_be_block_open(struct ctl_be_block_lun *be_lun,
464 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_move_done()  local
505 	be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);  in ctl_be_block_move_done()
507 		be_lun->dispatch(be_lun, beio);  in ctl_be_block_move_done()
509 		mtx_lock(&be_lun->queue_lock);  in ctl_be_block_move_done()
510 		STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);  in ctl_be_block_move_done()
511 		mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_move_done()
512 		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);  in ctl_be_block_move_done()
521 	struct ctl_be_block_lun *be_lun = beio->lun;  in ctl_be_block_biodone()  local
522 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_biodone()
531 	mtx_lock(&be_lun->io_lock);  in ctl_be_block_biodone()
553 		mtx_unlock(&be_lun->io_lock);  in ctl_be_block_biodone()
564 	mtx_unlock(&be_lun->io_lock);  in ctl_be_block_biodone()
608 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,  in ctl_be_block_flush_file()  argument
620 	(void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);  in ctl_be_block_flush_file()
622 	vn_lock(be_lun->vn, vn_lktype_write(mountpoint, be_lun->vn) |  in ctl_be_block_flush_file()
624 	error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,  in ctl_be_block_flush_file()
626 	VOP_UNLOCK(be_lun->vn);  in ctl_be_block_flush_file()
630 	mtx_lock(&be_lun->io_lock);  in ctl_be_block_flush_file()
634 	mtx_unlock(&be_lun->io_lock);  in ctl_be_block_flush_file()
652 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,  in ctl_be_block_dispatch_file()  argument
655 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_dispatch_file()
665 	file_data = &be_lun->backend.file;  in ctl_be_block_dispatch_file()
697 		vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);  in ctl_be_block_dispatch_file()
719 		error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);  in ctl_be_block_dispatch_file()
721 		VOP_UNLOCK(be_lun->vn);  in ctl_be_block_dispatch_file()
742 		(void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);  in ctl_be_block_dispatch_file()
743 		vn_lock(be_lun->vn, vn_lktype_write(mountpoint,  in ctl_be_block_dispatch_file()
744 		    be_lun->vn) | LK_RETRY);  in ctl_be_block_dispatch_file()
761 		error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);  in ctl_be_block_dispatch_file()
762 		VOP_UNLOCK(be_lun->vn);  in ctl_be_block_dispatch_file()
768 	mtx_lock(&be_lun->io_lock);  in ctl_be_block_dispatch_file()
772 	mtx_unlock(&be_lun->io_lock);  in ctl_be_block_dispatch_file()
810 ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,  in ctl_be_block_gls_file()  argument
823 	off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;  in ctl_be_block_gls_file()
824 	vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);  in ctl_be_block_gls_file()
825 	error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off,  in ctl_be_block_gls_file()
830 		error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off,  in ctl_be_block_gls_file()
836 			off = be_lun->size_bytes;  in ctl_be_block_gls_file()
839 	VOP_UNLOCK(be_lun->vn);  in ctl_be_block_gls_file()
843 	scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -  in ctl_be_block_gls_file()
851 ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname)  in ctl_be_block_getattr_file()  argument
859 	if (be_lun->vn == NULL)  in ctl_be_block_getattr_file()
861 	vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);  in ctl_be_block_getattr_file()
863 		error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);  in ctl_be_block_getattr_file()
865 			val = vattr.va_bytes / be_lun->cbe_lun.blocksize;  in ctl_be_block_getattr_file()
868 	    !VN_IS_DOOMED(be_lun->vn)) {  in ctl_be_block_getattr_file()
869 		error = VFS_STATFS(be_lun->vn->v_mount, &statfs);  in ctl_be_block_getattr_file()
872 			    be_lun->cbe_lun.blocksize;  in ctl_be_block_getattr_file()
874 	VOP_UNLOCK(be_lun->vn);  in ctl_be_block_getattr_file()
879 ctl_be_block_unmap_file(struct ctl_be_block_lun *be_lun,  in ctl_be_block_unmap_file()  argument
891 	file_data = &be_lun->backend.file;  in ctl_be_block_unmap_file()
896 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);  in ctl_be_block_unmap_file()
898 	(void)vn_start_write(be_lun->vn, &mp, V_WAIT);  in ctl_be_block_unmap_file()
899 	vn_lock(be_lun->vn, vn_lktype_write(mp, be_lun->vn) | LK_RETRY);  in ctl_be_block_unmap_file()
908 			    be_lun->cbe_lun.blocksize;  in ctl_be_block_unmap_file()
910 			    be_lun->cbe_lun.blocksize;  in ctl_be_block_unmap_file()
912 			error = vn_deallocate(be_lun->vn, &off, &len,  in ctl_be_block_unmap_file()
922 		error = vn_deallocate(be_lun->vn, &off, &len, 0,  in ctl_be_block_unmap_file()
925 	VOP_UNLOCK(be_lun->vn);  in ctl_be_block_unmap_file()
928 	mtx_lock(&be_lun->io_lock);  in ctl_be_block_unmap_file()
932 	mtx_unlock(&be_lun->io_lock);  in ctl_be_block_unmap_file()
957 ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,  in ctl_be_block_dispatch_zvol()  argument
960 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_dispatch_zvol()
1000 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_dispatch_zvol()
1018 	mtx_lock(&be_lun->io_lock);  in ctl_be_block_dispatch_zvol()
1022 	mtx_unlock(&be_lun->io_lock);  in ctl_be_block_dispatch_zvol()
1060 ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun,  in ctl_be_block_gls_zvol()  argument
1075 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_gls_zvol()
1078 		off = be_lun->size_bytes;  in ctl_be_block_gls_zvol()
1081 	off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;  in ctl_be_block_gls_zvol()
1093 			off = be_lun->size_bytes;  in ctl_be_block_gls_zvol()
1101 	scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -  in ctl_be_block_gls_zvol()
1109 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,  in ctl_be_block_flush_dev()  argument
1138 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);  in ctl_be_block_flush_dev()
1140 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_flush_dev()
1152 ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,  in ctl_be_block_unmap_dev_range()  argument
1162 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_unmap_dev_range()
1163 	maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize);  in ctl_be_block_unmap_dev_range()
1173 		bio->bio_pblkno     = off / be_lun->cbe_lun.blocksize;  in ctl_be_block_unmap_dev_range()
1178 		mtx_lock(&be_lun->io_lock);  in ctl_be_block_unmap_dev_range()
1182 		mtx_unlock(&be_lun->io_lock);  in ctl_be_block_unmap_dev_range()
1196 ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,  in ctl_be_block_unmap_dev()  argument
1209 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);  in ctl_be_block_unmap_dev()
1218 			    be_lun->cbe_lun.blocksize;  in ctl_be_block_unmap_dev()
1220 			ctl_be_block_unmap_dev_range(be_lun, beio,  in ctl_be_block_unmap_dev()
1221 			    scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize,  in ctl_be_block_unmap_dev()
1225 		ctl_be_block_unmap_dev_range(be_lun, beio,  in ctl_be_block_unmap_dev()
1230 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,  in ctl_be_block_dispatch_dev()  argument
1241 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_dispatch_dev()
1275 			bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize;  in ctl_be_block_dispatch_dev()
1287 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);  in ctl_be_block_dispatch_dev()
1306 ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname)  in ctl_be_block_getattr_dev()  argument
1313 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_getattr_dev()
1330 ctl_be_block_namespace_data(struct ctl_be_block_lun *be_lun,  in ctl_be_block_namespace_data()  argument
1333 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_namespace_data()
1338 	nsdata->nsze = htole64(be_lun->size_blocks);  in ctl_be_block_namespace_data()
1353 ctl_be_block_nvme_ids(struct ctl_be_block_lun *be_lun,  in ctl_be_block_nvme_ids()  argument
1356 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_nvme_ids()
1363 ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_sync()  argument
1366 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_cw_dispatch_sync()
1380 	be_lun->lun_flush(be_lun, beio);  in ctl_be_block_cw_dispatch_sync()
1401 ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_ws()  argument
1404 	struct ctl_be_block_softc *softc = be_lun->softc;  in ctl_be_block_cw_dispatch_ws()
1405 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_cw_dispatch_ws()
1421 	    (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {  in ctl_be_block_cw_dispatch_ws()
1439 		be_lun->unmap(be_lun, beio);  in ctl_be_block_cw_dispatch_ws()
1449 	pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;  in ctl_be_block_cw_dispatch_ws()
1450 	if (be_lun->cbe_lun.pblockoff > 0)  in ctl_be_block_cw_dispatch_ws()
1451 		pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;  in ctl_be_block_cw_dispatch_ws()
1502 	be_lun->dispatch(be_lun, beio);  in ctl_be_block_cw_dispatch_ws()
1506 ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_unmap()  argument
1519 	if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {  in ctl_be_block_cw_dispatch_unmap()
1536 	be_lun->unmap(be_lun, beio);  in ctl_be_block_cw_dispatch_unmap()
1540 ctl_be_block_cw_dispatch_flush(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_flush()  argument
1548 	beio->io_len = be_lun->size_bytes;  in ctl_be_block_cw_dispatch_flush()
1554 	be_lun->lun_flush(be_lun, beio);  in ctl_be_block_cw_dispatch_flush()
1558 ctl_be_block_cw_dispatch_wu(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_wu()  argument
1561 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_cw_dispatch_wu()
1579 	be_lun->unmap(be_lun, beio);  in ctl_be_block_cw_dispatch_wu()
1583 ctl_be_block_cw_dispatch_wz(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_wz()  argument
1586 	struct ctl_be_block_softc *softc = be_lun->softc;  in ctl_be_block_cw_dispatch_wz()
1587 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_cw_dispatch_wz()
1602 	    be_lun->unmap != NULL) {  in ctl_be_block_cw_dispatch_wz()
1608 		be_lun->unmap(be_lun, beio);  in ctl_be_block_cw_dispatch_wz()
1618 	pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;  in ctl_be_block_cw_dispatch_wz()
1619 	if (be_lun->cbe_lun.pblockoff > 0)  in ctl_be_block_cw_dispatch_wz()
1620 		pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;  in ctl_be_block_cw_dispatch_wz()
1660 	be_lun->dispatch(be_lun, beio);  in ctl_be_block_cw_dispatch_wz()
1664 ctl_be_block_cw_dispatch_dsm(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch_dsm()  argument
1667 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_cw_dispatch_dsm()
1678 	if (be_lun->unmap == NULL) {  in ctl_be_block_cw_dispatch_dsm()
1716 	be_lun->unmap(be_lun, beio);  in ctl_be_block_cw_dispatch_dsm()
1730 ctl_be_block_scsi_cr_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_scsi_cr_dispatch()  argument
1738 	softc = be_lun->softc;  in ctl_be_block_scsi_cr_dispatch()
1741 	beio->lun = be_lun;  in ctl_be_block_scsi_cr_dispatch()
1751 		if (be_lun->get_lba_status)  in ctl_be_block_scsi_cr_dispatch()
1752 			be_lun->get_lba_status(be_lun, beio);  in ctl_be_block_scsi_cr_dispatch()
1763 ctl_be_block_nvme_cr_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_nvme_cr_dispatch()  argument
1775 		ctl_be_block_namespace_data(be_lun, io);  in ctl_be_block_nvme_cr_dispatch()
1778 		ctl_be_block_nvme_ids(be_lun, io);  in ctl_be_block_nvme_cr_dispatch()
1786 ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cr_dispatch()  argument
1791 		ctl_be_block_scsi_cr_dispatch(be_lun, io);  in ctl_be_block_cr_dispatch()
1794 		ctl_be_block_nvme_cr_dispatch(be_lun, io);  in ctl_be_block_cr_dispatch()
1812 ctl_be_block_scsi_cw_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_scsi_cw_dispatch()  argument
1839 		ctl_be_block_cw_dispatch_sync(be_lun, io);  in ctl_be_block_scsi_cw_dispatch()
1843 		ctl_be_block_cw_dispatch_ws(be_lun, io);  in ctl_be_block_scsi_cw_dispatch()
1846 		ctl_be_block_cw_dispatch_unmap(be_lun, io);  in ctl_be_block_scsi_cw_dispatch()
1855 ctl_be_block_nvme_cw_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_nvme_cw_dispatch()  argument
1867 		ctl_be_block_cw_dispatch_flush(be_lun, io);  in ctl_be_block_nvme_cw_dispatch()
1870 		ctl_be_block_cw_dispatch_wu(be_lun, io);  in ctl_be_block_nvme_cw_dispatch()
1873 		ctl_be_block_cw_dispatch_wz(be_lun, io);  in ctl_be_block_nvme_cw_dispatch()
1876 		ctl_be_block_cw_dispatch_dsm(be_lun, io);  in ctl_be_block_nvme_cw_dispatch()
1884 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_cw_dispatch()  argument
1890 	softc = be_lun->softc;  in ctl_be_block_cw_dispatch()
1893 	beio->lun = be_lun;  in ctl_be_block_cw_dispatch()
1899 		ctl_be_block_scsi_cw_dispatch(be_lun, io);  in ctl_be_block_cw_dispatch()
1902 		ctl_be_block_nvme_cw_dispatch(be_lun, io);  in ctl_be_block_cw_dispatch()
1917 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_next()  local
1921 	be_lun = beio->lun;  in ctl_be_block_next()
1933 	mtx_lock(&be_lun->queue_lock);  in ctl_be_block_next()
1934 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);  in ctl_be_block_next()
1935 	mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_next()
1936 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);  in ctl_be_block_next()
1940 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,  in ctl_be_block_dispatch()  argument
1943 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_dispatch()
1951 	softc = be_lun->softc;  in ctl_be_block_dispatch()
1964 	beio->lun = be_lun;  in ctl_be_block_dispatch()
2057 		be_lun->dispatch(be_lun, beio);  in ctl_be_block_dispatch()
2067 	struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context;  in ctl_be_block_worker()  local
2068 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_worker()
2079 		mtx_lock(&be_lun->queue_lock);  in ctl_be_block_worker()
2080 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);  in ctl_be_block_worker()
2083 			STAILQ_REMOVE_HEAD(&be_lun->datamove_queue, links);  in ctl_be_block_worker()
2084 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_worker()
2091 			be_lun->dispatch(be_lun, beio);  in ctl_be_block_worker()
2094 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);  in ctl_be_block_worker()
2097 			STAILQ_REMOVE_HEAD(&be_lun->config_write_queue, links);  in ctl_be_block_worker()
2098 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_worker()
2104 			ctl_be_block_cw_dispatch(be_lun, io);  in ctl_be_block_worker()
2107 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue);  in ctl_be_block_worker()
2110 			STAILQ_REMOVE_HEAD(&be_lun->config_read_queue, links);  in ctl_be_block_worker()
2111 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_worker()
2117 			ctl_be_block_cr_dispatch(be_lun, io);  in ctl_be_block_worker()
2120 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);  in ctl_be_block_worker()
2123 			STAILQ_REMOVE_HEAD(&be_lun->input_queue, links);  in ctl_be_block_worker()
2124 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_worker()
2130 			ctl_be_block_dispatch(be_lun, io);  in ctl_be_block_worker()
2138 		mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_worker()
2151 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_submit()  local
2155 	be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);  in ctl_be_block_submit()
2161 	mtx_lock(&be_lun->queue_lock);  in ctl_be_block_submit()
2162 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);  in ctl_be_block_submit()
2163 	mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_submit()
2164 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);  in ctl_be_block_submit()
2211 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)  in ctl_be_block_open_file()  argument
2222 	cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_open_file()
2223 	file_data = &be_lun->backend.file;  in ctl_be_block_open_file()
2224 	params = &be_lun->params;  in ctl_be_block_open_file()
2226 	be_lun->dev_type = CTL_BE_BLOCK_FILE;  in ctl_be_block_open_file()
2227 	be_lun->dispatch = ctl_be_block_dispatch_file;  in ctl_be_block_open_file()
2228 	be_lun->lun_flush = ctl_be_block_flush_file;  in ctl_be_block_open_file()
2229 	be_lun->get_lba_status = ctl_be_block_gls_file;  in ctl_be_block_open_file()
2230 	be_lun->getattr = ctl_be_block_getattr_file;  in ctl_be_block_open_file()
2231 	be_lun->unmap = ctl_be_block_unmap_file;  in ctl_be_block_open_file()
2234 	error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);  in ctl_be_block_open_file()
2238 			 be_lun->dev_path);  in ctl_be_block_open_file()
2242 	error = VOP_PATHCONF(be_lun->vn, _PC_DEALLOC_PRESENT, &pconf);  in ctl_be_block_open_file()
2246 		    be_lun->dev_path);  in ctl_be_block_open_file()
2254 		be_lun->size_bytes = params->lun_size_bytes;  in ctl_be_block_open_file()
2256 		be_lun->size_bytes = vattr.va_size;  in ctl_be_block_open_file()
2270 	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;  in ctl_be_block_open_file()
2271 	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?  in ctl_be_block_open_file()
2272 	    0 : (be_lun->size_blocks - 1);  in ctl_be_block_open_file()
2309 	if (be_lun->size_bytes < cbe_lun->blocksize) {  in ctl_be_block_open_file()
2312 			 "file %s size %ju < block size %u", be_lun->dev_path,  in ctl_be_block_open_file()
2313 			 (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize);  in ctl_be_block_open_file()
2321 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)  in ctl_be_block_open_dev()  argument
2323 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_open_dev()
2331 	params = &be_lun->params;  in ctl_be_block_open_dev()
2333 	be_lun->dev_type = CTL_BE_BLOCK_DEV;  in ctl_be_block_open_dev()
2334 	csw = devvn_refthread(be_lun->vn, &dev, &ref);  in ctl_be_block_open_dev()
2338 		be_lun->dispatch = ctl_be_block_dispatch_zvol;  in ctl_be_block_open_dev()
2339 		be_lun->get_lba_status = ctl_be_block_gls_zvol;  in ctl_be_block_open_dev()
2342 		be_lun->dispatch = ctl_be_block_dispatch_dev;  in ctl_be_block_open_dev()
2343 		be_lun->get_lba_status = NULL;  in ctl_be_block_open_dev()
2351 	be_lun->lun_flush = ctl_be_block_flush_dev;  in ctl_be_block_open_dev()
2352 	be_lun->getattr = ctl_be_block_getattr_dev;  in ctl_be_block_open_dev()
2353 	be_lun->unmap = ctl_be_block_unmap_dev;  in ctl_be_block_open_dev()
2358 			 "no d_ioctl for device %s!", be_lun->dev_path);  in ctl_be_block_open_dev()
2368 			 "on %s!", error, be_lun->dev_path);  in ctl_be_block_open_dev()
2408 			 be_lun->dev_path);  in ctl_be_block_open_dev()
2423 		be_lun->size_bytes = params->lun_size_bytes;  in ctl_be_block_open_dev()
2425 		be_lun->size_bytes = otmp;  in ctl_be_block_open_dev()
2426 	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;  in ctl_be_block_open_dev()
2427 	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?  in ctl_be_block_open_dev()
2428 	    0 : (be_lun->size_blocks - 1);  in ctl_be_block_open_dev()
2474 	if (be_lun->dispatch == ctl_be_block_dispatch_zvol) {  in ctl_be_block_open_dev()
2498 ctl_be_block_close(struct ctl_be_block_lun *be_lun)  in ctl_be_block_close()  argument
2500 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_close()
2503 	if (be_lun->vn) {  in ctl_be_block_close()
2507 		(void)vn_close(be_lun->vn, flags, NOCRED, curthread);  in ctl_be_block_close()
2508 		be_lun->vn = NULL;  in ctl_be_block_close()
2510 		switch (be_lun->dev_type) {  in ctl_be_block_close()
2514 			if (be_lun->backend.file.cred != NULL) {  in ctl_be_block_close()
2515 				crfree(be_lun->backend.file.cred);  in ctl_be_block_close()
2516 				be_lun->backend.file.cred = NULL;  in ctl_be_block_close()
2522 			panic("Unexpected backend type %d", be_lun->dev_type);  in ctl_be_block_close()
2525 		be_lun->dev_type = CTL_BE_BLOCK_NONE;  in ctl_be_block_close()
2531 ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)  in ctl_be_block_open()  argument
2533 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_open()
2552 	free(be_lun->dev_path, M_CTLBLK);  in ctl_be_block_open()
2553 	be_lun->dev_path = strdup(value, M_CTLBLK);  in ctl_be_block_open()
2564 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path);  in ctl_be_block_open()
2577 		if (be_lun->dev_path[0] != '/') {  in ctl_be_block_open()
2581 				be_lun->dev_path);  in ctl_be_block_open()
2582 			free(be_lun->dev_path, M_CTLBLK);  in ctl_be_block_open()
2583 			be_lun->dev_path = dev_name;  in ctl_be_block_open()
2587 		    "error opening %s: %d", be_lun->dev_path, error);  in ctl_be_block_open()
2596 	be_lun->vn = nd.ni_vp;  in ctl_be_block_open()
2599 	if (vn_isdisk_error(be_lun->vn, &error)) {  in ctl_be_block_open()
2600 		error = ctl_be_block_open_dev(be_lun, req);  in ctl_be_block_open()
2601 	} else if (be_lun->vn->v_type == VREG) {  in ctl_be_block_open()
2602 		error = ctl_be_block_open_file(be_lun, req);  in ctl_be_block_open()
2606 			 "%s is not a disk or plain file", be_lun->dev_path);  in ctl_be_block_open()
2608 	VOP_UNLOCK(be_lun->vn);  in ctl_be_block_open()
2611 		ctl_be_block_close(be_lun);  in ctl_be_block_open()
2613 	if (be_lun->dispatch != ctl_be_block_dispatch_dev)  in ctl_be_block_open()
2631 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_create()  local
2643 	be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);  in ctl_be_block_create()
2644 	cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_create()
2645 	be_lun->params = req->reqdata.create;  in ctl_be_block_create()
2646 	be_lun->softc = softc;  in ctl_be_block_create()
2647 	STAILQ_INIT(&be_lun->input_queue);  in ctl_be_block_create()
2648 	STAILQ_INIT(&be_lun->config_read_queue);  in ctl_be_block_create()
2649 	STAILQ_INIT(&be_lun->config_write_queue);  in ctl_be_block_create()
2650 	STAILQ_INIT(&be_lun->datamove_queue);  in ctl_be_block_create()
2651 	mtx_init(&be_lun->io_lock, "ctlblock io", NULL, MTX_DEF);  in ctl_be_block_create()
2652 	mtx_init(&be_lun->queue_lock, "ctlblock queue", NULL, MTX_DEF);  in ctl_be_block_create()
2659 	be_lun->flags = 0;  in ctl_be_block_create()
2670 		be_lun->size_bytes = params->lun_size_bytes;  in ctl_be_block_create()
2677 		be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;  in ctl_be_block_create()
2678 		cbe_lun->maxlba = (be_lun->size_blocks == 0) ?  in ctl_be_block_create()
2679 		    0 : (be_lun->size_blocks - 1);  in ctl_be_block_create()
2683 			retval = ctl_be_block_open(be_lun, req);  in ctl_be_block_create()
2712 	if (be_lun->vn == NULL)  in ctl_be_block_create()
2715 	params->lun_size_bytes = be_lun->size_bytes;  in ctl_be_block_create()
2754 	TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);  in ctl_be_block_create()
2756 	be_lun->io_taskqueue = taskqueue_create("ctlblocktq", M_WAITOK,  in ctl_be_block_create()
2757 	    taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);  in ctl_be_block_create()
2759 	if (be_lun->io_taskqueue == NULL) {  in ctl_be_block_create()
2779 	retval = taskqueue_start_threads_in_proc(&be_lun->io_taskqueue,  in ctl_be_block_create()
2788 	be_lun->num_threads = num_threads;  in ctl_be_block_create()
2790 	retval = ctl_add_lun(&be_lun->cbe_lun);  in ctl_be_block_create()
2799 	be_lun->disk_stats = devstat_new_entry("cbb", cbe_lun->lun_id,  in ctl_be_block_create()
2808 	SLIST_INSERT_HEAD(&softc->lun_list, be_lun, links);  in ctl_be_block_create()
2818 	if (be_lun->io_taskqueue != NULL)  in ctl_be_block_create()
2819 		taskqueue_free(be_lun->io_taskqueue);  in ctl_be_block_create()
2820 	ctl_be_block_close(be_lun);  in ctl_be_block_create()
2821 	if (be_lun->dev_path != NULL)  in ctl_be_block_create()
2822 		free(be_lun->dev_path, M_CTLBLK);  in ctl_be_block_create()
2824 	mtx_destroy(&be_lun->queue_lock);  in ctl_be_block_create()
2825 	mtx_destroy(&be_lun->io_lock);  in ctl_be_block_create()
2826 	free(be_lun, M_CTLBLK);  in ctl_be_block_create()
2835 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_rm()  local
2843 	SLIST_FOREACH(be_lun, &softc->lun_list, links) {  in ctl_be_block_rm()
2844 		if (be_lun->cbe_lun.lun_id == params->lun_id) {  in ctl_be_block_rm()
2845 			SLIST_REMOVE(&softc->lun_list, be_lun,  in ctl_be_block_rm()
2853 	if (be_lun == NULL) {  in ctl_be_block_rm()
2859 	cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_rm()
2861 	if (be_lun->vn != NULL) {  in ctl_be_block_rm()
2864 		taskqueue_drain_all(be_lun->io_taskqueue);  in ctl_be_block_rm()
2865 		ctl_be_block_close(be_lun);  in ctl_be_block_rm()
2869 	be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;  in ctl_be_block_rm()
2878 		be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;  in ctl_be_block_rm()
2884 	while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {  in ctl_be_block_rm()
2885 		retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblockrm", 0);  in ctl_be_block_rm()
2889 	be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;  in ctl_be_block_rm()
2890 	if (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {  in ctl_be_block_rm()
2892 		free(be_lun, M_CTLBLK);  in ctl_be_block_rm()
2910 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_modify()  local
2920 	SLIST_FOREACH(be_lun, &softc->lun_list, links) {  in ctl_be_block_modify()
2921 		if (be_lun->cbe_lun.lun_id == params->lun_id)  in ctl_be_block_modify()
2925 	if (be_lun == NULL) {  in ctl_be_block_modify()
2931 	cbe_lun = &be_lun->cbe_lun;  in ctl_be_block_modify()
2934 		be_lun->params.lun_size_bytes = params->lun_size_bytes;  in ctl_be_block_modify()
2959 	oldsize = be_lun->size_blocks;  in ctl_be_block_modify()
2962 		if (be_lun->vn == NULL)  in ctl_be_block_modify()
2963 			error = ctl_be_block_open(be_lun, req);  in ctl_be_block_modify()
2964 		else if (vn_isdisk_error(be_lun->vn, &error))  in ctl_be_block_modify()
2965 			error = ctl_be_block_open_dev(be_lun, req);  in ctl_be_block_modify()
2966 		else if (be_lun->vn->v_type == VREG) {  in ctl_be_block_modify()
2967 			vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);  in ctl_be_block_modify()
2968 			error = ctl_be_block_open_file(be_lun, req);  in ctl_be_block_modify()
2969 			VOP_UNLOCK(be_lun->vn);  in ctl_be_block_modify()
2973 		    be_lun->vn != NULL) {  in ctl_be_block_modify()
2977 		    be_lun->vn == NULL) {  in ctl_be_block_modify()
2983 		if (be_lun->vn != NULL) {  in ctl_be_block_modify()
2986 			taskqueue_drain_all(be_lun->io_taskqueue);  in ctl_be_block_modify()
2987 			error = ctl_be_block_close(be_lun);  in ctl_be_block_modify()
2991 	if (be_lun->size_blocks != oldsize)  in ctl_be_block_modify()
2995 	params->lun_size_bytes = be_lun->size_bytes;  in ctl_be_block_modify()
3010 	struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)cbe_lun;  in ctl_be_block_lun_shutdown()  local
3011 	struct ctl_be_block_softc *softc = be_lun->softc;  in ctl_be_block_lun_shutdown()
3013 	taskqueue_drain_all(be_lun->io_taskqueue);  in ctl_be_block_lun_shutdown()
3014 	taskqueue_free(be_lun->io_taskqueue);  in ctl_be_block_lun_shutdown()
3015 	if (be_lun->disk_stats != NULL)  in ctl_be_block_lun_shutdown()
3016 		devstat_remove_entry(be_lun->disk_stats);  in ctl_be_block_lun_shutdown()
3017 	nvlist_destroy(be_lun->cbe_lun.options);  in ctl_be_block_lun_shutdown()
3018 	free(be_lun->dev_path, M_CTLBLK);  in ctl_be_block_lun_shutdown()
3019 	mtx_destroy(&be_lun->queue_lock);  in ctl_be_block_lun_shutdown()
3020 	mtx_destroy(&be_lun->io_lock);  in ctl_be_block_lun_shutdown()
3023 	be_lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;  in ctl_be_block_lun_shutdown()
3024 	if (be_lun->flags & CTL_BE_BLOCK_LUN_WAITING)  in ctl_be_block_lun_shutdown()
3025 		wakeup(be_lun);  in ctl_be_block_lun_shutdown()
3027 		free(be_lun, M_CTLBLK);  in ctl_be_block_lun_shutdown()
3034 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_scsi_config_write()  local
3041 	be_lun = (struct ctl_be_block_lun *)cbe_lun;  in ctl_be_block_scsi_config_write()
3058 		mtx_lock(&be_lun->queue_lock);  in ctl_be_block_scsi_config_write()
3059 		STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,  in ctl_be_block_scsi_config_write()
3061 		mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_scsi_config_write()
3062 		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);  in ctl_be_block_scsi_config_write()
3075 			if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) {  in ctl_be_block_scsi_config_write()
3076 				retval = ctl_be_block_open(be_lun, &req);  in ctl_be_block_scsi_config_write()
3093 				if (be_lun->vn != NULL)  in ctl_be_block_scsi_config_write()
3094 					ctl_be_block_close(be_lun);  in ctl_be_block_scsi_config_write()
3119 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_nvme_config_write()  local
3123 	be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);  in ctl_be_block_nvme_config_write()
3132 		mtx_lock(&be_lun->queue_lock);  in ctl_be_block_nvme_config_write()
3133 		STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,  in ctl_be_block_nvme_config_write()
3135 		mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_nvme_config_write()
3136 		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);  in ctl_be_block_nvme_config_write()
3162 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_scsi_config_read()  local
3167 	be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);  in ctl_be_block_scsi_config_read()
3172 			mtx_lock(&be_lun->queue_lock);  in ctl_be_block_scsi_config_read()
3173 			STAILQ_INSERT_TAIL(&be_lun->config_read_queue,  in ctl_be_block_scsi_config_read()
3175 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_scsi_config_read()
3176 			taskqueue_enqueue(be_lun->io_taskqueue,  in ctl_be_block_scsi_config_read()
3177 			    &be_lun->io_task);  in ctl_be_block_scsi_config_read()
3203 	struct ctl_be_block_lun *be_lun;  in ctl_be_block_nvme_config_read()  local
3207 	be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);  in ctl_be_block_nvme_config_read()
3218 			mtx_lock(&be_lun->queue_lock);  in ctl_be_block_nvme_config_read()
3219 			STAILQ_INSERT_TAIL(&be_lun->config_read_queue,  in ctl_be_block_nvme_config_read()
3221 			mtx_unlock(&be_lun->queue_lock);  in ctl_be_block_nvme_config_read()
3222 			taskqueue_enqueue(be_lun->io_taskqueue,  in ctl_be_block_nvme_config_read()
3223 			    &be_lun->io_task);  in ctl_be_block_nvme_config_read()