xref: /freebsd/sys/cam/ctl/ctl_backend_block.c (revision 71cd87c66c01245e6402e295cabedca2f8d48082)
1130f4520SKenneth D. Merry /*-
2130f4520SKenneth D. Merry  * Copyright (c) 2003 Silicon Graphics International Corp.
3130f4520SKenneth D. Merry  * Copyright (c) 2009-2011 Spectra Logic Corporation
481177295SEdward Tomasz Napierala  * Copyright (c) 2012 The FreeBSD Foundation
5648dfc1aSAlexander Motin  * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
6130f4520SKenneth D. Merry  * All rights reserved.
7130f4520SKenneth D. Merry  *
881177295SEdward Tomasz Napierala  * Portions of this software were developed by Edward Tomasz Napierala
981177295SEdward Tomasz Napierala  * under sponsorship from the FreeBSD Foundation.
1081177295SEdward Tomasz Napierala  *
11130f4520SKenneth D. Merry  * Redistribution and use in source and binary forms, with or without
12130f4520SKenneth D. Merry  * modification, are permitted provided that the following conditions
13130f4520SKenneth D. Merry  * are met:
14130f4520SKenneth D. Merry  * 1. Redistributions of source code must retain the above copyright
15130f4520SKenneth D. Merry  *    notice, this list of conditions, and the following disclaimer,
16130f4520SKenneth D. Merry  *    without modification.
17130f4520SKenneth D. Merry  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
18130f4520SKenneth D. Merry  *    substantially similar to the "NO WARRANTY" disclaimer below
19130f4520SKenneth D. Merry  *    ("Disclaimer") and any redistribution must be conditioned upon
20130f4520SKenneth D. Merry  *    including a substantially similar Disclaimer requirement for further
21130f4520SKenneth D. Merry  *    binary redistribution.
22130f4520SKenneth D. Merry  *
23130f4520SKenneth D. Merry  * NO WARRANTY
24130f4520SKenneth D. Merry  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25130f4520SKenneth D. Merry  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26130f4520SKenneth D. Merry  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
27130f4520SKenneth D. Merry  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28130f4520SKenneth D. Merry  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29130f4520SKenneth D. Merry  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30130f4520SKenneth D. Merry  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31130f4520SKenneth D. Merry  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
32130f4520SKenneth D. Merry  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
33130f4520SKenneth D. Merry  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34130f4520SKenneth D. Merry  * POSSIBILITY OF SUCH DAMAGES.
35130f4520SKenneth D. Merry  *
36130f4520SKenneth D. Merry  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
37130f4520SKenneth D. Merry  */
38130f4520SKenneth D. Merry /*
39130f4520SKenneth D. Merry  * CAM Target Layer driver backend for block devices.
40130f4520SKenneth D. Merry  *
41130f4520SKenneth D. Merry  * Author: Ken Merry <ken@FreeBSD.org>
42130f4520SKenneth D. Merry  */
43130f4520SKenneth D. Merry #include <sys/cdefs.h>
44130f4520SKenneth D. Merry __FBSDID("$FreeBSD$");
45130f4520SKenneth D. Merry 
46130f4520SKenneth D. Merry #include <sys/param.h>
47130f4520SKenneth D. Merry #include <sys/systm.h>
48130f4520SKenneth D. Merry #include <sys/kernel.h>
49130f4520SKenneth D. Merry #include <sys/types.h>
50130f4520SKenneth D. Merry #include <sys/kthread.h>
51130f4520SKenneth D. Merry #include <sys/bio.h>
52130f4520SKenneth D. Merry #include <sys/fcntl.h>
53ee7f31c0SAlexander Motin #include <sys/limits.h>
54130f4520SKenneth D. Merry #include <sys/lock.h>
55130f4520SKenneth D. Merry #include <sys/mutex.h>
56130f4520SKenneth D. Merry #include <sys/condvar.h>
57130f4520SKenneth D. Merry #include <sys/malloc.h>
58130f4520SKenneth D. Merry #include <sys/conf.h>
59130f4520SKenneth D. Merry #include <sys/ioccom.h>
60130f4520SKenneth D. Merry #include <sys/queue.h>
61130f4520SKenneth D. Merry #include <sys/sbuf.h>
62130f4520SKenneth D. Merry #include <sys/endian.h>
63130f4520SKenneth D. Merry #include <sys/uio.h>
64130f4520SKenneth D. Merry #include <sys/buf.h>
65130f4520SKenneth D. Merry #include <sys/taskqueue.h>
66130f4520SKenneth D. Merry #include <sys/vnode.h>
67130f4520SKenneth D. Merry #include <sys/namei.h>
68130f4520SKenneth D. Merry #include <sys/mount.h>
69130f4520SKenneth D. Merry #include <sys/disk.h>
70130f4520SKenneth D. Merry #include <sys/fcntl.h>
71130f4520SKenneth D. Merry #include <sys/filedesc.h>
72ef8daf3fSAlexander Motin #include <sys/filio.h>
73130f4520SKenneth D. Merry #include <sys/proc.h>
74130f4520SKenneth D. Merry #include <sys/pcpu.h>
75130f4520SKenneth D. Merry #include <sys/module.h>
76130f4520SKenneth D. Merry #include <sys/sdt.h>
77130f4520SKenneth D. Merry #include <sys/devicestat.h>
78130f4520SKenneth D. Merry #include <sys/sysctl.h>
79130f4520SKenneth D. Merry 
80130f4520SKenneth D. Merry #include <geom/geom.h>
81130f4520SKenneth D. Merry 
82130f4520SKenneth D. Merry #include <cam/cam.h>
83130f4520SKenneth D. Merry #include <cam/scsi/scsi_all.h>
84130f4520SKenneth D. Merry #include <cam/scsi/scsi_da.h>
85130f4520SKenneth D. Merry #include <cam/ctl/ctl_io.h>
86130f4520SKenneth D. Merry #include <cam/ctl/ctl.h>
87130f4520SKenneth D. Merry #include <cam/ctl/ctl_backend.h>
88130f4520SKenneth D. Merry #include <cam/ctl/ctl_ioctl.h>
897ac58230SAlexander Motin #include <cam/ctl/ctl_ha.h>
90130f4520SKenneth D. Merry #include <cam/ctl/ctl_scsi_all.h>
917ac58230SAlexander Motin #include <cam/ctl/ctl_private.h>
92130f4520SKenneth D. Merry #include <cam/ctl/ctl_error.h>
93130f4520SKenneth D. Merry 
94130f4520SKenneth D. Merry /*
9508a7cce5SAlexander Motin  * The idea here is that we'll allocate enough S/G space to hold a 1MB
9608a7cce5SAlexander Motin  * I/O.  If we get an I/O larger than that, we'll split it.
97130f4520SKenneth D. Merry  */
9811b569f7SAlexander Motin #define	CTLBLK_HALF_IO_SIZE	(512 * 1024)
9911b569f7SAlexander Motin #define	CTLBLK_MAX_IO_SIZE	(CTLBLK_HALF_IO_SIZE * 2)
10008a7cce5SAlexander Motin #define	CTLBLK_MAX_SEG		MAXPHYS
10111b569f7SAlexander Motin #define	CTLBLK_HALF_SEGS	MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1)
10211b569f7SAlexander Motin #define	CTLBLK_MAX_SEGS		(CTLBLK_HALF_SEGS * 2)
103130f4520SKenneth D. Merry 
104130f4520SKenneth D. Merry #ifdef CTLBLK_DEBUG
105130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) \
106130f4520SKenneth D. Merry     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
107130f4520SKenneth D. Merry #else
108130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) do {} while(0)
109130f4520SKenneth D. Merry #endif
110130f4520SKenneth D. Merry 
111e86a4142SAlexander Motin #define PRIV(io)	\
112e86a4142SAlexander Motin     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
11311b569f7SAlexander Motin #define ARGS(io)	\
11411b569f7SAlexander Motin     ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
115e86a4142SAlexander Motin 
116130f4520SKenneth D. Merry SDT_PROVIDER_DEFINE(cbb);
117130f4520SKenneth D. Merry 
118130f4520SKenneth D. Merry typedef enum {
119130f4520SKenneth D. Merry 	CTL_BE_BLOCK_LUN_UNCONFIGURED	= 0x01,
120130f4520SKenneth D. Merry 	CTL_BE_BLOCK_LUN_CONFIG_ERR	= 0x02,
121130f4520SKenneth D. Merry 	CTL_BE_BLOCK_LUN_WAITING	= 0x04,
122130f4520SKenneth D. Merry } ctl_be_block_lun_flags;
123130f4520SKenneth D. Merry 
124130f4520SKenneth D. Merry typedef enum {
125130f4520SKenneth D. Merry 	CTL_BE_BLOCK_NONE,
126130f4520SKenneth D. Merry 	CTL_BE_BLOCK_DEV,
127130f4520SKenneth D. Merry 	CTL_BE_BLOCK_FILE
128130f4520SKenneth D. Merry } ctl_be_block_type;
129130f4520SKenneth D. Merry 
130130f4520SKenneth D. Merry struct ctl_be_block_filedata {
131130f4520SKenneth D. Merry 	struct ucred *cred;
132130f4520SKenneth D. Merry };
133130f4520SKenneth D. Merry 
134130f4520SKenneth D. Merry union ctl_be_block_bedata {
135130f4520SKenneth D. Merry 	struct ctl_be_block_filedata file;
136130f4520SKenneth D. Merry };
137130f4520SKenneth D. Merry 
138130f4520SKenneth D. Merry struct ctl_be_block_io;
139130f4520SKenneth D. Merry struct ctl_be_block_lun;
140130f4520SKenneth D. Merry 
141130f4520SKenneth D. Merry typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
142130f4520SKenneth D. Merry 			       struct ctl_be_block_io *beio);
143c3e7ba3eSAlexander Motin typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun,
144c3e7ba3eSAlexander Motin 				  const char *attrname);
145130f4520SKenneth D. Merry 
146130f4520SKenneth D. Merry /*
147130f4520SKenneth D. Merry  * Backend LUN structure.  There is a 1:1 mapping between a block device
148130f4520SKenneth D. Merry  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
149130f4520SKenneth D. Merry  */
150130f4520SKenneth D. Merry struct ctl_be_block_lun {
15119720f41SAlexander Motin 	struct ctl_lun_create_params params;
152130f4520SKenneth D. Merry 	char lunname[32];
153130f4520SKenneth D. Merry 	char *dev_path;
154130f4520SKenneth D. Merry 	ctl_be_block_type dev_type;
155130f4520SKenneth D. Merry 	struct vnode *vn;
156130f4520SKenneth D. Merry 	union ctl_be_block_bedata backend;
157130f4520SKenneth D. Merry 	cbb_dispatch_t dispatch;
158130f4520SKenneth D. Merry 	cbb_dispatch_t lun_flush;
159ee7f31c0SAlexander Motin 	cbb_dispatch_t unmap;
160ef8daf3fSAlexander Motin 	cbb_dispatch_t get_lba_status;
161c3e7ba3eSAlexander Motin 	cbb_getattr_t getattr;
162130f4520SKenneth D. Merry 	uma_zone_t lun_zone;
163130f4520SKenneth D. Merry 	uint64_t size_blocks;
164130f4520SKenneth D. Merry 	uint64_t size_bytes;
165130f4520SKenneth D. Merry 	struct ctl_be_block_softc *softc;
166130f4520SKenneth D. Merry 	struct devstat *disk_stats;
167130f4520SKenneth D. Merry 	ctl_be_block_lun_flags flags;
168130f4520SKenneth D. Merry 	STAILQ_ENTRY(ctl_be_block_lun) links;
1690bcd4ab6SAlexander Motin 	struct ctl_be_lun cbe_lun;
170130f4520SKenneth D. Merry 	struct taskqueue *io_taskqueue;
171130f4520SKenneth D. Merry 	struct task io_task;
172130f4520SKenneth D. Merry 	int num_threads;
173130f4520SKenneth D. Merry 	STAILQ_HEAD(, ctl_io_hdr) input_queue;
174ef8daf3fSAlexander Motin 	STAILQ_HEAD(, ctl_io_hdr) config_read_queue;
175130f4520SKenneth D. Merry 	STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
176130f4520SKenneth D. Merry 	STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
17775c7a1d3SAlexander Motin 	struct mtx_padalign io_lock;
17875c7a1d3SAlexander Motin 	struct mtx_padalign queue_lock;
179130f4520SKenneth D. Merry };
180130f4520SKenneth D. Merry 
181130f4520SKenneth D. Merry /*
182130f4520SKenneth D. Merry  * Overall softc structure for the block backend module.
183130f4520SKenneth D. Merry  */
184130f4520SKenneth D. Merry struct ctl_be_block_softc {
185130f4520SKenneth D. Merry 	struct mtx			 lock;
1860c629e28SAlexander Motin 	uma_zone_t			 beio_zone;
187130f4520SKenneth D. Merry 	int				 num_luns;
188130f4520SKenneth D. Merry 	STAILQ_HEAD(, ctl_be_block_lun)	 lun_list;
189130f4520SKenneth D. Merry };
190130f4520SKenneth D. Merry 
191130f4520SKenneth D. Merry static struct ctl_be_block_softc backend_block_softc;
192130f4520SKenneth D. Merry 
193130f4520SKenneth D. Merry /*
194130f4520SKenneth D. Merry  * Per-I/O information.
195130f4520SKenneth D. Merry  */
196130f4520SKenneth D. Merry struct ctl_be_block_io {
197130f4520SKenneth D. Merry 	union ctl_io			*io;
198130f4520SKenneth D. Merry 	struct ctl_sg_entry		sg_segs[CTLBLK_MAX_SEGS];
199130f4520SKenneth D. Merry 	struct iovec			xiovecs[CTLBLK_MAX_SEGS];
200130f4520SKenneth D. Merry 	int				bio_cmd;
201130f4520SKenneth D. Merry 	int				num_segs;
202130f4520SKenneth D. Merry 	int				num_bios_sent;
203130f4520SKenneth D. Merry 	int				num_bios_done;
204130f4520SKenneth D. Merry 	int				send_complete;
2051f0694a6SAlexander Motin 	int				first_error;
2061f0694a6SAlexander Motin 	uint64_t			first_error_offset;
207130f4520SKenneth D. Merry 	struct bintime			ds_t0;
208130f4520SKenneth D. Merry 	devstat_tag_type		ds_tag_type;
209130f4520SKenneth D. Merry 	devstat_trans_flags		ds_trans_type;
210130f4520SKenneth D. Merry 	uint64_t			io_len;
211130f4520SKenneth D. Merry 	uint64_t			io_offset;
2127d0d4342SAlexander Motin 	int				io_arg;
213130f4520SKenneth D. Merry 	struct ctl_be_block_softc	*softc;
214130f4520SKenneth D. Merry 	struct ctl_be_block_lun		*lun;
215ee7f31c0SAlexander Motin 	void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */
216130f4520SKenneth D. Merry };
217130f4520SKenneth D. Merry 
2187ac58230SAlexander Motin extern struct ctl_softc *control_softc;
2197ac58230SAlexander Motin 
220130f4520SKenneth D. Merry static int cbb_num_threads = 14;
221130f4520SKenneth D. Merry SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
222130f4520SKenneth D. Merry 	    "CAM Target Layer Block Backend");
223af3b2549SHans Petter Selasky SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
224130f4520SKenneth D. Merry            &cbb_num_threads, 0, "Number of threads per backing file");
225130f4520SKenneth D. Merry 
226130f4520SKenneth D. Merry static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
227130f4520SKenneth D. Merry static void ctl_free_beio(struct ctl_be_block_io *beio);
228130f4520SKenneth D. Merry static void ctl_complete_beio(struct ctl_be_block_io *beio);
229130f4520SKenneth D. Merry static int ctl_be_block_move_done(union ctl_io *io);
230130f4520SKenneth D. Merry static void ctl_be_block_biodone(struct bio *bio);
231130f4520SKenneth D. Merry static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
232130f4520SKenneth D. Merry 				    struct ctl_be_block_io *beio);
233130f4520SKenneth D. Merry static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
234130f4520SKenneth D. Merry 				       struct ctl_be_block_io *beio);
235ef8daf3fSAlexander Motin static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
236ef8daf3fSAlexander Motin 				  struct ctl_be_block_io *beio);
23753c146deSAlexander Motin static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun,
23853c146deSAlexander Motin 					 const char *attrname);
239130f4520SKenneth D. Merry static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
240130f4520SKenneth D. Merry 				   struct ctl_be_block_io *beio);
241ee7f31c0SAlexander Motin static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
242ee7f31c0SAlexander Motin 				   struct ctl_be_block_io *beio);
243130f4520SKenneth D. Merry static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
244130f4520SKenneth D. Merry 				      struct ctl_be_block_io *beio);
245c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun,
246c3e7ba3eSAlexander Motin 					 const char *attrname);
247ef8daf3fSAlexander Motin static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
248ef8daf3fSAlexander Motin 				    union ctl_io *io);
249130f4520SKenneth D. Merry static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
250130f4520SKenneth D. Merry 				    union ctl_io *io);
251130f4520SKenneth D. Merry static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
252130f4520SKenneth D. Merry 				  union ctl_io *io);
253130f4520SKenneth D. Merry static void ctl_be_block_worker(void *context, int pending);
254130f4520SKenneth D. Merry static int ctl_be_block_submit(union ctl_io *io);
255130f4520SKenneth D. Merry static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
256130f4520SKenneth D. Merry 				   int flag, struct thread *td);
257130f4520SKenneth D. Merry static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
258130f4520SKenneth D. Merry 				  struct ctl_lun_req *req);
259130f4520SKenneth D. Merry static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
260130f4520SKenneth D. Merry 				 struct ctl_lun_req *req);
261130f4520SKenneth D. Merry static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
262648dfc1aSAlexander Motin static int ctl_be_block_open(struct ctl_be_block_lun *be_lun,
263130f4520SKenneth D. Merry 			     struct ctl_lun_req *req);
264130f4520SKenneth D. Merry static int ctl_be_block_create(struct ctl_be_block_softc *softc,
265130f4520SKenneth D. Merry 			       struct ctl_lun_req *req);
266130f4520SKenneth D. Merry static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
267130f4520SKenneth D. Merry 			   struct ctl_lun_req *req);
26881177295SEdward Tomasz Napierala static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
26981177295SEdward Tomasz Napierala 			   struct ctl_lun_req *req);
270130f4520SKenneth D. Merry static void ctl_be_block_lun_shutdown(void *be_lun);
271130f4520SKenneth D. Merry static void ctl_be_block_lun_config_status(void *be_lun,
272130f4520SKenneth D. Merry 					   ctl_lun_config_status status);
273130f4520SKenneth D. Merry static int ctl_be_block_config_write(union ctl_io *io);
274130f4520SKenneth D. Merry static int ctl_be_block_config_read(union ctl_io *io);
275130f4520SKenneth D. Merry static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
276c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname);
2770c629e28SAlexander Motin static int ctl_be_block_init(void);
2780c629e28SAlexander Motin static int ctl_be_block_shutdown(void);
279130f4520SKenneth D. Merry 
280130f4520SKenneth D. Merry static struct ctl_backend_driver ctl_be_block_driver =
281130f4520SKenneth D. Merry {
2822a2443d8SKenneth D. Merry 	.name = "block",
2832a2443d8SKenneth D. Merry 	.flags = CTL_BE_FLAG_HAS_CONFIG,
2842a2443d8SKenneth D. Merry 	.init = ctl_be_block_init,
2850c629e28SAlexander Motin 	.shutdown = ctl_be_block_shutdown,
2862a2443d8SKenneth D. Merry 	.data_submit = ctl_be_block_submit,
2872a2443d8SKenneth D. Merry 	.data_move_done = ctl_be_block_move_done,
2882a2443d8SKenneth D. Merry 	.config_read = ctl_be_block_config_read,
2892a2443d8SKenneth D. Merry 	.config_write = ctl_be_block_config_write,
2902a2443d8SKenneth D. Merry 	.ioctl = ctl_be_block_ioctl,
291c3e7ba3eSAlexander Motin 	.lun_info = ctl_be_block_lun_info,
292c3e7ba3eSAlexander Motin 	.lun_attr = ctl_be_block_lun_attr
293130f4520SKenneth D. Merry };
294130f4520SKenneth D. Merry 
295130f4520SKenneth D. Merry MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
296130f4520SKenneth D. Merry CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
297130f4520SKenneth D. Merry 
298130f4520SKenneth D. Merry static struct ctl_be_block_io *
299130f4520SKenneth D. Merry ctl_alloc_beio(struct ctl_be_block_softc *softc)
300130f4520SKenneth D. Merry {
301130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
302130f4520SKenneth D. Merry 
3030c629e28SAlexander Motin 	beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO);
304130f4520SKenneth D. Merry 	beio->softc = softc;
305130f4520SKenneth D. Merry 	return (beio);
306130f4520SKenneth D. Merry }
307130f4520SKenneth D. Merry 
308130f4520SKenneth D. Merry static void
309130f4520SKenneth D. Merry ctl_free_beio(struct ctl_be_block_io *beio)
310130f4520SKenneth D. Merry {
311130f4520SKenneth D. Merry 	int duplicate_free;
312130f4520SKenneth D. Merry 	int i;
313130f4520SKenneth D. Merry 
314130f4520SKenneth D. Merry 	duplicate_free = 0;
315130f4520SKenneth D. Merry 
316130f4520SKenneth D. Merry 	for (i = 0; i < beio->num_segs; i++) {
317130f4520SKenneth D. Merry 		if (beio->sg_segs[i].addr == NULL)
318130f4520SKenneth D. Merry 			duplicate_free++;
319130f4520SKenneth D. Merry 
320130f4520SKenneth D. Merry 		uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
321130f4520SKenneth D. Merry 		beio->sg_segs[i].addr = NULL;
32211b569f7SAlexander Motin 
32311b569f7SAlexander Motin 		/* For compare we had two equal S/G lists. */
32411b569f7SAlexander Motin 		if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) {
32511b569f7SAlexander Motin 			uma_zfree(beio->lun->lun_zone,
32611b569f7SAlexander Motin 			    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr);
32711b569f7SAlexander Motin 			beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL;
32811b569f7SAlexander Motin 		}
329130f4520SKenneth D. Merry 	}
330130f4520SKenneth D. Merry 
331130f4520SKenneth D. Merry 	if (duplicate_free > 0) {
332130f4520SKenneth D. Merry 		printf("%s: %d duplicate frees out of %d segments\n", __func__,
333130f4520SKenneth D. Merry 		       duplicate_free, beio->num_segs);
334130f4520SKenneth D. Merry 	}
335a0e36aeeSEdward Tomasz Napierala 
3360c629e28SAlexander Motin 	uma_zfree(beio->softc->beio_zone, beio);
337130f4520SKenneth D. Merry }
338130f4520SKenneth D. Merry 
339130f4520SKenneth D. Merry static void
340130f4520SKenneth D. Merry ctl_complete_beio(struct ctl_be_block_io *beio)
341130f4520SKenneth D. Merry {
34275c7a1d3SAlexander Motin 	union ctl_io *io = beio->io;
343130f4520SKenneth D. Merry 
344ee7f31c0SAlexander Motin 	if (beio->beio_cont != NULL) {
345ee7f31c0SAlexander Motin 		beio->beio_cont(beio);
346ee7f31c0SAlexander Motin 	} else {
347130f4520SKenneth D. Merry 		ctl_free_beio(beio);
34811b569f7SAlexander Motin 		ctl_data_submit_done(io);
349130f4520SKenneth D. Merry 	}
350ee7f31c0SAlexander Motin }
351130f4520SKenneth D. Merry 
352d6043e46SAlexander Motin static size_t
353d6043e46SAlexander Motin cmp(uint8_t *a, uint8_t *b, size_t size)
354d6043e46SAlexander Motin {
355d6043e46SAlexander Motin 	size_t i;
356d6043e46SAlexander Motin 
357d6043e46SAlexander Motin 	for (i = 0; i < size; i++) {
358d6043e46SAlexander Motin 		if (a[i] != b[i])
359d6043e46SAlexander Motin 			break;
360d6043e46SAlexander Motin 	}
361d6043e46SAlexander Motin 	return (i);
362d6043e46SAlexander Motin }
363d6043e46SAlexander Motin 
364d6043e46SAlexander Motin static void
365d6043e46SAlexander Motin ctl_be_block_compare(union ctl_io *io)
366d6043e46SAlexander Motin {
367d6043e46SAlexander Motin 	struct ctl_be_block_io *beio;
368d6043e46SAlexander Motin 	uint64_t off, res;
369d6043e46SAlexander Motin 	int i;
370d6043e46SAlexander Motin 	uint8_t info[8];
371d6043e46SAlexander Motin 
372d6043e46SAlexander Motin 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
373d6043e46SAlexander Motin 	off = 0;
374d6043e46SAlexander Motin 	for (i = 0; i < beio->num_segs; i++) {
375d6043e46SAlexander Motin 		res = cmp(beio->sg_segs[i].addr,
376d6043e46SAlexander Motin 		    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
377d6043e46SAlexander Motin 		    beio->sg_segs[i].len);
378d6043e46SAlexander Motin 		off += res;
379d6043e46SAlexander Motin 		if (res < beio->sg_segs[i].len)
380d6043e46SAlexander Motin 			break;
381d6043e46SAlexander Motin 	}
382d6043e46SAlexander Motin 	if (i < beio->num_segs) {
383d6043e46SAlexander Motin 		scsi_u64to8b(off, info);
384d6043e46SAlexander Motin 		ctl_set_sense(&io->scsiio, /*current_error*/ 1,
385d6043e46SAlexander Motin 		    /*sense_key*/ SSD_KEY_MISCOMPARE,
386d6043e46SAlexander Motin 		    /*asc*/ 0x1D, /*ascq*/ 0x00,
387d6043e46SAlexander Motin 		    /*type*/ SSD_ELEM_INFO,
388d6043e46SAlexander Motin 		    /*size*/ sizeof(info), /*data*/ &info,
389d6043e46SAlexander Motin 		    /*type*/ SSD_ELEM_NONE);
390d6043e46SAlexander Motin 	} else
391d6043e46SAlexander Motin 		ctl_set_success(&io->scsiio);
392d6043e46SAlexander Motin }
393d6043e46SAlexander Motin 
394130f4520SKenneth D. Merry static int
395130f4520SKenneth D. Merry ctl_be_block_move_done(union ctl_io *io)
396130f4520SKenneth D. Merry {
397130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
398130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
39911b569f7SAlexander Motin 	struct ctl_lba_len_flags *lbalen;
400130f4520SKenneth D. Merry #ifdef CTL_TIME_IO
401130f4520SKenneth D. Merry 	struct bintime cur_bt;
402130f4520SKenneth D. Merry #endif
403130f4520SKenneth D. Merry 
404e86a4142SAlexander Motin 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
405130f4520SKenneth D. Merry 	be_lun = beio->lun;
406130f4520SKenneth D. Merry 
407130f4520SKenneth D. Merry 	DPRINTF("entered\n");
408130f4520SKenneth D. Merry 
409130f4520SKenneth D. Merry #ifdef CTL_TIME_IO
410e675024aSAlexander Motin 	getbinuptime(&cur_bt);
411130f4520SKenneth D. Merry 	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
412130f4520SKenneth D. Merry 	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
413130f4520SKenneth D. Merry #endif
414e675024aSAlexander Motin 	io->io_hdr.num_dmas++;
41511b569f7SAlexander Motin 	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
416130f4520SKenneth D. Merry 
417130f4520SKenneth D. Merry 	/*
418130f4520SKenneth D. Merry 	 * We set status at this point for read commands, and write
419130f4520SKenneth D. Merry 	 * commands with errors.
420130f4520SKenneth D. Merry 	 */
421f7241cceSAlexander Motin 	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
422f7241cceSAlexander Motin 		;
423eb6ac6f9SAlexander Motin 	} else if ((io->io_hdr.port_status != 0) &&
424eb6ac6f9SAlexander Motin 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
425eb6ac6f9SAlexander Motin 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
426eb6ac6f9SAlexander Motin 		ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
427eb6ac6f9SAlexander Motin 		    /*retry_count*/ io->io_hdr.port_status);
428eb6ac6f9SAlexander Motin 	} else if (io->scsiio.kern_data_resid != 0 &&
429eb6ac6f9SAlexander Motin 	    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
430eb6ac6f9SAlexander Motin 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
431eb6ac6f9SAlexander Motin 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
432eb6ac6f9SAlexander Motin 		ctl_set_invalid_field_ciu(&io->scsiio);
433f7241cceSAlexander Motin 	} else if ((io->io_hdr.port_status == 0) &&
43411b569f7SAlexander Motin 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
43511b569f7SAlexander Motin 		lbalen = ARGS(beio->io);
43611b569f7SAlexander Motin 		if (lbalen->flags & CTL_LLF_READ) {
437130f4520SKenneth D. Merry 			ctl_set_success(&io->scsiio);
43811b569f7SAlexander Motin 		} else if (lbalen->flags & CTL_LLF_COMPARE) {
43911b569f7SAlexander Motin 			/* We have two data blocks ready for comparison. */
440d6043e46SAlexander Motin 			ctl_be_block_compare(io);
44111b569f7SAlexander Motin 		}
442130f4520SKenneth D. Merry 	}
443130f4520SKenneth D. Merry 
444130f4520SKenneth D. Merry 	/*
445130f4520SKenneth D. Merry 	 * If this is a read, or a write with errors, it is done.
446130f4520SKenneth D. Merry 	 */
447130f4520SKenneth D. Merry 	if ((beio->bio_cmd == BIO_READ)
448130f4520SKenneth D. Merry 	 || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
449130f4520SKenneth D. Merry 	 || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
450130f4520SKenneth D. Merry 		ctl_complete_beio(beio);
451130f4520SKenneth D. Merry 		return (0);
452130f4520SKenneth D. Merry 	}
453130f4520SKenneth D. Merry 
454130f4520SKenneth D. Merry 	/*
455130f4520SKenneth D. Merry 	 * At this point, we have a write and the DMA completed
456130f4520SKenneth D. Merry 	 * successfully.  We now have to queue it to the task queue to
457130f4520SKenneth D. Merry 	 * execute the backend I/O.  That is because we do blocking
458130f4520SKenneth D. Merry 	 * memory allocations, and in the file backing case, blocking I/O.
459130f4520SKenneth D. Merry 	 * This move done routine is generally called in the SIM's
460130f4520SKenneth D. Merry 	 * interrupt context, and therefore we cannot block.
461130f4520SKenneth D. Merry 	 */
46275c7a1d3SAlexander Motin 	mtx_lock(&be_lun->queue_lock);
463130f4520SKenneth D. Merry 	STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
46475c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->queue_lock);
465130f4520SKenneth D. Merry 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
466130f4520SKenneth D. Merry 
467130f4520SKenneth D. Merry 	return (0);
468130f4520SKenneth D. Merry }
469130f4520SKenneth D. Merry 
470130f4520SKenneth D. Merry static void
471130f4520SKenneth D. Merry ctl_be_block_biodone(struct bio *bio)
472130f4520SKenneth D. Merry {
473130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
474130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
475130f4520SKenneth D. Merry 	union ctl_io *io;
476e0c2f975SAlexander Motin 	int error;
477130f4520SKenneth D. Merry 
478130f4520SKenneth D. Merry 	beio = bio->bio_caller1;
479130f4520SKenneth D. Merry 	be_lun = beio->lun;
480130f4520SKenneth D. Merry 	io = beio->io;
481130f4520SKenneth D. Merry 
482130f4520SKenneth D. Merry 	DPRINTF("entered\n");
483130f4520SKenneth D. Merry 
484e0c2f975SAlexander Motin 	error = bio->bio_error;
48575c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
4861f0694a6SAlexander Motin 	if (error != 0 &&
4871f0694a6SAlexander Motin 	    (beio->first_error == 0 ||
4881f0694a6SAlexander Motin 	     bio->bio_offset < beio->first_error_offset)) {
4891f0694a6SAlexander Motin 		beio->first_error = error;
4901f0694a6SAlexander Motin 		beio->first_error_offset = bio->bio_offset;
4911f0694a6SAlexander Motin 	}
492130f4520SKenneth D. Merry 
493130f4520SKenneth D. Merry 	beio->num_bios_done++;
494130f4520SKenneth D. Merry 
495130f4520SKenneth D. Merry 	/*
496130f4520SKenneth D. Merry 	 * XXX KDM will this cause WITNESS to complain?  Holding a lock
497130f4520SKenneth D. Merry 	 * during the free might cause it to complain.
498130f4520SKenneth D. Merry 	 */
499130f4520SKenneth D. Merry 	g_destroy_bio(bio);
500130f4520SKenneth D. Merry 
501130f4520SKenneth D. Merry 	/*
502130f4520SKenneth D. Merry 	 * If the send complete bit isn't set, or we aren't the last I/O to
503130f4520SKenneth D. Merry 	 * complete, then we're done.
504130f4520SKenneth D. Merry 	 */
505130f4520SKenneth D. Merry 	if ((beio->send_complete == 0)
506130f4520SKenneth D. Merry 	 || (beio->num_bios_done < beio->num_bios_sent)) {
50775c7a1d3SAlexander Motin 		mtx_unlock(&be_lun->io_lock);
508130f4520SKenneth D. Merry 		return;
509130f4520SKenneth D. Merry 	}
510130f4520SKenneth D. Merry 
511130f4520SKenneth D. Merry 	/*
512130f4520SKenneth D. Merry 	 * At this point, we've verified that we are the last I/O to
513130f4520SKenneth D. Merry 	 * complete, so it's safe to drop the lock.
514130f4520SKenneth D. Merry 	 */
51575c7a1d3SAlexander Motin 	devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
51675c7a1d3SAlexander Motin 	    beio->ds_tag_type, beio->ds_trans_type,
51775c7a1d3SAlexander Motin 	    /*now*/ NULL, /*then*/&beio->ds_t0);
51875c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
519130f4520SKenneth D. Merry 
520130f4520SKenneth D. Merry 	/*
521130f4520SKenneth D. Merry 	 * If there are any errors from the backing device, we fail the
522130f4520SKenneth D. Merry 	 * entire I/O with a medium error.
523130f4520SKenneth D. Merry 	 */
5241f0694a6SAlexander Motin 	error = beio->first_error;
5251f0694a6SAlexander Motin 	if (error != 0) {
526e0c2f975SAlexander Motin 		if (error == EOPNOTSUPP) {
527e0c2f975SAlexander Motin 			ctl_set_invalid_opcode(&io->scsiio);
5280631de4aSAlexander Motin 		} else if (error == ENOSPC || error == EDQUOT) {
5294fc18ff9SAlexander Motin 			ctl_set_space_alloc_fail(&io->scsiio);
5306187d472SAlexander Motin 		} else if (error == EROFS || error == EACCES) {
5316187d472SAlexander Motin 			ctl_set_hw_write_protected(&io->scsiio);
532e0c2f975SAlexander Motin 		} else if (beio->bio_cmd == BIO_FLUSH) {
533130f4520SKenneth D. Merry 			/* XXX KDM is there is a better error here? */
534130f4520SKenneth D. Merry 			ctl_set_internal_failure(&io->scsiio,
535130f4520SKenneth D. Merry 						 /*sks_valid*/ 1,
536130f4520SKenneth D. Merry 						 /*retry_count*/ 0xbad2);
5377f7bb97aSAlexander Motin 		} else {
5387f7bb97aSAlexander Motin 			ctl_set_medium_error(&io->scsiio,
5397f7bb97aSAlexander Motin 			    beio->bio_cmd == BIO_READ);
5407f7bb97aSAlexander Motin 		}
541130f4520SKenneth D. Merry 		ctl_complete_beio(beio);
542130f4520SKenneth D. Merry 		return;
543130f4520SKenneth D. Merry 	}
544130f4520SKenneth D. Merry 
545130f4520SKenneth D. Merry 	/*
54611b569f7SAlexander Motin 	 * If this is a write, a flush, a delete or verify, we're all done.
547130f4520SKenneth D. Merry 	 * If this is a read, we can now send the data to the user.
548130f4520SKenneth D. Merry 	 */
549130f4520SKenneth D. Merry 	if ((beio->bio_cmd == BIO_WRITE)
550ee7f31c0SAlexander Motin 	 || (beio->bio_cmd == BIO_FLUSH)
55111b569f7SAlexander Motin 	 || (beio->bio_cmd == BIO_DELETE)
55211b569f7SAlexander Motin 	 || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
553130f4520SKenneth D. Merry 		ctl_set_success(&io->scsiio);
554130f4520SKenneth D. Merry 		ctl_complete_beio(beio);
555130f4520SKenneth D. Merry 	} else {
556f7241cceSAlexander Motin 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
55775a3108eSAlexander Motin 		    beio->beio_cont == NULL) {
558f7241cceSAlexander Motin 			ctl_set_success(&io->scsiio);
55975a3108eSAlexander Motin 			ctl_serseq_done(io);
56075a3108eSAlexander Motin 		}
561130f4520SKenneth D. Merry #ifdef CTL_TIME_IO
562e675024aSAlexander Motin 		getbinuptime(&io->io_hdr.dma_start_bt);
563130f4520SKenneth D. Merry #endif
564130f4520SKenneth D. Merry 		ctl_datamove(io);
565130f4520SKenneth D. Merry 	}
566130f4520SKenneth D. Merry }
567130f4520SKenneth D. Merry 
568130f4520SKenneth D. Merry static void
569130f4520SKenneth D. Merry ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
570130f4520SKenneth D. Merry 			struct ctl_be_block_io *beio)
571130f4520SKenneth D. Merry {
57275c7a1d3SAlexander Motin 	union ctl_io *io = beio->io;
573130f4520SKenneth D. Merry 	struct mount *mountpoint;
5745050aa86SKonstantin Belousov 	int error, lock_flags;
575130f4520SKenneth D. Merry 
576130f4520SKenneth D. Merry 	DPRINTF("entered\n");
577130f4520SKenneth D. Merry 
57875c7a1d3SAlexander Motin 	binuptime(&beio->ds_t0);
57975c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
58075c7a1d3SAlexander Motin 	devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
58175c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
582130f4520SKenneth D. Merry 
583130f4520SKenneth D. Merry 	(void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
584130f4520SKenneth D. Merry 
58567cc546dSAlexander Motin 	if (MNT_SHARED_WRITES(mountpoint) ||
58667cc546dSAlexander Motin 	    ((mountpoint == NULL) && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
587130f4520SKenneth D. Merry 		lock_flags = LK_SHARED;
588130f4520SKenneth D. Merry 	else
589130f4520SKenneth D. Merry 		lock_flags = LK_EXCLUSIVE;
590130f4520SKenneth D. Merry 	vn_lock(be_lun->vn, lock_flags | LK_RETRY);
5917d0d4342SAlexander Motin 	error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,
5927d0d4342SAlexander Motin 	    curthread);
593130f4520SKenneth D. Merry 	VOP_UNLOCK(be_lun->vn, 0);
594130f4520SKenneth D. Merry 
595130f4520SKenneth D. Merry 	vn_finished_write(mountpoint);
596130f4520SKenneth D. Merry 
59775c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
59875c7a1d3SAlexander Motin 	devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
59975c7a1d3SAlexander Motin 	    beio->ds_tag_type, beio->ds_trans_type,
60075c7a1d3SAlexander Motin 	    /*now*/ NULL, /*then*/&beio->ds_t0);
60175c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
60275c7a1d3SAlexander Motin 
603130f4520SKenneth D. Merry 	if (error == 0)
604130f4520SKenneth D. Merry 		ctl_set_success(&io->scsiio);
605130f4520SKenneth D. Merry 	else {
606130f4520SKenneth D. Merry 		/* XXX KDM is there is a better error here? */
607130f4520SKenneth D. Merry 		ctl_set_internal_failure(&io->scsiio,
608130f4520SKenneth D. Merry 					 /*sks_valid*/ 1,
609130f4520SKenneth D. Merry 					 /*retry_count*/ 0xbad1);
610130f4520SKenneth D. Merry 	}
611130f4520SKenneth D. Merry 
612130f4520SKenneth D. Merry 	ctl_complete_beio(beio);
613130f4520SKenneth D. Merry }
614130f4520SKenneth D. Merry 
61536160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_start, "uint64_t");
61636160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_start, "uint64_t");
61736160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_done,"uint64_t");
61836160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_done, "uint64_t");
619130f4520SKenneth D. Merry 
620130f4520SKenneth D. Merry static void
621130f4520SKenneth D. Merry ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
622130f4520SKenneth D. Merry 			   struct ctl_be_block_io *beio)
623130f4520SKenneth D. Merry {
624130f4520SKenneth D. Merry 	struct ctl_be_block_filedata *file_data;
625130f4520SKenneth D. Merry 	union ctl_io *io;
626130f4520SKenneth D. Merry 	struct uio xuio;
627130f4520SKenneth D. Merry 	struct iovec *xiovec;
62883981e31SAlexander Motin 	size_t s;
62983981e31SAlexander Motin 	int error, flags, i;
630130f4520SKenneth D. Merry 
631130f4520SKenneth D. Merry 	DPRINTF("entered\n");
632130f4520SKenneth D. Merry 
633130f4520SKenneth D. Merry 	file_data = &be_lun->backend.file;
634130f4520SKenneth D. Merry 	io = beio->io;
63555551d05SAlexander Motin 	flags = 0;
63655551d05SAlexander Motin 	if (ARGS(io)->flags & CTL_LLF_DPO)
63755551d05SAlexander Motin 		flags |= IO_DIRECT;
63855551d05SAlexander Motin 	if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
63955551d05SAlexander Motin 		flags |= IO_SYNC;
640130f4520SKenneth D. Merry 
64111b569f7SAlexander Motin 	bzero(&xuio, sizeof(xuio));
642130f4520SKenneth D. Merry 	if (beio->bio_cmd == BIO_READ) {
64336160958SMark Johnston 		SDT_PROBE0(cbb, , read, file_start);
64411b569f7SAlexander Motin 		xuio.uio_rw = UIO_READ;
645130f4520SKenneth D. Merry 	} else {
64636160958SMark Johnston 		SDT_PROBE0(cbb, , write, file_start);
647130f4520SKenneth D. Merry 		xuio.uio_rw = UIO_WRITE;
64811b569f7SAlexander Motin 	}
649130f4520SKenneth D. Merry 	xuio.uio_offset = beio->io_offset;
650130f4520SKenneth D. Merry 	xuio.uio_resid = beio->io_len;
651130f4520SKenneth D. Merry 	xuio.uio_segflg = UIO_SYSSPACE;
652130f4520SKenneth D. Merry 	xuio.uio_iov = beio->xiovecs;
653130f4520SKenneth D. Merry 	xuio.uio_iovcnt = beio->num_segs;
654130f4520SKenneth D. Merry 	xuio.uio_td = curthread;
655130f4520SKenneth D. Merry 
656130f4520SKenneth D. Merry 	for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
657130f4520SKenneth D. Merry 		xiovec->iov_base = beio->sg_segs[i].addr;
658130f4520SKenneth D. Merry 		xiovec->iov_len = beio->sg_segs[i].len;
659130f4520SKenneth D. Merry 	}
660130f4520SKenneth D. Merry 
66175c7a1d3SAlexander Motin 	binuptime(&beio->ds_t0);
66275c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
66375c7a1d3SAlexander Motin 	devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
66475c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
66575c7a1d3SAlexander Motin 
666130f4520SKenneth D. Merry 	if (beio->bio_cmd == BIO_READ) {
667130f4520SKenneth D. Merry 		vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
668130f4520SKenneth D. Merry 
669130f4520SKenneth D. Merry 		/*
670130f4520SKenneth D. Merry 		 * UFS pays attention to IO_DIRECT for reads.  If the
671130f4520SKenneth D. Merry 		 * DIRECTIO option is configured into the kernel, it calls
672130f4520SKenneth D. Merry 		 * ffs_rawread().  But that only works for single-segment
673130f4520SKenneth D. Merry 		 * uios with user space addresses.  In our case, with a
674130f4520SKenneth D. Merry 		 * kernel uio, it still reads into the buffer cache, but it
675130f4520SKenneth D. Merry 		 * will just try to release the buffer from the cache later
676130f4520SKenneth D. Merry 		 * on in ffs_read().
677130f4520SKenneth D. Merry 		 *
678130f4520SKenneth D. Merry 		 * ZFS does not pay attention to IO_DIRECT for reads.
679130f4520SKenneth D. Merry 		 *
680130f4520SKenneth D. Merry 		 * UFS does not pay attention to IO_SYNC for reads.
681130f4520SKenneth D. Merry 		 *
682130f4520SKenneth D. Merry 		 * ZFS pays attention to IO_SYNC (which translates into the
683130f4520SKenneth D. Merry 		 * Solaris define FRSYNC for zfs_read()) for reads.  It
684130f4520SKenneth D. Merry 		 * attempts to sync the file before reading.
685130f4520SKenneth D. Merry 		 */
68655551d05SAlexander Motin 		error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);
687130f4520SKenneth D. Merry 
688130f4520SKenneth D. Merry 		VOP_UNLOCK(be_lun->vn, 0);
68936160958SMark Johnston 		SDT_PROBE0(cbb, , read, file_done);
69083981e31SAlexander Motin 		if (error == 0 && xuio.uio_resid > 0) {
69183981e31SAlexander Motin 			/*
69283981e31SAlexander Motin 			 * If we red less then requested (EOF), then
69383981e31SAlexander Motin 			 * we should clean the rest of the buffer.
69483981e31SAlexander Motin 			 */
69583981e31SAlexander Motin 			s = beio->io_len - xuio.uio_resid;
69683981e31SAlexander Motin 			for (i = 0; i < beio->num_segs; i++) {
69783981e31SAlexander Motin 				if (s >= beio->sg_segs[i].len) {
69883981e31SAlexander Motin 					s -= beio->sg_segs[i].len;
69983981e31SAlexander Motin 					continue;
70083981e31SAlexander Motin 				}
70183981e31SAlexander Motin 				bzero((uint8_t *)beio->sg_segs[i].addr + s,
70283981e31SAlexander Motin 				    beio->sg_segs[i].len - s);
70383981e31SAlexander Motin 				s = 0;
70483981e31SAlexander Motin 			}
70583981e31SAlexander Motin 		}
706130f4520SKenneth D. Merry 	} else {
707130f4520SKenneth D. Merry 		struct mount *mountpoint;
708130f4520SKenneth D. Merry 		int lock_flags;
709130f4520SKenneth D. Merry 
710130f4520SKenneth D. Merry 		(void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
711130f4520SKenneth D. Merry 
71267cc546dSAlexander Motin 		if (MNT_SHARED_WRITES(mountpoint) || ((mountpoint == NULL)
713130f4520SKenneth D. Merry 		  && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
714130f4520SKenneth D. Merry 			lock_flags = LK_SHARED;
715130f4520SKenneth D. Merry 		else
716130f4520SKenneth D. Merry 			lock_flags = LK_EXCLUSIVE;
717130f4520SKenneth D. Merry 		vn_lock(be_lun->vn, lock_flags | LK_RETRY);
718130f4520SKenneth D. Merry 
719130f4520SKenneth D. Merry 		/*
720130f4520SKenneth D. Merry 		 * UFS pays attention to IO_DIRECT for writes.  The write
721130f4520SKenneth D. Merry 		 * is done asynchronously.  (Normally the write would just
722130f4520SKenneth D. Merry 		 * get put into cache.
723130f4520SKenneth D. Merry 		 *
724130f4520SKenneth D. Merry 		 * UFS pays attention to IO_SYNC for writes.  It will
725130f4520SKenneth D. Merry 		 * attempt to write the buffer out synchronously if that
726130f4520SKenneth D. Merry 		 * flag is set.
727130f4520SKenneth D. Merry 		 *
728130f4520SKenneth D. Merry 		 * ZFS does not pay attention to IO_DIRECT for writes.
729130f4520SKenneth D. Merry 		 *
730130f4520SKenneth D. Merry 		 * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
731130f4520SKenneth D. Merry 		 * for writes.  It will flush the transaction from the
732130f4520SKenneth D. Merry 		 * cache before returning.
733130f4520SKenneth D. Merry 		 */
73455551d05SAlexander Motin 		error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);
735130f4520SKenneth D. Merry 		VOP_UNLOCK(be_lun->vn, 0);
736130f4520SKenneth D. Merry 
737130f4520SKenneth D. Merry 		vn_finished_write(mountpoint);
73836160958SMark Johnston 		SDT_PROBE0(cbb, , write, file_done);
739130f4520SKenneth D. Merry         }
740130f4520SKenneth D. Merry 
74175c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
74275c7a1d3SAlexander Motin 	devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
74375c7a1d3SAlexander Motin 	    beio->ds_tag_type, beio->ds_trans_type,
74475c7a1d3SAlexander Motin 	    /*now*/ NULL, /*then*/&beio->ds_t0);
74575c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
74675c7a1d3SAlexander Motin 
747130f4520SKenneth D. Merry 	/*
748130f4520SKenneth D. Merry 	 * If we got an error, set the sense data to "MEDIUM ERROR" and
749130f4520SKenneth D. Merry 	 * return the I/O to the user.
750130f4520SKenneth D. Merry 	 */
751130f4520SKenneth D. Merry 	if (error != 0) {
7520631de4aSAlexander Motin 		if (error == ENOSPC || error == EDQUOT) {
7534fc18ff9SAlexander Motin 			ctl_set_space_alloc_fail(&io->scsiio);
7546187d472SAlexander Motin 		} else if (error == EROFS || error == EACCES) {
7556187d472SAlexander Motin 			ctl_set_hw_write_protected(&io->scsiio);
7567f7bb97aSAlexander Motin 		} else {
7577f7bb97aSAlexander Motin 			ctl_set_medium_error(&io->scsiio,
7587f7bb97aSAlexander Motin 			    beio->bio_cmd == BIO_READ);
7597f7bb97aSAlexander Motin 		}
760130f4520SKenneth D. Merry 		ctl_complete_beio(beio);
761130f4520SKenneth D. Merry 		return;
762130f4520SKenneth D. Merry 	}
763130f4520SKenneth D. Merry 
764130f4520SKenneth D. Merry 	/*
765696297adSAlexander Motin 	 * If this is a write or a verify, we're all done.
766130f4520SKenneth D. Merry 	 * If this is a read, we can now send the data to the user.
767130f4520SKenneth D. Merry 	 */
768696297adSAlexander Motin 	if ((beio->bio_cmd == BIO_WRITE) ||
769696297adSAlexander Motin 	    (ARGS(io)->flags & CTL_LLF_VERIFY)) {
770130f4520SKenneth D. Merry 		ctl_set_success(&io->scsiio);
771130f4520SKenneth D. Merry 		ctl_complete_beio(beio);
772130f4520SKenneth D. Merry 	} else {
773f7241cceSAlexander Motin 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
77475a3108eSAlexander Motin 		    beio->beio_cont == NULL) {
775f7241cceSAlexander Motin 			ctl_set_success(&io->scsiio);
77675a3108eSAlexander Motin 			ctl_serseq_done(io);
77775a3108eSAlexander Motin 		}
778130f4520SKenneth D. Merry #ifdef CTL_TIME_IO
779e675024aSAlexander Motin 		getbinuptime(&io->io_hdr.dma_start_bt);
780130f4520SKenneth D. Merry #endif
781130f4520SKenneth D. Merry 		ctl_datamove(io);
782130f4520SKenneth D. Merry 	}
783130f4520SKenneth D. Merry }
784130f4520SKenneth D. Merry 
785130f4520SKenneth D. Merry static void
786ef8daf3fSAlexander Motin ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
787ef8daf3fSAlexander Motin 			struct ctl_be_block_io *beio)
788ef8daf3fSAlexander Motin {
789ef8daf3fSAlexander Motin 	union ctl_io *io = beio->io;
790ef8daf3fSAlexander Motin 	struct ctl_lba_len_flags *lbalen = ARGS(io);
791ef8daf3fSAlexander Motin 	struct scsi_get_lba_status_data *data;
792ef8daf3fSAlexander Motin 	off_t roff, off;
793ef8daf3fSAlexander Motin 	int error, status;
794ef8daf3fSAlexander Motin 
795ef8daf3fSAlexander Motin 	DPRINTF("entered\n");
796ef8daf3fSAlexander Motin 
7970bcd4ab6SAlexander Motin 	off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
798ef8daf3fSAlexander Motin 	vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
799ef8daf3fSAlexander Motin 	error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off,
800ef8daf3fSAlexander Motin 	    0, curthread->td_ucred, curthread);
801ef8daf3fSAlexander Motin 	if (error == 0 && off > roff)
802ef8daf3fSAlexander Motin 		status = 0;	/* mapped up to off */
803ef8daf3fSAlexander Motin 	else {
804ef8daf3fSAlexander Motin 		error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off,
805ef8daf3fSAlexander Motin 		    0, curthread->td_ucred, curthread);
806ef8daf3fSAlexander Motin 		if (error == 0 && off > roff)
807ef8daf3fSAlexander Motin 			status = 1;	/* deallocated up to off */
808ef8daf3fSAlexander Motin 		else {
809ef8daf3fSAlexander Motin 			status = 0;	/* unknown up to the end */
810ef8daf3fSAlexander Motin 			off = be_lun->size_bytes;
811ef8daf3fSAlexander Motin 		}
812ef8daf3fSAlexander Motin 	}
813ef8daf3fSAlexander Motin 	VOP_UNLOCK(be_lun->vn, 0);
814ef8daf3fSAlexander Motin 
815ef8daf3fSAlexander Motin 	data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
816ef8daf3fSAlexander Motin 	scsi_u64to8b(lbalen->lba, data->descr[0].addr);
8170bcd4ab6SAlexander Motin 	scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
8180bcd4ab6SAlexander Motin 	    lbalen->lba), data->descr[0].length);
819ef8daf3fSAlexander Motin 	data->descr[0].status = status;
820ef8daf3fSAlexander Motin 
821ef8daf3fSAlexander Motin 	ctl_complete_beio(beio);
822ef8daf3fSAlexander Motin }
823ef8daf3fSAlexander Motin 
82453c146deSAlexander Motin static uint64_t
82553c146deSAlexander Motin ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname)
82653c146deSAlexander Motin {
82753c146deSAlexander Motin 	struct vattr		vattr;
82853c146deSAlexander Motin 	struct statfs		statfs;
829b9b4269cSAlexander Motin 	uint64_t		val;
83053c146deSAlexander Motin 	int			error;
83153c146deSAlexander Motin 
832b9b4269cSAlexander Motin 	val = UINT64_MAX;
83353c146deSAlexander Motin 	if (be_lun->vn == NULL)
834b9b4269cSAlexander Motin 		return (val);
835b9b4269cSAlexander Motin 	vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
83653c146deSAlexander Motin 	if (strcmp(attrname, "blocksused") == 0) {
83753c146deSAlexander Motin 		error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
838b9b4269cSAlexander Motin 		if (error == 0)
8390bcd4ab6SAlexander Motin 			val = vattr.va_bytes / be_lun->cbe_lun.blocksize;
84053c146deSAlexander Motin 	}
841b9b4269cSAlexander Motin 	if (strcmp(attrname, "blocksavail") == 0 &&
842b9b4269cSAlexander Motin 	    (be_lun->vn->v_iflag & VI_DOOMED) == 0) {
84353c146deSAlexander Motin 		error = VFS_STATFS(be_lun->vn->v_mount, &statfs);
844b9b4269cSAlexander Motin 		if (error == 0)
845a15bbf15SAlexander Motin 			val = statfs.f_bavail * statfs.f_bsize /
8460bcd4ab6SAlexander Motin 			    be_lun->cbe_lun.blocksize;
84753c146deSAlexander Motin 	}
848b9b4269cSAlexander Motin 	VOP_UNLOCK(be_lun->vn, 0);
849b9b4269cSAlexander Motin 	return (val);
85053c146deSAlexander Motin }
85153c146deSAlexander Motin 
852ef8daf3fSAlexander Motin static void
85367f586a8SAlexander Motin ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
85467f586a8SAlexander Motin 			   struct ctl_be_block_io *beio)
85567f586a8SAlexander Motin {
85667f586a8SAlexander Motin 	union ctl_io *io;
8573236151eSAlexander Motin 	struct cdevsw *csw;
8583236151eSAlexander Motin 	struct cdev *dev;
85967f586a8SAlexander Motin 	struct uio xuio;
86067f586a8SAlexander Motin 	struct iovec *xiovec;
8613236151eSAlexander Motin 	int error, flags, i, ref;
86267f586a8SAlexander Motin 
86367f586a8SAlexander Motin 	DPRINTF("entered\n");
86467f586a8SAlexander Motin 
86567f586a8SAlexander Motin 	io = beio->io;
86655551d05SAlexander Motin 	flags = 0;
86755551d05SAlexander Motin 	if (ARGS(io)->flags & CTL_LLF_DPO)
86855551d05SAlexander Motin 		flags |= IO_DIRECT;
86955551d05SAlexander Motin 	if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
87055551d05SAlexander Motin 		flags |= IO_SYNC;
87167f586a8SAlexander Motin 
87267f586a8SAlexander Motin 	bzero(&xuio, sizeof(xuio));
87367f586a8SAlexander Motin 	if (beio->bio_cmd == BIO_READ) {
87436160958SMark Johnston 		SDT_PROBE0(cbb, , read, file_start);
87567f586a8SAlexander Motin 		xuio.uio_rw = UIO_READ;
87667f586a8SAlexander Motin 	} else {
87736160958SMark Johnston 		SDT_PROBE0(cbb, , write, file_start);
87867f586a8SAlexander Motin 		xuio.uio_rw = UIO_WRITE;
87967f586a8SAlexander Motin 	}
88067f586a8SAlexander Motin 	xuio.uio_offset = beio->io_offset;
88167f586a8SAlexander Motin 	xuio.uio_resid = beio->io_len;
88267f586a8SAlexander Motin 	xuio.uio_segflg = UIO_SYSSPACE;
88367f586a8SAlexander Motin 	xuio.uio_iov = beio->xiovecs;
88467f586a8SAlexander Motin 	xuio.uio_iovcnt = beio->num_segs;
88567f586a8SAlexander Motin 	xuio.uio_td = curthread;
88667f586a8SAlexander Motin 
88767f586a8SAlexander Motin 	for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
88867f586a8SAlexander Motin 		xiovec->iov_base = beio->sg_segs[i].addr;
88967f586a8SAlexander Motin 		xiovec->iov_len = beio->sg_segs[i].len;
89067f586a8SAlexander Motin 	}
89167f586a8SAlexander Motin 
89267f586a8SAlexander Motin 	binuptime(&beio->ds_t0);
89367f586a8SAlexander Motin 	mtx_lock(&be_lun->io_lock);
89467f586a8SAlexander Motin 	devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
89567f586a8SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
89667f586a8SAlexander Motin 
8973236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
8983236151eSAlexander Motin 	if (csw) {
8993236151eSAlexander Motin 		if (beio->bio_cmd == BIO_READ)
9003236151eSAlexander Motin 			error = csw->d_read(dev, &xuio, flags);
9013236151eSAlexander Motin 		else
9023236151eSAlexander Motin 			error = csw->d_write(dev, &xuio, flags);
9033236151eSAlexander Motin 		dev_relthread(dev, ref);
9043236151eSAlexander Motin 	} else
9053236151eSAlexander Motin 		error = ENXIO;
9063236151eSAlexander Motin 
9073236151eSAlexander Motin 	if (beio->bio_cmd == BIO_READ)
90836160958SMark Johnston 		SDT_PROBE0(cbb, , read, file_done);
9093236151eSAlexander Motin 	else
91036160958SMark Johnston 		SDT_PROBE0(cbb, , write, file_done);
91167f586a8SAlexander Motin 
91267f586a8SAlexander Motin 	mtx_lock(&be_lun->io_lock);
91367f586a8SAlexander Motin 	devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
91467f586a8SAlexander Motin 	    beio->ds_tag_type, beio->ds_trans_type,
91567f586a8SAlexander Motin 	    /*now*/ NULL, /*then*/&beio->ds_t0);
91667f586a8SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
91767f586a8SAlexander Motin 
91867f586a8SAlexander Motin 	/*
91967f586a8SAlexander Motin 	 * If we got an error, set the sense data to "MEDIUM ERROR" and
92067f586a8SAlexander Motin 	 * return the I/O to the user.
92167f586a8SAlexander Motin 	 */
92267f586a8SAlexander Motin 	if (error != 0) {
9230631de4aSAlexander Motin 		if (error == ENOSPC || error == EDQUOT) {
9244fc18ff9SAlexander Motin 			ctl_set_space_alloc_fail(&io->scsiio);
9256187d472SAlexander Motin 		} else if (error == EROFS || error == EACCES) {
9266187d472SAlexander Motin 			ctl_set_hw_write_protected(&io->scsiio);
9277f7bb97aSAlexander Motin 		} else {
9287f7bb97aSAlexander Motin 			ctl_set_medium_error(&io->scsiio,
9297f7bb97aSAlexander Motin 			    beio->bio_cmd == BIO_READ);
9307f7bb97aSAlexander Motin 		}
93167f586a8SAlexander Motin 		ctl_complete_beio(beio);
93267f586a8SAlexander Motin 		return;
93367f586a8SAlexander Motin 	}
93467f586a8SAlexander Motin 
93567f586a8SAlexander Motin 	/*
93667f586a8SAlexander Motin 	 * If this is a write or a verify, we're all done.
93767f586a8SAlexander Motin 	 * If this is a read, we can now send the data to the user.
93867f586a8SAlexander Motin 	 */
93967f586a8SAlexander Motin 	if ((beio->bio_cmd == BIO_WRITE) ||
94067f586a8SAlexander Motin 	    (ARGS(io)->flags & CTL_LLF_VERIFY)) {
94167f586a8SAlexander Motin 		ctl_set_success(&io->scsiio);
94267f586a8SAlexander Motin 		ctl_complete_beio(beio);
94367f586a8SAlexander Motin 	} else {
944f7241cceSAlexander Motin 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
94575a3108eSAlexander Motin 		    beio->beio_cont == NULL) {
946f7241cceSAlexander Motin 			ctl_set_success(&io->scsiio);
94775a3108eSAlexander Motin 			ctl_serseq_done(io);
94875a3108eSAlexander Motin 		}
94967f586a8SAlexander Motin #ifdef CTL_TIME_IO
950e675024aSAlexander Motin 		getbinuptime(&io->io_hdr.dma_start_bt);
95167f586a8SAlexander Motin #endif
95267f586a8SAlexander Motin 		ctl_datamove(io);
95367f586a8SAlexander Motin 	}
95467f586a8SAlexander Motin }
95567f586a8SAlexander Motin 
95667f586a8SAlexander Motin static void
957ef8daf3fSAlexander Motin ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun,
958ef8daf3fSAlexander Motin 			struct ctl_be_block_io *beio)
959ef8daf3fSAlexander Motin {
960ef8daf3fSAlexander Motin 	union ctl_io *io = beio->io;
9613236151eSAlexander Motin 	struct cdevsw *csw;
9623236151eSAlexander Motin 	struct cdev *dev;
963ef8daf3fSAlexander Motin 	struct ctl_lba_len_flags *lbalen = ARGS(io);
964ef8daf3fSAlexander Motin 	struct scsi_get_lba_status_data *data;
965ef8daf3fSAlexander Motin 	off_t roff, off;
9663236151eSAlexander Motin 	int error, ref, status;
967ef8daf3fSAlexander Motin 
968ef8daf3fSAlexander Motin 	DPRINTF("entered\n");
969ef8daf3fSAlexander Motin 
9703236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
9713236151eSAlexander Motin 	if (csw == NULL) {
9723236151eSAlexander Motin 		status = 0;	/* unknown up to the end */
9733236151eSAlexander Motin 		off = be_lun->size_bytes;
9743236151eSAlexander Motin 		goto done;
9753236151eSAlexander Motin 	}
9760bcd4ab6SAlexander Motin 	off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
9773236151eSAlexander Motin 	error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD,
9783236151eSAlexander Motin 	    curthread);
979ef8daf3fSAlexander Motin 	if (error == 0 && off > roff)
980ef8daf3fSAlexander Motin 		status = 0;	/* mapped up to off */
981ef8daf3fSAlexander Motin 	else {
9823236151eSAlexander Motin 		error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD,
9833236151eSAlexander Motin 		    curthread);
984ef8daf3fSAlexander Motin 		if (error == 0 && off > roff)
985ef8daf3fSAlexander Motin 			status = 1;	/* deallocated up to off */
986ef8daf3fSAlexander Motin 		else {
987ef8daf3fSAlexander Motin 			status = 0;	/* unknown up to the end */
988ef8daf3fSAlexander Motin 			off = be_lun->size_bytes;
989ef8daf3fSAlexander Motin 		}
990ef8daf3fSAlexander Motin 	}
9913236151eSAlexander Motin 	dev_relthread(dev, ref);
992ef8daf3fSAlexander Motin 
9933236151eSAlexander Motin done:
994ef8daf3fSAlexander Motin 	data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
995ef8daf3fSAlexander Motin 	scsi_u64to8b(lbalen->lba, data->descr[0].addr);
9960bcd4ab6SAlexander Motin 	scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
9970bcd4ab6SAlexander Motin 	    lbalen->lba), data->descr[0].length);
998ef8daf3fSAlexander Motin 	data->descr[0].status = status;
999ef8daf3fSAlexander Motin 
1000ef8daf3fSAlexander Motin 	ctl_complete_beio(beio);
1001ef8daf3fSAlexander Motin }
1002ef8daf3fSAlexander Motin 
1003ef8daf3fSAlexander Motin static void
1004130f4520SKenneth D. Merry ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
1005130f4520SKenneth D. Merry 		       struct ctl_be_block_io *beio)
1006130f4520SKenneth D. Merry {
1007130f4520SKenneth D. Merry 	struct bio *bio;
10083236151eSAlexander Motin 	struct cdevsw *csw;
10093236151eSAlexander Motin 	struct cdev *dev;
10103236151eSAlexander Motin 	int ref;
1011130f4520SKenneth D. Merry 
1012130f4520SKenneth D. Merry 	DPRINTF("entered\n");
1013130f4520SKenneth D. Merry 
1014130f4520SKenneth D. Merry 	/* This can't fail, it's a blocking allocation. */
1015130f4520SKenneth D. Merry 	bio = g_alloc_bio();
1016130f4520SKenneth D. Merry 
1017130f4520SKenneth D. Merry 	bio->bio_cmd	    = BIO_FLUSH;
1018130f4520SKenneth D. Merry 	bio->bio_offset	    = 0;
1019130f4520SKenneth D. Merry 	bio->bio_data	    = 0;
1020130f4520SKenneth D. Merry 	bio->bio_done	    = ctl_be_block_biodone;
1021130f4520SKenneth D. Merry 	bio->bio_caller1    = beio;
1022130f4520SKenneth D. Merry 	bio->bio_pblkno	    = 0;
1023130f4520SKenneth D. Merry 
1024130f4520SKenneth D. Merry 	/*
1025130f4520SKenneth D. Merry 	 * We don't need to acquire the LUN lock here, because we are only
1026130f4520SKenneth D. Merry 	 * sending one bio, and so there is no other context to synchronize
1027130f4520SKenneth D. Merry 	 * with.
1028130f4520SKenneth D. Merry 	 */
1029130f4520SKenneth D. Merry 	beio->num_bios_sent = 1;
1030130f4520SKenneth D. Merry 	beio->send_complete = 1;
1031130f4520SKenneth D. Merry 
1032130f4520SKenneth D. Merry 	binuptime(&beio->ds_t0);
103375c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
1034130f4520SKenneth D. Merry 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
103575c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
1036130f4520SKenneth D. Merry 
10373236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
10383236151eSAlexander Motin 	if (csw) {
10393236151eSAlexander Motin 		bio->bio_dev = dev;
10403236151eSAlexander Motin 		csw->d_strategy(bio);
10413236151eSAlexander Motin 		dev_relthread(dev, ref);
10423236151eSAlexander Motin 	} else {
10433236151eSAlexander Motin 		bio->bio_error = ENXIO;
10443236151eSAlexander Motin 		ctl_be_block_biodone(bio);
10453236151eSAlexander Motin 	}
1046130f4520SKenneth D. Merry }
1047130f4520SKenneth D. Merry 
1048130f4520SKenneth D. Merry static void
1049ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,
1050ee7f31c0SAlexander Motin 		       struct ctl_be_block_io *beio,
1051ee7f31c0SAlexander Motin 		       uint64_t off, uint64_t len, int last)
1052ee7f31c0SAlexander Motin {
1053ee7f31c0SAlexander Motin 	struct bio *bio;
10548f5a226aSAlexander Motin 	uint64_t maxlen;
10553236151eSAlexander Motin 	struct cdevsw *csw;
10563236151eSAlexander Motin 	struct cdev *dev;
10573236151eSAlexander Motin 	int ref;
1058ee7f31c0SAlexander Motin 
10593236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
10600bcd4ab6SAlexander Motin 	maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize);
1061ee7f31c0SAlexander Motin 	while (len > 0) {
1062ee7f31c0SAlexander Motin 		bio = g_alloc_bio();
1063ee7f31c0SAlexander Motin 		bio->bio_cmd	    = BIO_DELETE;
10643236151eSAlexander Motin 		bio->bio_dev	    = dev;
1065ee7f31c0SAlexander Motin 		bio->bio_offset	    = off;
10668f5a226aSAlexander Motin 		bio->bio_length	    = MIN(len, maxlen);
1067ee7f31c0SAlexander Motin 		bio->bio_data	    = 0;
1068ee7f31c0SAlexander Motin 		bio->bio_done	    = ctl_be_block_biodone;
1069ee7f31c0SAlexander Motin 		bio->bio_caller1    = beio;
10700bcd4ab6SAlexander Motin 		bio->bio_pblkno     = off / be_lun->cbe_lun.blocksize;
1071ee7f31c0SAlexander Motin 
1072ee7f31c0SAlexander Motin 		off += bio->bio_length;
1073ee7f31c0SAlexander Motin 		len -= bio->bio_length;
1074ee7f31c0SAlexander Motin 
107575c7a1d3SAlexander Motin 		mtx_lock(&be_lun->io_lock);
1076ee7f31c0SAlexander Motin 		beio->num_bios_sent++;
1077ee7f31c0SAlexander Motin 		if (last && len == 0)
1078ee7f31c0SAlexander Motin 			beio->send_complete = 1;
107975c7a1d3SAlexander Motin 		mtx_unlock(&be_lun->io_lock);
1080ee7f31c0SAlexander Motin 
10813236151eSAlexander Motin 		if (csw) {
10823236151eSAlexander Motin 			csw->d_strategy(bio);
10833236151eSAlexander Motin 		} else {
10843236151eSAlexander Motin 			bio->bio_error = ENXIO;
10853236151eSAlexander Motin 			ctl_be_block_biodone(bio);
1086ee7f31c0SAlexander Motin 		}
1087ee7f31c0SAlexander Motin 	}
10883236151eSAlexander Motin 	if (csw)
10893236151eSAlexander Motin 		dev_relthread(dev, ref);
10903236151eSAlexander Motin }
1091ee7f31c0SAlexander Motin 
1092ee7f31c0SAlexander Motin static void
1093ee7f31c0SAlexander Motin ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
1094ee7f31c0SAlexander Motin 		       struct ctl_be_block_io *beio)
1095ee7f31c0SAlexander Motin {
1096ee7f31c0SAlexander Motin 	union ctl_io *io;
109766df9136SAlexander Motin 	struct ctl_ptr_len_flags *ptrlen;
1098ee7f31c0SAlexander Motin 	struct scsi_unmap_desc *buf, *end;
1099ee7f31c0SAlexander Motin 	uint64_t len;
1100ee7f31c0SAlexander Motin 
1101ee7f31c0SAlexander Motin 	io = beio->io;
1102ee7f31c0SAlexander Motin 
1103ee7f31c0SAlexander Motin 	DPRINTF("entered\n");
1104ee7f31c0SAlexander Motin 
1105ee7f31c0SAlexander Motin 	binuptime(&beio->ds_t0);
110675c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
1107ee7f31c0SAlexander Motin 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
110875c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
1109ee7f31c0SAlexander Motin 
1110ee7f31c0SAlexander Motin 	if (beio->io_offset == -1) {
1111ee7f31c0SAlexander Motin 		beio->io_len = 0;
111266df9136SAlexander Motin 		ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
111366df9136SAlexander Motin 		buf = (struct scsi_unmap_desc *)ptrlen->ptr;
111466df9136SAlexander Motin 		end = buf + ptrlen->len / sizeof(*buf);
1115ee7f31c0SAlexander Motin 		for (; buf < end; buf++) {
1116ee7f31c0SAlexander Motin 			len = (uint64_t)scsi_4btoul(buf->length) *
11170bcd4ab6SAlexander Motin 			    be_lun->cbe_lun.blocksize;
1118ee7f31c0SAlexander Motin 			beio->io_len += len;
1119ee7f31c0SAlexander Motin 			ctl_be_block_unmap_dev_range(be_lun, beio,
11200bcd4ab6SAlexander Motin 			    scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize,
11210bcd4ab6SAlexander Motin 			    len, (end - buf < 2) ? TRUE : FALSE);
1122ee7f31c0SAlexander Motin 		}
1123ee7f31c0SAlexander Motin 	} else
1124ee7f31c0SAlexander Motin 		ctl_be_block_unmap_dev_range(be_lun, beio,
1125ee7f31c0SAlexander Motin 		    beio->io_offset, beio->io_len, TRUE);
1126ee7f31c0SAlexander Motin }
1127ee7f31c0SAlexander Motin 
1128ee7f31c0SAlexander Motin static void
1129130f4520SKenneth D. Merry ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
1130130f4520SKenneth D. Merry 			  struct ctl_be_block_io *beio)
1131130f4520SKenneth D. Merry {
113275c7a1d3SAlexander Motin 	TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue);
1133130f4520SKenneth D. Merry 	struct bio *bio;
11343236151eSAlexander Motin 	struct cdevsw *csw;
11353236151eSAlexander Motin 	struct cdev *dev;
1136130f4520SKenneth D. Merry 	off_t cur_offset;
11373236151eSAlexander Motin 	int i, max_iosize, ref;
1138130f4520SKenneth D. Merry 
1139130f4520SKenneth D. Merry 	DPRINTF("entered\n");
11403236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
1141130f4520SKenneth D. Merry 
1142130f4520SKenneth D. Merry 	/*
1143130f4520SKenneth D. Merry 	 * We have to limit our I/O size to the maximum supported by the
1144130f4520SKenneth D. Merry 	 * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
1145130f4520SKenneth D. Merry 	 * set it properly, use DFLTPHYS.
1146130f4520SKenneth D. Merry 	 */
11473236151eSAlexander Motin 	if (csw) {
11483236151eSAlexander Motin 		max_iosize = dev->si_iosize_max;
1149130f4520SKenneth D. Merry 		if (max_iosize < PAGE_SIZE)
1150130f4520SKenneth D. Merry 			max_iosize = DFLTPHYS;
11513236151eSAlexander Motin 	} else
11523236151eSAlexander Motin 		max_iosize = DFLTPHYS;
1153130f4520SKenneth D. Merry 
1154130f4520SKenneth D. Merry 	cur_offset = beio->io_offset;
1155130f4520SKenneth D. Merry 	for (i = 0; i < beio->num_segs; i++) {
1156130f4520SKenneth D. Merry 		size_t cur_size;
1157130f4520SKenneth D. Merry 		uint8_t *cur_ptr;
1158130f4520SKenneth D. Merry 
1159130f4520SKenneth D. Merry 		cur_size = beio->sg_segs[i].len;
1160130f4520SKenneth D. Merry 		cur_ptr = beio->sg_segs[i].addr;
1161130f4520SKenneth D. Merry 
1162130f4520SKenneth D. Merry 		while (cur_size > 0) {
1163130f4520SKenneth D. Merry 			/* This can't fail, it's a blocking allocation. */
1164130f4520SKenneth D. Merry 			bio = g_alloc_bio();
1165130f4520SKenneth D. Merry 
1166130f4520SKenneth D. Merry 			KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
1167130f4520SKenneth D. Merry 
1168130f4520SKenneth D. Merry 			bio->bio_cmd = beio->bio_cmd;
11693236151eSAlexander Motin 			bio->bio_dev = dev;
1170130f4520SKenneth D. Merry 			bio->bio_caller1 = beio;
1171130f4520SKenneth D. Merry 			bio->bio_length = min(cur_size, max_iosize);
1172130f4520SKenneth D. Merry 			bio->bio_offset = cur_offset;
1173130f4520SKenneth D. Merry 			bio->bio_data = cur_ptr;
1174130f4520SKenneth D. Merry 			bio->bio_done = ctl_be_block_biodone;
11750bcd4ab6SAlexander Motin 			bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize;
1176130f4520SKenneth D. Merry 
1177130f4520SKenneth D. Merry 			cur_offset += bio->bio_length;
1178130f4520SKenneth D. Merry 			cur_ptr += bio->bio_length;
1179130f4520SKenneth D. Merry 			cur_size -= bio->bio_length;
1180130f4520SKenneth D. Merry 
118175c7a1d3SAlexander Motin 			TAILQ_INSERT_TAIL(&queue, bio, bio_queue);
1182130f4520SKenneth D. Merry 			beio->num_bios_sent++;
1183130f4520SKenneth D. Merry 		}
1184130f4520SKenneth D. Merry 	}
118575c7a1d3SAlexander Motin 	binuptime(&beio->ds_t0);
118675c7a1d3SAlexander Motin 	mtx_lock(&be_lun->io_lock);
118775c7a1d3SAlexander Motin 	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
118875c7a1d3SAlexander Motin 	beio->send_complete = 1;
118975c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->io_lock);
119075c7a1d3SAlexander Motin 
119175c7a1d3SAlexander Motin 	/*
119275c7a1d3SAlexander Motin 	 * Fire off all allocated requests!
119375c7a1d3SAlexander Motin 	 */
119475c7a1d3SAlexander Motin 	while ((bio = TAILQ_FIRST(&queue)) != NULL) {
119575c7a1d3SAlexander Motin 		TAILQ_REMOVE(&queue, bio, bio_queue);
11963236151eSAlexander Motin 		if (csw)
11973236151eSAlexander Motin 			csw->d_strategy(bio);
11983236151eSAlexander Motin 		else {
11993236151eSAlexander Motin 			bio->bio_error = ENXIO;
12003236151eSAlexander Motin 			ctl_be_block_biodone(bio);
120175c7a1d3SAlexander Motin 		}
1202130f4520SKenneth D. Merry 	}
12033236151eSAlexander Motin 	if (csw)
12043236151eSAlexander Motin 		dev_relthread(dev, ref);
12053236151eSAlexander Motin }
1206130f4520SKenneth D. Merry 
1207c3e7ba3eSAlexander Motin static uint64_t
1208c3e7ba3eSAlexander Motin ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname)
1209c3e7ba3eSAlexander Motin {
1210c3e7ba3eSAlexander Motin 	struct diocgattr_arg	arg;
12113236151eSAlexander Motin 	struct cdevsw *csw;
12123236151eSAlexander Motin 	struct cdev *dev;
12133236151eSAlexander Motin 	int error, ref;
1214c3e7ba3eSAlexander Motin 
12153236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
12163236151eSAlexander Motin 	if (csw == NULL)
1217c3e7ba3eSAlexander Motin 		return (UINT64_MAX);
1218c3e7ba3eSAlexander Motin 	strlcpy(arg.name, attrname, sizeof(arg.name));
1219c3e7ba3eSAlexander Motin 	arg.len = sizeof(arg.value.off);
12203236151eSAlexander Motin 	if (csw->d_ioctl) {
12213236151eSAlexander Motin 		error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
12223236151eSAlexander Motin 		    curthread);
12233236151eSAlexander Motin 	} else
12243236151eSAlexander Motin 		error = ENODEV;
12253236151eSAlexander Motin 	dev_relthread(dev, ref);
1226c3e7ba3eSAlexander Motin 	if (error != 0)
1227c3e7ba3eSAlexander Motin 		return (UINT64_MAX);
1228c3e7ba3eSAlexander Motin 	return (arg.value.off);
1229c3e7ba3eSAlexander Motin }
1230c3e7ba3eSAlexander Motin 
1231130f4520SKenneth D. Merry static void
12327d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun,
12337d0d4342SAlexander Motin 			    union ctl_io *io)
12347d0d4342SAlexander Motin {
12350bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
12367d0d4342SAlexander Motin 	struct ctl_be_block_io *beio;
12377d0d4342SAlexander Motin 	struct ctl_lba_len_flags *lbalen;
12387d0d4342SAlexander Motin 
12397d0d4342SAlexander Motin 	DPRINTF("entered\n");
12407d0d4342SAlexander Motin 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
12417d0d4342SAlexander Motin 	lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
12427d0d4342SAlexander Motin 
12430bcd4ab6SAlexander Motin 	beio->io_len = lbalen->len * cbe_lun->blocksize;
12440bcd4ab6SAlexander Motin 	beio->io_offset = lbalen->lba * cbe_lun->blocksize;
12457d0d4342SAlexander Motin 	beio->io_arg = (lbalen->flags & SSC_IMMED) != 0;
12467d0d4342SAlexander Motin 	beio->bio_cmd = BIO_FLUSH;
12477d0d4342SAlexander Motin 	beio->ds_trans_type = DEVSTAT_NO_DATA;
12487d0d4342SAlexander Motin 	DPRINTF("SYNC\n");
12497d0d4342SAlexander Motin 	be_lun->lun_flush(be_lun, beio);
12507d0d4342SAlexander Motin }
12517d0d4342SAlexander Motin 
12527d0d4342SAlexander Motin static void
1253ee7f31c0SAlexander Motin ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio)
1254ee7f31c0SAlexander Motin {
1255ee7f31c0SAlexander Motin 	union ctl_io *io;
1256ee7f31c0SAlexander Motin 
1257ee7f31c0SAlexander Motin 	io = beio->io;
1258ee7f31c0SAlexander Motin 	ctl_free_beio(beio);
1259ead2f117SAlexander Motin 	if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1260ead2f117SAlexander Motin 	    ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1261ead2f117SAlexander Motin 	     (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1262ee7f31c0SAlexander Motin 		ctl_config_write_done(io);
1263ee7f31c0SAlexander Motin 		return;
1264ee7f31c0SAlexander Motin 	}
1265ee7f31c0SAlexander Motin 
1266ee7f31c0SAlexander Motin 	ctl_be_block_config_write(io);
1267ee7f31c0SAlexander Motin }
1268ee7f31c0SAlexander Motin 
1269ee7f31c0SAlexander Motin static void
1270ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
1271ee7f31c0SAlexander Motin 			    union ctl_io *io)
1272ee7f31c0SAlexander Motin {
12730bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1274ee7f31c0SAlexander Motin 	struct ctl_be_block_io *beio;
127566df9136SAlexander Motin 	struct ctl_lba_len_flags *lbalen;
1276fee04ef7SAlexander Motin 	uint64_t len_left, lba;
1277fee04ef7SAlexander Motin 	uint32_t pb, pbo, adj;
1278ee7f31c0SAlexander Motin 	int i, seglen;
1279ee7f31c0SAlexander Motin 	uint8_t *buf, *end;
1280ee7f31c0SAlexander Motin 
1281ee7f31c0SAlexander Motin 	DPRINTF("entered\n");
1282ee7f31c0SAlexander Motin 
1283e86a4142SAlexander Motin 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
128411b569f7SAlexander Motin 	lbalen = ARGS(beio->io);
1285ee7f31c0SAlexander Motin 
128664c5167cSAlexander Motin 	if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) ||
12873406a2a0SAlexander Motin 	    (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {
1288ee7f31c0SAlexander Motin 		ctl_free_beio(beio);
1289ee7f31c0SAlexander Motin 		ctl_set_invalid_field(&io->scsiio,
1290ee7f31c0SAlexander Motin 				      /*sks_valid*/ 1,
1291ee7f31c0SAlexander Motin 				      /*command*/ 1,
1292ee7f31c0SAlexander Motin 				      /*field*/ 1,
1293ee7f31c0SAlexander Motin 				      /*bit_valid*/ 0,
1294ee7f31c0SAlexander Motin 				      /*bit*/ 0);
1295ee7f31c0SAlexander Motin 		ctl_config_write_done(io);
1296ee7f31c0SAlexander Motin 		return;
1297ee7f31c0SAlexander Motin 	}
1298ee7f31c0SAlexander Motin 
12993406a2a0SAlexander Motin 	if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) {
13000bcd4ab6SAlexander Motin 		beio->io_offset = lbalen->lba * cbe_lun->blocksize;
13010bcd4ab6SAlexander Motin 		beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize;
1302ee7f31c0SAlexander Motin 		beio->bio_cmd = BIO_DELETE;
1303ee7f31c0SAlexander Motin 		beio->ds_trans_type = DEVSTAT_FREE;
1304ee7f31c0SAlexander Motin 
1305ee7f31c0SAlexander Motin 		be_lun->unmap(be_lun, beio);
1306ee7f31c0SAlexander Motin 		return;
1307ee7f31c0SAlexander Motin 	}
1308ee7f31c0SAlexander Motin 
1309ee7f31c0SAlexander Motin 	beio->bio_cmd = BIO_WRITE;
1310ee7f31c0SAlexander Motin 	beio->ds_trans_type = DEVSTAT_WRITE;
1311ee7f31c0SAlexander Motin 
1312ee7f31c0SAlexander Motin 	DPRINTF("WRITE SAME at LBA %jx len %u\n",
131366df9136SAlexander Motin 	       (uintmax_t)lbalen->lba, lbalen->len);
1314ee7f31c0SAlexander Motin 
13150bcd4ab6SAlexander Motin 	pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;
13160bcd4ab6SAlexander Motin 	if (be_lun->cbe_lun.pblockoff > 0)
13170bcd4ab6SAlexander Motin 		pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;
1318fee04ef7SAlexander Motin 	else
1319fee04ef7SAlexander Motin 		pbo = 0;
13200bcd4ab6SAlexander Motin 	len_left = (uint64_t)lbalen->len * cbe_lun->blocksize;
1321ee7f31c0SAlexander Motin 	for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) {
1322ee7f31c0SAlexander Motin 
1323ee7f31c0SAlexander Motin 		/*
1324ee7f31c0SAlexander Motin 		 * Setup the S/G entry for this chunk.
1325ee7f31c0SAlexander Motin 		 */
132608a7cce5SAlexander Motin 		seglen = MIN(CTLBLK_MAX_SEG, len_left);
13270bcd4ab6SAlexander Motin 		if (pb > cbe_lun->blocksize) {
13280bcd4ab6SAlexander Motin 			adj = ((lbalen->lba + lba) * cbe_lun->blocksize +
132993b8c96cSAlexander Motin 			    seglen - pbo) % pb;
133093b8c96cSAlexander Motin 			if (seglen > adj)
133193b8c96cSAlexander Motin 				seglen -= adj;
133293b8c96cSAlexander Motin 			else
13330bcd4ab6SAlexander Motin 				seglen -= seglen % cbe_lun->blocksize;
133493b8c96cSAlexander Motin 		} else
13350bcd4ab6SAlexander Motin 			seglen -= seglen % cbe_lun->blocksize;
1336ee7f31c0SAlexander Motin 		beio->sg_segs[i].len = seglen;
1337ee7f31c0SAlexander Motin 		beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1338ee7f31c0SAlexander Motin 
1339ee7f31c0SAlexander Motin 		DPRINTF("segment %d addr %p len %zd\n", i,
1340ee7f31c0SAlexander Motin 			beio->sg_segs[i].addr, beio->sg_segs[i].len);
1341ee7f31c0SAlexander Motin 
1342ee7f31c0SAlexander Motin 		beio->num_segs++;
1343ee7f31c0SAlexander Motin 		len_left -= seglen;
1344ee7f31c0SAlexander Motin 
1345ee7f31c0SAlexander Motin 		buf = beio->sg_segs[i].addr;
1346ee7f31c0SAlexander Motin 		end = buf + seglen;
13470bcd4ab6SAlexander Motin 		for (; buf < end; buf += cbe_lun->blocksize) {
13486c2acea5SAlexander Motin 			if (lbalen->flags & SWS_NDOB) {
13496c2acea5SAlexander Motin 				memset(buf, 0, cbe_lun->blocksize);
13506c2acea5SAlexander Motin 			} else {
13516c2acea5SAlexander Motin 				memcpy(buf, io->scsiio.kern_data_ptr,
13526c2acea5SAlexander Motin 				    cbe_lun->blocksize);
13536c2acea5SAlexander Motin 			}
135466df9136SAlexander Motin 			if (lbalen->flags & SWS_LBDATA)
135566df9136SAlexander Motin 				scsi_ulto4b(lbalen->lba + lba, buf);
1356ee7f31c0SAlexander Motin 			lba++;
1357ee7f31c0SAlexander Motin 		}
1358ee7f31c0SAlexander Motin 	}
1359ee7f31c0SAlexander Motin 
13600bcd4ab6SAlexander Motin 	beio->io_offset = lbalen->lba * cbe_lun->blocksize;
13610bcd4ab6SAlexander Motin 	beio->io_len = lba * cbe_lun->blocksize;
1362ee7f31c0SAlexander Motin 
1363ee7f31c0SAlexander Motin 	/* We can not do all in one run. Correct and schedule rerun. */
1364ee7f31c0SAlexander Motin 	if (len_left > 0) {
136566df9136SAlexander Motin 		lbalen->lba += lba;
136666df9136SAlexander Motin 		lbalen->len -= lba;
1367ee7f31c0SAlexander Motin 		beio->beio_cont = ctl_be_block_cw_done_ws;
1368ee7f31c0SAlexander Motin 	}
1369ee7f31c0SAlexander Motin 
1370ee7f31c0SAlexander Motin 	be_lun->dispatch(be_lun, beio);
1371ee7f31c0SAlexander Motin }
1372ee7f31c0SAlexander Motin 
1373ee7f31c0SAlexander Motin static void
1374ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,
1375ee7f31c0SAlexander Motin 			    union ctl_io *io)
1376ee7f31c0SAlexander Motin {
1377ee7f31c0SAlexander Motin 	struct ctl_be_block_io *beio;
137866df9136SAlexander Motin 	struct ctl_ptr_len_flags *ptrlen;
1379ee7f31c0SAlexander Motin 
1380ee7f31c0SAlexander Motin 	DPRINTF("entered\n");
1381ee7f31c0SAlexander Motin 
1382e86a4142SAlexander Motin 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
138366df9136SAlexander Motin 	ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1384ee7f31c0SAlexander Motin 
13853406a2a0SAlexander Motin 	if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {
1386ee7f31c0SAlexander Motin 		ctl_free_beio(beio);
1387ee7f31c0SAlexander Motin 		ctl_set_invalid_field(&io->scsiio,
1388ee7f31c0SAlexander Motin 				      /*sks_valid*/ 0,
1389ee7f31c0SAlexander Motin 				      /*command*/ 1,
1390ee7f31c0SAlexander Motin 				      /*field*/ 0,
1391ee7f31c0SAlexander Motin 				      /*bit_valid*/ 0,
1392ee7f31c0SAlexander Motin 				      /*bit*/ 0);
1393ee7f31c0SAlexander Motin 		ctl_config_write_done(io);
1394ee7f31c0SAlexander Motin 		return;
1395ee7f31c0SAlexander Motin 	}
1396ee7f31c0SAlexander Motin 
1397ee7f31c0SAlexander Motin 	beio->io_len = 0;
1398ee7f31c0SAlexander Motin 	beio->io_offset = -1;
1399ee7f31c0SAlexander Motin 	beio->bio_cmd = BIO_DELETE;
1400ee7f31c0SAlexander Motin 	beio->ds_trans_type = DEVSTAT_FREE;
140166df9136SAlexander Motin 	DPRINTF("UNMAP\n");
1402ee7f31c0SAlexander Motin 	be_lun->unmap(be_lun, beio);
1403ee7f31c0SAlexander Motin }
1404ee7f31c0SAlexander Motin 
1405ee7f31c0SAlexander Motin static void
1406ef8daf3fSAlexander Motin ctl_be_block_cr_done(struct ctl_be_block_io *beio)
1407ef8daf3fSAlexander Motin {
1408ef8daf3fSAlexander Motin 	union ctl_io *io;
1409ef8daf3fSAlexander Motin 
1410ef8daf3fSAlexander Motin 	io = beio->io;
1411ef8daf3fSAlexander Motin 	ctl_free_beio(beio);
1412ef8daf3fSAlexander Motin 	ctl_config_read_done(io);
1413ef8daf3fSAlexander Motin }
1414ef8daf3fSAlexander Motin 
1415ef8daf3fSAlexander Motin static void
1416ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
1417ef8daf3fSAlexander Motin 			 union ctl_io *io)
1418ef8daf3fSAlexander Motin {
1419ef8daf3fSAlexander Motin 	struct ctl_be_block_io *beio;
1420ef8daf3fSAlexander Motin 	struct ctl_be_block_softc *softc;
1421ef8daf3fSAlexander Motin 
1422ef8daf3fSAlexander Motin 	DPRINTF("entered\n");
1423ef8daf3fSAlexander Motin 
1424ef8daf3fSAlexander Motin 	softc = be_lun->softc;
1425ef8daf3fSAlexander Motin 	beio = ctl_alloc_beio(softc);
1426ef8daf3fSAlexander Motin 	beio->io = io;
1427ef8daf3fSAlexander Motin 	beio->lun = be_lun;
1428ef8daf3fSAlexander Motin 	beio->beio_cont = ctl_be_block_cr_done;
1429ef8daf3fSAlexander Motin 	PRIV(io)->ptr = (void *)beio;
1430ef8daf3fSAlexander Motin 
1431ef8daf3fSAlexander Motin 	switch (io->scsiio.cdb[0]) {
1432ef8daf3fSAlexander Motin 	case SERVICE_ACTION_IN:		/* GET LBA STATUS */
1433ef8daf3fSAlexander Motin 		beio->bio_cmd = -1;
1434ef8daf3fSAlexander Motin 		beio->ds_trans_type = DEVSTAT_NO_DATA;
1435ef8daf3fSAlexander Motin 		beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1436ef8daf3fSAlexander Motin 		beio->io_len = 0;
1437ef8daf3fSAlexander Motin 		if (be_lun->get_lba_status)
1438ef8daf3fSAlexander Motin 			be_lun->get_lba_status(be_lun, beio);
1439ef8daf3fSAlexander Motin 		else
1440ef8daf3fSAlexander Motin 			ctl_be_block_cr_done(beio);
1441ef8daf3fSAlexander Motin 		break;
1442ef8daf3fSAlexander Motin 	default:
1443ef8daf3fSAlexander Motin 		panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1444ef8daf3fSAlexander Motin 		break;
1445ef8daf3fSAlexander Motin 	}
1446ef8daf3fSAlexander Motin }
1447ef8daf3fSAlexander Motin 
1448ef8daf3fSAlexander Motin static void
1449ee7f31c0SAlexander Motin ctl_be_block_cw_done(struct ctl_be_block_io *beio)
1450ee7f31c0SAlexander Motin {
1451ee7f31c0SAlexander Motin 	union ctl_io *io;
1452ee7f31c0SAlexander Motin 
1453ee7f31c0SAlexander Motin 	io = beio->io;
1454ee7f31c0SAlexander Motin 	ctl_free_beio(beio);
1455ee7f31c0SAlexander Motin 	ctl_config_write_done(io);
1456ee7f31c0SAlexander Motin }
1457ee7f31c0SAlexander Motin 
1458ee7f31c0SAlexander Motin static void
1459130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
1460130f4520SKenneth D. Merry 			 union ctl_io *io)
1461130f4520SKenneth D. Merry {
1462130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
1463130f4520SKenneth D. Merry 	struct ctl_be_block_softc *softc;
1464130f4520SKenneth D. Merry 
1465130f4520SKenneth D. Merry 	DPRINTF("entered\n");
1466130f4520SKenneth D. Merry 
1467130f4520SKenneth D. Merry 	softc = be_lun->softc;
1468130f4520SKenneth D. Merry 	beio = ctl_alloc_beio(softc);
1469130f4520SKenneth D. Merry 	beio->io = io;
1470130f4520SKenneth D. Merry 	beio->lun = be_lun;
1471ee7f31c0SAlexander Motin 	beio->beio_cont = ctl_be_block_cw_done;
14727d0d4342SAlexander Motin 	switch (io->scsiio.tag_type) {
14737d0d4342SAlexander Motin 	case CTL_TAG_ORDERED:
14747d0d4342SAlexander Motin 		beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
14757d0d4342SAlexander Motin 		break;
14767d0d4342SAlexander Motin 	case CTL_TAG_HEAD_OF_QUEUE:
14777d0d4342SAlexander Motin 		beio->ds_tag_type = DEVSTAT_TAG_HEAD;
14787d0d4342SAlexander Motin 		break;
14797d0d4342SAlexander Motin 	case CTL_TAG_UNTAGGED:
14807d0d4342SAlexander Motin 	case CTL_TAG_SIMPLE:
14817d0d4342SAlexander Motin 	case CTL_TAG_ACA:
14827d0d4342SAlexander Motin 	default:
14837d0d4342SAlexander Motin 		beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
14847d0d4342SAlexander Motin 		break;
14857d0d4342SAlexander Motin 	}
1486e86a4142SAlexander Motin 	PRIV(io)->ptr = (void *)beio;
1487130f4520SKenneth D. Merry 
1488130f4520SKenneth D. Merry 	switch (io->scsiio.cdb[0]) {
1489130f4520SKenneth D. Merry 	case SYNCHRONIZE_CACHE:
1490130f4520SKenneth D. Merry 	case SYNCHRONIZE_CACHE_16:
14917d0d4342SAlexander Motin 		ctl_be_block_cw_dispatch_sync(be_lun, io);
1492130f4520SKenneth D. Merry 		break;
1493ee7f31c0SAlexander Motin 	case WRITE_SAME_10:
1494ee7f31c0SAlexander Motin 	case WRITE_SAME_16:
1495ee7f31c0SAlexander Motin 		ctl_be_block_cw_dispatch_ws(be_lun, io);
1496ee7f31c0SAlexander Motin 		break;
1497ee7f31c0SAlexander Motin 	case UNMAP:
1498ee7f31c0SAlexander Motin 		ctl_be_block_cw_dispatch_unmap(be_lun, io);
1499ee7f31c0SAlexander Motin 		break;
1500130f4520SKenneth D. Merry 	default:
1501130f4520SKenneth D. Merry 		panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1502130f4520SKenneth D. Merry 		break;
1503130f4520SKenneth D. Merry 	}
1504130f4520SKenneth D. Merry }
1505130f4520SKenneth D. Merry 
150636160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, start, "uint64_t");
150736160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, start, "uint64_t");
150836160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, alloc_done, "uint64_t");
150936160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, alloc_done, "uint64_t");
1510130f4520SKenneth D. Merry 
1511130f4520SKenneth D. Merry static void
151208a7cce5SAlexander Motin ctl_be_block_next(struct ctl_be_block_io *beio)
151308a7cce5SAlexander Motin {
151408a7cce5SAlexander Motin 	struct ctl_be_block_lun *be_lun;
151508a7cce5SAlexander Motin 	union ctl_io *io;
151608a7cce5SAlexander Motin 
151708a7cce5SAlexander Motin 	io = beio->io;
151808a7cce5SAlexander Motin 	be_lun = beio->lun;
151908a7cce5SAlexander Motin 	ctl_free_beio(beio);
1520ead2f117SAlexander Motin 	if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1521ead2f117SAlexander Motin 	    ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1522ead2f117SAlexander Motin 	     (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
152311b569f7SAlexander Motin 		ctl_data_submit_done(io);
152408a7cce5SAlexander Motin 		return;
152508a7cce5SAlexander Motin 	}
152608a7cce5SAlexander Motin 
152708a7cce5SAlexander Motin 	io->io_hdr.status &= ~CTL_STATUS_MASK;
152808a7cce5SAlexander Motin 	io->io_hdr.status |= CTL_STATUS_NONE;
152908a7cce5SAlexander Motin 
153075c7a1d3SAlexander Motin 	mtx_lock(&be_lun->queue_lock);
153108a7cce5SAlexander Motin 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
153275c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->queue_lock);
153308a7cce5SAlexander Motin 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
153408a7cce5SAlexander Motin }
153508a7cce5SAlexander Motin 
153608a7cce5SAlexander Motin static void
1537130f4520SKenneth D. Merry ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
1538130f4520SKenneth D. Merry 			   union ctl_io *io)
1539130f4520SKenneth D. Merry {
15400bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1541130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
1542130f4520SKenneth D. Merry 	struct ctl_be_block_softc *softc;
154311b569f7SAlexander Motin 	struct ctl_lba_len_flags *lbalen;
1544e86a4142SAlexander Motin 	struct ctl_ptr_len_flags *bptrlen;
1545e86a4142SAlexander Motin 	uint64_t len_left, lbas;
1546130f4520SKenneth D. Merry 	int i;
1547130f4520SKenneth D. Merry 
1548130f4520SKenneth D. Merry 	softc = be_lun->softc;
1549130f4520SKenneth D. Merry 
1550130f4520SKenneth D. Merry 	DPRINTF("entered\n");
1551130f4520SKenneth D. Merry 
155211b569f7SAlexander Motin 	lbalen = ARGS(io);
155311b569f7SAlexander Motin 	if (lbalen->flags & CTL_LLF_WRITE) {
155436160958SMark Johnston 		SDT_PROBE0(cbb, , write, start);
155511b569f7SAlexander Motin 	} else {
155636160958SMark Johnston 		SDT_PROBE0(cbb, , read, start);
1557130f4520SKenneth D. Merry 	}
1558130f4520SKenneth D. Merry 
1559130f4520SKenneth D. Merry 	beio = ctl_alloc_beio(softc);
1560130f4520SKenneth D. Merry 	beio->io = io;
1561130f4520SKenneth D. Merry 	beio->lun = be_lun;
1562e86a4142SAlexander Motin 	bptrlen = PRIV(io);
1563e86a4142SAlexander Motin 	bptrlen->ptr = (void *)beio;
1564130f4520SKenneth D. Merry 
1565130f4520SKenneth D. Merry 	switch (io->scsiio.tag_type) {
1566130f4520SKenneth D. Merry 	case CTL_TAG_ORDERED:
1567130f4520SKenneth D. Merry 		beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1568130f4520SKenneth D. Merry 		break;
1569130f4520SKenneth D. Merry 	case CTL_TAG_HEAD_OF_QUEUE:
1570130f4520SKenneth D. Merry 		beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1571130f4520SKenneth D. Merry 		break;
1572130f4520SKenneth D. Merry 	case CTL_TAG_UNTAGGED:
1573130f4520SKenneth D. Merry 	case CTL_TAG_SIMPLE:
1574130f4520SKenneth D. Merry 	case CTL_TAG_ACA:
1575130f4520SKenneth D. Merry 	default:
1576130f4520SKenneth D. Merry 		beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1577130f4520SKenneth D. Merry 		break;
1578130f4520SKenneth D. Merry 	}
1579130f4520SKenneth D. Merry 
158011b569f7SAlexander Motin 	if (lbalen->flags & CTL_LLF_WRITE) {
1581130f4520SKenneth D. Merry 		beio->bio_cmd = BIO_WRITE;
1582130f4520SKenneth D. Merry 		beio->ds_trans_type = DEVSTAT_WRITE;
158311b569f7SAlexander Motin 	} else {
158411b569f7SAlexander Motin 		beio->bio_cmd = BIO_READ;
158511b569f7SAlexander Motin 		beio->ds_trans_type = DEVSTAT_READ;
1586130f4520SKenneth D. Merry 	}
1587130f4520SKenneth D. Merry 
158808a7cce5SAlexander Motin 	DPRINTF("%s at LBA %jx len %u @%ju\n",
1589130f4520SKenneth D. Merry 	       (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1590e86a4142SAlexander Motin 	       (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
159111b569f7SAlexander Motin 	if (lbalen->flags & CTL_LLF_COMPARE)
159211b569f7SAlexander Motin 		lbas = CTLBLK_HALF_IO_SIZE;
159311b569f7SAlexander Motin 	else
159411b569f7SAlexander Motin 		lbas = CTLBLK_MAX_IO_SIZE;
15950bcd4ab6SAlexander Motin 	lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize);
15960bcd4ab6SAlexander Motin 	beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize;
15970bcd4ab6SAlexander Motin 	beio->io_len = lbas * cbe_lun->blocksize;
1598e86a4142SAlexander Motin 	bptrlen->len += lbas;
1599130f4520SKenneth D. Merry 
160008a7cce5SAlexander Motin 	for (i = 0, len_left = beio->io_len; len_left > 0; i++) {
160108a7cce5SAlexander Motin 		KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)",
160208a7cce5SAlexander Motin 		    i, CTLBLK_MAX_SEGS));
1603130f4520SKenneth D. Merry 
1604130f4520SKenneth D. Merry 		/*
1605130f4520SKenneth D. Merry 		 * Setup the S/G entry for this chunk.
1606130f4520SKenneth D. Merry 		 */
160708a7cce5SAlexander Motin 		beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left);
1608130f4520SKenneth D. Merry 		beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1609130f4520SKenneth D. Merry 
1610130f4520SKenneth D. Merry 		DPRINTF("segment %d addr %p len %zd\n", i,
1611130f4520SKenneth D. Merry 			beio->sg_segs[i].addr, beio->sg_segs[i].len);
1612130f4520SKenneth D. Merry 
161311b569f7SAlexander Motin 		/* Set up second segment for compare operation. */
161411b569f7SAlexander Motin 		if (lbalen->flags & CTL_LLF_COMPARE) {
161511b569f7SAlexander Motin 			beio->sg_segs[i + CTLBLK_HALF_SEGS].len =
161611b569f7SAlexander Motin 			    beio->sg_segs[i].len;
161711b569f7SAlexander Motin 			beio->sg_segs[i + CTLBLK_HALF_SEGS].addr =
161811b569f7SAlexander Motin 			    uma_zalloc(be_lun->lun_zone, M_WAITOK);
161911b569f7SAlexander Motin 		}
162011b569f7SAlexander Motin 
1621130f4520SKenneth D. Merry 		beio->num_segs++;
1622130f4520SKenneth D. Merry 		len_left -= beio->sg_segs[i].len;
1623130f4520SKenneth D. Merry 	}
1624e86a4142SAlexander Motin 	if (bptrlen->len < lbalen->len)
162508a7cce5SAlexander Motin 		beio->beio_cont = ctl_be_block_next;
162608a7cce5SAlexander Motin 	io->scsiio.be_move_done = ctl_be_block_move_done;
162711b569f7SAlexander Motin 	/* For compare we have separate S/G lists for read and datamove. */
162811b569f7SAlexander Motin 	if (lbalen->flags & CTL_LLF_COMPARE)
162911b569f7SAlexander Motin 		io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
163011b569f7SAlexander Motin 	else
163108a7cce5SAlexander Motin 		io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
163208a7cce5SAlexander Motin 	io->scsiio.kern_data_len = beio->io_len;
163308a7cce5SAlexander Motin 	io->scsiio.kern_sg_entries = beio->num_segs;
1634b2221369SAlexander Motin 	io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
1635130f4520SKenneth D. Merry 
1636130f4520SKenneth D. Merry 	/*
1637130f4520SKenneth D. Merry 	 * For the read case, we need to read the data into our buffers and
1638130f4520SKenneth D. Merry 	 * then we can send it back to the user.  For the write case, we
1639130f4520SKenneth D. Merry 	 * need to get the data from the user first.
1640130f4520SKenneth D. Merry 	 */
1641130f4520SKenneth D. Merry 	if (beio->bio_cmd == BIO_READ) {
164236160958SMark Johnston 		SDT_PROBE0(cbb, , read, alloc_done);
1643130f4520SKenneth D. Merry 		be_lun->dispatch(be_lun, beio);
1644130f4520SKenneth D. Merry 	} else {
164536160958SMark Johnston 		SDT_PROBE0(cbb, , write, alloc_done);
1646130f4520SKenneth D. Merry #ifdef CTL_TIME_IO
1647e675024aSAlexander Motin 		getbinuptime(&io->io_hdr.dma_start_bt);
1648130f4520SKenneth D. Merry #endif
1649130f4520SKenneth D. Merry 		ctl_datamove(io);
1650130f4520SKenneth D. Merry 	}
1651130f4520SKenneth D. Merry }
1652130f4520SKenneth D. Merry 
1653130f4520SKenneth D. Merry static void
1654130f4520SKenneth D. Merry ctl_be_block_worker(void *context, int pending)
1655130f4520SKenneth D. Merry {
1656ee4ad294SAlexander Motin 	struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context;
1657ee4ad294SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1658130f4520SKenneth D. Merry 	union ctl_io *io;
1659130f4520SKenneth D. Merry 	struct ctl_be_block_io *beio;
1660130f4520SKenneth D. Merry 
1661ee4ad294SAlexander Motin 	DPRINTF("entered\n");
1662ee4ad294SAlexander Motin 	/*
1663ee4ad294SAlexander Motin 	 * Fetch and process I/Os from all queues.  If we detect LUN
1664648dfc1aSAlexander Motin 	 * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race,
1665ee4ad294SAlexander Motin 	 * so make response maximally opaque to not confuse initiator.
1666ee4ad294SAlexander Motin 	 */
1667ee4ad294SAlexander Motin 	for (;;) {
1668ee4ad294SAlexander Motin 		mtx_lock(&be_lun->queue_lock);
1669ee4ad294SAlexander Motin 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1670ee4ad294SAlexander Motin 		if (io != NULL) {
1671130f4520SKenneth D. Merry 			DPRINTF("datamove queue\n");
1672130f4520SKenneth D. Merry 			STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1673130f4520SKenneth D. Merry 				      ctl_io_hdr, links);
167475c7a1d3SAlexander Motin 			mtx_unlock(&be_lun->queue_lock);
1675e86a4142SAlexander Motin 			beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1676648dfc1aSAlexander Motin 			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1677ee4ad294SAlexander Motin 				ctl_set_busy(&io->scsiio);
1678ee4ad294SAlexander Motin 				ctl_complete_beio(beio);
1679ee4ad294SAlexander Motin 				return;
1680ee4ad294SAlexander Motin 			}
1681130f4520SKenneth D. Merry 			be_lun->dispatch(be_lun, beio);
1682130f4520SKenneth D. Merry 			continue;
1683130f4520SKenneth D. Merry 		}
1684130f4520SKenneth D. Merry 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1685130f4520SKenneth D. Merry 		if (io != NULL) {
1686130f4520SKenneth D. Merry 			DPRINTF("config write queue\n");
1687130f4520SKenneth D. Merry 			STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1688130f4520SKenneth D. Merry 				      ctl_io_hdr, links);
168975c7a1d3SAlexander Motin 			mtx_unlock(&be_lun->queue_lock);
1690648dfc1aSAlexander Motin 			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1691ee4ad294SAlexander Motin 				ctl_set_busy(&io->scsiio);
1692ee4ad294SAlexander Motin 				ctl_config_write_done(io);
1693ee4ad294SAlexander Motin 				return;
1694ee4ad294SAlexander Motin 			}
1695130f4520SKenneth D. Merry 			ctl_be_block_cw_dispatch(be_lun, io);
1696ef8daf3fSAlexander Motin 			continue;
1697ef8daf3fSAlexander Motin 		}
1698ef8daf3fSAlexander Motin 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue);
1699ef8daf3fSAlexander Motin 		if (io != NULL) {
1700ef8daf3fSAlexander Motin 			DPRINTF("config read queue\n");
1701ef8daf3fSAlexander Motin 			STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr,
1702ef8daf3fSAlexander Motin 				      ctl_io_hdr, links);
1703ef8daf3fSAlexander Motin 			mtx_unlock(&be_lun->queue_lock);
1704648dfc1aSAlexander Motin 			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1705ee4ad294SAlexander Motin 				ctl_set_busy(&io->scsiio);
1706ee4ad294SAlexander Motin 				ctl_config_read_done(io);
1707ee4ad294SAlexander Motin 				return;
1708ee4ad294SAlexander Motin 			}
1709ef8daf3fSAlexander Motin 			ctl_be_block_cr_dispatch(be_lun, io);
1710130f4520SKenneth D. Merry 			continue;
1711130f4520SKenneth D. Merry 		}
1712130f4520SKenneth D. Merry 		io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1713130f4520SKenneth D. Merry 		if (io != NULL) {
1714130f4520SKenneth D. Merry 			DPRINTF("input queue\n");
1715130f4520SKenneth D. Merry 			STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1716130f4520SKenneth D. Merry 				      ctl_io_hdr, links);
171775c7a1d3SAlexander Motin 			mtx_unlock(&be_lun->queue_lock);
1718648dfc1aSAlexander Motin 			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1719ee4ad294SAlexander Motin 				ctl_set_busy(&io->scsiio);
1720ee4ad294SAlexander Motin 				ctl_data_submit_done(io);
1721ee4ad294SAlexander Motin 				return;
1722ee4ad294SAlexander Motin 			}
1723130f4520SKenneth D. Merry 			ctl_be_block_dispatch(be_lun, io);
1724130f4520SKenneth D. Merry 			continue;
1725130f4520SKenneth D. Merry 		}
1726130f4520SKenneth D. Merry 
1727130f4520SKenneth D. Merry 		/*
1728130f4520SKenneth D. Merry 		 * If we get here, there is no work left in the queues, so
1729130f4520SKenneth D. Merry 		 * just break out and let the task queue go to sleep.
1730130f4520SKenneth D. Merry 		 */
1731ee4ad294SAlexander Motin 		mtx_unlock(&be_lun->queue_lock);
1732130f4520SKenneth D. Merry 		break;
1733130f4520SKenneth D. Merry 	}
1734130f4520SKenneth D. Merry }
1735130f4520SKenneth D. Merry 
1736130f4520SKenneth D. Merry /*
1737130f4520SKenneth D. Merry  * Entry point from CTL to the backend for I/O.  We queue everything to a
1738130f4520SKenneth D. Merry  * work thread, so this just puts the I/O on a queue and wakes up the
1739130f4520SKenneth D. Merry  * thread.
1740130f4520SKenneth D. Merry  */
1741130f4520SKenneth D. Merry static int
1742130f4520SKenneth D. Merry ctl_be_block_submit(union ctl_io *io)
1743130f4520SKenneth D. Merry {
1744130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
17450bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun;
1746130f4520SKenneth D. Merry 
1747130f4520SKenneth D. Merry 	DPRINTF("entered\n");
1748130f4520SKenneth D. Merry 
17499cbbfd2fSAlexander Motin 	cbe_lun = CTL_BACKEND_LUN(io);
17500bcd4ab6SAlexander Motin 	be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
1751130f4520SKenneth D. Merry 
1752130f4520SKenneth D. Merry 	/*
1753130f4520SKenneth D. Merry 	 * Make sure we only get SCSI I/O.
1754130f4520SKenneth D. Merry 	 */
1755130f4520SKenneth D. Merry 	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1756130f4520SKenneth D. Merry 		"%#x) encountered", io->io_hdr.io_type));
1757130f4520SKenneth D. Merry 
1758e86a4142SAlexander Motin 	PRIV(io)->len = 0;
1759e86a4142SAlexander Motin 
176075c7a1d3SAlexander Motin 	mtx_lock(&be_lun->queue_lock);
1761130f4520SKenneth D. Merry 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
176275c7a1d3SAlexander Motin 	mtx_unlock(&be_lun->queue_lock);
1763130f4520SKenneth D. Merry 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1764130f4520SKenneth D. Merry 
17659c71cd5aSAlexander Motin 	return (CTL_RETVAL_COMPLETE);
1766130f4520SKenneth D. Merry }
1767130f4520SKenneth D. Merry 
1768130f4520SKenneth D. Merry static int
1769130f4520SKenneth D. Merry ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1770130f4520SKenneth D. Merry 			int flag, struct thread *td)
1771130f4520SKenneth D. Merry {
1772130f4520SKenneth D. Merry 	struct ctl_be_block_softc *softc;
1773130f4520SKenneth D. Merry 	int error;
1774130f4520SKenneth D. Merry 
1775130f4520SKenneth D. Merry 	softc = &backend_block_softc;
1776130f4520SKenneth D. Merry 
1777130f4520SKenneth D. Merry 	error = 0;
1778130f4520SKenneth D. Merry 
1779130f4520SKenneth D. Merry 	switch (cmd) {
1780130f4520SKenneth D. Merry 	case CTL_LUN_REQ: {
1781130f4520SKenneth D. Merry 		struct ctl_lun_req *lun_req;
1782130f4520SKenneth D. Merry 
1783130f4520SKenneth D. Merry 		lun_req = (struct ctl_lun_req *)addr;
1784130f4520SKenneth D. Merry 
1785130f4520SKenneth D. Merry 		switch (lun_req->reqtype) {
1786130f4520SKenneth D. Merry 		case CTL_LUNREQ_CREATE:
1787130f4520SKenneth D. Merry 			error = ctl_be_block_create(softc, lun_req);
1788130f4520SKenneth D. Merry 			break;
1789130f4520SKenneth D. Merry 		case CTL_LUNREQ_RM:
1790130f4520SKenneth D. Merry 			error = ctl_be_block_rm(softc, lun_req);
1791130f4520SKenneth D. Merry 			break;
179281177295SEdward Tomasz Napierala 		case CTL_LUNREQ_MODIFY:
179381177295SEdward Tomasz Napierala 			error = ctl_be_block_modify(softc, lun_req);
179481177295SEdward Tomasz Napierala 			break;
1795130f4520SKenneth D. Merry 		default:
1796130f4520SKenneth D. Merry 			lun_req->status = CTL_LUN_ERROR;
1797130f4520SKenneth D. Merry 			snprintf(lun_req->error_str, sizeof(lun_req->error_str),
179819720f41SAlexander Motin 				 "invalid LUN request type %d",
1799130f4520SKenneth D. Merry 				 lun_req->reqtype);
1800130f4520SKenneth D. Merry 			break;
1801130f4520SKenneth D. Merry 		}
1802130f4520SKenneth D. Merry 		break;
1803130f4520SKenneth D. Merry 	}
1804130f4520SKenneth D. Merry 	default:
1805130f4520SKenneth D. Merry 		error = ENOTTY;
1806130f4520SKenneth D. Merry 		break;
1807130f4520SKenneth D. Merry 	}
1808130f4520SKenneth D. Merry 
1809130f4520SKenneth D. Merry 	return (error);
1810130f4520SKenneth D. Merry }
1811130f4520SKenneth D. Merry 
1812130f4520SKenneth D. Merry static int
1813130f4520SKenneth D. Merry ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1814130f4520SKenneth D. Merry {
18150bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun;
1816130f4520SKenneth D. Merry 	struct ctl_be_block_filedata *file_data;
1817130f4520SKenneth D. Merry 	struct ctl_lun_create_params *params;
181834961f40SAlexander Motin 	char			     *value;
1819130f4520SKenneth D. Merry 	struct vattr		      vattr;
182034961f40SAlexander Motin 	off_t			      ps, pss, po, pos, us, uss, uo, uos;
1821130f4520SKenneth D. Merry 	int			      error;
1822130f4520SKenneth D. Merry 
18230bcd4ab6SAlexander Motin 	cbe_lun = &be_lun->cbe_lun;
1824130f4520SKenneth D. Merry 	file_data = &be_lun->backend.file;
182519720f41SAlexander Motin 	params = &be_lun->params;
1826130f4520SKenneth D. Merry 
1827130f4520SKenneth D. Merry 	be_lun->dev_type = CTL_BE_BLOCK_FILE;
1828130f4520SKenneth D. Merry 	be_lun->dispatch = ctl_be_block_dispatch_file;
1829130f4520SKenneth D. Merry 	be_lun->lun_flush = ctl_be_block_flush_file;
1830ef8daf3fSAlexander Motin 	be_lun->get_lba_status = ctl_be_block_gls_file;
183153c146deSAlexander Motin 	be_lun->getattr = ctl_be_block_getattr_file;
18320bcd4ab6SAlexander Motin 	be_lun->unmap = NULL;
18330bcd4ab6SAlexander Motin 	cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
1834130f4520SKenneth D. Merry 
1835130f4520SKenneth D. Merry 	error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1836130f4520SKenneth D. Merry 	if (error != 0) {
1837130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
1838130f4520SKenneth D. Merry 			 "error calling VOP_GETATTR() for file %s",
1839130f4520SKenneth D. Merry 			 be_lun->dev_path);
1840130f4520SKenneth D. Merry 		return (error);
1841130f4520SKenneth D. Merry 	}
1842130f4520SKenneth D. Merry 
1843130f4520SKenneth D. Merry 	file_data->cred = crhold(curthread->td_ucred);
184481177295SEdward Tomasz Napierala 	if (params->lun_size_bytes != 0)
184581177295SEdward Tomasz Napierala 		be_lun->size_bytes = params->lun_size_bytes;
184681177295SEdward Tomasz Napierala 	else
1847130f4520SKenneth D. Merry 		be_lun->size_bytes = vattr.va_size;
1848130f4520SKenneth D. Merry 
1849130f4520SKenneth D. Merry 	/*
185020a28d6cSAlexander Motin 	 * For files we can use any logical block size.  Prefer 512 bytes
185120a28d6cSAlexander Motin 	 * for compatibility reasons.  If file's vattr.va_blocksize
185220a28d6cSAlexander Motin 	 * (preferred I/O block size) is bigger and multiple to chosen
185320a28d6cSAlexander Motin 	 * logical block size -- report it as physical block size.
1854130f4520SKenneth D. Merry 	 */
1855130f4520SKenneth D. Merry 	if (params->blocksize_bytes != 0)
18560bcd4ab6SAlexander Motin 		cbe_lun->blocksize = params->blocksize_bytes;
185791be33dcSAlexander Motin 	else if (cbe_lun->lun_type == T_CDROM)
185891be33dcSAlexander Motin 		cbe_lun->blocksize = 2048;
1859130f4520SKenneth D. Merry 	else
18600bcd4ab6SAlexander Motin 		cbe_lun->blocksize = 512;
18610bcd4ab6SAlexander Motin 	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
18620bcd4ab6SAlexander Motin 	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
18630bcd4ab6SAlexander Motin 	    0 : (be_lun->size_blocks - 1);
186434961f40SAlexander Motin 
186534961f40SAlexander Motin 	us = ps = vattr.va_blocksize;
186634961f40SAlexander Motin 	uo = po = 0;
186734961f40SAlexander Motin 
18680bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "pblocksize");
186934961f40SAlexander Motin 	if (value != NULL)
187034961f40SAlexander Motin 		ctl_expand_number(value, &ps);
18710bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
187234961f40SAlexander Motin 	if (value != NULL)
187334961f40SAlexander Motin 		ctl_expand_number(value, &po);
18740bcd4ab6SAlexander Motin 	pss = ps / cbe_lun->blocksize;
18750bcd4ab6SAlexander Motin 	pos = po / cbe_lun->blocksize;
18760bcd4ab6SAlexander Motin 	if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
18770bcd4ab6SAlexander Motin 	    ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
18780bcd4ab6SAlexander Motin 		cbe_lun->pblockexp = fls(pss) - 1;
18790bcd4ab6SAlexander Motin 		cbe_lun->pblockoff = (pss - pos) % pss;
188034961f40SAlexander Motin 	}
188134961f40SAlexander Motin 
18820bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ublocksize");
188334961f40SAlexander Motin 	if (value != NULL)
188434961f40SAlexander Motin 		ctl_expand_number(value, &us);
18850bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
188634961f40SAlexander Motin 	if (value != NULL)
188734961f40SAlexander Motin 		ctl_expand_number(value, &uo);
18880bcd4ab6SAlexander Motin 	uss = us / cbe_lun->blocksize;
18890bcd4ab6SAlexander Motin 	uos = uo / cbe_lun->blocksize;
18900bcd4ab6SAlexander Motin 	if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
18910bcd4ab6SAlexander Motin 	    ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
18920bcd4ab6SAlexander Motin 		cbe_lun->ublockexp = fls(uss) - 1;
18930bcd4ab6SAlexander Motin 		cbe_lun->ublockoff = (uss - uos) % uss;
189420a28d6cSAlexander Motin 	}
1895130f4520SKenneth D. Merry 
1896130f4520SKenneth D. Merry 	/*
1897130f4520SKenneth D. Merry 	 * Sanity check.  The media size has to be at least one
1898130f4520SKenneth D. Merry 	 * sector long.
1899130f4520SKenneth D. Merry 	 */
19000bcd4ab6SAlexander Motin 	if (be_lun->size_bytes < cbe_lun->blocksize) {
1901130f4520SKenneth D. Merry 		error = EINVAL;
1902130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
1903130f4520SKenneth D. Merry 			 "file %s size %ju < block size %u", be_lun->dev_path,
19040bcd4ab6SAlexander Motin 			 (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize);
1905130f4520SKenneth D. Merry 	}
1906cb8727e2SAlexander Motin 
19070bcd4ab6SAlexander Motin 	cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize;
1908130f4520SKenneth D. Merry 	return (error);
1909130f4520SKenneth D. Merry }
1910130f4520SKenneth D. Merry 
1911130f4520SKenneth D. Merry static int
1912130f4520SKenneth D. Merry ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1913130f4520SKenneth D. Merry {
19140bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1915130f4520SKenneth D. Merry 	struct ctl_lun_create_params *params;
19163236151eSAlexander Motin 	struct cdevsw		     *csw;
1917130f4520SKenneth D. Merry 	struct cdev		     *dev;
191834961f40SAlexander Motin 	char			     *value;
19193236151eSAlexander Motin 	int			      error, atomic, maxio, ref, unmap, tmp;
1920f6295033SAlexander Motin 	off_t			      ps, pss, po, pos, us, uss, uo, uos, otmp;
1921130f4520SKenneth D. Merry 
192219720f41SAlexander Motin 	params = &be_lun->params;
1923130f4520SKenneth D. Merry 
1924130f4520SKenneth D. Merry 	be_lun->dev_type = CTL_BE_BLOCK_DEV;
19253236151eSAlexander Motin 	csw = devvn_refthread(be_lun->vn, &dev, &ref);
19263236151eSAlexander Motin 	if (csw == NULL)
19273236151eSAlexander Motin 		return (ENXIO);
19283236151eSAlexander Motin 	if (strcmp(csw->d_name, "zvol") == 0) {
192967f586a8SAlexander Motin 		be_lun->dispatch = ctl_be_block_dispatch_zvol;
1930ef8daf3fSAlexander Motin 		be_lun->get_lba_status = ctl_be_block_gls_zvol;
1931cb8727e2SAlexander Motin 		atomic = maxio = CTLBLK_MAX_IO_SIZE;
1932cb8727e2SAlexander Motin 	} else {
193367f586a8SAlexander Motin 		be_lun->dispatch = ctl_be_block_dispatch_dev;
19340bcd4ab6SAlexander Motin 		be_lun->get_lba_status = NULL;
1935cb8727e2SAlexander Motin 		atomic = 0;
19363236151eSAlexander Motin 		maxio = dev->si_iosize_max;
1937cb8727e2SAlexander Motin 		if (maxio <= 0)
1938cb8727e2SAlexander Motin 			maxio = DFLTPHYS;
1939cb8727e2SAlexander Motin 		if (maxio > CTLBLK_MAX_IO_SIZE)
1940cb8727e2SAlexander Motin 			maxio = CTLBLK_MAX_IO_SIZE;
1941cb8727e2SAlexander Motin 	}
194267f586a8SAlexander Motin 	be_lun->lun_flush = ctl_be_block_flush_dev;
1943c3e7ba3eSAlexander Motin 	be_lun->getattr = ctl_be_block_getattr_dev;
19440bcd4ab6SAlexander Motin 	be_lun->unmap = ctl_be_block_unmap_dev;
1945130f4520SKenneth D. Merry 
19463236151eSAlexander Motin 	if (!csw->d_ioctl) {
19473236151eSAlexander Motin 		dev_relthread(dev, ref);
1948130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
19493236151eSAlexander Motin 			 "no d_ioctl for device %s!", be_lun->dev_path);
1950130f4520SKenneth D. Merry 		return (ENODEV);
1951130f4520SKenneth D. Merry 	}
1952130f4520SKenneth D. Merry 
19533236151eSAlexander Motin 	error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD,
1954130f4520SKenneth D. Merry 			       curthread);
1955130f4520SKenneth D. Merry 	if (error) {
19563236151eSAlexander Motin 		dev_relthread(dev, ref);
1957130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
195819720f41SAlexander Motin 			 "error %d returned for DIOCGSECTORSIZE ioctl "
195919720f41SAlexander Motin 			 "on %s!", error, be_lun->dev_path);
1960130f4520SKenneth D. Merry 		return (error);
1961130f4520SKenneth D. Merry 	}
1962130f4520SKenneth D. Merry 
1963130f4520SKenneth D. Merry 	/*
1964130f4520SKenneth D. Merry 	 * If the user has asked for a blocksize that is greater than the
1965130f4520SKenneth D. Merry 	 * backing device's blocksize, we can do it only if the blocksize
1966130f4520SKenneth D. Merry 	 * the user is asking for is an even multiple of the underlying
1967130f4520SKenneth D. Merry 	 * device's blocksize.
1968130f4520SKenneth D. Merry 	 */
1969a15bbf15SAlexander Motin 	if ((params->blocksize_bytes != 0) &&
1970a15bbf15SAlexander Motin 	    (params->blocksize_bytes >= tmp)) {
1971a15bbf15SAlexander Motin 		if (params->blocksize_bytes % tmp == 0) {
19720bcd4ab6SAlexander Motin 			cbe_lun->blocksize = params->blocksize_bytes;
1973130f4520SKenneth D. Merry 		} else {
19743236151eSAlexander Motin 			dev_relthread(dev, ref);
1975130f4520SKenneth D. Merry 			snprintf(req->error_str, sizeof(req->error_str),
197619720f41SAlexander Motin 				 "requested blocksize %u is not an even "
1977130f4520SKenneth D. Merry 				 "multiple of backing device blocksize %u",
1978f6295033SAlexander Motin 				 params->blocksize_bytes, tmp);
1979130f4520SKenneth D. Merry 			return (EINVAL);
1980130f4520SKenneth D. Merry 		}
1981a15bbf15SAlexander Motin 	} else if (params->blocksize_bytes != 0) {
19823236151eSAlexander Motin 		dev_relthread(dev, ref);
1983130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
198419720f41SAlexander Motin 			 "requested blocksize %u < backing device "
1985f6295033SAlexander Motin 			 "blocksize %u", params->blocksize_bytes, tmp);
1986130f4520SKenneth D. Merry 		return (EINVAL);
198791be33dcSAlexander Motin 	} else if (cbe_lun->lun_type == T_CDROM)
198891be33dcSAlexander Motin 		cbe_lun->blocksize = MAX(tmp, 2048);
198991be33dcSAlexander Motin 	else
19900bcd4ab6SAlexander Motin 		cbe_lun->blocksize = tmp;
1991130f4520SKenneth D. Merry 
19923236151eSAlexander Motin 	error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD,
1993130f4520SKenneth D. Merry 			     curthread);
1994130f4520SKenneth D. Merry 	if (error) {
19953236151eSAlexander Motin 		dev_relthread(dev, ref);
1996130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
199719720f41SAlexander Motin 			 "error %d returned for DIOCGMEDIASIZE "
199819720f41SAlexander Motin 			 " ioctl on %s!", error,
199981177295SEdward Tomasz Napierala 			 be_lun->dev_path);
2000130f4520SKenneth D. Merry 		return (error);
2001130f4520SKenneth D. Merry 	}
2002130f4520SKenneth D. Merry 
200381177295SEdward Tomasz Napierala 	if (params->lun_size_bytes != 0) {
2004f6295033SAlexander Motin 		if (params->lun_size_bytes > otmp) {
20053236151eSAlexander Motin 			dev_relthread(dev, ref);
200681177295SEdward Tomasz Napierala 			snprintf(req->error_str, sizeof(req->error_str),
200719720f41SAlexander Motin 				 "requested LUN size %ju > backing device "
200819720f41SAlexander Motin 				 "size %ju",
200981177295SEdward Tomasz Napierala 				 (uintmax_t)params->lun_size_bytes,
2010f6295033SAlexander Motin 				 (uintmax_t)otmp);
201181177295SEdward Tomasz Napierala 			return (EINVAL);
2012130f4520SKenneth D. Merry 		}
2013130f4520SKenneth D. Merry 
201481177295SEdward Tomasz Napierala 		be_lun->size_bytes = params->lun_size_bytes;
2015a15bbf15SAlexander Motin 	} else
2016f6295033SAlexander Motin 		be_lun->size_bytes = otmp;
20170bcd4ab6SAlexander Motin 	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
20180bcd4ab6SAlexander Motin 	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
20190bcd4ab6SAlexander Motin 	    0 : (be_lun->size_blocks - 1);
202081177295SEdward Tomasz Napierala 
20213236151eSAlexander Motin 	error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD,
20223236151eSAlexander Motin 	    curthread);
2023f6012722SAlexander Motin 	if (error)
2024f6012722SAlexander Motin 		ps = po = 0;
2025f6012722SAlexander Motin 	else {
20263236151eSAlexander Motin 		error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po,
20273236151eSAlexander Motin 		    FREAD, curthread);
2028f6012722SAlexander Motin 		if (error)
2029f6012722SAlexander Motin 			po = 0;
2030f6012722SAlexander Motin 	}
203134961f40SAlexander Motin 	us = ps;
203234961f40SAlexander Motin 	uo = po;
203334961f40SAlexander Motin 
20340bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "pblocksize");
203534961f40SAlexander Motin 	if (value != NULL)
203634961f40SAlexander Motin 		ctl_expand_number(value, &ps);
20370bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
203834961f40SAlexander Motin 	if (value != NULL)
203934961f40SAlexander Motin 		ctl_expand_number(value, &po);
20400bcd4ab6SAlexander Motin 	pss = ps / cbe_lun->blocksize;
20410bcd4ab6SAlexander Motin 	pos = po / cbe_lun->blocksize;
20420bcd4ab6SAlexander Motin 	if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
20430bcd4ab6SAlexander Motin 	    ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
20440bcd4ab6SAlexander Motin 		cbe_lun->pblockexp = fls(pss) - 1;
20450bcd4ab6SAlexander Motin 		cbe_lun->pblockoff = (pss - pos) % pss;
2046f6012722SAlexander Motin 	}
2047f6012722SAlexander Motin 
20480bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ublocksize");
204934961f40SAlexander Motin 	if (value != NULL)
205034961f40SAlexander Motin 		ctl_expand_number(value, &us);
20510bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
205234961f40SAlexander Motin 	if (value != NULL)
205334961f40SAlexander Motin 		ctl_expand_number(value, &uo);
20540bcd4ab6SAlexander Motin 	uss = us / cbe_lun->blocksize;
20550bcd4ab6SAlexander Motin 	uos = uo / cbe_lun->blocksize;
20560bcd4ab6SAlexander Motin 	if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
20570bcd4ab6SAlexander Motin 	    ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
20580bcd4ab6SAlexander Motin 		cbe_lun->ublockexp = fls(uss) - 1;
20590bcd4ab6SAlexander Motin 		cbe_lun->ublockoff = (uss - uos) % uss;
206034961f40SAlexander Motin 	}
206134961f40SAlexander Motin 
20620bcd4ab6SAlexander Motin 	cbe_lun->atomicblock = atomic / cbe_lun->blocksize;
20630bcd4ab6SAlexander Motin 	cbe_lun->opttxferlen = maxio / cbe_lun->blocksize;
2064fbc8d4ffSAlexander Motin 
2065fbc8d4ffSAlexander Motin 	if (be_lun->dispatch == ctl_be_block_dispatch_zvol) {
2066fbc8d4ffSAlexander Motin 		unmap = 1;
2067fbc8d4ffSAlexander Motin 	} else {
2068fbc8d4ffSAlexander Motin 		struct diocgattr_arg	arg;
2069fbc8d4ffSAlexander Motin 
2070fbc8d4ffSAlexander Motin 		strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name));
2071fbc8d4ffSAlexander Motin 		arg.len = sizeof(arg.value.i);
20723236151eSAlexander Motin 		error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
20733236151eSAlexander Motin 		    curthread);
2074fbc8d4ffSAlexander Motin 		unmap = (error == 0) ? arg.value.i : 0;
2075fbc8d4ffSAlexander Motin 	}
20760bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "unmap");
2077fbc8d4ffSAlexander Motin 	if (value != NULL)
2078fbc8d4ffSAlexander Motin 		unmap = (strcmp(value, "on") == 0);
2079fbc8d4ffSAlexander Motin 	if (unmap)
20800bcd4ab6SAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
20810bcd4ab6SAlexander Motin 	else
20820bcd4ab6SAlexander Motin 		cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
2083fbc8d4ffSAlexander Motin 
20843236151eSAlexander Motin 	dev_relthread(dev, ref);
208581177295SEdward Tomasz Napierala 	return (0);
208681177295SEdward Tomasz Napierala }
2087130f4520SKenneth D. Merry 
2088130f4520SKenneth D. Merry static int
2089130f4520SKenneth D. Merry ctl_be_block_close(struct ctl_be_block_lun *be_lun)
2090130f4520SKenneth D. Merry {
20910bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
20920bcd4ab6SAlexander Motin 	int flags;
2093130f4520SKenneth D. Merry 
20940bcd4ab6SAlexander Motin 	if (be_lun->vn) {
20950bcd4ab6SAlexander Motin 		flags = FREAD;
20960bcd4ab6SAlexander Motin 		if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0)
20970bcd4ab6SAlexander Motin 			flags |= FWRITE;
2098130f4520SKenneth D. Merry 		(void)vn_close(be_lun->vn, flags, NOCRED, curthread);
2099130f4520SKenneth D. Merry 		be_lun->vn = NULL;
2100130f4520SKenneth D. Merry 
2101130f4520SKenneth D. Merry 		switch (be_lun->dev_type) {
2102130f4520SKenneth D. Merry 		case CTL_BE_BLOCK_DEV:
2103130f4520SKenneth D. Merry 			break;
2104130f4520SKenneth D. Merry 		case CTL_BE_BLOCK_FILE:
2105130f4520SKenneth D. Merry 			if (be_lun->backend.file.cred != NULL) {
2106130f4520SKenneth D. Merry 				crfree(be_lun->backend.file.cred);
2107130f4520SKenneth D. Merry 				be_lun->backend.file.cred = NULL;
2108130f4520SKenneth D. Merry 			}
2109130f4520SKenneth D. Merry 			break;
2110130f4520SKenneth D. Merry 		case CTL_BE_BLOCK_NONE:
2111025a2301SEdward Tomasz Napierala 			break;
2112130f4520SKenneth D. Merry 		default:
21135124012aSAlexander Motin 			panic("Unexpected backend type %d", be_lun->dev_type);
2114130f4520SKenneth D. Merry 			break;
2115130f4520SKenneth D. Merry 		}
211619720f41SAlexander Motin 		be_lun->dev_type = CTL_BE_BLOCK_NONE;
2117130f4520SKenneth D. Merry 	}
2118130f4520SKenneth D. Merry 	return (0);
2119130f4520SKenneth D. Merry }
2120130f4520SKenneth D. Merry 
2121130f4520SKenneth D. Merry static int
2122648dfc1aSAlexander Motin ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
2123130f4520SKenneth D. Merry {
21240bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2125130f4520SKenneth D. Merry 	struct nameidata nd;
21260bcd4ab6SAlexander Motin 	char		*value;
21270bcd4ab6SAlexander Motin 	int		 error, flags;
2128130f4520SKenneth D. Merry 
2129130f4520SKenneth D. Merry 	error = 0;
2130130f4520SKenneth D. Merry 	if (rootvnode == NULL) {
2131130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
213219720f41SAlexander Motin 			 "Root filesystem is not mounted");
2133130f4520SKenneth D. Merry 		return (1);
2134130f4520SKenneth D. Merry 	}
21358a08cec1SMateusz Guzik 	pwd_ensure_dirs();
2136130f4520SKenneth D. Merry 
21370bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "file");
21380bcd4ab6SAlexander Motin 	if (value == NULL) {
21390bcd4ab6SAlexander Motin 		snprintf(req->error_str, sizeof(req->error_str),
21400bcd4ab6SAlexander Motin 			 "no file argument specified");
21410bcd4ab6SAlexander Motin 		return (1);
21420bcd4ab6SAlexander Motin 	}
21430bcd4ab6SAlexander Motin 	free(be_lun->dev_path, M_CTLBLK);
21440bcd4ab6SAlexander Motin 	be_lun->dev_path = strdup(value, M_CTLBLK);
21450bcd4ab6SAlexander Motin 
21460bcd4ab6SAlexander Motin 	flags = FREAD;
21470bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "readonly");
214891be33dcSAlexander Motin 	if (value != NULL) {
214991be33dcSAlexander Motin 		if (strcmp(value, "on") != 0)
215091be33dcSAlexander Motin 			flags |= FWRITE;
215191be33dcSAlexander Motin 	} else if (cbe_lun->lun_type == T_DIRECT)
21520bcd4ab6SAlexander Motin 		flags |= FWRITE;
21530bcd4ab6SAlexander Motin 
2154130f4520SKenneth D. Merry again:
2155130f4520SKenneth D. Merry 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
2156130f4520SKenneth D. Merry 	error = vn_open(&nd, &flags, 0, NULL);
21570bcd4ab6SAlexander Motin 	if ((error == EROFS || error == EACCES) && (flags & FWRITE)) {
21580bcd4ab6SAlexander Motin 		flags &= ~FWRITE;
21590bcd4ab6SAlexander Motin 		goto again;
21600bcd4ab6SAlexander Motin 	}
2161130f4520SKenneth D. Merry 	if (error) {
2162130f4520SKenneth D. Merry 		/*
2163130f4520SKenneth D. Merry 		 * This is the only reasonable guess we can make as far as
2164130f4520SKenneth D. Merry 		 * path if the user doesn't give us a fully qualified path.
2165130f4520SKenneth D. Merry 		 * If they want to specify a file, they need to specify the
2166130f4520SKenneth D. Merry 		 * full path.
2167130f4520SKenneth D. Merry 		 */
2168130f4520SKenneth D. Merry 		if (be_lun->dev_path[0] != '/') {
2169130f4520SKenneth D. Merry 			char *dev_name;
2170130f4520SKenneth D. Merry 
21710bcd4ab6SAlexander Motin 			asprintf(&dev_name, M_CTLBLK, "/dev/%s",
2172130f4520SKenneth D. Merry 				be_lun->dev_path);
2173130f4520SKenneth D. Merry 			free(be_lun->dev_path, M_CTLBLK);
2174130f4520SKenneth D. Merry 			be_lun->dev_path = dev_name;
2175130f4520SKenneth D. Merry 			goto again;
2176130f4520SKenneth D. Merry 		}
2177130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
217819720f41SAlexander Motin 		    "error opening %s: %d", be_lun->dev_path, error);
2179130f4520SKenneth D. Merry 		return (error);
2180130f4520SKenneth D. Merry 	}
21810bcd4ab6SAlexander Motin 	if (flags & FWRITE)
21820bcd4ab6SAlexander Motin 		cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY;
21830bcd4ab6SAlexander Motin 	else
21840bcd4ab6SAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
2185130f4520SKenneth D. Merry 
2186130f4520SKenneth D. Merry 	NDFREE(&nd, NDF_ONLY_PNBUF);
2187130f4520SKenneth D. Merry 	be_lun->vn = nd.ni_vp;
2188130f4520SKenneth D. Merry 
2189130f4520SKenneth D. Merry 	/* We only support disks and files. */
2190130f4520SKenneth D. Merry 	if (vn_isdisk(be_lun->vn, &error)) {
2191130f4520SKenneth D. Merry 		error = ctl_be_block_open_dev(be_lun, req);
2192130f4520SKenneth D. Merry 	} else if (be_lun->vn->v_type == VREG) {
2193130f4520SKenneth D. Merry 		error = ctl_be_block_open_file(be_lun, req);
2194130f4520SKenneth D. Merry 	} else {
2195130f4520SKenneth D. Merry 		error = EINVAL;
2196130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
2197025a2301SEdward Tomasz Napierala 			 "%s is not a disk or plain file", be_lun->dev_path);
2198130f4520SKenneth D. Merry 	}
2199130f4520SKenneth D. Merry 	VOP_UNLOCK(be_lun->vn, 0);
2200130f4520SKenneth D. Merry 
2201a15bbf15SAlexander Motin 	if (error != 0)
2202130f4520SKenneth D. Merry 		ctl_be_block_close(be_lun);
22030bcd4ab6SAlexander Motin 	cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
22040bcd4ab6SAlexander Motin 	if (be_lun->dispatch != ctl_be_block_dispatch_dev)
22050bcd4ab6SAlexander Motin 		cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
22060bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "serseq");
22070bcd4ab6SAlexander Motin 	if (value != NULL && strcmp(value, "on") == 0)
22080bcd4ab6SAlexander Motin 		cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
22090bcd4ab6SAlexander Motin 	else if (value != NULL && strcmp(value, "read") == 0)
22100bcd4ab6SAlexander Motin 		cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
22110bcd4ab6SAlexander Motin 	else if (value != NULL && strcmp(value, "off") == 0)
22120bcd4ab6SAlexander Motin 		cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2213130f4520SKenneth D. Merry 	return (0);
2214130f4520SKenneth D. Merry }
2215130f4520SKenneth D. Merry 
2216130f4520SKenneth D. Merry static int
2217130f4520SKenneth D. Merry ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2218130f4520SKenneth D. Merry {
22190bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun;
2220130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
2221130f4520SKenneth D. Merry 	struct ctl_lun_create_params *params;
222257a5db13SAlexander Motin 	char num_thread_str[16];
2223130f4520SKenneth D. Merry 	char tmpstr[32];
222457a5db13SAlexander Motin 	char *value;
2225fbc8d4ffSAlexander Motin 	int retval, num_threads;
222657a5db13SAlexander Motin 	int tmp_num_threads;
2227130f4520SKenneth D. Merry 
2228130f4520SKenneth D. Merry 	params = &req->reqdata.create;
2229130f4520SKenneth D. Merry 	retval = 0;
223019720f41SAlexander Motin 	req->status = CTL_LUN_OK;
2231130f4520SKenneth D. Merry 
2232130f4520SKenneth D. Merry 	be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
22330bcd4ab6SAlexander Motin 	cbe_lun = &be_lun->cbe_lun;
22340bcd4ab6SAlexander Motin 	cbe_lun->be_lun = be_lun;
223519720f41SAlexander Motin 	be_lun->params = req->reqdata.create;
2236130f4520SKenneth D. Merry 	be_lun->softc = softc;
2237130f4520SKenneth D. Merry 	STAILQ_INIT(&be_lun->input_queue);
2238ef8daf3fSAlexander Motin 	STAILQ_INIT(&be_lun->config_read_queue);
2239130f4520SKenneth D. Merry 	STAILQ_INIT(&be_lun->config_write_queue);
2240130f4520SKenneth D. Merry 	STAILQ_INIT(&be_lun->datamove_queue);
2241130f4520SKenneth D. Merry 	sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
224275c7a1d3SAlexander Motin 	mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF);
224375c7a1d3SAlexander Motin 	mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF);
22440bcd4ab6SAlexander Motin 	ctl_init_opts(&cbe_lun->options,
224543fb3a65SAlexander Motin 	    req->num_be_args, req->kern_be_args);
224608a7cce5SAlexander Motin 	be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG,
2247eeb94054SAlexander Motin 	    NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
2248130f4520SKenneth D. Merry 	if (be_lun->lun_zone == NULL) {
2249130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
225019720f41SAlexander Motin 			 "error allocating UMA zone");
2251130f4520SKenneth D. Merry 		goto bailout_error;
2252130f4520SKenneth D. Merry 	}
2253130f4520SKenneth D. Merry 
2254130f4520SKenneth D. Merry 	if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
22550bcd4ab6SAlexander Motin 		cbe_lun->lun_type = params->device_type;
2256130f4520SKenneth D. Merry 	else
22570bcd4ab6SAlexander Motin 		cbe_lun->lun_type = T_DIRECT;
22580bcd4ab6SAlexander Motin 	be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
22597ac58230SAlexander Motin 	cbe_lun->flags = 0;
22607ac58230SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ha_role");
22617ac58230SAlexander Motin 	if (value != NULL) {
22627ac58230SAlexander Motin 		if (strcmp(value, "primary") == 0)
22637ac58230SAlexander Motin 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
22647ac58230SAlexander Motin 	} else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
22657ac58230SAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2266130f4520SKenneth D. Merry 
226791be33dcSAlexander Motin 	if (cbe_lun->lun_type == T_DIRECT ||
226891be33dcSAlexander Motin 	    cbe_lun->lun_type == T_CDROM) {
2269a15bbf15SAlexander Motin 		be_lun->size_bytes = params->lun_size_bytes;
2270a15bbf15SAlexander Motin 		if (params->blocksize_bytes != 0)
22710bcd4ab6SAlexander Motin 			cbe_lun->blocksize = params->blocksize_bytes;
227291be33dcSAlexander Motin 		else if (cbe_lun->lun_type == T_CDROM)
227391be33dcSAlexander Motin 			cbe_lun->blocksize = 2048;
2274a15bbf15SAlexander Motin 		else
22750bcd4ab6SAlexander Motin 			cbe_lun->blocksize = 512;
22760bcd4ab6SAlexander Motin 		be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
22770bcd4ab6SAlexander Motin 		cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
22780bcd4ab6SAlexander Motin 		    0 : (be_lun->size_blocks - 1);
2279130f4520SKenneth D. Merry 
22807ac58230SAlexander Motin 		if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
22817ac58230SAlexander Motin 		    control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2282648dfc1aSAlexander Motin 			retval = ctl_be_block_open(be_lun, req);
2283130f4520SKenneth D. Merry 			if (retval != 0) {
2284130f4520SKenneth D. Merry 				retval = 0;
228519720f41SAlexander Motin 				req->status = CTL_LUN_WARNING;
2286130f4520SKenneth D. Merry 			}
22877ac58230SAlexander Motin 		}
22880bcd4ab6SAlexander Motin 		num_threads = cbb_num_threads;
2289130f4520SKenneth D. Merry 	} else {
2290130f4520SKenneth D. Merry 		num_threads = 1;
2291130f4520SKenneth D. Merry 	}
2292130f4520SKenneth D. Merry 
22930bcd4ab6SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "num_threads");
229457a5db13SAlexander Motin 	if (value != NULL) {
229557a5db13SAlexander Motin 		tmp_num_threads = strtol(value, NULL, 0);
2296130f4520SKenneth D. Merry 
2297130f4520SKenneth D. Merry 		/*
2298130f4520SKenneth D. Merry 		 * We don't let the user specify less than one
2299130f4520SKenneth D. Merry 		 * thread, but hope he's clueful enough not to
2300130f4520SKenneth D. Merry 		 * specify 1000 threads.
2301130f4520SKenneth D. Merry 		 */
2302130f4520SKenneth D. Merry 		if (tmp_num_threads < 1) {
2303130f4520SKenneth D. Merry 			snprintf(req->error_str, sizeof(req->error_str),
230419720f41SAlexander Motin 				 "invalid number of threads %s",
230519720f41SAlexander Motin 				 num_thread_str);
2306130f4520SKenneth D. Merry 			goto bailout_error;
2307130f4520SKenneth D. Merry 		}
2308130f4520SKenneth D. Merry 		num_threads = tmp_num_threads;
230957a5db13SAlexander Motin 	}
2310130f4520SKenneth D. Merry 
231119720f41SAlexander Motin 	if (be_lun->vn == NULL)
2312648dfc1aSAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2313130f4520SKenneth D. Merry 	/* Tell the user the blocksize we ended up using */
231419720f41SAlexander Motin 	params->lun_size_bytes = be_lun->size_bytes;
23150bcd4ab6SAlexander Motin 	params->blocksize_bytes = cbe_lun->blocksize;
2316130f4520SKenneth D. Merry 	if (params->flags & CTL_LUN_FLAG_ID_REQ) {
23170bcd4ab6SAlexander Motin 		cbe_lun->req_lun_id = params->req_lun_id;
23180bcd4ab6SAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ;
2319130f4520SKenneth D. Merry 	} else
23200bcd4ab6SAlexander Motin 		cbe_lun->req_lun_id = 0;
2321130f4520SKenneth D. Merry 
23220bcd4ab6SAlexander Motin 	cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown;
23230bcd4ab6SAlexander Motin 	cbe_lun->lun_config_status = ctl_be_block_lun_config_status;
23240bcd4ab6SAlexander Motin 	cbe_lun->be = &ctl_be_block_driver;
2325130f4520SKenneth D. Merry 
2326130f4520SKenneth D. Merry 	if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
2327*71cd87c6SAlan Somers 		snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%04d",
2328130f4520SKenneth D. Merry 			 softc->num_luns);
23290bcd4ab6SAlexander Motin 		strncpy((char *)cbe_lun->serial_num, tmpstr,
23300bcd4ab6SAlexander Motin 			MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr)));
2331130f4520SKenneth D. Merry 
2332130f4520SKenneth D. Merry 		/* Tell the user what we used for a serial number */
2333130f4520SKenneth D. Merry 		strncpy((char *)params->serial_num, tmpstr,
2334e7038eb7SAlexander Motin 			MIN(sizeof(params->serial_num), sizeof(tmpstr)));
2335130f4520SKenneth D. Merry 	} else {
23360bcd4ab6SAlexander Motin 		strncpy((char *)cbe_lun->serial_num, params->serial_num,
23370bcd4ab6SAlexander Motin 			MIN(sizeof(cbe_lun->serial_num),
2338130f4520SKenneth D. Merry 			sizeof(params->serial_num)));
2339130f4520SKenneth D. Merry 	}
2340130f4520SKenneth D. Merry 	if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
2341*71cd87c6SAlan Somers 		snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns);
23420bcd4ab6SAlexander Motin 		strncpy((char *)cbe_lun->device_id, tmpstr,
23430bcd4ab6SAlexander Motin 			MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr)));
2344130f4520SKenneth D. Merry 
2345130f4520SKenneth D. Merry 		/* Tell the user what we used for a device ID */
2346130f4520SKenneth D. Merry 		strncpy((char *)params->device_id, tmpstr,
2347e7038eb7SAlexander Motin 			MIN(sizeof(params->device_id), sizeof(tmpstr)));
2348130f4520SKenneth D. Merry 	} else {
23490bcd4ab6SAlexander Motin 		strncpy((char *)cbe_lun->device_id, params->device_id,
23500bcd4ab6SAlexander Motin 			MIN(sizeof(cbe_lun->device_id),
2351130f4520SKenneth D. Merry 			    sizeof(params->device_id)));
2352130f4520SKenneth D. Merry 	}
2353130f4520SKenneth D. Merry 
2354130f4520SKenneth D. Merry 	TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
2355130f4520SKenneth D. Merry 
2356130f4520SKenneth D. Merry 	be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
2357130f4520SKenneth D. Merry 	    taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
2358130f4520SKenneth D. Merry 
2359130f4520SKenneth D. Merry 	if (be_lun->io_taskqueue == NULL) {
2360130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
236119720f41SAlexander Motin 			 "unable to create taskqueue");
2362130f4520SKenneth D. Merry 		goto bailout_error;
2363130f4520SKenneth D. Merry 	}
2364130f4520SKenneth D. Merry 
2365130f4520SKenneth D. Merry 	/*
2366130f4520SKenneth D. Merry 	 * Note that we start the same number of threads by default for
2367130f4520SKenneth D. Merry 	 * both the file case and the block device case.  For the file
2368130f4520SKenneth D. Merry 	 * case, we need multiple threads to allow concurrency, because the
2369130f4520SKenneth D. Merry 	 * vnode interface is designed to be a blocking interface.  For the
2370130f4520SKenneth D. Merry 	 * block device case, ZFS zvols at least will block the caller's
2371130f4520SKenneth D. Merry 	 * context in many instances, and so we need multiple threads to
2372130f4520SKenneth D. Merry 	 * overcome that problem.  Other block devices don't need as many
2373130f4520SKenneth D. Merry 	 * threads, but they shouldn't cause too many problems.
2374130f4520SKenneth D. Merry 	 *
2375130f4520SKenneth D. Merry 	 * If the user wants to just have a single thread for a block
2376130f4520SKenneth D. Merry 	 * device, he can specify that when the LUN is created, or change
2377130f4520SKenneth D. Merry 	 * the tunable/sysctl to alter the default number of threads.
2378130f4520SKenneth D. Merry 	 */
2379130f4520SKenneth D. Merry 	retval = taskqueue_start_threads(&be_lun->io_taskqueue,
2380130f4520SKenneth D. Merry 					 /*num threads*/num_threads,
2381130f4520SKenneth D. Merry 					 /*priority*/PWAIT,
2382130f4520SKenneth D. Merry 					 /*thread name*/
2383130f4520SKenneth D. Merry 					 "%s taskq", be_lun->lunname);
2384130f4520SKenneth D. Merry 
2385130f4520SKenneth D. Merry 	if (retval != 0)
2386130f4520SKenneth D. Merry 		goto bailout_error;
2387130f4520SKenneth D. Merry 
2388130f4520SKenneth D. Merry 	be_lun->num_threads = num_threads;
2389130f4520SKenneth D. Merry 
2390130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2391130f4520SKenneth D. Merry 	softc->num_luns++;
2392130f4520SKenneth D. Merry 	STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
2393130f4520SKenneth D. Merry 
2394130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2395130f4520SKenneth D. Merry 
23960bcd4ab6SAlexander Motin 	retval = ctl_add_lun(&be_lun->cbe_lun);
2397130f4520SKenneth D. Merry 	if (retval != 0) {
2398130f4520SKenneth D. Merry 		mtx_lock(&softc->lock);
2399130f4520SKenneth D. Merry 		STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2400130f4520SKenneth D. Merry 			      links);
2401130f4520SKenneth D. Merry 		softc->num_luns--;
2402130f4520SKenneth D. Merry 		mtx_unlock(&softc->lock);
2403130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
240419720f41SAlexander Motin 			 "ctl_add_lun() returned error %d, see dmesg for "
240519720f41SAlexander Motin 			 "details", retval);
2406130f4520SKenneth D. Merry 		retval = 0;
2407130f4520SKenneth D. Merry 		goto bailout_error;
2408130f4520SKenneth D. Merry 	}
2409130f4520SKenneth D. Merry 
2410130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2411130f4520SKenneth D. Merry 
2412130f4520SKenneth D. Merry 	/*
2413130f4520SKenneth D. Merry 	 * Tell the config_status routine that we're waiting so it won't
2414130f4520SKenneth D. Merry 	 * clean up the LUN in the event of an error.
2415130f4520SKenneth D. Merry 	 */
2416130f4520SKenneth D. Merry 	be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2417130f4520SKenneth D. Merry 
2418130f4520SKenneth D. Merry 	while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
2419130f4520SKenneth D. Merry 		retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2420130f4520SKenneth D. Merry 		if (retval == EINTR)
2421130f4520SKenneth D. Merry 			break;
2422130f4520SKenneth D. Merry 	}
2423130f4520SKenneth D. Merry 	be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2424130f4520SKenneth D. Merry 
2425130f4520SKenneth D. Merry 	if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
2426130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
242719720f41SAlexander Motin 			 "LUN configuration error, see dmesg for details");
2428130f4520SKenneth D. Merry 		STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2429130f4520SKenneth D. Merry 			      links);
2430130f4520SKenneth D. Merry 		softc->num_luns--;
2431130f4520SKenneth D. Merry 		mtx_unlock(&softc->lock);
2432130f4520SKenneth D. Merry 		goto bailout_error;
2433130f4520SKenneth D. Merry 	} else {
24340bcd4ab6SAlexander Motin 		params->req_lun_id = cbe_lun->lun_id;
2435130f4520SKenneth D. Merry 	}
2436130f4520SKenneth D. Merry 
2437130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2438130f4520SKenneth D. Merry 
2439130f4520SKenneth D. Merry 	be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
24400bcd4ab6SAlexander Motin 					       cbe_lun->blocksize,
2441130f4520SKenneth D. Merry 					       DEVSTAT_ALL_SUPPORTED,
24420bcd4ab6SAlexander Motin 					       cbe_lun->lun_type
2443130f4520SKenneth D. Merry 					       | DEVSTAT_TYPE_IF_OTHER,
2444130f4520SKenneth D. Merry 					       DEVSTAT_PRIORITY_OTHER);
2445130f4520SKenneth D. Merry 
2446130f4520SKenneth D. Merry 	return (retval);
2447130f4520SKenneth D. Merry 
2448130f4520SKenneth D. Merry bailout_error:
2449130f4520SKenneth D. Merry 	req->status = CTL_LUN_ERROR;
2450130f4520SKenneth D. Merry 
24519e005bbcSAlexander Motin 	if (be_lun->io_taskqueue != NULL)
24529e005bbcSAlexander Motin 		taskqueue_free(be_lun->io_taskqueue);
2453130f4520SKenneth D. Merry 	ctl_be_block_close(be_lun);
24549e005bbcSAlexander Motin 	if (be_lun->dev_path != NULL)
2455130f4520SKenneth D. Merry 		free(be_lun->dev_path, M_CTLBLK);
24569e005bbcSAlexander Motin 	if (be_lun->lun_zone != NULL)
24579e005bbcSAlexander Motin 		uma_zdestroy(be_lun->lun_zone);
24580bcd4ab6SAlexander Motin 	ctl_free_opts(&cbe_lun->options);
245975c7a1d3SAlexander Motin 	mtx_destroy(&be_lun->queue_lock);
246075c7a1d3SAlexander Motin 	mtx_destroy(&be_lun->io_lock);
2461130f4520SKenneth D. Merry 	free(be_lun, M_CTLBLK);
2462130f4520SKenneth D. Merry 
2463130f4520SKenneth D. Merry 	return (retval);
2464130f4520SKenneth D. Merry }
2465130f4520SKenneth D. Merry 
2466130f4520SKenneth D. Merry static int
2467130f4520SKenneth D. Merry ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2468130f4520SKenneth D. Merry {
2469130f4520SKenneth D. Merry 	struct ctl_lun_rm_params *params;
2470130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
2471ee4ad294SAlexander Motin 	struct ctl_be_lun *cbe_lun;
2472130f4520SKenneth D. Merry 	int retval;
2473130f4520SKenneth D. Merry 
2474130f4520SKenneth D. Merry 	params = &req->reqdata.rm;
2475130f4520SKenneth D. Merry 
2476130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2477130f4520SKenneth D. Merry 	STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
24780bcd4ab6SAlexander Motin 		if (be_lun->cbe_lun.lun_id == params->lun_id)
2479130f4520SKenneth D. Merry 			break;
2480130f4520SKenneth D. Merry 	}
2481130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2482130f4520SKenneth D. Merry 	if (be_lun == NULL) {
2483130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
248419720f41SAlexander Motin 			 "LUN %u is not managed by the block backend",
248519720f41SAlexander Motin 			 params->lun_id);
2486130f4520SKenneth D. Merry 		goto bailout_error;
2487130f4520SKenneth D. Merry 	}
2488ee4ad294SAlexander Motin 	cbe_lun = &be_lun->cbe_lun;
2489130f4520SKenneth D. Merry 
2490ee4ad294SAlexander Motin 	retval = ctl_disable_lun(cbe_lun);
2491130f4520SKenneth D. Merry 	if (retval != 0) {
2492130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
249319720f41SAlexander Motin 			 "error %d returned from ctl_disable_lun() for "
249419720f41SAlexander Motin 			 "LUN %d", retval, params->lun_id);
2495130f4520SKenneth D. Merry 		goto bailout_error;
2496130f4520SKenneth D. Merry 	}
2497130f4520SKenneth D. Merry 
2498ee4ad294SAlexander Motin 	if (be_lun->vn != NULL) {
2499648dfc1aSAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2500648dfc1aSAlexander Motin 		ctl_lun_no_media(cbe_lun);
2501ee4ad294SAlexander Motin 		taskqueue_drain_all(be_lun->io_taskqueue);
2502ee4ad294SAlexander Motin 		ctl_be_block_close(be_lun);
2503ee4ad294SAlexander Motin 	}
2504ee4ad294SAlexander Motin 
2505ee4ad294SAlexander Motin 	retval = ctl_invalidate_lun(cbe_lun);
2506130f4520SKenneth D. Merry 	if (retval != 0) {
2507130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
250819720f41SAlexander Motin 			 "error %d returned from ctl_invalidate_lun() for "
250919720f41SAlexander Motin 			 "LUN %d", retval, params->lun_id);
2510130f4520SKenneth D. Merry 		goto bailout_error;
2511130f4520SKenneth D. Merry 	}
2512130f4520SKenneth D. Merry 
2513130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2514130f4520SKenneth D. Merry 	be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2515130f4520SKenneth D. Merry 	while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2516130f4520SKenneth D. Merry                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2517130f4520SKenneth D. Merry                 if (retval == EINTR)
2518130f4520SKenneth D. Merry                         break;
2519130f4520SKenneth D. Merry         }
2520130f4520SKenneth D. Merry 	be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2521130f4520SKenneth D. Merry 
2522130f4520SKenneth D. Merry 	if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2523130f4520SKenneth D. Merry 		snprintf(req->error_str, sizeof(req->error_str),
252419720f41SAlexander Motin 			 "interrupted waiting for LUN to be freed");
2525130f4520SKenneth D. Merry 		mtx_unlock(&softc->lock);
2526130f4520SKenneth D. Merry 		goto bailout_error;
2527130f4520SKenneth D. Merry 	}
2528130f4520SKenneth D. Merry 
2529130f4520SKenneth D. Merry 	STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
2530130f4520SKenneth D. Merry 
2531130f4520SKenneth D. Merry 	softc->num_luns--;
2532130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2533130f4520SKenneth D. Merry 
2534ee4ad294SAlexander Motin 	taskqueue_drain_all(be_lun->io_taskqueue);
2535130f4520SKenneth D. Merry 	taskqueue_free(be_lun->io_taskqueue);
2536130f4520SKenneth D. Merry 
2537130f4520SKenneth D. Merry 	if (be_lun->disk_stats != NULL)
2538130f4520SKenneth D. Merry 		devstat_remove_entry(be_lun->disk_stats);
2539130f4520SKenneth D. Merry 
2540130f4520SKenneth D. Merry 	uma_zdestroy(be_lun->lun_zone);
2541130f4520SKenneth D. Merry 
2542ee4ad294SAlexander Motin 	ctl_free_opts(&cbe_lun->options);
2543130f4520SKenneth D. Merry 	free(be_lun->dev_path, M_CTLBLK);
254475c7a1d3SAlexander Motin 	mtx_destroy(&be_lun->queue_lock);
254575c7a1d3SAlexander Motin 	mtx_destroy(&be_lun->io_lock);
2546130f4520SKenneth D. Merry 	free(be_lun, M_CTLBLK);
2547130f4520SKenneth D. Merry 
2548130f4520SKenneth D. Merry 	req->status = CTL_LUN_OK;
2549130f4520SKenneth D. Merry 	return (0);
2550130f4520SKenneth D. Merry 
2551130f4520SKenneth D. Merry bailout_error:
2552130f4520SKenneth D. Merry 	req->status = CTL_LUN_ERROR;
2553130f4520SKenneth D. Merry 	return (0);
2554130f4520SKenneth D. Merry }
2555130f4520SKenneth D. Merry 
255681177295SEdward Tomasz Napierala static int
255781177295SEdward Tomasz Napierala ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
255881177295SEdward Tomasz Napierala {
255981177295SEdward Tomasz Napierala 	struct ctl_lun_modify_params *params;
256081177295SEdward Tomasz Napierala 	struct ctl_be_block_lun *be_lun;
2561a3977beaSAlexander Motin 	struct ctl_be_lun *cbe_lun;
25627ac58230SAlexander Motin 	char *value;
256371d8e97eSAlexander Motin 	uint64_t oldsize;
25647ac58230SAlexander Motin 	int error, wasprim;
256581177295SEdward Tomasz Napierala 
256681177295SEdward Tomasz Napierala 	params = &req->reqdata.modify;
256781177295SEdward Tomasz Napierala 
256881177295SEdward Tomasz Napierala 	mtx_lock(&softc->lock);
256981177295SEdward Tomasz Napierala 	STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
25700bcd4ab6SAlexander Motin 		if (be_lun->cbe_lun.lun_id == params->lun_id)
257181177295SEdward Tomasz Napierala 			break;
257281177295SEdward Tomasz Napierala 	}
257381177295SEdward Tomasz Napierala 	mtx_unlock(&softc->lock);
257481177295SEdward Tomasz Napierala 	if (be_lun == NULL) {
257581177295SEdward Tomasz Napierala 		snprintf(req->error_str, sizeof(req->error_str),
257619720f41SAlexander Motin 			 "LUN %u is not managed by the block backend",
257719720f41SAlexander Motin 			 params->lun_id);
257881177295SEdward Tomasz Napierala 		goto bailout_error;
257981177295SEdward Tomasz Napierala 	}
2580a3977beaSAlexander Motin 	cbe_lun = &be_lun->cbe_lun;
258181177295SEdward Tomasz Napierala 
2582a3977beaSAlexander Motin 	if (params->lun_size_bytes != 0)
258319720f41SAlexander Motin 		be_lun->params.lun_size_bytes = params->lun_size_bytes;
2584a3977beaSAlexander Motin 	ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
258581177295SEdward Tomasz Napierala 
25867ac58230SAlexander Motin 	wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
25877ac58230SAlexander Motin 	value = ctl_get_opt(&cbe_lun->options, "ha_role");
25887ac58230SAlexander Motin 	if (value != NULL) {
25897ac58230SAlexander Motin 		if (strcmp(value, "primary") == 0)
25907ac58230SAlexander Motin 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
25917ac58230SAlexander Motin 		else
25927ac58230SAlexander Motin 			cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
25937ac58230SAlexander Motin 	} else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
25947ac58230SAlexander Motin 		cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
25957ac58230SAlexander Motin 	else
25967ac58230SAlexander Motin 		cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
25977ac58230SAlexander Motin 	if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) {
25987ac58230SAlexander Motin 		if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)
25997ac58230SAlexander Motin 			ctl_lun_primary(cbe_lun);
26007ac58230SAlexander Motin 		else
26017ac58230SAlexander Motin 			ctl_lun_secondary(cbe_lun);
26027ac58230SAlexander Motin 	}
26037ac58230SAlexander Motin 
26040bcd4ab6SAlexander Motin 	oldsize = be_lun->size_blocks;
26057ac58230SAlexander Motin 	if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
26067ac58230SAlexander Motin 	    control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
260719720f41SAlexander Motin 		if (be_lun->vn == NULL)
2608648dfc1aSAlexander Motin 			error = ctl_be_block_open(be_lun, req);
2609b9b4269cSAlexander Motin 		else if (vn_isdisk(be_lun->vn, &error))
26104ce7a086SAlexander Motin 			error = ctl_be_block_open_dev(be_lun, req);
26113d5cb709SAlexander Motin 		else if (be_lun->vn->v_type == VREG) {
26123d5cb709SAlexander Motin 			vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
26134ce7a086SAlexander Motin 			error = ctl_be_block_open_file(be_lun, req);
26143d5cb709SAlexander Motin 			VOP_UNLOCK(be_lun->vn, 0);
26153d5cb709SAlexander Motin 		} else
2616b9b4269cSAlexander Motin 			error = EINVAL;
2617648dfc1aSAlexander Motin 		if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) &&
26180bcd4ab6SAlexander Motin 		    be_lun->vn != NULL) {
2619648dfc1aSAlexander Motin 			cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2620648dfc1aSAlexander Motin 			ctl_lun_has_media(cbe_lun);
2621648dfc1aSAlexander Motin 		} else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 &&
2622648dfc1aSAlexander Motin 		    be_lun->vn == NULL) {
2623648dfc1aSAlexander Motin 			cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2624648dfc1aSAlexander Motin 			ctl_lun_no_media(cbe_lun);
262571d8e97eSAlexander Motin 		}
2626648dfc1aSAlexander Motin 		cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
26277ac58230SAlexander Motin 	} else {
26287ac58230SAlexander Motin 		if (be_lun->vn != NULL) {
2629648dfc1aSAlexander Motin 			cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2630648dfc1aSAlexander Motin 			ctl_lun_no_media(cbe_lun);
2631ee4ad294SAlexander Motin 			taskqueue_drain_all(be_lun->io_taskqueue);
26327ac58230SAlexander Motin 			error = ctl_be_block_close(be_lun);
26337ac58230SAlexander Motin 		} else
26347ac58230SAlexander Motin 			error = 0;
26357ac58230SAlexander Motin 	}
26367ac58230SAlexander Motin 	if (be_lun->size_blocks != oldsize)
26377ac58230SAlexander Motin 		ctl_lun_capacity_changed(cbe_lun);
263881177295SEdward Tomasz Napierala 
263981177295SEdward Tomasz Napierala 	/* Tell the user the exact size we ended up using */
264081177295SEdward Tomasz Napierala 	params->lun_size_bytes = be_lun->size_bytes;
264181177295SEdward Tomasz Napierala 
264219720f41SAlexander Motin 	req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK;
264381177295SEdward Tomasz Napierala 	return (0);
264481177295SEdward Tomasz Napierala 
264581177295SEdward Tomasz Napierala bailout_error:
264681177295SEdward Tomasz Napierala 	req->status = CTL_LUN_ERROR;
264781177295SEdward Tomasz Napierala 	return (0);
264881177295SEdward Tomasz Napierala }
264981177295SEdward Tomasz Napierala 
2650130f4520SKenneth D. Merry static void
2651130f4520SKenneth D. Merry ctl_be_block_lun_shutdown(void *be_lun)
2652130f4520SKenneth D. Merry {
265368bf823fSAlexander Motin 	struct ctl_be_block_lun *lun = be_lun;
265468bf823fSAlexander Motin 	struct ctl_be_block_softc *softc = lun->softc;
2655130f4520SKenneth D. Merry 
2656130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2657130f4520SKenneth D. Merry 	lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2658130f4520SKenneth D. Merry 	if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2659130f4520SKenneth D. Merry 		wakeup(lun);
2660130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2661130f4520SKenneth D. Merry }
2662130f4520SKenneth D. Merry 
2663130f4520SKenneth D. Merry static void
2664130f4520SKenneth D. Merry ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2665130f4520SKenneth D. Merry {
2666130f4520SKenneth D. Merry 	struct ctl_be_block_lun *lun;
2667130f4520SKenneth D. Merry 	struct ctl_be_block_softc *softc;
2668130f4520SKenneth D. Merry 
2669130f4520SKenneth D. Merry 	lun = (struct ctl_be_block_lun *)be_lun;
2670130f4520SKenneth D. Merry 	softc = lun->softc;
2671130f4520SKenneth D. Merry 
2672130f4520SKenneth D. Merry 	if (status == CTL_LUN_CONFIG_OK) {
2673130f4520SKenneth D. Merry 		mtx_lock(&softc->lock);
2674130f4520SKenneth D. Merry 		lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2675130f4520SKenneth D. Merry 		if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2676130f4520SKenneth D. Merry 			wakeup(lun);
2677130f4520SKenneth D. Merry 		mtx_unlock(&softc->lock);
2678130f4520SKenneth D. Merry 
2679130f4520SKenneth D. Merry 		/*
2680130f4520SKenneth D. Merry 		 * We successfully added the LUN, attempt to enable it.
2681130f4520SKenneth D. Merry 		 */
26820bcd4ab6SAlexander Motin 		if (ctl_enable_lun(&lun->cbe_lun) != 0) {
2683130f4520SKenneth D. Merry 			printf("%s: ctl_enable_lun() failed!\n", __func__);
26840bcd4ab6SAlexander Motin 			if (ctl_invalidate_lun(&lun->cbe_lun) != 0) {
2685130f4520SKenneth D. Merry 				printf("%s: ctl_invalidate_lun() failed!\n",
2686130f4520SKenneth D. Merry 				       __func__);
2687130f4520SKenneth D. Merry 			}
2688130f4520SKenneth D. Merry 		}
2689130f4520SKenneth D. Merry 
2690130f4520SKenneth D. Merry 		return;
2691130f4520SKenneth D. Merry 	}
2692130f4520SKenneth D. Merry 
2693130f4520SKenneth D. Merry 
2694130f4520SKenneth D. Merry 	mtx_lock(&softc->lock);
2695130f4520SKenneth D. Merry 	lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2696130f4520SKenneth D. Merry 	lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2697130f4520SKenneth D. Merry 	wakeup(lun);
2698130f4520SKenneth D. Merry 	mtx_unlock(&softc->lock);
2699130f4520SKenneth D. Merry }
2700130f4520SKenneth D. Merry 
2701130f4520SKenneth D. Merry 
2702130f4520SKenneth D. Merry static int
2703130f4520SKenneth D. Merry ctl_be_block_config_write(union ctl_io *io)
2704130f4520SKenneth D. Merry {
2705130f4520SKenneth D. Merry 	struct ctl_be_block_lun *be_lun;
27060bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun;
2707130f4520SKenneth D. Merry 	int retval;
2708130f4520SKenneth D. Merry 
2709130f4520SKenneth D. Merry 	DPRINTF("entered\n");
2710130f4520SKenneth D. Merry 
27119cbbfd2fSAlexander Motin 	cbe_lun = CTL_BACKEND_LUN(io);
27120bcd4ab6SAlexander Motin 	be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2713130f4520SKenneth D. Merry 
271467cc546dSAlexander Motin 	retval = 0;
2715130f4520SKenneth D. Merry 	switch (io->scsiio.cdb[0]) {
2716130f4520SKenneth D. Merry 	case SYNCHRONIZE_CACHE:
2717130f4520SKenneth D. Merry 	case SYNCHRONIZE_CACHE_16:
2718ee7f31c0SAlexander Motin 	case WRITE_SAME_10:
2719ee7f31c0SAlexander Motin 	case WRITE_SAME_16:
2720ee7f31c0SAlexander Motin 	case UNMAP:
2721130f4520SKenneth D. Merry 		/*
2722130f4520SKenneth D. Merry 		 * The upper level CTL code will filter out any CDBs with
2723130f4520SKenneth D. Merry 		 * the immediate bit set and return the proper error.
2724130f4520SKenneth D. Merry 		 *
2725130f4520SKenneth D. Merry 		 * We don't really need to worry about what LBA range the
2726130f4520SKenneth D. Merry 		 * user asked to be synced out.  When they issue a sync
2727130f4520SKenneth D. Merry 		 * cache command, we'll sync out the whole thing.
2728130f4520SKenneth D. Merry 		 */
272975c7a1d3SAlexander Motin 		mtx_lock(&be_lun->queue_lock);
2730130f4520SKenneth D. Merry 		STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2731130f4520SKenneth D. Merry 				   links);
273275c7a1d3SAlexander Motin 		mtx_unlock(&be_lun->queue_lock);
2733130f4520SKenneth D. Merry 		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2734130f4520SKenneth D. Merry 		break;
2735130f4520SKenneth D. Merry 	case START_STOP_UNIT: {
2736130f4520SKenneth D. Merry 		struct scsi_start_stop_unit *cdb;
2737648dfc1aSAlexander Motin 		struct ctl_lun_req req;
2738130f4520SKenneth D. Merry 
2739130f4520SKenneth D. Merry 		cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
274066b69676SAlexander Motin 		if ((cdb->how & SSS_PC_MASK) != 0) {
274166b69676SAlexander Motin 			ctl_set_success(&io->scsiio);
274266b69676SAlexander Motin 			ctl_config_write_done(io);
274366b69676SAlexander Motin 			break;
274466b69676SAlexander Motin 		}
2745648dfc1aSAlexander Motin 		if (cdb->how & SSS_START) {
2746648dfc1aSAlexander Motin 			if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) {
2747648dfc1aSAlexander Motin 				retval = ctl_be_block_open(be_lun, &req);
2748648dfc1aSAlexander Motin 				cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
2749648dfc1aSAlexander Motin 				if (retval == 0) {
2750648dfc1aSAlexander Motin 					cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2751648dfc1aSAlexander Motin 					ctl_lun_has_media(cbe_lun);
2752130f4520SKenneth D. Merry 				} else {
2753648dfc1aSAlexander Motin 					cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2754648dfc1aSAlexander Motin 					ctl_lun_no_media(cbe_lun);
2755130f4520SKenneth D. Merry 				}
2756648dfc1aSAlexander Motin 			}
2757648dfc1aSAlexander Motin 			ctl_start_lun(cbe_lun);
2758648dfc1aSAlexander Motin 		} else {
2759648dfc1aSAlexander Motin 			ctl_stop_lun(cbe_lun);
2760648dfc1aSAlexander Motin 			if (cdb->how & SSS_LOEJ) {
2761648dfc1aSAlexander Motin 				cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2762648dfc1aSAlexander Motin 				cbe_lun->flags |= CTL_LUN_FLAG_EJECTED;
2763648dfc1aSAlexander Motin 				ctl_lun_ejected(cbe_lun);
2764648dfc1aSAlexander Motin 				if (be_lun->vn != NULL)
2765648dfc1aSAlexander Motin 					ctl_be_block_close(be_lun);
2766648dfc1aSAlexander Motin 			}
2767648dfc1aSAlexander Motin 		}
2768648dfc1aSAlexander Motin 
2769648dfc1aSAlexander Motin 		ctl_set_success(&io->scsiio);
2770130f4520SKenneth D. Merry 		ctl_config_write_done(io);
2771130f4520SKenneth D. Merry 		break;
2772130f4520SKenneth D. Merry 	}
277391be33dcSAlexander Motin 	case PREVENT_ALLOW:
277491be33dcSAlexander Motin 		ctl_set_success(&io->scsiio);
277591be33dcSAlexander Motin 		ctl_config_write_done(io);
277691be33dcSAlexander Motin 		break;
2777130f4520SKenneth D. Merry 	default:
2778130f4520SKenneth D. Merry 		ctl_set_invalid_opcode(&io->scsiio);
2779130f4520SKenneth D. Merry 		ctl_config_write_done(io);
2780130f4520SKenneth D. Merry 		retval = CTL_RETVAL_COMPLETE;
2781130f4520SKenneth D. Merry 		break;
2782130f4520SKenneth D. Merry 	}
2783130f4520SKenneth D. Merry 
2784130f4520SKenneth D. Merry 	return (retval);
2785130f4520SKenneth D. Merry }
2786130f4520SKenneth D. Merry 
2787130f4520SKenneth D. Merry static int
2788130f4520SKenneth D. Merry ctl_be_block_config_read(union ctl_io *io)
2789130f4520SKenneth D. Merry {
2790ef8daf3fSAlexander Motin 	struct ctl_be_block_lun *be_lun;
27910bcd4ab6SAlexander Motin 	struct ctl_be_lun *cbe_lun;
2792ef8daf3fSAlexander Motin 	int retval = 0;
2793ef8daf3fSAlexander Motin 
2794ef8daf3fSAlexander Motin 	DPRINTF("entered\n");
2795ef8daf3fSAlexander Motin 
27969cbbfd2fSAlexander Motin 	cbe_lun = CTL_BACKEND_LUN(io);
27970bcd4ab6SAlexander Motin 	be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2798ef8daf3fSAlexander Motin 
2799ef8daf3fSAlexander Motin 	switch (io->scsiio.cdb[0]) {
2800ef8daf3fSAlexander Motin 	case SERVICE_ACTION_IN:
2801ef8daf3fSAlexander Motin 		if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) {
2802ef8daf3fSAlexander Motin 			mtx_lock(&be_lun->queue_lock);
2803ef8daf3fSAlexander Motin 			STAILQ_INSERT_TAIL(&be_lun->config_read_queue,
2804ef8daf3fSAlexander Motin 			    &io->io_hdr, links);
2805ef8daf3fSAlexander Motin 			mtx_unlock(&be_lun->queue_lock);
2806ef8daf3fSAlexander Motin 			taskqueue_enqueue(be_lun->io_taskqueue,
2807ef8daf3fSAlexander Motin 			    &be_lun->io_task);
2808ef8daf3fSAlexander Motin 			retval = CTL_RETVAL_QUEUED;
2809ef8daf3fSAlexander Motin 			break;
2810ef8daf3fSAlexander Motin 		}
2811ef8daf3fSAlexander Motin 		ctl_set_invalid_field(&io->scsiio,
2812ef8daf3fSAlexander Motin 				      /*sks_valid*/ 1,
2813ef8daf3fSAlexander Motin 				      /*command*/ 1,
2814ef8daf3fSAlexander Motin 				      /*field*/ 1,
2815ef8daf3fSAlexander Motin 				      /*bit_valid*/ 1,
2816ef8daf3fSAlexander Motin 				      /*bit*/ 4);
2817ef8daf3fSAlexander Motin 		ctl_config_read_done(io);
2818ef8daf3fSAlexander Motin 		retval = CTL_RETVAL_COMPLETE;
2819ef8daf3fSAlexander Motin 		break;
2820ef8daf3fSAlexander Motin 	default:
2821ef8daf3fSAlexander Motin 		ctl_set_invalid_opcode(&io->scsiio);
2822ef8daf3fSAlexander Motin 		ctl_config_read_done(io);
2823ef8daf3fSAlexander Motin 		retval = CTL_RETVAL_COMPLETE;
2824ef8daf3fSAlexander Motin 		break;
2825ef8daf3fSAlexander Motin 	}
2826ef8daf3fSAlexander Motin 
2827ef8daf3fSAlexander Motin 	return (retval);
2828130f4520SKenneth D. Merry }
2829130f4520SKenneth D. Merry 
2830130f4520SKenneth D. Merry static int
2831130f4520SKenneth D. Merry ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2832130f4520SKenneth D. Merry {
2833130f4520SKenneth D. Merry 	struct ctl_be_block_lun *lun;
2834130f4520SKenneth D. Merry 	int retval;
2835130f4520SKenneth D. Merry 
2836130f4520SKenneth D. Merry 	lun = (struct ctl_be_block_lun *)be_lun;
2837130f4520SKenneth D. Merry 
28382cfbcb9bSAlexander Motin 	retval = sbuf_printf(sb, "\t<num_threads>");
2839130f4520SKenneth D. Merry 	if (retval != 0)
2840130f4520SKenneth D. Merry 		goto bailout;
2841130f4520SKenneth D. Merry 	retval = sbuf_printf(sb, "%d", lun->num_threads);
2842130f4520SKenneth D. Merry 	if (retval != 0)
2843130f4520SKenneth D. Merry 		goto bailout;
28442cfbcb9bSAlexander Motin 	retval = sbuf_printf(sb, "</num_threads>\n");
2845130f4520SKenneth D. Merry 
2846130f4520SKenneth D. Merry bailout:
2847130f4520SKenneth D. Merry 	return (retval);
2848130f4520SKenneth D. Merry }
2849130f4520SKenneth D. Merry 
2850c3e7ba3eSAlexander Motin static uint64_t
2851c3e7ba3eSAlexander Motin ctl_be_block_lun_attr(void *be_lun, const char *attrname)
2852c3e7ba3eSAlexander Motin {
2853c3e7ba3eSAlexander Motin 	struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun;
2854c3e7ba3eSAlexander Motin 
2855c3e7ba3eSAlexander Motin 	if (lun->getattr == NULL)
2856c3e7ba3eSAlexander Motin 		return (UINT64_MAX);
2857c3e7ba3eSAlexander Motin 	return (lun->getattr(lun, attrname));
2858c3e7ba3eSAlexander Motin }
2859c3e7ba3eSAlexander Motin 
28600c629e28SAlexander Motin static int
2861130f4520SKenneth D. Merry ctl_be_block_init(void)
2862130f4520SKenneth D. Merry {
28630c629e28SAlexander Motin 	struct ctl_be_block_softc *softc = &backend_block_softc;
2864130f4520SKenneth D. Merry 
286575c7a1d3SAlexander Motin 	mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
28660c629e28SAlexander Motin 	softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
2867a0e36aeeSEdward Tomasz Napierala 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
2868130f4520SKenneth D. Merry 	STAILQ_INIT(&softc->lun_list);
28690c629e28SAlexander Motin 	return (0);
28700c629e28SAlexander Motin }
2871130f4520SKenneth D. Merry 
28720c629e28SAlexander Motin 
28730c629e28SAlexander Motin static int
28740c629e28SAlexander Motin ctl_be_block_shutdown(void)
28750c629e28SAlexander Motin {
28760c629e28SAlexander Motin 	struct ctl_be_block_softc *softc = &backend_block_softc;
28770c629e28SAlexander Motin 	struct ctl_be_block_lun *lun, *next_lun;
28780c629e28SAlexander Motin 
28790c629e28SAlexander Motin 	mtx_lock(&softc->lock);
28800c629e28SAlexander Motin 	STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) {
28810c629e28SAlexander Motin 		/*
28820c629e28SAlexander Motin 		 * Drop our lock here.  Since ctl_invalidate_lun() can call
28830c629e28SAlexander Motin 		 * back into us, this could potentially lead to a recursive
28840c629e28SAlexander Motin 		 * lock of the same mutex, which would cause a hang.
28850c629e28SAlexander Motin 		 */
28860c629e28SAlexander Motin 		mtx_unlock(&softc->lock);
28870c629e28SAlexander Motin 		ctl_disable_lun(&lun->cbe_lun);
28880c629e28SAlexander Motin 		ctl_invalidate_lun(&lun->cbe_lun);
28890c629e28SAlexander Motin 		mtx_lock(&softc->lock);
28900c629e28SAlexander Motin 	}
28910c629e28SAlexander Motin 	mtx_unlock(&softc->lock);
28920c629e28SAlexander Motin 
28930c629e28SAlexander Motin 	uma_zdestroy(softc->beio_zone);
28940c629e28SAlexander Motin 	mtx_destroy(&softc->lock);
28950c629e28SAlexander Motin 	return (0);
2896130f4520SKenneth D. Merry }
2897