1130f4520SKenneth D. Merry /*- 2130f4520SKenneth D. Merry * Copyright (c) 2003 Silicon Graphics International Corp. 3130f4520SKenneth D. Merry * Copyright (c) 2009-2011 Spectra Logic Corporation 481177295SEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 5648dfc1aSAlexander Motin * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org> 6130f4520SKenneth D. Merry * All rights reserved. 7130f4520SKenneth D. Merry * 881177295SEdward Tomasz Napierala * Portions of this software were developed by Edward Tomasz Napierala 981177295SEdward Tomasz Napierala * under sponsorship from the FreeBSD Foundation. 1081177295SEdward Tomasz Napierala * 11130f4520SKenneth D. Merry * Redistribution and use in source and binary forms, with or without 12130f4520SKenneth D. Merry * modification, are permitted provided that the following conditions 13130f4520SKenneth D. Merry * are met: 14130f4520SKenneth D. Merry * 1. Redistributions of source code must retain the above copyright 15130f4520SKenneth D. Merry * notice, this list of conditions, and the following disclaimer, 16130f4520SKenneth D. Merry * without modification. 17130f4520SKenneth D. Merry * 2. Redistributions in binary form must reproduce at minimum a disclaimer 18130f4520SKenneth D. Merry * substantially similar to the "NO WARRANTY" disclaimer below 19130f4520SKenneth D. Merry * ("Disclaimer") and any redistribution must be conditioned upon 20130f4520SKenneth D. Merry * including a substantially similar Disclaimer requirement for further 21130f4520SKenneth D. Merry * binary redistribution. 22130f4520SKenneth D. Merry * 23130f4520SKenneth D. Merry * NO WARRANTY 24130f4520SKenneth D. Merry * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25130f4520SKenneth D. Merry * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26130f4520SKenneth D. Merry * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 27130f4520SKenneth D. Merry * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28130f4520SKenneth D. Merry * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29130f4520SKenneth D. Merry * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30130f4520SKenneth D. Merry * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31130f4520SKenneth D. Merry * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 32130f4520SKenneth D. Merry * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 33130f4520SKenneth D. Merry * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34130f4520SKenneth D. Merry * POSSIBILITY OF SUCH DAMAGES. 35130f4520SKenneth D. Merry * 36130f4520SKenneth D. Merry * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $ 37130f4520SKenneth D. Merry */ 38130f4520SKenneth D. Merry /* 39130f4520SKenneth D. Merry * CAM Target Layer driver backend for block devices. 40130f4520SKenneth D. Merry * 41130f4520SKenneth D. Merry * Author: Ken Merry <ken@FreeBSD.org> 42130f4520SKenneth D. Merry */ 43130f4520SKenneth D. Merry #include <sys/cdefs.h> 44130f4520SKenneth D. Merry __FBSDID("$FreeBSD$"); 45130f4520SKenneth D. Merry 46130f4520SKenneth D. Merry #include <sys/param.h> 47130f4520SKenneth D. Merry #include <sys/systm.h> 48130f4520SKenneth D. Merry #include <sys/kernel.h> 49130f4520SKenneth D. Merry #include <sys/types.h> 50130f4520SKenneth D. Merry #include <sys/kthread.h> 51130f4520SKenneth D. Merry #include <sys/bio.h> 52130f4520SKenneth D. Merry #include <sys/fcntl.h> 53ee7f31c0SAlexander Motin #include <sys/limits.h> 54130f4520SKenneth D. Merry #include <sys/lock.h> 55130f4520SKenneth D. Merry #include <sys/mutex.h> 56130f4520SKenneth D. Merry #include <sys/condvar.h> 57130f4520SKenneth D. Merry #include <sys/malloc.h> 58130f4520SKenneth D. Merry #include <sys/conf.h> 59130f4520SKenneth D. Merry #include <sys/ioccom.h> 60130f4520SKenneth D. Merry #include <sys/queue.h> 61130f4520SKenneth D. Merry #include <sys/sbuf.h> 62130f4520SKenneth D. Merry #include <sys/endian.h> 63130f4520SKenneth D. Merry #include <sys/uio.h> 64130f4520SKenneth D. Merry #include <sys/buf.h> 65130f4520SKenneth D. Merry #include <sys/taskqueue.h> 66130f4520SKenneth D. Merry #include <sys/vnode.h> 67130f4520SKenneth D. Merry #include <sys/namei.h> 68130f4520SKenneth D. Merry #include <sys/mount.h> 69130f4520SKenneth D. Merry #include <sys/disk.h> 70130f4520SKenneth D. Merry #include <sys/fcntl.h> 71130f4520SKenneth D. Merry #include <sys/filedesc.h> 72ef8daf3fSAlexander Motin #include <sys/filio.h> 73130f4520SKenneth D. Merry #include <sys/proc.h> 74130f4520SKenneth D. Merry #include <sys/pcpu.h> 75130f4520SKenneth D. Merry #include <sys/module.h> 76130f4520SKenneth D. Merry #include <sys/sdt.h> 77130f4520SKenneth D. Merry #include <sys/devicestat.h> 78130f4520SKenneth D. Merry #include <sys/sysctl.h> 79130f4520SKenneth D. Merry 80130f4520SKenneth D. Merry #include <geom/geom.h> 81130f4520SKenneth D. Merry 82130f4520SKenneth D. Merry #include <cam/cam.h> 83130f4520SKenneth D. Merry #include <cam/scsi/scsi_all.h> 84130f4520SKenneth D. Merry #include <cam/scsi/scsi_da.h> 85130f4520SKenneth D. Merry #include <cam/ctl/ctl_io.h> 86130f4520SKenneth D. Merry #include <cam/ctl/ctl.h> 87130f4520SKenneth D. Merry #include <cam/ctl/ctl_backend.h> 88130f4520SKenneth D. Merry #include <cam/ctl/ctl_ioctl.h> 897ac58230SAlexander Motin #include <cam/ctl/ctl_ha.h> 90130f4520SKenneth D. Merry #include <cam/ctl/ctl_scsi_all.h> 917ac58230SAlexander Motin #include <cam/ctl/ctl_private.h> 92130f4520SKenneth D. Merry #include <cam/ctl/ctl_error.h> 93130f4520SKenneth D. Merry 94130f4520SKenneth D. Merry /* 9508a7cce5SAlexander Motin * The idea here is that we'll allocate enough S/G space to hold a 1MB 9608a7cce5SAlexander Motin * I/O. If we get an I/O larger than that, we'll split it. 97130f4520SKenneth D. Merry */ 9811b569f7SAlexander Motin #define CTLBLK_HALF_IO_SIZE (512 * 1024) 9911b569f7SAlexander Motin #define CTLBLK_MAX_IO_SIZE (CTLBLK_HALF_IO_SIZE * 2) 10008a7cce5SAlexander Motin #define CTLBLK_MAX_SEG MAXPHYS 10111b569f7SAlexander Motin #define CTLBLK_HALF_SEGS MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1) 10211b569f7SAlexander Motin #define CTLBLK_MAX_SEGS (CTLBLK_HALF_SEGS * 2) 103130f4520SKenneth D. Merry 104130f4520SKenneth D. Merry #ifdef CTLBLK_DEBUG 105130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) \ 106130f4520SKenneth D. Merry printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) 107130f4520SKenneth D. Merry #else 108130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) do {} while(0) 109130f4520SKenneth D. Merry #endif 110130f4520SKenneth D. Merry 111e86a4142SAlexander Motin #define PRIV(io) \ 112e86a4142SAlexander Motin ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND]) 11311b569f7SAlexander Motin #define ARGS(io) \ 11411b569f7SAlexander Motin ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]) 115e86a4142SAlexander Motin 116130f4520SKenneth D. Merry SDT_PROVIDER_DEFINE(cbb); 117130f4520SKenneth D. Merry 118130f4520SKenneth D. Merry typedef enum { 119130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_UNCONFIGURED = 0x01, 120130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_CONFIG_ERR = 0x02, 121130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_WAITING = 0x04, 122130f4520SKenneth D. Merry } ctl_be_block_lun_flags; 123130f4520SKenneth D. Merry 124130f4520SKenneth D. Merry typedef enum { 125130f4520SKenneth D. Merry CTL_BE_BLOCK_NONE, 126130f4520SKenneth D. Merry CTL_BE_BLOCK_DEV, 127130f4520SKenneth D. Merry CTL_BE_BLOCK_FILE 128130f4520SKenneth D. Merry } ctl_be_block_type; 129130f4520SKenneth D. Merry 130130f4520SKenneth D. Merry struct ctl_be_block_filedata { 131130f4520SKenneth D. Merry struct ucred *cred; 132130f4520SKenneth D. Merry }; 133130f4520SKenneth D. Merry 134130f4520SKenneth D. Merry union ctl_be_block_bedata { 135130f4520SKenneth D. Merry struct ctl_be_block_filedata file; 136130f4520SKenneth D. Merry }; 137130f4520SKenneth D. Merry 138130f4520SKenneth D. Merry struct ctl_be_block_io; 139130f4520SKenneth D. Merry struct ctl_be_block_lun; 140130f4520SKenneth D. Merry 141130f4520SKenneth D. Merry typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun, 142130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 143c3e7ba3eSAlexander Motin typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun, 144c3e7ba3eSAlexander Motin const char *attrname); 145130f4520SKenneth D. Merry 146130f4520SKenneth D. Merry /* 147130f4520SKenneth D. Merry * Backend LUN structure. There is a 1:1 mapping between a block device 148130f4520SKenneth D. Merry * and a backend block LUN, and between a backend block LUN and a CTL LUN. 149130f4520SKenneth D. Merry */ 150130f4520SKenneth D. Merry struct ctl_be_block_lun { 15119720f41SAlexander Motin struct ctl_lun_create_params params; 152130f4520SKenneth D. Merry char lunname[32]; 153130f4520SKenneth D. Merry char *dev_path; 154130f4520SKenneth D. Merry ctl_be_block_type dev_type; 155130f4520SKenneth D. Merry struct vnode *vn; 156130f4520SKenneth D. Merry union ctl_be_block_bedata backend; 157130f4520SKenneth D. Merry cbb_dispatch_t dispatch; 158130f4520SKenneth D. Merry cbb_dispatch_t lun_flush; 159ee7f31c0SAlexander Motin cbb_dispatch_t unmap; 160ef8daf3fSAlexander Motin cbb_dispatch_t get_lba_status; 161c3e7ba3eSAlexander Motin cbb_getattr_t getattr; 162130f4520SKenneth D. Merry uma_zone_t lun_zone; 163130f4520SKenneth D. Merry uint64_t size_blocks; 164130f4520SKenneth D. Merry uint64_t size_bytes; 165130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 166130f4520SKenneth D. Merry struct devstat *disk_stats; 167130f4520SKenneth D. Merry ctl_be_block_lun_flags flags; 168130f4520SKenneth D. Merry STAILQ_ENTRY(ctl_be_block_lun) links; 1690bcd4ab6SAlexander Motin struct ctl_be_lun cbe_lun; 170130f4520SKenneth D. Merry struct taskqueue *io_taskqueue; 171130f4520SKenneth D. Merry struct task io_task; 172130f4520SKenneth D. Merry int num_threads; 173130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) input_queue; 174ef8daf3fSAlexander Motin STAILQ_HEAD(, ctl_io_hdr) config_read_queue; 175130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) config_write_queue; 176130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 17775c7a1d3SAlexander Motin struct mtx_padalign io_lock; 17875c7a1d3SAlexander Motin struct mtx_padalign queue_lock; 179130f4520SKenneth D. Merry }; 180130f4520SKenneth D. Merry 181130f4520SKenneth D. Merry /* 182130f4520SKenneth D. Merry * Overall softc structure for the block backend module. 183130f4520SKenneth D. Merry */ 184130f4520SKenneth D. Merry struct ctl_be_block_softc { 185130f4520SKenneth D. Merry struct mtx lock; 1860c629e28SAlexander Motin uma_zone_t beio_zone; 187130f4520SKenneth D. Merry int num_luns; 188130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_be_block_lun) lun_list; 189130f4520SKenneth D. Merry }; 190130f4520SKenneth D. Merry 191130f4520SKenneth D. Merry static struct ctl_be_block_softc backend_block_softc; 192130f4520SKenneth D. Merry 193130f4520SKenneth D. Merry /* 194130f4520SKenneth D. Merry * Per-I/O information. 195130f4520SKenneth D. Merry */ 196130f4520SKenneth D. Merry struct ctl_be_block_io { 197130f4520SKenneth D. Merry union ctl_io *io; 198130f4520SKenneth D. Merry struct ctl_sg_entry sg_segs[CTLBLK_MAX_SEGS]; 199130f4520SKenneth D. Merry struct iovec xiovecs[CTLBLK_MAX_SEGS]; 200130f4520SKenneth D. Merry int bio_cmd; 201130f4520SKenneth D. Merry int num_segs; 202130f4520SKenneth D. Merry int num_bios_sent; 203130f4520SKenneth D. Merry int num_bios_done; 204130f4520SKenneth D. Merry int send_complete; 2051f0694a6SAlexander Motin int first_error; 2061f0694a6SAlexander Motin uint64_t first_error_offset; 207130f4520SKenneth D. Merry struct bintime ds_t0; 208130f4520SKenneth D. Merry devstat_tag_type ds_tag_type; 209130f4520SKenneth D. Merry devstat_trans_flags ds_trans_type; 210130f4520SKenneth D. Merry uint64_t io_len; 211130f4520SKenneth D. Merry uint64_t io_offset; 2127d0d4342SAlexander Motin int io_arg; 213130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 214130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 215ee7f31c0SAlexander Motin void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */ 216130f4520SKenneth D. Merry }; 217130f4520SKenneth D. Merry 2187ac58230SAlexander Motin extern struct ctl_softc *control_softc; 2197ac58230SAlexander Motin 220130f4520SKenneth D. Merry static int cbb_num_threads = 14; 221130f4520SKenneth D. Merry SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0, 222130f4520SKenneth D. Merry "CAM Target Layer Block Backend"); 223af3b2549SHans Petter Selasky SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, 224130f4520SKenneth D. Merry &cbb_num_threads, 0, "Number of threads per backing file"); 225130f4520SKenneth D. Merry 226130f4520SKenneth D. Merry static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); 227130f4520SKenneth D. Merry static void ctl_free_beio(struct ctl_be_block_io *beio); 228130f4520SKenneth D. Merry static void ctl_complete_beio(struct ctl_be_block_io *beio); 229130f4520SKenneth D. Merry static int ctl_be_block_move_done(union ctl_io *io); 230130f4520SKenneth D. Merry static void ctl_be_block_biodone(struct bio *bio); 231130f4520SKenneth D. Merry static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 232130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 233130f4520SKenneth D. Merry static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 234130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 235ef8daf3fSAlexander Motin static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 236ef8daf3fSAlexander Motin struct ctl_be_block_io *beio); 23753c146deSAlexander Motin static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, 23853c146deSAlexander Motin const char *attrname); 239130f4520SKenneth D. Merry static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 240130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 241ee7f31c0SAlexander Motin static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 242ee7f31c0SAlexander Motin struct ctl_be_block_io *beio); 243130f4520SKenneth D. Merry static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 244130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 245c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, 246c3e7ba3eSAlexander Motin const char *attrname); 247ef8daf3fSAlexander Motin static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 248ef8daf3fSAlexander Motin union ctl_io *io); 249130f4520SKenneth D. Merry static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 250130f4520SKenneth D. Merry union ctl_io *io); 251130f4520SKenneth D. Merry static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 252130f4520SKenneth D. Merry union ctl_io *io); 253130f4520SKenneth D. Merry static void ctl_be_block_worker(void *context, int pending); 254130f4520SKenneth D. Merry static int ctl_be_block_submit(union ctl_io *io); 255130f4520SKenneth D. Merry static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 256130f4520SKenneth D. Merry int flag, struct thread *td); 257130f4520SKenneth D. Merry static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, 258130f4520SKenneth D. Merry struct ctl_lun_req *req); 259130f4520SKenneth D. Merry static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, 260130f4520SKenneth D. Merry struct ctl_lun_req *req); 261130f4520SKenneth D. Merry static int ctl_be_block_close(struct ctl_be_block_lun *be_lun); 262648dfc1aSAlexander Motin static int ctl_be_block_open(struct ctl_be_block_lun *be_lun, 263130f4520SKenneth D. Merry struct ctl_lun_req *req); 264130f4520SKenneth D. Merry static int ctl_be_block_create(struct ctl_be_block_softc *softc, 265130f4520SKenneth D. Merry struct ctl_lun_req *req); 266130f4520SKenneth D. Merry static int ctl_be_block_rm(struct ctl_be_block_softc *softc, 267130f4520SKenneth D. Merry struct ctl_lun_req *req); 26881177295SEdward Tomasz Napierala static int ctl_be_block_modify(struct ctl_be_block_softc *softc, 26981177295SEdward Tomasz Napierala struct ctl_lun_req *req); 270130f4520SKenneth D. Merry static void ctl_be_block_lun_shutdown(void *be_lun); 271130f4520SKenneth D. Merry static void ctl_be_block_lun_config_status(void *be_lun, 272130f4520SKenneth D. Merry ctl_lun_config_status status); 273130f4520SKenneth D. Merry static int ctl_be_block_config_write(union ctl_io *io); 274130f4520SKenneth D. Merry static int ctl_be_block_config_read(union ctl_io *io); 275130f4520SKenneth D. Merry static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); 276c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); 2770c629e28SAlexander Motin static int ctl_be_block_init(void); 2780c629e28SAlexander Motin static int ctl_be_block_shutdown(void); 279130f4520SKenneth D. Merry 280130f4520SKenneth D. Merry static struct ctl_backend_driver ctl_be_block_driver = 281130f4520SKenneth D. Merry { 2822a2443d8SKenneth D. Merry .name = "block", 2832a2443d8SKenneth D. Merry .flags = CTL_BE_FLAG_HAS_CONFIG, 2842a2443d8SKenneth D. Merry .init = ctl_be_block_init, 2850c629e28SAlexander Motin .shutdown = ctl_be_block_shutdown, 2862a2443d8SKenneth D. Merry .data_submit = ctl_be_block_submit, 2872a2443d8SKenneth D. Merry .data_move_done = ctl_be_block_move_done, 2882a2443d8SKenneth D. Merry .config_read = ctl_be_block_config_read, 2892a2443d8SKenneth D. Merry .config_write = ctl_be_block_config_write, 2902a2443d8SKenneth D. Merry .ioctl = ctl_be_block_ioctl, 291c3e7ba3eSAlexander Motin .lun_info = ctl_be_block_lun_info, 292c3e7ba3eSAlexander Motin .lun_attr = ctl_be_block_lun_attr 293130f4520SKenneth D. Merry }; 294130f4520SKenneth D. Merry 295130f4520SKenneth D. Merry MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); 296130f4520SKenneth D. Merry CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); 297130f4520SKenneth D. Merry 298130f4520SKenneth D. Merry static struct ctl_be_block_io * 299130f4520SKenneth D. Merry ctl_alloc_beio(struct ctl_be_block_softc *softc) 300130f4520SKenneth D. Merry { 301130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 302130f4520SKenneth D. Merry 3030c629e28SAlexander Motin beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO); 304130f4520SKenneth D. Merry beio->softc = softc; 305130f4520SKenneth D. Merry return (beio); 306130f4520SKenneth D. Merry } 307130f4520SKenneth D. Merry 308130f4520SKenneth D. Merry static void 309130f4520SKenneth D. Merry ctl_free_beio(struct ctl_be_block_io *beio) 310130f4520SKenneth D. Merry { 311130f4520SKenneth D. Merry int duplicate_free; 312130f4520SKenneth D. Merry int i; 313130f4520SKenneth D. Merry 314130f4520SKenneth D. Merry duplicate_free = 0; 315130f4520SKenneth D. Merry 316130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 317130f4520SKenneth D. Merry if (beio->sg_segs[i].addr == NULL) 318130f4520SKenneth D. Merry duplicate_free++; 319130f4520SKenneth D. Merry 320130f4520SKenneth D. Merry uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr); 321130f4520SKenneth D. Merry beio->sg_segs[i].addr = NULL; 32211b569f7SAlexander Motin 32311b569f7SAlexander Motin /* For compare we had two equal S/G lists. */ 32411b569f7SAlexander Motin if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) { 32511b569f7SAlexander Motin uma_zfree(beio->lun->lun_zone, 32611b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr); 32711b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL; 32811b569f7SAlexander Motin } 329130f4520SKenneth D. Merry } 330130f4520SKenneth D. Merry 331130f4520SKenneth D. Merry if (duplicate_free > 0) { 332130f4520SKenneth D. Merry printf("%s: %d duplicate frees out of %d segments\n", __func__, 333130f4520SKenneth D. Merry duplicate_free, beio->num_segs); 334130f4520SKenneth D. Merry } 335a0e36aeeSEdward Tomasz Napierala 3360c629e28SAlexander Motin uma_zfree(beio->softc->beio_zone, beio); 337130f4520SKenneth D. Merry } 338130f4520SKenneth D. Merry 339130f4520SKenneth D. Merry static void 340130f4520SKenneth D. Merry ctl_complete_beio(struct ctl_be_block_io *beio) 341130f4520SKenneth D. Merry { 34275c7a1d3SAlexander Motin union ctl_io *io = beio->io; 343130f4520SKenneth D. Merry 344ee7f31c0SAlexander Motin if (beio->beio_cont != NULL) { 345ee7f31c0SAlexander Motin beio->beio_cont(beio); 346ee7f31c0SAlexander Motin } else { 347130f4520SKenneth D. Merry ctl_free_beio(beio); 34811b569f7SAlexander Motin ctl_data_submit_done(io); 349130f4520SKenneth D. Merry } 350ee7f31c0SAlexander Motin } 351130f4520SKenneth D. Merry 352d6043e46SAlexander Motin static size_t 353d6043e46SAlexander Motin cmp(uint8_t *a, uint8_t *b, size_t size) 354d6043e46SAlexander Motin { 355d6043e46SAlexander Motin size_t i; 356d6043e46SAlexander Motin 357d6043e46SAlexander Motin for (i = 0; i < size; i++) { 358d6043e46SAlexander Motin if (a[i] != b[i]) 359d6043e46SAlexander Motin break; 360d6043e46SAlexander Motin } 361d6043e46SAlexander Motin return (i); 362d6043e46SAlexander Motin } 363d6043e46SAlexander Motin 364d6043e46SAlexander Motin static void 365d6043e46SAlexander Motin ctl_be_block_compare(union ctl_io *io) 366d6043e46SAlexander Motin { 367d6043e46SAlexander Motin struct ctl_be_block_io *beio; 368d6043e46SAlexander Motin uint64_t off, res; 369d6043e46SAlexander Motin int i; 370d6043e46SAlexander Motin uint8_t info[8]; 371d6043e46SAlexander Motin 372d6043e46SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 373d6043e46SAlexander Motin off = 0; 374d6043e46SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 375d6043e46SAlexander Motin res = cmp(beio->sg_segs[i].addr, 376d6043e46SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, 377d6043e46SAlexander Motin beio->sg_segs[i].len); 378d6043e46SAlexander Motin off += res; 379d6043e46SAlexander Motin if (res < beio->sg_segs[i].len) 380d6043e46SAlexander Motin break; 381d6043e46SAlexander Motin } 382d6043e46SAlexander Motin if (i < beio->num_segs) { 383d6043e46SAlexander Motin scsi_u64to8b(off, info); 384d6043e46SAlexander Motin ctl_set_sense(&io->scsiio, /*current_error*/ 1, 385d6043e46SAlexander Motin /*sense_key*/ SSD_KEY_MISCOMPARE, 386d6043e46SAlexander Motin /*asc*/ 0x1D, /*ascq*/ 0x00, 387d6043e46SAlexander Motin /*type*/ SSD_ELEM_INFO, 388d6043e46SAlexander Motin /*size*/ sizeof(info), /*data*/ &info, 389d6043e46SAlexander Motin /*type*/ SSD_ELEM_NONE); 390d6043e46SAlexander Motin } else 391d6043e46SAlexander Motin ctl_set_success(&io->scsiio); 392d6043e46SAlexander Motin } 393d6043e46SAlexander Motin 394130f4520SKenneth D. Merry static int 395130f4520SKenneth D. Merry ctl_be_block_move_done(union ctl_io *io) 396130f4520SKenneth D. Merry { 397130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 398130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 39911b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 400130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 401130f4520SKenneth D. Merry struct bintime cur_bt; 402130f4520SKenneth D. Merry #endif 403130f4520SKenneth D. Merry 404e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 405130f4520SKenneth D. Merry be_lun = beio->lun; 406130f4520SKenneth D. Merry 407130f4520SKenneth D. Merry DPRINTF("entered\n"); 408130f4520SKenneth D. Merry 409130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 410e675024aSAlexander Motin getbinuptime(&cur_bt); 411130f4520SKenneth D. Merry bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); 412130f4520SKenneth D. Merry bintime_add(&io->io_hdr.dma_bt, &cur_bt); 413130f4520SKenneth D. Merry #endif 414e675024aSAlexander Motin io->io_hdr.num_dmas++; 41511b569f7SAlexander Motin io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; 416130f4520SKenneth D. Merry 417130f4520SKenneth D. Merry /* 418130f4520SKenneth D. Merry * We set status at this point for read commands, and write 419130f4520SKenneth D. Merry * commands with errors. 420130f4520SKenneth D. Merry */ 421f7241cceSAlexander Motin if (io->io_hdr.flags & CTL_FLAG_ABORT) { 422f7241cceSAlexander Motin ; 423eb6ac6f9SAlexander Motin } else if ((io->io_hdr.port_status != 0) && 424eb6ac6f9SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 425eb6ac6f9SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 426eb6ac6f9SAlexander Motin ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, 427eb6ac6f9SAlexander Motin /*retry_count*/ io->io_hdr.port_status); 428eb6ac6f9SAlexander Motin } else if (io->scsiio.kern_data_resid != 0 && 429eb6ac6f9SAlexander Motin (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && 430eb6ac6f9SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 431eb6ac6f9SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 432eb6ac6f9SAlexander Motin ctl_set_invalid_field_ciu(&io->scsiio); 433f7241cceSAlexander Motin } else if ((io->io_hdr.port_status == 0) && 43411b569f7SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { 43511b569f7SAlexander Motin lbalen = ARGS(beio->io); 43611b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_READ) { 437130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 43811b569f7SAlexander Motin } else if (lbalen->flags & CTL_LLF_COMPARE) { 43911b569f7SAlexander Motin /* We have two data blocks ready for comparison. */ 440d6043e46SAlexander Motin ctl_be_block_compare(io); 44111b569f7SAlexander Motin } 442130f4520SKenneth D. Merry } 443130f4520SKenneth D. Merry 444130f4520SKenneth D. Merry /* 445130f4520SKenneth D. Merry * If this is a read, or a write with errors, it is done. 446130f4520SKenneth D. Merry */ 447130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_READ) 448130f4520SKenneth D. Merry || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) 449130f4520SKenneth D. Merry || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { 450130f4520SKenneth D. Merry ctl_complete_beio(beio); 451130f4520SKenneth D. Merry return (0); 452130f4520SKenneth D. Merry } 453130f4520SKenneth D. Merry 454130f4520SKenneth D. Merry /* 455130f4520SKenneth D. Merry * At this point, we have a write and the DMA completed 456130f4520SKenneth D. Merry * successfully. We now have to queue it to the task queue to 457130f4520SKenneth D. Merry * execute the backend I/O. That is because we do blocking 458130f4520SKenneth D. Merry * memory allocations, and in the file backing case, blocking I/O. 459130f4520SKenneth D. Merry * This move done routine is generally called in the SIM's 460130f4520SKenneth D. Merry * interrupt context, and therefore we cannot block. 461130f4520SKenneth D. Merry */ 46275c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 463130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); 46475c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 465130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 466130f4520SKenneth D. Merry 467130f4520SKenneth D. Merry return (0); 468130f4520SKenneth D. Merry } 469130f4520SKenneth D. Merry 470130f4520SKenneth D. Merry static void 471130f4520SKenneth D. Merry ctl_be_block_biodone(struct bio *bio) 472130f4520SKenneth D. Merry { 473130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 474130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 475130f4520SKenneth D. Merry union ctl_io *io; 476e0c2f975SAlexander Motin int error; 477130f4520SKenneth D. Merry 478130f4520SKenneth D. Merry beio = bio->bio_caller1; 479130f4520SKenneth D. Merry be_lun = beio->lun; 480130f4520SKenneth D. Merry io = beio->io; 481130f4520SKenneth D. Merry 482130f4520SKenneth D. Merry DPRINTF("entered\n"); 483130f4520SKenneth D. Merry 484e0c2f975SAlexander Motin error = bio->bio_error; 48575c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 4861f0694a6SAlexander Motin if (error != 0 && 4871f0694a6SAlexander Motin (beio->first_error == 0 || 4881f0694a6SAlexander Motin bio->bio_offset < beio->first_error_offset)) { 4891f0694a6SAlexander Motin beio->first_error = error; 4901f0694a6SAlexander Motin beio->first_error_offset = bio->bio_offset; 4911f0694a6SAlexander Motin } 492130f4520SKenneth D. Merry 493130f4520SKenneth D. Merry beio->num_bios_done++; 494130f4520SKenneth D. Merry 495130f4520SKenneth D. Merry /* 496130f4520SKenneth D. Merry * XXX KDM will this cause WITNESS to complain? Holding a lock 497130f4520SKenneth D. Merry * during the free might cause it to complain. 498130f4520SKenneth D. Merry */ 499130f4520SKenneth D. Merry g_destroy_bio(bio); 500130f4520SKenneth D. Merry 501130f4520SKenneth D. Merry /* 502130f4520SKenneth D. Merry * If the send complete bit isn't set, or we aren't the last I/O to 503130f4520SKenneth D. Merry * complete, then we're done. 504130f4520SKenneth D. Merry */ 505130f4520SKenneth D. Merry if ((beio->send_complete == 0) 506130f4520SKenneth D. Merry || (beio->num_bios_done < beio->num_bios_sent)) { 50775c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 508130f4520SKenneth D. Merry return; 509130f4520SKenneth D. Merry } 510130f4520SKenneth D. Merry 511130f4520SKenneth D. Merry /* 512130f4520SKenneth D. Merry * At this point, we've verified that we are the last I/O to 513130f4520SKenneth D. Merry * complete, so it's safe to drop the lock. 514130f4520SKenneth D. Merry */ 51575c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 51675c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 51775c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 51875c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 519130f4520SKenneth D. Merry 520130f4520SKenneth D. Merry /* 521130f4520SKenneth D. Merry * If there are any errors from the backing device, we fail the 522130f4520SKenneth D. Merry * entire I/O with a medium error. 523130f4520SKenneth D. Merry */ 5241f0694a6SAlexander Motin error = beio->first_error; 5251f0694a6SAlexander Motin if (error != 0) { 526e0c2f975SAlexander Motin if (error == EOPNOTSUPP) { 527e0c2f975SAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 5280631de4aSAlexander Motin } else if (error == ENOSPC || error == EDQUOT) { 5294fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 5306187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 5316187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 532e0c2f975SAlexander Motin } else if (beio->bio_cmd == BIO_FLUSH) { 533130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 534130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 535130f4520SKenneth D. Merry /*sks_valid*/ 1, 536130f4520SKenneth D. Merry /*retry_count*/ 0xbad2); 5377f7bb97aSAlexander Motin } else { 5387f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 5397f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 5407f7bb97aSAlexander Motin } 541130f4520SKenneth D. Merry ctl_complete_beio(beio); 542130f4520SKenneth D. Merry return; 543130f4520SKenneth D. Merry } 544130f4520SKenneth D. Merry 545130f4520SKenneth D. Merry /* 54611b569f7SAlexander Motin * If this is a write, a flush, a delete or verify, we're all done. 547130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 548130f4520SKenneth D. Merry */ 549130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_WRITE) 550ee7f31c0SAlexander Motin || (beio->bio_cmd == BIO_FLUSH) 55111b569f7SAlexander Motin || (beio->bio_cmd == BIO_DELETE) 55211b569f7SAlexander Motin || (ARGS(io)->flags & CTL_LLF_VERIFY)) { 553130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 554130f4520SKenneth D. Merry ctl_complete_beio(beio); 555130f4520SKenneth D. Merry } else { 556f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 55775a3108eSAlexander Motin beio->beio_cont == NULL) { 558f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 55975a3108eSAlexander Motin ctl_serseq_done(io); 56075a3108eSAlexander Motin } 561130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 562e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 563130f4520SKenneth D. Merry #endif 564130f4520SKenneth D. Merry ctl_datamove(io); 565130f4520SKenneth D. Merry } 566130f4520SKenneth D. Merry } 567130f4520SKenneth D. Merry 568130f4520SKenneth D. Merry static void 569130f4520SKenneth D. Merry ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 570130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 571130f4520SKenneth D. Merry { 57275c7a1d3SAlexander Motin union ctl_io *io = beio->io; 573130f4520SKenneth D. Merry struct mount *mountpoint; 5745050aa86SKonstantin Belousov int error, lock_flags; 575130f4520SKenneth D. Merry 576130f4520SKenneth D. Merry DPRINTF("entered\n"); 577130f4520SKenneth D. Merry 57875c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 57975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 58075c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 58175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 582130f4520SKenneth D. Merry 583130f4520SKenneth D. Merry (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 584130f4520SKenneth D. Merry 58567cc546dSAlexander Motin if (MNT_SHARED_WRITES(mountpoint) || 58667cc546dSAlexander Motin ((mountpoint == NULL) && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 587130f4520SKenneth D. Merry lock_flags = LK_SHARED; 588130f4520SKenneth D. Merry else 589130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 590130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 5917d0d4342SAlexander Motin error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT, 5927d0d4342SAlexander Motin curthread); 593130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 594130f4520SKenneth D. Merry 595130f4520SKenneth D. Merry vn_finished_write(mountpoint); 596130f4520SKenneth D. Merry 59775c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 59875c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 59975c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 60075c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 60175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 60275c7a1d3SAlexander Motin 603130f4520SKenneth D. Merry if (error == 0) 604130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 605130f4520SKenneth D. Merry else { 606130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 607130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 608130f4520SKenneth D. Merry /*sks_valid*/ 1, 609130f4520SKenneth D. Merry /*retry_count*/ 0xbad1); 610130f4520SKenneth D. Merry } 611130f4520SKenneth D. Merry 612130f4520SKenneth D. Merry ctl_complete_beio(beio); 613130f4520SKenneth D. Merry } 614130f4520SKenneth D. Merry 61536160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_start, "uint64_t"); 61636160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_start, "uint64_t"); 61736160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_done,"uint64_t"); 61836160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_done, "uint64_t"); 619130f4520SKenneth D. Merry 620130f4520SKenneth D. Merry static void 621130f4520SKenneth D. Merry ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 622130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 623130f4520SKenneth D. Merry { 624130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 625130f4520SKenneth D. Merry union ctl_io *io; 626130f4520SKenneth D. Merry struct uio xuio; 627130f4520SKenneth D. Merry struct iovec *xiovec; 62883981e31SAlexander Motin size_t s; 62983981e31SAlexander Motin int error, flags, i; 630130f4520SKenneth D. Merry 631130f4520SKenneth D. Merry DPRINTF("entered\n"); 632130f4520SKenneth D. Merry 633130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 634130f4520SKenneth D. Merry io = beio->io; 63555551d05SAlexander Motin flags = 0; 63655551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 63755551d05SAlexander Motin flags |= IO_DIRECT; 63855551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 63955551d05SAlexander Motin flags |= IO_SYNC; 640130f4520SKenneth D. Merry 64111b569f7SAlexander Motin bzero(&xuio, sizeof(xuio)); 642130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 64336160958SMark Johnston SDT_PROBE0(cbb, , read, file_start); 64411b569f7SAlexander Motin xuio.uio_rw = UIO_READ; 645130f4520SKenneth D. Merry } else { 64636160958SMark Johnston SDT_PROBE0(cbb, , write, file_start); 647130f4520SKenneth D. Merry xuio.uio_rw = UIO_WRITE; 64811b569f7SAlexander Motin } 649130f4520SKenneth D. Merry xuio.uio_offset = beio->io_offset; 650130f4520SKenneth D. Merry xuio.uio_resid = beio->io_len; 651130f4520SKenneth D. Merry xuio.uio_segflg = UIO_SYSSPACE; 652130f4520SKenneth D. Merry xuio.uio_iov = beio->xiovecs; 653130f4520SKenneth D. Merry xuio.uio_iovcnt = beio->num_segs; 654130f4520SKenneth D. Merry xuio.uio_td = curthread; 655130f4520SKenneth D. Merry 656130f4520SKenneth D. Merry for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 657130f4520SKenneth D. Merry xiovec->iov_base = beio->sg_segs[i].addr; 658130f4520SKenneth D. Merry xiovec->iov_len = beio->sg_segs[i].len; 659130f4520SKenneth D. Merry } 660130f4520SKenneth D. Merry 66175c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 66275c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 66375c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 66475c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 66575c7a1d3SAlexander Motin 666130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 667130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 668130f4520SKenneth D. Merry 669130f4520SKenneth D. Merry /* 670130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for reads. If the 671130f4520SKenneth D. Merry * DIRECTIO option is configured into the kernel, it calls 672130f4520SKenneth D. Merry * ffs_rawread(). But that only works for single-segment 673130f4520SKenneth D. Merry * uios with user space addresses. In our case, with a 674130f4520SKenneth D. Merry * kernel uio, it still reads into the buffer cache, but it 675130f4520SKenneth D. Merry * will just try to release the buffer from the cache later 676130f4520SKenneth D. Merry * on in ffs_read(). 677130f4520SKenneth D. Merry * 678130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for reads. 679130f4520SKenneth D. Merry * 680130f4520SKenneth D. Merry * UFS does not pay attention to IO_SYNC for reads. 681130f4520SKenneth D. Merry * 682130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (which translates into the 683130f4520SKenneth D. Merry * Solaris define FRSYNC for zfs_read()) for reads. It 684130f4520SKenneth D. Merry * attempts to sync the file before reading. 685130f4520SKenneth D. Merry */ 68655551d05SAlexander Motin error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); 687130f4520SKenneth D. Merry 688130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 68936160958SMark Johnston SDT_PROBE0(cbb, , read, file_done); 69083981e31SAlexander Motin if (error == 0 && xuio.uio_resid > 0) { 69183981e31SAlexander Motin /* 69283981e31SAlexander Motin * If we red less then requested (EOF), then 69383981e31SAlexander Motin * we should clean the rest of the buffer. 69483981e31SAlexander Motin */ 69583981e31SAlexander Motin s = beio->io_len - xuio.uio_resid; 69683981e31SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 69783981e31SAlexander Motin if (s >= beio->sg_segs[i].len) { 69883981e31SAlexander Motin s -= beio->sg_segs[i].len; 69983981e31SAlexander Motin continue; 70083981e31SAlexander Motin } 70183981e31SAlexander Motin bzero((uint8_t *)beio->sg_segs[i].addr + s, 70283981e31SAlexander Motin beio->sg_segs[i].len - s); 70383981e31SAlexander Motin s = 0; 70483981e31SAlexander Motin } 70583981e31SAlexander Motin } 706130f4520SKenneth D. Merry } else { 707130f4520SKenneth D. Merry struct mount *mountpoint; 708130f4520SKenneth D. Merry int lock_flags; 709130f4520SKenneth D. Merry 710130f4520SKenneth D. Merry (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 711130f4520SKenneth D. Merry 71267cc546dSAlexander Motin if (MNT_SHARED_WRITES(mountpoint) || ((mountpoint == NULL) 713130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 714130f4520SKenneth D. Merry lock_flags = LK_SHARED; 715130f4520SKenneth D. Merry else 716130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 717130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 718130f4520SKenneth D. Merry 719130f4520SKenneth D. Merry /* 720130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for writes. The write 721130f4520SKenneth D. Merry * is done asynchronously. (Normally the write would just 722130f4520SKenneth D. Merry * get put into cache. 723130f4520SKenneth D. Merry * 724130f4520SKenneth D. Merry * UFS pays attention to IO_SYNC for writes. It will 725130f4520SKenneth D. Merry * attempt to write the buffer out synchronously if that 726130f4520SKenneth D. Merry * flag is set. 727130f4520SKenneth D. Merry * 728130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for writes. 729130f4520SKenneth D. Merry * 730130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) 731130f4520SKenneth D. Merry * for writes. It will flush the transaction from the 732130f4520SKenneth D. Merry * cache before returning. 733130f4520SKenneth D. Merry */ 73455551d05SAlexander Motin error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); 735130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 736130f4520SKenneth D. Merry 737130f4520SKenneth D. Merry vn_finished_write(mountpoint); 73836160958SMark Johnston SDT_PROBE0(cbb, , write, file_done); 739130f4520SKenneth D. Merry } 740130f4520SKenneth D. Merry 74175c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 74275c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 74375c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 74475c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 74575c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 74675c7a1d3SAlexander Motin 747130f4520SKenneth D. Merry /* 748130f4520SKenneth D. Merry * If we got an error, set the sense data to "MEDIUM ERROR" and 749130f4520SKenneth D. Merry * return the I/O to the user. 750130f4520SKenneth D. Merry */ 751130f4520SKenneth D. Merry if (error != 0) { 7520631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 7534fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 7546187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 7556187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 7567f7bb97aSAlexander Motin } else { 7577f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 7587f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 7597f7bb97aSAlexander Motin } 760130f4520SKenneth D. Merry ctl_complete_beio(beio); 761130f4520SKenneth D. Merry return; 762130f4520SKenneth D. Merry } 763130f4520SKenneth D. Merry 764130f4520SKenneth D. Merry /* 765696297adSAlexander Motin * If this is a write or a verify, we're all done. 766130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 767130f4520SKenneth D. Merry */ 768696297adSAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 769696297adSAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 770130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 771130f4520SKenneth D. Merry ctl_complete_beio(beio); 772130f4520SKenneth D. Merry } else { 773f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 77475a3108eSAlexander Motin beio->beio_cont == NULL) { 775f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 77675a3108eSAlexander Motin ctl_serseq_done(io); 77775a3108eSAlexander Motin } 778130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 779e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 780130f4520SKenneth D. Merry #endif 781130f4520SKenneth D. Merry ctl_datamove(io); 782130f4520SKenneth D. Merry } 783130f4520SKenneth D. Merry } 784130f4520SKenneth D. Merry 785130f4520SKenneth D. Merry static void 786ef8daf3fSAlexander Motin ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 787ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 788ef8daf3fSAlexander Motin { 789ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 790ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 791ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 792ef8daf3fSAlexander Motin off_t roff, off; 793ef8daf3fSAlexander Motin int error, status; 794ef8daf3fSAlexander Motin 795ef8daf3fSAlexander Motin DPRINTF("entered\n"); 796ef8daf3fSAlexander Motin 7970bcd4ab6SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; 798ef8daf3fSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 799ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, 800ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 801ef8daf3fSAlexander Motin if (error == 0 && off > roff) 802ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 803ef8daf3fSAlexander Motin else { 804ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, 805ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 806ef8daf3fSAlexander Motin if (error == 0 && off > roff) 807ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 808ef8daf3fSAlexander Motin else { 809ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 810ef8daf3fSAlexander Motin off = be_lun->size_bytes; 811ef8daf3fSAlexander Motin } 812ef8daf3fSAlexander Motin } 813ef8daf3fSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 814ef8daf3fSAlexander Motin 815ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 816ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 8170bcd4ab6SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - 8180bcd4ab6SAlexander Motin lbalen->lba), data->descr[0].length); 819ef8daf3fSAlexander Motin data->descr[0].status = status; 820ef8daf3fSAlexander Motin 821ef8daf3fSAlexander Motin ctl_complete_beio(beio); 822ef8daf3fSAlexander Motin } 823ef8daf3fSAlexander Motin 82453c146deSAlexander Motin static uint64_t 82553c146deSAlexander Motin ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname) 82653c146deSAlexander Motin { 82753c146deSAlexander Motin struct vattr vattr; 82853c146deSAlexander Motin struct statfs statfs; 829b9b4269cSAlexander Motin uint64_t val; 83053c146deSAlexander Motin int error; 83153c146deSAlexander Motin 832b9b4269cSAlexander Motin val = UINT64_MAX; 83353c146deSAlexander Motin if (be_lun->vn == NULL) 834b9b4269cSAlexander Motin return (val); 835b9b4269cSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 83653c146deSAlexander Motin if (strcmp(attrname, "blocksused") == 0) { 83753c146deSAlexander Motin error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 838b9b4269cSAlexander Motin if (error == 0) 8390bcd4ab6SAlexander Motin val = vattr.va_bytes / be_lun->cbe_lun.blocksize; 84053c146deSAlexander Motin } 841b9b4269cSAlexander Motin if (strcmp(attrname, "blocksavail") == 0 && 842b9b4269cSAlexander Motin (be_lun->vn->v_iflag & VI_DOOMED) == 0) { 84353c146deSAlexander Motin error = VFS_STATFS(be_lun->vn->v_mount, &statfs); 844b9b4269cSAlexander Motin if (error == 0) 845a15bbf15SAlexander Motin val = statfs.f_bavail * statfs.f_bsize / 8460bcd4ab6SAlexander Motin be_lun->cbe_lun.blocksize; 84753c146deSAlexander Motin } 848b9b4269cSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 849b9b4269cSAlexander Motin return (val); 85053c146deSAlexander Motin } 85153c146deSAlexander Motin 852ef8daf3fSAlexander Motin static void 85367f586a8SAlexander Motin ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, 85467f586a8SAlexander Motin struct ctl_be_block_io *beio) 85567f586a8SAlexander Motin { 85667f586a8SAlexander Motin union ctl_io *io; 8573236151eSAlexander Motin struct cdevsw *csw; 8583236151eSAlexander Motin struct cdev *dev; 85967f586a8SAlexander Motin struct uio xuio; 86067f586a8SAlexander Motin struct iovec *xiovec; 8613236151eSAlexander Motin int error, flags, i, ref; 86267f586a8SAlexander Motin 86367f586a8SAlexander Motin DPRINTF("entered\n"); 86467f586a8SAlexander Motin 86567f586a8SAlexander Motin io = beio->io; 86655551d05SAlexander Motin flags = 0; 86755551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 86855551d05SAlexander Motin flags |= IO_DIRECT; 86955551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 87055551d05SAlexander Motin flags |= IO_SYNC; 87167f586a8SAlexander Motin 87267f586a8SAlexander Motin bzero(&xuio, sizeof(xuio)); 87367f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 87436160958SMark Johnston SDT_PROBE0(cbb, , read, file_start); 87567f586a8SAlexander Motin xuio.uio_rw = UIO_READ; 87667f586a8SAlexander Motin } else { 87736160958SMark Johnston SDT_PROBE0(cbb, , write, file_start); 87867f586a8SAlexander Motin xuio.uio_rw = UIO_WRITE; 87967f586a8SAlexander Motin } 88067f586a8SAlexander Motin xuio.uio_offset = beio->io_offset; 88167f586a8SAlexander Motin xuio.uio_resid = beio->io_len; 88267f586a8SAlexander Motin xuio.uio_segflg = UIO_SYSSPACE; 88367f586a8SAlexander Motin xuio.uio_iov = beio->xiovecs; 88467f586a8SAlexander Motin xuio.uio_iovcnt = beio->num_segs; 88567f586a8SAlexander Motin xuio.uio_td = curthread; 88667f586a8SAlexander Motin 88767f586a8SAlexander Motin for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 88867f586a8SAlexander Motin xiovec->iov_base = beio->sg_segs[i].addr; 88967f586a8SAlexander Motin xiovec->iov_len = beio->sg_segs[i].len; 89067f586a8SAlexander Motin } 89167f586a8SAlexander Motin 89267f586a8SAlexander Motin binuptime(&beio->ds_t0); 89367f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 89467f586a8SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 89567f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 89667f586a8SAlexander Motin 8973236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 8983236151eSAlexander Motin if (csw) { 8993236151eSAlexander Motin if (beio->bio_cmd == BIO_READ) 9003236151eSAlexander Motin error = csw->d_read(dev, &xuio, flags); 9013236151eSAlexander Motin else 9023236151eSAlexander Motin error = csw->d_write(dev, &xuio, flags); 9033236151eSAlexander Motin dev_relthread(dev, ref); 9043236151eSAlexander Motin } else 9053236151eSAlexander Motin error = ENXIO; 9063236151eSAlexander Motin 9073236151eSAlexander Motin if (beio->bio_cmd == BIO_READ) 90836160958SMark Johnston SDT_PROBE0(cbb, , read, file_done); 9093236151eSAlexander Motin else 91036160958SMark Johnston SDT_PROBE0(cbb, , write, file_done); 91167f586a8SAlexander Motin 91267f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 91367f586a8SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 91467f586a8SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 91567f586a8SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 91667f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 91767f586a8SAlexander Motin 91867f586a8SAlexander Motin /* 91967f586a8SAlexander Motin * If we got an error, set the sense data to "MEDIUM ERROR" and 92067f586a8SAlexander Motin * return the I/O to the user. 92167f586a8SAlexander Motin */ 92267f586a8SAlexander Motin if (error != 0) { 9230631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 9244fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 9256187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 9266187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 9277f7bb97aSAlexander Motin } else { 9287f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 9297f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 9307f7bb97aSAlexander Motin } 93167f586a8SAlexander Motin ctl_complete_beio(beio); 93267f586a8SAlexander Motin return; 93367f586a8SAlexander Motin } 93467f586a8SAlexander Motin 93567f586a8SAlexander Motin /* 93667f586a8SAlexander Motin * If this is a write or a verify, we're all done. 93767f586a8SAlexander Motin * If this is a read, we can now send the data to the user. 93867f586a8SAlexander Motin */ 93967f586a8SAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 94067f586a8SAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 94167f586a8SAlexander Motin ctl_set_success(&io->scsiio); 94267f586a8SAlexander Motin ctl_complete_beio(beio); 94367f586a8SAlexander Motin } else { 944f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 94575a3108eSAlexander Motin beio->beio_cont == NULL) { 946f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 94775a3108eSAlexander Motin ctl_serseq_done(io); 94875a3108eSAlexander Motin } 94967f586a8SAlexander Motin #ifdef CTL_TIME_IO 950e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 95167f586a8SAlexander Motin #endif 95267f586a8SAlexander Motin ctl_datamove(io); 95367f586a8SAlexander Motin } 95467f586a8SAlexander Motin } 95567f586a8SAlexander Motin 95667f586a8SAlexander Motin static void 957ef8daf3fSAlexander Motin ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun, 958ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 959ef8daf3fSAlexander Motin { 960ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 9613236151eSAlexander Motin struct cdevsw *csw; 9623236151eSAlexander Motin struct cdev *dev; 963ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 964ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 965ef8daf3fSAlexander Motin off_t roff, off; 9663236151eSAlexander Motin int error, ref, status; 967ef8daf3fSAlexander Motin 968ef8daf3fSAlexander Motin DPRINTF("entered\n"); 969ef8daf3fSAlexander Motin 9703236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 9713236151eSAlexander Motin if (csw == NULL) { 9723236151eSAlexander Motin status = 0; /* unknown up to the end */ 9733236151eSAlexander Motin off = be_lun->size_bytes; 9743236151eSAlexander Motin goto done; 9753236151eSAlexander Motin } 9760bcd4ab6SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; 9773236151eSAlexander Motin error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD, 9783236151eSAlexander Motin curthread); 979ef8daf3fSAlexander Motin if (error == 0 && off > roff) 980ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 981ef8daf3fSAlexander Motin else { 9823236151eSAlexander Motin error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD, 9833236151eSAlexander Motin curthread); 984ef8daf3fSAlexander Motin if (error == 0 && off > roff) 985ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 986ef8daf3fSAlexander Motin else { 987ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 988ef8daf3fSAlexander Motin off = be_lun->size_bytes; 989ef8daf3fSAlexander Motin } 990ef8daf3fSAlexander Motin } 9913236151eSAlexander Motin dev_relthread(dev, ref); 992ef8daf3fSAlexander Motin 9933236151eSAlexander Motin done: 994ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 995ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 9960bcd4ab6SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - 9970bcd4ab6SAlexander Motin lbalen->lba), data->descr[0].length); 998ef8daf3fSAlexander Motin data->descr[0].status = status; 999ef8daf3fSAlexander Motin 1000ef8daf3fSAlexander Motin ctl_complete_beio(beio); 1001ef8daf3fSAlexander Motin } 1002ef8daf3fSAlexander Motin 1003ef8daf3fSAlexander Motin static void 1004130f4520SKenneth D. Merry ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 1005130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1006130f4520SKenneth D. Merry { 1007130f4520SKenneth D. Merry struct bio *bio; 10083236151eSAlexander Motin struct cdevsw *csw; 10093236151eSAlexander Motin struct cdev *dev; 10103236151eSAlexander Motin int ref; 1011130f4520SKenneth D. Merry 1012130f4520SKenneth D. Merry DPRINTF("entered\n"); 1013130f4520SKenneth D. Merry 1014130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1015130f4520SKenneth D. Merry bio = g_alloc_bio(); 1016130f4520SKenneth D. Merry 1017130f4520SKenneth D. Merry bio->bio_cmd = BIO_FLUSH; 1018130f4520SKenneth D. Merry bio->bio_offset = 0; 1019130f4520SKenneth D. Merry bio->bio_data = 0; 1020130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 1021130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1022130f4520SKenneth D. Merry bio->bio_pblkno = 0; 1023130f4520SKenneth D. Merry 1024130f4520SKenneth D. Merry /* 1025130f4520SKenneth D. Merry * We don't need to acquire the LUN lock here, because we are only 1026130f4520SKenneth D. Merry * sending one bio, and so there is no other context to synchronize 1027130f4520SKenneth D. Merry * with. 1028130f4520SKenneth D. Merry */ 1029130f4520SKenneth D. Merry beio->num_bios_sent = 1; 1030130f4520SKenneth D. Merry beio->send_complete = 1; 1031130f4520SKenneth D. Merry 1032130f4520SKenneth D. Merry binuptime(&beio->ds_t0); 103375c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1034130f4520SKenneth D. Merry devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 103575c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1036130f4520SKenneth D. Merry 10373236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 10383236151eSAlexander Motin if (csw) { 10393236151eSAlexander Motin bio->bio_dev = dev; 10403236151eSAlexander Motin csw->d_strategy(bio); 10413236151eSAlexander Motin dev_relthread(dev, ref); 10423236151eSAlexander Motin } else { 10433236151eSAlexander Motin bio->bio_error = ENXIO; 10443236151eSAlexander Motin ctl_be_block_biodone(bio); 10453236151eSAlexander Motin } 1046130f4520SKenneth D. Merry } 1047130f4520SKenneth D. Merry 1048130f4520SKenneth D. Merry static void 1049ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun, 1050ee7f31c0SAlexander Motin struct ctl_be_block_io *beio, 1051ee7f31c0SAlexander Motin uint64_t off, uint64_t len, int last) 1052ee7f31c0SAlexander Motin { 1053ee7f31c0SAlexander Motin struct bio *bio; 10548f5a226aSAlexander Motin uint64_t maxlen; 10553236151eSAlexander Motin struct cdevsw *csw; 10563236151eSAlexander Motin struct cdev *dev; 10573236151eSAlexander Motin int ref; 1058ee7f31c0SAlexander Motin 10593236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 10600bcd4ab6SAlexander Motin maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize); 1061ee7f31c0SAlexander Motin while (len > 0) { 1062ee7f31c0SAlexander Motin bio = g_alloc_bio(); 1063ee7f31c0SAlexander Motin bio->bio_cmd = BIO_DELETE; 10643236151eSAlexander Motin bio->bio_dev = dev; 1065ee7f31c0SAlexander Motin bio->bio_offset = off; 10668f5a226aSAlexander Motin bio->bio_length = MIN(len, maxlen); 1067ee7f31c0SAlexander Motin bio->bio_data = 0; 1068ee7f31c0SAlexander Motin bio->bio_done = ctl_be_block_biodone; 1069ee7f31c0SAlexander Motin bio->bio_caller1 = beio; 10700bcd4ab6SAlexander Motin bio->bio_pblkno = off / be_lun->cbe_lun.blocksize; 1071ee7f31c0SAlexander Motin 1072ee7f31c0SAlexander Motin off += bio->bio_length; 1073ee7f31c0SAlexander Motin len -= bio->bio_length; 1074ee7f31c0SAlexander Motin 107575c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1076ee7f31c0SAlexander Motin beio->num_bios_sent++; 1077ee7f31c0SAlexander Motin if (last && len == 0) 1078ee7f31c0SAlexander Motin beio->send_complete = 1; 107975c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1080ee7f31c0SAlexander Motin 10813236151eSAlexander Motin if (csw) { 10823236151eSAlexander Motin csw->d_strategy(bio); 10833236151eSAlexander Motin } else { 10843236151eSAlexander Motin bio->bio_error = ENXIO; 10853236151eSAlexander Motin ctl_be_block_biodone(bio); 1086ee7f31c0SAlexander Motin } 1087ee7f31c0SAlexander Motin } 10883236151eSAlexander Motin if (csw) 10893236151eSAlexander Motin dev_relthread(dev, ref); 10903236151eSAlexander Motin } 1091ee7f31c0SAlexander Motin 1092ee7f31c0SAlexander Motin static void 1093ee7f31c0SAlexander Motin ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 1094ee7f31c0SAlexander Motin struct ctl_be_block_io *beio) 1095ee7f31c0SAlexander Motin { 1096ee7f31c0SAlexander Motin union ctl_io *io; 109766df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1098ee7f31c0SAlexander Motin struct scsi_unmap_desc *buf, *end; 1099ee7f31c0SAlexander Motin uint64_t len; 1100ee7f31c0SAlexander Motin 1101ee7f31c0SAlexander Motin io = beio->io; 1102ee7f31c0SAlexander Motin 1103ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1104ee7f31c0SAlexander Motin 1105ee7f31c0SAlexander Motin binuptime(&beio->ds_t0); 110675c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1107ee7f31c0SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 110875c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1109ee7f31c0SAlexander Motin 1110ee7f31c0SAlexander Motin if (beio->io_offset == -1) { 1111ee7f31c0SAlexander Motin beio->io_len = 0; 111266df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 111366df9136SAlexander Motin buf = (struct scsi_unmap_desc *)ptrlen->ptr; 111466df9136SAlexander Motin end = buf + ptrlen->len / sizeof(*buf); 1115ee7f31c0SAlexander Motin for (; buf < end; buf++) { 1116ee7f31c0SAlexander Motin len = (uint64_t)scsi_4btoul(buf->length) * 11170bcd4ab6SAlexander Motin be_lun->cbe_lun.blocksize; 1118ee7f31c0SAlexander Motin beio->io_len += len; 1119ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 11200bcd4ab6SAlexander Motin scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize, 11210bcd4ab6SAlexander Motin len, (end - buf < 2) ? TRUE : FALSE); 1122ee7f31c0SAlexander Motin } 1123ee7f31c0SAlexander Motin } else 1124ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1125ee7f31c0SAlexander Motin beio->io_offset, beio->io_len, TRUE); 1126ee7f31c0SAlexander Motin } 1127ee7f31c0SAlexander Motin 1128ee7f31c0SAlexander Motin static void 1129130f4520SKenneth D. Merry ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 1130130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1131130f4520SKenneth D. Merry { 113275c7a1d3SAlexander Motin TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue); 1133130f4520SKenneth D. Merry struct bio *bio; 11343236151eSAlexander Motin struct cdevsw *csw; 11353236151eSAlexander Motin struct cdev *dev; 1136130f4520SKenneth D. Merry off_t cur_offset; 11373236151eSAlexander Motin int i, max_iosize, ref; 1138130f4520SKenneth D. Merry 1139130f4520SKenneth D. Merry DPRINTF("entered\n"); 11403236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 1141130f4520SKenneth D. Merry 1142130f4520SKenneth D. Merry /* 1143130f4520SKenneth D. Merry * We have to limit our I/O size to the maximum supported by the 1144130f4520SKenneth D. Merry * backend device. Hopefully it is MAXPHYS. If the driver doesn't 1145130f4520SKenneth D. Merry * set it properly, use DFLTPHYS. 1146130f4520SKenneth D. Merry */ 11473236151eSAlexander Motin if (csw) { 11483236151eSAlexander Motin max_iosize = dev->si_iosize_max; 1149130f4520SKenneth D. Merry if (max_iosize < PAGE_SIZE) 1150130f4520SKenneth D. Merry max_iosize = DFLTPHYS; 11513236151eSAlexander Motin } else 11523236151eSAlexander Motin max_iosize = DFLTPHYS; 1153130f4520SKenneth D. Merry 1154130f4520SKenneth D. Merry cur_offset = beio->io_offset; 1155130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 1156130f4520SKenneth D. Merry size_t cur_size; 1157130f4520SKenneth D. Merry uint8_t *cur_ptr; 1158130f4520SKenneth D. Merry 1159130f4520SKenneth D. Merry cur_size = beio->sg_segs[i].len; 1160130f4520SKenneth D. Merry cur_ptr = beio->sg_segs[i].addr; 1161130f4520SKenneth D. Merry 1162130f4520SKenneth D. Merry while (cur_size > 0) { 1163130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1164130f4520SKenneth D. Merry bio = g_alloc_bio(); 1165130f4520SKenneth D. Merry 1166130f4520SKenneth D. Merry KASSERT(bio != NULL, ("g_alloc_bio() failed!\n")); 1167130f4520SKenneth D. Merry 1168130f4520SKenneth D. Merry bio->bio_cmd = beio->bio_cmd; 11693236151eSAlexander Motin bio->bio_dev = dev; 1170130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1171130f4520SKenneth D. Merry bio->bio_length = min(cur_size, max_iosize); 1172130f4520SKenneth D. Merry bio->bio_offset = cur_offset; 1173130f4520SKenneth D. Merry bio->bio_data = cur_ptr; 1174130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 11750bcd4ab6SAlexander Motin bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize; 1176130f4520SKenneth D. Merry 1177130f4520SKenneth D. Merry cur_offset += bio->bio_length; 1178130f4520SKenneth D. Merry cur_ptr += bio->bio_length; 1179130f4520SKenneth D. Merry cur_size -= bio->bio_length; 1180130f4520SKenneth D. Merry 118175c7a1d3SAlexander Motin TAILQ_INSERT_TAIL(&queue, bio, bio_queue); 1182130f4520SKenneth D. Merry beio->num_bios_sent++; 1183130f4520SKenneth D. Merry } 1184130f4520SKenneth D. Merry } 118575c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 118675c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 118775c7a1d3SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 118875c7a1d3SAlexander Motin beio->send_complete = 1; 118975c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 119075c7a1d3SAlexander Motin 119175c7a1d3SAlexander Motin /* 119275c7a1d3SAlexander Motin * Fire off all allocated requests! 119375c7a1d3SAlexander Motin */ 119475c7a1d3SAlexander Motin while ((bio = TAILQ_FIRST(&queue)) != NULL) { 119575c7a1d3SAlexander Motin TAILQ_REMOVE(&queue, bio, bio_queue); 11963236151eSAlexander Motin if (csw) 11973236151eSAlexander Motin csw->d_strategy(bio); 11983236151eSAlexander Motin else { 11993236151eSAlexander Motin bio->bio_error = ENXIO; 12003236151eSAlexander Motin ctl_be_block_biodone(bio); 120175c7a1d3SAlexander Motin } 1202130f4520SKenneth D. Merry } 12033236151eSAlexander Motin if (csw) 12043236151eSAlexander Motin dev_relthread(dev, ref); 12053236151eSAlexander Motin } 1206130f4520SKenneth D. Merry 1207c3e7ba3eSAlexander Motin static uint64_t 1208c3e7ba3eSAlexander Motin ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname) 1209c3e7ba3eSAlexander Motin { 1210c3e7ba3eSAlexander Motin struct diocgattr_arg arg; 12113236151eSAlexander Motin struct cdevsw *csw; 12123236151eSAlexander Motin struct cdev *dev; 12133236151eSAlexander Motin int error, ref; 1214c3e7ba3eSAlexander Motin 12153236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 12163236151eSAlexander Motin if (csw == NULL) 1217c3e7ba3eSAlexander Motin return (UINT64_MAX); 1218c3e7ba3eSAlexander Motin strlcpy(arg.name, attrname, sizeof(arg.name)); 1219c3e7ba3eSAlexander Motin arg.len = sizeof(arg.value.off); 12203236151eSAlexander Motin if (csw->d_ioctl) { 12213236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, 12223236151eSAlexander Motin curthread); 12233236151eSAlexander Motin } else 12243236151eSAlexander Motin error = ENODEV; 12253236151eSAlexander Motin dev_relthread(dev, ref); 1226c3e7ba3eSAlexander Motin if (error != 0) 1227c3e7ba3eSAlexander Motin return (UINT64_MAX); 1228c3e7ba3eSAlexander Motin return (arg.value.off); 1229c3e7ba3eSAlexander Motin } 1230c3e7ba3eSAlexander Motin 1231130f4520SKenneth D. Merry static void 12327d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun, 12337d0d4342SAlexander Motin union ctl_io *io) 12347d0d4342SAlexander Motin { 12350bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 12367d0d4342SAlexander Motin struct ctl_be_block_io *beio; 12377d0d4342SAlexander Motin struct ctl_lba_len_flags *lbalen; 12387d0d4342SAlexander Motin 12397d0d4342SAlexander Motin DPRINTF("entered\n"); 12407d0d4342SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 12417d0d4342SAlexander Motin lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 12427d0d4342SAlexander Motin 12430bcd4ab6SAlexander Motin beio->io_len = lbalen->len * cbe_lun->blocksize; 12440bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 12457d0d4342SAlexander Motin beio->io_arg = (lbalen->flags & SSC_IMMED) != 0; 12467d0d4342SAlexander Motin beio->bio_cmd = BIO_FLUSH; 12477d0d4342SAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 12487d0d4342SAlexander Motin DPRINTF("SYNC\n"); 12497d0d4342SAlexander Motin be_lun->lun_flush(be_lun, beio); 12507d0d4342SAlexander Motin } 12517d0d4342SAlexander Motin 12527d0d4342SAlexander Motin static void 1253ee7f31c0SAlexander Motin ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio) 1254ee7f31c0SAlexander Motin { 1255ee7f31c0SAlexander Motin union ctl_io *io; 1256ee7f31c0SAlexander Motin 1257ee7f31c0SAlexander Motin io = beio->io; 1258ee7f31c0SAlexander Motin ctl_free_beio(beio); 1259ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1260ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1261ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1262ee7f31c0SAlexander Motin ctl_config_write_done(io); 1263ee7f31c0SAlexander Motin return; 1264ee7f31c0SAlexander Motin } 1265ee7f31c0SAlexander Motin 1266ee7f31c0SAlexander Motin ctl_be_block_config_write(io); 1267ee7f31c0SAlexander Motin } 1268ee7f31c0SAlexander Motin 1269ee7f31c0SAlexander Motin static void 1270ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun, 1271ee7f31c0SAlexander Motin union ctl_io *io) 1272ee7f31c0SAlexander Motin { 12730bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1274ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 127566df9136SAlexander Motin struct ctl_lba_len_flags *lbalen; 1276fee04ef7SAlexander Motin uint64_t len_left, lba; 1277fee04ef7SAlexander Motin uint32_t pb, pbo, adj; 1278ee7f31c0SAlexander Motin int i, seglen; 1279ee7f31c0SAlexander Motin uint8_t *buf, *end; 1280ee7f31c0SAlexander Motin 1281ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1282ee7f31c0SAlexander Motin 1283e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 128411b569f7SAlexander Motin lbalen = ARGS(beio->io); 1285ee7f31c0SAlexander Motin 128664c5167cSAlexander Motin if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || 12873406a2a0SAlexander Motin (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { 1288ee7f31c0SAlexander Motin ctl_free_beio(beio); 1289ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1290ee7f31c0SAlexander Motin /*sks_valid*/ 1, 1291ee7f31c0SAlexander Motin /*command*/ 1, 1292ee7f31c0SAlexander Motin /*field*/ 1, 1293ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1294ee7f31c0SAlexander Motin /*bit*/ 0); 1295ee7f31c0SAlexander Motin ctl_config_write_done(io); 1296ee7f31c0SAlexander Motin return; 1297ee7f31c0SAlexander Motin } 1298ee7f31c0SAlexander Motin 12993406a2a0SAlexander Motin if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { 13000bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 13010bcd4ab6SAlexander Motin beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize; 1302ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1303ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 1304ee7f31c0SAlexander Motin 1305ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1306ee7f31c0SAlexander Motin return; 1307ee7f31c0SAlexander Motin } 1308ee7f31c0SAlexander Motin 1309ee7f31c0SAlexander Motin beio->bio_cmd = BIO_WRITE; 1310ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_WRITE; 1311ee7f31c0SAlexander Motin 1312ee7f31c0SAlexander Motin DPRINTF("WRITE SAME at LBA %jx len %u\n", 131366df9136SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len); 1314ee7f31c0SAlexander Motin 13150bcd4ab6SAlexander Motin pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp; 13160bcd4ab6SAlexander Motin if (be_lun->cbe_lun.pblockoff > 0) 13170bcd4ab6SAlexander Motin pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff; 1318fee04ef7SAlexander Motin else 1319fee04ef7SAlexander Motin pbo = 0; 13200bcd4ab6SAlexander Motin len_left = (uint64_t)lbalen->len * cbe_lun->blocksize; 1321ee7f31c0SAlexander Motin for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) { 1322ee7f31c0SAlexander Motin 1323ee7f31c0SAlexander Motin /* 1324ee7f31c0SAlexander Motin * Setup the S/G entry for this chunk. 1325ee7f31c0SAlexander Motin */ 132608a7cce5SAlexander Motin seglen = MIN(CTLBLK_MAX_SEG, len_left); 13270bcd4ab6SAlexander Motin if (pb > cbe_lun->blocksize) { 13280bcd4ab6SAlexander Motin adj = ((lbalen->lba + lba) * cbe_lun->blocksize + 132993b8c96cSAlexander Motin seglen - pbo) % pb; 133093b8c96cSAlexander Motin if (seglen > adj) 133193b8c96cSAlexander Motin seglen -= adj; 133293b8c96cSAlexander Motin else 13330bcd4ab6SAlexander Motin seglen -= seglen % cbe_lun->blocksize; 133493b8c96cSAlexander Motin } else 13350bcd4ab6SAlexander Motin seglen -= seglen % cbe_lun->blocksize; 1336ee7f31c0SAlexander Motin beio->sg_segs[i].len = seglen; 1337ee7f31c0SAlexander Motin beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1338ee7f31c0SAlexander Motin 1339ee7f31c0SAlexander Motin DPRINTF("segment %d addr %p len %zd\n", i, 1340ee7f31c0SAlexander Motin beio->sg_segs[i].addr, beio->sg_segs[i].len); 1341ee7f31c0SAlexander Motin 1342ee7f31c0SAlexander Motin beio->num_segs++; 1343ee7f31c0SAlexander Motin len_left -= seglen; 1344ee7f31c0SAlexander Motin 1345ee7f31c0SAlexander Motin buf = beio->sg_segs[i].addr; 1346ee7f31c0SAlexander Motin end = buf + seglen; 13470bcd4ab6SAlexander Motin for (; buf < end; buf += cbe_lun->blocksize) { 13486c2acea5SAlexander Motin if (lbalen->flags & SWS_NDOB) { 13496c2acea5SAlexander Motin memset(buf, 0, cbe_lun->blocksize); 13506c2acea5SAlexander Motin } else { 13516c2acea5SAlexander Motin memcpy(buf, io->scsiio.kern_data_ptr, 13526c2acea5SAlexander Motin cbe_lun->blocksize); 13536c2acea5SAlexander Motin } 135466df9136SAlexander Motin if (lbalen->flags & SWS_LBDATA) 135566df9136SAlexander Motin scsi_ulto4b(lbalen->lba + lba, buf); 1356ee7f31c0SAlexander Motin lba++; 1357ee7f31c0SAlexander Motin } 1358ee7f31c0SAlexander Motin } 1359ee7f31c0SAlexander Motin 13600bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 13610bcd4ab6SAlexander Motin beio->io_len = lba * cbe_lun->blocksize; 1362ee7f31c0SAlexander Motin 1363ee7f31c0SAlexander Motin /* We can not do all in one run. Correct and schedule rerun. */ 1364ee7f31c0SAlexander Motin if (len_left > 0) { 136566df9136SAlexander Motin lbalen->lba += lba; 136666df9136SAlexander Motin lbalen->len -= lba; 1367ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done_ws; 1368ee7f31c0SAlexander Motin } 1369ee7f31c0SAlexander Motin 1370ee7f31c0SAlexander Motin be_lun->dispatch(be_lun, beio); 1371ee7f31c0SAlexander Motin } 1372ee7f31c0SAlexander Motin 1373ee7f31c0SAlexander Motin static void 1374ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun, 1375ee7f31c0SAlexander Motin union ctl_io *io) 1376ee7f31c0SAlexander Motin { 1377ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 137866df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1379ee7f31c0SAlexander Motin 1380ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1381ee7f31c0SAlexander Motin 1382e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 138366df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1384ee7f31c0SAlexander Motin 13853406a2a0SAlexander Motin if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { 1386ee7f31c0SAlexander Motin ctl_free_beio(beio); 1387ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1388ee7f31c0SAlexander Motin /*sks_valid*/ 0, 1389ee7f31c0SAlexander Motin /*command*/ 1, 1390ee7f31c0SAlexander Motin /*field*/ 0, 1391ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1392ee7f31c0SAlexander Motin /*bit*/ 0); 1393ee7f31c0SAlexander Motin ctl_config_write_done(io); 1394ee7f31c0SAlexander Motin return; 1395ee7f31c0SAlexander Motin } 1396ee7f31c0SAlexander Motin 1397ee7f31c0SAlexander Motin beio->io_len = 0; 1398ee7f31c0SAlexander Motin beio->io_offset = -1; 1399ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1400ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 140166df9136SAlexander Motin DPRINTF("UNMAP\n"); 1402ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1403ee7f31c0SAlexander Motin } 1404ee7f31c0SAlexander Motin 1405ee7f31c0SAlexander Motin static void 1406ef8daf3fSAlexander Motin ctl_be_block_cr_done(struct ctl_be_block_io *beio) 1407ef8daf3fSAlexander Motin { 1408ef8daf3fSAlexander Motin union ctl_io *io; 1409ef8daf3fSAlexander Motin 1410ef8daf3fSAlexander Motin io = beio->io; 1411ef8daf3fSAlexander Motin ctl_free_beio(beio); 1412ef8daf3fSAlexander Motin ctl_config_read_done(io); 1413ef8daf3fSAlexander Motin } 1414ef8daf3fSAlexander Motin 1415ef8daf3fSAlexander Motin static void 1416ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 1417ef8daf3fSAlexander Motin union ctl_io *io) 1418ef8daf3fSAlexander Motin { 1419ef8daf3fSAlexander Motin struct ctl_be_block_io *beio; 1420ef8daf3fSAlexander Motin struct ctl_be_block_softc *softc; 1421ef8daf3fSAlexander Motin 1422ef8daf3fSAlexander Motin DPRINTF("entered\n"); 1423ef8daf3fSAlexander Motin 1424ef8daf3fSAlexander Motin softc = be_lun->softc; 1425ef8daf3fSAlexander Motin beio = ctl_alloc_beio(softc); 1426ef8daf3fSAlexander Motin beio->io = io; 1427ef8daf3fSAlexander Motin beio->lun = be_lun; 1428ef8daf3fSAlexander Motin beio->beio_cont = ctl_be_block_cr_done; 1429ef8daf3fSAlexander Motin PRIV(io)->ptr = (void *)beio; 1430ef8daf3fSAlexander Motin 1431ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 1432ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: /* GET LBA STATUS */ 1433ef8daf3fSAlexander Motin beio->bio_cmd = -1; 1434ef8daf3fSAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 1435ef8daf3fSAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1436ef8daf3fSAlexander Motin beio->io_len = 0; 1437ef8daf3fSAlexander Motin if (be_lun->get_lba_status) 1438ef8daf3fSAlexander Motin be_lun->get_lba_status(be_lun, beio); 1439ef8daf3fSAlexander Motin else 1440ef8daf3fSAlexander Motin ctl_be_block_cr_done(beio); 1441ef8daf3fSAlexander Motin break; 1442ef8daf3fSAlexander Motin default: 1443ef8daf3fSAlexander Motin panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1444ef8daf3fSAlexander Motin break; 1445ef8daf3fSAlexander Motin } 1446ef8daf3fSAlexander Motin } 1447ef8daf3fSAlexander Motin 1448ef8daf3fSAlexander Motin static void 1449ee7f31c0SAlexander Motin ctl_be_block_cw_done(struct ctl_be_block_io *beio) 1450ee7f31c0SAlexander Motin { 1451ee7f31c0SAlexander Motin union ctl_io *io; 1452ee7f31c0SAlexander Motin 1453ee7f31c0SAlexander Motin io = beio->io; 1454ee7f31c0SAlexander Motin ctl_free_beio(beio); 1455ee7f31c0SAlexander Motin ctl_config_write_done(io); 1456ee7f31c0SAlexander Motin } 1457ee7f31c0SAlexander Motin 1458ee7f31c0SAlexander Motin static void 1459130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 1460130f4520SKenneth D. Merry union ctl_io *io) 1461130f4520SKenneth D. Merry { 1462130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1463130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1464130f4520SKenneth D. Merry 1465130f4520SKenneth D. Merry DPRINTF("entered\n"); 1466130f4520SKenneth D. Merry 1467130f4520SKenneth D. Merry softc = be_lun->softc; 1468130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1469130f4520SKenneth D. Merry beio->io = io; 1470130f4520SKenneth D. Merry beio->lun = be_lun; 1471ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done; 14727d0d4342SAlexander Motin switch (io->scsiio.tag_type) { 14737d0d4342SAlexander Motin case CTL_TAG_ORDERED: 14747d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 14757d0d4342SAlexander Motin break; 14767d0d4342SAlexander Motin case CTL_TAG_HEAD_OF_QUEUE: 14777d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_HEAD; 14787d0d4342SAlexander Motin break; 14797d0d4342SAlexander Motin case CTL_TAG_UNTAGGED: 14807d0d4342SAlexander Motin case CTL_TAG_SIMPLE: 14817d0d4342SAlexander Motin case CTL_TAG_ACA: 14827d0d4342SAlexander Motin default: 14837d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 14847d0d4342SAlexander Motin break; 14857d0d4342SAlexander Motin } 1486e86a4142SAlexander Motin PRIV(io)->ptr = (void *)beio; 1487130f4520SKenneth D. Merry 1488130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 1489130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 1490130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 14917d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(be_lun, io); 1492130f4520SKenneth D. Merry break; 1493ee7f31c0SAlexander Motin case WRITE_SAME_10: 1494ee7f31c0SAlexander Motin case WRITE_SAME_16: 1495ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(be_lun, io); 1496ee7f31c0SAlexander Motin break; 1497ee7f31c0SAlexander Motin case UNMAP: 1498ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(be_lun, io); 1499ee7f31c0SAlexander Motin break; 1500130f4520SKenneth D. Merry default: 1501130f4520SKenneth D. Merry panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1502130f4520SKenneth D. Merry break; 1503130f4520SKenneth D. Merry } 1504130f4520SKenneth D. Merry } 1505130f4520SKenneth D. Merry 150636160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, start, "uint64_t"); 150736160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, start, "uint64_t"); 150836160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, alloc_done, "uint64_t"); 150936160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, alloc_done, "uint64_t"); 1510130f4520SKenneth D. Merry 1511130f4520SKenneth D. Merry static void 151208a7cce5SAlexander Motin ctl_be_block_next(struct ctl_be_block_io *beio) 151308a7cce5SAlexander Motin { 151408a7cce5SAlexander Motin struct ctl_be_block_lun *be_lun; 151508a7cce5SAlexander Motin union ctl_io *io; 151608a7cce5SAlexander Motin 151708a7cce5SAlexander Motin io = beio->io; 151808a7cce5SAlexander Motin be_lun = beio->lun; 151908a7cce5SAlexander Motin ctl_free_beio(beio); 1520ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1521ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1522ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 152311b569f7SAlexander Motin ctl_data_submit_done(io); 152408a7cce5SAlexander Motin return; 152508a7cce5SAlexander Motin } 152608a7cce5SAlexander Motin 152708a7cce5SAlexander Motin io->io_hdr.status &= ~CTL_STATUS_MASK; 152808a7cce5SAlexander Motin io->io_hdr.status |= CTL_STATUS_NONE; 152908a7cce5SAlexander Motin 153075c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 153108a7cce5SAlexander Motin STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 153275c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 153308a7cce5SAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 153408a7cce5SAlexander Motin } 153508a7cce5SAlexander Motin 153608a7cce5SAlexander Motin static void 1537130f4520SKenneth D. Merry ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 1538130f4520SKenneth D. Merry union ctl_io *io) 1539130f4520SKenneth D. Merry { 15400bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1541130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1542130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 154311b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 1544e86a4142SAlexander Motin struct ctl_ptr_len_flags *bptrlen; 1545e86a4142SAlexander Motin uint64_t len_left, lbas; 1546130f4520SKenneth D. Merry int i; 1547130f4520SKenneth D. Merry 1548130f4520SKenneth D. Merry softc = be_lun->softc; 1549130f4520SKenneth D. Merry 1550130f4520SKenneth D. Merry DPRINTF("entered\n"); 1551130f4520SKenneth D. Merry 155211b569f7SAlexander Motin lbalen = ARGS(io); 155311b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 155436160958SMark Johnston SDT_PROBE0(cbb, , write, start); 155511b569f7SAlexander Motin } else { 155636160958SMark Johnston SDT_PROBE0(cbb, , read, start); 1557130f4520SKenneth D. Merry } 1558130f4520SKenneth D. Merry 1559130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1560130f4520SKenneth D. Merry beio->io = io; 1561130f4520SKenneth D. Merry beio->lun = be_lun; 1562e86a4142SAlexander Motin bptrlen = PRIV(io); 1563e86a4142SAlexander Motin bptrlen->ptr = (void *)beio; 1564130f4520SKenneth D. Merry 1565130f4520SKenneth D. Merry switch (io->scsiio.tag_type) { 1566130f4520SKenneth D. Merry case CTL_TAG_ORDERED: 1567130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1568130f4520SKenneth D. Merry break; 1569130f4520SKenneth D. Merry case CTL_TAG_HEAD_OF_QUEUE: 1570130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1571130f4520SKenneth D. Merry break; 1572130f4520SKenneth D. Merry case CTL_TAG_UNTAGGED: 1573130f4520SKenneth D. Merry case CTL_TAG_SIMPLE: 1574130f4520SKenneth D. Merry case CTL_TAG_ACA: 1575130f4520SKenneth D. Merry default: 1576130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1577130f4520SKenneth D. Merry break; 1578130f4520SKenneth D. Merry } 1579130f4520SKenneth D. Merry 158011b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1581130f4520SKenneth D. Merry beio->bio_cmd = BIO_WRITE; 1582130f4520SKenneth D. Merry beio->ds_trans_type = DEVSTAT_WRITE; 158311b569f7SAlexander Motin } else { 158411b569f7SAlexander Motin beio->bio_cmd = BIO_READ; 158511b569f7SAlexander Motin beio->ds_trans_type = DEVSTAT_READ; 1586130f4520SKenneth D. Merry } 1587130f4520SKenneth D. Merry 158808a7cce5SAlexander Motin DPRINTF("%s at LBA %jx len %u @%ju\n", 1589130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", 1590e86a4142SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); 159111b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 159211b569f7SAlexander Motin lbas = CTLBLK_HALF_IO_SIZE; 159311b569f7SAlexander Motin else 159411b569f7SAlexander Motin lbas = CTLBLK_MAX_IO_SIZE; 15950bcd4ab6SAlexander Motin lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize); 15960bcd4ab6SAlexander Motin beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize; 15970bcd4ab6SAlexander Motin beio->io_len = lbas * cbe_lun->blocksize; 1598e86a4142SAlexander Motin bptrlen->len += lbas; 1599130f4520SKenneth D. Merry 160008a7cce5SAlexander Motin for (i = 0, len_left = beio->io_len; len_left > 0; i++) { 160108a7cce5SAlexander Motin KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)", 160208a7cce5SAlexander Motin i, CTLBLK_MAX_SEGS)); 1603130f4520SKenneth D. Merry 1604130f4520SKenneth D. Merry /* 1605130f4520SKenneth D. Merry * Setup the S/G entry for this chunk. 1606130f4520SKenneth D. Merry */ 160708a7cce5SAlexander Motin beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left); 1608130f4520SKenneth D. Merry beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1609130f4520SKenneth D. Merry 1610130f4520SKenneth D. Merry DPRINTF("segment %d addr %p len %zd\n", i, 1611130f4520SKenneth D. Merry beio->sg_segs[i].addr, beio->sg_segs[i].len); 1612130f4520SKenneth D. Merry 161311b569f7SAlexander Motin /* Set up second segment for compare operation. */ 161411b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) { 161511b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].len = 161611b569f7SAlexander Motin beio->sg_segs[i].len; 161711b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = 161811b569f7SAlexander Motin uma_zalloc(be_lun->lun_zone, M_WAITOK); 161911b569f7SAlexander Motin } 162011b569f7SAlexander Motin 1621130f4520SKenneth D. Merry beio->num_segs++; 1622130f4520SKenneth D. Merry len_left -= beio->sg_segs[i].len; 1623130f4520SKenneth D. Merry } 1624e86a4142SAlexander Motin if (bptrlen->len < lbalen->len) 162508a7cce5SAlexander Motin beio->beio_cont = ctl_be_block_next; 162608a7cce5SAlexander Motin io->scsiio.be_move_done = ctl_be_block_move_done; 162711b569f7SAlexander Motin /* For compare we have separate S/G lists for read and datamove. */ 162811b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 162911b569f7SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS]; 163011b569f7SAlexander Motin else 163108a7cce5SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs; 163208a7cce5SAlexander Motin io->scsiio.kern_data_len = beio->io_len; 163308a7cce5SAlexander Motin io->scsiio.kern_sg_entries = beio->num_segs; 1634b2221369SAlexander Motin io->io_hdr.flags |= CTL_FLAG_ALLOCATED; 1635130f4520SKenneth D. Merry 1636130f4520SKenneth D. Merry /* 1637130f4520SKenneth D. Merry * For the read case, we need to read the data into our buffers and 1638130f4520SKenneth D. Merry * then we can send it back to the user. For the write case, we 1639130f4520SKenneth D. Merry * need to get the data from the user first. 1640130f4520SKenneth D. Merry */ 1641130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 164236160958SMark Johnston SDT_PROBE0(cbb, , read, alloc_done); 1643130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1644130f4520SKenneth D. Merry } else { 164536160958SMark Johnston SDT_PROBE0(cbb, , write, alloc_done); 1646130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 1647e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 1648130f4520SKenneth D. Merry #endif 1649130f4520SKenneth D. Merry ctl_datamove(io); 1650130f4520SKenneth D. Merry } 1651130f4520SKenneth D. Merry } 1652130f4520SKenneth D. Merry 1653130f4520SKenneth D. Merry static void 1654130f4520SKenneth D. Merry ctl_be_block_worker(void *context, int pending) 1655130f4520SKenneth D. Merry { 1656ee4ad294SAlexander Motin struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context; 1657ee4ad294SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1658130f4520SKenneth D. Merry union ctl_io *io; 1659130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1660130f4520SKenneth D. Merry 1661ee4ad294SAlexander Motin DPRINTF("entered\n"); 1662ee4ad294SAlexander Motin /* 1663ee4ad294SAlexander Motin * Fetch and process I/Os from all queues. If we detect LUN 1664648dfc1aSAlexander Motin * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race, 1665ee4ad294SAlexander Motin * so make response maximally opaque to not confuse initiator. 1666ee4ad294SAlexander Motin */ 1667ee4ad294SAlexander Motin for (;;) { 1668ee4ad294SAlexander Motin mtx_lock(&be_lun->queue_lock); 1669ee4ad294SAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); 1670ee4ad294SAlexander Motin if (io != NULL) { 1671130f4520SKenneth D. Merry DPRINTF("datamove queue\n"); 1672130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, 1673130f4520SKenneth D. Merry ctl_io_hdr, links); 167475c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1675e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1676648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1677ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1678ee4ad294SAlexander Motin ctl_complete_beio(beio); 1679ee4ad294SAlexander Motin return; 1680ee4ad294SAlexander Motin } 1681130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1682130f4520SKenneth D. Merry continue; 1683130f4520SKenneth D. Merry } 1684130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); 1685130f4520SKenneth D. Merry if (io != NULL) { 1686130f4520SKenneth D. Merry DPRINTF("config write queue\n"); 1687130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, 1688130f4520SKenneth D. Merry ctl_io_hdr, links); 168975c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1690648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1691ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1692ee4ad294SAlexander Motin ctl_config_write_done(io); 1693ee4ad294SAlexander Motin return; 1694ee4ad294SAlexander Motin } 1695130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(be_lun, io); 1696ef8daf3fSAlexander Motin continue; 1697ef8daf3fSAlexander Motin } 1698ef8daf3fSAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); 1699ef8daf3fSAlexander Motin if (io != NULL) { 1700ef8daf3fSAlexander Motin DPRINTF("config read queue\n"); 1701ef8daf3fSAlexander Motin STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, 1702ef8daf3fSAlexander Motin ctl_io_hdr, links); 1703ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 1704648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1705ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1706ee4ad294SAlexander Motin ctl_config_read_done(io); 1707ee4ad294SAlexander Motin return; 1708ee4ad294SAlexander Motin } 1709ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(be_lun, io); 1710130f4520SKenneth D. Merry continue; 1711130f4520SKenneth D. Merry } 1712130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); 1713130f4520SKenneth D. Merry if (io != NULL) { 1714130f4520SKenneth D. Merry DPRINTF("input queue\n"); 1715130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, 1716130f4520SKenneth D. Merry ctl_io_hdr, links); 171775c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1718648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1719ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1720ee4ad294SAlexander Motin ctl_data_submit_done(io); 1721ee4ad294SAlexander Motin return; 1722ee4ad294SAlexander Motin } 1723130f4520SKenneth D. Merry ctl_be_block_dispatch(be_lun, io); 1724130f4520SKenneth D. Merry continue; 1725130f4520SKenneth D. Merry } 1726130f4520SKenneth D. Merry 1727130f4520SKenneth D. Merry /* 1728130f4520SKenneth D. Merry * If we get here, there is no work left in the queues, so 1729130f4520SKenneth D. Merry * just break out and let the task queue go to sleep. 1730130f4520SKenneth D. Merry */ 1731ee4ad294SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1732130f4520SKenneth D. Merry break; 1733130f4520SKenneth D. Merry } 1734130f4520SKenneth D. Merry } 1735130f4520SKenneth D. Merry 1736130f4520SKenneth D. Merry /* 1737130f4520SKenneth D. Merry * Entry point from CTL to the backend for I/O. We queue everything to a 1738130f4520SKenneth D. Merry * work thread, so this just puts the I/O on a queue and wakes up the 1739130f4520SKenneth D. Merry * thread. 1740130f4520SKenneth D. Merry */ 1741130f4520SKenneth D. Merry static int 1742130f4520SKenneth D. Merry ctl_be_block_submit(union ctl_io *io) 1743130f4520SKenneth D. Merry { 1744130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 17450bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 1746130f4520SKenneth D. Merry 1747130f4520SKenneth D. Merry DPRINTF("entered\n"); 1748130f4520SKenneth D. Merry 17499cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 17500bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 1751130f4520SKenneth D. Merry 1752130f4520SKenneth D. Merry /* 1753130f4520SKenneth D. Merry * Make sure we only get SCSI I/O. 1754130f4520SKenneth D. Merry */ 1755130f4520SKenneth D. Merry KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type " 1756130f4520SKenneth D. Merry "%#x) encountered", io->io_hdr.io_type)); 1757130f4520SKenneth D. Merry 1758e86a4142SAlexander Motin PRIV(io)->len = 0; 1759e86a4142SAlexander Motin 176075c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1761130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 176275c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1763130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1764130f4520SKenneth D. Merry 17659c71cd5aSAlexander Motin return (CTL_RETVAL_COMPLETE); 1766130f4520SKenneth D. Merry } 1767130f4520SKenneth D. Merry 1768130f4520SKenneth D. Merry static int 1769130f4520SKenneth D. Merry ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 1770130f4520SKenneth D. Merry int flag, struct thread *td) 1771130f4520SKenneth D. Merry { 1772130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1773130f4520SKenneth D. Merry int error; 1774130f4520SKenneth D. Merry 1775130f4520SKenneth D. Merry softc = &backend_block_softc; 1776130f4520SKenneth D. Merry 1777130f4520SKenneth D. Merry error = 0; 1778130f4520SKenneth D. Merry 1779130f4520SKenneth D. Merry switch (cmd) { 1780130f4520SKenneth D. Merry case CTL_LUN_REQ: { 1781130f4520SKenneth D. Merry struct ctl_lun_req *lun_req; 1782130f4520SKenneth D. Merry 1783130f4520SKenneth D. Merry lun_req = (struct ctl_lun_req *)addr; 1784130f4520SKenneth D. Merry 1785130f4520SKenneth D. Merry switch (lun_req->reqtype) { 1786130f4520SKenneth D. Merry case CTL_LUNREQ_CREATE: 1787130f4520SKenneth D. Merry error = ctl_be_block_create(softc, lun_req); 1788130f4520SKenneth D. Merry break; 1789130f4520SKenneth D. Merry case CTL_LUNREQ_RM: 1790130f4520SKenneth D. Merry error = ctl_be_block_rm(softc, lun_req); 1791130f4520SKenneth D. Merry break; 179281177295SEdward Tomasz Napierala case CTL_LUNREQ_MODIFY: 179381177295SEdward Tomasz Napierala error = ctl_be_block_modify(softc, lun_req); 179481177295SEdward Tomasz Napierala break; 1795130f4520SKenneth D. Merry default: 1796130f4520SKenneth D. Merry lun_req->status = CTL_LUN_ERROR; 1797130f4520SKenneth D. Merry snprintf(lun_req->error_str, sizeof(lun_req->error_str), 179819720f41SAlexander Motin "invalid LUN request type %d", 1799130f4520SKenneth D. Merry lun_req->reqtype); 1800130f4520SKenneth D. Merry break; 1801130f4520SKenneth D. Merry } 1802130f4520SKenneth D. Merry break; 1803130f4520SKenneth D. Merry } 1804130f4520SKenneth D. Merry default: 1805130f4520SKenneth D. Merry error = ENOTTY; 1806130f4520SKenneth D. Merry break; 1807130f4520SKenneth D. Merry } 1808130f4520SKenneth D. Merry 1809130f4520SKenneth D. Merry return (error); 1810130f4520SKenneth D. Merry } 1811130f4520SKenneth D. Merry 1812130f4520SKenneth D. Merry static int 1813130f4520SKenneth D. Merry ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1814130f4520SKenneth D. Merry { 18150bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 1816130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 1817130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 181834961f40SAlexander Motin char *value; 1819130f4520SKenneth D. Merry struct vattr vattr; 182034961f40SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos; 1821130f4520SKenneth D. Merry int error; 1822130f4520SKenneth D. Merry 18230bcd4ab6SAlexander Motin cbe_lun = &be_lun->cbe_lun; 1824130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 182519720f41SAlexander Motin params = &be_lun->params; 1826130f4520SKenneth D. Merry 1827130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_FILE; 1828130f4520SKenneth D. Merry be_lun->dispatch = ctl_be_block_dispatch_file; 1829130f4520SKenneth D. Merry be_lun->lun_flush = ctl_be_block_flush_file; 1830ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_file; 183153c146deSAlexander Motin be_lun->getattr = ctl_be_block_getattr_file; 18320bcd4ab6SAlexander Motin be_lun->unmap = NULL; 18330bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; 1834130f4520SKenneth D. Merry 1835130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 1836130f4520SKenneth D. Merry if (error != 0) { 1837130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1838130f4520SKenneth D. Merry "error calling VOP_GETATTR() for file %s", 1839130f4520SKenneth D. Merry be_lun->dev_path); 1840130f4520SKenneth D. Merry return (error); 1841130f4520SKenneth D. Merry } 1842130f4520SKenneth D. Merry 1843130f4520SKenneth D. Merry file_data->cred = crhold(curthread->td_ucred); 184481177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) 184581177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 184681177295SEdward Tomasz Napierala else 1847130f4520SKenneth D. Merry be_lun->size_bytes = vattr.va_size; 1848130f4520SKenneth D. Merry 1849130f4520SKenneth D. Merry /* 185020a28d6cSAlexander Motin * For files we can use any logical block size. Prefer 512 bytes 185120a28d6cSAlexander Motin * for compatibility reasons. If file's vattr.va_blocksize 185220a28d6cSAlexander Motin * (preferred I/O block size) is bigger and multiple to chosen 185320a28d6cSAlexander Motin * logical block size -- report it as physical block size. 1854130f4520SKenneth D. Merry */ 1855130f4520SKenneth D. Merry if (params->blocksize_bytes != 0) 18560bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 185791be33dcSAlexander Motin else if (cbe_lun->lun_type == T_CDROM) 185891be33dcSAlexander Motin cbe_lun->blocksize = 2048; 1859130f4520SKenneth D. Merry else 18600bcd4ab6SAlexander Motin cbe_lun->blocksize = 512; 18610bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 18620bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 18630bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 186434961f40SAlexander Motin 186534961f40SAlexander Motin us = ps = vattr.va_blocksize; 186634961f40SAlexander Motin uo = po = 0; 186734961f40SAlexander Motin 18680bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "pblocksize"); 186934961f40SAlexander Motin if (value != NULL) 187034961f40SAlexander Motin ctl_expand_number(value, &ps); 18710bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "pblockoffset"); 187234961f40SAlexander Motin if (value != NULL) 187334961f40SAlexander Motin ctl_expand_number(value, &po); 18740bcd4ab6SAlexander Motin pss = ps / cbe_lun->blocksize; 18750bcd4ab6SAlexander Motin pos = po / cbe_lun->blocksize; 18760bcd4ab6SAlexander Motin if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && 18770bcd4ab6SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { 18780bcd4ab6SAlexander Motin cbe_lun->pblockexp = fls(pss) - 1; 18790bcd4ab6SAlexander Motin cbe_lun->pblockoff = (pss - pos) % pss; 188034961f40SAlexander Motin } 188134961f40SAlexander Motin 18820bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ublocksize"); 188334961f40SAlexander Motin if (value != NULL) 188434961f40SAlexander Motin ctl_expand_number(value, &us); 18850bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ublockoffset"); 188634961f40SAlexander Motin if (value != NULL) 188734961f40SAlexander Motin ctl_expand_number(value, &uo); 18880bcd4ab6SAlexander Motin uss = us / cbe_lun->blocksize; 18890bcd4ab6SAlexander Motin uos = uo / cbe_lun->blocksize; 18900bcd4ab6SAlexander Motin if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && 18910bcd4ab6SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { 18920bcd4ab6SAlexander Motin cbe_lun->ublockexp = fls(uss) - 1; 18930bcd4ab6SAlexander Motin cbe_lun->ublockoff = (uss - uos) % uss; 189420a28d6cSAlexander Motin } 1895130f4520SKenneth D. Merry 1896130f4520SKenneth D. Merry /* 1897130f4520SKenneth D. Merry * Sanity check. The media size has to be at least one 1898130f4520SKenneth D. Merry * sector long. 1899130f4520SKenneth D. Merry */ 19000bcd4ab6SAlexander Motin if (be_lun->size_bytes < cbe_lun->blocksize) { 1901130f4520SKenneth D. Merry error = EINVAL; 1902130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1903130f4520SKenneth D. Merry "file %s size %ju < block size %u", be_lun->dev_path, 19040bcd4ab6SAlexander Motin (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize); 1905130f4520SKenneth D. Merry } 1906cb8727e2SAlexander Motin 19070bcd4ab6SAlexander Motin cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize; 1908130f4520SKenneth D. Merry return (error); 1909130f4520SKenneth D. Merry } 1910130f4520SKenneth D. Merry 1911130f4520SKenneth D. Merry static int 1912130f4520SKenneth D. Merry ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1913130f4520SKenneth D. Merry { 19140bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1915130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 19163236151eSAlexander Motin struct cdevsw *csw; 1917130f4520SKenneth D. Merry struct cdev *dev; 191834961f40SAlexander Motin char *value; 19193236151eSAlexander Motin int error, atomic, maxio, ref, unmap, tmp; 1920f6295033SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos, otmp; 1921130f4520SKenneth D. Merry 192219720f41SAlexander Motin params = &be_lun->params; 1923130f4520SKenneth D. Merry 1924130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_DEV; 19253236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 19263236151eSAlexander Motin if (csw == NULL) 19273236151eSAlexander Motin return (ENXIO); 19283236151eSAlexander Motin if (strcmp(csw->d_name, "zvol") == 0) { 192967f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_zvol; 1930ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_zvol; 1931cb8727e2SAlexander Motin atomic = maxio = CTLBLK_MAX_IO_SIZE; 1932cb8727e2SAlexander Motin } else { 193367f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_dev; 19340bcd4ab6SAlexander Motin be_lun->get_lba_status = NULL; 1935cb8727e2SAlexander Motin atomic = 0; 19363236151eSAlexander Motin maxio = dev->si_iosize_max; 1937cb8727e2SAlexander Motin if (maxio <= 0) 1938cb8727e2SAlexander Motin maxio = DFLTPHYS; 1939cb8727e2SAlexander Motin if (maxio > CTLBLK_MAX_IO_SIZE) 1940cb8727e2SAlexander Motin maxio = CTLBLK_MAX_IO_SIZE; 1941cb8727e2SAlexander Motin } 194267f586a8SAlexander Motin be_lun->lun_flush = ctl_be_block_flush_dev; 1943c3e7ba3eSAlexander Motin be_lun->getattr = ctl_be_block_getattr_dev; 19440bcd4ab6SAlexander Motin be_lun->unmap = ctl_be_block_unmap_dev; 1945130f4520SKenneth D. Merry 19463236151eSAlexander Motin if (!csw->d_ioctl) { 19473236151eSAlexander Motin dev_relthread(dev, ref); 1948130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 19493236151eSAlexander Motin "no d_ioctl for device %s!", be_lun->dev_path); 1950130f4520SKenneth D. Merry return (ENODEV); 1951130f4520SKenneth D. Merry } 1952130f4520SKenneth D. Merry 19533236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD, 1954130f4520SKenneth D. Merry curthread); 1955130f4520SKenneth D. Merry if (error) { 19563236151eSAlexander Motin dev_relthread(dev, ref); 1957130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 195819720f41SAlexander Motin "error %d returned for DIOCGSECTORSIZE ioctl " 195919720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 1960130f4520SKenneth D. Merry return (error); 1961130f4520SKenneth D. Merry } 1962130f4520SKenneth D. Merry 1963130f4520SKenneth D. Merry /* 1964130f4520SKenneth D. Merry * If the user has asked for a blocksize that is greater than the 1965130f4520SKenneth D. Merry * backing device's blocksize, we can do it only if the blocksize 1966130f4520SKenneth D. Merry * the user is asking for is an even multiple of the underlying 1967130f4520SKenneth D. Merry * device's blocksize. 1968130f4520SKenneth D. Merry */ 1969a15bbf15SAlexander Motin if ((params->blocksize_bytes != 0) && 1970a15bbf15SAlexander Motin (params->blocksize_bytes >= tmp)) { 1971a15bbf15SAlexander Motin if (params->blocksize_bytes % tmp == 0) { 19720bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 1973130f4520SKenneth D. Merry } else { 19743236151eSAlexander Motin dev_relthread(dev, ref); 1975130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 197619720f41SAlexander Motin "requested blocksize %u is not an even " 1977130f4520SKenneth D. Merry "multiple of backing device blocksize %u", 1978f6295033SAlexander Motin params->blocksize_bytes, tmp); 1979130f4520SKenneth D. Merry return (EINVAL); 1980130f4520SKenneth D. Merry } 1981a15bbf15SAlexander Motin } else if (params->blocksize_bytes != 0) { 19823236151eSAlexander Motin dev_relthread(dev, ref); 1983130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 198419720f41SAlexander Motin "requested blocksize %u < backing device " 1985f6295033SAlexander Motin "blocksize %u", params->blocksize_bytes, tmp); 1986130f4520SKenneth D. Merry return (EINVAL); 198791be33dcSAlexander Motin } else if (cbe_lun->lun_type == T_CDROM) 198891be33dcSAlexander Motin cbe_lun->blocksize = MAX(tmp, 2048); 198991be33dcSAlexander Motin else 19900bcd4ab6SAlexander Motin cbe_lun->blocksize = tmp; 1991130f4520SKenneth D. Merry 19923236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD, 1993130f4520SKenneth D. Merry curthread); 1994130f4520SKenneth D. Merry if (error) { 19953236151eSAlexander Motin dev_relthread(dev, ref); 1996130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 199719720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE " 199819720f41SAlexander Motin " ioctl on %s!", error, 199981177295SEdward Tomasz Napierala be_lun->dev_path); 2000130f4520SKenneth D. Merry return (error); 2001130f4520SKenneth D. Merry } 2002130f4520SKenneth D. Merry 200381177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 2004f6295033SAlexander Motin if (params->lun_size_bytes > otmp) { 20053236151eSAlexander Motin dev_relthread(dev, ref); 200681177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 200719720f41SAlexander Motin "requested LUN size %ju > backing device " 200819720f41SAlexander Motin "size %ju", 200981177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 2010f6295033SAlexander Motin (uintmax_t)otmp); 201181177295SEdward Tomasz Napierala return (EINVAL); 2012130f4520SKenneth D. Merry } 2013130f4520SKenneth D. Merry 201481177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 2015a15bbf15SAlexander Motin } else 2016f6295033SAlexander Motin be_lun->size_bytes = otmp; 20170bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 20180bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 20190bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 202081177295SEdward Tomasz Napierala 20213236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD, 20223236151eSAlexander Motin curthread); 2023f6012722SAlexander Motin if (error) 2024f6012722SAlexander Motin ps = po = 0; 2025f6012722SAlexander Motin else { 20263236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po, 20273236151eSAlexander Motin FREAD, curthread); 2028f6012722SAlexander Motin if (error) 2029f6012722SAlexander Motin po = 0; 2030f6012722SAlexander Motin } 203134961f40SAlexander Motin us = ps; 203234961f40SAlexander Motin uo = po; 203334961f40SAlexander Motin 20340bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "pblocksize"); 203534961f40SAlexander Motin if (value != NULL) 203634961f40SAlexander Motin ctl_expand_number(value, &ps); 20370bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "pblockoffset"); 203834961f40SAlexander Motin if (value != NULL) 203934961f40SAlexander Motin ctl_expand_number(value, &po); 20400bcd4ab6SAlexander Motin pss = ps / cbe_lun->blocksize; 20410bcd4ab6SAlexander Motin pos = po / cbe_lun->blocksize; 20420bcd4ab6SAlexander Motin if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && 20430bcd4ab6SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { 20440bcd4ab6SAlexander Motin cbe_lun->pblockexp = fls(pss) - 1; 20450bcd4ab6SAlexander Motin cbe_lun->pblockoff = (pss - pos) % pss; 2046f6012722SAlexander Motin } 2047f6012722SAlexander Motin 20480bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ublocksize"); 204934961f40SAlexander Motin if (value != NULL) 205034961f40SAlexander Motin ctl_expand_number(value, &us); 20510bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ublockoffset"); 205234961f40SAlexander Motin if (value != NULL) 205334961f40SAlexander Motin ctl_expand_number(value, &uo); 20540bcd4ab6SAlexander Motin uss = us / cbe_lun->blocksize; 20550bcd4ab6SAlexander Motin uos = uo / cbe_lun->blocksize; 20560bcd4ab6SAlexander Motin if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && 20570bcd4ab6SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { 20580bcd4ab6SAlexander Motin cbe_lun->ublockexp = fls(uss) - 1; 20590bcd4ab6SAlexander Motin cbe_lun->ublockoff = (uss - uos) % uss; 206034961f40SAlexander Motin } 206134961f40SAlexander Motin 20620bcd4ab6SAlexander Motin cbe_lun->atomicblock = atomic / cbe_lun->blocksize; 20630bcd4ab6SAlexander Motin cbe_lun->opttxferlen = maxio / cbe_lun->blocksize; 2064fbc8d4ffSAlexander Motin 2065fbc8d4ffSAlexander Motin if (be_lun->dispatch == ctl_be_block_dispatch_zvol) { 2066fbc8d4ffSAlexander Motin unmap = 1; 2067fbc8d4ffSAlexander Motin } else { 2068fbc8d4ffSAlexander Motin struct diocgattr_arg arg; 2069fbc8d4ffSAlexander Motin 2070fbc8d4ffSAlexander Motin strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); 2071fbc8d4ffSAlexander Motin arg.len = sizeof(arg.value.i); 20723236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, 20733236151eSAlexander Motin curthread); 2074fbc8d4ffSAlexander Motin unmap = (error == 0) ? arg.value.i : 0; 2075fbc8d4ffSAlexander Motin } 20760bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "unmap"); 2077fbc8d4ffSAlexander Motin if (value != NULL) 2078fbc8d4ffSAlexander Motin unmap = (strcmp(value, "on") == 0); 2079fbc8d4ffSAlexander Motin if (unmap) 20800bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_UNMAP; 20810bcd4ab6SAlexander Motin else 20820bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; 2083fbc8d4ffSAlexander Motin 20843236151eSAlexander Motin dev_relthread(dev, ref); 208581177295SEdward Tomasz Napierala return (0); 208681177295SEdward Tomasz Napierala } 2087130f4520SKenneth D. Merry 2088130f4520SKenneth D. Merry static int 2089130f4520SKenneth D. Merry ctl_be_block_close(struct ctl_be_block_lun *be_lun) 2090130f4520SKenneth D. Merry { 20910bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 20920bcd4ab6SAlexander Motin int flags; 2093130f4520SKenneth D. Merry 20940bcd4ab6SAlexander Motin if (be_lun->vn) { 20950bcd4ab6SAlexander Motin flags = FREAD; 20960bcd4ab6SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0) 20970bcd4ab6SAlexander Motin flags |= FWRITE; 2098130f4520SKenneth D. Merry (void)vn_close(be_lun->vn, flags, NOCRED, curthread); 2099130f4520SKenneth D. Merry be_lun->vn = NULL; 2100130f4520SKenneth D. Merry 2101130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2102130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2103130f4520SKenneth D. Merry break; 2104130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2105130f4520SKenneth D. Merry if (be_lun->backend.file.cred != NULL) { 2106130f4520SKenneth D. Merry crfree(be_lun->backend.file.cred); 2107130f4520SKenneth D. Merry be_lun->backend.file.cred = NULL; 2108130f4520SKenneth D. Merry } 2109130f4520SKenneth D. Merry break; 2110130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2111025a2301SEdward Tomasz Napierala break; 2112130f4520SKenneth D. Merry default: 21135124012aSAlexander Motin panic("Unexpected backend type %d", be_lun->dev_type); 2114130f4520SKenneth D. Merry break; 2115130f4520SKenneth D. Merry } 211619720f41SAlexander Motin be_lun->dev_type = CTL_BE_BLOCK_NONE; 2117130f4520SKenneth D. Merry } 2118130f4520SKenneth D. Merry return (0); 2119130f4520SKenneth D. Merry } 2120130f4520SKenneth D. Merry 2121130f4520SKenneth D. Merry static int 2122648dfc1aSAlexander Motin ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 2123130f4520SKenneth D. Merry { 21240bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 2125130f4520SKenneth D. Merry struct nameidata nd; 21260bcd4ab6SAlexander Motin char *value; 21270bcd4ab6SAlexander Motin int error, flags; 2128130f4520SKenneth D. Merry 2129130f4520SKenneth D. Merry error = 0; 2130130f4520SKenneth D. Merry if (rootvnode == NULL) { 2131130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 213219720f41SAlexander Motin "Root filesystem is not mounted"); 2133130f4520SKenneth D. Merry return (1); 2134130f4520SKenneth D. Merry } 21358a08cec1SMateusz Guzik pwd_ensure_dirs(); 2136130f4520SKenneth D. Merry 21370bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "file"); 21380bcd4ab6SAlexander Motin if (value == NULL) { 21390bcd4ab6SAlexander Motin snprintf(req->error_str, sizeof(req->error_str), 21400bcd4ab6SAlexander Motin "no file argument specified"); 21410bcd4ab6SAlexander Motin return (1); 21420bcd4ab6SAlexander Motin } 21430bcd4ab6SAlexander Motin free(be_lun->dev_path, M_CTLBLK); 21440bcd4ab6SAlexander Motin be_lun->dev_path = strdup(value, M_CTLBLK); 21450bcd4ab6SAlexander Motin 21460bcd4ab6SAlexander Motin flags = FREAD; 21470bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "readonly"); 214891be33dcSAlexander Motin if (value != NULL) { 214991be33dcSAlexander Motin if (strcmp(value, "on") != 0) 215091be33dcSAlexander Motin flags |= FWRITE; 215191be33dcSAlexander Motin } else if (cbe_lun->lun_type == T_DIRECT) 21520bcd4ab6SAlexander Motin flags |= FWRITE; 21530bcd4ab6SAlexander Motin 2154130f4520SKenneth D. Merry again: 2155130f4520SKenneth D. Merry NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread); 2156130f4520SKenneth D. Merry error = vn_open(&nd, &flags, 0, NULL); 21570bcd4ab6SAlexander Motin if ((error == EROFS || error == EACCES) && (flags & FWRITE)) { 21580bcd4ab6SAlexander Motin flags &= ~FWRITE; 21590bcd4ab6SAlexander Motin goto again; 21600bcd4ab6SAlexander Motin } 2161130f4520SKenneth D. Merry if (error) { 2162130f4520SKenneth D. Merry /* 2163130f4520SKenneth D. Merry * This is the only reasonable guess we can make as far as 2164130f4520SKenneth D. Merry * path if the user doesn't give us a fully qualified path. 2165130f4520SKenneth D. Merry * If they want to specify a file, they need to specify the 2166130f4520SKenneth D. Merry * full path. 2167130f4520SKenneth D. Merry */ 2168130f4520SKenneth D. Merry if (be_lun->dev_path[0] != '/') { 2169130f4520SKenneth D. Merry char *dev_name; 2170130f4520SKenneth D. Merry 21710bcd4ab6SAlexander Motin asprintf(&dev_name, M_CTLBLK, "/dev/%s", 2172130f4520SKenneth D. Merry be_lun->dev_path); 2173130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 2174130f4520SKenneth D. Merry be_lun->dev_path = dev_name; 2175130f4520SKenneth D. Merry goto again; 2176130f4520SKenneth D. Merry } 2177130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 217819720f41SAlexander Motin "error opening %s: %d", be_lun->dev_path, error); 2179130f4520SKenneth D. Merry return (error); 2180130f4520SKenneth D. Merry } 21810bcd4ab6SAlexander Motin if (flags & FWRITE) 21820bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY; 21830bcd4ab6SAlexander Motin else 21840bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_READONLY; 2185130f4520SKenneth D. Merry 2186130f4520SKenneth D. Merry NDFREE(&nd, NDF_ONLY_PNBUF); 2187130f4520SKenneth D. Merry be_lun->vn = nd.ni_vp; 2188130f4520SKenneth D. Merry 2189130f4520SKenneth D. Merry /* We only support disks and files. */ 2190130f4520SKenneth D. Merry if (vn_isdisk(be_lun->vn, &error)) { 2191130f4520SKenneth D. Merry error = ctl_be_block_open_dev(be_lun, req); 2192130f4520SKenneth D. Merry } else if (be_lun->vn->v_type == VREG) { 2193130f4520SKenneth D. Merry error = ctl_be_block_open_file(be_lun, req); 2194130f4520SKenneth D. Merry } else { 2195130f4520SKenneth D. Merry error = EINVAL; 2196130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 2197025a2301SEdward Tomasz Napierala "%s is not a disk or plain file", be_lun->dev_path); 2198130f4520SKenneth D. Merry } 2199130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 2200130f4520SKenneth D. Merry 2201a15bbf15SAlexander Motin if (error != 0) 2202130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 22030bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; 22040bcd4ab6SAlexander Motin if (be_lun->dispatch != ctl_be_block_dispatch_dev) 22050bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_READ; 22060bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "serseq"); 22070bcd4ab6SAlexander Motin if (value != NULL && strcmp(value, "on") == 0) 22080bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_ON; 22090bcd4ab6SAlexander Motin else if (value != NULL && strcmp(value, "read") == 0) 22100bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_READ; 22110bcd4ab6SAlexander Motin else if (value != NULL && strcmp(value, "off") == 0) 22120bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; 2213130f4520SKenneth D. Merry return (0); 2214130f4520SKenneth D. Merry } 2215130f4520SKenneth D. Merry 2216130f4520SKenneth D. Merry static int 2217130f4520SKenneth D. Merry ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2218130f4520SKenneth D. Merry { 22190bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2220130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2221130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 222257a5db13SAlexander Motin char num_thread_str[16]; 2223130f4520SKenneth D. Merry char tmpstr[32]; 222457a5db13SAlexander Motin char *value; 2225fbc8d4ffSAlexander Motin int retval, num_threads; 222657a5db13SAlexander Motin int tmp_num_threads; 2227130f4520SKenneth D. Merry 2228130f4520SKenneth D. Merry params = &req->reqdata.create; 2229130f4520SKenneth D. Merry retval = 0; 223019720f41SAlexander Motin req->status = CTL_LUN_OK; 2231130f4520SKenneth D. Merry 2232130f4520SKenneth D. Merry be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK); 22330bcd4ab6SAlexander Motin cbe_lun = &be_lun->cbe_lun; 22340bcd4ab6SAlexander Motin cbe_lun->be_lun = be_lun; 223519720f41SAlexander Motin be_lun->params = req->reqdata.create; 2236130f4520SKenneth D. Merry be_lun->softc = softc; 2237130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->input_queue); 2238ef8daf3fSAlexander Motin STAILQ_INIT(&be_lun->config_read_queue); 2239130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->config_write_queue); 2240130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->datamove_queue); 2241130f4520SKenneth D. Merry sprintf(be_lun->lunname, "cblk%d", softc->num_luns); 224275c7a1d3SAlexander Motin mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF); 224375c7a1d3SAlexander Motin mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF); 22440bcd4ab6SAlexander Motin ctl_init_opts(&cbe_lun->options, 224543fb3a65SAlexander Motin req->num_be_args, req->kern_be_args); 224608a7cce5SAlexander Motin be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG, 2247eeb94054SAlexander Motin NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0); 2248130f4520SKenneth D. Merry if (be_lun->lun_zone == NULL) { 2249130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 225019720f41SAlexander Motin "error allocating UMA zone"); 2251130f4520SKenneth D. Merry goto bailout_error; 2252130f4520SKenneth D. Merry } 2253130f4520SKenneth D. Merry 2254130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_DEV_TYPE) 22550bcd4ab6SAlexander Motin cbe_lun->lun_type = params->device_type; 2256130f4520SKenneth D. Merry else 22570bcd4ab6SAlexander Motin cbe_lun->lun_type = T_DIRECT; 22580bcd4ab6SAlexander Motin be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED; 22597ac58230SAlexander Motin cbe_lun->flags = 0; 22607ac58230SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ha_role"); 22617ac58230SAlexander Motin if (value != NULL) { 22627ac58230SAlexander Motin if (strcmp(value, "primary") == 0) 22637ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 22647ac58230SAlexander Motin } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) 22657ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 2266130f4520SKenneth D. Merry 226791be33dcSAlexander Motin if (cbe_lun->lun_type == T_DIRECT || 226891be33dcSAlexander Motin cbe_lun->lun_type == T_CDROM) { 2269a15bbf15SAlexander Motin be_lun->size_bytes = params->lun_size_bytes; 2270a15bbf15SAlexander Motin if (params->blocksize_bytes != 0) 22710bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 227291be33dcSAlexander Motin else if (cbe_lun->lun_type == T_CDROM) 227391be33dcSAlexander Motin cbe_lun->blocksize = 2048; 2274a15bbf15SAlexander Motin else 22750bcd4ab6SAlexander Motin cbe_lun->blocksize = 512; 22760bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 22770bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 22780bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 2279130f4520SKenneth D. Merry 22807ac58230SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || 22817ac58230SAlexander Motin control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { 2282648dfc1aSAlexander Motin retval = ctl_be_block_open(be_lun, req); 2283130f4520SKenneth D. Merry if (retval != 0) { 2284130f4520SKenneth D. Merry retval = 0; 228519720f41SAlexander Motin req->status = CTL_LUN_WARNING; 2286130f4520SKenneth D. Merry } 22877ac58230SAlexander Motin } 22880bcd4ab6SAlexander Motin num_threads = cbb_num_threads; 2289130f4520SKenneth D. Merry } else { 2290130f4520SKenneth D. Merry num_threads = 1; 2291130f4520SKenneth D. Merry } 2292130f4520SKenneth D. Merry 22930bcd4ab6SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "num_threads"); 229457a5db13SAlexander Motin if (value != NULL) { 229557a5db13SAlexander Motin tmp_num_threads = strtol(value, NULL, 0); 2296130f4520SKenneth D. Merry 2297130f4520SKenneth D. Merry /* 2298130f4520SKenneth D. Merry * We don't let the user specify less than one 2299130f4520SKenneth D. Merry * thread, but hope he's clueful enough not to 2300130f4520SKenneth D. Merry * specify 1000 threads. 2301130f4520SKenneth D. Merry */ 2302130f4520SKenneth D. Merry if (tmp_num_threads < 1) { 2303130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 230419720f41SAlexander Motin "invalid number of threads %s", 230519720f41SAlexander Motin num_thread_str); 2306130f4520SKenneth D. Merry goto bailout_error; 2307130f4520SKenneth D. Merry } 2308130f4520SKenneth D. Merry num_threads = tmp_num_threads; 230957a5db13SAlexander Motin } 2310130f4520SKenneth D. Merry 231119720f41SAlexander Motin if (be_lun->vn == NULL) 2312648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2313130f4520SKenneth D. Merry /* Tell the user the blocksize we ended up using */ 231419720f41SAlexander Motin params->lun_size_bytes = be_lun->size_bytes; 23150bcd4ab6SAlexander Motin params->blocksize_bytes = cbe_lun->blocksize; 2316130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_ID_REQ) { 23170bcd4ab6SAlexander Motin cbe_lun->req_lun_id = params->req_lun_id; 23180bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ; 2319130f4520SKenneth D. Merry } else 23200bcd4ab6SAlexander Motin cbe_lun->req_lun_id = 0; 2321130f4520SKenneth D. Merry 23220bcd4ab6SAlexander Motin cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown; 23230bcd4ab6SAlexander Motin cbe_lun->lun_config_status = ctl_be_block_lun_config_status; 23240bcd4ab6SAlexander Motin cbe_lun->be = &ctl_be_block_driver; 2325130f4520SKenneth D. Merry 2326130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { 2327*71cd87c6SAlan Somers snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%04d", 2328130f4520SKenneth D. Merry softc->num_luns); 23290bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->serial_num, tmpstr, 23300bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr))); 2331130f4520SKenneth D. Merry 2332130f4520SKenneth D. Merry /* Tell the user what we used for a serial number */ 2333130f4520SKenneth D. Merry strncpy((char *)params->serial_num, tmpstr, 2334e7038eb7SAlexander Motin MIN(sizeof(params->serial_num), sizeof(tmpstr))); 2335130f4520SKenneth D. Merry } else { 23360bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->serial_num, params->serial_num, 23370bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->serial_num), 2338130f4520SKenneth D. Merry sizeof(params->serial_num))); 2339130f4520SKenneth D. Merry } 2340130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { 2341*71cd87c6SAlan Somers snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns); 23420bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->device_id, tmpstr, 23430bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr))); 2344130f4520SKenneth D. Merry 2345130f4520SKenneth D. Merry /* Tell the user what we used for a device ID */ 2346130f4520SKenneth D. Merry strncpy((char *)params->device_id, tmpstr, 2347e7038eb7SAlexander Motin MIN(sizeof(params->device_id), sizeof(tmpstr))); 2348130f4520SKenneth D. Merry } else { 23490bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->device_id, params->device_id, 23500bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->device_id), 2351130f4520SKenneth D. Merry sizeof(params->device_id))); 2352130f4520SKenneth D. Merry } 2353130f4520SKenneth D. Merry 2354130f4520SKenneth D. Merry TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); 2355130f4520SKenneth D. Merry 2356130f4520SKenneth D. Merry be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK, 2357130f4520SKenneth D. Merry taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); 2358130f4520SKenneth D. Merry 2359130f4520SKenneth D. Merry if (be_lun->io_taskqueue == NULL) { 2360130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 236119720f41SAlexander Motin "unable to create taskqueue"); 2362130f4520SKenneth D. Merry goto bailout_error; 2363130f4520SKenneth D. Merry } 2364130f4520SKenneth D. Merry 2365130f4520SKenneth D. Merry /* 2366130f4520SKenneth D. Merry * Note that we start the same number of threads by default for 2367130f4520SKenneth D. Merry * both the file case and the block device case. For the file 2368130f4520SKenneth D. Merry * case, we need multiple threads to allow concurrency, because the 2369130f4520SKenneth D. Merry * vnode interface is designed to be a blocking interface. For the 2370130f4520SKenneth D. Merry * block device case, ZFS zvols at least will block the caller's 2371130f4520SKenneth D. Merry * context in many instances, and so we need multiple threads to 2372130f4520SKenneth D. Merry * overcome that problem. Other block devices don't need as many 2373130f4520SKenneth D. Merry * threads, but they shouldn't cause too many problems. 2374130f4520SKenneth D. Merry * 2375130f4520SKenneth D. Merry * If the user wants to just have a single thread for a block 2376130f4520SKenneth D. Merry * device, he can specify that when the LUN is created, or change 2377130f4520SKenneth D. Merry * the tunable/sysctl to alter the default number of threads. 2378130f4520SKenneth D. Merry */ 2379130f4520SKenneth D. Merry retval = taskqueue_start_threads(&be_lun->io_taskqueue, 2380130f4520SKenneth D. Merry /*num threads*/num_threads, 2381130f4520SKenneth D. Merry /*priority*/PWAIT, 2382130f4520SKenneth D. Merry /*thread name*/ 2383130f4520SKenneth D. Merry "%s taskq", be_lun->lunname); 2384130f4520SKenneth D. Merry 2385130f4520SKenneth D. Merry if (retval != 0) 2386130f4520SKenneth D. Merry goto bailout_error; 2387130f4520SKenneth D. Merry 2388130f4520SKenneth D. Merry be_lun->num_threads = num_threads; 2389130f4520SKenneth D. Merry 2390130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2391130f4520SKenneth D. Merry softc->num_luns++; 2392130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links); 2393130f4520SKenneth D. Merry 2394130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2395130f4520SKenneth D. Merry 23960bcd4ab6SAlexander Motin retval = ctl_add_lun(&be_lun->cbe_lun); 2397130f4520SKenneth D. Merry if (retval != 0) { 2398130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2399130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2400130f4520SKenneth D. Merry links); 2401130f4520SKenneth D. Merry softc->num_luns--; 2402130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2403130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 240419720f41SAlexander Motin "ctl_add_lun() returned error %d, see dmesg for " 240519720f41SAlexander Motin "details", retval); 2406130f4520SKenneth D. Merry retval = 0; 2407130f4520SKenneth D. Merry goto bailout_error; 2408130f4520SKenneth D. Merry } 2409130f4520SKenneth D. Merry 2410130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2411130f4520SKenneth D. Merry 2412130f4520SKenneth D. Merry /* 2413130f4520SKenneth D. Merry * Tell the config_status routine that we're waiting so it won't 2414130f4520SKenneth D. Merry * clean up the LUN in the event of an error. 2415130f4520SKenneth D. Merry */ 2416130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2417130f4520SKenneth D. Merry 2418130f4520SKenneth D. Merry while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { 2419130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2420130f4520SKenneth D. Merry if (retval == EINTR) 2421130f4520SKenneth D. Merry break; 2422130f4520SKenneth D. Merry } 2423130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2424130f4520SKenneth D. Merry 2425130f4520SKenneth D. Merry if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) { 2426130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 242719720f41SAlexander Motin "LUN configuration error, see dmesg for details"); 2428130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2429130f4520SKenneth D. Merry links); 2430130f4520SKenneth D. Merry softc->num_luns--; 2431130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2432130f4520SKenneth D. Merry goto bailout_error; 2433130f4520SKenneth D. Merry } else { 24340bcd4ab6SAlexander Motin params->req_lun_id = cbe_lun->lun_id; 2435130f4520SKenneth D. Merry } 2436130f4520SKenneth D. Merry 2437130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2438130f4520SKenneth D. Merry 2439130f4520SKenneth D. Merry be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id, 24400bcd4ab6SAlexander Motin cbe_lun->blocksize, 2441130f4520SKenneth D. Merry DEVSTAT_ALL_SUPPORTED, 24420bcd4ab6SAlexander Motin cbe_lun->lun_type 2443130f4520SKenneth D. Merry | DEVSTAT_TYPE_IF_OTHER, 2444130f4520SKenneth D. Merry DEVSTAT_PRIORITY_OTHER); 2445130f4520SKenneth D. Merry 2446130f4520SKenneth D. Merry return (retval); 2447130f4520SKenneth D. Merry 2448130f4520SKenneth D. Merry bailout_error: 2449130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2450130f4520SKenneth D. Merry 24519e005bbcSAlexander Motin if (be_lun->io_taskqueue != NULL) 24529e005bbcSAlexander Motin taskqueue_free(be_lun->io_taskqueue); 2453130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 24549e005bbcSAlexander Motin if (be_lun->dev_path != NULL) 2455130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 24569e005bbcSAlexander Motin if (be_lun->lun_zone != NULL) 24579e005bbcSAlexander Motin uma_zdestroy(be_lun->lun_zone); 24580bcd4ab6SAlexander Motin ctl_free_opts(&cbe_lun->options); 245975c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 246075c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2461130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2462130f4520SKenneth D. Merry 2463130f4520SKenneth D. Merry return (retval); 2464130f4520SKenneth D. Merry } 2465130f4520SKenneth D. Merry 2466130f4520SKenneth D. Merry static int 2467130f4520SKenneth D. Merry ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2468130f4520SKenneth D. Merry { 2469130f4520SKenneth D. Merry struct ctl_lun_rm_params *params; 2470130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2471ee4ad294SAlexander Motin struct ctl_be_lun *cbe_lun; 2472130f4520SKenneth D. Merry int retval; 2473130f4520SKenneth D. Merry 2474130f4520SKenneth D. Merry params = &req->reqdata.rm; 2475130f4520SKenneth D. Merry 2476130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2477130f4520SKenneth D. Merry STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 24780bcd4ab6SAlexander Motin if (be_lun->cbe_lun.lun_id == params->lun_id) 2479130f4520SKenneth D. Merry break; 2480130f4520SKenneth D. Merry } 2481130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2482130f4520SKenneth D. Merry if (be_lun == NULL) { 2483130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 248419720f41SAlexander Motin "LUN %u is not managed by the block backend", 248519720f41SAlexander Motin params->lun_id); 2486130f4520SKenneth D. Merry goto bailout_error; 2487130f4520SKenneth D. Merry } 2488ee4ad294SAlexander Motin cbe_lun = &be_lun->cbe_lun; 2489130f4520SKenneth D. Merry 2490ee4ad294SAlexander Motin retval = ctl_disable_lun(cbe_lun); 2491130f4520SKenneth D. Merry if (retval != 0) { 2492130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 249319720f41SAlexander Motin "error %d returned from ctl_disable_lun() for " 249419720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2495130f4520SKenneth D. Merry goto bailout_error; 2496130f4520SKenneth D. Merry } 2497130f4520SKenneth D. Merry 2498ee4ad294SAlexander Motin if (be_lun->vn != NULL) { 2499648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2500648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2501ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 2502ee4ad294SAlexander Motin ctl_be_block_close(be_lun); 2503ee4ad294SAlexander Motin } 2504ee4ad294SAlexander Motin 2505ee4ad294SAlexander Motin retval = ctl_invalidate_lun(cbe_lun); 2506130f4520SKenneth D. Merry if (retval != 0) { 2507130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 250819720f41SAlexander Motin "error %d returned from ctl_invalidate_lun() for " 250919720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2510130f4520SKenneth D. Merry goto bailout_error; 2511130f4520SKenneth D. Merry } 2512130f4520SKenneth D. Merry 2513130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2514130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2515130f4520SKenneth D. Merry while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2516130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2517130f4520SKenneth D. Merry if (retval == EINTR) 2518130f4520SKenneth D. Merry break; 2519130f4520SKenneth D. Merry } 2520130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2521130f4520SKenneth D. Merry 2522130f4520SKenneth D. Merry if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2523130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 252419720f41SAlexander Motin "interrupted waiting for LUN to be freed"); 2525130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2526130f4520SKenneth D. Merry goto bailout_error; 2527130f4520SKenneth D. Merry } 2528130f4520SKenneth D. Merry 2529130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links); 2530130f4520SKenneth D. Merry 2531130f4520SKenneth D. Merry softc->num_luns--; 2532130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2533130f4520SKenneth D. Merry 2534ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 2535130f4520SKenneth D. Merry taskqueue_free(be_lun->io_taskqueue); 2536130f4520SKenneth D. Merry 2537130f4520SKenneth D. Merry if (be_lun->disk_stats != NULL) 2538130f4520SKenneth D. Merry devstat_remove_entry(be_lun->disk_stats); 2539130f4520SKenneth D. Merry 2540130f4520SKenneth D. Merry uma_zdestroy(be_lun->lun_zone); 2541130f4520SKenneth D. Merry 2542ee4ad294SAlexander Motin ctl_free_opts(&cbe_lun->options); 2543130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 254475c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 254575c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2546130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2547130f4520SKenneth D. Merry 2548130f4520SKenneth D. Merry req->status = CTL_LUN_OK; 2549130f4520SKenneth D. Merry return (0); 2550130f4520SKenneth D. Merry 2551130f4520SKenneth D. Merry bailout_error: 2552130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2553130f4520SKenneth D. Merry return (0); 2554130f4520SKenneth D. Merry } 2555130f4520SKenneth D. Merry 255681177295SEdward Tomasz Napierala static int 255781177295SEdward Tomasz Napierala ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 255881177295SEdward Tomasz Napierala { 255981177295SEdward Tomasz Napierala struct ctl_lun_modify_params *params; 256081177295SEdward Tomasz Napierala struct ctl_be_block_lun *be_lun; 2561a3977beaSAlexander Motin struct ctl_be_lun *cbe_lun; 25627ac58230SAlexander Motin char *value; 256371d8e97eSAlexander Motin uint64_t oldsize; 25647ac58230SAlexander Motin int error, wasprim; 256581177295SEdward Tomasz Napierala 256681177295SEdward Tomasz Napierala params = &req->reqdata.modify; 256781177295SEdward Tomasz Napierala 256881177295SEdward Tomasz Napierala mtx_lock(&softc->lock); 256981177295SEdward Tomasz Napierala STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 25700bcd4ab6SAlexander Motin if (be_lun->cbe_lun.lun_id == params->lun_id) 257181177295SEdward Tomasz Napierala break; 257281177295SEdward Tomasz Napierala } 257381177295SEdward Tomasz Napierala mtx_unlock(&softc->lock); 257481177295SEdward Tomasz Napierala if (be_lun == NULL) { 257581177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 257619720f41SAlexander Motin "LUN %u is not managed by the block backend", 257719720f41SAlexander Motin params->lun_id); 257881177295SEdward Tomasz Napierala goto bailout_error; 257981177295SEdward Tomasz Napierala } 2580a3977beaSAlexander Motin cbe_lun = &be_lun->cbe_lun; 258181177295SEdward Tomasz Napierala 2582a3977beaSAlexander Motin if (params->lun_size_bytes != 0) 258319720f41SAlexander Motin be_lun->params.lun_size_bytes = params->lun_size_bytes; 2584a3977beaSAlexander Motin ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args); 258581177295SEdward Tomasz Napierala 25867ac58230SAlexander Motin wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY); 25877ac58230SAlexander Motin value = ctl_get_opt(&cbe_lun->options, "ha_role"); 25887ac58230SAlexander Motin if (value != NULL) { 25897ac58230SAlexander Motin if (strcmp(value, "primary") == 0) 25907ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 25917ac58230SAlexander Motin else 25927ac58230SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; 25937ac58230SAlexander Motin } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) 25947ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 25957ac58230SAlexander Motin else 25967ac58230SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; 25977ac58230SAlexander Motin if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) { 25987ac58230SAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) 25997ac58230SAlexander Motin ctl_lun_primary(cbe_lun); 26007ac58230SAlexander Motin else 26017ac58230SAlexander Motin ctl_lun_secondary(cbe_lun); 26027ac58230SAlexander Motin } 26037ac58230SAlexander Motin 26040bcd4ab6SAlexander Motin oldsize = be_lun->size_blocks; 26057ac58230SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || 26067ac58230SAlexander Motin control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { 260719720f41SAlexander Motin if (be_lun->vn == NULL) 2608648dfc1aSAlexander Motin error = ctl_be_block_open(be_lun, req); 2609b9b4269cSAlexander Motin else if (vn_isdisk(be_lun->vn, &error)) 26104ce7a086SAlexander Motin error = ctl_be_block_open_dev(be_lun, req); 26113d5cb709SAlexander Motin else if (be_lun->vn->v_type == VREG) { 26123d5cb709SAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 26134ce7a086SAlexander Motin error = ctl_be_block_open_file(be_lun, req); 26143d5cb709SAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 26153d5cb709SAlexander Motin } else 2616b9b4269cSAlexander Motin error = EINVAL; 2617648dfc1aSAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) && 26180bcd4ab6SAlexander Motin be_lun->vn != NULL) { 2619648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; 2620648dfc1aSAlexander Motin ctl_lun_has_media(cbe_lun); 2621648dfc1aSAlexander Motin } else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 && 2622648dfc1aSAlexander Motin be_lun->vn == NULL) { 2623648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2624648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 262571d8e97eSAlexander Motin } 2626648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; 26277ac58230SAlexander Motin } else { 26287ac58230SAlexander Motin if (be_lun->vn != NULL) { 2629648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2630648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2631ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 26327ac58230SAlexander Motin error = ctl_be_block_close(be_lun); 26337ac58230SAlexander Motin } else 26347ac58230SAlexander Motin error = 0; 26357ac58230SAlexander Motin } 26367ac58230SAlexander Motin if (be_lun->size_blocks != oldsize) 26377ac58230SAlexander Motin ctl_lun_capacity_changed(cbe_lun); 263881177295SEdward Tomasz Napierala 263981177295SEdward Tomasz Napierala /* Tell the user the exact size we ended up using */ 264081177295SEdward Tomasz Napierala params->lun_size_bytes = be_lun->size_bytes; 264181177295SEdward Tomasz Napierala 264219720f41SAlexander Motin req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; 264381177295SEdward Tomasz Napierala return (0); 264481177295SEdward Tomasz Napierala 264581177295SEdward Tomasz Napierala bailout_error: 264681177295SEdward Tomasz Napierala req->status = CTL_LUN_ERROR; 264781177295SEdward Tomasz Napierala return (0); 264881177295SEdward Tomasz Napierala } 264981177295SEdward Tomasz Napierala 2650130f4520SKenneth D. Merry static void 2651130f4520SKenneth D. Merry ctl_be_block_lun_shutdown(void *be_lun) 2652130f4520SKenneth D. Merry { 265368bf823fSAlexander Motin struct ctl_be_block_lun *lun = be_lun; 265468bf823fSAlexander Motin struct ctl_be_block_softc *softc = lun->softc; 2655130f4520SKenneth D. Merry 2656130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2657130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; 2658130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2659130f4520SKenneth D. Merry wakeup(lun); 2660130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2661130f4520SKenneth D. Merry } 2662130f4520SKenneth D. Merry 2663130f4520SKenneth D. Merry static void 2664130f4520SKenneth D. Merry ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status) 2665130f4520SKenneth D. Merry { 2666130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2667130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2668130f4520SKenneth D. Merry 2669130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2670130f4520SKenneth D. Merry softc = lun->softc; 2671130f4520SKenneth D. Merry 2672130f4520SKenneth D. Merry if (status == CTL_LUN_CONFIG_OK) { 2673130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2674130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2675130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2676130f4520SKenneth D. Merry wakeup(lun); 2677130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2678130f4520SKenneth D. Merry 2679130f4520SKenneth D. Merry /* 2680130f4520SKenneth D. Merry * We successfully added the LUN, attempt to enable it. 2681130f4520SKenneth D. Merry */ 26820bcd4ab6SAlexander Motin if (ctl_enable_lun(&lun->cbe_lun) != 0) { 2683130f4520SKenneth D. Merry printf("%s: ctl_enable_lun() failed!\n", __func__); 26840bcd4ab6SAlexander Motin if (ctl_invalidate_lun(&lun->cbe_lun) != 0) { 2685130f4520SKenneth D. Merry printf("%s: ctl_invalidate_lun() failed!\n", 2686130f4520SKenneth D. Merry __func__); 2687130f4520SKenneth D. Merry } 2688130f4520SKenneth D. Merry } 2689130f4520SKenneth D. Merry 2690130f4520SKenneth D. Merry return; 2691130f4520SKenneth D. Merry } 2692130f4520SKenneth D. Merry 2693130f4520SKenneth D. Merry 2694130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2695130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2696130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR; 2697130f4520SKenneth D. Merry wakeup(lun); 2698130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2699130f4520SKenneth D. Merry } 2700130f4520SKenneth D. Merry 2701130f4520SKenneth D. Merry 2702130f4520SKenneth D. Merry static int 2703130f4520SKenneth D. Merry ctl_be_block_config_write(union ctl_io *io) 2704130f4520SKenneth D. Merry { 2705130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 27060bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2707130f4520SKenneth D. Merry int retval; 2708130f4520SKenneth D. Merry 2709130f4520SKenneth D. Merry DPRINTF("entered\n"); 2710130f4520SKenneth D. Merry 27119cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 27120bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 2713130f4520SKenneth D. Merry 271467cc546dSAlexander Motin retval = 0; 2715130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 2716130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 2717130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 2718ee7f31c0SAlexander Motin case WRITE_SAME_10: 2719ee7f31c0SAlexander Motin case WRITE_SAME_16: 2720ee7f31c0SAlexander Motin case UNMAP: 2721130f4520SKenneth D. Merry /* 2722130f4520SKenneth D. Merry * The upper level CTL code will filter out any CDBs with 2723130f4520SKenneth D. Merry * the immediate bit set and return the proper error. 2724130f4520SKenneth D. Merry * 2725130f4520SKenneth D. Merry * We don't really need to worry about what LBA range the 2726130f4520SKenneth D. Merry * user asked to be synced out. When they issue a sync 2727130f4520SKenneth D. Merry * cache command, we'll sync out the whole thing. 2728130f4520SKenneth D. Merry */ 272975c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 2730130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, 2731130f4520SKenneth D. Merry links); 273275c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 2733130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 2734130f4520SKenneth D. Merry break; 2735130f4520SKenneth D. Merry case START_STOP_UNIT: { 2736130f4520SKenneth D. Merry struct scsi_start_stop_unit *cdb; 2737648dfc1aSAlexander Motin struct ctl_lun_req req; 2738130f4520SKenneth D. Merry 2739130f4520SKenneth D. Merry cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; 274066b69676SAlexander Motin if ((cdb->how & SSS_PC_MASK) != 0) { 274166b69676SAlexander Motin ctl_set_success(&io->scsiio); 274266b69676SAlexander Motin ctl_config_write_done(io); 274366b69676SAlexander Motin break; 274466b69676SAlexander Motin } 2745648dfc1aSAlexander Motin if (cdb->how & SSS_START) { 2746648dfc1aSAlexander Motin if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) { 2747648dfc1aSAlexander Motin retval = ctl_be_block_open(be_lun, &req); 2748648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; 2749648dfc1aSAlexander Motin if (retval == 0) { 2750648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; 2751648dfc1aSAlexander Motin ctl_lun_has_media(cbe_lun); 2752130f4520SKenneth D. Merry } else { 2753648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2754648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2755130f4520SKenneth D. Merry } 2756648dfc1aSAlexander Motin } 2757648dfc1aSAlexander Motin ctl_start_lun(cbe_lun); 2758648dfc1aSAlexander Motin } else { 2759648dfc1aSAlexander Motin ctl_stop_lun(cbe_lun); 2760648dfc1aSAlexander Motin if (cdb->how & SSS_LOEJ) { 2761648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2762648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_EJECTED; 2763648dfc1aSAlexander Motin ctl_lun_ejected(cbe_lun); 2764648dfc1aSAlexander Motin if (be_lun->vn != NULL) 2765648dfc1aSAlexander Motin ctl_be_block_close(be_lun); 2766648dfc1aSAlexander Motin } 2767648dfc1aSAlexander Motin } 2768648dfc1aSAlexander Motin 2769648dfc1aSAlexander Motin ctl_set_success(&io->scsiio); 2770130f4520SKenneth D. Merry ctl_config_write_done(io); 2771130f4520SKenneth D. Merry break; 2772130f4520SKenneth D. Merry } 277391be33dcSAlexander Motin case PREVENT_ALLOW: 277491be33dcSAlexander Motin ctl_set_success(&io->scsiio); 277591be33dcSAlexander Motin ctl_config_write_done(io); 277691be33dcSAlexander Motin break; 2777130f4520SKenneth D. Merry default: 2778130f4520SKenneth D. Merry ctl_set_invalid_opcode(&io->scsiio); 2779130f4520SKenneth D. Merry ctl_config_write_done(io); 2780130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2781130f4520SKenneth D. Merry break; 2782130f4520SKenneth D. Merry } 2783130f4520SKenneth D. Merry 2784130f4520SKenneth D. Merry return (retval); 2785130f4520SKenneth D. Merry } 2786130f4520SKenneth D. Merry 2787130f4520SKenneth D. Merry static int 2788130f4520SKenneth D. Merry ctl_be_block_config_read(union ctl_io *io) 2789130f4520SKenneth D. Merry { 2790ef8daf3fSAlexander Motin struct ctl_be_block_lun *be_lun; 27910bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2792ef8daf3fSAlexander Motin int retval = 0; 2793ef8daf3fSAlexander Motin 2794ef8daf3fSAlexander Motin DPRINTF("entered\n"); 2795ef8daf3fSAlexander Motin 27969cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 27970bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 2798ef8daf3fSAlexander Motin 2799ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 2800ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: 2801ef8daf3fSAlexander Motin if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { 2802ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 2803ef8daf3fSAlexander Motin STAILQ_INSERT_TAIL(&be_lun->config_read_queue, 2804ef8daf3fSAlexander Motin &io->io_hdr, links); 2805ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 2806ef8daf3fSAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, 2807ef8daf3fSAlexander Motin &be_lun->io_task); 2808ef8daf3fSAlexander Motin retval = CTL_RETVAL_QUEUED; 2809ef8daf3fSAlexander Motin break; 2810ef8daf3fSAlexander Motin } 2811ef8daf3fSAlexander Motin ctl_set_invalid_field(&io->scsiio, 2812ef8daf3fSAlexander Motin /*sks_valid*/ 1, 2813ef8daf3fSAlexander Motin /*command*/ 1, 2814ef8daf3fSAlexander Motin /*field*/ 1, 2815ef8daf3fSAlexander Motin /*bit_valid*/ 1, 2816ef8daf3fSAlexander Motin /*bit*/ 4); 2817ef8daf3fSAlexander Motin ctl_config_read_done(io); 2818ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2819ef8daf3fSAlexander Motin break; 2820ef8daf3fSAlexander Motin default: 2821ef8daf3fSAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 2822ef8daf3fSAlexander Motin ctl_config_read_done(io); 2823ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2824ef8daf3fSAlexander Motin break; 2825ef8daf3fSAlexander Motin } 2826ef8daf3fSAlexander Motin 2827ef8daf3fSAlexander Motin return (retval); 2828130f4520SKenneth D. Merry } 2829130f4520SKenneth D. Merry 2830130f4520SKenneth D. Merry static int 2831130f4520SKenneth D. Merry ctl_be_block_lun_info(void *be_lun, struct sbuf *sb) 2832130f4520SKenneth D. Merry { 2833130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2834130f4520SKenneth D. Merry int retval; 2835130f4520SKenneth D. Merry 2836130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2837130f4520SKenneth D. Merry 28382cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "\t<num_threads>"); 2839130f4520SKenneth D. Merry if (retval != 0) 2840130f4520SKenneth D. Merry goto bailout; 2841130f4520SKenneth D. Merry retval = sbuf_printf(sb, "%d", lun->num_threads); 2842130f4520SKenneth D. Merry if (retval != 0) 2843130f4520SKenneth D. Merry goto bailout; 28442cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "</num_threads>\n"); 2845130f4520SKenneth D. Merry 2846130f4520SKenneth D. Merry bailout: 2847130f4520SKenneth D. Merry return (retval); 2848130f4520SKenneth D. Merry } 2849130f4520SKenneth D. Merry 2850c3e7ba3eSAlexander Motin static uint64_t 2851c3e7ba3eSAlexander Motin ctl_be_block_lun_attr(void *be_lun, const char *attrname) 2852c3e7ba3eSAlexander Motin { 2853c3e7ba3eSAlexander Motin struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun; 2854c3e7ba3eSAlexander Motin 2855c3e7ba3eSAlexander Motin if (lun->getattr == NULL) 2856c3e7ba3eSAlexander Motin return (UINT64_MAX); 2857c3e7ba3eSAlexander Motin return (lun->getattr(lun, attrname)); 2858c3e7ba3eSAlexander Motin } 2859c3e7ba3eSAlexander Motin 28600c629e28SAlexander Motin static int 2861130f4520SKenneth D. Merry ctl_be_block_init(void) 2862130f4520SKenneth D. Merry { 28630c629e28SAlexander Motin struct ctl_be_block_softc *softc = &backend_block_softc; 2864130f4520SKenneth D. Merry 286575c7a1d3SAlexander Motin mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); 28660c629e28SAlexander Motin softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), 2867a0e36aeeSEdward Tomasz Napierala NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 2868130f4520SKenneth D. Merry STAILQ_INIT(&softc->lun_list); 28690c629e28SAlexander Motin return (0); 28700c629e28SAlexander Motin } 2871130f4520SKenneth D. Merry 28720c629e28SAlexander Motin 28730c629e28SAlexander Motin static int 28740c629e28SAlexander Motin ctl_be_block_shutdown(void) 28750c629e28SAlexander Motin { 28760c629e28SAlexander Motin struct ctl_be_block_softc *softc = &backend_block_softc; 28770c629e28SAlexander Motin struct ctl_be_block_lun *lun, *next_lun; 28780c629e28SAlexander Motin 28790c629e28SAlexander Motin mtx_lock(&softc->lock); 28800c629e28SAlexander Motin STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) { 28810c629e28SAlexander Motin /* 28820c629e28SAlexander Motin * Drop our lock here. Since ctl_invalidate_lun() can call 28830c629e28SAlexander Motin * back into us, this could potentially lead to a recursive 28840c629e28SAlexander Motin * lock of the same mutex, which would cause a hang. 28850c629e28SAlexander Motin */ 28860c629e28SAlexander Motin mtx_unlock(&softc->lock); 28870c629e28SAlexander Motin ctl_disable_lun(&lun->cbe_lun); 28880c629e28SAlexander Motin ctl_invalidate_lun(&lun->cbe_lun); 28890c629e28SAlexander Motin mtx_lock(&softc->lock); 28900c629e28SAlexander Motin } 28910c629e28SAlexander Motin mtx_unlock(&softc->lock); 28920c629e28SAlexander Motin 28930c629e28SAlexander Motin uma_zdestroy(softc->beio_zone); 28940c629e28SAlexander Motin mtx_destroy(&softc->lock); 28950c629e28SAlexander Motin return (0); 2896130f4520SKenneth D. Merry } 2897