1130f4520SKenneth D. Merry /*- 2bec9534dSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3bec9534dSPedro F. Giffuni * 4130f4520SKenneth D. Merry * Copyright (c) 2003 Silicon Graphics International Corp. 5130f4520SKenneth D. Merry * Copyright (c) 2009-2011 Spectra Logic Corporation 681177295SEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 7648dfc1aSAlexander Motin * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org> 8130f4520SKenneth D. Merry * All rights reserved. 9130f4520SKenneth D. Merry * 1081177295SEdward Tomasz Napierala * Portions of this software were developed by Edward Tomasz Napierala 1181177295SEdward Tomasz Napierala * under sponsorship from the FreeBSD Foundation. 1281177295SEdward Tomasz Napierala * 13130f4520SKenneth D. Merry * Redistribution and use in source and binary forms, with or without 14130f4520SKenneth D. Merry * modification, are permitted provided that the following conditions 15130f4520SKenneth D. Merry * are met: 16130f4520SKenneth D. Merry * 1. Redistributions of source code must retain the above copyright 17130f4520SKenneth D. Merry * notice, this list of conditions, and the following disclaimer, 18130f4520SKenneth D. Merry * without modification. 19130f4520SKenneth D. Merry * 2. Redistributions in binary form must reproduce at minimum a disclaimer 20130f4520SKenneth D. Merry * substantially similar to the "NO WARRANTY" disclaimer below 21130f4520SKenneth D. Merry * ("Disclaimer") and any redistribution must be conditioned upon 22130f4520SKenneth D. Merry * including a substantially similar Disclaimer requirement for further 23130f4520SKenneth D. Merry * binary redistribution. 24130f4520SKenneth D. Merry * 25130f4520SKenneth D. Merry * NO WARRANTY 26130f4520SKenneth D. Merry * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27130f4520SKenneth D. Merry * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28130f4520SKenneth D. Merry * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 29130f4520SKenneth D. Merry * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30130f4520SKenneth D. Merry * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31130f4520SKenneth D. Merry * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32130f4520SKenneth D. Merry * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33130f4520SKenneth D. Merry * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 34130f4520SKenneth D. Merry * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 35130f4520SKenneth D. Merry * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36130f4520SKenneth D. Merry * POSSIBILITY OF SUCH DAMAGES. 37130f4520SKenneth D. Merry * 38130f4520SKenneth D. Merry * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $ 39130f4520SKenneth D. Merry */ 40130f4520SKenneth D. Merry /* 41130f4520SKenneth D. Merry * CAM Target Layer driver backend for block devices. 42130f4520SKenneth D. Merry * 43130f4520SKenneth D. Merry * Author: Ken Merry <ken@FreeBSD.org> 44130f4520SKenneth D. Merry */ 45130f4520SKenneth D. Merry #include <sys/cdefs.h> 46130f4520SKenneth D. Merry __FBSDID("$FreeBSD$"); 47130f4520SKenneth D. Merry 48130f4520SKenneth D. Merry #include <sys/param.h> 49130f4520SKenneth D. Merry #include <sys/systm.h> 50130f4520SKenneth D. Merry #include <sys/kernel.h> 51130f4520SKenneth D. Merry #include <sys/types.h> 52130f4520SKenneth D. Merry #include <sys/kthread.h> 53130f4520SKenneth D. Merry #include <sys/bio.h> 54130f4520SKenneth D. Merry #include <sys/fcntl.h> 55ee7f31c0SAlexander Motin #include <sys/limits.h> 56130f4520SKenneth D. Merry #include <sys/lock.h> 57130f4520SKenneth D. Merry #include <sys/mutex.h> 58130f4520SKenneth D. Merry #include <sys/condvar.h> 59130f4520SKenneth D. Merry #include <sys/malloc.h> 60130f4520SKenneth D. Merry #include <sys/conf.h> 61130f4520SKenneth D. Merry #include <sys/ioccom.h> 62130f4520SKenneth D. Merry #include <sys/queue.h> 63130f4520SKenneth D. Merry #include <sys/sbuf.h> 64130f4520SKenneth D. Merry #include <sys/endian.h> 65130f4520SKenneth D. Merry #include <sys/uio.h> 66130f4520SKenneth D. Merry #include <sys/buf.h> 67130f4520SKenneth D. Merry #include <sys/taskqueue.h> 68130f4520SKenneth D. Merry #include <sys/vnode.h> 69130f4520SKenneth D. Merry #include <sys/namei.h> 70130f4520SKenneth D. Merry #include <sys/mount.h> 71130f4520SKenneth D. Merry #include <sys/disk.h> 72130f4520SKenneth D. Merry #include <sys/fcntl.h> 73130f4520SKenneth D. Merry #include <sys/filedesc.h> 74ef8daf3fSAlexander Motin #include <sys/filio.h> 75130f4520SKenneth D. Merry #include <sys/proc.h> 76130f4520SKenneth D. Merry #include <sys/pcpu.h> 77130f4520SKenneth D. Merry #include <sys/module.h> 78130f4520SKenneth D. Merry #include <sys/sdt.h> 79130f4520SKenneth D. Merry #include <sys/devicestat.h> 80130f4520SKenneth D. Merry #include <sys/sysctl.h> 818951f055SMarcelo Araujo #include <sys/nv.h> 828951f055SMarcelo Araujo #include <sys/dnv.h> 83130f4520SKenneth D. Merry 84130f4520SKenneth D. Merry #include <geom/geom.h> 85130f4520SKenneth D. Merry 86130f4520SKenneth D. Merry #include <cam/cam.h> 87130f4520SKenneth D. Merry #include <cam/scsi/scsi_all.h> 88130f4520SKenneth D. Merry #include <cam/scsi/scsi_da.h> 89130f4520SKenneth D. Merry #include <cam/ctl/ctl_io.h> 90130f4520SKenneth D. Merry #include <cam/ctl/ctl.h> 91130f4520SKenneth D. Merry #include <cam/ctl/ctl_backend.h> 92130f4520SKenneth D. Merry #include <cam/ctl/ctl_ioctl.h> 937ac58230SAlexander Motin #include <cam/ctl/ctl_ha.h> 94130f4520SKenneth D. Merry #include <cam/ctl/ctl_scsi_all.h> 957ac58230SAlexander Motin #include <cam/ctl/ctl_private.h> 96130f4520SKenneth D. Merry #include <cam/ctl/ctl_error.h> 97130f4520SKenneth D. Merry 98130f4520SKenneth D. Merry /* 9908a7cce5SAlexander Motin * The idea here is that we'll allocate enough S/G space to hold a 1MB 10008a7cce5SAlexander Motin * I/O. If we get an I/O larger than that, we'll split it. 101130f4520SKenneth D. Merry */ 10211b569f7SAlexander Motin #define CTLBLK_HALF_IO_SIZE (512 * 1024) 10311b569f7SAlexander Motin #define CTLBLK_MAX_IO_SIZE (CTLBLK_HALF_IO_SIZE * 2) 10408a7cce5SAlexander Motin #define CTLBLK_MAX_SEG MAXPHYS 10511b569f7SAlexander Motin #define CTLBLK_HALF_SEGS MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1) 10611b569f7SAlexander Motin #define CTLBLK_MAX_SEGS (CTLBLK_HALF_SEGS * 2) 107130f4520SKenneth D. Merry 108130f4520SKenneth D. Merry #ifdef CTLBLK_DEBUG 109130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) \ 110130f4520SKenneth D. Merry printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) 111130f4520SKenneth D. Merry #else 112130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) do {} while(0) 113130f4520SKenneth D. Merry #endif 114130f4520SKenneth D. Merry 115e86a4142SAlexander Motin #define PRIV(io) \ 116e86a4142SAlexander Motin ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND]) 11711b569f7SAlexander Motin #define ARGS(io) \ 11811b569f7SAlexander Motin ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]) 119e86a4142SAlexander Motin 120130f4520SKenneth D. Merry SDT_PROVIDER_DEFINE(cbb); 121130f4520SKenneth D. Merry 122130f4520SKenneth D. Merry typedef enum { 123130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_UNCONFIGURED = 0x01, 124130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_CONFIG_ERR = 0x02, 125130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_WAITING = 0x04, 126130f4520SKenneth D. Merry } ctl_be_block_lun_flags; 127130f4520SKenneth D. Merry 128130f4520SKenneth D. Merry typedef enum { 129130f4520SKenneth D. Merry CTL_BE_BLOCK_NONE, 130130f4520SKenneth D. Merry CTL_BE_BLOCK_DEV, 131130f4520SKenneth D. Merry CTL_BE_BLOCK_FILE 132130f4520SKenneth D. Merry } ctl_be_block_type; 133130f4520SKenneth D. Merry 134130f4520SKenneth D. Merry struct ctl_be_block_filedata { 135130f4520SKenneth D. Merry struct ucred *cred; 136130f4520SKenneth D. Merry }; 137130f4520SKenneth D. Merry 138130f4520SKenneth D. Merry union ctl_be_block_bedata { 139130f4520SKenneth D. Merry struct ctl_be_block_filedata file; 140130f4520SKenneth D. Merry }; 141130f4520SKenneth D. Merry 142130f4520SKenneth D. Merry struct ctl_be_block_io; 143130f4520SKenneth D. Merry struct ctl_be_block_lun; 144130f4520SKenneth D. Merry 145130f4520SKenneth D. Merry typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun, 146130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 147c3e7ba3eSAlexander Motin typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun, 148c3e7ba3eSAlexander Motin const char *attrname); 149130f4520SKenneth D. Merry 150130f4520SKenneth D. Merry /* 151130f4520SKenneth D. Merry * Backend LUN structure. There is a 1:1 mapping between a block device 152130f4520SKenneth D. Merry * and a backend block LUN, and between a backend block LUN and a CTL LUN. 153130f4520SKenneth D. Merry */ 154130f4520SKenneth D. Merry struct ctl_be_block_lun { 15519720f41SAlexander Motin struct ctl_lun_create_params params; 156130f4520SKenneth D. Merry char lunname[32]; 157130f4520SKenneth D. Merry char *dev_path; 158130f4520SKenneth D. Merry ctl_be_block_type dev_type; 159130f4520SKenneth D. Merry struct vnode *vn; 160130f4520SKenneth D. Merry union ctl_be_block_bedata backend; 161130f4520SKenneth D. Merry cbb_dispatch_t dispatch; 162130f4520SKenneth D. Merry cbb_dispatch_t lun_flush; 163ee7f31c0SAlexander Motin cbb_dispatch_t unmap; 164ef8daf3fSAlexander Motin cbb_dispatch_t get_lba_status; 165c3e7ba3eSAlexander Motin cbb_getattr_t getattr; 166130f4520SKenneth D. Merry uma_zone_t lun_zone; 167130f4520SKenneth D. Merry uint64_t size_blocks; 168130f4520SKenneth D. Merry uint64_t size_bytes; 169130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 170130f4520SKenneth D. Merry struct devstat *disk_stats; 171130f4520SKenneth D. Merry ctl_be_block_lun_flags flags; 172130f4520SKenneth D. Merry STAILQ_ENTRY(ctl_be_block_lun) links; 1730bcd4ab6SAlexander Motin struct ctl_be_lun cbe_lun; 174130f4520SKenneth D. Merry struct taskqueue *io_taskqueue; 175130f4520SKenneth D. Merry struct task io_task; 176130f4520SKenneth D. Merry int num_threads; 177130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) input_queue; 178ef8daf3fSAlexander Motin STAILQ_HEAD(, ctl_io_hdr) config_read_queue; 179130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) config_write_queue; 180130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 18175c7a1d3SAlexander Motin struct mtx_padalign io_lock; 18275c7a1d3SAlexander Motin struct mtx_padalign queue_lock; 183130f4520SKenneth D. Merry }; 184130f4520SKenneth D. Merry 185130f4520SKenneth D. Merry /* 186130f4520SKenneth D. Merry * Overall softc structure for the block backend module. 187130f4520SKenneth D. Merry */ 188130f4520SKenneth D. Merry struct ctl_be_block_softc { 189130f4520SKenneth D. Merry struct mtx lock; 1900c629e28SAlexander Motin uma_zone_t beio_zone; 191130f4520SKenneth D. Merry int num_luns; 192130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_be_block_lun) lun_list; 193130f4520SKenneth D. Merry }; 194130f4520SKenneth D. Merry 195130f4520SKenneth D. Merry static struct ctl_be_block_softc backend_block_softc; 196130f4520SKenneth D. Merry 197130f4520SKenneth D. Merry /* 198130f4520SKenneth D. Merry * Per-I/O information. 199130f4520SKenneth D. Merry */ 200130f4520SKenneth D. Merry struct ctl_be_block_io { 201130f4520SKenneth D. Merry union ctl_io *io; 202130f4520SKenneth D. Merry struct ctl_sg_entry sg_segs[CTLBLK_MAX_SEGS]; 203130f4520SKenneth D. Merry struct iovec xiovecs[CTLBLK_MAX_SEGS]; 204130f4520SKenneth D. Merry int bio_cmd; 205130f4520SKenneth D. Merry int num_segs; 206130f4520SKenneth D. Merry int num_bios_sent; 207130f4520SKenneth D. Merry int num_bios_done; 208130f4520SKenneth D. Merry int send_complete; 2091f0694a6SAlexander Motin int first_error; 2101f0694a6SAlexander Motin uint64_t first_error_offset; 211130f4520SKenneth D. Merry struct bintime ds_t0; 212130f4520SKenneth D. Merry devstat_tag_type ds_tag_type; 213130f4520SKenneth D. Merry devstat_trans_flags ds_trans_type; 214130f4520SKenneth D. Merry uint64_t io_len; 215130f4520SKenneth D. Merry uint64_t io_offset; 2167d0d4342SAlexander Motin int io_arg; 217130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 218130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 219ee7f31c0SAlexander Motin void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */ 220130f4520SKenneth D. Merry }; 221130f4520SKenneth D. Merry 2227ac58230SAlexander Motin extern struct ctl_softc *control_softc; 2237ac58230SAlexander Motin 224130f4520SKenneth D. Merry static int cbb_num_threads = 14; 225130f4520SKenneth D. Merry SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0, 226130f4520SKenneth D. Merry "CAM Target Layer Block Backend"); 227af3b2549SHans Petter Selasky SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, 228130f4520SKenneth D. Merry &cbb_num_threads, 0, "Number of threads per backing file"); 229130f4520SKenneth D. Merry 230130f4520SKenneth D. Merry static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); 231130f4520SKenneth D. Merry static void ctl_free_beio(struct ctl_be_block_io *beio); 232130f4520SKenneth D. Merry static void ctl_complete_beio(struct ctl_be_block_io *beio); 233130f4520SKenneth D. Merry static int ctl_be_block_move_done(union ctl_io *io); 234130f4520SKenneth D. Merry static void ctl_be_block_biodone(struct bio *bio); 235130f4520SKenneth D. Merry static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 236130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 237130f4520SKenneth D. Merry static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 238130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 239ef8daf3fSAlexander Motin static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 240ef8daf3fSAlexander Motin struct ctl_be_block_io *beio); 24153c146deSAlexander Motin static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, 24253c146deSAlexander Motin const char *attrname); 243130f4520SKenneth D. Merry static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 244130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 245ee7f31c0SAlexander Motin static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 246ee7f31c0SAlexander Motin struct ctl_be_block_io *beio); 247130f4520SKenneth D. Merry static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 248130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 249c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, 250c3e7ba3eSAlexander Motin const char *attrname); 251ef8daf3fSAlexander Motin static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 252ef8daf3fSAlexander Motin union ctl_io *io); 253130f4520SKenneth D. Merry static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 254130f4520SKenneth D. Merry union ctl_io *io); 255130f4520SKenneth D. Merry static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 256130f4520SKenneth D. Merry union ctl_io *io); 257130f4520SKenneth D. Merry static void ctl_be_block_worker(void *context, int pending); 258130f4520SKenneth D. Merry static int ctl_be_block_submit(union ctl_io *io); 259130f4520SKenneth D. Merry static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 260130f4520SKenneth D. Merry int flag, struct thread *td); 261130f4520SKenneth D. Merry static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, 262130f4520SKenneth D. Merry struct ctl_lun_req *req); 263130f4520SKenneth D. Merry static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, 264130f4520SKenneth D. Merry struct ctl_lun_req *req); 265130f4520SKenneth D. Merry static int ctl_be_block_close(struct ctl_be_block_lun *be_lun); 266648dfc1aSAlexander Motin static int ctl_be_block_open(struct ctl_be_block_lun *be_lun, 267130f4520SKenneth D. Merry struct ctl_lun_req *req); 268130f4520SKenneth D. Merry static int ctl_be_block_create(struct ctl_be_block_softc *softc, 269130f4520SKenneth D. Merry struct ctl_lun_req *req); 270130f4520SKenneth D. Merry static int ctl_be_block_rm(struct ctl_be_block_softc *softc, 271130f4520SKenneth D. Merry struct ctl_lun_req *req); 27281177295SEdward Tomasz Napierala static int ctl_be_block_modify(struct ctl_be_block_softc *softc, 27381177295SEdward Tomasz Napierala struct ctl_lun_req *req); 274130f4520SKenneth D. Merry static void ctl_be_block_lun_shutdown(void *be_lun); 275130f4520SKenneth D. Merry static void ctl_be_block_lun_config_status(void *be_lun, 276130f4520SKenneth D. Merry ctl_lun_config_status status); 277130f4520SKenneth D. Merry static int ctl_be_block_config_write(union ctl_io *io); 278130f4520SKenneth D. Merry static int ctl_be_block_config_read(union ctl_io *io); 279130f4520SKenneth D. Merry static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); 280c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); 2810c629e28SAlexander Motin static int ctl_be_block_init(void); 2820c629e28SAlexander Motin static int ctl_be_block_shutdown(void); 283130f4520SKenneth D. Merry 284130f4520SKenneth D. Merry static struct ctl_backend_driver ctl_be_block_driver = 285130f4520SKenneth D. Merry { 2862a2443d8SKenneth D. Merry .name = "block", 2872a2443d8SKenneth D. Merry .flags = CTL_BE_FLAG_HAS_CONFIG, 2882a2443d8SKenneth D. Merry .init = ctl_be_block_init, 2890c629e28SAlexander Motin .shutdown = ctl_be_block_shutdown, 2902a2443d8SKenneth D. Merry .data_submit = ctl_be_block_submit, 2912a2443d8SKenneth D. Merry .data_move_done = ctl_be_block_move_done, 2922a2443d8SKenneth D. Merry .config_read = ctl_be_block_config_read, 2932a2443d8SKenneth D. Merry .config_write = ctl_be_block_config_write, 2942a2443d8SKenneth D. Merry .ioctl = ctl_be_block_ioctl, 295c3e7ba3eSAlexander Motin .lun_info = ctl_be_block_lun_info, 296c3e7ba3eSAlexander Motin .lun_attr = ctl_be_block_lun_attr 297130f4520SKenneth D. Merry }; 298130f4520SKenneth D. Merry 299130f4520SKenneth D. Merry MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); 300130f4520SKenneth D. Merry CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); 301130f4520SKenneth D. Merry 302130f4520SKenneth D. Merry static struct ctl_be_block_io * 303130f4520SKenneth D. Merry ctl_alloc_beio(struct ctl_be_block_softc *softc) 304130f4520SKenneth D. Merry { 305130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 306130f4520SKenneth D. Merry 3070c629e28SAlexander Motin beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO); 308130f4520SKenneth D. Merry beio->softc = softc; 309130f4520SKenneth D. Merry return (beio); 310130f4520SKenneth D. Merry } 311130f4520SKenneth D. Merry 312130f4520SKenneth D. Merry static void 313130f4520SKenneth D. Merry ctl_free_beio(struct ctl_be_block_io *beio) 314130f4520SKenneth D. Merry { 315130f4520SKenneth D. Merry int duplicate_free; 316130f4520SKenneth D. Merry int i; 317130f4520SKenneth D. Merry 318130f4520SKenneth D. Merry duplicate_free = 0; 319130f4520SKenneth D. Merry 320130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 321130f4520SKenneth D. Merry if (beio->sg_segs[i].addr == NULL) 322130f4520SKenneth D. Merry duplicate_free++; 323130f4520SKenneth D. Merry 324130f4520SKenneth D. Merry uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr); 325130f4520SKenneth D. Merry beio->sg_segs[i].addr = NULL; 32611b569f7SAlexander Motin 32711b569f7SAlexander Motin /* For compare we had two equal S/G lists. */ 32811b569f7SAlexander Motin if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) { 32911b569f7SAlexander Motin uma_zfree(beio->lun->lun_zone, 33011b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr); 33111b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL; 33211b569f7SAlexander Motin } 333130f4520SKenneth D. Merry } 334130f4520SKenneth D. Merry 335130f4520SKenneth D. Merry if (duplicate_free > 0) { 336130f4520SKenneth D. Merry printf("%s: %d duplicate frees out of %d segments\n", __func__, 337130f4520SKenneth D. Merry duplicate_free, beio->num_segs); 338130f4520SKenneth D. Merry } 339a0e36aeeSEdward Tomasz Napierala 3400c629e28SAlexander Motin uma_zfree(beio->softc->beio_zone, beio); 341130f4520SKenneth D. Merry } 342130f4520SKenneth D. Merry 343130f4520SKenneth D. Merry static void 344130f4520SKenneth D. Merry ctl_complete_beio(struct ctl_be_block_io *beio) 345130f4520SKenneth D. Merry { 34675c7a1d3SAlexander Motin union ctl_io *io = beio->io; 347130f4520SKenneth D. Merry 348ee7f31c0SAlexander Motin if (beio->beio_cont != NULL) { 349ee7f31c0SAlexander Motin beio->beio_cont(beio); 350ee7f31c0SAlexander Motin } else { 351130f4520SKenneth D. Merry ctl_free_beio(beio); 35211b569f7SAlexander Motin ctl_data_submit_done(io); 353130f4520SKenneth D. Merry } 354ee7f31c0SAlexander Motin } 355130f4520SKenneth D. Merry 356d6043e46SAlexander Motin static size_t 357d6043e46SAlexander Motin cmp(uint8_t *a, uint8_t *b, size_t size) 358d6043e46SAlexander Motin { 359d6043e46SAlexander Motin size_t i; 360d6043e46SAlexander Motin 361d6043e46SAlexander Motin for (i = 0; i < size; i++) { 362d6043e46SAlexander Motin if (a[i] != b[i]) 363d6043e46SAlexander Motin break; 364d6043e46SAlexander Motin } 365d6043e46SAlexander Motin return (i); 366d6043e46SAlexander Motin } 367d6043e46SAlexander Motin 368d6043e46SAlexander Motin static void 369d6043e46SAlexander Motin ctl_be_block_compare(union ctl_io *io) 370d6043e46SAlexander Motin { 371d6043e46SAlexander Motin struct ctl_be_block_io *beio; 372d6043e46SAlexander Motin uint64_t off, res; 373d6043e46SAlexander Motin int i; 374d6043e46SAlexander Motin uint8_t info[8]; 375d6043e46SAlexander Motin 376d6043e46SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 377d6043e46SAlexander Motin off = 0; 378d6043e46SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 379d6043e46SAlexander Motin res = cmp(beio->sg_segs[i].addr, 380d6043e46SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, 381d6043e46SAlexander Motin beio->sg_segs[i].len); 382d6043e46SAlexander Motin off += res; 383d6043e46SAlexander Motin if (res < beio->sg_segs[i].len) 384d6043e46SAlexander Motin break; 385d6043e46SAlexander Motin } 386d6043e46SAlexander Motin if (i < beio->num_segs) { 387d6043e46SAlexander Motin scsi_u64to8b(off, info); 388d6043e46SAlexander Motin ctl_set_sense(&io->scsiio, /*current_error*/ 1, 389d6043e46SAlexander Motin /*sense_key*/ SSD_KEY_MISCOMPARE, 390d6043e46SAlexander Motin /*asc*/ 0x1D, /*ascq*/ 0x00, 391d6043e46SAlexander Motin /*type*/ SSD_ELEM_INFO, 392d6043e46SAlexander Motin /*size*/ sizeof(info), /*data*/ &info, 393d6043e46SAlexander Motin /*type*/ SSD_ELEM_NONE); 394d6043e46SAlexander Motin } else 395d6043e46SAlexander Motin ctl_set_success(&io->scsiio); 396d6043e46SAlexander Motin } 397d6043e46SAlexander Motin 398130f4520SKenneth D. Merry static int 399130f4520SKenneth D. Merry ctl_be_block_move_done(union ctl_io *io) 400130f4520SKenneth D. Merry { 401130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 402130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 40311b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 404130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 405130f4520SKenneth D. Merry struct bintime cur_bt; 406130f4520SKenneth D. Merry #endif 407130f4520SKenneth D. Merry 408e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 409130f4520SKenneth D. Merry be_lun = beio->lun; 410130f4520SKenneth D. Merry 411130f4520SKenneth D. Merry DPRINTF("entered\n"); 412130f4520SKenneth D. Merry 413130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 414e675024aSAlexander Motin getbinuptime(&cur_bt); 415130f4520SKenneth D. Merry bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); 416130f4520SKenneth D. Merry bintime_add(&io->io_hdr.dma_bt, &cur_bt); 417130f4520SKenneth D. Merry #endif 418e675024aSAlexander Motin io->io_hdr.num_dmas++; 41911b569f7SAlexander Motin io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; 420130f4520SKenneth D. Merry 421130f4520SKenneth D. Merry /* 422130f4520SKenneth D. Merry * We set status at this point for read commands, and write 423130f4520SKenneth D. Merry * commands with errors. 424130f4520SKenneth D. Merry */ 425f7241cceSAlexander Motin if (io->io_hdr.flags & CTL_FLAG_ABORT) { 426f7241cceSAlexander Motin ; 427eb6ac6f9SAlexander Motin } else if ((io->io_hdr.port_status != 0) && 428eb6ac6f9SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 429eb6ac6f9SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 430eb6ac6f9SAlexander Motin ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, 431eb6ac6f9SAlexander Motin /*retry_count*/ io->io_hdr.port_status); 432eb6ac6f9SAlexander Motin } else if (io->scsiio.kern_data_resid != 0 && 433eb6ac6f9SAlexander Motin (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && 434eb6ac6f9SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 435eb6ac6f9SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 436eb6ac6f9SAlexander Motin ctl_set_invalid_field_ciu(&io->scsiio); 437f7241cceSAlexander Motin } else if ((io->io_hdr.port_status == 0) && 43811b569f7SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { 43911b569f7SAlexander Motin lbalen = ARGS(beio->io); 44011b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_READ) { 441130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 44211b569f7SAlexander Motin } else if (lbalen->flags & CTL_LLF_COMPARE) { 44311b569f7SAlexander Motin /* We have two data blocks ready for comparison. */ 444d6043e46SAlexander Motin ctl_be_block_compare(io); 44511b569f7SAlexander Motin } 446130f4520SKenneth D. Merry } 447130f4520SKenneth D. Merry 448130f4520SKenneth D. Merry /* 449130f4520SKenneth D. Merry * If this is a read, or a write with errors, it is done. 450130f4520SKenneth D. Merry */ 451130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_READ) 452130f4520SKenneth D. Merry || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) 453130f4520SKenneth D. Merry || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { 454130f4520SKenneth D. Merry ctl_complete_beio(beio); 455130f4520SKenneth D. Merry return (0); 456130f4520SKenneth D. Merry } 457130f4520SKenneth D. Merry 458130f4520SKenneth D. Merry /* 459130f4520SKenneth D. Merry * At this point, we have a write and the DMA completed 460130f4520SKenneth D. Merry * successfully. We now have to queue it to the task queue to 461130f4520SKenneth D. Merry * execute the backend I/O. That is because we do blocking 462130f4520SKenneth D. Merry * memory allocations, and in the file backing case, blocking I/O. 463130f4520SKenneth D. Merry * This move done routine is generally called in the SIM's 464130f4520SKenneth D. Merry * interrupt context, and therefore we cannot block. 465130f4520SKenneth D. Merry */ 46675c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 467130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); 46875c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 469130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 470130f4520SKenneth D. Merry 471130f4520SKenneth D. Merry return (0); 472130f4520SKenneth D. Merry } 473130f4520SKenneth D. Merry 474130f4520SKenneth D. Merry static void 475130f4520SKenneth D. Merry ctl_be_block_biodone(struct bio *bio) 476130f4520SKenneth D. Merry { 477130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 478130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 479130f4520SKenneth D. Merry union ctl_io *io; 480e0c2f975SAlexander Motin int error; 481130f4520SKenneth D. Merry 482130f4520SKenneth D. Merry beio = bio->bio_caller1; 483130f4520SKenneth D. Merry be_lun = beio->lun; 484130f4520SKenneth D. Merry io = beio->io; 485130f4520SKenneth D. Merry 486130f4520SKenneth D. Merry DPRINTF("entered\n"); 487130f4520SKenneth D. Merry 488e0c2f975SAlexander Motin error = bio->bio_error; 48975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 4901f0694a6SAlexander Motin if (error != 0 && 4911f0694a6SAlexander Motin (beio->first_error == 0 || 4921f0694a6SAlexander Motin bio->bio_offset < beio->first_error_offset)) { 4931f0694a6SAlexander Motin beio->first_error = error; 4941f0694a6SAlexander Motin beio->first_error_offset = bio->bio_offset; 4951f0694a6SAlexander Motin } 496130f4520SKenneth D. Merry 497130f4520SKenneth D. Merry beio->num_bios_done++; 498130f4520SKenneth D. Merry 499130f4520SKenneth D. Merry /* 500130f4520SKenneth D. Merry * XXX KDM will this cause WITNESS to complain? Holding a lock 501130f4520SKenneth D. Merry * during the free might cause it to complain. 502130f4520SKenneth D. Merry */ 503130f4520SKenneth D. Merry g_destroy_bio(bio); 504130f4520SKenneth D. Merry 505130f4520SKenneth D. Merry /* 506130f4520SKenneth D. Merry * If the send complete bit isn't set, or we aren't the last I/O to 507130f4520SKenneth D. Merry * complete, then we're done. 508130f4520SKenneth D. Merry */ 509130f4520SKenneth D. Merry if ((beio->send_complete == 0) 510130f4520SKenneth D. Merry || (beio->num_bios_done < beio->num_bios_sent)) { 51175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 512130f4520SKenneth D. Merry return; 513130f4520SKenneth D. Merry } 514130f4520SKenneth D. Merry 515130f4520SKenneth D. Merry /* 516130f4520SKenneth D. Merry * At this point, we've verified that we are the last I/O to 517130f4520SKenneth D. Merry * complete, so it's safe to drop the lock. 518130f4520SKenneth D. Merry */ 51975c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 52075c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 52175c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 52275c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 523130f4520SKenneth D. Merry 524130f4520SKenneth D. Merry /* 525130f4520SKenneth D. Merry * If there are any errors from the backing device, we fail the 526130f4520SKenneth D. Merry * entire I/O with a medium error. 527130f4520SKenneth D. Merry */ 5281f0694a6SAlexander Motin error = beio->first_error; 5291f0694a6SAlexander Motin if (error != 0) { 530e0c2f975SAlexander Motin if (error == EOPNOTSUPP) { 531e0c2f975SAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 5320631de4aSAlexander Motin } else if (error == ENOSPC || error == EDQUOT) { 5334fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 5346187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 5356187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 536e0c2f975SAlexander Motin } else if (beio->bio_cmd == BIO_FLUSH) { 537130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 538130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 539130f4520SKenneth D. Merry /*sks_valid*/ 1, 540130f4520SKenneth D. Merry /*retry_count*/ 0xbad2); 5417f7bb97aSAlexander Motin } else { 5427f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 5437f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 5447f7bb97aSAlexander Motin } 545130f4520SKenneth D. Merry ctl_complete_beio(beio); 546130f4520SKenneth D. Merry return; 547130f4520SKenneth D. Merry } 548130f4520SKenneth D. Merry 549130f4520SKenneth D. Merry /* 55011b569f7SAlexander Motin * If this is a write, a flush, a delete or verify, we're all done. 551130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 552130f4520SKenneth D. Merry */ 553130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_WRITE) 554ee7f31c0SAlexander Motin || (beio->bio_cmd == BIO_FLUSH) 55511b569f7SAlexander Motin || (beio->bio_cmd == BIO_DELETE) 55611b569f7SAlexander Motin || (ARGS(io)->flags & CTL_LLF_VERIFY)) { 557130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 558130f4520SKenneth D. Merry ctl_complete_beio(beio); 559130f4520SKenneth D. Merry } else { 560f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 56175a3108eSAlexander Motin beio->beio_cont == NULL) { 562f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 56375a3108eSAlexander Motin ctl_serseq_done(io); 56475a3108eSAlexander Motin } 565130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 566e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 567130f4520SKenneth D. Merry #endif 568130f4520SKenneth D. Merry ctl_datamove(io); 569130f4520SKenneth D. Merry } 570130f4520SKenneth D. Merry } 571130f4520SKenneth D. Merry 572130f4520SKenneth D. Merry static void 573130f4520SKenneth D. Merry ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 574130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 575130f4520SKenneth D. Merry { 57675c7a1d3SAlexander Motin union ctl_io *io = beio->io; 577130f4520SKenneth D. Merry struct mount *mountpoint; 5785050aa86SKonstantin Belousov int error, lock_flags; 579130f4520SKenneth D. Merry 580130f4520SKenneth D. Merry DPRINTF("entered\n"); 581130f4520SKenneth D. Merry 58275c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 58375c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 58475c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 58575c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 586130f4520SKenneth D. Merry 587130f4520SKenneth D. Merry (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 588130f4520SKenneth D. Merry 58967cc546dSAlexander Motin if (MNT_SHARED_WRITES(mountpoint) || 59067cc546dSAlexander Motin ((mountpoint == NULL) && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 591130f4520SKenneth D. Merry lock_flags = LK_SHARED; 592130f4520SKenneth D. Merry else 593130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 594130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 5957d0d4342SAlexander Motin error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT, 5967d0d4342SAlexander Motin curthread); 597130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 598130f4520SKenneth D. Merry 599130f4520SKenneth D. Merry vn_finished_write(mountpoint); 600130f4520SKenneth D. Merry 60175c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 60275c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 60375c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 60475c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 60575c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 60675c7a1d3SAlexander Motin 607130f4520SKenneth D. Merry if (error == 0) 608130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 609130f4520SKenneth D. Merry else { 610130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 611130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 612130f4520SKenneth D. Merry /*sks_valid*/ 1, 613130f4520SKenneth D. Merry /*retry_count*/ 0xbad1); 614130f4520SKenneth D. Merry } 615130f4520SKenneth D. Merry 616130f4520SKenneth D. Merry ctl_complete_beio(beio); 617130f4520SKenneth D. Merry } 618130f4520SKenneth D. Merry 61936160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_start, "uint64_t"); 62036160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_start, "uint64_t"); 62136160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, file_done,"uint64_t"); 62236160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, file_done, "uint64_t"); 623130f4520SKenneth D. Merry 624130f4520SKenneth D. Merry static void 625130f4520SKenneth D. Merry ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 626130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 627130f4520SKenneth D. Merry { 628130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 629130f4520SKenneth D. Merry union ctl_io *io; 630130f4520SKenneth D. Merry struct uio xuio; 631130f4520SKenneth D. Merry struct iovec *xiovec; 63283981e31SAlexander Motin size_t s; 63383981e31SAlexander Motin int error, flags, i; 634130f4520SKenneth D. Merry 635130f4520SKenneth D. Merry DPRINTF("entered\n"); 636130f4520SKenneth D. Merry 637130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 638130f4520SKenneth D. Merry io = beio->io; 63955551d05SAlexander Motin flags = 0; 64055551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 64155551d05SAlexander Motin flags |= IO_DIRECT; 64255551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 64355551d05SAlexander Motin flags |= IO_SYNC; 644130f4520SKenneth D. Merry 64511b569f7SAlexander Motin bzero(&xuio, sizeof(xuio)); 646130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 64736160958SMark Johnston SDT_PROBE0(cbb, , read, file_start); 64811b569f7SAlexander Motin xuio.uio_rw = UIO_READ; 649130f4520SKenneth D. Merry } else { 65036160958SMark Johnston SDT_PROBE0(cbb, , write, file_start); 651130f4520SKenneth D. Merry xuio.uio_rw = UIO_WRITE; 65211b569f7SAlexander Motin } 653130f4520SKenneth D. Merry xuio.uio_offset = beio->io_offset; 654130f4520SKenneth D. Merry xuio.uio_resid = beio->io_len; 655130f4520SKenneth D. Merry xuio.uio_segflg = UIO_SYSSPACE; 656130f4520SKenneth D. Merry xuio.uio_iov = beio->xiovecs; 657130f4520SKenneth D. Merry xuio.uio_iovcnt = beio->num_segs; 658130f4520SKenneth D. Merry xuio.uio_td = curthread; 659130f4520SKenneth D. Merry 660130f4520SKenneth D. Merry for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 661130f4520SKenneth D. Merry xiovec->iov_base = beio->sg_segs[i].addr; 662130f4520SKenneth D. Merry xiovec->iov_len = beio->sg_segs[i].len; 663130f4520SKenneth D. Merry } 664130f4520SKenneth D. Merry 66575c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 66675c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 66775c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 66875c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 66975c7a1d3SAlexander Motin 670130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 671130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 672130f4520SKenneth D. Merry 673130f4520SKenneth D. Merry /* 674130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for reads. If the 675130f4520SKenneth D. Merry * DIRECTIO option is configured into the kernel, it calls 676130f4520SKenneth D. Merry * ffs_rawread(). But that only works for single-segment 677130f4520SKenneth D. Merry * uios with user space addresses. In our case, with a 678130f4520SKenneth D. Merry * kernel uio, it still reads into the buffer cache, but it 679130f4520SKenneth D. Merry * will just try to release the buffer from the cache later 680130f4520SKenneth D. Merry * on in ffs_read(). 681130f4520SKenneth D. Merry * 682130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for reads. 683130f4520SKenneth D. Merry * 684130f4520SKenneth D. Merry * UFS does not pay attention to IO_SYNC for reads. 685130f4520SKenneth D. Merry * 686130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (which translates into the 687130f4520SKenneth D. Merry * Solaris define FRSYNC for zfs_read()) for reads. It 688130f4520SKenneth D. Merry * attempts to sync the file before reading. 689130f4520SKenneth D. Merry */ 69055551d05SAlexander Motin error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); 691130f4520SKenneth D. Merry 692130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 69336160958SMark Johnston SDT_PROBE0(cbb, , read, file_done); 69483981e31SAlexander Motin if (error == 0 && xuio.uio_resid > 0) { 69583981e31SAlexander Motin /* 69683981e31SAlexander Motin * If we red less then requested (EOF), then 69783981e31SAlexander Motin * we should clean the rest of the buffer. 69883981e31SAlexander Motin */ 69983981e31SAlexander Motin s = beio->io_len - xuio.uio_resid; 70083981e31SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 70183981e31SAlexander Motin if (s >= beio->sg_segs[i].len) { 70283981e31SAlexander Motin s -= beio->sg_segs[i].len; 70383981e31SAlexander Motin continue; 70483981e31SAlexander Motin } 70583981e31SAlexander Motin bzero((uint8_t *)beio->sg_segs[i].addr + s, 70683981e31SAlexander Motin beio->sg_segs[i].len - s); 70783981e31SAlexander Motin s = 0; 70883981e31SAlexander Motin } 70983981e31SAlexander Motin } 710130f4520SKenneth D. Merry } else { 711130f4520SKenneth D. Merry struct mount *mountpoint; 712130f4520SKenneth D. Merry int lock_flags; 713130f4520SKenneth D. Merry 714130f4520SKenneth D. Merry (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 715130f4520SKenneth D. Merry 71667cc546dSAlexander Motin if (MNT_SHARED_WRITES(mountpoint) || ((mountpoint == NULL) 717130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 718130f4520SKenneth D. Merry lock_flags = LK_SHARED; 719130f4520SKenneth D. Merry else 720130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 721130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 722130f4520SKenneth D. Merry 723130f4520SKenneth D. Merry /* 724130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for writes. The write 725130f4520SKenneth D. Merry * is done asynchronously. (Normally the write would just 726130f4520SKenneth D. Merry * get put into cache. 727130f4520SKenneth D. Merry * 728130f4520SKenneth D. Merry * UFS pays attention to IO_SYNC for writes. It will 729130f4520SKenneth D. Merry * attempt to write the buffer out synchronously if that 730130f4520SKenneth D. Merry * flag is set. 731130f4520SKenneth D. Merry * 732130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for writes. 733130f4520SKenneth D. Merry * 734130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) 735130f4520SKenneth D. Merry * for writes. It will flush the transaction from the 736130f4520SKenneth D. Merry * cache before returning. 737130f4520SKenneth D. Merry */ 73855551d05SAlexander Motin error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); 739130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 740130f4520SKenneth D. Merry 741130f4520SKenneth D. Merry vn_finished_write(mountpoint); 74236160958SMark Johnston SDT_PROBE0(cbb, , write, file_done); 743130f4520SKenneth D. Merry } 744130f4520SKenneth D. Merry 74575c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 74675c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 74775c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 74875c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 74975c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 75075c7a1d3SAlexander Motin 751130f4520SKenneth D. Merry /* 752130f4520SKenneth D. Merry * If we got an error, set the sense data to "MEDIUM ERROR" and 753130f4520SKenneth D. Merry * return the I/O to the user. 754130f4520SKenneth D. Merry */ 755130f4520SKenneth D. Merry if (error != 0) { 7560631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 7574fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 7586187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 7596187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 7607f7bb97aSAlexander Motin } else { 7617f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 7627f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 7637f7bb97aSAlexander Motin } 764130f4520SKenneth D. Merry ctl_complete_beio(beio); 765130f4520SKenneth D. Merry return; 766130f4520SKenneth D. Merry } 767130f4520SKenneth D. Merry 768130f4520SKenneth D. Merry /* 769696297adSAlexander Motin * If this is a write or a verify, we're all done. 770130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 771130f4520SKenneth D. Merry */ 772696297adSAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 773696297adSAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 774130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 775130f4520SKenneth D. Merry ctl_complete_beio(beio); 776130f4520SKenneth D. Merry } else { 777f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 77875a3108eSAlexander Motin beio->beio_cont == NULL) { 779f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 78075a3108eSAlexander Motin ctl_serseq_done(io); 78175a3108eSAlexander Motin } 782130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 783e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 784130f4520SKenneth D. Merry #endif 785130f4520SKenneth D. Merry ctl_datamove(io); 786130f4520SKenneth D. Merry } 787130f4520SKenneth D. Merry } 788130f4520SKenneth D. Merry 789130f4520SKenneth D. Merry static void 790ef8daf3fSAlexander Motin ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 791ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 792ef8daf3fSAlexander Motin { 793ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 794ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 795ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 796ef8daf3fSAlexander Motin off_t roff, off; 797ef8daf3fSAlexander Motin int error, status; 798ef8daf3fSAlexander Motin 799ef8daf3fSAlexander Motin DPRINTF("entered\n"); 800ef8daf3fSAlexander Motin 8010bcd4ab6SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; 802ef8daf3fSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 803ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, 804ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 805ef8daf3fSAlexander Motin if (error == 0 && off > roff) 806ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 807ef8daf3fSAlexander Motin else { 808ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, 809ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 810ef8daf3fSAlexander Motin if (error == 0 && off > roff) 811ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 812ef8daf3fSAlexander Motin else { 813ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 814ef8daf3fSAlexander Motin off = be_lun->size_bytes; 815ef8daf3fSAlexander Motin } 816ef8daf3fSAlexander Motin } 817ef8daf3fSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 818ef8daf3fSAlexander Motin 819ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 820ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 8210bcd4ab6SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - 8220bcd4ab6SAlexander Motin lbalen->lba), data->descr[0].length); 823ef8daf3fSAlexander Motin data->descr[0].status = status; 824ef8daf3fSAlexander Motin 825ef8daf3fSAlexander Motin ctl_complete_beio(beio); 826ef8daf3fSAlexander Motin } 827ef8daf3fSAlexander Motin 82853c146deSAlexander Motin static uint64_t 82953c146deSAlexander Motin ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname) 83053c146deSAlexander Motin { 83153c146deSAlexander Motin struct vattr vattr; 83253c146deSAlexander Motin struct statfs statfs; 833b9b4269cSAlexander Motin uint64_t val; 83453c146deSAlexander Motin int error; 83553c146deSAlexander Motin 836b9b4269cSAlexander Motin val = UINT64_MAX; 83753c146deSAlexander Motin if (be_lun->vn == NULL) 838b9b4269cSAlexander Motin return (val); 839b9b4269cSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 84053c146deSAlexander Motin if (strcmp(attrname, "blocksused") == 0) { 84153c146deSAlexander Motin error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 842b9b4269cSAlexander Motin if (error == 0) 8430bcd4ab6SAlexander Motin val = vattr.va_bytes / be_lun->cbe_lun.blocksize; 84453c146deSAlexander Motin } 845b9b4269cSAlexander Motin if (strcmp(attrname, "blocksavail") == 0 && 846b9b4269cSAlexander Motin (be_lun->vn->v_iflag & VI_DOOMED) == 0) { 84753c146deSAlexander Motin error = VFS_STATFS(be_lun->vn->v_mount, &statfs); 848b9b4269cSAlexander Motin if (error == 0) 849a15bbf15SAlexander Motin val = statfs.f_bavail * statfs.f_bsize / 8500bcd4ab6SAlexander Motin be_lun->cbe_lun.blocksize; 85153c146deSAlexander Motin } 852b9b4269cSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 853b9b4269cSAlexander Motin return (val); 85453c146deSAlexander Motin } 85553c146deSAlexander Motin 856ef8daf3fSAlexander Motin static void 85767f586a8SAlexander Motin ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, 85867f586a8SAlexander Motin struct ctl_be_block_io *beio) 85967f586a8SAlexander Motin { 86067f586a8SAlexander Motin union ctl_io *io; 8613236151eSAlexander Motin struct cdevsw *csw; 8623236151eSAlexander Motin struct cdev *dev; 86367f586a8SAlexander Motin struct uio xuio; 86467f586a8SAlexander Motin struct iovec *xiovec; 8653236151eSAlexander Motin int error, flags, i, ref; 86667f586a8SAlexander Motin 86767f586a8SAlexander Motin DPRINTF("entered\n"); 86867f586a8SAlexander Motin 86967f586a8SAlexander Motin io = beio->io; 87055551d05SAlexander Motin flags = 0; 87155551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 87255551d05SAlexander Motin flags |= IO_DIRECT; 87355551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 87455551d05SAlexander Motin flags |= IO_SYNC; 87567f586a8SAlexander Motin 87667f586a8SAlexander Motin bzero(&xuio, sizeof(xuio)); 87767f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 87836160958SMark Johnston SDT_PROBE0(cbb, , read, file_start); 87967f586a8SAlexander Motin xuio.uio_rw = UIO_READ; 88067f586a8SAlexander Motin } else { 88136160958SMark Johnston SDT_PROBE0(cbb, , write, file_start); 88267f586a8SAlexander Motin xuio.uio_rw = UIO_WRITE; 88367f586a8SAlexander Motin } 88467f586a8SAlexander Motin xuio.uio_offset = beio->io_offset; 88567f586a8SAlexander Motin xuio.uio_resid = beio->io_len; 88667f586a8SAlexander Motin xuio.uio_segflg = UIO_SYSSPACE; 88767f586a8SAlexander Motin xuio.uio_iov = beio->xiovecs; 88867f586a8SAlexander Motin xuio.uio_iovcnt = beio->num_segs; 88967f586a8SAlexander Motin xuio.uio_td = curthread; 89067f586a8SAlexander Motin 89167f586a8SAlexander Motin for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 89267f586a8SAlexander Motin xiovec->iov_base = beio->sg_segs[i].addr; 89367f586a8SAlexander Motin xiovec->iov_len = beio->sg_segs[i].len; 89467f586a8SAlexander Motin } 89567f586a8SAlexander Motin 89667f586a8SAlexander Motin binuptime(&beio->ds_t0); 89767f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 89867f586a8SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 89967f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 90067f586a8SAlexander Motin 9013236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 9023236151eSAlexander Motin if (csw) { 9033236151eSAlexander Motin if (beio->bio_cmd == BIO_READ) 9043236151eSAlexander Motin error = csw->d_read(dev, &xuio, flags); 9053236151eSAlexander Motin else 9063236151eSAlexander Motin error = csw->d_write(dev, &xuio, flags); 9073236151eSAlexander Motin dev_relthread(dev, ref); 9083236151eSAlexander Motin } else 9093236151eSAlexander Motin error = ENXIO; 9103236151eSAlexander Motin 9113236151eSAlexander Motin if (beio->bio_cmd == BIO_READ) 91236160958SMark Johnston SDT_PROBE0(cbb, , read, file_done); 9133236151eSAlexander Motin else 91436160958SMark Johnston SDT_PROBE0(cbb, , write, file_done); 91567f586a8SAlexander Motin 91667f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 91767f586a8SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 91867f586a8SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 91967f586a8SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 92067f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 92167f586a8SAlexander Motin 92267f586a8SAlexander Motin /* 92367f586a8SAlexander Motin * If we got an error, set the sense data to "MEDIUM ERROR" and 92467f586a8SAlexander Motin * return the I/O to the user. 92567f586a8SAlexander Motin */ 92667f586a8SAlexander Motin if (error != 0) { 9270631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 9284fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 9296187d472SAlexander Motin } else if (error == EROFS || error == EACCES) { 9306187d472SAlexander Motin ctl_set_hw_write_protected(&io->scsiio); 9317f7bb97aSAlexander Motin } else { 9327f7bb97aSAlexander Motin ctl_set_medium_error(&io->scsiio, 9337f7bb97aSAlexander Motin beio->bio_cmd == BIO_READ); 9347f7bb97aSAlexander Motin } 93567f586a8SAlexander Motin ctl_complete_beio(beio); 93667f586a8SAlexander Motin return; 93767f586a8SAlexander Motin } 93867f586a8SAlexander Motin 93967f586a8SAlexander Motin /* 94067f586a8SAlexander Motin * If this is a write or a verify, we're all done. 94167f586a8SAlexander Motin * If this is a read, we can now send the data to the user. 94267f586a8SAlexander Motin */ 94367f586a8SAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 94467f586a8SAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 94567f586a8SAlexander Motin ctl_set_success(&io->scsiio); 94667f586a8SAlexander Motin ctl_complete_beio(beio); 94767f586a8SAlexander Motin } else { 948f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 94975a3108eSAlexander Motin beio->beio_cont == NULL) { 950f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 95175a3108eSAlexander Motin ctl_serseq_done(io); 95275a3108eSAlexander Motin } 95367f586a8SAlexander Motin #ifdef CTL_TIME_IO 954e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 95567f586a8SAlexander Motin #endif 95667f586a8SAlexander Motin ctl_datamove(io); 95767f586a8SAlexander Motin } 95867f586a8SAlexander Motin } 95967f586a8SAlexander Motin 96067f586a8SAlexander Motin static void 961ef8daf3fSAlexander Motin ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun, 962ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 963ef8daf3fSAlexander Motin { 964ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 9653236151eSAlexander Motin struct cdevsw *csw; 9663236151eSAlexander Motin struct cdev *dev; 967ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 968ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 969ef8daf3fSAlexander Motin off_t roff, off; 9703236151eSAlexander Motin int error, ref, status; 971ef8daf3fSAlexander Motin 972ef8daf3fSAlexander Motin DPRINTF("entered\n"); 973ef8daf3fSAlexander Motin 9743236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 9753236151eSAlexander Motin if (csw == NULL) { 9763236151eSAlexander Motin status = 0; /* unknown up to the end */ 9773236151eSAlexander Motin off = be_lun->size_bytes; 9783236151eSAlexander Motin goto done; 9793236151eSAlexander Motin } 9800bcd4ab6SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; 9813236151eSAlexander Motin error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD, 9823236151eSAlexander Motin curthread); 983ef8daf3fSAlexander Motin if (error == 0 && off > roff) 984ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 985ef8daf3fSAlexander Motin else { 9863236151eSAlexander Motin error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD, 9873236151eSAlexander Motin curthread); 988ef8daf3fSAlexander Motin if (error == 0 && off > roff) 989ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 990ef8daf3fSAlexander Motin else { 991ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 992ef8daf3fSAlexander Motin off = be_lun->size_bytes; 993ef8daf3fSAlexander Motin } 994ef8daf3fSAlexander Motin } 9953236151eSAlexander Motin dev_relthread(dev, ref); 996ef8daf3fSAlexander Motin 9973236151eSAlexander Motin done: 998ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 999ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 10000bcd4ab6SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - 10010bcd4ab6SAlexander Motin lbalen->lba), data->descr[0].length); 1002ef8daf3fSAlexander Motin data->descr[0].status = status; 1003ef8daf3fSAlexander Motin 1004ef8daf3fSAlexander Motin ctl_complete_beio(beio); 1005ef8daf3fSAlexander Motin } 1006ef8daf3fSAlexander Motin 1007ef8daf3fSAlexander Motin static void 1008130f4520SKenneth D. Merry ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 1009130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1010130f4520SKenneth D. Merry { 1011130f4520SKenneth D. Merry struct bio *bio; 10123236151eSAlexander Motin struct cdevsw *csw; 10133236151eSAlexander Motin struct cdev *dev; 10143236151eSAlexander Motin int ref; 1015130f4520SKenneth D. Merry 1016130f4520SKenneth D. Merry DPRINTF("entered\n"); 1017130f4520SKenneth D. Merry 1018130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1019130f4520SKenneth D. Merry bio = g_alloc_bio(); 1020130f4520SKenneth D. Merry 1021130f4520SKenneth D. Merry bio->bio_cmd = BIO_FLUSH; 1022130f4520SKenneth D. Merry bio->bio_offset = 0; 1023130f4520SKenneth D. Merry bio->bio_data = 0; 1024130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 1025130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1026130f4520SKenneth D. Merry bio->bio_pblkno = 0; 1027130f4520SKenneth D. Merry 1028130f4520SKenneth D. Merry /* 1029130f4520SKenneth D. Merry * We don't need to acquire the LUN lock here, because we are only 1030130f4520SKenneth D. Merry * sending one bio, and so there is no other context to synchronize 1031130f4520SKenneth D. Merry * with. 1032130f4520SKenneth D. Merry */ 1033130f4520SKenneth D. Merry beio->num_bios_sent = 1; 1034130f4520SKenneth D. Merry beio->send_complete = 1; 1035130f4520SKenneth D. Merry 1036130f4520SKenneth D. Merry binuptime(&beio->ds_t0); 103775c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1038130f4520SKenneth D. Merry devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 103975c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1040130f4520SKenneth D. Merry 10413236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 10423236151eSAlexander Motin if (csw) { 10433236151eSAlexander Motin bio->bio_dev = dev; 10443236151eSAlexander Motin csw->d_strategy(bio); 10453236151eSAlexander Motin dev_relthread(dev, ref); 10463236151eSAlexander Motin } else { 10473236151eSAlexander Motin bio->bio_error = ENXIO; 10483236151eSAlexander Motin ctl_be_block_biodone(bio); 10493236151eSAlexander Motin } 1050130f4520SKenneth D. Merry } 1051130f4520SKenneth D. Merry 1052130f4520SKenneth D. Merry static void 1053ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun, 1054ee7f31c0SAlexander Motin struct ctl_be_block_io *beio, 1055ee7f31c0SAlexander Motin uint64_t off, uint64_t len, int last) 1056ee7f31c0SAlexander Motin { 1057ee7f31c0SAlexander Motin struct bio *bio; 10588f5a226aSAlexander Motin uint64_t maxlen; 10593236151eSAlexander Motin struct cdevsw *csw; 10603236151eSAlexander Motin struct cdev *dev; 10613236151eSAlexander Motin int ref; 1062ee7f31c0SAlexander Motin 10633236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 10640bcd4ab6SAlexander Motin maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize); 1065ee7f31c0SAlexander Motin while (len > 0) { 1066ee7f31c0SAlexander Motin bio = g_alloc_bio(); 1067ee7f31c0SAlexander Motin bio->bio_cmd = BIO_DELETE; 10683236151eSAlexander Motin bio->bio_dev = dev; 1069ee7f31c0SAlexander Motin bio->bio_offset = off; 10708f5a226aSAlexander Motin bio->bio_length = MIN(len, maxlen); 1071ee7f31c0SAlexander Motin bio->bio_data = 0; 1072ee7f31c0SAlexander Motin bio->bio_done = ctl_be_block_biodone; 1073ee7f31c0SAlexander Motin bio->bio_caller1 = beio; 10740bcd4ab6SAlexander Motin bio->bio_pblkno = off / be_lun->cbe_lun.blocksize; 1075ee7f31c0SAlexander Motin 1076ee7f31c0SAlexander Motin off += bio->bio_length; 1077ee7f31c0SAlexander Motin len -= bio->bio_length; 1078ee7f31c0SAlexander Motin 107975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1080ee7f31c0SAlexander Motin beio->num_bios_sent++; 1081ee7f31c0SAlexander Motin if (last && len == 0) 1082ee7f31c0SAlexander Motin beio->send_complete = 1; 108375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1084ee7f31c0SAlexander Motin 10853236151eSAlexander Motin if (csw) { 10863236151eSAlexander Motin csw->d_strategy(bio); 10873236151eSAlexander Motin } else { 10883236151eSAlexander Motin bio->bio_error = ENXIO; 10893236151eSAlexander Motin ctl_be_block_biodone(bio); 1090ee7f31c0SAlexander Motin } 1091ee7f31c0SAlexander Motin } 10923236151eSAlexander Motin if (csw) 10933236151eSAlexander Motin dev_relthread(dev, ref); 10943236151eSAlexander Motin } 1095ee7f31c0SAlexander Motin 1096ee7f31c0SAlexander Motin static void 1097ee7f31c0SAlexander Motin ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 1098ee7f31c0SAlexander Motin struct ctl_be_block_io *beio) 1099ee7f31c0SAlexander Motin { 1100ee7f31c0SAlexander Motin union ctl_io *io; 110166df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1102ee7f31c0SAlexander Motin struct scsi_unmap_desc *buf, *end; 1103ee7f31c0SAlexander Motin uint64_t len; 1104ee7f31c0SAlexander Motin 1105ee7f31c0SAlexander Motin io = beio->io; 1106ee7f31c0SAlexander Motin 1107ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1108ee7f31c0SAlexander Motin 1109ee7f31c0SAlexander Motin binuptime(&beio->ds_t0); 111075c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1111ee7f31c0SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 111275c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1113ee7f31c0SAlexander Motin 1114ee7f31c0SAlexander Motin if (beio->io_offset == -1) { 1115ee7f31c0SAlexander Motin beio->io_len = 0; 111666df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 111766df9136SAlexander Motin buf = (struct scsi_unmap_desc *)ptrlen->ptr; 111866df9136SAlexander Motin end = buf + ptrlen->len / sizeof(*buf); 1119ee7f31c0SAlexander Motin for (; buf < end; buf++) { 1120ee7f31c0SAlexander Motin len = (uint64_t)scsi_4btoul(buf->length) * 11210bcd4ab6SAlexander Motin be_lun->cbe_lun.blocksize; 1122ee7f31c0SAlexander Motin beio->io_len += len; 1123ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 11240bcd4ab6SAlexander Motin scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize, 11250bcd4ab6SAlexander Motin len, (end - buf < 2) ? TRUE : FALSE); 1126ee7f31c0SAlexander Motin } 1127ee7f31c0SAlexander Motin } else 1128ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1129ee7f31c0SAlexander Motin beio->io_offset, beio->io_len, TRUE); 1130ee7f31c0SAlexander Motin } 1131ee7f31c0SAlexander Motin 1132ee7f31c0SAlexander Motin static void 1133130f4520SKenneth D. Merry ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 1134130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1135130f4520SKenneth D. Merry { 113675c7a1d3SAlexander Motin TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue); 1137130f4520SKenneth D. Merry struct bio *bio; 11383236151eSAlexander Motin struct cdevsw *csw; 11393236151eSAlexander Motin struct cdev *dev; 1140130f4520SKenneth D. Merry off_t cur_offset; 11413236151eSAlexander Motin int i, max_iosize, ref; 1142130f4520SKenneth D. Merry 1143130f4520SKenneth D. Merry DPRINTF("entered\n"); 11443236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 1145130f4520SKenneth D. Merry 1146130f4520SKenneth D. Merry /* 1147130f4520SKenneth D. Merry * We have to limit our I/O size to the maximum supported by the 1148130f4520SKenneth D. Merry * backend device. Hopefully it is MAXPHYS. If the driver doesn't 1149130f4520SKenneth D. Merry * set it properly, use DFLTPHYS. 1150130f4520SKenneth D. Merry */ 11513236151eSAlexander Motin if (csw) { 11523236151eSAlexander Motin max_iosize = dev->si_iosize_max; 1153130f4520SKenneth D. Merry if (max_iosize < PAGE_SIZE) 1154130f4520SKenneth D. Merry max_iosize = DFLTPHYS; 11553236151eSAlexander Motin } else 11563236151eSAlexander Motin max_iosize = DFLTPHYS; 1157130f4520SKenneth D. Merry 1158130f4520SKenneth D. Merry cur_offset = beio->io_offset; 1159130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 1160130f4520SKenneth D. Merry size_t cur_size; 1161130f4520SKenneth D. Merry uint8_t *cur_ptr; 1162130f4520SKenneth D. Merry 1163130f4520SKenneth D. Merry cur_size = beio->sg_segs[i].len; 1164130f4520SKenneth D. Merry cur_ptr = beio->sg_segs[i].addr; 1165130f4520SKenneth D. Merry 1166130f4520SKenneth D. Merry while (cur_size > 0) { 1167130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1168130f4520SKenneth D. Merry bio = g_alloc_bio(); 1169130f4520SKenneth D. Merry 1170130f4520SKenneth D. Merry KASSERT(bio != NULL, ("g_alloc_bio() failed!\n")); 1171130f4520SKenneth D. Merry 1172130f4520SKenneth D. Merry bio->bio_cmd = beio->bio_cmd; 11733236151eSAlexander Motin bio->bio_dev = dev; 1174130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1175130f4520SKenneth D. Merry bio->bio_length = min(cur_size, max_iosize); 1176130f4520SKenneth D. Merry bio->bio_offset = cur_offset; 1177130f4520SKenneth D. Merry bio->bio_data = cur_ptr; 1178130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 11790bcd4ab6SAlexander Motin bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize; 1180130f4520SKenneth D. Merry 1181130f4520SKenneth D. Merry cur_offset += bio->bio_length; 1182130f4520SKenneth D. Merry cur_ptr += bio->bio_length; 1183130f4520SKenneth D. Merry cur_size -= bio->bio_length; 1184130f4520SKenneth D. Merry 118575c7a1d3SAlexander Motin TAILQ_INSERT_TAIL(&queue, bio, bio_queue); 1186130f4520SKenneth D. Merry beio->num_bios_sent++; 1187130f4520SKenneth D. Merry } 1188130f4520SKenneth D. Merry } 118975c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 119075c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 119175c7a1d3SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 119275c7a1d3SAlexander Motin beio->send_complete = 1; 119375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 119475c7a1d3SAlexander Motin 119575c7a1d3SAlexander Motin /* 119675c7a1d3SAlexander Motin * Fire off all allocated requests! 119775c7a1d3SAlexander Motin */ 119875c7a1d3SAlexander Motin while ((bio = TAILQ_FIRST(&queue)) != NULL) { 119975c7a1d3SAlexander Motin TAILQ_REMOVE(&queue, bio, bio_queue); 12003236151eSAlexander Motin if (csw) 12013236151eSAlexander Motin csw->d_strategy(bio); 12023236151eSAlexander Motin else { 12033236151eSAlexander Motin bio->bio_error = ENXIO; 12043236151eSAlexander Motin ctl_be_block_biodone(bio); 120575c7a1d3SAlexander Motin } 1206130f4520SKenneth D. Merry } 12073236151eSAlexander Motin if (csw) 12083236151eSAlexander Motin dev_relthread(dev, ref); 12093236151eSAlexander Motin } 1210130f4520SKenneth D. Merry 1211c3e7ba3eSAlexander Motin static uint64_t 1212c3e7ba3eSAlexander Motin ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname) 1213c3e7ba3eSAlexander Motin { 1214c3e7ba3eSAlexander Motin struct diocgattr_arg arg; 12153236151eSAlexander Motin struct cdevsw *csw; 12163236151eSAlexander Motin struct cdev *dev; 12173236151eSAlexander Motin int error, ref; 1218c3e7ba3eSAlexander Motin 12193236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 12203236151eSAlexander Motin if (csw == NULL) 1221c3e7ba3eSAlexander Motin return (UINT64_MAX); 1222c3e7ba3eSAlexander Motin strlcpy(arg.name, attrname, sizeof(arg.name)); 1223c3e7ba3eSAlexander Motin arg.len = sizeof(arg.value.off); 12243236151eSAlexander Motin if (csw->d_ioctl) { 12253236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, 12263236151eSAlexander Motin curthread); 12273236151eSAlexander Motin } else 12283236151eSAlexander Motin error = ENODEV; 12293236151eSAlexander Motin dev_relthread(dev, ref); 1230c3e7ba3eSAlexander Motin if (error != 0) 1231c3e7ba3eSAlexander Motin return (UINT64_MAX); 1232c3e7ba3eSAlexander Motin return (arg.value.off); 1233c3e7ba3eSAlexander Motin } 1234c3e7ba3eSAlexander Motin 1235130f4520SKenneth D. Merry static void 12367d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun, 12377d0d4342SAlexander Motin union ctl_io *io) 12387d0d4342SAlexander Motin { 12390bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 12407d0d4342SAlexander Motin struct ctl_be_block_io *beio; 12417d0d4342SAlexander Motin struct ctl_lba_len_flags *lbalen; 12427d0d4342SAlexander Motin 12437d0d4342SAlexander Motin DPRINTF("entered\n"); 12447d0d4342SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 12457d0d4342SAlexander Motin lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 12467d0d4342SAlexander Motin 12470bcd4ab6SAlexander Motin beio->io_len = lbalen->len * cbe_lun->blocksize; 12480bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 12497d0d4342SAlexander Motin beio->io_arg = (lbalen->flags & SSC_IMMED) != 0; 12507d0d4342SAlexander Motin beio->bio_cmd = BIO_FLUSH; 12517d0d4342SAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 12527d0d4342SAlexander Motin DPRINTF("SYNC\n"); 12537d0d4342SAlexander Motin be_lun->lun_flush(be_lun, beio); 12547d0d4342SAlexander Motin } 12557d0d4342SAlexander Motin 12567d0d4342SAlexander Motin static void 1257ee7f31c0SAlexander Motin ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio) 1258ee7f31c0SAlexander Motin { 1259ee7f31c0SAlexander Motin union ctl_io *io; 1260ee7f31c0SAlexander Motin 1261ee7f31c0SAlexander Motin io = beio->io; 1262ee7f31c0SAlexander Motin ctl_free_beio(beio); 1263ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1264ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1265ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1266ee7f31c0SAlexander Motin ctl_config_write_done(io); 1267ee7f31c0SAlexander Motin return; 1268ee7f31c0SAlexander Motin } 1269ee7f31c0SAlexander Motin 1270ee7f31c0SAlexander Motin ctl_be_block_config_write(io); 1271ee7f31c0SAlexander Motin } 1272ee7f31c0SAlexander Motin 1273ee7f31c0SAlexander Motin static void 1274ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun, 1275ee7f31c0SAlexander Motin union ctl_io *io) 1276ee7f31c0SAlexander Motin { 12770bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1278ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 127966df9136SAlexander Motin struct ctl_lba_len_flags *lbalen; 1280fee04ef7SAlexander Motin uint64_t len_left, lba; 1281fee04ef7SAlexander Motin uint32_t pb, pbo, adj; 1282ee7f31c0SAlexander Motin int i, seglen; 1283ee7f31c0SAlexander Motin uint8_t *buf, *end; 1284ee7f31c0SAlexander Motin 1285ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1286ee7f31c0SAlexander Motin 1287e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 128811b569f7SAlexander Motin lbalen = ARGS(beio->io); 1289ee7f31c0SAlexander Motin 129064c5167cSAlexander Motin if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || 12913406a2a0SAlexander Motin (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { 1292ee7f31c0SAlexander Motin ctl_free_beio(beio); 1293ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1294ee7f31c0SAlexander Motin /*sks_valid*/ 1, 1295ee7f31c0SAlexander Motin /*command*/ 1, 1296ee7f31c0SAlexander Motin /*field*/ 1, 1297ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1298ee7f31c0SAlexander Motin /*bit*/ 0); 1299ee7f31c0SAlexander Motin ctl_config_write_done(io); 1300ee7f31c0SAlexander Motin return; 1301ee7f31c0SAlexander Motin } 1302ee7f31c0SAlexander Motin 13033406a2a0SAlexander Motin if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { 13040bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 13050bcd4ab6SAlexander Motin beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize; 1306ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1307ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 1308ee7f31c0SAlexander Motin 1309ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1310ee7f31c0SAlexander Motin return; 1311ee7f31c0SAlexander Motin } 1312ee7f31c0SAlexander Motin 1313ee7f31c0SAlexander Motin beio->bio_cmd = BIO_WRITE; 1314ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_WRITE; 1315ee7f31c0SAlexander Motin 1316ee7f31c0SAlexander Motin DPRINTF("WRITE SAME at LBA %jx len %u\n", 131766df9136SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len); 1318ee7f31c0SAlexander Motin 13190bcd4ab6SAlexander Motin pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp; 13200bcd4ab6SAlexander Motin if (be_lun->cbe_lun.pblockoff > 0) 13210bcd4ab6SAlexander Motin pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff; 1322fee04ef7SAlexander Motin else 1323fee04ef7SAlexander Motin pbo = 0; 13240bcd4ab6SAlexander Motin len_left = (uint64_t)lbalen->len * cbe_lun->blocksize; 1325ee7f31c0SAlexander Motin for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) { 1326ee7f31c0SAlexander Motin 1327ee7f31c0SAlexander Motin /* 1328ee7f31c0SAlexander Motin * Setup the S/G entry for this chunk. 1329ee7f31c0SAlexander Motin */ 133008a7cce5SAlexander Motin seglen = MIN(CTLBLK_MAX_SEG, len_left); 13310bcd4ab6SAlexander Motin if (pb > cbe_lun->blocksize) { 13320bcd4ab6SAlexander Motin adj = ((lbalen->lba + lba) * cbe_lun->blocksize + 133393b8c96cSAlexander Motin seglen - pbo) % pb; 133493b8c96cSAlexander Motin if (seglen > adj) 133593b8c96cSAlexander Motin seglen -= adj; 133693b8c96cSAlexander Motin else 13370bcd4ab6SAlexander Motin seglen -= seglen % cbe_lun->blocksize; 133893b8c96cSAlexander Motin } else 13390bcd4ab6SAlexander Motin seglen -= seglen % cbe_lun->blocksize; 1340ee7f31c0SAlexander Motin beio->sg_segs[i].len = seglen; 1341ee7f31c0SAlexander Motin beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1342ee7f31c0SAlexander Motin 1343ee7f31c0SAlexander Motin DPRINTF("segment %d addr %p len %zd\n", i, 1344ee7f31c0SAlexander Motin beio->sg_segs[i].addr, beio->sg_segs[i].len); 1345ee7f31c0SAlexander Motin 1346ee7f31c0SAlexander Motin beio->num_segs++; 1347ee7f31c0SAlexander Motin len_left -= seglen; 1348ee7f31c0SAlexander Motin 1349ee7f31c0SAlexander Motin buf = beio->sg_segs[i].addr; 1350ee7f31c0SAlexander Motin end = buf + seglen; 13510bcd4ab6SAlexander Motin for (; buf < end; buf += cbe_lun->blocksize) { 13526c2acea5SAlexander Motin if (lbalen->flags & SWS_NDOB) { 13536c2acea5SAlexander Motin memset(buf, 0, cbe_lun->blocksize); 13546c2acea5SAlexander Motin } else { 13556c2acea5SAlexander Motin memcpy(buf, io->scsiio.kern_data_ptr, 13566c2acea5SAlexander Motin cbe_lun->blocksize); 13576c2acea5SAlexander Motin } 135866df9136SAlexander Motin if (lbalen->flags & SWS_LBDATA) 135966df9136SAlexander Motin scsi_ulto4b(lbalen->lba + lba, buf); 1360ee7f31c0SAlexander Motin lba++; 1361ee7f31c0SAlexander Motin } 1362ee7f31c0SAlexander Motin } 1363ee7f31c0SAlexander Motin 13640bcd4ab6SAlexander Motin beio->io_offset = lbalen->lba * cbe_lun->blocksize; 13650bcd4ab6SAlexander Motin beio->io_len = lba * cbe_lun->blocksize; 1366ee7f31c0SAlexander Motin 1367ee7f31c0SAlexander Motin /* We can not do all in one run. Correct and schedule rerun. */ 1368ee7f31c0SAlexander Motin if (len_left > 0) { 136966df9136SAlexander Motin lbalen->lba += lba; 137066df9136SAlexander Motin lbalen->len -= lba; 1371ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done_ws; 1372ee7f31c0SAlexander Motin } 1373ee7f31c0SAlexander Motin 1374ee7f31c0SAlexander Motin be_lun->dispatch(be_lun, beio); 1375ee7f31c0SAlexander Motin } 1376ee7f31c0SAlexander Motin 1377ee7f31c0SAlexander Motin static void 1378ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun, 1379ee7f31c0SAlexander Motin union ctl_io *io) 1380ee7f31c0SAlexander Motin { 1381ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 138266df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1383ee7f31c0SAlexander Motin 1384ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1385ee7f31c0SAlexander Motin 1386e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 138766df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1388ee7f31c0SAlexander Motin 13893406a2a0SAlexander Motin if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { 1390ee7f31c0SAlexander Motin ctl_free_beio(beio); 1391ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1392ee7f31c0SAlexander Motin /*sks_valid*/ 0, 1393ee7f31c0SAlexander Motin /*command*/ 1, 1394ee7f31c0SAlexander Motin /*field*/ 0, 1395ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1396ee7f31c0SAlexander Motin /*bit*/ 0); 1397ee7f31c0SAlexander Motin ctl_config_write_done(io); 1398ee7f31c0SAlexander Motin return; 1399ee7f31c0SAlexander Motin } 1400ee7f31c0SAlexander Motin 1401ee7f31c0SAlexander Motin beio->io_len = 0; 1402ee7f31c0SAlexander Motin beio->io_offset = -1; 1403ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1404ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 140566df9136SAlexander Motin DPRINTF("UNMAP\n"); 1406ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1407ee7f31c0SAlexander Motin } 1408ee7f31c0SAlexander Motin 1409ee7f31c0SAlexander Motin static void 1410ef8daf3fSAlexander Motin ctl_be_block_cr_done(struct ctl_be_block_io *beio) 1411ef8daf3fSAlexander Motin { 1412ef8daf3fSAlexander Motin union ctl_io *io; 1413ef8daf3fSAlexander Motin 1414ef8daf3fSAlexander Motin io = beio->io; 1415ef8daf3fSAlexander Motin ctl_free_beio(beio); 1416ef8daf3fSAlexander Motin ctl_config_read_done(io); 1417ef8daf3fSAlexander Motin } 1418ef8daf3fSAlexander Motin 1419ef8daf3fSAlexander Motin static void 1420ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 1421ef8daf3fSAlexander Motin union ctl_io *io) 1422ef8daf3fSAlexander Motin { 1423ef8daf3fSAlexander Motin struct ctl_be_block_io *beio; 1424ef8daf3fSAlexander Motin struct ctl_be_block_softc *softc; 1425ef8daf3fSAlexander Motin 1426ef8daf3fSAlexander Motin DPRINTF("entered\n"); 1427ef8daf3fSAlexander Motin 1428ef8daf3fSAlexander Motin softc = be_lun->softc; 1429ef8daf3fSAlexander Motin beio = ctl_alloc_beio(softc); 1430ef8daf3fSAlexander Motin beio->io = io; 1431ef8daf3fSAlexander Motin beio->lun = be_lun; 1432ef8daf3fSAlexander Motin beio->beio_cont = ctl_be_block_cr_done; 1433ef8daf3fSAlexander Motin PRIV(io)->ptr = (void *)beio; 1434ef8daf3fSAlexander Motin 1435ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 1436ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: /* GET LBA STATUS */ 1437ef8daf3fSAlexander Motin beio->bio_cmd = -1; 1438ef8daf3fSAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 1439ef8daf3fSAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1440ef8daf3fSAlexander Motin beio->io_len = 0; 1441ef8daf3fSAlexander Motin if (be_lun->get_lba_status) 1442ef8daf3fSAlexander Motin be_lun->get_lba_status(be_lun, beio); 1443ef8daf3fSAlexander Motin else 1444ef8daf3fSAlexander Motin ctl_be_block_cr_done(beio); 1445ef8daf3fSAlexander Motin break; 1446ef8daf3fSAlexander Motin default: 1447ef8daf3fSAlexander Motin panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1448ef8daf3fSAlexander Motin break; 1449ef8daf3fSAlexander Motin } 1450ef8daf3fSAlexander Motin } 1451ef8daf3fSAlexander Motin 1452ef8daf3fSAlexander Motin static void 1453ee7f31c0SAlexander Motin ctl_be_block_cw_done(struct ctl_be_block_io *beio) 1454ee7f31c0SAlexander Motin { 1455ee7f31c0SAlexander Motin union ctl_io *io; 1456ee7f31c0SAlexander Motin 1457ee7f31c0SAlexander Motin io = beio->io; 1458ee7f31c0SAlexander Motin ctl_free_beio(beio); 1459ee7f31c0SAlexander Motin ctl_config_write_done(io); 1460ee7f31c0SAlexander Motin } 1461ee7f31c0SAlexander Motin 1462ee7f31c0SAlexander Motin static void 1463130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 1464130f4520SKenneth D. Merry union ctl_io *io) 1465130f4520SKenneth D. Merry { 1466130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1467130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1468130f4520SKenneth D. Merry 1469130f4520SKenneth D. Merry DPRINTF("entered\n"); 1470130f4520SKenneth D. Merry 1471130f4520SKenneth D. Merry softc = be_lun->softc; 1472130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1473130f4520SKenneth D. Merry beio->io = io; 1474130f4520SKenneth D. Merry beio->lun = be_lun; 1475ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done; 14767d0d4342SAlexander Motin switch (io->scsiio.tag_type) { 14777d0d4342SAlexander Motin case CTL_TAG_ORDERED: 14787d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 14797d0d4342SAlexander Motin break; 14807d0d4342SAlexander Motin case CTL_TAG_HEAD_OF_QUEUE: 14817d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_HEAD; 14827d0d4342SAlexander Motin break; 14837d0d4342SAlexander Motin case CTL_TAG_UNTAGGED: 14847d0d4342SAlexander Motin case CTL_TAG_SIMPLE: 14857d0d4342SAlexander Motin case CTL_TAG_ACA: 14867d0d4342SAlexander Motin default: 14877d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 14887d0d4342SAlexander Motin break; 14897d0d4342SAlexander Motin } 1490e86a4142SAlexander Motin PRIV(io)->ptr = (void *)beio; 1491130f4520SKenneth D. Merry 1492130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 1493130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 1494130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 14957d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(be_lun, io); 1496130f4520SKenneth D. Merry break; 1497ee7f31c0SAlexander Motin case WRITE_SAME_10: 1498ee7f31c0SAlexander Motin case WRITE_SAME_16: 1499ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(be_lun, io); 1500ee7f31c0SAlexander Motin break; 1501ee7f31c0SAlexander Motin case UNMAP: 1502ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(be_lun, io); 1503ee7f31c0SAlexander Motin break; 1504130f4520SKenneth D. Merry default: 1505130f4520SKenneth D. Merry panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1506130f4520SKenneth D. Merry break; 1507130f4520SKenneth D. Merry } 1508130f4520SKenneth D. Merry } 1509130f4520SKenneth D. Merry 151036160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, start, "uint64_t"); 151136160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, start, "uint64_t"); 151236160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , read, alloc_done, "uint64_t"); 151336160958SMark Johnston SDT_PROBE_DEFINE1(cbb, , write, alloc_done, "uint64_t"); 1514130f4520SKenneth D. Merry 1515130f4520SKenneth D. Merry static void 151608a7cce5SAlexander Motin ctl_be_block_next(struct ctl_be_block_io *beio) 151708a7cce5SAlexander Motin { 151808a7cce5SAlexander Motin struct ctl_be_block_lun *be_lun; 151908a7cce5SAlexander Motin union ctl_io *io; 152008a7cce5SAlexander Motin 152108a7cce5SAlexander Motin io = beio->io; 152208a7cce5SAlexander Motin be_lun = beio->lun; 152308a7cce5SAlexander Motin ctl_free_beio(beio); 1524ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1525ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1526ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 152711b569f7SAlexander Motin ctl_data_submit_done(io); 152808a7cce5SAlexander Motin return; 152908a7cce5SAlexander Motin } 153008a7cce5SAlexander Motin 153108a7cce5SAlexander Motin io->io_hdr.status &= ~CTL_STATUS_MASK; 153208a7cce5SAlexander Motin io->io_hdr.status |= CTL_STATUS_NONE; 153308a7cce5SAlexander Motin 153475c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 153508a7cce5SAlexander Motin STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 153675c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 153708a7cce5SAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 153808a7cce5SAlexander Motin } 153908a7cce5SAlexander Motin 154008a7cce5SAlexander Motin static void 1541130f4520SKenneth D. Merry ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 1542130f4520SKenneth D. Merry union ctl_io *io) 1543130f4520SKenneth D. Merry { 15440bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1545130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1546130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 154711b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 1548e86a4142SAlexander Motin struct ctl_ptr_len_flags *bptrlen; 1549e86a4142SAlexander Motin uint64_t len_left, lbas; 1550130f4520SKenneth D. Merry int i; 1551130f4520SKenneth D. Merry 1552130f4520SKenneth D. Merry softc = be_lun->softc; 1553130f4520SKenneth D. Merry 1554130f4520SKenneth D. Merry DPRINTF("entered\n"); 1555130f4520SKenneth D. Merry 155611b569f7SAlexander Motin lbalen = ARGS(io); 155711b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 155836160958SMark Johnston SDT_PROBE0(cbb, , write, start); 155911b569f7SAlexander Motin } else { 156036160958SMark Johnston SDT_PROBE0(cbb, , read, start); 1561130f4520SKenneth D. Merry } 1562130f4520SKenneth D. Merry 1563130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1564130f4520SKenneth D. Merry beio->io = io; 1565130f4520SKenneth D. Merry beio->lun = be_lun; 1566e86a4142SAlexander Motin bptrlen = PRIV(io); 1567e86a4142SAlexander Motin bptrlen->ptr = (void *)beio; 1568130f4520SKenneth D. Merry 1569130f4520SKenneth D. Merry switch (io->scsiio.tag_type) { 1570130f4520SKenneth D. Merry case CTL_TAG_ORDERED: 1571130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1572130f4520SKenneth D. Merry break; 1573130f4520SKenneth D. Merry case CTL_TAG_HEAD_OF_QUEUE: 1574130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1575130f4520SKenneth D. Merry break; 1576130f4520SKenneth D. Merry case CTL_TAG_UNTAGGED: 1577130f4520SKenneth D. Merry case CTL_TAG_SIMPLE: 1578130f4520SKenneth D. Merry case CTL_TAG_ACA: 1579130f4520SKenneth D. Merry default: 1580130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1581130f4520SKenneth D. Merry break; 1582130f4520SKenneth D. Merry } 1583130f4520SKenneth D. Merry 158411b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1585130f4520SKenneth D. Merry beio->bio_cmd = BIO_WRITE; 1586130f4520SKenneth D. Merry beio->ds_trans_type = DEVSTAT_WRITE; 158711b569f7SAlexander Motin } else { 158811b569f7SAlexander Motin beio->bio_cmd = BIO_READ; 158911b569f7SAlexander Motin beio->ds_trans_type = DEVSTAT_READ; 1590130f4520SKenneth D. Merry } 1591130f4520SKenneth D. Merry 159208a7cce5SAlexander Motin DPRINTF("%s at LBA %jx len %u @%ju\n", 1593130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", 1594e86a4142SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); 159511b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 159611b569f7SAlexander Motin lbas = CTLBLK_HALF_IO_SIZE; 159711b569f7SAlexander Motin else 159811b569f7SAlexander Motin lbas = CTLBLK_MAX_IO_SIZE; 15990bcd4ab6SAlexander Motin lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize); 16000bcd4ab6SAlexander Motin beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize; 16010bcd4ab6SAlexander Motin beio->io_len = lbas * cbe_lun->blocksize; 1602e86a4142SAlexander Motin bptrlen->len += lbas; 1603130f4520SKenneth D. Merry 160408a7cce5SAlexander Motin for (i = 0, len_left = beio->io_len; len_left > 0; i++) { 160508a7cce5SAlexander Motin KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)", 160608a7cce5SAlexander Motin i, CTLBLK_MAX_SEGS)); 1607130f4520SKenneth D. Merry 1608130f4520SKenneth D. Merry /* 1609130f4520SKenneth D. Merry * Setup the S/G entry for this chunk. 1610130f4520SKenneth D. Merry */ 161108a7cce5SAlexander Motin beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left); 1612130f4520SKenneth D. Merry beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1613130f4520SKenneth D. Merry 1614130f4520SKenneth D. Merry DPRINTF("segment %d addr %p len %zd\n", i, 1615130f4520SKenneth D. Merry beio->sg_segs[i].addr, beio->sg_segs[i].len); 1616130f4520SKenneth D. Merry 161711b569f7SAlexander Motin /* Set up second segment for compare operation. */ 161811b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) { 161911b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].len = 162011b569f7SAlexander Motin beio->sg_segs[i].len; 162111b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = 162211b569f7SAlexander Motin uma_zalloc(be_lun->lun_zone, M_WAITOK); 162311b569f7SAlexander Motin } 162411b569f7SAlexander Motin 1625130f4520SKenneth D. Merry beio->num_segs++; 1626130f4520SKenneth D. Merry len_left -= beio->sg_segs[i].len; 1627130f4520SKenneth D. Merry } 1628e86a4142SAlexander Motin if (bptrlen->len < lbalen->len) 162908a7cce5SAlexander Motin beio->beio_cont = ctl_be_block_next; 163008a7cce5SAlexander Motin io->scsiio.be_move_done = ctl_be_block_move_done; 163111b569f7SAlexander Motin /* For compare we have separate S/G lists for read and datamove. */ 163211b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 163311b569f7SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS]; 163411b569f7SAlexander Motin else 163508a7cce5SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs; 163608a7cce5SAlexander Motin io->scsiio.kern_data_len = beio->io_len; 163708a7cce5SAlexander Motin io->scsiio.kern_sg_entries = beio->num_segs; 1638b2221369SAlexander Motin io->io_hdr.flags |= CTL_FLAG_ALLOCATED; 1639130f4520SKenneth D. Merry 1640130f4520SKenneth D. Merry /* 1641130f4520SKenneth D. Merry * For the read case, we need to read the data into our buffers and 1642130f4520SKenneth D. Merry * then we can send it back to the user. For the write case, we 1643130f4520SKenneth D. Merry * need to get the data from the user first. 1644130f4520SKenneth D. Merry */ 1645130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 164636160958SMark Johnston SDT_PROBE0(cbb, , read, alloc_done); 1647130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1648130f4520SKenneth D. Merry } else { 164936160958SMark Johnston SDT_PROBE0(cbb, , write, alloc_done); 1650130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 1651e675024aSAlexander Motin getbinuptime(&io->io_hdr.dma_start_bt); 1652130f4520SKenneth D. Merry #endif 1653130f4520SKenneth D. Merry ctl_datamove(io); 1654130f4520SKenneth D. Merry } 1655130f4520SKenneth D. Merry } 1656130f4520SKenneth D. Merry 1657130f4520SKenneth D. Merry static void 1658130f4520SKenneth D. Merry ctl_be_block_worker(void *context, int pending) 1659130f4520SKenneth D. Merry { 1660ee4ad294SAlexander Motin struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context; 1661ee4ad294SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1662130f4520SKenneth D. Merry union ctl_io *io; 1663130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1664130f4520SKenneth D. Merry 1665ee4ad294SAlexander Motin DPRINTF("entered\n"); 1666ee4ad294SAlexander Motin /* 1667ee4ad294SAlexander Motin * Fetch and process I/Os from all queues. If we detect LUN 1668648dfc1aSAlexander Motin * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race, 1669ee4ad294SAlexander Motin * so make response maximally opaque to not confuse initiator. 1670ee4ad294SAlexander Motin */ 1671ee4ad294SAlexander Motin for (;;) { 1672ee4ad294SAlexander Motin mtx_lock(&be_lun->queue_lock); 1673ee4ad294SAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); 1674ee4ad294SAlexander Motin if (io != NULL) { 1675130f4520SKenneth D. Merry DPRINTF("datamove queue\n"); 1676130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, 1677130f4520SKenneth D. Merry ctl_io_hdr, links); 167875c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1679e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1680648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1681ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1682ee4ad294SAlexander Motin ctl_complete_beio(beio); 1683ee4ad294SAlexander Motin return; 1684ee4ad294SAlexander Motin } 1685130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1686130f4520SKenneth D. Merry continue; 1687130f4520SKenneth D. Merry } 1688130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); 1689130f4520SKenneth D. Merry if (io != NULL) { 1690130f4520SKenneth D. Merry DPRINTF("config write queue\n"); 1691130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, 1692130f4520SKenneth D. Merry ctl_io_hdr, links); 169375c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1694648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1695ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1696ee4ad294SAlexander Motin ctl_config_write_done(io); 1697ee4ad294SAlexander Motin return; 1698ee4ad294SAlexander Motin } 1699130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(be_lun, io); 1700ef8daf3fSAlexander Motin continue; 1701ef8daf3fSAlexander Motin } 1702ef8daf3fSAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); 1703ef8daf3fSAlexander Motin if (io != NULL) { 1704ef8daf3fSAlexander Motin DPRINTF("config read queue\n"); 1705ef8daf3fSAlexander Motin STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, 1706ef8daf3fSAlexander Motin ctl_io_hdr, links); 1707ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 1708648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1709ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1710ee4ad294SAlexander Motin ctl_config_read_done(io); 1711ee4ad294SAlexander Motin return; 1712ee4ad294SAlexander Motin } 1713ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(be_lun, io); 1714130f4520SKenneth D. Merry continue; 1715130f4520SKenneth D. Merry } 1716130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); 1717130f4520SKenneth D. Merry if (io != NULL) { 1718130f4520SKenneth D. Merry DPRINTF("input queue\n"); 1719130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, 1720130f4520SKenneth D. Merry ctl_io_hdr, links); 172175c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1722648dfc1aSAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { 1723ee4ad294SAlexander Motin ctl_set_busy(&io->scsiio); 1724ee4ad294SAlexander Motin ctl_data_submit_done(io); 1725ee4ad294SAlexander Motin return; 1726ee4ad294SAlexander Motin } 1727130f4520SKenneth D. Merry ctl_be_block_dispatch(be_lun, io); 1728130f4520SKenneth D. Merry continue; 1729130f4520SKenneth D. Merry } 1730130f4520SKenneth D. Merry 1731130f4520SKenneth D. Merry /* 1732130f4520SKenneth D. Merry * If we get here, there is no work left in the queues, so 1733130f4520SKenneth D. Merry * just break out and let the task queue go to sleep. 1734130f4520SKenneth D. Merry */ 1735ee4ad294SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1736130f4520SKenneth D. Merry break; 1737130f4520SKenneth D. Merry } 1738130f4520SKenneth D. Merry } 1739130f4520SKenneth D. Merry 1740130f4520SKenneth D. Merry /* 1741130f4520SKenneth D. Merry * Entry point from CTL to the backend for I/O. We queue everything to a 1742130f4520SKenneth D. Merry * work thread, so this just puts the I/O on a queue and wakes up the 1743130f4520SKenneth D. Merry * thread. 1744130f4520SKenneth D. Merry */ 1745130f4520SKenneth D. Merry static int 1746130f4520SKenneth D. Merry ctl_be_block_submit(union ctl_io *io) 1747130f4520SKenneth D. Merry { 1748130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 17490bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 1750130f4520SKenneth D. Merry 1751130f4520SKenneth D. Merry DPRINTF("entered\n"); 1752130f4520SKenneth D. Merry 17539cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 17540bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 1755130f4520SKenneth D. Merry 1756130f4520SKenneth D. Merry /* 1757130f4520SKenneth D. Merry * Make sure we only get SCSI I/O. 1758130f4520SKenneth D. Merry */ 1759130f4520SKenneth D. Merry KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type " 1760130f4520SKenneth D. Merry "%#x) encountered", io->io_hdr.io_type)); 1761130f4520SKenneth D. Merry 1762e86a4142SAlexander Motin PRIV(io)->len = 0; 1763e86a4142SAlexander Motin 176475c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1765130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 176675c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1767130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1768130f4520SKenneth D. Merry 17699c71cd5aSAlexander Motin return (CTL_RETVAL_COMPLETE); 1770130f4520SKenneth D. Merry } 1771130f4520SKenneth D. Merry 1772130f4520SKenneth D. Merry static int 1773130f4520SKenneth D. Merry ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 1774130f4520SKenneth D. Merry int flag, struct thread *td) 1775130f4520SKenneth D. Merry { 1776130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1777130f4520SKenneth D. Merry int error; 1778130f4520SKenneth D. Merry 1779130f4520SKenneth D. Merry softc = &backend_block_softc; 1780130f4520SKenneth D. Merry 1781130f4520SKenneth D. Merry error = 0; 1782130f4520SKenneth D. Merry 1783130f4520SKenneth D. Merry switch (cmd) { 1784130f4520SKenneth D. Merry case CTL_LUN_REQ: { 1785130f4520SKenneth D. Merry struct ctl_lun_req *lun_req; 1786130f4520SKenneth D. Merry 1787130f4520SKenneth D. Merry lun_req = (struct ctl_lun_req *)addr; 1788130f4520SKenneth D. Merry 1789130f4520SKenneth D. Merry switch (lun_req->reqtype) { 1790130f4520SKenneth D. Merry case CTL_LUNREQ_CREATE: 1791130f4520SKenneth D. Merry error = ctl_be_block_create(softc, lun_req); 1792130f4520SKenneth D. Merry break; 1793130f4520SKenneth D. Merry case CTL_LUNREQ_RM: 1794130f4520SKenneth D. Merry error = ctl_be_block_rm(softc, lun_req); 1795130f4520SKenneth D. Merry break; 179681177295SEdward Tomasz Napierala case CTL_LUNREQ_MODIFY: 179781177295SEdward Tomasz Napierala error = ctl_be_block_modify(softc, lun_req); 179881177295SEdward Tomasz Napierala break; 1799130f4520SKenneth D. Merry default: 1800130f4520SKenneth D. Merry lun_req->status = CTL_LUN_ERROR; 1801130f4520SKenneth D. Merry snprintf(lun_req->error_str, sizeof(lun_req->error_str), 180219720f41SAlexander Motin "invalid LUN request type %d", 1803130f4520SKenneth D. Merry lun_req->reqtype); 1804130f4520SKenneth D. Merry break; 1805130f4520SKenneth D. Merry } 1806130f4520SKenneth D. Merry break; 1807130f4520SKenneth D. Merry } 1808130f4520SKenneth D. Merry default: 1809130f4520SKenneth D. Merry error = ENOTTY; 1810130f4520SKenneth D. Merry break; 1811130f4520SKenneth D. Merry } 1812130f4520SKenneth D. Merry 1813130f4520SKenneth D. Merry return (error); 1814130f4520SKenneth D. Merry } 1815130f4520SKenneth D. Merry 1816130f4520SKenneth D. Merry static int 1817130f4520SKenneth D. Merry ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1818130f4520SKenneth D. Merry { 18190bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 1820130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 1821130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 18228951f055SMarcelo Araujo const char *value; 1823130f4520SKenneth D. Merry struct vattr vattr; 182434961f40SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos; 1825130f4520SKenneth D. Merry int error; 1826130f4520SKenneth D. Merry 18270bcd4ab6SAlexander Motin cbe_lun = &be_lun->cbe_lun; 1828130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 182919720f41SAlexander Motin params = &be_lun->params; 1830130f4520SKenneth D. Merry 1831130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_FILE; 1832130f4520SKenneth D. Merry be_lun->dispatch = ctl_be_block_dispatch_file; 1833130f4520SKenneth D. Merry be_lun->lun_flush = ctl_be_block_flush_file; 1834ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_file; 183553c146deSAlexander Motin be_lun->getattr = ctl_be_block_getattr_file; 18360bcd4ab6SAlexander Motin be_lun->unmap = NULL; 18370bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; 1838130f4520SKenneth D. Merry 1839130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 1840130f4520SKenneth D. Merry if (error != 0) { 1841130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1842130f4520SKenneth D. Merry "error calling VOP_GETATTR() for file %s", 1843130f4520SKenneth D. Merry be_lun->dev_path); 1844130f4520SKenneth D. Merry return (error); 1845130f4520SKenneth D. Merry } 1846130f4520SKenneth D. Merry 1847130f4520SKenneth D. Merry file_data->cred = crhold(curthread->td_ucred); 184881177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) 184981177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 185081177295SEdward Tomasz Napierala else 1851130f4520SKenneth D. Merry be_lun->size_bytes = vattr.va_size; 1852130f4520SKenneth D. Merry 1853130f4520SKenneth D. Merry /* 185420a28d6cSAlexander Motin * For files we can use any logical block size. Prefer 512 bytes 185520a28d6cSAlexander Motin * for compatibility reasons. If file's vattr.va_blocksize 185620a28d6cSAlexander Motin * (preferred I/O block size) is bigger and multiple to chosen 185720a28d6cSAlexander Motin * logical block size -- report it as physical block size. 1858130f4520SKenneth D. Merry */ 1859130f4520SKenneth D. Merry if (params->blocksize_bytes != 0) 18600bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 186191be33dcSAlexander Motin else if (cbe_lun->lun_type == T_CDROM) 186291be33dcSAlexander Motin cbe_lun->blocksize = 2048; 1863130f4520SKenneth D. Merry else 18640bcd4ab6SAlexander Motin cbe_lun->blocksize = 512; 18650bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 18660bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 18670bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 186834961f40SAlexander Motin 186934961f40SAlexander Motin us = ps = vattr.va_blocksize; 187034961f40SAlexander Motin uo = po = 0; 187134961f40SAlexander Motin 18728951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL); 187334961f40SAlexander Motin if (value != NULL) 187434961f40SAlexander Motin ctl_expand_number(value, &ps); 18758951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL); 187634961f40SAlexander Motin if (value != NULL) 187734961f40SAlexander Motin ctl_expand_number(value, &po); 18780bcd4ab6SAlexander Motin pss = ps / cbe_lun->blocksize; 18790bcd4ab6SAlexander Motin pos = po / cbe_lun->blocksize; 18800bcd4ab6SAlexander Motin if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && 18810bcd4ab6SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { 18820bcd4ab6SAlexander Motin cbe_lun->pblockexp = fls(pss) - 1; 18830bcd4ab6SAlexander Motin cbe_lun->pblockoff = (pss - pos) % pss; 188434961f40SAlexander Motin } 188534961f40SAlexander Motin 18868951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL); 188734961f40SAlexander Motin if (value != NULL) 188834961f40SAlexander Motin ctl_expand_number(value, &us); 18898951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL); 189034961f40SAlexander Motin if (value != NULL) 189134961f40SAlexander Motin ctl_expand_number(value, &uo); 18920bcd4ab6SAlexander Motin uss = us / cbe_lun->blocksize; 18930bcd4ab6SAlexander Motin uos = uo / cbe_lun->blocksize; 18940bcd4ab6SAlexander Motin if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && 18950bcd4ab6SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { 18960bcd4ab6SAlexander Motin cbe_lun->ublockexp = fls(uss) - 1; 18970bcd4ab6SAlexander Motin cbe_lun->ublockoff = (uss - uos) % uss; 189820a28d6cSAlexander Motin } 1899130f4520SKenneth D. Merry 1900130f4520SKenneth D. Merry /* 1901130f4520SKenneth D. Merry * Sanity check. The media size has to be at least one 1902130f4520SKenneth D. Merry * sector long. 1903130f4520SKenneth D. Merry */ 19040bcd4ab6SAlexander Motin if (be_lun->size_bytes < cbe_lun->blocksize) { 1905130f4520SKenneth D. Merry error = EINVAL; 1906130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1907130f4520SKenneth D. Merry "file %s size %ju < block size %u", be_lun->dev_path, 19080bcd4ab6SAlexander Motin (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize); 1909130f4520SKenneth D. Merry } 1910cb8727e2SAlexander Motin 19110bcd4ab6SAlexander Motin cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize; 1912130f4520SKenneth D. Merry return (error); 1913130f4520SKenneth D. Merry } 1914130f4520SKenneth D. Merry 1915130f4520SKenneth D. Merry static int 1916130f4520SKenneth D. Merry ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1917130f4520SKenneth D. Merry { 19180bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 1919130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 19203236151eSAlexander Motin struct cdevsw *csw; 1921130f4520SKenneth D. Merry struct cdev *dev; 19228951f055SMarcelo Araujo const char *value; 19233236151eSAlexander Motin int error, atomic, maxio, ref, unmap, tmp; 1924f6295033SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos, otmp; 1925130f4520SKenneth D. Merry 192619720f41SAlexander Motin params = &be_lun->params; 1927130f4520SKenneth D. Merry 1928130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_DEV; 19293236151eSAlexander Motin csw = devvn_refthread(be_lun->vn, &dev, &ref); 19303236151eSAlexander Motin if (csw == NULL) 19313236151eSAlexander Motin return (ENXIO); 19323236151eSAlexander Motin if (strcmp(csw->d_name, "zvol") == 0) { 193367f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_zvol; 1934ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_zvol; 1935cb8727e2SAlexander Motin atomic = maxio = CTLBLK_MAX_IO_SIZE; 1936cb8727e2SAlexander Motin } else { 193767f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_dev; 19380bcd4ab6SAlexander Motin be_lun->get_lba_status = NULL; 1939cb8727e2SAlexander Motin atomic = 0; 19403236151eSAlexander Motin maxio = dev->si_iosize_max; 1941cb8727e2SAlexander Motin if (maxio <= 0) 1942cb8727e2SAlexander Motin maxio = DFLTPHYS; 1943cb8727e2SAlexander Motin if (maxio > CTLBLK_MAX_IO_SIZE) 1944cb8727e2SAlexander Motin maxio = CTLBLK_MAX_IO_SIZE; 1945cb8727e2SAlexander Motin } 194667f586a8SAlexander Motin be_lun->lun_flush = ctl_be_block_flush_dev; 1947c3e7ba3eSAlexander Motin be_lun->getattr = ctl_be_block_getattr_dev; 19480bcd4ab6SAlexander Motin be_lun->unmap = ctl_be_block_unmap_dev; 1949130f4520SKenneth D. Merry 19503236151eSAlexander Motin if (!csw->d_ioctl) { 19513236151eSAlexander Motin dev_relthread(dev, ref); 1952130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 19533236151eSAlexander Motin "no d_ioctl for device %s!", be_lun->dev_path); 1954130f4520SKenneth D. Merry return (ENODEV); 1955130f4520SKenneth D. Merry } 1956130f4520SKenneth D. Merry 19573236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD, 1958130f4520SKenneth D. Merry curthread); 1959130f4520SKenneth D. Merry if (error) { 19603236151eSAlexander Motin dev_relthread(dev, ref); 1961130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 196219720f41SAlexander Motin "error %d returned for DIOCGSECTORSIZE ioctl " 196319720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 1964130f4520SKenneth D. Merry return (error); 1965130f4520SKenneth D. Merry } 1966130f4520SKenneth D. Merry 1967130f4520SKenneth D. Merry /* 1968130f4520SKenneth D. Merry * If the user has asked for a blocksize that is greater than the 1969130f4520SKenneth D. Merry * backing device's blocksize, we can do it only if the blocksize 1970130f4520SKenneth D. Merry * the user is asking for is an even multiple of the underlying 1971130f4520SKenneth D. Merry * device's blocksize. 1972130f4520SKenneth D. Merry */ 1973a15bbf15SAlexander Motin if ((params->blocksize_bytes != 0) && 1974a15bbf15SAlexander Motin (params->blocksize_bytes >= tmp)) { 1975a15bbf15SAlexander Motin if (params->blocksize_bytes % tmp == 0) { 19760bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 1977130f4520SKenneth D. Merry } else { 19783236151eSAlexander Motin dev_relthread(dev, ref); 1979130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 198019720f41SAlexander Motin "requested blocksize %u is not an even " 1981130f4520SKenneth D. Merry "multiple of backing device blocksize %u", 1982f6295033SAlexander Motin params->blocksize_bytes, tmp); 1983130f4520SKenneth D. Merry return (EINVAL); 1984130f4520SKenneth D. Merry } 1985a15bbf15SAlexander Motin } else if (params->blocksize_bytes != 0) { 19863236151eSAlexander Motin dev_relthread(dev, ref); 1987130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 198819720f41SAlexander Motin "requested blocksize %u < backing device " 1989f6295033SAlexander Motin "blocksize %u", params->blocksize_bytes, tmp); 1990130f4520SKenneth D. Merry return (EINVAL); 199191be33dcSAlexander Motin } else if (cbe_lun->lun_type == T_CDROM) 199291be33dcSAlexander Motin cbe_lun->blocksize = MAX(tmp, 2048); 199391be33dcSAlexander Motin else 19940bcd4ab6SAlexander Motin cbe_lun->blocksize = tmp; 1995130f4520SKenneth D. Merry 19963236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD, 1997130f4520SKenneth D. Merry curthread); 1998130f4520SKenneth D. Merry if (error) { 19993236151eSAlexander Motin dev_relthread(dev, ref); 2000130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 200119720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE " 200219720f41SAlexander Motin " ioctl on %s!", error, 200381177295SEdward Tomasz Napierala be_lun->dev_path); 2004130f4520SKenneth D. Merry return (error); 2005130f4520SKenneth D. Merry } 2006130f4520SKenneth D. Merry 200781177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 2008f6295033SAlexander Motin if (params->lun_size_bytes > otmp) { 20093236151eSAlexander Motin dev_relthread(dev, ref); 201081177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 201119720f41SAlexander Motin "requested LUN size %ju > backing device " 201219720f41SAlexander Motin "size %ju", 201381177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 2014f6295033SAlexander Motin (uintmax_t)otmp); 201581177295SEdward Tomasz Napierala return (EINVAL); 2016130f4520SKenneth D. Merry } 2017130f4520SKenneth D. Merry 201881177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 2019a15bbf15SAlexander Motin } else 2020f6295033SAlexander Motin be_lun->size_bytes = otmp; 20210bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 20220bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 20230bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 202481177295SEdward Tomasz Napierala 20253236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD, 20263236151eSAlexander Motin curthread); 2027f6012722SAlexander Motin if (error) 2028f6012722SAlexander Motin ps = po = 0; 2029f6012722SAlexander Motin else { 20303236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po, 20313236151eSAlexander Motin FREAD, curthread); 2032f6012722SAlexander Motin if (error) 2033f6012722SAlexander Motin po = 0; 2034f6012722SAlexander Motin } 203534961f40SAlexander Motin us = ps; 203634961f40SAlexander Motin uo = po; 203734961f40SAlexander Motin 20388951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL); 203934961f40SAlexander Motin if (value != NULL) 204034961f40SAlexander Motin ctl_expand_number(value, &ps); 20418951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL); 204234961f40SAlexander Motin if (value != NULL) 204334961f40SAlexander Motin ctl_expand_number(value, &po); 20440bcd4ab6SAlexander Motin pss = ps / cbe_lun->blocksize; 20450bcd4ab6SAlexander Motin pos = po / cbe_lun->blocksize; 20460bcd4ab6SAlexander Motin if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && 20470bcd4ab6SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { 20480bcd4ab6SAlexander Motin cbe_lun->pblockexp = fls(pss) - 1; 20490bcd4ab6SAlexander Motin cbe_lun->pblockoff = (pss - pos) % pss; 2050f6012722SAlexander Motin } 2051f6012722SAlexander Motin 20528951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL); 205334961f40SAlexander Motin if (value != NULL) 205434961f40SAlexander Motin ctl_expand_number(value, &us); 20558951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL); 205634961f40SAlexander Motin if (value != NULL) 205734961f40SAlexander Motin ctl_expand_number(value, &uo); 20580bcd4ab6SAlexander Motin uss = us / cbe_lun->blocksize; 20590bcd4ab6SAlexander Motin uos = uo / cbe_lun->blocksize; 20600bcd4ab6SAlexander Motin if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && 20610bcd4ab6SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { 20620bcd4ab6SAlexander Motin cbe_lun->ublockexp = fls(uss) - 1; 20630bcd4ab6SAlexander Motin cbe_lun->ublockoff = (uss - uos) % uss; 206434961f40SAlexander Motin } 206534961f40SAlexander Motin 20660bcd4ab6SAlexander Motin cbe_lun->atomicblock = atomic / cbe_lun->blocksize; 20670bcd4ab6SAlexander Motin cbe_lun->opttxferlen = maxio / cbe_lun->blocksize; 2068fbc8d4ffSAlexander Motin 2069fbc8d4ffSAlexander Motin if (be_lun->dispatch == ctl_be_block_dispatch_zvol) { 2070fbc8d4ffSAlexander Motin unmap = 1; 2071fbc8d4ffSAlexander Motin } else { 2072fbc8d4ffSAlexander Motin struct diocgattr_arg arg; 2073fbc8d4ffSAlexander Motin 2074fbc8d4ffSAlexander Motin strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); 2075fbc8d4ffSAlexander Motin arg.len = sizeof(arg.value.i); 20763236151eSAlexander Motin error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, 20773236151eSAlexander Motin curthread); 2078fbc8d4ffSAlexander Motin unmap = (error == 0) ? arg.value.i : 0; 2079fbc8d4ffSAlexander Motin } 20808951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "unmap", NULL); 2081fbc8d4ffSAlexander Motin if (value != NULL) 2082fbc8d4ffSAlexander Motin unmap = (strcmp(value, "on") == 0); 2083fbc8d4ffSAlexander Motin if (unmap) 20840bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_UNMAP; 20850bcd4ab6SAlexander Motin else 20860bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; 2087fbc8d4ffSAlexander Motin 20883236151eSAlexander Motin dev_relthread(dev, ref); 208981177295SEdward Tomasz Napierala return (0); 209081177295SEdward Tomasz Napierala } 2091130f4520SKenneth D. Merry 2092130f4520SKenneth D. Merry static int 2093130f4520SKenneth D. Merry ctl_be_block_close(struct ctl_be_block_lun *be_lun) 2094130f4520SKenneth D. Merry { 20950bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 20960bcd4ab6SAlexander Motin int flags; 2097130f4520SKenneth D. Merry 20980bcd4ab6SAlexander Motin if (be_lun->vn) { 20990bcd4ab6SAlexander Motin flags = FREAD; 21000bcd4ab6SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0) 21010bcd4ab6SAlexander Motin flags |= FWRITE; 2102130f4520SKenneth D. Merry (void)vn_close(be_lun->vn, flags, NOCRED, curthread); 2103130f4520SKenneth D. Merry be_lun->vn = NULL; 2104130f4520SKenneth D. Merry 2105130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2106130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2107130f4520SKenneth D. Merry break; 2108130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2109130f4520SKenneth D. Merry if (be_lun->backend.file.cred != NULL) { 2110130f4520SKenneth D. Merry crfree(be_lun->backend.file.cred); 2111130f4520SKenneth D. Merry be_lun->backend.file.cred = NULL; 2112130f4520SKenneth D. Merry } 2113130f4520SKenneth D. Merry break; 2114130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2115025a2301SEdward Tomasz Napierala break; 2116130f4520SKenneth D. Merry default: 21175124012aSAlexander Motin panic("Unexpected backend type %d", be_lun->dev_type); 2118130f4520SKenneth D. Merry break; 2119130f4520SKenneth D. Merry } 212019720f41SAlexander Motin be_lun->dev_type = CTL_BE_BLOCK_NONE; 2121130f4520SKenneth D. Merry } 2122130f4520SKenneth D. Merry return (0); 2123130f4520SKenneth D. Merry } 2124130f4520SKenneth D. Merry 2125130f4520SKenneth D. Merry static int 2126648dfc1aSAlexander Motin ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 2127130f4520SKenneth D. Merry { 21280bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; 2129130f4520SKenneth D. Merry struct nameidata nd; 21308951f055SMarcelo Araujo const char *value; 21310bcd4ab6SAlexander Motin int error, flags; 2132130f4520SKenneth D. Merry 2133130f4520SKenneth D. Merry error = 0; 2134130f4520SKenneth D. Merry if (rootvnode == NULL) { 2135130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 213619720f41SAlexander Motin "Root filesystem is not mounted"); 2137130f4520SKenneth D. Merry return (1); 2138130f4520SKenneth D. Merry } 21398a08cec1SMateusz Guzik pwd_ensure_dirs(); 2140130f4520SKenneth D. Merry 21418951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "file", NULL); 21420bcd4ab6SAlexander Motin if (value == NULL) { 21430bcd4ab6SAlexander Motin snprintf(req->error_str, sizeof(req->error_str), 21440bcd4ab6SAlexander Motin "no file argument specified"); 21450bcd4ab6SAlexander Motin return (1); 21460bcd4ab6SAlexander Motin } 21470bcd4ab6SAlexander Motin free(be_lun->dev_path, M_CTLBLK); 21480bcd4ab6SAlexander Motin be_lun->dev_path = strdup(value, M_CTLBLK); 21490bcd4ab6SAlexander Motin 21500bcd4ab6SAlexander Motin flags = FREAD; 21518951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "readonly", NULL); 215291be33dcSAlexander Motin if (value != NULL) { 215391be33dcSAlexander Motin if (strcmp(value, "on") != 0) 215491be33dcSAlexander Motin flags |= FWRITE; 215591be33dcSAlexander Motin } else if (cbe_lun->lun_type == T_DIRECT) 21560bcd4ab6SAlexander Motin flags |= FWRITE; 21570bcd4ab6SAlexander Motin 2158130f4520SKenneth D. Merry again: 2159130f4520SKenneth D. Merry NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread); 2160130f4520SKenneth D. Merry error = vn_open(&nd, &flags, 0, NULL); 21610bcd4ab6SAlexander Motin if ((error == EROFS || error == EACCES) && (flags & FWRITE)) { 21620bcd4ab6SAlexander Motin flags &= ~FWRITE; 21630bcd4ab6SAlexander Motin goto again; 21640bcd4ab6SAlexander Motin } 2165130f4520SKenneth D. Merry if (error) { 2166130f4520SKenneth D. Merry /* 2167130f4520SKenneth D. Merry * This is the only reasonable guess we can make as far as 2168130f4520SKenneth D. Merry * path if the user doesn't give us a fully qualified path. 2169130f4520SKenneth D. Merry * If they want to specify a file, they need to specify the 2170130f4520SKenneth D. Merry * full path. 2171130f4520SKenneth D. Merry */ 2172130f4520SKenneth D. Merry if (be_lun->dev_path[0] != '/') { 2173130f4520SKenneth D. Merry char *dev_name; 2174130f4520SKenneth D. Merry 21750bcd4ab6SAlexander Motin asprintf(&dev_name, M_CTLBLK, "/dev/%s", 2176130f4520SKenneth D. Merry be_lun->dev_path); 2177130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 2178130f4520SKenneth D. Merry be_lun->dev_path = dev_name; 2179130f4520SKenneth D. Merry goto again; 2180130f4520SKenneth D. Merry } 2181130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 218219720f41SAlexander Motin "error opening %s: %d", be_lun->dev_path, error); 2183130f4520SKenneth D. Merry return (error); 2184130f4520SKenneth D. Merry } 21850bcd4ab6SAlexander Motin if (flags & FWRITE) 21860bcd4ab6SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY; 21870bcd4ab6SAlexander Motin else 21880bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_READONLY; 2189130f4520SKenneth D. Merry 2190130f4520SKenneth D. Merry NDFREE(&nd, NDF_ONLY_PNBUF); 2191130f4520SKenneth D. Merry be_lun->vn = nd.ni_vp; 2192130f4520SKenneth D. Merry 2193130f4520SKenneth D. Merry /* We only support disks and files. */ 2194130f4520SKenneth D. Merry if (vn_isdisk(be_lun->vn, &error)) { 2195130f4520SKenneth D. Merry error = ctl_be_block_open_dev(be_lun, req); 2196130f4520SKenneth D. Merry } else if (be_lun->vn->v_type == VREG) { 2197130f4520SKenneth D. Merry error = ctl_be_block_open_file(be_lun, req); 2198130f4520SKenneth D. Merry } else { 2199130f4520SKenneth D. Merry error = EINVAL; 2200130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 2201025a2301SEdward Tomasz Napierala "%s is not a disk or plain file", be_lun->dev_path); 2202130f4520SKenneth D. Merry } 2203130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 2204130f4520SKenneth D. Merry 2205a15bbf15SAlexander Motin if (error != 0) 2206130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 22070bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; 22080bcd4ab6SAlexander Motin if (be_lun->dispatch != ctl_be_block_dispatch_dev) 22090bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_READ; 22108951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "serseq", NULL); 22110bcd4ab6SAlexander Motin if (value != NULL && strcmp(value, "on") == 0) 22120bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_ON; 22130bcd4ab6SAlexander Motin else if (value != NULL && strcmp(value, "read") == 0) 22140bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_READ; 22150bcd4ab6SAlexander Motin else if (value != NULL && strcmp(value, "off") == 0) 22160bcd4ab6SAlexander Motin cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; 2217130f4520SKenneth D. Merry return (0); 2218130f4520SKenneth D. Merry } 2219130f4520SKenneth D. Merry 2220130f4520SKenneth D. Merry static int 2221130f4520SKenneth D. Merry ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2222130f4520SKenneth D. Merry { 22230bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2224130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2225130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 222657a5db13SAlexander Motin char num_thread_str[16]; 2227130f4520SKenneth D. Merry char tmpstr[32]; 22288951f055SMarcelo Araujo const char *value; 2229fbc8d4ffSAlexander Motin int retval, num_threads; 223057a5db13SAlexander Motin int tmp_num_threads; 2231130f4520SKenneth D. Merry 2232130f4520SKenneth D. Merry params = &req->reqdata.create; 2233130f4520SKenneth D. Merry retval = 0; 223419720f41SAlexander Motin req->status = CTL_LUN_OK; 2235130f4520SKenneth D. Merry 2236130f4520SKenneth D. Merry be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK); 22370bcd4ab6SAlexander Motin cbe_lun = &be_lun->cbe_lun; 22380bcd4ab6SAlexander Motin cbe_lun->be_lun = be_lun; 223919720f41SAlexander Motin be_lun->params = req->reqdata.create; 2240130f4520SKenneth D. Merry be_lun->softc = softc; 2241130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->input_queue); 2242ef8daf3fSAlexander Motin STAILQ_INIT(&be_lun->config_read_queue); 2243130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->config_write_queue); 2244130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->datamove_queue); 2245130f4520SKenneth D. Merry sprintf(be_lun->lunname, "cblk%d", softc->num_luns); 224675c7a1d3SAlexander Motin mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF); 224775c7a1d3SAlexander Motin mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF); 22488951f055SMarcelo Araujo cbe_lun->options = nvlist_clone(req->args_nvl); 224908a7cce5SAlexander Motin be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG, 2250eeb94054SAlexander Motin NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0); 2251130f4520SKenneth D. Merry if (be_lun->lun_zone == NULL) { 2252130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 225319720f41SAlexander Motin "error allocating UMA zone"); 2254130f4520SKenneth D. Merry goto bailout_error; 2255130f4520SKenneth D. Merry } 2256130f4520SKenneth D. Merry 2257130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_DEV_TYPE) 22580bcd4ab6SAlexander Motin cbe_lun->lun_type = params->device_type; 2259130f4520SKenneth D. Merry else 22600bcd4ab6SAlexander Motin cbe_lun->lun_type = T_DIRECT; 22610bcd4ab6SAlexander Motin be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED; 22627ac58230SAlexander Motin cbe_lun->flags = 0; 22638951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL); 22647ac58230SAlexander Motin if (value != NULL) { 22657ac58230SAlexander Motin if (strcmp(value, "primary") == 0) 22667ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 22677ac58230SAlexander Motin } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) 22687ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 2269130f4520SKenneth D. Merry 227091be33dcSAlexander Motin if (cbe_lun->lun_type == T_DIRECT || 227191be33dcSAlexander Motin cbe_lun->lun_type == T_CDROM) { 2272a15bbf15SAlexander Motin be_lun->size_bytes = params->lun_size_bytes; 2273a15bbf15SAlexander Motin if (params->blocksize_bytes != 0) 22740bcd4ab6SAlexander Motin cbe_lun->blocksize = params->blocksize_bytes; 227591be33dcSAlexander Motin else if (cbe_lun->lun_type == T_CDROM) 227691be33dcSAlexander Motin cbe_lun->blocksize = 2048; 2277a15bbf15SAlexander Motin else 22780bcd4ab6SAlexander Motin cbe_lun->blocksize = 512; 22790bcd4ab6SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; 22800bcd4ab6SAlexander Motin cbe_lun->maxlba = (be_lun->size_blocks == 0) ? 22810bcd4ab6SAlexander Motin 0 : (be_lun->size_blocks - 1); 2282130f4520SKenneth D. Merry 22837ac58230SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || 22847ac58230SAlexander Motin control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { 2285648dfc1aSAlexander Motin retval = ctl_be_block_open(be_lun, req); 2286130f4520SKenneth D. Merry if (retval != 0) { 2287130f4520SKenneth D. Merry retval = 0; 228819720f41SAlexander Motin req->status = CTL_LUN_WARNING; 2289130f4520SKenneth D. Merry } 22907ac58230SAlexander Motin } 22910bcd4ab6SAlexander Motin num_threads = cbb_num_threads; 2292130f4520SKenneth D. Merry } else { 2293130f4520SKenneth D. Merry num_threads = 1; 2294130f4520SKenneth D. Merry } 2295130f4520SKenneth D. Merry 22968951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "num_threads", NULL); 229757a5db13SAlexander Motin if (value != NULL) { 229857a5db13SAlexander Motin tmp_num_threads = strtol(value, NULL, 0); 2299130f4520SKenneth D. Merry 2300130f4520SKenneth D. Merry /* 2301130f4520SKenneth D. Merry * We don't let the user specify less than one 2302130f4520SKenneth D. Merry * thread, but hope he's clueful enough not to 2303130f4520SKenneth D. Merry * specify 1000 threads. 2304130f4520SKenneth D. Merry */ 2305130f4520SKenneth D. Merry if (tmp_num_threads < 1) { 2306130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 230719720f41SAlexander Motin "invalid number of threads %s", 230819720f41SAlexander Motin num_thread_str); 2309130f4520SKenneth D. Merry goto bailout_error; 2310130f4520SKenneth D. Merry } 2311130f4520SKenneth D. Merry num_threads = tmp_num_threads; 231257a5db13SAlexander Motin } 2313130f4520SKenneth D. Merry 231419720f41SAlexander Motin if (be_lun->vn == NULL) 2315648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2316130f4520SKenneth D. Merry /* Tell the user the blocksize we ended up using */ 231719720f41SAlexander Motin params->lun_size_bytes = be_lun->size_bytes; 23180bcd4ab6SAlexander Motin params->blocksize_bytes = cbe_lun->blocksize; 2319130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_ID_REQ) { 23200bcd4ab6SAlexander Motin cbe_lun->req_lun_id = params->req_lun_id; 23210bcd4ab6SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ; 2322130f4520SKenneth D. Merry } else 23230bcd4ab6SAlexander Motin cbe_lun->req_lun_id = 0; 2324130f4520SKenneth D. Merry 23250bcd4ab6SAlexander Motin cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown; 23260bcd4ab6SAlexander Motin cbe_lun->lun_config_status = ctl_be_block_lun_config_status; 23270bcd4ab6SAlexander Motin cbe_lun->be = &ctl_be_block_driver; 2328130f4520SKenneth D. Merry 2329130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { 233071cd87c6SAlan Somers snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%04d", 2331130f4520SKenneth D. Merry softc->num_luns); 23320bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->serial_num, tmpstr, 23330bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr))); 2334130f4520SKenneth D. Merry 2335130f4520SKenneth D. Merry /* Tell the user what we used for a serial number */ 2336130f4520SKenneth D. Merry strncpy((char *)params->serial_num, tmpstr, 2337e7038eb7SAlexander Motin MIN(sizeof(params->serial_num), sizeof(tmpstr))); 2338130f4520SKenneth D. Merry } else { 23390bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->serial_num, params->serial_num, 23400bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->serial_num), 2341130f4520SKenneth D. Merry sizeof(params->serial_num))); 2342130f4520SKenneth D. Merry } 2343130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { 234471cd87c6SAlan Somers snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns); 23450bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->device_id, tmpstr, 23460bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr))); 2347130f4520SKenneth D. Merry 2348130f4520SKenneth D. Merry /* Tell the user what we used for a device ID */ 2349130f4520SKenneth D. Merry strncpy((char *)params->device_id, tmpstr, 2350e7038eb7SAlexander Motin MIN(sizeof(params->device_id), sizeof(tmpstr))); 2351130f4520SKenneth D. Merry } else { 23520bcd4ab6SAlexander Motin strncpy((char *)cbe_lun->device_id, params->device_id, 23530bcd4ab6SAlexander Motin MIN(sizeof(cbe_lun->device_id), 2354130f4520SKenneth D. Merry sizeof(params->device_id))); 2355130f4520SKenneth D. Merry } 2356130f4520SKenneth D. Merry 2357130f4520SKenneth D. Merry TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); 2358130f4520SKenneth D. Merry 2359130f4520SKenneth D. Merry be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK, 2360130f4520SKenneth D. Merry taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); 2361130f4520SKenneth D. Merry 2362130f4520SKenneth D. Merry if (be_lun->io_taskqueue == NULL) { 2363130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 236419720f41SAlexander Motin "unable to create taskqueue"); 2365130f4520SKenneth D. Merry goto bailout_error; 2366130f4520SKenneth D. Merry } 2367130f4520SKenneth D. Merry 2368130f4520SKenneth D. Merry /* 2369130f4520SKenneth D. Merry * Note that we start the same number of threads by default for 2370130f4520SKenneth D. Merry * both the file case and the block device case. For the file 2371130f4520SKenneth D. Merry * case, we need multiple threads to allow concurrency, because the 2372130f4520SKenneth D. Merry * vnode interface is designed to be a blocking interface. For the 2373130f4520SKenneth D. Merry * block device case, ZFS zvols at least will block the caller's 2374130f4520SKenneth D. Merry * context in many instances, and so we need multiple threads to 2375130f4520SKenneth D. Merry * overcome that problem. Other block devices don't need as many 2376130f4520SKenneth D. Merry * threads, but they shouldn't cause too many problems. 2377130f4520SKenneth D. Merry * 2378130f4520SKenneth D. Merry * If the user wants to just have a single thread for a block 2379130f4520SKenneth D. Merry * device, he can specify that when the LUN is created, or change 2380130f4520SKenneth D. Merry * the tunable/sysctl to alter the default number of threads. 2381130f4520SKenneth D. Merry */ 2382130f4520SKenneth D. Merry retval = taskqueue_start_threads(&be_lun->io_taskqueue, 2383130f4520SKenneth D. Merry /*num threads*/num_threads, 2384*053db1feSAlexander Motin /*priority*/PUSER, 2385130f4520SKenneth D. Merry /*thread name*/ 2386130f4520SKenneth D. Merry "%s taskq", be_lun->lunname); 2387130f4520SKenneth D. Merry 2388130f4520SKenneth D. Merry if (retval != 0) 2389130f4520SKenneth D. Merry goto bailout_error; 2390130f4520SKenneth D. Merry 2391130f4520SKenneth D. Merry be_lun->num_threads = num_threads; 2392130f4520SKenneth D. Merry 2393130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2394130f4520SKenneth D. Merry softc->num_luns++; 2395130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links); 2396130f4520SKenneth D. Merry 2397130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2398130f4520SKenneth D. Merry 23990bcd4ab6SAlexander Motin retval = ctl_add_lun(&be_lun->cbe_lun); 2400130f4520SKenneth D. Merry if (retval != 0) { 2401130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2402130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2403130f4520SKenneth D. Merry links); 2404130f4520SKenneth D. Merry softc->num_luns--; 2405130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2406130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 240719720f41SAlexander Motin "ctl_add_lun() returned error %d, see dmesg for " 240819720f41SAlexander Motin "details", retval); 2409130f4520SKenneth D. Merry retval = 0; 2410130f4520SKenneth D. Merry goto bailout_error; 2411130f4520SKenneth D. Merry } 2412130f4520SKenneth D. Merry 2413130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2414130f4520SKenneth D. Merry 2415130f4520SKenneth D. Merry /* 2416130f4520SKenneth D. Merry * Tell the config_status routine that we're waiting so it won't 2417130f4520SKenneth D. Merry * clean up the LUN in the event of an error. 2418130f4520SKenneth D. Merry */ 2419130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2420130f4520SKenneth D. Merry 2421130f4520SKenneth D. Merry while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { 2422130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2423130f4520SKenneth D. Merry if (retval == EINTR) 2424130f4520SKenneth D. Merry break; 2425130f4520SKenneth D. Merry } 2426130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2427130f4520SKenneth D. Merry 2428130f4520SKenneth D. Merry if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) { 2429130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 243019720f41SAlexander Motin "LUN configuration error, see dmesg for details"); 2431130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2432130f4520SKenneth D. Merry links); 2433130f4520SKenneth D. Merry softc->num_luns--; 2434130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2435130f4520SKenneth D. Merry goto bailout_error; 2436130f4520SKenneth D. Merry } else { 24370bcd4ab6SAlexander Motin params->req_lun_id = cbe_lun->lun_id; 2438130f4520SKenneth D. Merry } 2439130f4520SKenneth D. Merry 2440130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2441130f4520SKenneth D. Merry 2442130f4520SKenneth D. Merry be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id, 24430bcd4ab6SAlexander Motin cbe_lun->blocksize, 2444130f4520SKenneth D. Merry DEVSTAT_ALL_SUPPORTED, 24450bcd4ab6SAlexander Motin cbe_lun->lun_type 2446130f4520SKenneth D. Merry | DEVSTAT_TYPE_IF_OTHER, 2447130f4520SKenneth D. Merry DEVSTAT_PRIORITY_OTHER); 2448130f4520SKenneth D. Merry 2449130f4520SKenneth D. Merry return (retval); 2450130f4520SKenneth D. Merry 2451130f4520SKenneth D. Merry bailout_error: 2452130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2453130f4520SKenneth D. Merry 24549e005bbcSAlexander Motin if (be_lun->io_taskqueue != NULL) 24559e005bbcSAlexander Motin taskqueue_free(be_lun->io_taskqueue); 2456130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 24579e005bbcSAlexander Motin if (be_lun->dev_path != NULL) 2458130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 24599e005bbcSAlexander Motin if (be_lun->lun_zone != NULL) 24609e005bbcSAlexander Motin uma_zdestroy(be_lun->lun_zone); 24618951f055SMarcelo Araujo nvlist_destroy(cbe_lun->options); 246275c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 246375c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2464130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2465130f4520SKenneth D. Merry 2466130f4520SKenneth D. Merry return (retval); 2467130f4520SKenneth D. Merry } 2468130f4520SKenneth D. Merry 2469130f4520SKenneth D. Merry static int 2470130f4520SKenneth D. Merry ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2471130f4520SKenneth D. Merry { 2472130f4520SKenneth D. Merry struct ctl_lun_rm_params *params; 2473130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2474ee4ad294SAlexander Motin struct ctl_be_lun *cbe_lun; 2475130f4520SKenneth D. Merry int retval; 2476130f4520SKenneth D. Merry 2477130f4520SKenneth D. Merry params = &req->reqdata.rm; 2478130f4520SKenneth D. Merry 2479130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2480130f4520SKenneth D. Merry STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 24810bcd4ab6SAlexander Motin if (be_lun->cbe_lun.lun_id == params->lun_id) 2482130f4520SKenneth D. Merry break; 2483130f4520SKenneth D. Merry } 2484130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2485130f4520SKenneth D. Merry if (be_lun == NULL) { 2486130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 248719720f41SAlexander Motin "LUN %u is not managed by the block backend", 248819720f41SAlexander Motin params->lun_id); 2489130f4520SKenneth D. Merry goto bailout_error; 2490130f4520SKenneth D. Merry } 2491ee4ad294SAlexander Motin cbe_lun = &be_lun->cbe_lun; 2492130f4520SKenneth D. Merry 2493ee4ad294SAlexander Motin retval = ctl_disable_lun(cbe_lun); 2494130f4520SKenneth D. Merry if (retval != 0) { 2495130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 249619720f41SAlexander Motin "error %d returned from ctl_disable_lun() for " 249719720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2498130f4520SKenneth D. Merry goto bailout_error; 2499130f4520SKenneth D. Merry } 2500130f4520SKenneth D. Merry 2501ee4ad294SAlexander Motin if (be_lun->vn != NULL) { 2502648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2503648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2504ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 2505ee4ad294SAlexander Motin ctl_be_block_close(be_lun); 2506ee4ad294SAlexander Motin } 2507ee4ad294SAlexander Motin 2508ee4ad294SAlexander Motin retval = ctl_invalidate_lun(cbe_lun); 2509130f4520SKenneth D. Merry if (retval != 0) { 2510130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 251119720f41SAlexander Motin "error %d returned from ctl_invalidate_lun() for " 251219720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2513130f4520SKenneth D. Merry goto bailout_error; 2514130f4520SKenneth D. Merry } 2515130f4520SKenneth D. Merry 2516130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2517130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2518130f4520SKenneth D. Merry while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2519130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2520130f4520SKenneth D. Merry if (retval == EINTR) 2521130f4520SKenneth D. Merry break; 2522130f4520SKenneth D. Merry } 2523130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2524130f4520SKenneth D. Merry 2525130f4520SKenneth D. Merry if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2526130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 252719720f41SAlexander Motin "interrupted waiting for LUN to be freed"); 2528130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2529130f4520SKenneth D. Merry goto bailout_error; 2530130f4520SKenneth D. Merry } 2531130f4520SKenneth D. Merry 2532130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links); 2533130f4520SKenneth D. Merry 2534130f4520SKenneth D. Merry softc->num_luns--; 2535130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2536130f4520SKenneth D. Merry 2537ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 2538130f4520SKenneth D. Merry taskqueue_free(be_lun->io_taskqueue); 2539130f4520SKenneth D. Merry 2540130f4520SKenneth D. Merry if (be_lun->disk_stats != NULL) 2541130f4520SKenneth D. Merry devstat_remove_entry(be_lun->disk_stats); 2542130f4520SKenneth D. Merry 2543130f4520SKenneth D. Merry uma_zdestroy(be_lun->lun_zone); 2544130f4520SKenneth D. Merry 25458951f055SMarcelo Araujo nvlist_destroy(cbe_lun->options); 2546130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 254775c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 254875c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2549130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2550130f4520SKenneth D. Merry 2551130f4520SKenneth D. Merry req->status = CTL_LUN_OK; 2552130f4520SKenneth D. Merry return (0); 2553130f4520SKenneth D. Merry 2554130f4520SKenneth D. Merry bailout_error: 2555130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2556130f4520SKenneth D. Merry return (0); 2557130f4520SKenneth D. Merry } 2558130f4520SKenneth D. Merry 255981177295SEdward Tomasz Napierala static int 256081177295SEdward Tomasz Napierala ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 256181177295SEdward Tomasz Napierala { 256281177295SEdward Tomasz Napierala struct ctl_lun_modify_params *params; 256381177295SEdward Tomasz Napierala struct ctl_be_block_lun *be_lun; 2564a3977beaSAlexander Motin struct ctl_be_lun *cbe_lun; 25658951f055SMarcelo Araujo const char *value; 256671d8e97eSAlexander Motin uint64_t oldsize; 25677ac58230SAlexander Motin int error, wasprim; 256881177295SEdward Tomasz Napierala 256981177295SEdward Tomasz Napierala params = &req->reqdata.modify; 257081177295SEdward Tomasz Napierala 257181177295SEdward Tomasz Napierala mtx_lock(&softc->lock); 257281177295SEdward Tomasz Napierala STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 25730bcd4ab6SAlexander Motin if (be_lun->cbe_lun.lun_id == params->lun_id) 257481177295SEdward Tomasz Napierala break; 257581177295SEdward Tomasz Napierala } 257681177295SEdward Tomasz Napierala mtx_unlock(&softc->lock); 257781177295SEdward Tomasz Napierala if (be_lun == NULL) { 257881177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 257919720f41SAlexander Motin "LUN %u is not managed by the block backend", 258019720f41SAlexander Motin params->lun_id); 258181177295SEdward Tomasz Napierala goto bailout_error; 258281177295SEdward Tomasz Napierala } 2583a3977beaSAlexander Motin cbe_lun = &be_lun->cbe_lun; 258481177295SEdward Tomasz Napierala 2585a3977beaSAlexander Motin if (params->lun_size_bytes != 0) 258619720f41SAlexander Motin be_lun->params.lun_size_bytes = params->lun_size_bytes; 25878951f055SMarcelo Araujo 25888951f055SMarcelo Araujo nvlist_destroy(cbe_lun->options); 25898951f055SMarcelo Araujo cbe_lun->options = nvlist_clone(req->args_nvl); 259081177295SEdward Tomasz Napierala 25917ac58230SAlexander Motin wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY); 25928951f055SMarcelo Araujo value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL); 25937ac58230SAlexander Motin if (value != NULL) { 25947ac58230SAlexander Motin if (strcmp(value, "primary") == 0) 25957ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 25967ac58230SAlexander Motin else 25977ac58230SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; 25987ac58230SAlexander Motin } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) 25997ac58230SAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; 26007ac58230SAlexander Motin else 26017ac58230SAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; 26027ac58230SAlexander Motin if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) { 26037ac58230SAlexander Motin if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) 26047ac58230SAlexander Motin ctl_lun_primary(cbe_lun); 26057ac58230SAlexander Motin else 26067ac58230SAlexander Motin ctl_lun_secondary(cbe_lun); 26077ac58230SAlexander Motin } 26087ac58230SAlexander Motin 26090bcd4ab6SAlexander Motin oldsize = be_lun->size_blocks; 26107ac58230SAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || 26117ac58230SAlexander Motin control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { 261219720f41SAlexander Motin if (be_lun->vn == NULL) 2613648dfc1aSAlexander Motin error = ctl_be_block_open(be_lun, req); 2614b9b4269cSAlexander Motin else if (vn_isdisk(be_lun->vn, &error)) 26154ce7a086SAlexander Motin error = ctl_be_block_open_dev(be_lun, req); 26163d5cb709SAlexander Motin else if (be_lun->vn->v_type == VREG) { 26173d5cb709SAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 26184ce7a086SAlexander Motin error = ctl_be_block_open_file(be_lun, req); 26193d5cb709SAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 26203d5cb709SAlexander Motin } else 2621b9b4269cSAlexander Motin error = EINVAL; 2622648dfc1aSAlexander Motin if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) && 26230bcd4ab6SAlexander Motin be_lun->vn != NULL) { 2624648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; 2625648dfc1aSAlexander Motin ctl_lun_has_media(cbe_lun); 2626648dfc1aSAlexander Motin } else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 && 2627648dfc1aSAlexander Motin be_lun->vn == NULL) { 2628648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2629648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 263071d8e97eSAlexander Motin } 2631648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; 26327ac58230SAlexander Motin } else { 26337ac58230SAlexander Motin if (be_lun->vn != NULL) { 2634648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2635648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2636ee4ad294SAlexander Motin taskqueue_drain_all(be_lun->io_taskqueue); 26377ac58230SAlexander Motin error = ctl_be_block_close(be_lun); 26387ac58230SAlexander Motin } else 26397ac58230SAlexander Motin error = 0; 26407ac58230SAlexander Motin } 26417ac58230SAlexander Motin if (be_lun->size_blocks != oldsize) 26427ac58230SAlexander Motin ctl_lun_capacity_changed(cbe_lun); 264381177295SEdward Tomasz Napierala 264481177295SEdward Tomasz Napierala /* Tell the user the exact size we ended up using */ 264581177295SEdward Tomasz Napierala params->lun_size_bytes = be_lun->size_bytes; 264681177295SEdward Tomasz Napierala 264719720f41SAlexander Motin req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; 264881177295SEdward Tomasz Napierala return (0); 264981177295SEdward Tomasz Napierala 265081177295SEdward Tomasz Napierala bailout_error: 265181177295SEdward Tomasz Napierala req->status = CTL_LUN_ERROR; 265281177295SEdward Tomasz Napierala return (0); 265381177295SEdward Tomasz Napierala } 265481177295SEdward Tomasz Napierala 2655130f4520SKenneth D. Merry static void 2656130f4520SKenneth D. Merry ctl_be_block_lun_shutdown(void *be_lun) 2657130f4520SKenneth D. Merry { 265868bf823fSAlexander Motin struct ctl_be_block_lun *lun = be_lun; 265968bf823fSAlexander Motin struct ctl_be_block_softc *softc = lun->softc; 2660130f4520SKenneth D. Merry 2661130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2662130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; 2663130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2664130f4520SKenneth D. Merry wakeup(lun); 2665130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2666130f4520SKenneth D. Merry } 2667130f4520SKenneth D. Merry 2668130f4520SKenneth D. Merry static void 2669130f4520SKenneth D. Merry ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status) 2670130f4520SKenneth D. Merry { 2671130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2672130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2673130f4520SKenneth D. Merry 2674130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2675130f4520SKenneth D. Merry softc = lun->softc; 2676130f4520SKenneth D. Merry 2677130f4520SKenneth D. Merry if (status == CTL_LUN_CONFIG_OK) { 2678130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2679130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2680130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2681130f4520SKenneth D. Merry wakeup(lun); 2682130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2683130f4520SKenneth D. Merry 2684130f4520SKenneth D. Merry /* 2685130f4520SKenneth D. Merry * We successfully added the LUN, attempt to enable it. 2686130f4520SKenneth D. Merry */ 26870bcd4ab6SAlexander Motin if (ctl_enable_lun(&lun->cbe_lun) != 0) { 2688130f4520SKenneth D. Merry printf("%s: ctl_enable_lun() failed!\n", __func__); 26890bcd4ab6SAlexander Motin if (ctl_invalidate_lun(&lun->cbe_lun) != 0) { 2690130f4520SKenneth D. Merry printf("%s: ctl_invalidate_lun() failed!\n", 2691130f4520SKenneth D. Merry __func__); 2692130f4520SKenneth D. Merry } 2693130f4520SKenneth D. Merry } 2694130f4520SKenneth D. Merry 2695130f4520SKenneth D. Merry return; 2696130f4520SKenneth D. Merry } 2697130f4520SKenneth D. Merry 2698130f4520SKenneth D. Merry 2699130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2700130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2701130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR; 2702130f4520SKenneth D. Merry wakeup(lun); 2703130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2704130f4520SKenneth D. Merry } 2705130f4520SKenneth D. Merry 2706130f4520SKenneth D. Merry 2707130f4520SKenneth D. Merry static int 2708130f4520SKenneth D. Merry ctl_be_block_config_write(union ctl_io *io) 2709130f4520SKenneth D. Merry { 2710130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 27110bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2712130f4520SKenneth D. Merry int retval; 2713130f4520SKenneth D. Merry 2714130f4520SKenneth D. Merry DPRINTF("entered\n"); 2715130f4520SKenneth D. Merry 27169cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 27170bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 2718130f4520SKenneth D. Merry 271967cc546dSAlexander Motin retval = 0; 2720130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 2721130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 2722130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 2723ee7f31c0SAlexander Motin case WRITE_SAME_10: 2724ee7f31c0SAlexander Motin case WRITE_SAME_16: 2725ee7f31c0SAlexander Motin case UNMAP: 2726130f4520SKenneth D. Merry /* 2727130f4520SKenneth D. Merry * The upper level CTL code will filter out any CDBs with 2728130f4520SKenneth D. Merry * the immediate bit set and return the proper error. 2729130f4520SKenneth D. Merry * 2730130f4520SKenneth D. Merry * We don't really need to worry about what LBA range the 2731130f4520SKenneth D. Merry * user asked to be synced out. When they issue a sync 2732130f4520SKenneth D. Merry * cache command, we'll sync out the whole thing. 2733130f4520SKenneth D. Merry */ 273475c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 2735130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, 2736130f4520SKenneth D. Merry links); 273775c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 2738130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 2739130f4520SKenneth D. Merry break; 2740130f4520SKenneth D. Merry case START_STOP_UNIT: { 2741130f4520SKenneth D. Merry struct scsi_start_stop_unit *cdb; 2742648dfc1aSAlexander Motin struct ctl_lun_req req; 2743130f4520SKenneth D. Merry 2744130f4520SKenneth D. Merry cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; 274566b69676SAlexander Motin if ((cdb->how & SSS_PC_MASK) != 0) { 274666b69676SAlexander Motin ctl_set_success(&io->scsiio); 274766b69676SAlexander Motin ctl_config_write_done(io); 274866b69676SAlexander Motin break; 274966b69676SAlexander Motin } 2750648dfc1aSAlexander Motin if (cdb->how & SSS_START) { 2751648dfc1aSAlexander Motin if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) { 2752648dfc1aSAlexander Motin retval = ctl_be_block_open(be_lun, &req); 2753648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; 2754648dfc1aSAlexander Motin if (retval == 0) { 2755648dfc1aSAlexander Motin cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; 2756648dfc1aSAlexander Motin ctl_lun_has_media(cbe_lun); 2757130f4520SKenneth D. Merry } else { 2758648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2759648dfc1aSAlexander Motin ctl_lun_no_media(cbe_lun); 2760130f4520SKenneth D. Merry } 2761648dfc1aSAlexander Motin } 2762648dfc1aSAlexander Motin ctl_start_lun(cbe_lun); 2763648dfc1aSAlexander Motin } else { 2764648dfc1aSAlexander Motin ctl_stop_lun(cbe_lun); 2765648dfc1aSAlexander Motin if (cdb->how & SSS_LOEJ) { 2766648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; 2767648dfc1aSAlexander Motin cbe_lun->flags |= CTL_LUN_FLAG_EJECTED; 2768648dfc1aSAlexander Motin ctl_lun_ejected(cbe_lun); 2769648dfc1aSAlexander Motin if (be_lun->vn != NULL) 2770648dfc1aSAlexander Motin ctl_be_block_close(be_lun); 2771648dfc1aSAlexander Motin } 2772648dfc1aSAlexander Motin } 2773648dfc1aSAlexander Motin 2774648dfc1aSAlexander Motin ctl_set_success(&io->scsiio); 2775130f4520SKenneth D. Merry ctl_config_write_done(io); 2776130f4520SKenneth D. Merry break; 2777130f4520SKenneth D. Merry } 277891be33dcSAlexander Motin case PREVENT_ALLOW: 277991be33dcSAlexander Motin ctl_set_success(&io->scsiio); 278091be33dcSAlexander Motin ctl_config_write_done(io); 278191be33dcSAlexander Motin break; 2782130f4520SKenneth D. Merry default: 2783130f4520SKenneth D. Merry ctl_set_invalid_opcode(&io->scsiio); 2784130f4520SKenneth D. Merry ctl_config_write_done(io); 2785130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2786130f4520SKenneth D. Merry break; 2787130f4520SKenneth D. Merry } 2788130f4520SKenneth D. Merry 2789130f4520SKenneth D. Merry return (retval); 2790130f4520SKenneth D. Merry } 2791130f4520SKenneth D. Merry 2792130f4520SKenneth D. Merry static int 2793130f4520SKenneth D. Merry ctl_be_block_config_read(union ctl_io *io) 2794130f4520SKenneth D. Merry { 2795ef8daf3fSAlexander Motin struct ctl_be_block_lun *be_lun; 27960bcd4ab6SAlexander Motin struct ctl_be_lun *cbe_lun; 2797ef8daf3fSAlexander Motin int retval = 0; 2798ef8daf3fSAlexander Motin 2799ef8daf3fSAlexander Motin DPRINTF("entered\n"); 2800ef8daf3fSAlexander Motin 28019cbbfd2fSAlexander Motin cbe_lun = CTL_BACKEND_LUN(io); 28020bcd4ab6SAlexander Motin be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun; 2803ef8daf3fSAlexander Motin 2804ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 2805ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: 2806ef8daf3fSAlexander Motin if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { 2807ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 2808ef8daf3fSAlexander Motin STAILQ_INSERT_TAIL(&be_lun->config_read_queue, 2809ef8daf3fSAlexander Motin &io->io_hdr, links); 2810ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 2811ef8daf3fSAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, 2812ef8daf3fSAlexander Motin &be_lun->io_task); 2813ef8daf3fSAlexander Motin retval = CTL_RETVAL_QUEUED; 2814ef8daf3fSAlexander Motin break; 2815ef8daf3fSAlexander Motin } 2816ef8daf3fSAlexander Motin ctl_set_invalid_field(&io->scsiio, 2817ef8daf3fSAlexander Motin /*sks_valid*/ 1, 2818ef8daf3fSAlexander Motin /*command*/ 1, 2819ef8daf3fSAlexander Motin /*field*/ 1, 2820ef8daf3fSAlexander Motin /*bit_valid*/ 1, 2821ef8daf3fSAlexander Motin /*bit*/ 4); 2822ef8daf3fSAlexander Motin ctl_config_read_done(io); 2823ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2824ef8daf3fSAlexander Motin break; 2825ef8daf3fSAlexander Motin default: 2826ef8daf3fSAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 2827ef8daf3fSAlexander Motin ctl_config_read_done(io); 2828ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2829ef8daf3fSAlexander Motin break; 2830ef8daf3fSAlexander Motin } 2831ef8daf3fSAlexander Motin 2832ef8daf3fSAlexander Motin return (retval); 2833130f4520SKenneth D. Merry } 2834130f4520SKenneth D. Merry 2835130f4520SKenneth D. Merry static int 2836130f4520SKenneth D. Merry ctl_be_block_lun_info(void *be_lun, struct sbuf *sb) 2837130f4520SKenneth D. Merry { 2838130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2839130f4520SKenneth D. Merry int retval; 2840130f4520SKenneth D. Merry 2841130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2842130f4520SKenneth D. Merry 28432cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "\t<num_threads>"); 2844130f4520SKenneth D. Merry if (retval != 0) 2845130f4520SKenneth D. Merry goto bailout; 2846130f4520SKenneth D. Merry retval = sbuf_printf(sb, "%d", lun->num_threads); 2847130f4520SKenneth D. Merry if (retval != 0) 2848130f4520SKenneth D. Merry goto bailout; 28492cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "</num_threads>\n"); 2850130f4520SKenneth D. Merry 2851130f4520SKenneth D. Merry bailout: 2852130f4520SKenneth D. Merry return (retval); 2853130f4520SKenneth D. Merry } 2854130f4520SKenneth D. Merry 2855c3e7ba3eSAlexander Motin static uint64_t 2856c3e7ba3eSAlexander Motin ctl_be_block_lun_attr(void *be_lun, const char *attrname) 2857c3e7ba3eSAlexander Motin { 2858c3e7ba3eSAlexander Motin struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun; 2859c3e7ba3eSAlexander Motin 2860c3e7ba3eSAlexander Motin if (lun->getattr == NULL) 2861c3e7ba3eSAlexander Motin return (UINT64_MAX); 2862c3e7ba3eSAlexander Motin return (lun->getattr(lun, attrname)); 2863c3e7ba3eSAlexander Motin } 2864c3e7ba3eSAlexander Motin 28650c629e28SAlexander Motin static int 2866130f4520SKenneth D. Merry ctl_be_block_init(void) 2867130f4520SKenneth D. Merry { 28680c629e28SAlexander Motin struct ctl_be_block_softc *softc = &backend_block_softc; 2869130f4520SKenneth D. Merry 287075c7a1d3SAlexander Motin mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); 28710c629e28SAlexander Motin softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), 2872a0e36aeeSEdward Tomasz Napierala NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 2873130f4520SKenneth D. Merry STAILQ_INIT(&softc->lun_list); 28740c629e28SAlexander Motin return (0); 28750c629e28SAlexander Motin } 2876130f4520SKenneth D. Merry 28770c629e28SAlexander Motin 28780c629e28SAlexander Motin static int 28790c629e28SAlexander Motin ctl_be_block_shutdown(void) 28800c629e28SAlexander Motin { 28810c629e28SAlexander Motin struct ctl_be_block_softc *softc = &backend_block_softc; 28820c629e28SAlexander Motin struct ctl_be_block_lun *lun, *next_lun; 28830c629e28SAlexander Motin 28840c629e28SAlexander Motin mtx_lock(&softc->lock); 28850c629e28SAlexander Motin STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) { 28860c629e28SAlexander Motin /* 28870c629e28SAlexander Motin * Drop our lock here. Since ctl_invalidate_lun() can call 28880c629e28SAlexander Motin * back into us, this could potentially lead to a recursive 28890c629e28SAlexander Motin * lock of the same mutex, which would cause a hang. 28900c629e28SAlexander Motin */ 28910c629e28SAlexander Motin mtx_unlock(&softc->lock); 28920c629e28SAlexander Motin ctl_disable_lun(&lun->cbe_lun); 28930c629e28SAlexander Motin ctl_invalidate_lun(&lun->cbe_lun); 28940c629e28SAlexander Motin mtx_lock(&softc->lock); 28950c629e28SAlexander Motin } 28960c629e28SAlexander Motin mtx_unlock(&softc->lock); 28970c629e28SAlexander Motin 28980c629e28SAlexander Motin uma_zdestroy(softc->beio_zone); 28990c629e28SAlexander Motin mtx_destroy(&softc->lock); 29000c629e28SAlexander Motin return (0); 2901130f4520SKenneth D. Merry } 2902