1130f4520SKenneth D. Merry /*- 2130f4520SKenneth D. Merry * Copyright (c) 2003 Silicon Graphics International Corp. 3130f4520SKenneth D. Merry * Copyright (c) 2009-2011 Spectra Logic Corporation 481177295SEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 5130f4520SKenneth D. Merry * All rights reserved. 6130f4520SKenneth D. Merry * 781177295SEdward Tomasz Napierala * Portions of this software were developed by Edward Tomasz Napierala 881177295SEdward Tomasz Napierala * under sponsorship from the FreeBSD Foundation. 981177295SEdward Tomasz Napierala * 10130f4520SKenneth D. Merry * Redistribution and use in source and binary forms, with or without 11130f4520SKenneth D. Merry * modification, are permitted provided that the following conditions 12130f4520SKenneth D. Merry * are met: 13130f4520SKenneth D. Merry * 1. Redistributions of source code must retain the above copyright 14130f4520SKenneth D. Merry * notice, this list of conditions, and the following disclaimer, 15130f4520SKenneth D. Merry * without modification. 16130f4520SKenneth D. Merry * 2. Redistributions in binary form must reproduce at minimum a disclaimer 17130f4520SKenneth D. Merry * substantially similar to the "NO WARRANTY" disclaimer below 18130f4520SKenneth D. Merry * ("Disclaimer") and any redistribution must be conditioned upon 19130f4520SKenneth D. Merry * including a substantially similar Disclaimer requirement for further 20130f4520SKenneth D. Merry * binary redistribution. 21130f4520SKenneth D. Merry * 22130f4520SKenneth D. Merry * NO WARRANTY 23130f4520SKenneth D. Merry * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 24130f4520SKenneth D. Merry * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 25130f4520SKenneth D. Merry * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 26130f4520SKenneth D. Merry * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 27130f4520SKenneth D. Merry * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28130f4520SKenneth D. Merry * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29130f4520SKenneth D. Merry * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30130f4520SKenneth D. Merry * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 31130f4520SKenneth D. Merry * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 32130f4520SKenneth D. Merry * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33130f4520SKenneth D. Merry * POSSIBILITY OF SUCH DAMAGES. 34130f4520SKenneth D. Merry * 35130f4520SKenneth D. Merry * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $ 36130f4520SKenneth D. Merry */ 37130f4520SKenneth D. Merry /* 38130f4520SKenneth D. Merry * CAM Target Layer driver backend for block devices. 39130f4520SKenneth D. Merry * 40130f4520SKenneth D. Merry * Author: Ken Merry <ken@FreeBSD.org> 41130f4520SKenneth D. Merry */ 42130f4520SKenneth D. Merry #include <sys/cdefs.h> 43130f4520SKenneth D. Merry __FBSDID("$FreeBSD$"); 44130f4520SKenneth D. Merry 45130f4520SKenneth D. Merry #include <sys/param.h> 46130f4520SKenneth D. Merry #include <sys/systm.h> 47130f4520SKenneth D. Merry #include <sys/kernel.h> 48130f4520SKenneth D. Merry #include <sys/types.h> 49130f4520SKenneth D. Merry #include <sys/kthread.h> 50130f4520SKenneth D. Merry #include <sys/bio.h> 51130f4520SKenneth D. Merry #include <sys/fcntl.h> 52ee7f31c0SAlexander Motin #include <sys/limits.h> 53130f4520SKenneth D. Merry #include <sys/lock.h> 54130f4520SKenneth D. Merry #include <sys/mutex.h> 55130f4520SKenneth D. Merry #include <sys/condvar.h> 56130f4520SKenneth D. Merry #include <sys/malloc.h> 57130f4520SKenneth D. Merry #include <sys/conf.h> 58130f4520SKenneth D. Merry #include <sys/ioccom.h> 59130f4520SKenneth D. Merry #include <sys/queue.h> 60130f4520SKenneth D. Merry #include <sys/sbuf.h> 61130f4520SKenneth D. Merry #include <sys/endian.h> 62130f4520SKenneth D. Merry #include <sys/uio.h> 63130f4520SKenneth D. Merry #include <sys/buf.h> 64130f4520SKenneth D. Merry #include <sys/taskqueue.h> 65130f4520SKenneth D. Merry #include <sys/vnode.h> 66130f4520SKenneth D. Merry #include <sys/namei.h> 67130f4520SKenneth D. Merry #include <sys/mount.h> 68130f4520SKenneth D. Merry #include <sys/disk.h> 69130f4520SKenneth D. Merry #include <sys/fcntl.h> 70130f4520SKenneth D. Merry #include <sys/filedesc.h> 71ef8daf3fSAlexander Motin #include <sys/filio.h> 72130f4520SKenneth D. Merry #include <sys/proc.h> 73130f4520SKenneth D. Merry #include <sys/pcpu.h> 74130f4520SKenneth D. Merry #include <sys/module.h> 75130f4520SKenneth D. Merry #include <sys/sdt.h> 76130f4520SKenneth D. Merry #include <sys/devicestat.h> 77130f4520SKenneth D. Merry #include <sys/sysctl.h> 78130f4520SKenneth D. Merry 79130f4520SKenneth D. Merry #include <geom/geom.h> 80130f4520SKenneth D. Merry 81130f4520SKenneth D. Merry #include <cam/cam.h> 82130f4520SKenneth D. Merry #include <cam/scsi/scsi_all.h> 83130f4520SKenneth D. Merry #include <cam/scsi/scsi_da.h> 84130f4520SKenneth D. Merry #include <cam/ctl/ctl_io.h> 85130f4520SKenneth D. Merry #include <cam/ctl/ctl.h> 86130f4520SKenneth D. Merry #include <cam/ctl/ctl_backend.h> 87130f4520SKenneth D. Merry #include <cam/ctl/ctl_ioctl.h> 88130f4520SKenneth D. Merry #include <cam/ctl/ctl_scsi_all.h> 89130f4520SKenneth D. Merry #include <cam/ctl/ctl_error.h> 90130f4520SKenneth D. Merry 91130f4520SKenneth D. Merry /* 9208a7cce5SAlexander Motin * The idea here is that we'll allocate enough S/G space to hold a 1MB 9308a7cce5SAlexander Motin * I/O. If we get an I/O larger than that, we'll split it. 94130f4520SKenneth D. Merry */ 9511b569f7SAlexander Motin #define CTLBLK_HALF_IO_SIZE (512 * 1024) 9611b569f7SAlexander Motin #define CTLBLK_MAX_IO_SIZE (CTLBLK_HALF_IO_SIZE * 2) 9708a7cce5SAlexander Motin #define CTLBLK_MAX_SEG MAXPHYS 9811b569f7SAlexander Motin #define CTLBLK_HALF_SEGS MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1) 9911b569f7SAlexander Motin #define CTLBLK_MAX_SEGS (CTLBLK_HALF_SEGS * 2) 100130f4520SKenneth D. Merry 101130f4520SKenneth D. Merry #ifdef CTLBLK_DEBUG 102130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) \ 103130f4520SKenneth D. Merry printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) 104130f4520SKenneth D. Merry #else 105130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) do {} while(0) 106130f4520SKenneth D. Merry #endif 107130f4520SKenneth D. Merry 108e86a4142SAlexander Motin #define PRIV(io) \ 109e86a4142SAlexander Motin ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND]) 11011b569f7SAlexander Motin #define ARGS(io) \ 11111b569f7SAlexander Motin ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]) 112e86a4142SAlexander Motin 113130f4520SKenneth D. Merry SDT_PROVIDER_DEFINE(cbb); 114130f4520SKenneth D. Merry 115130f4520SKenneth D. Merry typedef enum { 116130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_UNCONFIGURED = 0x01, 117130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_CONFIG_ERR = 0x02, 118130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_WAITING = 0x04, 119130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_MULTI_THREAD = 0x08 120130f4520SKenneth D. Merry } ctl_be_block_lun_flags; 121130f4520SKenneth D. Merry 122130f4520SKenneth D. Merry typedef enum { 123130f4520SKenneth D. Merry CTL_BE_BLOCK_NONE, 124130f4520SKenneth D. Merry CTL_BE_BLOCK_DEV, 125130f4520SKenneth D. Merry CTL_BE_BLOCK_FILE 126130f4520SKenneth D. Merry } ctl_be_block_type; 127130f4520SKenneth D. Merry 128130f4520SKenneth D. Merry struct ctl_be_block_devdata { 129130f4520SKenneth D. Merry struct cdev *cdev; 130130f4520SKenneth D. Merry struct cdevsw *csw; 131130f4520SKenneth D. Merry int dev_ref; 132130f4520SKenneth D. Merry }; 133130f4520SKenneth D. Merry 134130f4520SKenneth D. Merry struct ctl_be_block_filedata { 135130f4520SKenneth D. Merry struct ucred *cred; 136130f4520SKenneth D. Merry }; 137130f4520SKenneth D. Merry 138130f4520SKenneth D. Merry union ctl_be_block_bedata { 139130f4520SKenneth D. Merry struct ctl_be_block_devdata dev; 140130f4520SKenneth D. Merry struct ctl_be_block_filedata file; 141130f4520SKenneth D. Merry }; 142130f4520SKenneth D. Merry 143130f4520SKenneth D. Merry struct ctl_be_block_io; 144130f4520SKenneth D. Merry struct ctl_be_block_lun; 145130f4520SKenneth D. Merry 146130f4520SKenneth D. Merry typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun, 147130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 148c3e7ba3eSAlexander Motin typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun, 149c3e7ba3eSAlexander Motin const char *attrname); 150130f4520SKenneth D. Merry 151130f4520SKenneth D. Merry /* 152130f4520SKenneth D. Merry * Backend LUN structure. There is a 1:1 mapping between a block device 153130f4520SKenneth D. Merry * and a backend block LUN, and between a backend block LUN and a CTL LUN. 154130f4520SKenneth D. Merry */ 155130f4520SKenneth D. Merry struct ctl_be_block_lun { 15619720f41SAlexander Motin struct ctl_lun_create_params params; 157130f4520SKenneth D. Merry struct ctl_block_disk *disk; 158130f4520SKenneth D. Merry char lunname[32]; 159130f4520SKenneth D. Merry char *dev_path; 160130f4520SKenneth D. Merry ctl_be_block_type dev_type; 161130f4520SKenneth D. Merry struct vnode *vn; 162130f4520SKenneth D. Merry union ctl_be_block_bedata backend; 163130f4520SKenneth D. Merry cbb_dispatch_t dispatch; 164130f4520SKenneth D. Merry cbb_dispatch_t lun_flush; 165ee7f31c0SAlexander Motin cbb_dispatch_t unmap; 166ef8daf3fSAlexander Motin cbb_dispatch_t get_lba_status; 167c3e7ba3eSAlexander Motin cbb_getattr_t getattr; 168130f4520SKenneth D. Merry uma_zone_t lun_zone; 169130f4520SKenneth D. Merry uint64_t size_blocks; 170130f4520SKenneth D. Merry uint64_t size_bytes; 171130f4520SKenneth D. Merry uint32_t blocksize; 172f6012722SAlexander Motin uint16_t pblockexp; 173f6012722SAlexander Motin uint16_t pblockoff; 17434961f40SAlexander Motin uint16_t ublockexp; 17534961f40SAlexander Motin uint16_t ublockoff; 176cb8727e2SAlexander Motin uint32_t atomicblock; 177cb8727e2SAlexander Motin uint32_t opttxferlen; 178130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 179130f4520SKenneth D. Merry struct devstat *disk_stats; 180130f4520SKenneth D. Merry ctl_be_block_lun_flags flags; 181130f4520SKenneth D. Merry STAILQ_ENTRY(ctl_be_block_lun) links; 182130f4520SKenneth D. Merry struct ctl_be_lun ctl_be_lun; 183130f4520SKenneth D. Merry struct taskqueue *io_taskqueue; 184130f4520SKenneth D. Merry struct task io_task; 185130f4520SKenneth D. Merry int num_threads; 186130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) input_queue; 187ef8daf3fSAlexander Motin STAILQ_HEAD(, ctl_io_hdr) config_read_queue; 188130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) config_write_queue; 189130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 19075c7a1d3SAlexander Motin struct mtx_padalign io_lock; 19175c7a1d3SAlexander Motin struct mtx_padalign queue_lock; 192130f4520SKenneth D. Merry }; 193130f4520SKenneth D. Merry 194130f4520SKenneth D. Merry /* 195130f4520SKenneth D. Merry * Overall softc structure for the block backend module. 196130f4520SKenneth D. Merry */ 197130f4520SKenneth D. Merry struct ctl_be_block_softc { 198130f4520SKenneth D. Merry struct mtx lock; 199130f4520SKenneth D. Merry int num_disks; 200130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_block_disk) disk_list; 201130f4520SKenneth D. Merry int num_luns; 202130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_be_block_lun) lun_list; 203130f4520SKenneth D. Merry }; 204130f4520SKenneth D. Merry 205130f4520SKenneth D. Merry static struct ctl_be_block_softc backend_block_softc; 206130f4520SKenneth D. Merry 207130f4520SKenneth D. Merry /* 208130f4520SKenneth D. Merry * Per-I/O information. 209130f4520SKenneth D. Merry */ 210130f4520SKenneth D. Merry struct ctl_be_block_io { 211130f4520SKenneth D. Merry union ctl_io *io; 212130f4520SKenneth D. Merry struct ctl_sg_entry sg_segs[CTLBLK_MAX_SEGS]; 213130f4520SKenneth D. Merry struct iovec xiovecs[CTLBLK_MAX_SEGS]; 214130f4520SKenneth D. Merry int bio_cmd; 215130f4520SKenneth D. Merry int num_segs; 216130f4520SKenneth D. Merry int num_bios_sent; 217130f4520SKenneth D. Merry int num_bios_done; 218130f4520SKenneth D. Merry int send_complete; 219130f4520SKenneth D. Merry int num_errors; 220130f4520SKenneth D. Merry struct bintime ds_t0; 221130f4520SKenneth D. Merry devstat_tag_type ds_tag_type; 222130f4520SKenneth D. Merry devstat_trans_flags ds_trans_type; 223130f4520SKenneth D. Merry uint64_t io_len; 224130f4520SKenneth D. Merry uint64_t io_offset; 2257d0d4342SAlexander Motin int io_arg; 226130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 227130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 228ee7f31c0SAlexander Motin void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */ 229130f4520SKenneth D. Merry }; 230130f4520SKenneth D. Merry 231130f4520SKenneth D. Merry static int cbb_num_threads = 14; 232130f4520SKenneth D. Merry SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0, 233130f4520SKenneth D. Merry "CAM Target Layer Block Backend"); 234af3b2549SHans Petter Selasky SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, 235130f4520SKenneth D. Merry &cbb_num_threads, 0, "Number of threads per backing file"); 236130f4520SKenneth D. Merry 237130f4520SKenneth D. Merry static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); 238130f4520SKenneth D. Merry static void ctl_free_beio(struct ctl_be_block_io *beio); 239130f4520SKenneth D. Merry static void ctl_complete_beio(struct ctl_be_block_io *beio); 240130f4520SKenneth D. Merry static int ctl_be_block_move_done(union ctl_io *io); 241130f4520SKenneth D. Merry static void ctl_be_block_biodone(struct bio *bio); 242130f4520SKenneth D. Merry static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 243130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 244130f4520SKenneth D. Merry static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 245130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 246ef8daf3fSAlexander Motin static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 247ef8daf3fSAlexander Motin struct ctl_be_block_io *beio); 24853c146deSAlexander Motin static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, 24953c146deSAlexander Motin const char *attrname); 250130f4520SKenneth D. Merry static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 251130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 252ee7f31c0SAlexander Motin static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 253ee7f31c0SAlexander Motin struct ctl_be_block_io *beio); 254130f4520SKenneth D. Merry static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 255130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 256c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, 257c3e7ba3eSAlexander Motin const char *attrname); 258ef8daf3fSAlexander Motin static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 259ef8daf3fSAlexander Motin union ctl_io *io); 260130f4520SKenneth D. Merry static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 261130f4520SKenneth D. Merry union ctl_io *io); 262130f4520SKenneth D. Merry static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 263130f4520SKenneth D. Merry union ctl_io *io); 264130f4520SKenneth D. Merry static void ctl_be_block_worker(void *context, int pending); 265130f4520SKenneth D. Merry static int ctl_be_block_submit(union ctl_io *io); 266130f4520SKenneth D. Merry static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 267130f4520SKenneth D. Merry int flag, struct thread *td); 268130f4520SKenneth D. Merry static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, 269130f4520SKenneth D. Merry struct ctl_lun_req *req); 270130f4520SKenneth D. Merry static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, 271130f4520SKenneth D. Merry struct ctl_lun_req *req); 272130f4520SKenneth D. Merry static int ctl_be_block_close(struct ctl_be_block_lun *be_lun); 273130f4520SKenneth D. Merry static int ctl_be_block_open(struct ctl_be_block_softc *softc, 274130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun, 275130f4520SKenneth D. Merry struct ctl_lun_req *req); 276130f4520SKenneth D. Merry static int ctl_be_block_create(struct ctl_be_block_softc *softc, 277130f4520SKenneth D. Merry struct ctl_lun_req *req); 278130f4520SKenneth D. Merry static int ctl_be_block_rm(struct ctl_be_block_softc *softc, 279130f4520SKenneth D. Merry struct ctl_lun_req *req); 28081177295SEdward Tomasz Napierala static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 28181177295SEdward Tomasz Napierala struct ctl_lun_req *req); 28281177295SEdward Tomasz Napierala static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 28381177295SEdward Tomasz Napierala struct ctl_lun_req *req); 28481177295SEdward Tomasz Napierala static int ctl_be_block_modify(struct ctl_be_block_softc *softc, 28581177295SEdward Tomasz Napierala struct ctl_lun_req *req); 286130f4520SKenneth D. Merry static void ctl_be_block_lun_shutdown(void *be_lun); 287130f4520SKenneth D. Merry static void ctl_be_block_lun_config_status(void *be_lun, 288130f4520SKenneth D. Merry ctl_lun_config_status status); 289130f4520SKenneth D. Merry static int ctl_be_block_config_write(union ctl_io *io); 290130f4520SKenneth D. Merry static int ctl_be_block_config_read(union ctl_io *io); 291130f4520SKenneth D. Merry static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); 292c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); 293130f4520SKenneth D. Merry int ctl_be_block_init(void); 294130f4520SKenneth D. Merry 295130f4520SKenneth D. Merry static struct ctl_backend_driver ctl_be_block_driver = 296130f4520SKenneth D. Merry { 2972a2443d8SKenneth D. Merry .name = "block", 2982a2443d8SKenneth D. Merry .flags = CTL_BE_FLAG_HAS_CONFIG, 2992a2443d8SKenneth D. Merry .init = ctl_be_block_init, 3002a2443d8SKenneth D. Merry .data_submit = ctl_be_block_submit, 3012a2443d8SKenneth D. Merry .data_move_done = ctl_be_block_move_done, 3022a2443d8SKenneth D. Merry .config_read = ctl_be_block_config_read, 3032a2443d8SKenneth D. Merry .config_write = ctl_be_block_config_write, 3042a2443d8SKenneth D. Merry .ioctl = ctl_be_block_ioctl, 305c3e7ba3eSAlexander Motin .lun_info = ctl_be_block_lun_info, 306c3e7ba3eSAlexander Motin .lun_attr = ctl_be_block_lun_attr 307130f4520SKenneth D. Merry }; 308130f4520SKenneth D. Merry 309130f4520SKenneth D. Merry MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); 310130f4520SKenneth D. Merry CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); 311130f4520SKenneth D. Merry 312a0e36aeeSEdward Tomasz Napierala static uma_zone_t beio_zone; 313a0e36aeeSEdward Tomasz Napierala 314130f4520SKenneth D. Merry static struct ctl_be_block_io * 315130f4520SKenneth D. Merry ctl_alloc_beio(struct ctl_be_block_softc *softc) 316130f4520SKenneth D. Merry { 317130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 318130f4520SKenneth D. Merry 319a0e36aeeSEdward Tomasz Napierala beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO); 320130f4520SKenneth D. Merry beio->softc = softc; 321130f4520SKenneth D. Merry return (beio); 322130f4520SKenneth D. Merry } 323130f4520SKenneth D. Merry 324130f4520SKenneth D. Merry static void 325130f4520SKenneth D. Merry ctl_free_beio(struct ctl_be_block_io *beio) 326130f4520SKenneth D. Merry { 327130f4520SKenneth D. Merry int duplicate_free; 328130f4520SKenneth D. Merry int i; 329130f4520SKenneth D. Merry 330130f4520SKenneth D. Merry duplicate_free = 0; 331130f4520SKenneth D. Merry 332130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 333130f4520SKenneth D. Merry if (beio->sg_segs[i].addr == NULL) 334130f4520SKenneth D. Merry duplicate_free++; 335130f4520SKenneth D. Merry 336130f4520SKenneth D. Merry uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr); 337130f4520SKenneth D. Merry beio->sg_segs[i].addr = NULL; 33811b569f7SAlexander Motin 33911b569f7SAlexander Motin /* For compare we had two equal S/G lists. */ 34011b569f7SAlexander Motin if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) { 34111b569f7SAlexander Motin uma_zfree(beio->lun->lun_zone, 34211b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr); 34311b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL; 34411b569f7SAlexander Motin } 345130f4520SKenneth D. Merry } 346130f4520SKenneth D. Merry 347130f4520SKenneth D. Merry if (duplicate_free > 0) { 348130f4520SKenneth D. Merry printf("%s: %d duplicate frees out of %d segments\n", __func__, 349130f4520SKenneth D. Merry duplicate_free, beio->num_segs); 350130f4520SKenneth D. Merry } 351a0e36aeeSEdward Tomasz Napierala 352a0e36aeeSEdward Tomasz Napierala uma_zfree(beio_zone, beio); 353130f4520SKenneth D. Merry } 354130f4520SKenneth D. Merry 355130f4520SKenneth D. Merry static void 356130f4520SKenneth D. Merry ctl_complete_beio(struct ctl_be_block_io *beio) 357130f4520SKenneth D. Merry { 35875c7a1d3SAlexander Motin union ctl_io *io = beio->io; 359130f4520SKenneth D. Merry 360ee7f31c0SAlexander Motin if (beio->beio_cont != NULL) { 361ee7f31c0SAlexander Motin beio->beio_cont(beio); 362ee7f31c0SAlexander Motin } else { 363130f4520SKenneth D. Merry ctl_free_beio(beio); 36411b569f7SAlexander Motin ctl_data_submit_done(io); 365130f4520SKenneth D. Merry } 366ee7f31c0SAlexander Motin } 367130f4520SKenneth D. Merry 368130f4520SKenneth D. Merry static int 369130f4520SKenneth D. Merry ctl_be_block_move_done(union ctl_io *io) 370130f4520SKenneth D. Merry { 371130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 372130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 37311b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 374130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 375130f4520SKenneth D. Merry struct bintime cur_bt; 376130f4520SKenneth D. Merry #endif 37711b569f7SAlexander Motin int i; 378130f4520SKenneth D. Merry 379e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 380130f4520SKenneth D. Merry be_lun = beio->lun; 381130f4520SKenneth D. Merry 382130f4520SKenneth D. Merry DPRINTF("entered\n"); 383130f4520SKenneth D. Merry 384130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 385130f4520SKenneth D. Merry getbintime(&cur_bt); 386130f4520SKenneth D. Merry bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); 387130f4520SKenneth D. Merry bintime_add(&io->io_hdr.dma_bt, &cur_bt); 388130f4520SKenneth D. Merry io->io_hdr.num_dmas++; 389130f4520SKenneth D. Merry #endif 39011b569f7SAlexander Motin io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; 391130f4520SKenneth D. Merry 392130f4520SKenneth D. Merry /* 393130f4520SKenneth D. Merry * We set status at this point for read commands, and write 394130f4520SKenneth D. Merry * commands with errors. 395130f4520SKenneth D. Merry */ 396f7241cceSAlexander Motin if (io->io_hdr.flags & CTL_FLAG_ABORT) { 397f7241cceSAlexander Motin ; 398f7241cceSAlexander Motin } else if ((io->io_hdr.port_status == 0) && 39911b569f7SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { 40011b569f7SAlexander Motin lbalen = ARGS(beio->io); 40111b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_READ) { 402130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 40311b569f7SAlexander Motin } else if (lbalen->flags & CTL_LLF_COMPARE) { 40411b569f7SAlexander Motin /* We have two data blocks ready for comparison. */ 40511b569f7SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 40611b569f7SAlexander Motin if (memcmp(beio->sg_segs[i].addr, 40711b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, 40811b569f7SAlexander Motin beio->sg_segs[i].len) != 0) 40911b569f7SAlexander Motin break; 41011b569f7SAlexander Motin } 41111b569f7SAlexander Motin if (i < beio->num_segs) 41211b569f7SAlexander Motin ctl_set_sense(&io->scsiio, 41311b569f7SAlexander Motin /*current_error*/ 1, 41411b569f7SAlexander Motin /*sense_key*/ SSD_KEY_MISCOMPARE, 41511b569f7SAlexander Motin /*asc*/ 0x1D, 41611b569f7SAlexander Motin /*ascq*/ 0x00, 41711b569f7SAlexander Motin SSD_ELEM_NONE); 41811b569f7SAlexander Motin else 41911b569f7SAlexander Motin ctl_set_success(&io->scsiio); 42011b569f7SAlexander Motin } 421f7241cceSAlexander Motin } else if ((io->io_hdr.port_status != 0) && 422f7241cceSAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 423f7241cceSAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 424130f4520SKenneth D. Merry /* 425130f4520SKenneth D. Merry * For hardware error sense keys, the sense key 426130f4520SKenneth D. Merry * specific value is defined to be a retry count, 427130f4520SKenneth D. Merry * but we use it to pass back an internal FETD 428130f4520SKenneth D. Merry * error code. XXX KDM Hopefully the FETD is only 429130f4520SKenneth D. Merry * using 16 bits for an error code, since that's 430130f4520SKenneth D. Merry * all the space we have in the sks field. 431130f4520SKenneth D. Merry */ 432130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 433130f4520SKenneth D. Merry /*sks_valid*/ 1, 434130f4520SKenneth D. Merry /*retry_count*/ 435130f4520SKenneth D. Merry io->io_hdr.port_status); 436130f4520SKenneth D. Merry } 437130f4520SKenneth D. Merry 438130f4520SKenneth D. Merry /* 439130f4520SKenneth D. Merry * If this is a read, or a write with errors, it is done. 440130f4520SKenneth D. Merry */ 441130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_READ) 442130f4520SKenneth D. Merry || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) 443130f4520SKenneth D. Merry || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { 444130f4520SKenneth D. Merry ctl_complete_beio(beio); 445130f4520SKenneth D. Merry return (0); 446130f4520SKenneth D. Merry } 447130f4520SKenneth D. Merry 448130f4520SKenneth D. Merry /* 449130f4520SKenneth D. Merry * At this point, we have a write and the DMA completed 450130f4520SKenneth D. Merry * successfully. We now have to queue it to the task queue to 451130f4520SKenneth D. Merry * execute the backend I/O. That is because we do blocking 452130f4520SKenneth D. Merry * memory allocations, and in the file backing case, blocking I/O. 453130f4520SKenneth D. Merry * This move done routine is generally called in the SIM's 454130f4520SKenneth D. Merry * interrupt context, and therefore we cannot block. 455130f4520SKenneth D. Merry */ 45675c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 457130f4520SKenneth D. Merry /* 458130f4520SKenneth D. Merry * XXX KDM make sure that links is okay to use at this point. 459130f4520SKenneth D. Merry * Otherwise, we either need to add another field to ctl_io_hdr, 460130f4520SKenneth D. Merry * or deal with resource allocation here. 461130f4520SKenneth D. Merry */ 462130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); 46375c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 464130f4520SKenneth D. Merry 465130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 466130f4520SKenneth D. Merry 467130f4520SKenneth D. Merry return (0); 468130f4520SKenneth D. Merry } 469130f4520SKenneth D. Merry 470130f4520SKenneth D. Merry static void 471130f4520SKenneth D. Merry ctl_be_block_biodone(struct bio *bio) 472130f4520SKenneth D. Merry { 473130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 474130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 475130f4520SKenneth D. Merry union ctl_io *io; 476e0c2f975SAlexander Motin int error; 477130f4520SKenneth D. Merry 478130f4520SKenneth D. Merry beio = bio->bio_caller1; 479130f4520SKenneth D. Merry be_lun = beio->lun; 480130f4520SKenneth D. Merry io = beio->io; 481130f4520SKenneth D. Merry 482130f4520SKenneth D. Merry DPRINTF("entered\n"); 483130f4520SKenneth D. Merry 484e0c2f975SAlexander Motin error = bio->bio_error; 48575c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 486e0c2f975SAlexander Motin if (error != 0) 487130f4520SKenneth D. Merry beio->num_errors++; 488130f4520SKenneth D. Merry 489130f4520SKenneth D. Merry beio->num_bios_done++; 490130f4520SKenneth D. Merry 491130f4520SKenneth D. Merry /* 492130f4520SKenneth D. Merry * XXX KDM will this cause WITNESS to complain? Holding a lock 493130f4520SKenneth D. Merry * during the free might cause it to complain. 494130f4520SKenneth D. Merry */ 495130f4520SKenneth D. Merry g_destroy_bio(bio); 496130f4520SKenneth D. Merry 497130f4520SKenneth D. Merry /* 498130f4520SKenneth D. Merry * If the send complete bit isn't set, or we aren't the last I/O to 499130f4520SKenneth D. Merry * complete, then we're done. 500130f4520SKenneth D. Merry */ 501130f4520SKenneth D. Merry if ((beio->send_complete == 0) 502130f4520SKenneth D. Merry || (beio->num_bios_done < beio->num_bios_sent)) { 50375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 504130f4520SKenneth D. Merry return; 505130f4520SKenneth D. Merry } 506130f4520SKenneth D. Merry 507130f4520SKenneth D. Merry /* 508130f4520SKenneth D. Merry * At this point, we've verified that we are the last I/O to 509130f4520SKenneth D. Merry * complete, so it's safe to drop the lock. 510130f4520SKenneth D. Merry */ 51175c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 51275c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 51375c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 51475c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 515130f4520SKenneth D. Merry 516130f4520SKenneth D. Merry /* 517130f4520SKenneth D. Merry * If there are any errors from the backing device, we fail the 518130f4520SKenneth D. Merry * entire I/O with a medium error. 519130f4520SKenneth D. Merry */ 520130f4520SKenneth D. Merry if (beio->num_errors > 0) { 521e0c2f975SAlexander Motin if (error == EOPNOTSUPP) { 522e0c2f975SAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 5230631de4aSAlexander Motin } else if (error == ENOSPC || error == EDQUOT) { 5244fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 525e0c2f975SAlexander Motin } else if (beio->bio_cmd == BIO_FLUSH) { 526130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 527130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 528130f4520SKenneth D. Merry /*sks_valid*/ 1, 529130f4520SKenneth D. Merry /*retry_count*/ 0xbad2); 530130f4520SKenneth D. Merry } else 531130f4520SKenneth D. Merry ctl_set_medium_error(&io->scsiio); 532130f4520SKenneth D. Merry ctl_complete_beio(beio); 533130f4520SKenneth D. Merry return; 534130f4520SKenneth D. Merry } 535130f4520SKenneth D. Merry 536130f4520SKenneth D. Merry /* 53711b569f7SAlexander Motin * If this is a write, a flush, a delete or verify, we're all done. 538130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 539130f4520SKenneth D. Merry */ 540130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_WRITE) 541ee7f31c0SAlexander Motin || (beio->bio_cmd == BIO_FLUSH) 54211b569f7SAlexander Motin || (beio->bio_cmd == BIO_DELETE) 54311b569f7SAlexander Motin || (ARGS(io)->flags & CTL_LLF_VERIFY)) { 544130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 545130f4520SKenneth D. Merry ctl_complete_beio(beio); 546130f4520SKenneth D. Merry } else { 547f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 548f7241cceSAlexander Motin beio->beio_cont == NULL) 549f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 550130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 551130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 552130f4520SKenneth D. Merry #endif 553130f4520SKenneth D. Merry ctl_datamove(io); 554130f4520SKenneth D. Merry } 555130f4520SKenneth D. Merry } 556130f4520SKenneth D. Merry 557130f4520SKenneth D. Merry static void 558130f4520SKenneth D. Merry ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 559130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 560130f4520SKenneth D. Merry { 56175c7a1d3SAlexander Motin union ctl_io *io = beio->io; 562130f4520SKenneth D. Merry struct mount *mountpoint; 5635050aa86SKonstantin Belousov int error, lock_flags; 564130f4520SKenneth D. Merry 565130f4520SKenneth D. Merry DPRINTF("entered\n"); 566130f4520SKenneth D. Merry 56775c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 56875c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 56975c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 57075c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 571130f4520SKenneth D. Merry 572130f4520SKenneth D. Merry (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 573130f4520SKenneth D. Merry 574130f4520SKenneth D. Merry if (MNT_SHARED_WRITES(mountpoint) 575130f4520SKenneth D. Merry || ((mountpoint == NULL) 576130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 577130f4520SKenneth D. Merry lock_flags = LK_SHARED; 578130f4520SKenneth D. Merry else 579130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 580130f4520SKenneth D. Merry 581130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 582130f4520SKenneth D. Merry 5837d0d4342SAlexander Motin error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT, 5847d0d4342SAlexander Motin curthread); 585130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 586130f4520SKenneth D. Merry 587130f4520SKenneth D. Merry vn_finished_write(mountpoint); 588130f4520SKenneth D. Merry 58975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 59075c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 59175c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 59275c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 59375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 59475c7a1d3SAlexander Motin 595130f4520SKenneth D. Merry if (error == 0) 596130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 597130f4520SKenneth D. Merry else { 598130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 599130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 600130f4520SKenneth D. Merry /*sks_valid*/ 1, 601130f4520SKenneth D. Merry /*retry_count*/ 0xbad1); 602130f4520SKenneth D. Merry } 603130f4520SKenneth D. Merry 604130f4520SKenneth D. Merry ctl_complete_beio(beio); 605130f4520SKenneth D. Merry } 606130f4520SKenneth D. Merry 607d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t"); 608d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t"); 609d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t"); 610d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t"); 611130f4520SKenneth D. Merry 612130f4520SKenneth D. Merry static void 613130f4520SKenneth D. Merry ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 614130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 615130f4520SKenneth D. Merry { 616130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 617130f4520SKenneth D. Merry union ctl_io *io; 618130f4520SKenneth D. Merry struct uio xuio; 619130f4520SKenneth D. Merry struct iovec *xiovec; 6205050aa86SKonstantin Belousov int flags; 621130f4520SKenneth D. Merry int error, i; 622130f4520SKenneth D. Merry 623130f4520SKenneth D. Merry DPRINTF("entered\n"); 624130f4520SKenneth D. Merry 625130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 626130f4520SKenneth D. Merry io = beio->io; 62755551d05SAlexander Motin flags = 0; 62855551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 62955551d05SAlexander Motin flags |= IO_DIRECT; 63055551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 63155551d05SAlexander Motin flags |= IO_SYNC; 632130f4520SKenneth D. Merry 63311b569f7SAlexander Motin bzero(&xuio, sizeof(xuio)); 634130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 635130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 63611b569f7SAlexander Motin xuio.uio_rw = UIO_READ; 637130f4520SKenneth D. Merry } else { 638130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 639130f4520SKenneth D. Merry xuio.uio_rw = UIO_WRITE; 64011b569f7SAlexander Motin } 641130f4520SKenneth D. Merry xuio.uio_offset = beio->io_offset; 642130f4520SKenneth D. Merry xuio.uio_resid = beio->io_len; 643130f4520SKenneth D. Merry xuio.uio_segflg = UIO_SYSSPACE; 644130f4520SKenneth D. Merry xuio.uio_iov = beio->xiovecs; 645130f4520SKenneth D. Merry xuio.uio_iovcnt = beio->num_segs; 646130f4520SKenneth D. Merry xuio.uio_td = curthread; 647130f4520SKenneth D. Merry 648130f4520SKenneth D. Merry for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 649130f4520SKenneth D. Merry xiovec->iov_base = beio->sg_segs[i].addr; 650130f4520SKenneth D. Merry xiovec->iov_len = beio->sg_segs[i].len; 651130f4520SKenneth D. Merry } 652130f4520SKenneth D. Merry 65375c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 65475c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 65575c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 65675c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 65775c7a1d3SAlexander Motin 658130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 659130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 660130f4520SKenneth D. Merry 661130f4520SKenneth D. Merry /* 662130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for reads. If the 663130f4520SKenneth D. Merry * DIRECTIO option is configured into the kernel, it calls 664130f4520SKenneth D. Merry * ffs_rawread(). But that only works for single-segment 665130f4520SKenneth D. Merry * uios with user space addresses. In our case, with a 666130f4520SKenneth D. Merry * kernel uio, it still reads into the buffer cache, but it 667130f4520SKenneth D. Merry * will just try to release the buffer from the cache later 668130f4520SKenneth D. Merry * on in ffs_read(). 669130f4520SKenneth D. Merry * 670130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for reads. 671130f4520SKenneth D. Merry * 672130f4520SKenneth D. Merry * UFS does not pay attention to IO_SYNC for reads. 673130f4520SKenneth D. Merry * 674130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (which translates into the 675130f4520SKenneth D. Merry * Solaris define FRSYNC for zfs_read()) for reads. It 676130f4520SKenneth D. Merry * attempts to sync the file before reading. 677130f4520SKenneth D. Merry */ 67855551d05SAlexander Motin error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); 679130f4520SKenneth D. Merry 680130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 68111b569f7SAlexander Motin SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 682130f4520SKenneth D. Merry } else { 683130f4520SKenneth D. Merry struct mount *mountpoint; 684130f4520SKenneth D. Merry int lock_flags; 685130f4520SKenneth D. Merry 686130f4520SKenneth D. Merry (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 687130f4520SKenneth D. Merry 688130f4520SKenneth D. Merry if (MNT_SHARED_WRITES(mountpoint) 689130f4520SKenneth D. Merry || ((mountpoint == NULL) 690130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 691130f4520SKenneth D. Merry lock_flags = LK_SHARED; 692130f4520SKenneth D. Merry else 693130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 694130f4520SKenneth D. Merry 695130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 696130f4520SKenneth D. Merry 697130f4520SKenneth D. Merry /* 698130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for writes. The write 699130f4520SKenneth D. Merry * is done asynchronously. (Normally the write would just 700130f4520SKenneth D. Merry * get put into cache. 701130f4520SKenneth D. Merry * 702130f4520SKenneth D. Merry * UFS pays attention to IO_SYNC for writes. It will 703130f4520SKenneth D. Merry * attempt to write the buffer out synchronously if that 704130f4520SKenneth D. Merry * flag is set. 705130f4520SKenneth D. Merry * 706130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for writes. 707130f4520SKenneth D. Merry * 708130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) 709130f4520SKenneth D. Merry * for writes. It will flush the transaction from the 710130f4520SKenneth D. Merry * cache before returning. 711130f4520SKenneth D. Merry */ 71255551d05SAlexander Motin error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); 713130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 714130f4520SKenneth D. Merry 715130f4520SKenneth D. Merry vn_finished_write(mountpoint); 71611b569f7SAlexander Motin SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 717130f4520SKenneth D. Merry } 718130f4520SKenneth D. Merry 71975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 72075c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 72175c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 72275c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 72375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 72475c7a1d3SAlexander Motin 725130f4520SKenneth D. Merry /* 726130f4520SKenneth D. Merry * If we got an error, set the sense data to "MEDIUM ERROR" and 727130f4520SKenneth D. Merry * return the I/O to the user. 728130f4520SKenneth D. Merry */ 729130f4520SKenneth D. Merry if (error != 0) { 730130f4520SKenneth D. Merry char path_str[32]; 731130f4520SKenneth D. Merry 732130f4520SKenneth D. Merry ctl_scsi_path_string(io, path_str, sizeof(path_str)); 733130f4520SKenneth D. Merry printf("%s%s command returned errno %d\n", path_str, 734130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error); 7350631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 7364fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 7374fc18ff9SAlexander Motin } else 738130f4520SKenneth D. Merry ctl_set_medium_error(&io->scsiio); 739130f4520SKenneth D. Merry ctl_complete_beio(beio); 740130f4520SKenneth D. Merry return; 741130f4520SKenneth D. Merry } 742130f4520SKenneth D. Merry 743130f4520SKenneth D. Merry /* 744696297adSAlexander Motin * If this is a write or a verify, we're all done. 745130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 746130f4520SKenneth D. Merry */ 747696297adSAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 748696297adSAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 749130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 750130f4520SKenneth D. Merry ctl_complete_beio(beio); 751130f4520SKenneth D. Merry } else { 752f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 753f7241cceSAlexander Motin beio->beio_cont == NULL) 754f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 755130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 756130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 757130f4520SKenneth D. Merry #endif 758130f4520SKenneth D. Merry ctl_datamove(io); 759130f4520SKenneth D. Merry } 760130f4520SKenneth D. Merry } 761130f4520SKenneth D. Merry 762130f4520SKenneth D. Merry static void 763ef8daf3fSAlexander Motin ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 764ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 765ef8daf3fSAlexander Motin { 766ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 767ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 768ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 769ef8daf3fSAlexander Motin off_t roff, off; 770ef8daf3fSAlexander Motin int error, status; 771ef8daf3fSAlexander Motin 772ef8daf3fSAlexander Motin DPRINTF("entered\n"); 773ef8daf3fSAlexander Motin 774*a15bbf15SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->blocksize; 775ef8daf3fSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 776ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, 777ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 778ef8daf3fSAlexander Motin if (error == 0 && off > roff) 779ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 780ef8daf3fSAlexander Motin else { 781ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, 782ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 783ef8daf3fSAlexander Motin if (error == 0 && off > roff) 784ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 785ef8daf3fSAlexander Motin else { 786ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 787ef8daf3fSAlexander Motin off = be_lun->size_bytes; 788ef8daf3fSAlexander Motin } 789ef8daf3fSAlexander Motin } 790ef8daf3fSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 791ef8daf3fSAlexander Motin 792ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 793ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 794*a15bbf15SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->blocksize - lbalen->lba), 795ef8daf3fSAlexander Motin data->descr[0].length); 796ef8daf3fSAlexander Motin data->descr[0].status = status; 797ef8daf3fSAlexander Motin 798ef8daf3fSAlexander Motin ctl_complete_beio(beio); 799ef8daf3fSAlexander Motin } 800ef8daf3fSAlexander Motin 80153c146deSAlexander Motin static uint64_t 80253c146deSAlexander Motin ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname) 80353c146deSAlexander Motin { 80453c146deSAlexander Motin struct vattr vattr; 80553c146deSAlexander Motin struct statfs statfs; 806b9b4269cSAlexander Motin uint64_t val; 80753c146deSAlexander Motin int error; 80853c146deSAlexander Motin 809b9b4269cSAlexander Motin val = UINT64_MAX; 81053c146deSAlexander Motin if (be_lun->vn == NULL) 811b9b4269cSAlexander Motin return (val); 812b9b4269cSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 81353c146deSAlexander Motin if (strcmp(attrname, "blocksused") == 0) { 81453c146deSAlexander Motin error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 815b9b4269cSAlexander Motin if (error == 0) 816*a15bbf15SAlexander Motin val = vattr.va_bytes / be_lun->blocksize; 81753c146deSAlexander Motin } 818b9b4269cSAlexander Motin if (strcmp(attrname, "blocksavail") == 0 && 819b9b4269cSAlexander Motin (be_lun->vn->v_iflag & VI_DOOMED) == 0) { 82053c146deSAlexander Motin error = VFS_STATFS(be_lun->vn->v_mount, &statfs); 821b9b4269cSAlexander Motin if (error == 0) 822*a15bbf15SAlexander Motin val = statfs.f_bavail * statfs.f_bsize / 823*a15bbf15SAlexander Motin be_lun->blocksize; 82453c146deSAlexander Motin } 825b9b4269cSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 826b9b4269cSAlexander Motin return (val); 82753c146deSAlexander Motin } 82853c146deSAlexander Motin 829ef8daf3fSAlexander Motin static void 83067f586a8SAlexander Motin ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, 83167f586a8SAlexander Motin struct ctl_be_block_io *beio) 83267f586a8SAlexander Motin { 83367f586a8SAlexander Motin struct ctl_be_block_devdata *dev_data; 83467f586a8SAlexander Motin union ctl_io *io; 83567f586a8SAlexander Motin struct uio xuio; 83667f586a8SAlexander Motin struct iovec *xiovec; 83767f586a8SAlexander Motin int flags; 83867f586a8SAlexander Motin int error, i; 83967f586a8SAlexander Motin 84067f586a8SAlexander Motin DPRINTF("entered\n"); 84167f586a8SAlexander Motin 84267f586a8SAlexander Motin dev_data = &be_lun->backend.dev; 84367f586a8SAlexander Motin io = beio->io; 84455551d05SAlexander Motin flags = 0; 84555551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 84655551d05SAlexander Motin flags |= IO_DIRECT; 84755551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 84855551d05SAlexander Motin flags |= IO_SYNC; 84967f586a8SAlexander Motin 85067f586a8SAlexander Motin bzero(&xuio, sizeof(xuio)); 85167f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 85267f586a8SAlexander Motin SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 85367f586a8SAlexander Motin xuio.uio_rw = UIO_READ; 85467f586a8SAlexander Motin } else { 85567f586a8SAlexander Motin SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 85667f586a8SAlexander Motin xuio.uio_rw = UIO_WRITE; 85767f586a8SAlexander Motin } 85867f586a8SAlexander Motin xuio.uio_offset = beio->io_offset; 85967f586a8SAlexander Motin xuio.uio_resid = beio->io_len; 86067f586a8SAlexander Motin xuio.uio_segflg = UIO_SYSSPACE; 86167f586a8SAlexander Motin xuio.uio_iov = beio->xiovecs; 86267f586a8SAlexander Motin xuio.uio_iovcnt = beio->num_segs; 86367f586a8SAlexander Motin xuio.uio_td = curthread; 86467f586a8SAlexander Motin 86567f586a8SAlexander Motin for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 86667f586a8SAlexander Motin xiovec->iov_base = beio->sg_segs[i].addr; 86767f586a8SAlexander Motin xiovec->iov_len = beio->sg_segs[i].len; 86867f586a8SAlexander Motin } 86967f586a8SAlexander Motin 87067f586a8SAlexander Motin binuptime(&beio->ds_t0); 87167f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 87267f586a8SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 87367f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 87467f586a8SAlexander Motin 87567f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 87655551d05SAlexander Motin error = (*dev_data->csw->d_read)(dev_data->cdev, &xuio, flags); 87767f586a8SAlexander Motin SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 87867f586a8SAlexander Motin } else { 87955551d05SAlexander Motin error = (*dev_data->csw->d_write)(dev_data->cdev, &xuio, flags); 88067f586a8SAlexander Motin SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 88167f586a8SAlexander Motin } 88267f586a8SAlexander Motin 88367f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 88467f586a8SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 88567f586a8SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 88667f586a8SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 88767f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 88867f586a8SAlexander Motin 88967f586a8SAlexander Motin /* 89067f586a8SAlexander Motin * If we got an error, set the sense data to "MEDIUM ERROR" and 89167f586a8SAlexander Motin * return the I/O to the user. 89267f586a8SAlexander Motin */ 89367f586a8SAlexander Motin if (error != 0) { 8940631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 8954fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 8964fc18ff9SAlexander Motin } else 89767f586a8SAlexander Motin ctl_set_medium_error(&io->scsiio); 89867f586a8SAlexander Motin ctl_complete_beio(beio); 89967f586a8SAlexander Motin return; 90067f586a8SAlexander Motin } 90167f586a8SAlexander Motin 90267f586a8SAlexander Motin /* 90367f586a8SAlexander Motin * If this is a write or a verify, we're all done. 90467f586a8SAlexander Motin * If this is a read, we can now send the data to the user. 90567f586a8SAlexander Motin */ 90667f586a8SAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 90767f586a8SAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 90867f586a8SAlexander Motin ctl_set_success(&io->scsiio); 90967f586a8SAlexander Motin ctl_complete_beio(beio); 91067f586a8SAlexander Motin } else { 911f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 912f7241cceSAlexander Motin beio->beio_cont == NULL) 913f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 91467f586a8SAlexander Motin #ifdef CTL_TIME_IO 91567f586a8SAlexander Motin getbintime(&io->io_hdr.dma_start_bt); 91667f586a8SAlexander Motin #endif 91767f586a8SAlexander Motin ctl_datamove(io); 91867f586a8SAlexander Motin } 91967f586a8SAlexander Motin } 92067f586a8SAlexander Motin 92167f586a8SAlexander Motin static void 922ef8daf3fSAlexander Motin ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun, 923ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 924ef8daf3fSAlexander Motin { 925ef8daf3fSAlexander Motin struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 926ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 927ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 928ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 929ef8daf3fSAlexander Motin off_t roff, off; 930ef8daf3fSAlexander Motin int error, status; 931ef8daf3fSAlexander Motin 932ef8daf3fSAlexander Motin DPRINTF("entered\n"); 933ef8daf3fSAlexander Motin 934*a15bbf15SAlexander Motin off = roff = ((off_t)lbalen->lba) * be_lun->blocksize; 935ef8daf3fSAlexander Motin error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKHOLE, 936ef8daf3fSAlexander Motin (caddr_t)&off, FREAD, curthread); 937ef8daf3fSAlexander Motin if (error == 0 && off > roff) 938ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 939ef8daf3fSAlexander Motin else { 940ef8daf3fSAlexander Motin error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKDATA, 941ef8daf3fSAlexander Motin (caddr_t)&off, FREAD, curthread); 942ef8daf3fSAlexander Motin if (error == 0 && off > roff) 943ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 944ef8daf3fSAlexander Motin else { 945ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 946ef8daf3fSAlexander Motin off = be_lun->size_bytes; 947ef8daf3fSAlexander Motin } 948ef8daf3fSAlexander Motin } 949ef8daf3fSAlexander Motin 950ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 951ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 952*a15bbf15SAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->blocksize - lbalen->lba), 953ef8daf3fSAlexander Motin data->descr[0].length); 954ef8daf3fSAlexander Motin data->descr[0].status = status; 955ef8daf3fSAlexander Motin 956ef8daf3fSAlexander Motin ctl_complete_beio(beio); 957ef8daf3fSAlexander Motin } 958ef8daf3fSAlexander Motin 959ef8daf3fSAlexander Motin static void 960130f4520SKenneth D. Merry ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 961130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 962130f4520SKenneth D. Merry { 963130f4520SKenneth D. Merry struct bio *bio; 964130f4520SKenneth D. Merry union ctl_io *io; 965130f4520SKenneth D. Merry struct ctl_be_block_devdata *dev_data; 966130f4520SKenneth D. Merry 967130f4520SKenneth D. Merry dev_data = &be_lun->backend.dev; 968130f4520SKenneth D. Merry io = beio->io; 969130f4520SKenneth D. Merry 970130f4520SKenneth D. Merry DPRINTF("entered\n"); 971130f4520SKenneth D. Merry 972130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 973130f4520SKenneth D. Merry bio = g_alloc_bio(); 974130f4520SKenneth D. Merry 975130f4520SKenneth D. Merry bio->bio_cmd = BIO_FLUSH; 976130f4520SKenneth D. Merry bio->bio_dev = dev_data->cdev; 977130f4520SKenneth D. Merry bio->bio_offset = 0; 978130f4520SKenneth D. Merry bio->bio_data = 0; 979130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 980130f4520SKenneth D. Merry bio->bio_caller1 = beio; 981130f4520SKenneth D. Merry bio->bio_pblkno = 0; 982130f4520SKenneth D. Merry 983130f4520SKenneth D. Merry /* 984130f4520SKenneth D. Merry * We don't need to acquire the LUN lock here, because we are only 985130f4520SKenneth D. Merry * sending one bio, and so there is no other context to synchronize 986130f4520SKenneth D. Merry * with. 987130f4520SKenneth D. Merry */ 988130f4520SKenneth D. Merry beio->num_bios_sent = 1; 989130f4520SKenneth D. Merry beio->send_complete = 1; 990130f4520SKenneth D. Merry 991130f4520SKenneth D. Merry binuptime(&beio->ds_t0); 99275c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 993130f4520SKenneth D. Merry devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 99475c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 995130f4520SKenneth D. Merry 996130f4520SKenneth D. Merry (*dev_data->csw->d_strategy)(bio); 997130f4520SKenneth D. Merry } 998130f4520SKenneth D. Merry 999130f4520SKenneth D. Merry static void 1000ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun, 1001ee7f31c0SAlexander Motin struct ctl_be_block_io *beio, 1002ee7f31c0SAlexander Motin uint64_t off, uint64_t len, int last) 1003ee7f31c0SAlexander Motin { 1004ee7f31c0SAlexander Motin struct bio *bio; 1005ee7f31c0SAlexander Motin struct ctl_be_block_devdata *dev_data; 10068f5a226aSAlexander Motin uint64_t maxlen; 1007ee7f31c0SAlexander Motin 1008ee7f31c0SAlexander Motin dev_data = &be_lun->backend.dev; 10098f5a226aSAlexander Motin maxlen = LONG_MAX - (LONG_MAX % be_lun->blocksize); 1010ee7f31c0SAlexander Motin while (len > 0) { 1011ee7f31c0SAlexander Motin bio = g_alloc_bio(); 1012ee7f31c0SAlexander Motin bio->bio_cmd = BIO_DELETE; 1013ee7f31c0SAlexander Motin bio->bio_dev = dev_data->cdev; 1014ee7f31c0SAlexander Motin bio->bio_offset = off; 10158f5a226aSAlexander Motin bio->bio_length = MIN(len, maxlen); 1016ee7f31c0SAlexander Motin bio->bio_data = 0; 1017ee7f31c0SAlexander Motin bio->bio_done = ctl_be_block_biodone; 1018ee7f31c0SAlexander Motin bio->bio_caller1 = beio; 10198f5a226aSAlexander Motin bio->bio_pblkno = off / be_lun->blocksize; 1020ee7f31c0SAlexander Motin 1021ee7f31c0SAlexander Motin off += bio->bio_length; 1022ee7f31c0SAlexander Motin len -= bio->bio_length; 1023ee7f31c0SAlexander Motin 102475c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1025ee7f31c0SAlexander Motin beio->num_bios_sent++; 1026ee7f31c0SAlexander Motin if (last && len == 0) 1027ee7f31c0SAlexander Motin beio->send_complete = 1; 102875c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1029ee7f31c0SAlexander Motin 1030ee7f31c0SAlexander Motin (*dev_data->csw->d_strategy)(bio); 1031ee7f31c0SAlexander Motin } 1032ee7f31c0SAlexander Motin } 1033ee7f31c0SAlexander Motin 1034ee7f31c0SAlexander Motin static void 1035ee7f31c0SAlexander Motin ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 1036ee7f31c0SAlexander Motin struct ctl_be_block_io *beio) 1037ee7f31c0SAlexander Motin { 1038ee7f31c0SAlexander Motin union ctl_io *io; 1039ee7f31c0SAlexander Motin struct ctl_be_block_devdata *dev_data; 104066df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1041ee7f31c0SAlexander Motin struct scsi_unmap_desc *buf, *end; 1042ee7f31c0SAlexander Motin uint64_t len; 1043ee7f31c0SAlexander Motin 1044ee7f31c0SAlexander Motin dev_data = &be_lun->backend.dev; 1045ee7f31c0SAlexander Motin io = beio->io; 1046ee7f31c0SAlexander Motin 1047ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1048ee7f31c0SAlexander Motin 1049ee7f31c0SAlexander Motin binuptime(&beio->ds_t0); 105075c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1051ee7f31c0SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 105275c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1053ee7f31c0SAlexander Motin 1054ee7f31c0SAlexander Motin if (beio->io_offset == -1) { 1055ee7f31c0SAlexander Motin beio->io_len = 0; 105666df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 105766df9136SAlexander Motin buf = (struct scsi_unmap_desc *)ptrlen->ptr; 105866df9136SAlexander Motin end = buf + ptrlen->len / sizeof(*buf); 1059ee7f31c0SAlexander Motin for (; buf < end; buf++) { 1060ee7f31c0SAlexander Motin len = (uint64_t)scsi_4btoul(buf->length) * 1061ee7f31c0SAlexander Motin be_lun->blocksize; 1062ee7f31c0SAlexander Motin beio->io_len += len; 1063ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1064ee7f31c0SAlexander Motin scsi_8btou64(buf->lba) * be_lun->blocksize, len, 10657e0be022SAlexander Motin (end - buf < 2) ? TRUE : FALSE); 1066ee7f31c0SAlexander Motin } 1067ee7f31c0SAlexander Motin } else 1068ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1069ee7f31c0SAlexander Motin beio->io_offset, beio->io_len, TRUE); 1070ee7f31c0SAlexander Motin } 1071ee7f31c0SAlexander Motin 1072ee7f31c0SAlexander Motin static void 1073130f4520SKenneth D. Merry ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 1074130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1075130f4520SKenneth D. Merry { 107675c7a1d3SAlexander Motin TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue); 1077130f4520SKenneth D. Merry int i; 1078130f4520SKenneth D. Merry struct bio *bio; 1079130f4520SKenneth D. Merry struct ctl_be_block_devdata *dev_data; 1080130f4520SKenneth D. Merry off_t cur_offset; 1081130f4520SKenneth D. Merry int max_iosize; 1082130f4520SKenneth D. Merry 1083130f4520SKenneth D. Merry DPRINTF("entered\n"); 1084130f4520SKenneth D. Merry 1085130f4520SKenneth D. Merry dev_data = &be_lun->backend.dev; 1086130f4520SKenneth D. Merry 1087130f4520SKenneth D. Merry /* 1088130f4520SKenneth D. Merry * We have to limit our I/O size to the maximum supported by the 1089130f4520SKenneth D. Merry * backend device. Hopefully it is MAXPHYS. If the driver doesn't 1090130f4520SKenneth D. Merry * set it properly, use DFLTPHYS. 1091130f4520SKenneth D. Merry */ 1092130f4520SKenneth D. Merry max_iosize = dev_data->cdev->si_iosize_max; 1093130f4520SKenneth D. Merry if (max_iosize < PAGE_SIZE) 1094130f4520SKenneth D. Merry max_iosize = DFLTPHYS; 1095130f4520SKenneth D. Merry 1096130f4520SKenneth D. Merry cur_offset = beio->io_offset; 1097130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 1098130f4520SKenneth D. Merry size_t cur_size; 1099130f4520SKenneth D. Merry uint8_t *cur_ptr; 1100130f4520SKenneth D. Merry 1101130f4520SKenneth D. Merry cur_size = beio->sg_segs[i].len; 1102130f4520SKenneth D. Merry cur_ptr = beio->sg_segs[i].addr; 1103130f4520SKenneth D. Merry 1104130f4520SKenneth D. Merry while (cur_size > 0) { 1105130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1106130f4520SKenneth D. Merry bio = g_alloc_bio(); 1107130f4520SKenneth D. Merry 1108130f4520SKenneth D. Merry KASSERT(bio != NULL, ("g_alloc_bio() failed!\n")); 1109130f4520SKenneth D. Merry 1110130f4520SKenneth D. Merry bio->bio_cmd = beio->bio_cmd; 1111130f4520SKenneth D. Merry bio->bio_dev = dev_data->cdev; 1112130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1113130f4520SKenneth D. Merry bio->bio_length = min(cur_size, max_iosize); 1114130f4520SKenneth D. Merry bio->bio_offset = cur_offset; 1115130f4520SKenneth D. Merry bio->bio_data = cur_ptr; 1116130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 1117130f4520SKenneth D. Merry bio->bio_pblkno = cur_offset / be_lun->blocksize; 1118130f4520SKenneth D. Merry 1119130f4520SKenneth D. Merry cur_offset += bio->bio_length; 1120130f4520SKenneth D. Merry cur_ptr += bio->bio_length; 1121130f4520SKenneth D. Merry cur_size -= bio->bio_length; 1122130f4520SKenneth D. Merry 112375c7a1d3SAlexander Motin TAILQ_INSERT_TAIL(&queue, bio, bio_queue); 1124130f4520SKenneth D. Merry beio->num_bios_sent++; 1125130f4520SKenneth D. Merry } 1126130f4520SKenneth D. Merry } 112775c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 112875c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 112975c7a1d3SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 113075c7a1d3SAlexander Motin beio->send_complete = 1; 113175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 113275c7a1d3SAlexander Motin 113375c7a1d3SAlexander Motin /* 113475c7a1d3SAlexander Motin * Fire off all allocated requests! 113575c7a1d3SAlexander Motin */ 113675c7a1d3SAlexander Motin while ((bio = TAILQ_FIRST(&queue)) != NULL) { 113775c7a1d3SAlexander Motin TAILQ_REMOVE(&queue, bio, bio_queue); 113875c7a1d3SAlexander Motin (*dev_data->csw->d_strategy)(bio); 113975c7a1d3SAlexander Motin } 1140130f4520SKenneth D. Merry } 1141130f4520SKenneth D. Merry 1142c3e7ba3eSAlexander Motin static uint64_t 1143c3e7ba3eSAlexander Motin ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname) 1144c3e7ba3eSAlexander Motin { 1145c3e7ba3eSAlexander Motin struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 1146c3e7ba3eSAlexander Motin struct diocgattr_arg arg; 1147c3e7ba3eSAlexander Motin int error; 1148c3e7ba3eSAlexander Motin 1149c3e7ba3eSAlexander Motin if (dev_data->csw == NULL || dev_data->csw->d_ioctl == NULL) 1150c3e7ba3eSAlexander Motin return (UINT64_MAX); 1151c3e7ba3eSAlexander Motin strlcpy(arg.name, attrname, sizeof(arg.name)); 1152c3e7ba3eSAlexander Motin arg.len = sizeof(arg.value.off); 1153c3e7ba3eSAlexander Motin error = dev_data->csw->d_ioctl(dev_data->cdev, 1154c3e7ba3eSAlexander Motin DIOCGATTR, (caddr_t)&arg, FREAD, curthread); 1155c3e7ba3eSAlexander Motin if (error != 0) 1156c3e7ba3eSAlexander Motin return (UINT64_MAX); 1157c3e7ba3eSAlexander Motin return (arg.value.off); 1158c3e7ba3eSAlexander Motin } 1159c3e7ba3eSAlexander Motin 1160130f4520SKenneth D. Merry static void 11617d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun, 11627d0d4342SAlexander Motin union ctl_io *io) 11637d0d4342SAlexander Motin { 11647d0d4342SAlexander Motin struct ctl_be_block_io *beio; 11657d0d4342SAlexander Motin struct ctl_lba_len_flags *lbalen; 11667d0d4342SAlexander Motin 11677d0d4342SAlexander Motin DPRINTF("entered\n"); 11687d0d4342SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 11697d0d4342SAlexander Motin lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 11707d0d4342SAlexander Motin 11717d0d4342SAlexander Motin beio->io_len = lbalen->len * be_lun->blocksize; 11727d0d4342SAlexander Motin beio->io_offset = lbalen->lba * be_lun->blocksize; 11737d0d4342SAlexander Motin beio->io_arg = (lbalen->flags & SSC_IMMED) != 0; 11747d0d4342SAlexander Motin beio->bio_cmd = BIO_FLUSH; 11757d0d4342SAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 11767d0d4342SAlexander Motin DPRINTF("SYNC\n"); 11777d0d4342SAlexander Motin be_lun->lun_flush(be_lun, beio); 11787d0d4342SAlexander Motin } 11797d0d4342SAlexander Motin 11807d0d4342SAlexander Motin static void 1181ee7f31c0SAlexander Motin ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio) 1182ee7f31c0SAlexander Motin { 1183ee7f31c0SAlexander Motin union ctl_io *io; 1184ee7f31c0SAlexander Motin 1185ee7f31c0SAlexander Motin io = beio->io; 1186ee7f31c0SAlexander Motin ctl_free_beio(beio); 1187ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1188ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1189ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1190ee7f31c0SAlexander Motin ctl_config_write_done(io); 1191ee7f31c0SAlexander Motin return; 1192ee7f31c0SAlexander Motin } 1193ee7f31c0SAlexander Motin 1194ee7f31c0SAlexander Motin ctl_be_block_config_write(io); 1195ee7f31c0SAlexander Motin } 1196ee7f31c0SAlexander Motin 1197ee7f31c0SAlexander Motin static void 1198ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun, 1199ee7f31c0SAlexander Motin union ctl_io *io) 1200ee7f31c0SAlexander Motin { 1201ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 120266df9136SAlexander Motin struct ctl_lba_len_flags *lbalen; 1203fee04ef7SAlexander Motin uint64_t len_left, lba; 1204fee04ef7SAlexander Motin uint32_t pb, pbo, adj; 1205ee7f31c0SAlexander Motin int i, seglen; 1206ee7f31c0SAlexander Motin uint8_t *buf, *end; 1207ee7f31c0SAlexander Motin 1208ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1209ee7f31c0SAlexander Motin 1210e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 121111b569f7SAlexander Motin lbalen = ARGS(beio->io); 1212ee7f31c0SAlexander Motin 121364c5167cSAlexander Motin if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || 12143406a2a0SAlexander Motin (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { 1215ee7f31c0SAlexander Motin ctl_free_beio(beio); 1216ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1217ee7f31c0SAlexander Motin /*sks_valid*/ 1, 1218ee7f31c0SAlexander Motin /*command*/ 1, 1219ee7f31c0SAlexander Motin /*field*/ 1, 1220ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1221ee7f31c0SAlexander Motin /*bit*/ 0); 1222ee7f31c0SAlexander Motin ctl_config_write_done(io); 1223ee7f31c0SAlexander Motin return; 1224ee7f31c0SAlexander Motin } 1225ee7f31c0SAlexander Motin 12263406a2a0SAlexander Motin if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { 122766df9136SAlexander Motin beio->io_offset = lbalen->lba * be_lun->blocksize; 122866df9136SAlexander Motin beio->io_len = (uint64_t)lbalen->len * be_lun->blocksize; 1229ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1230ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 1231ee7f31c0SAlexander Motin 1232ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1233ee7f31c0SAlexander Motin return; 1234ee7f31c0SAlexander Motin } 1235ee7f31c0SAlexander Motin 1236ee7f31c0SAlexander Motin beio->bio_cmd = BIO_WRITE; 1237ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_WRITE; 1238ee7f31c0SAlexander Motin 1239ee7f31c0SAlexander Motin DPRINTF("WRITE SAME at LBA %jx len %u\n", 124066df9136SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len); 1241ee7f31c0SAlexander Motin 1242fee04ef7SAlexander Motin pb = be_lun->blocksize << be_lun->pblockexp; 1243fee04ef7SAlexander Motin if (be_lun->pblockoff > 0) 1244fee04ef7SAlexander Motin pbo = pb - be_lun->blocksize * be_lun->pblockoff; 1245fee04ef7SAlexander Motin else 1246fee04ef7SAlexander Motin pbo = 0; 124766df9136SAlexander Motin len_left = (uint64_t)lbalen->len * be_lun->blocksize; 1248ee7f31c0SAlexander Motin for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) { 1249ee7f31c0SAlexander Motin 1250ee7f31c0SAlexander Motin /* 1251ee7f31c0SAlexander Motin * Setup the S/G entry for this chunk. 1252ee7f31c0SAlexander Motin */ 125308a7cce5SAlexander Motin seglen = MIN(CTLBLK_MAX_SEG, len_left); 125493b8c96cSAlexander Motin if (pb > be_lun->blocksize) { 125593b8c96cSAlexander Motin adj = ((lbalen->lba + lba) * be_lun->blocksize + 125693b8c96cSAlexander Motin seglen - pbo) % pb; 125793b8c96cSAlexander Motin if (seglen > adj) 125893b8c96cSAlexander Motin seglen -= adj; 125993b8c96cSAlexander Motin else 126093b8c96cSAlexander Motin seglen -= seglen % be_lun->blocksize; 126193b8c96cSAlexander Motin } else 1262ee7f31c0SAlexander Motin seglen -= seglen % be_lun->blocksize; 1263ee7f31c0SAlexander Motin beio->sg_segs[i].len = seglen; 1264ee7f31c0SAlexander Motin beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1265ee7f31c0SAlexander Motin 1266ee7f31c0SAlexander Motin DPRINTF("segment %d addr %p len %zd\n", i, 1267ee7f31c0SAlexander Motin beio->sg_segs[i].addr, beio->sg_segs[i].len); 1268ee7f31c0SAlexander Motin 1269ee7f31c0SAlexander Motin beio->num_segs++; 1270ee7f31c0SAlexander Motin len_left -= seglen; 1271ee7f31c0SAlexander Motin 1272ee7f31c0SAlexander Motin buf = beio->sg_segs[i].addr; 1273ee7f31c0SAlexander Motin end = buf + seglen; 1274ee7f31c0SAlexander Motin for (; buf < end; buf += be_lun->blocksize) { 1275ee7f31c0SAlexander Motin memcpy(buf, io->scsiio.kern_data_ptr, be_lun->blocksize); 127666df9136SAlexander Motin if (lbalen->flags & SWS_LBDATA) 127766df9136SAlexander Motin scsi_ulto4b(lbalen->lba + lba, buf); 1278ee7f31c0SAlexander Motin lba++; 1279ee7f31c0SAlexander Motin } 1280ee7f31c0SAlexander Motin } 1281ee7f31c0SAlexander Motin 128266df9136SAlexander Motin beio->io_offset = lbalen->lba * be_lun->blocksize; 1283ee7f31c0SAlexander Motin beio->io_len = lba * be_lun->blocksize; 1284ee7f31c0SAlexander Motin 1285ee7f31c0SAlexander Motin /* We can not do all in one run. Correct and schedule rerun. */ 1286ee7f31c0SAlexander Motin if (len_left > 0) { 128766df9136SAlexander Motin lbalen->lba += lba; 128866df9136SAlexander Motin lbalen->len -= lba; 1289ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done_ws; 1290ee7f31c0SAlexander Motin } 1291ee7f31c0SAlexander Motin 1292ee7f31c0SAlexander Motin be_lun->dispatch(be_lun, beio); 1293ee7f31c0SAlexander Motin } 1294ee7f31c0SAlexander Motin 1295ee7f31c0SAlexander Motin static void 1296ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun, 1297ee7f31c0SAlexander Motin union ctl_io *io) 1298ee7f31c0SAlexander Motin { 1299ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 130066df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1301ee7f31c0SAlexander Motin 1302ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1303ee7f31c0SAlexander Motin 1304e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 130566df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1306ee7f31c0SAlexander Motin 13073406a2a0SAlexander Motin if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { 1308ee7f31c0SAlexander Motin ctl_free_beio(beio); 1309ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1310ee7f31c0SAlexander Motin /*sks_valid*/ 0, 1311ee7f31c0SAlexander Motin /*command*/ 1, 1312ee7f31c0SAlexander Motin /*field*/ 0, 1313ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1314ee7f31c0SAlexander Motin /*bit*/ 0); 1315ee7f31c0SAlexander Motin ctl_config_write_done(io); 1316ee7f31c0SAlexander Motin return; 1317ee7f31c0SAlexander Motin } 1318ee7f31c0SAlexander Motin 1319ee7f31c0SAlexander Motin beio->io_len = 0; 1320ee7f31c0SAlexander Motin beio->io_offset = -1; 1321ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1322ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 132366df9136SAlexander Motin DPRINTF("UNMAP\n"); 1324ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1325ee7f31c0SAlexander Motin } 1326ee7f31c0SAlexander Motin 1327ee7f31c0SAlexander Motin static void 1328ef8daf3fSAlexander Motin ctl_be_block_cr_done(struct ctl_be_block_io *beio) 1329ef8daf3fSAlexander Motin { 1330ef8daf3fSAlexander Motin union ctl_io *io; 1331ef8daf3fSAlexander Motin 1332ef8daf3fSAlexander Motin io = beio->io; 1333ef8daf3fSAlexander Motin ctl_free_beio(beio); 1334ef8daf3fSAlexander Motin ctl_config_read_done(io); 1335ef8daf3fSAlexander Motin } 1336ef8daf3fSAlexander Motin 1337ef8daf3fSAlexander Motin static void 1338ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 1339ef8daf3fSAlexander Motin union ctl_io *io) 1340ef8daf3fSAlexander Motin { 1341ef8daf3fSAlexander Motin struct ctl_be_block_io *beio; 1342ef8daf3fSAlexander Motin struct ctl_be_block_softc *softc; 1343ef8daf3fSAlexander Motin 1344ef8daf3fSAlexander Motin DPRINTF("entered\n"); 1345ef8daf3fSAlexander Motin 1346ef8daf3fSAlexander Motin softc = be_lun->softc; 1347ef8daf3fSAlexander Motin beio = ctl_alloc_beio(softc); 1348ef8daf3fSAlexander Motin beio->io = io; 1349ef8daf3fSAlexander Motin beio->lun = be_lun; 1350ef8daf3fSAlexander Motin beio->beio_cont = ctl_be_block_cr_done; 1351ef8daf3fSAlexander Motin PRIV(io)->ptr = (void *)beio; 1352ef8daf3fSAlexander Motin 1353ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 1354ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: /* GET LBA STATUS */ 1355ef8daf3fSAlexander Motin beio->bio_cmd = -1; 1356ef8daf3fSAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 1357ef8daf3fSAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1358ef8daf3fSAlexander Motin beio->io_len = 0; 1359ef8daf3fSAlexander Motin if (be_lun->get_lba_status) 1360ef8daf3fSAlexander Motin be_lun->get_lba_status(be_lun, beio); 1361ef8daf3fSAlexander Motin else 1362ef8daf3fSAlexander Motin ctl_be_block_cr_done(beio); 1363ef8daf3fSAlexander Motin break; 1364ef8daf3fSAlexander Motin default: 1365ef8daf3fSAlexander Motin panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1366ef8daf3fSAlexander Motin break; 1367ef8daf3fSAlexander Motin } 1368ef8daf3fSAlexander Motin } 1369ef8daf3fSAlexander Motin 1370ef8daf3fSAlexander Motin static void 1371ee7f31c0SAlexander Motin ctl_be_block_cw_done(struct ctl_be_block_io *beio) 1372ee7f31c0SAlexander Motin { 1373ee7f31c0SAlexander Motin union ctl_io *io; 1374ee7f31c0SAlexander Motin 1375ee7f31c0SAlexander Motin io = beio->io; 1376ee7f31c0SAlexander Motin ctl_free_beio(beio); 1377ee7f31c0SAlexander Motin ctl_config_write_done(io); 1378ee7f31c0SAlexander Motin } 1379ee7f31c0SAlexander Motin 1380ee7f31c0SAlexander Motin static void 1381130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 1382130f4520SKenneth D. Merry union ctl_io *io) 1383130f4520SKenneth D. Merry { 1384130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1385130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1386130f4520SKenneth D. Merry 1387130f4520SKenneth D. Merry DPRINTF("entered\n"); 1388130f4520SKenneth D. Merry 1389130f4520SKenneth D. Merry softc = be_lun->softc; 1390130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1391130f4520SKenneth D. Merry beio->io = io; 1392130f4520SKenneth D. Merry beio->lun = be_lun; 1393ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done; 13947d0d4342SAlexander Motin switch (io->scsiio.tag_type) { 13957d0d4342SAlexander Motin case CTL_TAG_ORDERED: 13967d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 13977d0d4342SAlexander Motin break; 13987d0d4342SAlexander Motin case CTL_TAG_HEAD_OF_QUEUE: 13997d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_HEAD; 14007d0d4342SAlexander Motin break; 14017d0d4342SAlexander Motin case CTL_TAG_UNTAGGED: 14027d0d4342SAlexander Motin case CTL_TAG_SIMPLE: 14037d0d4342SAlexander Motin case CTL_TAG_ACA: 14047d0d4342SAlexander Motin default: 14057d0d4342SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 14067d0d4342SAlexander Motin break; 14077d0d4342SAlexander Motin } 1408e86a4142SAlexander Motin PRIV(io)->ptr = (void *)beio; 1409130f4520SKenneth D. Merry 1410130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 1411130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 1412130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 14137d0d4342SAlexander Motin ctl_be_block_cw_dispatch_sync(be_lun, io); 1414130f4520SKenneth D. Merry break; 1415ee7f31c0SAlexander Motin case WRITE_SAME_10: 1416ee7f31c0SAlexander Motin case WRITE_SAME_16: 1417ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(be_lun, io); 1418ee7f31c0SAlexander Motin break; 1419ee7f31c0SAlexander Motin case UNMAP: 1420ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(be_lun, io); 1421ee7f31c0SAlexander Motin break; 1422130f4520SKenneth D. Merry default: 1423130f4520SKenneth D. Merry panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1424130f4520SKenneth D. Merry break; 1425130f4520SKenneth D. Merry } 1426130f4520SKenneth D. Merry } 1427130f4520SKenneth D. Merry 1428d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t"); 1429d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t"); 1430d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t"); 1431d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t"); 1432130f4520SKenneth D. Merry 1433130f4520SKenneth D. Merry static void 143408a7cce5SAlexander Motin ctl_be_block_next(struct ctl_be_block_io *beio) 143508a7cce5SAlexander Motin { 143608a7cce5SAlexander Motin struct ctl_be_block_lun *be_lun; 143708a7cce5SAlexander Motin union ctl_io *io; 143808a7cce5SAlexander Motin 143908a7cce5SAlexander Motin io = beio->io; 144008a7cce5SAlexander Motin be_lun = beio->lun; 144108a7cce5SAlexander Motin ctl_free_beio(beio); 1442ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1443ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1444ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 144511b569f7SAlexander Motin ctl_data_submit_done(io); 144608a7cce5SAlexander Motin return; 144708a7cce5SAlexander Motin } 144808a7cce5SAlexander Motin 144908a7cce5SAlexander Motin io->io_hdr.status &= ~CTL_STATUS_MASK; 145008a7cce5SAlexander Motin io->io_hdr.status |= CTL_STATUS_NONE; 145108a7cce5SAlexander Motin 145275c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 145308a7cce5SAlexander Motin /* 145408a7cce5SAlexander Motin * XXX KDM make sure that links is okay to use at this point. 145508a7cce5SAlexander Motin * Otherwise, we either need to add another field to ctl_io_hdr, 145608a7cce5SAlexander Motin * or deal with resource allocation here. 145708a7cce5SAlexander Motin */ 145808a7cce5SAlexander Motin STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 145975c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 146008a7cce5SAlexander Motin 146108a7cce5SAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 146208a7cce5SAlexander Motin } 146308a7cce5SAlexander Motin 146408a7cce5SAlexander Motin static void 1465130f4520SKenneth D. Merry ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 1466130f4520SKenneth D. Merry union ctl_io *io) 1467130f4520SKenneth D. Merry { 1468130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1469130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 147011b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 1471e86a4142SAlexander Motin struct ctl_ptr_len_flags *bptrlen; 1472e86a4142SAlexander Motin uint64_t len_left, lbas; 1473130f4520SKenneth D. Merry int i; 1474130f4520SKenneth D. Merry 1475130f4520SKenneth D. Merry softc = be_lun->softc; 1476130f4520SKenneth D. Merry 1477130f4520SKenneth D. Merry DPRINTF("entered\n"); 1478130f4520SKenneth D. Merry 147911b569f7SAlexander Motin lbalen = ARGS(io); 148011b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1481130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0); 148211b569f7SAlexander Motin } else { 148311b569f7SAlexander Motin SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0); 1484130f4520SKenneth D. Merry } 1485130f4520SKenneth D. Merry 1486130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1487130f4520SKenneth D. Merry beio->io = io; 1488130f4520SKenneth D. Merry beio->lun = be_lun; 1489e86a4142SAlexander Motin bptrlen = PRIV(io); 1490e86a4142SAlexander Motin bptrlen->ptr = (void *)beio; 1491130f4520SKenneth D. Merry 1492130f4520SKenneth D. Merry switch (io->scsiio.tag_type) { 1493130f4520SKenneth D. Merry case CTL_TAG_ORDERED: 1494130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1495130f4520SKenneth D. Merry break; 1496130f4520SKenneth D. Merry case CTL_TAG_HEAD_OF_QUEUE: 1497130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1498130f4520SKenneth D. Merry break; 1499130f4520SKenneth D. Merry case CTL_TAG_UNTAGGED: 1500130f4520SKenneth D. Merry case CTL_TAG_SIMPLE: 1501130f4520SKenneth D. Merry case CTL_TAG_ACA: 1502130f4520SKenneth D. Merry default: 1503130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1504130f4520SKenneth D. Merry break; 1505130f4520SKenneth D. Merry } 1506130f4520SKenneth D. Merry 150711b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1508130f4520SKenneth D. Merry beio->bio_cmd = BIO_WRITE; 1509130f4520SKenneth D. Merry beio->ds_trans_type = DEVSTAT_WRITE; 151011b569f7SAlexander Motin } else { 151111b569f7SAlexander Motin beio->bio_cmd = BIO_READ; 151211b569f7SAlexander Motin beio->ds_trans_type = DEVSTAT_READ; 1513130f4520SKenneth D. Merry } 1514130f4520SKenneth D. Merry 151508a7cce5SAlexander Motin DPRINTF("%s at LBA %jx len %u @%ju\n", 1516130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", 1517e86a4142SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); 151811b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 151911b569f7SAlexander Motin lbas = CTLBLK_HALF_IO_SIZE; 152011b569f7SAlexander Motin else 152111b569f7SAlexander Motin lbas = CTLBLK_MAX_IO_SIZE; 152211b569f7SAlexander Motin lbas = MIN(lbalen->len - bptrlen->len, lbas / be_lun->blocksize); 1523e86a4142SAlexander Motin beio->io_offset = (lbalen->lba + bptrlen->len) * be_lun->blocksize; 1524e86a4142SAlexander Motin beio->io_len = lbas * be_lun->blocksize; 1525e86a4142SAlexander Motin bptrlen->len += lbas; 1526130f4520SKenneth D. Merry 152708a7cce5SAlexander Motin for (i = 0, len_left = beio->io_len; len_left > 0; i++) { 152808a7cce5SAlexander Motin KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)", 152908a7cce5SAlexander Motin i, CTLBLK_MAX_SEGS)); 1530130f4520SKenneth D. Merry 1531130f4520SKenneth D. Merry /* 1532130f4520SKenneth D. Merry * Setup the S/G entry for this chunk. 1533130f4520SKenneth D. Merry */ 153408a7cce5SAlexander Motin beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left); 1535130f4520SKenneth D. Merry beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1536130f4520SKenneth D. Merry 1537130f4520SKenneth D. Merry DPRINTF("segment %d addr %p len %zd\n", i, 1538130f4520SKenneth D. Merry beio->sg_segs[i].addr, beio->sg_segs[i].len); 1539130f4520SKenneth D. Merry 154011b569f7SAlexander Motin /* Set up second segment for compare operation. */ 154111b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) { 154211b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].len = 154311b569f7SAlexander Motin beio->sg_segs[i].len; 154411b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = 154511b569f7SAlexander Motin uma_zalloc(be_lun->lun_zone, M_WAITOK); 154611b569f7SAlexander Motin } 154711b569f7SAlexander Motin 1548130f4520SKenneth D. Merry beio->num_segs++; 1549130f4520SKenneth D. Merry len_left -= beio->sg_segs[i].len; 1550130f4520SKenneth D. Merry } 1551e86a4142SAlexander Motin if (bptrlen->len < lbalen->len) 155208a7cce5SAlexander Motin beio->beio_cont = ctl_be_block_next; 155308a7cce5SAlexander Motin io->scsiio.be_move_done = ctl_be_block_move_done; 155411b569f7SAlexander Motin /* For compare we have separate S/G lists for read and datamove. */ 155511b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 155611b569f7SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS]; 155711b569f7SAlexander Motin else 155808a7cce5SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs; 155908a7cce5SAlexander Motin io->scsiio.kern_data_len = beio->io_len; 156008a7cce5SAlexander Motin io->scsiio.kern_data_resid = 0; 156108a7cce5SAlexander Motin io->scsiio.kern_sg_entries = beio->num_segs; 156208a7cce5SAlexander Motin io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST; 1563130f4520SKenneth D. Merry 1564130f4520SKenneth D. Merry /* 1565130f4520SKenneth D. Merry * For the read case, we need to read the data into our buffers and 1566130f4520SKenneth D. Merry * then we can send it back to the user. For the write case, we 1567130f4520SKenneth D. Merry * need to get the data from the user first. 1568130f4520SKenneth D. Merry */ 1569130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 1570130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0); 1571130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1572130f4520SKenneth D. Merry } else { 1573130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0); 1574130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 1575130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 1576130f4520SKenneth D. Merry #endif 1577130f4520SKenneth D. Merry ctl_datamove(io); 1578130f4520SKenneth D. Merry } 1579130f4520SKenneth D. Merry } 1580130f4520SKenneth D. Merry 1581130f4520SKenneth D. Merry static void 1582130f4520SKenneth D. Merry ctl_be_block_worker(void *context, int pending) 1583130f4520SKenneth D. Merry { 1584130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 1585130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1586130f4520SKenneth D. Merry union ctl_io *io; 1587130f4520SKenneth D. Merry 1588130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)context; 1589130f4520SKenneth D. Merry softc = be_lun->softc; 1590130f4520SKenneth D. Merry 1591130f4520SKenneth D. Merry DPRINTF("entered\n"); 1592130f4520SKenneth D. Merry 159375c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1594130f4520SKenneth D. Merry for (;;) { 1595130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); 1596130f4520SKenneth D. Merry if (io != NULL) { 1597130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1598130f4520SKenneth D. Merry 1599130f4520SKenneth D. Merry DPRINTF("datamove queue\n"); 1600130f4520SKenneth D. Merry 1601130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, 1602130f4520SKenneth D. Merry ctl_io_hdr, links); 1603130f4520SKenneth D. Merry 160475c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1605130f4520SKenneth D. Merry 1606e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1607130f4520SKenneth D. Merry 1608130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1609130f4520SKenneth D. Merry 161075c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1611130f4520SKenneth D. Merry continue; 1612130f4520SKenneth D. Merry } 1613130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); 1614130f4520SKenneth D. Merry if (io != NULL) { 1615130f4520SKenneth D. Merry DPRINTF("config write queue\n"); 1616130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, 1617130f4520SKenneth D. Merry ctl_io_hdr, links); 161875c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1619130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(be_lun, io); 1620ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 1621ef8daf3fSAlexander Motin continue; 1622ef8daf3fSAlexander Motin } 1623ef8daf3fSAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); 1624ef8daf3fSAlexander Motin if (io != NULL) { 1625ef8daf3fSAlexander Motin DPRINTF("config read queue\n"); 1626ef8daf3fSAlexander Motin STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, 1627ef8daf3fSAlexander Motin ctl_io_hdr, links); 1628ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 1629ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(be_lun, io); 163075c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1631130f4520SKenneth D. Merry continue; 1632130f4520SKenneth D. Merry } 1633130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); 1634130f4520SKenneth D. Merry if (io != NULL) { 1635130f4520SKenneth D. Merry DPRINTF("input queue\n"); 1636130f4520SKenneth D. Merry 1637130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, 1638130f4520SKenneth D. Merry ctl_io_hdr, links); 163975c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1640130f4520SKenneth D. Merry 1641130f4520SKenneth D. Merry /* 1642130f4520SKenneth D. Merry * We must drop the lock, since this routine and 1643130f4520SKenneth D. Merry * its children may sleep. 1644130f4520SKenneth D. Merry */ 1645130f4520SKenneth D. Merry ctl_be_block_dispatch(be_lun, io); 1646130f4520SKenneth D. Merry 164775c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1648130f4520SKenneth D. Merry continue; 1649130f4520SKenneth D. Merry } 1650130f4520SKenneth D. Merry 1651130f4520SKenneth D. Merry /* 1652130f4520SKenneth D. Merry * If we get here, there is no work left in the queues, so 1653130f4520SKenneth D. Merry * just break out and let the task queue go to sleep. 1654130f4520SKenneth D. Merry */ 1655130f4520SKenneth D. Merry break; 1656130f4520SKenneth D. Merry } 165775c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1658130f4520SKenneth D. Merry } 1659130f4520SKenneth D. Merry 1660130f4520SKenneth D. Merry /* 1661130f4520SKenneth D. Merry * Entry point from CTL to the backend for I/O. We queue everything to a 1662130f4520SKenneth D. Merry * work thread, so this just puts the I/O on a queue and wakes up the 1663130f4520SKenneth D. Merry * thread. 1664130f4520SKenneth D. Merry */ 1665130f4520SKenneth D. Merry static int 1666130f4520SKenneth D. Merry ctl_be_block_submit(union ctl_io *io) 1667130f4520SKenneth D. Merry { 1668130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 1669130f4520SKenneth D. Merry struct ctl_be_lun *ctl_be_lun; 1670130f4520SKenneth D. Merry 1671130f4520SKenneth D. Merry DPRINTF("entered\n"); 1672130f4520SKenneth D. Merry 1673130f4520SKenneth D. Merry ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 1674130f4520SKenneth D. Merry CTL_PRIV_BACKEND_LUN].ptr; 1675130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 1676130f4520SKenneth D. Merry 1677130f4520SKenneth D. Merry /* 1678130f4520SKenneth D. Merry * Make sure we only get SCSI I/O. 1679130f4520SKenneth D. Merry */ 1680130f4520SKenneth D. Merry KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type " 1681130f4520SKenneth D. Merry "%#x) encountered", io->io_hdr.io_type)); 1682130f4520SKenneth D. Merry 1683e86a4142SAlexander Motin PRIV(io)->len = 0; 1684e86a4142SAlexander Motin 168575c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1686130f4520SKenneth D. Merry /* 1687130f4520SKenneth D. Merry * XXX KDM make sure that links is okay to use at this point. 1688130f4520SKenneth D. Merry * Otherwise, we either need to add another field to ctl_io_hdr, 1689130f4520SKenneth D. Merry * or deal with resource allocation here. 1690130f4520SKenneth D. Merry */ 1691130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 169275c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1693130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1694130f4520SKenneth D. Merry 16959c71cd5aSAlexander Motin return (CTL_RETVAL_COMPLETE); 1696130f4520SKenneth D. Merry } 1697130f4520SKenneth D. Merry 1698130f4520SKenneth D. Merry static int 1699130f4520SKenneth D. Merry ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 1700130f4520SKenneth D. Merry int flag, struct thread *td) 1701130f4520SKenneth D. Merry { 1702130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1703130f4520SKenneth D. Merry int error; 1704130f4520SKenneth D. Merry 1705130f4520SKenneth D. Merry softc = &backend_block_softc; 1706130f4520SKenneth D. Merry 1707130f4520SKenneth D. Merry error = 0; 1708130f4520SKenneth D. Merry 1709130f4520SKenneth D. Merry switch (cmd) { 1710130f4520SKenneth D. Merry case CTL_LUN_REQ: { 1711130f4520SKenneth D. Merry struct ctl_lun_req *lun_req; 1712130f4520SKenneth D. Merry 1713130f4520SKenneth D. Merry lun_req = (struct ctl_lun_req *)addr; 1714130f4520SKenneth D. Merry 1715130f4520SKenneth D. Merry switch (lun_req->reqtype) { 1716130f4520SKenneth D. Merry case CTL_LUNREQ_CREATE: 1717130f4520SKenneth D. Merry error = ctl_be_block_create(softc, lun_req); 1718130f4520SKenneth D. Merry break; 1719130f4520SKenneth D. Merry case CTL_LUNREQ_RM: 1720130f4520SKenneth D. Merry error = ctl_be_block_rm(softc, lun_req); 1721130f4520SKenneth D. Merry break; 172281177295SEdward Tomasz Napierala case CTL_LUNREQ_MODIFY: 172381177295SEdward Tomasz Napierala error = ctl_be_block_modify(softc, lun_req); 172481177295SEdward Tomasz Napierala break; 1725130f4520SKenneth D. Merry default: 1726130f4520SKenneth D. Merry lun_req->status = CTL_LUN_ERROR; 1727130f4520SKenneth D. Merry snprintf(lun_req->error_str, sizeof(lun_req->error_str), 172819720f41SAlexander Motin "invalid LUN request type %d", 1729130f4520SKenneth D. Merry lun_req->reqtype); 1730130f4520SKenneth D. Merry break; 1731130f4520SKenneth D. Merry } 1732130f4520SKenneth D. Merry break; 1733130f4520SKenneth D. Merry } 1734130f4520SKenneth D. Merry default: 1735130f4520SKenneth D. Merry error = ENOTTY; 1736130f4520SKenneth D. Merry break; 1737130f4520SKenneth D. Merry } 1738130f4520SKenneth D. Merry 1739130f4520SKenneth D. Merry return (error); 1740130f4520SKenneth D. Merry } 1741130f4520SKenneth D. Merry 1742130f4520SKenneth D. Merry static int 1743130f4520SKenneth D. Merry ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1744130f4520SKenneth D. Merry { 1745130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 1746130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 174734961f40SAlexander Motin char *value; 1748130f4520SKenneth D. Merry struct vattr vattr; 174934961f40SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos; 1750130f4520SKenneth D. Merry int error; 1751130f4520SKenneth D. Merry 1752130f4520SKenneth D. Merry error = 0; 1753130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 175419720f41SAlexander Motin params = &be_lun->params; 1755130f4520SKenneth D. Merry 1756130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_FILE; 1757130f4520SKenneth D. Merry be_lun->dispatch = ctl_be_block_dispatch_file; 1758130f4520SKenneth D. Merry be_lun->lun_flush = ctl_be_block_flush_file; 1759ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_file; 176053c146deSAlexander Motin be_lun->getattr = ctl_be_block_getattr_file; 1761130f4520SKenneth D. Merry 1762130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 1763130f4520SKenneth D. Merry if (error != 0) { 1764130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1765130f4520SKenneth D. Merry "error calling VOP_GETATTR() for file %s", 1766130f4520SKenneth D. Merry be_lun->dev_path); 1767130f4520SKenneth D. Merry return (error); 1768130f4520SKenneth D. Merry } 1769130f4520SKenneth D. Merry 1770130f4520SKenneth D. Merry /* 1771130f4520SKenneth D. Merry * Verify that we have the ability to upgrade to exclusive 1772130f4520SKenneth D. Merry * access on this file so we can trap errors at open instead 1773130f4520SKenneth D. Merry * of reporting them during first access. 1774130f4520SKenneth D. Merry */ 1775130f4520SKenneth D. Merry if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) { 1776130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY); 1777130f4520SKenneth D. Merry if (be_lun->vn->v_iflag & VI_DOOMED) { 1778130f4520SKenneth D. Merry error = EBADF; 1779130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1780130f4520SKenneth D. Merry "error locking file %s", be_lun->dev_path); 1781130f4520SKenneth D. Merry return (error); 1782130f4520SKenneth D. Merry } 1783130f4520SKenneth D. Merry } 1784130f4520SKenneth D. Merry 1785130f4520SKenneth D. Merry 1786130f4520SKenneth D. Merry file_data->cred = crhold(curthread->td_ucred); 178781177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) 178881177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 178981177295SEdward Tomasz Napierala else 1790130f4520SKenneth D. Merry be_lun->size_bytes = vattr.va_size; 1791130f4520SKenneth D. Merry /* 1792130f4520SKenneth D. Merry * We set the multi thread flag for file operations because all 1793130f4520SKenneth D. Merry * filesystems (in theory) are capable of allowing multiple readers 1794130f4520SKenneth D. Merry * of a file at once. So we want to get the maximum possible 1795130f4520SKenneth D. Merry * concurrency. 1796130f4520SKenneth D. Merry */ 1797130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD; 1798130f4520SKenneth D. Merry 1799130f4520SKenneth D. Merry /* 180020a28d6cSAlexander Motin * For files we can use any logical block size. Prefer 512 bytes 180120a28d6cSAlexander Motin * for compatibility reasons. If file's vattr.va_blocksize 180220a28d6cSAlexander Motin * (preferred I/O block size) is bigger and multiple to chosen 180320a28d6cSAlexander Motin * logical block size -- report it as physical block size. 1804130f4520SKenneth D. Merry */ 1805130f4520SKenneth D. Merry if (params->blocksize_bytes != 0) 1806130f4520SKenneth D. Merry be_lun->blocksize = params->blocksize_bytes; 1807130f4520SKenneth D. Merry else 1808130f4520SKenneth D. Merry be_lun->blocksize = 512; 180934961f40SAlexander Motin 181034961f40SAlexander Motin us = ps = vattr.va_blocksize; 181134961f40SAlexander Motin uo = po = 0; 181234961f40SAlexander Motin 181334961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblocksize"); 181434961f40SAlexander Motin if (value != NULL) 181534961f40SAlexander Motin ctl_expand_number(value, &ps); 181634961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblockoffset"); 181734961f40SAlexander Motin if (value != NULL) 181834961f40SAlexander Motin ctl_expand_number(value, &po); 181934961f40SAlexander Motin pss = ps / be_lun->blocksize; 182034961f40SAlexander Motin pos = po / be_lun->blocksize; 182134961f40SAlexander Motin if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) && 182234961f40SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) { 182320a28d6cSAlexander Motin be_lun->pblockexp = fls(pss) - 1; 182434961f40SAlexander Motin be_lun->pblockoff = (pss - pos) % pss; 182534961f40SAlexander Motin } 182634961f40SAlexander Motin 182734961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublocksize"); 182834961f40SAlexander Motin if (value != NULL) 182934961f40SAlexander Motin ctl_expand_number(value, &us); 183034961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublockoffset"); 183134961f40SAlexander Motin if (value != NULL) 183234961f40SAlexander Motin ctl_expand_number(value, &uo); 183334961f40SAlexander Motin uss = us / be_lun->blocksize; 183434961f40SAlexander Motin uos = uo / be_lun->blocksize; 183534961f40SAlexander Motin if ((uss > 0) && (uss * be_lun->blocksize == us) && (uss >= uos) && 183634961f40SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * be_lun->blocksize == uo)) { 183734961f40SAlexander Motin be_lun->ublockexp = fls(uss) - 1; 183834961f40SAlexander Motin be_lun->ublockoff = (uss - uos) % uss; 183920a28d6cSAlexander Motin } 1840130f4520SKenneth D. Merry 1841130f4520SKenneth D. Merry /* 1842130f4520SKenneth D. Merry * Sanity check. The media size has to be at least one 1843130f4520SKenneth D. Merry * sector long. 1844130f4520SKenneth D. Merry */ 1845130f4520SKenneth D. Merry if (be_lun->size_bytes < be_lun->blocksize) { 1846130f4520SKenneth D. Merry error = EINVAL; 1847130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1848130f4520SKenneth D. Merry "file %s size %ju < block size %u", be_lun->dev_path, 1849130f4520SKenneth D. Merry (uintmax_t)be_lun->size_bytes, be_lun->blocksize); 1850130f4520SKenneth D. Merry } 1851cb8727e2SAlexander Motin 1852cb8727e2SAlexander Motin be_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / be_lun->blocksize; 1853130f4520SKenneth D. Merry return (error); 1854130f4520SKenneth D. Merry } 1855130f4520SKenneth D. Merry 1856130f4520SKenneth D. Merry static int 1857130f4520SKenneth D. Merry ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1858130f4520SKenneth D. Merry { 1859130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 1860130f4520SKenneth D. Merry struct vattr vattr; 1861130f4520SKenneth D. Merry struct cdev *dev; 1862130f4520SKenneth D. Merry struct cdevsw *devsw; 186334961f40SAlexander Motin char *value; 1864fbc8d4ffSAlexander Motin int error, atomic, maxio, unmap; 1865*a15bbf15SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos, tmp; 1866130f4520SKenneth D. Merry 186719720f41SAlexander Motin params = &be_lun->params; 1868130f4520SKenneth D. Merry 1869130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_DEV; 1870130f4520SKenneth D. Merry be_lun->backend.dev.cdev = be_lun->vn->v_rdev; 1871130f4520SKenneth D. Merry be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev, 1872130f4520SKenneth D. Merry &be_lun->backend.dev.dev_ref); 1873130f4520SKenneth D. Merry if (be_lun->backend.dev.csw == NULL) 1874130f4520SKenneth D. Merry panic("Unable to retrieve device switch"); 1875ef8daf3fSAlexander Motin if (strcmp(be_lun->backend.dev.csw->d_name, "zvol") == 0) { 187667f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_zvol; 1877ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_zvol; 1878cb8727e2SAlexander Motin atomic = maxio = CTLBLK_MAX_IO_SIZE; 1879cb8727e2SAlexander Motin } else { 188067f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_dev; 1881cb8727e2SAlexander Motin atomic = 0; 1882cb8727e2SAlexander Motin maxio = be_lun->backend.dev.cdev->si_iosize_max; 1883cb8727e2SAlexander Motin if (maxio <= 0) 1884cb8727e2SAlexander Motin maxio = DFLTPHYS; 1885cb8727e2SAlexander Motin if (maxio > CTLBLK_MAX_IO_SIZE) 1886cb8727e2SAlexander Motin maxio = CTLBLK_MAX_IO_SIZE; 1887cb8727e2SAlexander Motin } 188867f586a8SAlexander Motin be_lun->lun_flush = ctl_be_block_flush_dev; 1889c3e7ba3eSAlexander Motin be_lun->getattr = ctl_be_block_getattr_dev; 1890130f4520SKenneth D. Merry 1891130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED); 1892130f4520SKenneth D. Merry if (error) { 1893130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 189419720f41SAlexander Motin "error getting vnode attributes for device %s", 189519720f41SAlexander Motin be_lun->dev_path); 1896130f4520SKenneth D. Merry return (error); 1897130f4520SKenneth D. Merry } 1898130f4520SKenneth D. Merry 1899130f4520SKenneth D. Merry dev = be_lun->vn->v_rdev; 1900130f4520SKenneth D. Merry devsw = dev->si_devsw; 1901130f4520SKenneth D. Merry if (!devsw->d_ioctl) { 1902130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 190319720f41SAlexander Motin "no d_ioctl for device %s!", 1904130f4520SKenneth D. Merry be_lun->dev_path); 1905130f4520SKenneth D. Merry return (ENODEV); 1906130f4520SKenneth D. Merry } 1907130f4520SKenneth D. Merry 1908*a15bbf15SAlexander Motin error = devsw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD, 1909130f4520SKenneth D. Merry curthread); 1910130f4520SKenneth D. Merry if (error) { 1911130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 191219720f41SAlexander Motin "error %d returned for DIOCGSECTORSIZE ioctl " 191319720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 1914130f4520SKenneth D. Merry return (error); 1915130f4520SKenneth D. Merry } 1916130f4520SKenneth D. Merry 1917130f4520SKenneth D. Merry /* 1918130f4520SKenneth D. Merry * If the user has asked for a blocksize that is greater than the 1919130f4520SKenneth D. Merry * backing device's blocksize, we can do it only if the blocksize 1920130f4520SKenneth D. Merry * the user is asking for is an even multiple of the underlying 1921130f4520SKenneth D. Merry * device's blocksize. 1922130f4520SKenneth D. Merry */ 1923*a15bbf15SAlexander Motin if ((params->blocksize_bytes != 0) && 1924*a15bbf15SAlexander Motin (params->blocksize_bytes >= tmp)) { 1925*a15bbf15SAlexander Motin if (params->blocksize_bytes % tmp == 0) { 1926130f4520SKenneth D. Merry be_lun->blocksize = params->blocksize_bytes; 1927130f4520SKenneth D. Merry } else { 1928130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 192919720f41SAlexander Motin "requested blocksize %u is not an even " 1930130f4520SKenneth D. Merry "multiple of backing device blocksize %u", 193119720f41SAlexander Motin params->blocksize_bytes, 1932130f4520SKenneth D. Merry be_lun->blocksize); 1933130f4520SKenneth D. Merry return (EINVAL); 1934130f4520SKenneth D. Merry 1935130f4520SKenneth D. Merry } 1936*a15bbf15SAlexander Motin } else if (params->blocksize_bytes != 0) { 1937130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 193819720f41SAlexander Motin "requested blocksize %u < backing device " 193919720f41SAlexander Motin "blocksize %u", params->blocksize_bytes, 1940130f4520SKenneth D. Merry be_lun->blocksize); 1941130f4520SKenneth D. Merry return (EINVAL); 1942*a15bbf15SAlexander Motin } else 1943*a15bbf15SAlexander Motin be_lun->blocksize = tmp; 1944130f4520SKenneth D. Merry 1945*a15bbf15SAlexander Motin error = devsw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&tmp, FREAD, 1946130f4520SKenneth D. Merry curthread); 1947130f4520SKenneth D. Merry if (error) { 1948130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 194919720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE " 195019720f41SAlexander Motin " ioctl on %s!", error, 195181177295SEdward Tomasz Napierala be_lun->dev_path); 1952130f4520SKenneth D. Merry return (error); 1953130f4520SKenneth D. Merry } 1954130f4520SKenneth D. Merry 195581177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 1956*a15bbf15SAlexander Motin if (params->lun_size_bytes > tmp) { 195781177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 195819720f41SAlexander Motin "requested LUN size %ju > backing device " 195919720f41SAlexander Motin "size %ju", 196081177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 196181177295SEdward Tomasz Napierala (uintmax_t)be_lun->size_bytes); 196281177295SEdward Tomasz Napierala return (EINVAL); 1963130f4520SKenneth D. Merry } 1964130f4520SKenneth D. Merry 196581177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 1966*a15bbf15SAlexander Motin } else 1967*a15bbf15SAlexander Motin be_lun->size_bytes = tmp; 196881177295SEdward Tomasz Napierala 1969f6012722SAlexander Motin error = devsw->d_ioctl(dev, DIOCGSTRIPESIZE, 1970f6012722SAlexander Motin (caddr_t)&ps, FREAD, curthread); 1971f6012722SAlexander Motin if (error) 1972f6012722SAlexander Motin ps = po = 0; 1973f6012722SAlexander Motin else { 1974f6012722SAlexander Motin error = devsw->d_ioctl(dev, DIOCGSTRIPEOFFSET, 1975f6012722SAlexander Motin (caddr_t)&po, FREAD, curthread); 1976f6012722SAlexander Motin if (error) 1977f6012722SAlexander Motin po = 0; 1978f6012722SAlexander Motin } 197934961f40SAlexander Motin us = ps; 198034961f40SAlexander Motin uo = po; 198134961f40SAlexander Motin 198234961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblocksize"); 198334961f40SAlexander Motin if (value != NULL) 198434961f40SAlexander Motin ctl_expand_number(value, &ps); 198534961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblockoffset"); 198634961f40SAlexander Motin if (value != NULL) 198734961f40SAlexander Motin ctl_expand_number(value, &po); 1988f6012722SAlexander Motin pss = ps / be_lun->blocksize; 1989f6012722SAlexander Motin pos = po / be_lun->blocksize; 1990f6012722SAlexander Motin if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) && 1991f6012722SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) { 1992f6012722SAlexander Motin be_lun->pblockexp = fls(pss) - 1; 1993f6012722SAlexander Motin be_lun->pblockoff = (pss - pos) % pss; 1994f6012722SAlexander Motin } 1995f6012722SAlexander Motin 199634961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublocksize"); 199734961f40SAlexander Motin if (value != NULL) 199834961f40SAlexander Motin ctl_expand_number(value, &us); 199934961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublockoffset"); 200034961f40SAlexander Motin if (value != NULL) 200134961f40SAlexander Motin ctl_expand_number(value, &uo); 200234961f40SAlexander Motin uss = us / be_lun->blocksize; 200334961f40SAlexander Motin uos = uo / be_lun->blocksize; 200434961f40SAlexander Motin if ((uss > 0) && (uss * be_lun->blocksize == us) && (uss >= uos) && 200534961f40SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * be_lun->blocksize == uo)) { 200634961f40SAlexander Motin be_lun->ublockexp = fls(uss) - 1; 200734961f40SAlexander Motin be_lun->ublockoff = (uss - uos) % uss; 200834961f40SAlexander Motin } 200934961f40SAlexander Motin 2010cb8727e2SAlexander Motin be_lun->atomicblock = atomic / be_lun->blocksize; 2011cb8727e2SAlexander Motin be_lun->opttxferlen = maxio / be_lun->blocksize; 2012fbc8d4ffSAlexander Motin 2013fbc8d4ffSAlexander Motin if (be_lun->dispatch == ctl_be_block_dispatch_zvol) { 2014fbc8d4ffSAlexander Motin unmap = 1; 2015fbc8d4ffSAlexander Motin } else { 2016fbc8d4ffSAlexander Motin struct diocgattr_arg arg; 2017fbc8d4ffSAlexander Motin 2018fbc8d4ffSAlexander Motin strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); 2019fbc8d4ffSAlexander Motin arg.len = sizeof(arg.value.i); 2020fbc8d4ffSAlexander Motin error = devsw->d_ioctl(dev, DIOCGATTR, 2021fbc8d4ffSAlexander Motin (caddr_t)&arg, FREAD, curthread); 2022fbc8d4ffSAlexander Motin unmap = (error == 0) ? arg.value.i : 0; 2023fbc8d4ffSAlexander Motin } 2024fbc8d4ffSAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "unmap"); 2025fbc8d4ffSAlexander Motin if (value != NULL) 2026fbc8d4ffSAlexander Motin unmap = (strcmp(value, "on") == 0); 2027fbc8d4ffSAlexander Motin if (unmap) 2028fbc8d4ffSAlexander Motin be_lun->unmap = ctl_be_block_unmap_dev; 2029fbc8d4ffSAlexander Motin 203081177295SEdward Tomasz Napierala return (0); 203181177295SEdward Tomasz Napierala } 2032130f4520SKenneth D. Merry 2033130f4520SKenneth D. Merry static int 2034130f4520SKenneth D. Merry ctl_be_block_close(struct ctl_be_block_lun *be_lun) 2035130f4520SKenneth D. Merry { 2036130f4520SKenneth D. Merry DROP_GIANT(); 2037130f4520SKenneth D. Merry if (be_lun->vn) { 2038130f4520SKenneth D. Merry int flags = FREAD | FWRITE; 2039130f4520SKenneth D. Merry 2040130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2041130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2042130f4520SKenneth D. Merry if (be_lun->backend.dev.csw) { 2043130f4520SKenneth D. Merry dev_relthread(be_lun->backend.dev.cdev, 2044130f4520SKenneth D. Merry be_lun->backend.dev.dev_ref); 2045130f4520SKenneth D. Merry be_lun->backend.dev.csw = NULL; 2046130f4520SKenneth D. Merry be_lun->backend.dev.cdev = NULL; 2047130f4520SKenneth D. Merry } 2048130f4520SKenneth D. Merry break; 2049130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2050130f4520SKenneth D. Merry break; 2051130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2052025a2301SEdward Tomasz Napierala break; 2053130f4520SKenneth D. Merry default: 2054130f4520SKenneth D. Merry panic("Unexpected backend type."); 2055130f4520SKenneth D. Merry break; 2056130f4520SKenneth D. Merry } 2057130f4520SKenneth D. Merry 2058130f4520SKenneth D. Merry (void)vn_close(be_lun->vn, flags, NOCRED, curthread); 2059130f4520SKenneth D. Merry be_lun->vn = NULL; 2060130f4520SKenneth D. Merry 2061130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2062130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2063130f4520SKenneth D. Merry break; 2064130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2065130f4520SKenneth D. Merry if (be_lun->backend.file.cred != NULL) { 2066130f4520SKenneth D. Merry crfree(be_lun->backend.file.cred); 2067130f4520SKenneth D. Merry be_lun->backend.file.cred = NULL; 2068130f4520SKenneth D. Merry } 2069130f4520SKenneth D. Merry break; 2070130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2071025a2301SEdward Tomasz Napierala break; 2072130f4520SKenneth D. Merry default: 2073130f4520SKenneth D. Merry panic("Unexpected backend type."); 2074130f4520SKenneth D. Merry break; 2075130f4520SKenneth D. Merry } 207619720f41SAlexander Motin be_lun->dev_type = CTL_BE_BLOCK_NONE; 2077130f4520SKenneth D. Merry } 2078130f4520SKenneth D. Merry PICKUP_GIANT(); 2079130f4520SKenneth D. Merry 2080130f4520SKenneth D. Merry return (0); 2081130f4520SKenneth D. Merry } 2082130f4520SKenneth D. Merry 2083130f4520SKenneth D. Merry static int 2084130f4520SKenneth D. Merry ctl_be_block_open(struct ctl_be_block_softc *softc, 2085130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 2086130f4520SKenneth D. Merry { 2087130f4520SKenneth D. Merry struct nameidata nd; 2088130f4520SKenneth D. Merry int flags; 2089130f4520SKenneth D. Merry int error; 2090130f4520SKenneth D. Merry 2091130f4520SKenneth D. Merry /* 2092130f4520SKenneth D. Merry * XXX KDM allow a read-only option? 2093130f4520SKenneth D. Merry */ 2094130f4520SKenneth D. Merry flags = FREAD | FWRITE; 2095130f4520SKenneth D. Merry error = 0; 2096130f4520SKenneth D. Merry 2097130f4520SKenneth D. Merry if (rootvnode == NULL) { 2098130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 209919720f41SAlexander Motin "Root filesystem is not mounted"); 2100130f4520SKenneth D. Merry return (1); 2101130f4520SKenneth D. Merry } 2102130f4520SKenneth D. Merry 21038a08cec1SMateusz Guzik pwd_ensure_dirs(); 2104130f4520SKenneth D. Merry 2105130f4520SKenneth D. Merry again: 2106130f4520SKenneth D. Merry NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread); 2107130f4520SKenneth D. Merry error = vn_open(&nd, &flags, 0, NULL); 2108130f4520SKenneth D. Merry if (error) { 2109130f4520SKenneth D. Merry /* 2110130f4520SKenneth D. Merry * This is the only reasonable guess we can make as far as 2111130f4520SKenneth D. Merry * path if the user doesn't give us a fully qualified path. 2112130f4520SKenneth D. Merry * If they want to specify a file, they need to specify the 2113130f4520SKenneth D. Merry * full path. 2114130f4520SKenneth D. Merry */ 2115130f4520SKenneth D. Merry if (be_lun->dev_path[0] != '/') { 2116130f4520SKenneth D. Merry char *dev_path = "/dev/"; 2117130f4520SKenneth D. Merry char *dev_name; 2118130f4520SKenneth D. Merry 2119130f4520SKenneth D. Merry /* Try adding device path at beginning of name */ 2120130f4520SKenneth D. Merry dev_name = malloc(strlen(be_lun->dev_path) 2121130f4520SKenneth D. Merry + strlen(dev_path) + 1, 2122130f4520SKenneth D. Merry M_CTLBLK, M_WAITOK); 2123130f4520SKenneth D. Merry if (dev_name) { 2124130f4520SKenneth D. Merry sprintf(dev_name, "%s%s", dev_path, 2125130f4520SKenneth D. Merry be_lun->dev_path); 2126130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 2127130f4520SKenneth D. Merry be_lun->dev_path = dev_name; 2128130f4520SKenneth D. Merry goto again; 2129130f4520SKenneth D. Merry } 2130130f4520SKenneth D. Merry } 2131130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 213219720f41SAlexander Motin "error opening %s: %d", be_lun->dev_path, error); 2133130f4520SKenneth D. Merry return (error); 2134130f4520SKenneth D. Merry } 2135130f4520SKenneth D. Merry 2136130f4520SKenneth D. Merry NDFREE(&nd, NDF_ONLY_PNBUF); 2137130f4520SKenneth D. Merry 2138130f4520SKenneth D. Merry be_lun->vn = nd.ni_vp; 2139130f4520SKenneth D. Merry 2140130f4520SKenneth D. Merry /* We only support disks and files. */ 2141130f4520SKenneth D. Merry if (vn_isdisk(be_lun->vn, &error)) { 2142130f4520SKenneth D. Merry error = ctl_be_block_open_dev(be_lun, req); 2143130f4520SKenneth D. Merry } else if (be_lun->vn->v_type == VREG) { 2144130f4520SKenneth D. Merry error = ctl_be_block_open_file(be_lun, req); 2145130f4520SKenneth D. Merry } else { 2146130f4520SKenneth D. Merry error = EINVAL; 2147130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 2148025a2301SEdward Tomasz Napierala "%s is not a disk or plain file", be_lun->dev_path); 2149130f4520SKenneth D. Merry } 2150130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 2151130f4520SKenneth D. Merry 2152*a15bbf15SAlexander Motin if (error != 0) 2153130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 2154130f4520SKenneth D. Merry return (0); 2155130f4520SKenneth D. Merry } 2156130f4520SKenneth D. Merry 2157130f4520SKenneth D. Merry static int 2158130f4520SKenneth D. Merry ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2159130f4520SKenneth D. Merry { 2160130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2161130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 216257a5db13SAlexander Motin char num_thread_str[16]; 2163130f4520SKenneth D. Merry char tmpstr[32]; 216457a5db13SAlexander Motin char *value; 2165fbc8d4ffSAlexander Motin int retval, num_threads; 216657a5db13SAlexander Motin int tmp_num_threads; 2167130f4520SKenneth D. Merry 2168130f4520SKenneth D. Merry params = &req->reqdata.create; 2169130f4520SKenneth D. Merry retval = 0; 217019720f41SAlexander Motin req->status = CTL_LUN_OK; 2171130f4520SKenneth D. Merry 2172130f4520SKenneth D. Merry num_threads = cbb_num_threads; 2173130f4520SKenneth D. Merry 2174130f4520SKenneth D. Merry be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK); 2175130f4520SKenneth D. Merry 217619720f41SAlexander Motin be_lun->params = req->reqdata.create; 2177130f4520SKenneth D. Merry be_lun->softc = softc; 2178130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->input_queue); 2179ef8daf3fSAlexander Motin STAILQ_INIT(&be_lun->config_read_queue); 2180130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->config_write_queue); 2181130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->datamove_queue); 2182130f4520SKenneth D. Merry sprintf(be_lun->lunname, "cblk%d", softc->num_luns); 218375c7a1d3SAlexander Motin mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF); 218475c7a1d3SAlexander Motin mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF); 218543fb3a65SAlexander Motin ctl_init_opts(&be_lun->ctl_be_lun.options, 218643fb3a65SAlexander Motin req->num_be_args, req->kern_be_args); 2187130f4520SKenneth D. Merry 218808a7cce5SAlexander Motin be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG, 2189eeb94054SAlexander Motin NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0); 2190130f4520SKenneth D. Merry 2191130f4520SKenneth D. Merry if (be_lun->lun_zone == NULL) { 2192130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 219319720f41SAlexander Motin "error allocating UMA zone"); 2194130f4520SKenneth D. Merry goto bailout_error; 2195130f4520SKenneth D. Merry } 2196130f4520SKenneth D. Merry 2197130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_DEV_TYPE) 2198130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type = params->device_type; 2199130f4520SKenneth D. Merry else 2200130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type = T_DIRECT; 2201130f4520SKenneth D. Merry 2202130f4520SKenneth D. Merry if (be_lun->ctl_be_lun.lun_type == T_DIRECT) { 220343fb3a65SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "file"); 2204eb3687a6SAlexander Motin if (value == NULL) { 2205130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 220619720f41SAlexander Motin "no file argument specified"); 2207130f4520SKenneth D. Merry goto bailout_error; 2208130f4520SKenneth D. Merry } 2209eb3687a6SAlexander Motin be_lun->dev_path = strdup(value, M_CTLBLK); 2210*a15bbf15SAlexander Motin be_lun->size_bytes = params->lun_size_bytes; 2211*a15bbf15SAlexander Motin if (params->blocksize_bytes != 0) 2212*a15bbf15SAlexander Motin be_lun->blocksize = params->blocksize_bytes; 2213*a15bbf15SAlexander Motin else 221419720f41SAlexander Motin be_lun->blocksize = 512; 2215130f4520SKenneth D. Merry 2216130f4520SKenneth D. Merry retval = ctl_be_block_open(softc, be_lun, req); 2217*a15bbf15SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / be_lun->blocksize; 2218130f4520SKenneth D. Merry if (retval != 0) { 2219130f4520SKenneth D. Merry retval = 0; 222019720f41SAlexander Motin req->status = CTL_LUN_WARNING; 2221130f4520SKenneth D. Merry } 2222130f4520SKenneth D. Merry } else { 2223130f4520SKenneth D. Merry /* 2224130f4520SKenneth D. Merry * For processor devices, we don't have any size. 2225130f4520SKenneth D. Merry */ 2226130f4520SKenneth D. Merry be_lun->blocksize = 0; 2227f6012722SAlexander Motin be_lun->pblockexp = 0; 2228f6012722SAlexander Motin be_lun->pblockoff = 0; 222934961f40SAlexander Motin be_lun->ublockexp = 0; 223034961f40SAlexander Motin be_lun->ublockoff = 0; 2231130f4520SKenneth D. Merry be_lun->size_blocks = 0; 2232130f4520SKenneth D. Merry be_lun->size_bytes = 0; 2233130f4520SKenneth D. Merry be_lun->ctl_be_lun.maxlba = 0; 2234130f4520SKenneth D. Merry 2235130f4520SKenneth D. Merry /* 2236130f4520SKenneth D. Merry * Default to just 1 thread for processor devices. 2237130f4520SKenneth D. Merry */ 2238130f4520SKenneth D. Merry num_threads = 1; 2239130f4520SKenneth D. Merry } 2240130f4520SKenneth D. Merry 2241130f4520SKenneth D. Merry /* 2242130f4520SKenneth D. Merry * XXX This searching loop might be refactored to be combined with 2243130f4520SKenneth D. Merry * the loop above, 2244130f4520SKenneth D. Merry */ 224543fb3a65SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "num_threads"); 224657a5db13SAlexander Motin if (value != NULL) { 224757a5db13SAlexander Motin tmp_num_threads = strtol(value, NULL, 0); 2248130f4520SKenneth D. Merry 2249130f4520SKenneth D. Merry /* 2250130f4520SKenneth D. Merry * We don't let the user specify less than one 2251130f4520SKenneth D. Merry * thread, but hope he's clueful enough not to 2252130f4520SKenneth D. Merry * specify 1000 threads. 2253130f4520SKenneth D. Merry */ 2254130f4520SKenneth D. Merry if (tmp_num_threads < 1) { 2255130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 225619720f41SAlexander Motin "invalid number of threads %s", 225719720f41SAlexander Motin num_thread_str); 2258130f4520SKenneth D. Merry goto bailout_error; 2259130f4520SKenneth D. Merry } 2260130f4520SKenneth D. Merry num_threads = tmp_num_threads; 226157a5db13SAlexander Motin } 2262130f4520SKenneth D. Merry 2263130f4520SKenneth D. Merry be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED; 2264130f4520SKenneth D. Merry be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY; 226519720f41SAlexander Motin if (be_lun->vn == NULL) 226619720f41SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_OFFLINE; 2267fbc8d4ffSAlexander Motin if (be_lun->unmap != NULL) 2268ee7f31c0SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP; 2269bfbfc4a3SAlexander Motin if (be_lun->dispatch != ctl_be_block_dispatch_dev) 2270bfbfc4a3SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_SERSEQ_READ; 2271130f4520SKenneth D. Merry be_lun->ctl_be_lun.be_lun = be_lun; 227219720f41SAlexander Motin be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 227319720f41SAlexander Motin 0 : (be_lun->size_blocks - 1); 2274130f4520SKenneth D. Merry be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 2275f6012722SAlexander Motin be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 2276f6012722SAlexander Motin be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 227734961f40SAlexander Motin be_lun->ctl_be_lun.ublockexp = be_lun->ublockexp; 227834961f40SAlexander Motin be_lun->ctl_be_lun.ublockoff = be_lun->ublockoff; 2279cb8727e2SAlexander Motin be_lun->ctl_be_lun.atomicblock = be_lun->atomicblock; 2280cb8727e2SAlexander Motin be_lun->ctl_be_lun.opttxferlen = be_lun->opttxferlen; 2281130f4520SKenneth D. Merry /* Tell the user the blocksize we ended up using */ 228219720f41SAlexander Motin params->lun_size_bytes = be_lun->size_bytes; 2283130f4520SKenneth D. Merry params->blocksize_bytes = be_lun->blocksize; 2284130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_ID_REQ) { 2285130f4520SKenneth D. Merry be_lun->ctl_be_lun.req_lun_id = params->req_lun_id; 2286130f4520SKenneth D. Merry be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ; 2287130f4520SKenneth D. Merry } else 2288130f4520SKenneth D. Merry be_lun->ctl_be_lun.req_lun_id = 0; 2289130f4520SKenneth D. Merry 2290130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown; 2291130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_config_status = 2292130f4520SKenneth D. Merry ctl_be_block_lun_config_status; 2293130f4520SKenneth D. Merry be_lun->ctl_be_lun.be = &ctl_be_block_driver; 2294130f4520SKenneth D. Merry 2295130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { 2296130f4520SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d", 2297130f4520SKenneth D. Merry softc->num_luns); 2298130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr, 2299e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.serial_num), 2300130f4520SKenneth D. Merry sizeof(tmpstr))); 2301130f4520SKenneth D. Merry 2302130f4520SKenneth D. Merry /* Tell the user what we used for a serial number */ 2303130f4520SKenneth D. Merry strncpy((char *)params->serial_num, tmpstr, 2304e7038eb7SAlexander Motin MIN(sizeof(params->serial_num), sizeof(tmpstr))); 2305130f4520SKenneth D. Merry } else { 2306130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.serial_num, 2307130f4520SKenneth D. Merry params->serial_num, 2308e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.serial_num), 2309130f4520SKenneth D. Merry sizeof(params->serial_num))); 2310130f4520SKenneth D. Merry } 2311130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { 2312130f4520SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns); 2313130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr, 2314e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.device_id), 2315130f4520SKenneth D. Merry sizeof(tmpstr))); 2316130f4520SKenneth D. Merry 2317130f4520SKenneth D. Merry /* Tell the user what we used for a device ID */ 2318130f4520SKenneth D. Merry strncpy((char *)params->device_id, tmpstr, 2319e7038eb7SAlexander Motin MIN(sizeof(params->device_id), sizeof(tmpstr))); 2320130f4520SKenneth D. Merry } else { 2321130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.device_id, 2322130f4520SKenneth D. Merry params->device_id, 2323e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.device_id), 2324130f4520SKenneth D. Merry sizeof(params->device_id))); 2325130f4520SKenneth D. Merry } 2326130f4520SKenneth D. Merry 2327130f4520SKenneth D. Merry TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); 2328130f4520SKenneth D. Merry 2329130f4520SKenneth D. Merry be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK, 2330130f4520SKenneth D. Merry taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); 2331130f4520SKenneth D. Merry 2332130f4520SKenneth D. Merry if (be_lun->io_taskqueue == NULL) { 2333130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 233419720f41SAlexander Motin "unable to create taskqueue"); 2335130f4520SKenneth D. Merry goto bailout_error; 2336130f4520SKenneth D. Merry } 2337130f4520SKenneth D. Merry 2338130f4520SKenneth D. Merry /* 2339130f4520SKenneth D. Merry * Note that we start the same number of threads by default for 2340130f4520SKenneth D. Merry * both the file case and the block device case. For the file 2341130f4520SKenneth D. Merry * case, we need multiple threads to allow concurrency, because the 2342130f4520SKenneth D. Merry * vnode interface is designed to be a blocking interface. For the 2343130f4520SKenneth D. Merry * block device case, ZFS zvols at least will block the caller's 2344130f4520SKenneth D. Merry * context in many instances, and so we need multiple threads to 2345130f4520SKenneth D. Merry * overcome that problem. Other block devices don't need as many 2346130f4520SKenneth D. Merry * threads, but they shouldn't cause too many problems. 2347130f4520SKenneth D. Merry * 2348130f4520SKenneth D. Merry * If the user wants to just have a single thread for a block 2349130f4520SKenneth D. Merry * device, he can specify that when the LUN is created, or change 2350130f4520SKenneth D. Merry * the tunable/sysctl to alter the default number of threads. 2351130f4520SKenneth D. Merry */ 2352130f4520SKenneth D. Merry retval = taskqueue_start_threads(&be_lun->io_taskqueue, 2353130f4520SKenneth D. Merry /*num threads*/num_threads, 2354130f4520SKenneth D. Merry /*priority*/PWAIT, 2355130f4520SKenneth D. Merry /*thread name*/ 2356130f4520SKenneth D. Merry "%s taskq", be_lun->lunname); 2357130f4520SKenneth D. Merry 2358130f4520SKenneth D. Merry if (retval != 0) 2359130f4520SKenneth D. Merry goto bailout_error; 2360130f4520SKenneth D. Merry 2361130f4520SKenneth D. Merry be_lun->num_threads = num_threads; 2362130f4520SKenneth D. Merry 2363130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2364130f4520SKenneth D. Merry softc->num_luns++; 2365130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links); 2366130f4520SKenneth D. Merry 2367130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2368130f4520SKenneth D. Merry 2369130f4520SKenneth D. Merry retval = ctl_add_lun(&be_lun->ctl_be_lun); 2370130f4520SKenneth D. Merry if (retval != 0) { 2371130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2372130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2373130f4520SKenneth D. Merry links); 2374130f4520SKenneth D. Merry softc->num_luns--; 2375130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2376130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 237719720f41SAlexander Motin "ctl_add_lun() returned error %d, see dmesg for " 237819720f41SAlexander Motin "details", retval); 2379130f4520SKenneth D. Merry retval = 0; 2380130f4520SKenneth D. Merry goto bailout_error; 2381130f4520SKenneth D. Merry } 2382130f4520SKenneth D. Merry 2383130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2384130f4520SKenneth D. Merry 2385130f4520SKenneth D. Merry /* 2386130f4520SKenneth D. Merry * Tell the config_status routine that we're waiting so it won't 2387130f4520SKenneth D. Merry * clean up the LUN in the event of an error. 2388130f4520SKenneth D. Merry */ 2389130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2390130f4520SKenneth D. Merry 2391130f4520SKenneth D. Merry while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { 2392130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2393130f4520SKenneth D. Merry if (retval == EINTR) 2394130f4520SKenneth D. Merry break; 2395130f4520SKenneth D. Merry } 2396130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2397130f4520SKenneth D. Merry 2398130f4520SKenneth D. Merry if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) { 2399130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 240019720f41SAlexander Motin "LUN configuration error, see dmesg for details"); 2401130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2402130f4520SKenneth D. Merry links); 2403130f4520SKenneth D. Merry softc->num_luns--; 2404130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2405130f4520SKenneth D. Merry goto bailout_error; 2406130f4520SKenneth D. Merry } else { 2407130f4520SKenneth D. Merry params->req_lun_id = be_lun->ctl_be_lun.lun_id; 2408130f4520SKenneth D. Merry } 2409130f4520SKenneth D. Merry 2410130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2411130f4520SKenneth D. Merry 2412130f4520SKenneth D. Merry be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id, 2413130f4520SKenneth D. Merry be_lun->blocksize, 2414130f4520SKenneth D. Merry DEVSTAT_ALL_SUPPORTED, 2415130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type 2416130f4520SKenneth D. Merry | DEVSTAT_TYPE_IF_OTHER, 2417130f4520SKenneth D. Merry DEVSTAT_PRIORITY_OTHER); 2418130f4520SKenneth D. Merry 2419130f4520SKenneth D. Merry return (retval); 2420130f4520SKenneth D. Merry 2421130f4520SKenneth D. Merry bailout_error: 2422130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2423130f4520SKenneth D. Merry 24249e005bbcSAlexander Motin if (be_lun->io_taskqueue != NULL) 24259e005bbcSAlexander Motin taskqueue_free(be_lun->io_taskqueue); 2426130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 24279e005bbcSAlexander Motin if (be_lun->dev_path != NULL) 2428130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 24299e005bbcSAlexander Motin if (be_lun->lun_zone != NULL) 24309e005bbcSAlexander Motin uma_zdestroy(be_lun->lun_zone); 243143fb3a65SAlexander Motin ctl_free_opts(&be_lun->ctl_be_lun.options); 243275c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 243375c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2434130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2435130f4520SKenneth D. Merry 2436130f4520SKenneth D. Merry return (retval); 2437130f4520SKenneth D. Merry } 2438130f4520SKenneth D. Merry 2439130f4520SKenneth D. Merry static int 2440130f4520SKenneth D. Merry ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2441130f4520SKenneth D. Merry { 2442130f4520SKenneth D. Merry struct ctl_lun_rm_params *params; 2443130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2444130f4520SKenneth D. Merry int retval; 2445130f4520SKenneth D. Merry 2446130f4520SKenneth D. Merry params = &req->reqdata.rm; 2447130f4520SKenneth D. Merry 2448130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2449130f4520SKenneth D. Merry 2450130f4520SKenneth D. Merry be_lun = NULL; 2451130f4520SKenneth D. Merry 2452130f4520SKenneth D. Merry STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 2453130f4520SKenneth D. Merry if (be_lun->ctl_be_lun.lun_id == params->lun_id) 2454130f4520SKenneth D. Merry break; 2455130f4520SKenneth D. Merry } 2456130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2457130f4520SKenneth D. Merry 2458130f4520SKenneth D. Merry if (be_lun == NULL) { 2459130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 246019720f41SAlexander Motin "LUN %u is not managed by the block backend", 246119720f41SAlexander Motin params->lun_id); 2462130f4520SKenneth D. Merry goto bailout_error; 2463130f4520SKenneth D. Merry } 2464130f4520SKenneth D. Merry 2465130f4520SKenneth D. Merry retval = ctl_disable_lun(&be_lun->ctl_be_lun); 2466130f4520SKenneth D. Merry 2467130f4520SKenneth D. Merry if (retval != 0) { 2468130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 246919720f41SAlexander Motin "error %d returned from ctl_disable_lun() for " 247019720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2471130f4520SKenneth D. Merry goto bailout_error; 2472130f4520SKenneth D. Merry 2473130f4520SKenneth D. Merry } 2474130f4520SKenneth D. Merry 2475130f4520SKenneth D. Merry retval = ctl_invalidate_lun(&be_lun->ctl_be_lun); 2476130f4520SKenneth D. Merry if (retval != 0) { 2477130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 247819720f41SAlexander Motin "error %d returned from ctl_invalidate_lun() for " 247919720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2480130f4520SKenneth D. Merry goto bailout_error; 2481130f4520SKenneth D. Merry } 2482130f4520SKenneth D. Merry 2483130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2484130f4520SKenneth D. Merry 2485130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2486130f4520SKenneth D. Merry 2487130f4520SKenneth D. Merry while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2488130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2489130f4520SKenneth D. Merry if (retval == EINTR) 2490130f4520SKenneth D. Merry break; 2491130f4520SKenneth D. Merry } 2492130f4520SKenneth D. Merry 2493130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2494130f4520SKenneth D. Merry 2495130f4520SKenneth D. Merry if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2496130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 249719720f41SAlexander Motin "interrupted waiting for LUN to be freed"); 2498130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2499130f4520SKenneth D. Merry goto bailout_error; 2500130f4520SKenneth D. Merry } 2501130f4520SKenneth D. Merry 2502130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links); 2503130f4520SKenneth D. Merry 2504130f4520SKenneth D. Merry softc->num_luns--; 2505130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2506130f4520SKenneth D. Merry 2507130f4520SKenneth D. Merry taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task); 2508130f4520SKenneth D. Merry 2509130f4520SKenneth D. Merry taskqueue_free(be_lun->io_taskqueue); 2510130f4520SKenneth D. Merry 2511130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 2512130f4520SKenneth D. Merry 2513130f4520SKenneth D. Merry if (be_lun->disk_stats != NULL) 2514130f4520SKenneth D. Merry devstat_remove_entry(be_lun->disk_stats); 2515130f4520SKenneth D. Merry 2516130f4520SKenneth D. Merry uma_zdestroy(be_lun->lun_zone); 2517130f4520SKenneth D. Merry 251843fb3a65SAlexander Motin ctl_free_opts(&be_lun->ctl_be_lun.options); 2519130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 252075c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 252175c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2522130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2523130f4520SKenneth D. Merry 2524130f4520SKenneth D. Merry req->status = CTL_LUN_OK; 2525130f4520SKenneth D. Merry 2526130f4520SKenneth D. Merry return (0); 2527130f4520SKenneth D. Merry 2528130f4520SKenneth D. Merry bailout_error: 2529130f4520SKenneth D. Merry 2530130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2531130f4520SKenneth D. Merry 2532130f4520SKenneth D. Merry return (0); 2533130f4520SKenneth D. Merry } 2534130f4520SKenneth D. Merry 253581177295SEdward Tomasz Napierala static int 253681177295SEdward Tomasz Napierala ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 253781177295SEdward Tomasz Napierala struct ctl_lun_req *req) 253881177295SEdward Tomasz Napierala { 253981177295SEdward Tomasz Napierala struct vattr vattr; 254081177295SEdward Tomasz Napierala int error; 254119720f41SAlexander Motin struct ctl_lun_create_params *params = &be_lun->params; 254281177295SEdward Tomasz Napierala 254381177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 254481177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 254581177295SEdward Tomasz Napierala } else { 254671d8e97eSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 254781177295SEdward Tomasz Napierala error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 254871d8e97eSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 254981177295SEdward Tomasz Napierala if (error != 0) { 255081177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 255181177295SEdward Tomasz Napierala "error calling VOP_GETATTR() for file %s", 255281177295SEdward Tomasz Napierala be_lun->dev_path); 255381177295SEdward Tomasz Napierala return (error); 255481177295SEdward Tomasz Napierala } 255581177295SEdward Tomasz Napierala 255681177295SEdward Tomasz Napierala be_lun->size_bytes = vattr.va_size; 255781177295SEdward Tomasz Napierala } 255881177295SEdward Tomasz Napierala 255981177295SEdward Tomasz Napierala return (0); 256081177295SEdward Tomasz Napierala } 256181177295SEdward Tomasz Napierala 256281177295SEdward Tomasz Napierala static int 256381177295SEdward Tomasz Napierala ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 256481177295SEdward Tomasz Napierala struct ctl_lun_req *req) 256581177295SEdward Tomasz Napierala { 256671d8e97eSAlexander Motin struct ctl_be_block_devdata *dev_data; 256781177295SEdward Tomasz Napierala int error; 256819720f41SAlexander Motin struct ctl_lun_create_params *params = &be_lun->params; 256981177295SEdward Tomasz Napierala uint64_t size_bytes; 257081177295SEdward Tomasz Napierala 257171d8e97eSAlexander Motin dev_data = &be_lun->backend.dev; 257271d8e97eSAlexander Motin if (!dev_data->csw->d_ioctl) { 257381177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 257419720f41SAlexander Motin "no d_ioctl for device %s!", be_lun->dev_path); 257581177295SEdward Tomasz Napierala return (ENODEV); 257681177295SEdward Tomasz Napierala } 257781177295SEdward Tomasz Napierala 257871d8e97eSAlexander Motin error = dev_data->csw->d_ioctl(dev_data->cdev, DIOCGMEDIASIZE, 257981177295SEdward Tomasz Napierala (caddr_t)&size_bytes, FREAD, 258081177295SEdward Tomasz Napierala curthread); 258181177295SEdward Tomasz Napierala if (error) { 258281177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 258319720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE ioctl " 258419720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 258581177295SEdward Tomasz Napierala return (error); 258681177295SEdward Tomasz Napierala } 258781177295SEdward Tomasz Napierala 258881177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 258981177295SEdward Tomasz Napierala if (params->lun_size_bytes > size_bytes) { 259081177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 259119720f41SAlexander Motin "requested LUN size %ju > backing device " 259219720f41SAlexander Motin "size %ju", 259381177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 259481177295SEdward Tomasz Napierala (uintmax_t)size_bytes); 259581177295SEdward Tomasz Napierala return (EINVAL); 259681177295SEdward Tomasz Napierala } 259781177295SEdward Tomasz Napierala 259881177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 259981177295SEdward Tomasz Napierala } else { 260081177295SEdward Tomasz Napierala be_lun->size_bytes = size_bytes; 260181177295SEdward Tomasz Napierala } 260281177295SEdward Tomasz Napierala 260381177295SEdward Tomasz Napierala return (0); 260481177295SEdward Tomasz Napierala } 260581177295SEdward Tomasz Napierala 260681177295SEdward Tomasz Napierala static int 260781177295SEdward Tomasz Napierala ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 260881177295SEdward Tomasz Napierala { 260981177295SEdward Tomasz Napierala struct ctl_lun_modify_params *params; 261081177295SEdward Tomasz Napierala struct ctl_be_block_lun *be_lun; 261171d8e97eSAlexander Motin uint64_t oldsize; 26125050aa86SKonstantin Belousov int error; 261381177295SEdward Tomasz Napierala 261481177295SEdward Tomasz Napierala params = &req->reqdata.modify; 261581177295SEdward Tomasz Napierala 261681177295SEdward Tomasz Napierala mtx_lock(&softc->lock); 261781177295SEdward Tomasz Napierala be_lun = NULL; 261881177295SEdward Tomasz Napierala STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 261981177295SEdward Tomasz Napierala if (be_lun->ctl_be_lun.lun_id == params->lun_id) 262081177295SEdward Tomasz Napierala break; 262181177295SEdward Tomasz Napierala } 262281177295SEdward Tomasz Napierala mtx_unlock(&softc->lock); 262381177295SEdward Tomasz Napierala 262481177295SEdward Tomasz Napierala if (be_lun == NULL) { 262581177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 262619720f41SAlexander Motin "LUN %u is not managed by the block backend", 262719720f41SAlexander Motin params->lun_id); 262881177295SEdward Tomasz Napierala goto bailout_error; 262981177295SEdward Tomasz Napierala } 263081177295SEdward Tomasz Napierala 263119720f41SAlexander Motin be_lun->params.lun_size_bytes = params->lun_size_bytes; 263281177295SEdward Tomasz Napierala 26333f829b0cSAlexander Motin oldsize = be_lun->size_bytes; 263419720f41SAlexander Motin if (be_lun->vn == NULL) 263519720f41SAlexander Motin error = ctl_be_block_open(softc, be_lun, req); 2636b9b4269cSAlexander Motin else if (vn_isdisk(be_lun->vn, &error)) 2637b9b4269cSAlexander Motin error = ctl_be_block_modify_dev(be_lun, req); 263819720f41SAlexander Motin else if (be_lun->vn->v_type == VREG) 263981177295SEdward Tomasz Napierala error = ctl_be_block_modify_file(be_lun, req); 264081177295SEdward Tomasz Napierala else 2641b9b4269cSAlexander Motin error = EINVAL; 2642*a15bbf15SAlexander Motin be_lun->size_blocks = be_lun->size_bytes / be_lun->blocksize; 264381177295SEdward Tomasz Napierala 26443f829b0cSAlexander Motin if (error == 0 && be_lun->size_bytes != oldsize) { 264581177295SEdward Tomasz Napierala 264681177295SEdward Tomasz Napierala /* 264781177295SEdward Tomasz Napierala * The maximum LBA is the size - 1. 264881177295SEdward Tomasz Napierala * 264981177295SEdward Tomasz Napierala * XXX: Note that this field is being updated without locking, 265081177295SEdward Tomasz Napierala * which might cause problems on 32-bit architectures. 265181177295SEdward Tomasz Napierala */ 2652fbc8d4ffSAlexander Motin if (be_lun->unmap != NULL) 2653fbc8d4ffSAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP; 265419720f41SAlexander Motin be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 265519720f41SAlexander Motin 0 : (be_lun->size_blocks - 1); 265619720f41SAlexander Motin be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 265719720f41SAlexander Motin be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 265819720f41SAlexander Motin be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 265934961f40SAlexander Motin be_lun->ctl_be_lun.ublockexp = be_lun->ublockexp; 266034961f40SAlexander Motin be_lun->ctl_be_lun.ublockoff = be_lun->ublockoff; 2661cb8727e2SAlexander Motin be_lun->ctl_be_lun.atomicblock = be_lun->atomicblock; 2662cb8727e2SAlexander Motin be_lun->ctl_be_lun.opttxferlen = be_lun->opttxferlen; 266381177295SEdward Tomasz Napierala ctl_lun_capacity_changed(&be_lun->ctl_be_lun); 266419720f41SAlexander Motin if (oldsize == 0 && be_lun->size_blocks != 0) 266519720f41SAlexander Motin ctl_lun_online(&be_lun->ctl_be_lun); 266671d8e97eSAlexander Motin } 266781177295SEdward Tomasz Napierala 266881177295SEdward Tomasz Napierala /* Tell the user the exact size we ended up using */ 266981177295SEdward Tomasz Napierala params->lun_size_bytes = be_lun->size_bytes; 267081177295SEdward Tomasz Napierala 267119720f41SAlexander Motin req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; 267281177295SEdward Tomasz Napierala 267381177295SEdward Tomasz Napierala return (0); 267481177295SEdward Tomasz Napierala 267581177295SEdward Tomasz Napierala bailout_error: 267681177295SEdward Tomasz Napierala req->status = CTL_LUN_ERROR; 267781177295SEdward Tomasz Napierala 267881177295SEdward Tomasz Napierala return (0); 267981177295SEdward Tomasz Napierala } 268081177295SEdward Tomasz Napierala 2681130f4520SKenneth D. Merry static void 2682130f4520SKenneth D. Merry ctl_be_block_lun_shutdown(void *be_lun) 2683130f4520SKenneth D. Merry { 2684130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2685130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2686130f4520SKenneth D. Merry 2687130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2688130f4520SKenneth D. Merry 2689130f4520SKenneth D. Merry softc = lun->softc; 2690130f4520SKenneth D. Merry 2691130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2692130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; 2693130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2694130f4520SKenneth D. Merry wakeup(lun); 2695130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2696130f4520SKenneth D. Merry 2697130f4520SKenneth D. Merry } 2698130f4520SKenneth D. Merry 2699130f4520SKenneth D. Merry static void 2700130f4520SKenneth D. Merry ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status) 2701130f4520SKenneth D. Merry { 2702130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2703130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2704130f4520SKenneth D. Merry 2705130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2706130f4520SKenneth D. Merry softc = lun->softc; 2707130f4520SKenneth D. Merry 2708130f4520SKenneth D. Merry if (status == CTL_LUN_CONFIG_OK) { 2709130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2710130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2711130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2712130f4520SKenneth D. Merry wakeup(lun); 2713130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2714130f4520SKenneth D. Merry 2715130f4520SKenneth D. Merry /* 2716130f4520SKenneth D. Merry * We successfully added the LUN, attempt to enable it. 2717130f4520SKenneth D. Merry */ 2718130f4520SKenneth D. Merry if (ctl_enable_lun(&lun->ctl_be_lun) != 0) { 2719130f4520SKenneth D. Merry printf("%s: ctl_enable_lun() failed!\n", __func__); 2720130f4520SKenneth D. Merry if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) { 2721130f4520SKenneth D. Merry printf("%s: ctl_invalidate_lun() failed!\n", 2722130f4520SKenneth D. Merry __func__); 2723130f4520SKenneth D. Merry } 2724130f4520SKenneth D. Merry } 2725130f4520SKenneth D. Merry 2726130f4520SKenneth D. Merry return; 2727130f4520SKenneth D. Merry } 2728130f4520SKenneth D. Merry 2729130f4520SKenneth D. Merry 2730130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2731130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2732130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR; 2733130f4520SKenneth D. Merry wakeup(lun); 2734130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2735130f4520SKenneth D. Merry } 2736130f4520SKenneth D. Merry 2737130f4520SKenneth D. Merry 2738130f4520SKenneth D. Merry static int 2739130f4520SKenneth D. Merry ctl_be_block_config_write(union ctl_io *io) 2740130f4520SKenneth D. Merry { 2741130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2742130f4520SKenneth D. Merry struct ctl_be_lun *ctl_be_lun; 2743130f4520SKenneth D. Merry int retval; 2744130f4520SKenneth D. Merry 2745130f4520SKenneth D. Merry retval = 0; 2746130f4520SKenneth D. Merry 2747130f4520SKenneth D. Merry DPRINTF("entered\n"); 2748130f4520SKenneth D. Merry 2749130f4520SKenneth D. Merry ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2750130f4520SKenneth D. Merry CTL_PRIV_BACKEND_LUN].ptr; 2751130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2752130f4520SKenneth D. Merry 2753130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 2754130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 2755130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 2756ee7f31c0SAlexander Motin case WRITE_SAME_10: 2757ee7f31c0SAlexander Motin case WRITE_SAME_16: 2758ee7f31c0SAlexander Motin case UNMAP: 2759130f4520SKenneth D. Merry /* 2760130f4520SKenneth D. Merry * The upper level CTL code will filter out any CDBs with 2761130f4520SKenneth D. Merry * the immediate bit set and return the proper error. 2762130f4520SKenneth D. Merry * 2763130f4520SKenneth D. Merry * We don't really need to worry about what LBA range the 2764130f4520SKenneth D. Merry * user asked to be synced out. When they issue a sync 2765130f4520SKenneth D. Merry * cache command, we'll sync out the whole thing. 2766130f4520SKenneth D. Merry */ 276775c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 2768130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, 2769130f4520SKenneth D. Merry links); 277075c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 2771130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 2772130f4520SKenneth D. Merry break; 2773130f4520SKenneth D. Merry case START_STOP_UNIT: { 2774130f4520SKenneth D. Merry struct scsi_start_stop_unit *cdb; 2775130f4520SKenneth D. Merry 2776130f4520SKenneth D. Merry cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; 2777130f4520SKenneth D. Merry 2778130f4520SKenneth D. Merry if (cdb->how & SSS_START) 2779130f4520SKenneth D. Merry retval = ctl_start_lun(ctl_be_lun); 2780130f4520SKenneth D. Merry else { 2781130f4520SKenneth D. Merry retval = ctl_stop_lun(ctl_be_lun); 2782130f4520SKenneth D. Merry /* 2783130f4520SKenneth D. Merry * XXX KDM Copan-specific offline behavior. 2784130f4520SKenneth D. Merry * Figure out a reasonable way to port this? 2785130f4520SKenneth D. Merry */ 2786130f4520SKenneth D. Merry #ifdef NEEDTOPORT 2787130f4520SKenneth D. Merry if ((retval == 0) 2788130f4520SKenneth D. Merry && (cdb->byte2 & SSS_ONOFFLINE)) 2789130f4520SKenneth D. Merry retval = ctl_lun_offline(ctl_be_lun); 2790130f4520SKenneth D. Merry #endif 2791130f4520SKenneth D. Merry } 2792130f4520SKenneth D. Merry 2793130f4520SKenneth D. Merry /* 2794130f4520SKenneth D. Merry * In general, the above routines should not fail. They 2795130f4520SKenneth D. Merry * just set state for the LUN. So we've got something 2796130f4520SKenneth D. Merry * pretty wrong here if we can't start or stop the LUN. 2797130f4520SKenneth D. Merry */ 2798130f4520SKenneth D. Merry if (retval != 0) { 2799130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 2800130f4520SKenneth D. Merry /*sks_valid*/ 1, 2801130f4520SKenneth D. Merry /*retry_count*/ 0xf051); 2802130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2803130f4520SKenneth D. Merry } else { 2804130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 2805130f4520SKenneth D. Merry } 2806130f4520SKenneth D. Merry ctl_config_write_done(io); 2807130f4520SKenneth D. Merry break; 2808130f4520SKenneth D. Merry } 2809130f4520SKenneth D. Merry default: 2810130f4520SKenneth D. Merry ctl_set_invalid_opcode(&io->scsiio); 2811130f4520SKenneth D. Merry ctl_config_write_done(io); 2812130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2813130f4520SKenneth D. Merry break; 2814130f4520SKenneth D. Merry } 2815130f4520SKenneth D. Merry 2816130f4520SKenneth D. Merry return (retval); 2817130f4520SKenneth D. Merry } 2818130f4520SKenneth D. Merry 2819130f4520SKenneth D. Merry static int 2820130f4520SKenneth D. Merry ctl_be_block_config_read(union ctl_io *io) 2821130f4520SKenneth D. Merry { 2822ef8daf3fSAlexander Motin struct ctl_be_block_lun *be_lun; 2823ef8daf3fSAlexander Motin struct ctl_be_lun *ctl_be_lun; 2824ef8daf3fSAlexander Motin int retval = 0; 2825ef8daf3fSAlexander Motin 2826ef8daf3fSAlexander Motin DPRINTF("entered\n"); 2827ef8daf3fSAlexander Motin 2828ef8daf3fSAlexander Motin ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2829ef8daf3fSAlexander Motin CTL_PRIV_BACKEND_LUN].ptr; 2830ef8daf3fSAlexander Motin be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2831ef8daf3fSAlexander Motin 2832ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 2833ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: 2834ef8daf3fSAlexander Motin if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { 2835ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 2836ef8daf3fSAlexander Motin STAILQ_INSERT_TAIL(&be_lun->config_read_queue, 2837ef8daf3fSAlexander Motin &io->io_hdr, links); 2838ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 2839ef8daf3fSAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, 2840ef8daf3fSAlexander Motin &be_lun->io_task); 2841ef8daf3fSAlexander Motin retval = CTL_RETVAL_QUEUED; 2842ef8daf3fSAlexander Motin break; 2843ef8daf3fSAlexander Motin } 2844ef8daf3fSAlexander Motin ctl_set_invalid_field(&io->scsiio, 2845ef8daf3fSAlexander Motin /*sks_valid*/ 1, 2846ef8daf3fSAlexander Motin /*command*/ 1, 2847ef8daf3fSAlexander Motin /*field*/ 1, 2848ef8daf3fSAlexander Motin /*bit_valid*/ 1, 2849ef8daf3fSAlexander Motin /*bit*/ 4); 2850ef8daf3fSAlexander Motin ctl_config_read_done(io); 2851ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2852ef8daf3fSAlexander Motin break; 2853ef8daf3fSAlexander Motin default: 2854ef8daf3fSAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 2855ef8daf3fSAlexander Motin ctl_config_read_done(io); 2856ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2857ef8daf3fSAlexander Motin break; 2858ef8daf3fSAlexander Motin } 2859ef8daf3fSAlexander Motin 2860ef8daf3fSAlexander Motin return (retval); 2861130f4520SKenneth D. Merry } 2862130f4520SKenneth D. Merry 2863130f4520SKenneth D. Merry static int 2864130f4520SKenneth D. Merry ctl_be_block_lun_info(void *be_lun, struct sbuf *sb) 2865130f4520SKenneth D. Merry { 2866130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2867130f4520SKenneth D. Merry int retval; 2868130f4520SKenneth D. Merry 2869130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2870130f4520SKenneth D. Merry retval = 0; 2871130f4520SKenneth D. Merry 28722cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "\t<num_threads>"); 2873130f4520SKenneth D. Merry 2874130f4520SKenneth D. Merry if (retval != 0) 2875130f4520SKenneth D. Merry goto bailout; 2876130f4520SKenneth D. Merry 2877130f4520SKenneth D. Merry retval = sbuf_printf(sb, "%d", lun->num_threads); 2878130f4520SKenneth D. Merry 2879130f4520SKenneth D. Merry if (retval != 0) 2880130f4520SKenneth D. Merry goto bailout; 2881130f4520SKenneth D. Merry 28822cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "</num_threads>\n"); 2883130f4520SKenneth D. Merry 2884130f4520SKenneth D. Merry bailout: 2885130f4520SKenneth D. Merry 2886130f4520SKenneth D. Merry return (retval); 2887130f4520SKenneth D. Merry } 2888130f4520SKenneth D. Merry 2889c3e7ba3eSAlexander Motin static uint64_t 2890c3e7ba3eSAlexander Motin ctl_be_block_lun_attr(void *be_lun, const char *attrname) 2891c3e7ba3eSAlexander Motin { 2892c3e7ba3eSAlexander Motin struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun; 2893c3e7ba3eSAlexander Motin 2894c3e7ba3eSAlexander Motin if (lun->getattr == NULL) 2895c3e7ba3eSAlexander Motin return (UINT64_MAX); 2896c3e7ba3eSAlexander Motin return (lun->getattr(lun, attrname)); 2897c3e7ba3eSAlexander Motin } 2898c3e7ba3eSAlexander Motin 2899130f4520SKenneth D. Merry int 2900130f4520SKenneth D. Merry ctl_be_block_init(void) 2901130f4520SKenneth D. Merry { 2902130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2903130f4520SKenneth D. Merry int retval; 2904130f4520SKenneth D. Merry 2905130f4520SKenneth D. Merry softc = &backend_block_softc; 2906130f4520SKenneth D. Merry retval = 0; 2907130f4520SKenneth D. Merry 290875c7a1d3SAlexander Motin mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); 2909a0e36aeeSEdward Tomasz Napierala beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), 2910a0e36aeeSEdward Tomasz Napierala NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 2911130f4520SKenneth D. Merry STAILQ_INIT(&softc->disk_list); 2912130f4520SKenneth D. Merry STAILQ_INIT(&softc->lun_list); 2913130f4520SKenneth D. Merry 2914130f4520SKenneth D. Merry return (retval); 2915130f4520SKenneth D. Merry } 2916