1130f4520SKenneth D. Merry /*- 2130f4520SKenneth D. Merry * Copyright (c) 2003 Silicon Graphics International Corp. 3130f4520SKenneth D. Merry * Copyright (c) 2009-2011 Spectra Logic Corporation 481177295SEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 5130f4520SKenneth D. Merry * All rights reserved. 6130f4520SKenneth D. Merry * 781177295SEdward Tomasz Napierala * Portions of this software were developed by Edward Tomasz Napierala 881177295SEdward Tomasz Napierala * under sponsorship from the FreeBSD Foundation. 981177295SEdward Tomasz Napierala * 10130f4520SKenneth D. Merry * Redistribution and use in source and binary forms, with or without 11130f4520SKenneth D. Merry * modification, are permitted provided that the following conditions 12130f4520SKenneth D. Merry * are met: 13130f4520SKenneth D. Merry * 1. Redistributions of source code must retain the above copyright 14130f4520SKenneth D. Merry * notice, this list of conditions, and the following disclaimer, 15130f4520SKenneth D. Merry * without modification. 16130f4520SKenneth D. Merry * 2. Redistributions in binary form must reproduce at minimum a disclaimer 17130f4520SKenneth D. Merry * substantially similar to the "NO WARRANTY" disclaimer below 18130f4520SKenneth D. Merry * ("Disclaimer") and any redistribution must be conditioned upon 19130f4520SKenneth D. Merry * including a substantially similar Disclaimer requirement for further 20130f4520SKenneth D. Merry * binary redistribution. 21130f4520SKenneth D. Merry * 22130f4520SKenneth D. Merry * NO WARRANTY 23130f4520SKenneth D. Merry * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 24130f4520SKenneth D. Merry * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 25130f4520SKenneth D. Merry * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 26130f4520SKenneth D. Merry * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 27130f4520SKenneth D. Merry * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28130f4520SKenneth D. Merry * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29130f4520SKenneth D. Merry * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30130f4520SKenneth D. Merry * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 31130f4520SKenneth D. Merry * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 32130f4520SKenneth D. Merry * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33130f4520SKenneth D. Merry * POSSIBILITY OF SUCH DAMAGES. 34130f4520SKenneth D. Merry * 35130f4520SKenneth D. Merry * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $ 36130f4520SKenneth D. Merry */ 37130f4520SKenneth D. Merry /* 38130f4520SKenneth D. Merry * CAM Target Layer driver backend for block devices. 39130f4520SKenneth D. Merry * 40130f4520SKenneth D. Merry * Author: Ken Merry <ken@FreeBSD.org> 41130f4520SKenneth D. Merry */ 42130f4520SKenneth D. Merry #include <sys/cdefs.h> 43130f4520SKenneth D. Merry __FBSDID("$FreeBSD$"); 44130f4520SKenneth D. Merry 45130f4520SKenneth D. Merry #include <sys/param.h> 46130f4520SKenneth D. Merry #include <sys/systm.h> 47130f4520SKenneth D. Merry #include <sys/kernel.h> 48130f4520SKenneth D. Merry #include <sys/types.h> 49130f4520SKenneth D. Merry #include <sys/kthread.h> 50130f4520SKenneth D. Merry #include <sys/bio.h> 51130f4520SKenneth D. Merry #include <sys/fcntl.h> 52ee7f31c0SAlexander Motin #include <sys/limits.h> 53130f4520SKenneth D. Merry #include <sys/lock.h> 54130f4520SKenneth D. Merry #include <sys/mutex.h> 55130f4520SKenneth D. Merry #include <sys/condvar.h> 56130f4520SKenneth D. Merry #include <sys/malloc.h> 57130f4520SKenneth D. Merry #include <sys/conf.h> 58130f4520SKenneth D. Merry #include <sys/ioccom.h> 59130f4520SKenneth D. Merry #include <sys/queue.h> 60130f4520SKenneth D. Merry #include <sys/sbuf.h> 61130f4520SKenneth D. Merry #include <sys/endian.h> 62130f4520SKenneth D. Merry #include <sys/uio.h> 63130f4520SKenneth D. Merry #include <sys/buf.h> 64130f4520SKenneth D. Merry #include <sys/taskqueue.h> 65130f4520SKenneth D. Merry #include <sys/vnode.h> 66130f4520SKenneth D. Merry #include <sys/namei.h> 67130f4520SKenneth D. Merry #include <sys/mount.h> 68130f4520SKenneth D. Merry #include <sys/disk.h> 69130f4520SKenneth D. Merry #include <sys/fcntl.h> 70130f4520SKenneth D. Merry #include <sys/filedesc.h> 71ef8daf3fSAlexander Motin #include <sys/filio.h> 72130f4520SKenneth D. Merry #include <sys/proc.h> 73130f4520SKenneth D. Merry #include <sys/pcpu.h> 74130f4520SKenneth D. Merry #include <sys/module.h> 75130f4520SKenneth D. Merry #include <sys/sdt.h> 76130f4520SKenneth D. Merry #include <sys/devicestat.h> 77130f4520SKenneth D. Merry #include <sys/sysctl.h> 78130f4520SKenneth D. Merry 79130f4520SKenneth D. Merry #include <geom/geom.h> 80130f4520SKenneth D. Merry 81130f4520SKenneth D. Merry #include <cam/cam.h> 82130f4520SKenneth D. Merry #include <cam/scsi/scsi_all.h> 83130f4520SKenneth D. Merry #include <cam/scsi/scsi_da.h> 84130f4520SKenneth D. Merry #include <cam/ctl/ctl_io.h> 85130f4520SKenneth D. Merry #include <cam/ctl/ctl.h> 86130f4520SKenneth D. Merry #include <cam/ctl/ctl_backend.h> 87130f4520SKenneth D. Merry #include <cam/ctl/ctl_frontend_internal.h> 88130f4520SKenneth D. Merry #include <cam/ctl/ctl_ioctl.h> 89130f4520SKenneth D. Merry #include <cam/ctl/ctl_scsi_all.h> 90130f4520SKenneth D. Merry #include <cam/ctl/ctl_error.h> 91130f4520SKenneth D. Merry 92130f4520SKenneth D. Merry /* 9308a7cce5SAlexander Motin * The idea here is that we'll allocate enough S/G space to hold a 1MB 9408a7cce5SAlexander Motin * I/O. If we get an I/O larger than that, we'll split it. 95130f4520SKenneth D. Merry */ 9611b569f7SAlexander Motin #define CTLBLK_HALF_IO_SIZE (512 * 1024) 9711b569f7SAlexander Motin #define CTLBLK_MAX_IO_SIZE (CTLBLK_HALF_IO_SIZE * 2) 9808a7cce5SAlexander Motin #define CTLBLK_MAX_SEG MAXPHYS 9911b569f7SAlexander Motin #define CTLBLK_HALF_SEGS MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1) 10011b569f7SAlexander Motin #define CTLBLK_MAX_SEGS (CTLBLK_HALF_SEGS * 2) 101130f4520SKenneth D. Merry 102130f4520SKenneth D. Merry #ifdef CTLBLK_DEBUG 103130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) \ 104130f4520SKenneth D. Merry printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) 105130f4520SKenneth D. Merry #else 106130f4520SKenneth D. Merry #define DPRINTF(fmt, args...) do {} while(0) 107130f4520SKenneth D. Merry #endif 108130f4520SKenneth D. Merry 109e86a4142SAlexander Motin #define PRIV(io) \ 110e86a4142SAlexander Motin ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND]) 11111b569f7SAlexander Motin #define ARGS(io) \ 11211b569f7SAlexander Motin ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]) 113e86a4142SAlexander Motin 114130f4520SKenneth D. Merry SDT_PROVIDER_DEFINE(cbb); 115130f4520SKenneth D. Merry 116130f4520SKenneth D. Merry typedef enum { 117130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_UNCONFIGURED = 0x01, 118130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_CONFIG_ERR = 0x02, 119130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_WAITING = 0x04, 120130f4520SKenneth D. Merry CTL_BE_BLOCK_LUN_MULTI_THREAD = 0x08 121130f4520SKenneth D. Merry } ctl_be_block_lun_flags; 122130f4520SKenneth D. Merry 123130f4520SKenneth D. Merry typedef enum { 124130f4520SKenneth D. Merry CTL_BE_BLOCK_NONE, 125130f4520SKenneth D. Merry CTL_BE_BLOCK_DEV, 126130f4520SKenneth D. Merry CTL_BE_BLOCK_FILE 127130f4520SKenneth D. Merry } ctl_be_block_type; 128130f4520SKenneth D. Merry 129130f4520SKenneth D. Merry struct ctl_be_block_devdata { 130130f4520SKenneth D. Merry struct cdev *cdev; 131130f4520SKenneth D. Merry struct cdevsw *csw; 132130f4520SKenneth D. Merry int dev_ref; 133130f4520SKenneth D. Merry }; 134130f4520SKenneth D. Merry 135130f4520SKenneth D. Merry struct ctl_be_block_filedata { 136130f4520SKenneth D. Merry struct ucred *cred; 137130f4520SKenneth D. Merry }; 138130f4520SKenneth D. Merry 139130f4520SKenneth D. Merry union ctl_be_block_bedata { 140130f4520SKenneth D. Merry struct ctl_be_block_devdata dev; 141130f4520SKenneth D. Merry struct ctl_be_block_filedata file; 142130f4520SKenneth D. Merry }; 143130f4520SKenneth D. Merry 144130f4520SKenneth D. Merry struct ctl_be_block_io; 145130f4520SKenneth D. Merry struct ctl_be_block_lun; 146130f4520SKenneth D. Merry 147130f4520SKenneth D. Merry typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun, 148130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 149c3e7ba3eSAlexander Motin typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun, 150c3e7ba3eSAlexander Motin const char *attrname); 151130f4520SKenneth D. Merry 152130f4520SKenneth D. Merry /* 153130f4520SKenneth D. Merry * Backend LUN structure. There is a 1:1 mapping between a block device 154130f4520SKenneth D. Merry * and a backend block LUN, and between a backend block LUN and a CTL LUN. 155130f4520SKenneth D. Merry */ 156130f4520SKenneth D. Merry struct ctl_be_block_lun { 15719720f41SAlexander Motin struct ctl_lun_create_params params; 158130f4520SKenneth D. Merry struct ctl_block_disk *disk; 159130f4520SKenneth D. Merry char lunname[32]; 160130f4520SKenneth D. Merry char *dev_path; 161130f4520SKenneth D. Merry ctl_be_block_type dev_type; 162130f4520SKenneth D. Merry struct vnode *vn; 163130f4520SKenneth D. Merry union ctl_be_block_bedata backend; 164130f4520SKenneth D. Merry cbb_dispatch_t dispatch; 165130f4520SKenneth D. Merry cbb_dispatch_t lun_flush; 166ee7f31c0SAlexander Motin cbb_dispatch_t unmap; 167ef8daf3fSAlexander Motin cbb_dispatch_t get_lba_status; 168c3e7ba3eSAlexander Motin cbb_getattr_t getattr; 169130f4520SKenneth D. Merry uma_zone_t lun_zone; 170130f4520SKenneth D. Merry uint64_t size_blocks; 171130f4520SKenneth D. Merry uint64_t size_bytes; 172130f4520SKenneth D. Merry uint32_t blocksize; 173130f4520SKenneth D. Merry int blocksize_shift; 174f6012722SAlexander Motin uint16_t pblockexp; 175f6012722SAlexander Motin uint16_t pblockoff; 17634961f40SAlexander Motin uint16_t ublockexp; 17734961f40SAlexander Motin uint16_t ublockoff; 178cb8727e2SAlexander Motin uint32_t atomicblock; 179cb8727e2SAlexander Motin uint32_t opttxferlen; 180130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 181130f4520SKenneth D. Merry struct devstat *disk_stats; 182130f4520SKenneth D. Merry ctl_be_block_lun_flags flags; 183130f4520SKenneth D. Merry STAILQ_ENTRY(ctl_be_block_lun) links; 184130f4520SKenneth D. Merry struct ctl_be_lun ctl_be_lun; 185130f4520SKenneth D. Merry struct taskqueue *io_taskqueue; 186130f4520SKenneth D. Merry struct task io_task; 187130f4520SKenneth D. Merry int num_threads; 188130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) input_queue; 189ef8daf3fSAlexander Motin STAILQ_HEAD(, ctl_io_hdr) config_read_queue; 190130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) config_write_queue; 191130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 19275c7a1d3SAlexander Motin struct mtx_padalign io_lock; 19375c7a1d3SAlexander Motin struct mtx_padalign queue_lock; 194130f4520SKenneth D. Merry }; 195130f4520SKenneth D. Merry 196130f4520SKenneth D. Merry /* 197130f4520SKenneth D. Merry * Overall softc structure for the block backend module. 198130f4520SKenneth D. Merry */ 199130f4520SKenneth D. Merry struct ctl_be_block_softc { 200130f4520SKenneth D. Merry struct mtx lock; 201130f4520SKenneth D. Merry int num_disks; 202130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_block_disk) disk_list; 203130f4520SKenneth D. Merry int num_luns; 204130f4520SKenneth D. Merry STAILQ_HEAD(, ctl_be_block_lun) lun_list; 205130f4520SKenneth D. Merry }; 206130f4520SKenneth D. Merry 207130f4520SKenneth D. Merry static struct ctl_be_block_softc backend_block_softc; 208130f4520SKenneth D. Merry 209130f4520SKenneth D. Merry /* 210130f4520SKenneth D. Merry * Per-I/O information. 211130f4520SKenneth D. Merry */ 212130f4520SKenneth D. Merry struct ctl_be_block_io { 213130f4520SKenneth D. Merry union ctl_io *io; 214130f4520SKenneth D. Merry struct ctl_sg_entry sg_segs[CTLBLK_MAX_SEGS]; 215130f4520SKenneth D. Merry struct iovec xiovecs[CTLBLK_MAX_SEGS]; 216130f4520SKenneth D. Merry int bio_cmd; 217130f4520SKenneth D. Merry int num_segs; 218130f4520SKenneth D. Merry int num_bios_sent; 219130f4520SKenneth D. Merry int num_bios_done; 220130f4520SKenneth D. Merry int send_complete; 221130f4520SKenneth D. Merry int num_errors; 222130f4520SKenneth D. Merry struct bintime ds_t0; 223130f4520SKenneth D. Merry devstat_tag_type ds_tag_type; 224130f4520SKenneth D. Merry devstat_trans_flags ds_trans_type; 225130f4520SKenneth D. Merry uint64_t io_len; 226130f4520SKenneth D. Merry uint64_t io_offset; 227130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 228130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 229ee7f31c0SAlexander Motin void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */ 230130f4520SKenneth D. Merry }; 231130f4520SKenneth D. Merry 232130f4520SKenneth D. Merry static int cbb_num_threads = 14; 233130f4520SKenneth D. Merry SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0, 234130f4520SKenneth D. Merry "CAM Target Layer Block Backend"); 235af3b2549SHans Petter Selasky SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, 236130f4520SKenneth D. Merry &cbb_num_threads, 0, "Number of threads per backing file"); 237130f4520SKenneth D. Merry 238130f4520SKenneth D. Merry static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); 239130f4520SKenneth D. Merry static void ctl_free_beio(struct ctl_be_block_io *beio); 240130f4520SKenneth D. Merry static void ctl_complete_beio(struct ctl_be_block_io *beio); 241130f4520SKenneth D. Merry static int ctl_be_block_move_done(union ctl_io *io); 242130f4520SKenneth D. Merry static void ctl_be_block_biodone(struct bio *bio); 243130f4520SKenneth D. Merry static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 244130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 245130f4520SKenneth D. Merry static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 246130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 247ef8daf3fSAlexander Motin static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 248ef8daf3fSAlexander Motin struct ctl_be_block_io *beio); 24953c146deSAlexander Motin static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, 25053c146deSAlexander Motin const char *attrname); 251130f4520SKenneth D. Merry static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 252130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 253ee7f31c0SAlexander Motin static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 254ee7f31c0SAlexander Motin struct ctl_be_block_io *beio); 255130f4520SKenneth D. Merry static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 256130f4520SKenneth D. Merry struct ctl_be_block_io *beio); 257c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, 258c3e7ba3eSAlexander Motin const char *attrname); 259ef8daf3fSAlexander Motin static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 260ef8daf3fSAlexander Motin union ctl_io *io); 261130f4520SKenneth D. Merry static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 262130f4520SKenneth D. Merry union ctl_io *io); 263130f4520SKenneth D. Merry static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 264130f4520SKenneth D. Merry union ctl_io *io); 265130f4520SKenneth D. Merry static void ctl_be_block_worker(void *context, int pending); 266130f4520SKenneth D. Merry static int ctl_be_block_submit(union ctl_io *io); 267130f4520SKenneth D. Merry static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 268130f4520SKenneth D. Merry int flag, struct thread *td); 269130f4520SKenneth D. Merry static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, 270130f4520SKenneth D. Merry struct ctl_lun_req *req); 271130f4520SKenneth D. Merry static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, 272130f4520SKenneth D. Merry struct ctl_lun_req *req); 273130f4520SKenneth D. Merry static int ctl_be_block_close(struct ctl_be_block_lun *be_lun); 274130f4520SKenneth D. Merry static int ctl_be_block_open(struct ctl_be_block_softc *softc, 275130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun, 276130f4520SKenneth D. Merry struct ctl_lun_req *req); 277130f4520SKenneth D. Merry static int ctl_be_block_create(struct ctl_be_block_softc *softc, 278130f4520SKenneth D. Merry struct ctl_lun_req *req); 279130f4520SKenneth D. Merry static int ctl_be_block_rm(struct ctl_be_block_softc *softc, 280130f4520SKenneth D. Merry struct ctl_lun_req *req); 28181177295SEdward Tomasz Napierala static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 28281177295SEdward Tomasz Napierala struct ctl_lun_req *req); 28381177295SEdward Tomasz Napierala static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 28481177295SEdward Tomasz Napierala struct ctl_lun_req *req); 28581177295SEdward Tomasz Napierala static int ctl_be_block_modify(struct ctl_be_block_softc *softc, 28681177295SEdward Tomasz Napierala struct ctl_lun_req *req); 287130f4520SKenneth D. Merry static void ctl_be_block_lun_shutdown(void *be_lun); 288130f4520SKenneth D. Merry static void ctl_be_block_lun_config_status(void *be_lun, 289130f4520SKenneth D. Merry ctl_lun_config_status status); 290130f4520SKenneth D. Merry static int ctl_be_block_config_write(union ctl_io *io); 291130f4520SKenneth D. Merry static int ctl_be_block_config_read(union ctl_io *io); 292130f4520SKenneth D. Merry static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); 293c3e7ba3eSAlexander Motin static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); 294130f4520SKenneth D. Merry int ctl_be_block_init(void); 295130f4520SKenneth D. Merry 296130f4520SKenneth D. Merry static struct ctl_backend_driver ctl_be_block_driver = 297130f4520SKenneth D. Merry { 2982a2443d8SKenneth D. Merry .name = "block", 2992a2443d8SKenneth D. Merry .flags = CTL_BE_FLAG_HAS_CONFIG, 3002a2443d8SKenneth D. Merry .init = ctl_be_block_init, 3012a2443d8SKenneth D. Merry .data_submit = ctl_be_block_submit, 3022a2443d8SKenneth D. Merry .data_move_done = ctl_be_block_move_done, 3032a2443d8SKenneth D. Merry .config_read = ctl_be_block_config_read, 3042a2443d8SKenneth D. Merry .config_write = ctl_be_block_config_write, 3052a2443d8SKenneth D. Merry .ioctl = ctl_be_block_ioctl, 306c3e7ba3eSAlexander Motin .lun_info = ctl_be_block_lun_info, 307c3e7ba3eSAlexander Motin .lun_attr = ctl_be_block_lun_attr 308130f4520SKenneth D. Merry }; 309130f4520SKenneth D. Merry 310130f4520SKenneth D. Merry MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); 311130f4520SKenneth D. Merry CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); 312130f4520SKenneth D. Merry 313a0e36aeeSEdward Tomasz Napierala static uma_zone_t beio_zone; 314a0e36aeeSEdward Tomasz Napierala 315130f4520SKenneth D. Merry static struct ctl_be_block_io * 316130f4520SKenneth D. Merry ctl_alloc_beio(struct ctl_be_block_softc *softc) 317130f4520SKenneth D. Merry { 318130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 319130f4520SKenneth D. Merry 320a0e36aeeSEdward Tomasz Napierala beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO); 321130f4520SKenneth D. Merry beio->softc = softc; 322130f4520SKenneth D. Merry return (beio); 323130f4520SKenneth D. Merry } 324130f4520SKenneth D. Merry 325130f4520SKenneth D. Merry static void 326130f4520SKenneth D. Merry ctl_free_beio(struct ctl_be_block_io *beio) 327130f4520SKenneth D. Merry { 328130f4520SKenneth D. Merry int duplicate_free; 329130f4520SKenneth D. Merry int i; 330130f4520SKenneth D. Merry 331130f4520SKenneth D. Merry duplicate_free = 0; 332130f4520SKenneth D. Merry 333130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 334130f4520SKenneth D. Merry if (beio->sg_segs[i].addr == NULL) 335130f4520SKenneth D. Merry duplicate_free++; 336130f4520SKenneth D. Merry 337130f4520SKenneth D. Merry uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr); 338130f4520SKenneth D. Merry beio->sg_segs[i].addr = NULL; 33911b569f7SAlexander Motin 34011b569f7SAlexander Motin /* For compare we had two equal S/G lists. */ 34111b569f7SAlexander Motin if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) { 34211b569f7SAlexander Motin uma_zfree(beio->lun->lun_zone, 34311b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr); 34411b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL; 34511b569f7SAlexander Motin } 346130f4520SKenneth D. Merry } 347130f4520SKenneth D. Merry 348130f4520SKenneth D. Merry if (duplicate_free > 0) { 349130f4520SKenneth D. Merry printf("%s: %d duplicate frees out of %d segments\n", __func__, 350130f4520SKenneth D. Merry duplicate_free, beio->num_segs); 351130f4520SKenneth D. Merry } 352a0e36aeeSEdward Tomasz Napierala 353a0e36aeeSEdward Tomasz Napierala uma_zfree(beio_zone, beio); 354130f4520SKenneth D. Merry } 355130f4520SKenneth D. Merry 356130f4520SKenneth D. Merry static void 357130f4520SKenneth D. Merry ctl_complete_beio(struct ctl_be_block_io *beio) 358130f4520SKenneth D. Merry { 35975c7a1d3SAlexander Motin union ctl_io *io = beio->io; 360130f4520SKenneth D. Merry 361ee7f31c0SAlexander Motin if (beio->beio_cont != NULL) { 362ee7f31c0SAlexander Motin beio->beio_cont(beio); 363ee7f31c0SAlexander Motin } else { 364130f4520SKenneth D. Merry ctl_free_beio(beio); 36511b569f7SAlexander Motin ctl_data_submit_done(io); 366130f4520SKenneth D. Merry } 367ee7f31c0SAlexander Motin } 368130f4520SKenneth D. Merry 369130f4520SKenneth D. Merry static int 370130f4520SKenneth D. Merry ctl_be_block_move_done(union ctl_io *io) 371130f4520SKenneth D. Merry { 372130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 373130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 37411b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 375130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 376130f4520SKenneth D. Merry struct bintime cur_bt; 377130f4520SKenneth D. Merry #endif 37811b569f7SAlexander Motin int i; 379130f4520SKenneth D. Merry 380e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 381130f4520SKenneth D. Merry be_lun = beio->lun; 382130f4520SKenneth D. Merry 383130f4520SKenneth D. Merry DPRINTF("entered\n"); 384130f4520SKenneth D. Merry 385130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 386130f4520SKenneth D. Merry getbintime(&cur_bt); 387130f4520SKenneth D. Merry bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); 388130f4520SKenneth D. Merry bintime_add(&io->io_hdr.dma_bt, &cur_bt); 389130f4520SKenneth D. Merry io->io_hdr.num_dmas++; 390130f4520SKenneth D. Merry #endif 39111b569f7SAlexander Motin io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; 392130f4520SKenneth D. Merry 393130f4520SKenneth D. Merry /* 394130f4520SKenneth D. Merry * We set status at this point for read commands, and write 395130f4520SKenneth D. Merry * commands with errors. 396130f4520SKenneth D. Merry */ 397f7241cceSAlexander Motin if (io->io_hdr.flags & CTL_FLAG_ABORT) { 398f7241cceSAlexander Motin ; 399f7241cceSAlexander Motin } else if ((io->io_hdr.port_status == 0) && 40011b569f7SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { 40111b569f7SAlexander Motin lbalen = ARGS(beio->io); 40211b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_READ) { 403130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 40411b569f7SAlexander Motin } else if (lbalen->flags & CTL_LLF_COMPARE) { 40511b569f7SAlexander Motin /* We have two data blocks ready for comparison. */ 40611b569f7SAlexander Motin for (i = 0; i < beio->num_segs; i++) { 40711b569f7SAlexander Motin if (memcmp(beio->sg_segs[i].addr, 40811b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, 40911b569f7SAlexander Motin beio->sg_segs[i].len) != 0) 41011b569f7SAlexander Motin break; 41111b569f7SAlexander Motin } 41211b569f7SAlexander Motin if (i < beio->num_segs) 41311b569f7SAlexander Motin ctl_set_sense(&io->scsiio, 41411b569f7SAlexander Motin /*current_error*/ 1, 41511b569f7SAlexander Motin /*sense_key*/ SSD_KEY_MISCOMPARE, 41611b569f7SAlexander Motin /*asc*/ 0x1D, 41711b569f7SAlexander Motin /*ascq*/ 0x00, 41811b569f7SAlexander Motin SSD_ELEM_NONE); 41911b569f7SAlexander Motin else 42011b569f7SAlexander Motin ctl_set_success(&io->scsiio); 42111b569f7SAlexander Motin } 422f7241cceSAlexander Motin } else if ((io->io_hdr.port_status != 0) && 423f7241cceSAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 424f7241cceSAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 425130f4520SKenneth D. Merry /* 426130f4520SKenneth D. Merry * For hardware error sense keys, the sense key 427130f4520SKenneth D. Merry * specific value is defined to be a retry count, 428130f4520SKenneth D. Merry * but we use it to pass back an internal FETD 429130f4520SKenneth D. Merry * error code. XXX KDM Hopefully the FETD is only 430130f4520SKenneth D. Merry * using 16 bits for an error code, since that's 431130f4520SKenneth D. Merry * all the space we have in the sks field. 432130f4520SKenneth D. Merry */ 433130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 434130f4520SKenneth D. Merry /*sks_valid*/ 1, 435130f4520SKenneth D. Merry /*retry_count*/ 436130f4520SKenneth D. Merry io->io_hdr.port_status); 437130f4520SKenneth D. Merry } 438130f4520SKenneth D. Merry 439130f4520SKenneth D. Merry /* 440130f4520SKenneth D. Merry * If this is a read, or a write with errors, it is done. 441130f4520SKenneth D. Merry */ 442130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_READ) 443130f4520SKenneth D. Merry || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) 444130f4520SKenneth D. Merry || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { 445130f4520SKenneth D. Merry ctl_complete_beio(beio); 446130f4520SKenneth D. Merry return (0); 447130f4520SKenneth D. Merry } 448130f4520SKenneth D. Merry 449130f4520SKenneth D. Merry /* 450130f4520SKenneth D. Merry * At this point, we have a write and the DMA completed 451130f4520SKenneth D. Merry * successfully. We now have to queue it to the task queue to 452130f4520SKenneth D. Merry * execute the backend I/O. That is because we do blocking 453130f4520SKenneth D. Merry * memory allocations, and in the file backing case, blocking I/O. 454130f4520SKenneth D. Merry * This move done routine is generally called in the SIM's 455130f4520SKenneth D. Merry * interrupt context, and therefore we cannot block. 456130f4520SKenneth D. Merry */ 45775c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 458130f4520SKenneth D. Merry /* 459130f4520SKenneth D. Merry * XXX KDM make sure that links is okay to use at this point. 460130f4520SKenneth D. Merry * Otherwise, we either need to add another field to ctl_io_hdr, 461130f4520SKenneth D. Merry * or deal with resource allocation here. 462130f4520SKenneth D. Merry */ 463130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); 46475c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 465130f4520SKenneth D. Merry 466130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 467130f4520SKenneth D. Merry 468130f4520SKenneth D. Merry return (0); 469130f4520SKenneth D. Merry } 470130f4520SKenneth D. Merry 471130f4520SKenneth D. Merry static void 472130f4520SKenneth D. Merry ctl_be_block_biodone(struct bio *bio) 473130f4520SKenneth D. Merry { 474130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 475130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 476130f4520SKenneth D. Merry union ctl_io *io; 477e0c2f975SAlexander Motin int error; 478130f4520SKenneth D. Merry 479130f4520SKenneth D. Merry beio = bio->bio_caller1; 480130f4520SKenneth D. Merry be_lun = beio->lun; 481130f4520SKenneth D. Merry io = beio->io; 482130f4520SKenneth D. Merry 483130f4520SKenneth D. Merry DPRINTF("entered\n"); 484130f4520SKenneth D. Merry 485e0c2f975SAlexander Motin error = bio->bio_error; 48675c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 487e0c2f975SAlexander Motin if (error != 0) 488130f4520SKenneth D. Merry beio->num_errors++; 489130f4520SKenneth D. Merry 490130f4520SKenneth D. Merry beio->num_bios_done++; 491130f4520SKenneth D. Merry 492130f4520SKenneth D. Merry /* 493130f4520SKenneth D. Merry * XXX KDM will this cause WITNESS to complain? Holding a lock 494130f4520SKenneth D. Merry * during the free might cause it to complain. 495130f4520SKenneth D. Merry */ 496130f4520SKenneth D. Merry g_destroy_bio(bio); 497130f4520SKenneth D. Merry 498130f4520SKenneth D. Merry /* 499130f4520SKenneth D. Merry * If the send complete bit isn't set, or we aren't the last I/O to 500130f4520SKenneth D. Merry * complete, then we're done. 501130f4520SKenneth D. Merry */ 502130f4520SKenneth D. Merry if ((beio->send_complete == 0) 503130f4520SKenneth D. Merry || (beio->num_bios_done < beio->num_bios_sent)) { 50475c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 505130f4520SKenneth D. Merry return; 506130f4520SKenneth D. Merry } 507130f4520SKenneth D. Merry 508130f4520SKenneth D. Merry /* 509130f4520SKenneth D. Merry * At this point, we've verified that we are the last I/O to 510130f4520SKenneth D. Merry * complete, so it's safe to drop the lock. 511130f4520SKenneth D. Merry */ 51275c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 51375c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 51475c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 51575c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 516130f4520SKenneth D. Merry 517130f4520SKenneth D. Merry /* 518130f4520SKenneth D. Merry * If there are any errors from the backing device, we fail the 519130f4520SKenneth D. Merry * entire I/O with a medium error. 520130f4520SKenneth D. Merry */ 521130f4520SKenneth D. Merry if (beio->num_errors > 0) { 522e0c2f975SAlexander Motin if (error == EOPNOTSUPP) { 523e0c2f975SAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 5240631de4aSAlexander Motin } else if (error == ENOSPC || error == EDQUOT) { 5254fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 526e0c2f975SAlexander Motin } else if (beio->bio_cmd == BIO_FLUSH) { 527130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 528130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 529130f4520SKenneth D. Merry /*sks_valid*/ 1, 530130f4520SKenneth D. Merry /*retry_count*/ 0xbad2); 531130f4520SKenneth D. Merry } else 532130f4520SKenneth D. Merry ctl_set_medium_error(&io->scsiio); 533130f4520SKenneth D. Merry ctl_complete_beio(beio); 534130f4520SKenneth D. Merry return; 535130f4520SKenneth D. Merry } 536130f4520SKenneth D. Merry 537130f4520SKenneth D. Merry /* 53811b569f7SAlexander Motin * If this is a write, a flush, a delete or verify, we're all done. 539130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 540130f4520SKenneth D. Merry */ 541130f4520SKenneth D. Merry if ((beio->bio_cmd == BIO_WRITE) 542ee7f31c0SAlexander Motin || (beio->bio_cmd == BIO_FLUSH) 54311b569f7SAlexander Motin || (beio->bio_cmd == BIO_DELETE) 54411b569f7SAlexander Motin || (ARGS(io)->flags & CTL_LLF_VERIFY)) { 545130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 546130f4520SKenneth D. Merry ctl_complete_beio(beio); 547130f4520SKenneth D. Merry } else { 548f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 549f7241cceSAlexander Motin beio->beio_cont == NULL) 550f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 551130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 552130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 553130f4520SKenneth D. Merry #endif 554130f4520SKenneth D. Merry ctl_datamove(io); 555130f4520SKenneth D. Merry } 556130f4520SKenneth D. Merry } 557130f4520SKenneth D. Merry 558130f4520SKenneth D. Merry static void 559130f4520SKenneth D. Merry ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 560130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 561130f4520SKenneth D. Merry { 56275c7a1d3SAlexander Motin union ctl_io *io = beio->io; 563130f4520SKenneth D. Merry struct mount *mountpoint; 5645050aa86SKonstantin Belousov int error, lock_flags; 565130f4520SKenneth D. Merry 566130f4520SKenneth D. Merry DPRINTF("entered\n"); 567130f4520SKenneth D. Merry 56875c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 56975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 57075c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 57175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 572130f4520SKenneth D. Merry 573130f4520SKenneth D. Merry (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 574130f4520SKenneth D. Merry 575130f4520SKenneth D. Merry if (MNT_SHARED_WRITES(mountpoint) 576130f4520SKenneth D. Merry || ((mountpoint == NULL) 577130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 578130f4520SKenneth D. Merry lock_flags = LK_SHARED; 579130f4520SKenneth D. Merry else 580130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 581130f4520SKenneth D. Merry 582130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 583130f4520SKenneth D. Merry 584130f4520SKenneth D. Merry error = VOP_FSYNC(be_lun->vn, MNT_WAIT, curthread); 585130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 586130f4520SKenneth D. Merry 587130f4520SKenneth D. Merry vn_finished_write(mountpoint); 588130f4520SKenneth D. Merry 58975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 59075c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 59175c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 59275c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 59375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 59475c7a1d3SAlexander Motin 595130f4520SKenneth D. Merry if (error == 0) 596130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 597130f4520SKenneth D. Merry else { 598130f4520SKenneth D. Merry /* XXX KDM is there is a better error here? */ 599130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 600130f4520SKenneth D. Merry /*sks_valid*/ 1, 601130f4520SKenneth D. Merry /*retry_count*/ 0xbad1); 602130f4520SKenneth D. Merry } 603130f4520SKenneth D. Merry 604130f4520SKenneth D. Merry ctl_complete_beio(beio); 605130f4520SKenneth D. Merry } 606130f4520SKenneth D. Merry 607d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t"); 608d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t"); 609d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t"); 610d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t"); 611130f4520SKenneth D. Merry 612130f4520SKenneth D. Merry static void 613130f4520SKenneth D. Merry ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 614130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 615130f4520SKenneth D. Merry { 616130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 617130f4520SKenneth D. Merry union ctl_io *io; 618130f4520SKenneth D. Merry struct uio xuio; 619130f4520SKenneth D. Merry struct iovec *xiovec; 6205050aa86SKonstantin Belousov int flags; 621130f4520SKenneth D. Merry int error, i; 622130f4520SKenneth D. Merry 623130f4520SKenneth D. Merry DPRINTF("entered\n"); 624130f4520SKenneth D. Merry 625130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 626130f4520SKenneth D. Merry io = beio->io; 62755551d05SAlexander Motin flags = 0; 62855551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 62955551d05SAlexander Motin flags |= IO_DIRECT; 63055551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 63155551d05SAlexander Motin flags |= IO_SYNC; 632130f4520SKenneth D. Merry 63311b569f7SAlexander Motin bzero(&xuio, sizeof(xuio)); 634130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 635130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 63611b569f7SAlexander Motin xuio.uio_rw = UIO_READ; 637130f4520SKenneth D. Merry } else { 638130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 639130f4520SKenneth D. Merry xuio.uio_rw = UIO_WRITE; 64011b569f7SAlexander Motin } 641130f4520SKenneth D. Merry xuio.uio_offset = beio->io_offset; 642130f4520SKenneth D. Merry xuio.uio_resid = beio->io_len; 643130f4520SKenneth D. Merry xuio.uio_segflg = UIO_SYSSPACE; 644130f4520SKenneth D. Merry xuio.uio_iov = beio->xiovecs; 645130f4520SKenneth D. Merry xuio.uio_iovcnt = beio->num_segs; 646130f4520SKenneth D. Merry xuio.uio_td = curthread; 647130f4520SKenneth D. Merry 648130f4520SKenneth D. Merry for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 649130f4520SKenneth D. Merry xiovec->iov_base = beio->sg_segs[i].addr; 650130f4520SKenneth D. Merry xiovec->iov_len = beio->sg_segs[i].len; 651130f4520SKenneth D. Merry } 652130f4520SKenneth D. Merry 65375c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 65475c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 65575c7a1d3SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 65675c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 65775c7a1d3SAlexander Motin 658130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 659130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 660130f4520SKenneth D. Merry 661130f4520SKenneth D. Merry /* 662130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for reads. If the 663130f4520SKenneth D. Merry * DIRECTIO option is configured into the kernel, it calls 664130f4520SKenneth D. Merry * ffs_rawread(). But that only works for single-segment 665130f4520SKenneth D. Merry * uios with user space addresses. In our case, with a 666130f4520SKenneth D. Merry * kernel uio, it still reads into the buffer cache, but it 667130f4520SKenneth D. Merry * will just try to release the buffer from the cache later 668130f4520SKenneth D. Merry * on in ffs_read(). 669130f4520SKenneth D. Merry * 670130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for reads. 671130f4520SKenneth D. Merry * 672130f4520SKenneth D. Merry * UFS does not pay attention to IO_SYNC for reads. 673130f4520SKenneth D. Merry * 674130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (which translates into the 675130f4520SKenneth D. Merry * Solaris define FRSYNC for zfs_read()) for reads. It 676130f4520SKenneth D. Merry * attempts to sync the file before reading. 677130f4520SKenneth D. Merry * 678130f4520SKenneth D. Merry * So, to attempt to provide some barrier semantics in the 679130f4520SKenneth D. Merry * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC. 680130f4520SKenneth D. Merry */ 68155551d05SAlexander Motin error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); 682130f4520SKenneth D. Merry 683130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 68411b569f7SAlexander Motin SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 685130f4520SKenneth D. Merry } else { 686130f4520SKenneth D. Merry struct mount *mountpoint; 687130f4520SKenneth D. Merry int lock_flags; 688130f4520SKenneth D. Merry 689130f4520SKenneth D. Merry (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 690130f4520SKenneth D. Merry 691130f4520SKenneth D. Merry if (MNT_SHARED_WRITES(mountpoint) 692130f4520SKenneth D. Merry || ((mountpoint == NULL) 693130f4520SKenneth D. Merry && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 694130f4520SKenneth D. Merry lock_flags = LK_SHARED; 695130f4520SKenneth D. Merry else 696130f4520SKenneth D. Merry lock_flags = LK_EXCLUSIVE; 697130f4520SKenneth D. Merry 698130f4520SKenneth D. Merry vn_lock(be_lun->vn, lock_flags | LK_RETRY); 699130f4520SKenneth D. Merry 700130f4520SKenneth D. Merry /* 701130f4520SKenneth D. Merry * UFS pays attention to IO_DIRECT for writes. The write 702130f4520SKenneth D. Merry * is done asynchronously. (Normally the write would just 703130f4520SKenneth D. Merry * get put into cache. 704130f4520SKenneth D. Merry * 705130f4520SKenneth D. Merry * UFS pays attention to IO_SYNC for writes. It will 706130f4520SKenneth D. Merry * attempt to write the buffer out synchronously if that 707130f4520SKenneth D. Merry * flag is set. 708130f4520SKenneth D. Merry * 709130f4520SKenneth D. Merry * ZFS does not pay attention to IO_DIRECT for writes. 710130f4520SKenneth D. Merry * 711130f4520SKenneth D. Merry * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) 712130f4520SKenneth D. Merry * for writes. It will flush the transaction from the 713130f4520SKenneth D. Merry * cache before returning. 714130f4520SKenneth D. Merry * 715130f4520SKenneth D. Merry * So if we've got the BIO_ORDERED flag set, we want 716130f4520SKenneth D. Merry * IO_SYNC in either the UFS or ZFS case. 717130f4520SKenneth D. Merry */ 71855551d05SAlexander Motin error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); 719130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 720130f4520SKenneth D. Merry 721130f4520SKenneth D. Merry vn_finished_write(mountpoint); 72211b569f7SAlexander Motin SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 723130f4520SKenneth D. Merry } 724130f4520SKenneth D. Merry 72575c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 72675c7a1d3SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 72775c7a1d3SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 72875c7a1d3SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 72975c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 73075c7a1d3SAlexander Motin 731130f4520SKenneth D. Merry /* 732130f4520SKenneth D. Merry * If we got an error, set the sense data to "MEDIUM ERROR" and 733130f4520SKenneth D. Merry * return the I/O to the user. 734130f4520SKenneth D. Merry */ 735130f4520SKenneth D. Merry if (error != 0) { 736130f4520SKenneth D. Merry char path_str[32]; 737130f4520SKenneth D. Merry 738130f4520SKenneth D. Merry ctl_scsi_path_string(io, path_str, sizeof(path_str)); 739130f4520SKenneth D. Merry printf("%s%s command returned errno %d\n", path_str, 740130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error); 7410631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 7424fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 7434fc18ff9SAlexander Motin } else 744130f4520SKenneth D. Merry ctl_set_medium_error(&io->scsiio); 745130f4520SKenneth D. Merry ctl_complete_beio(beio); 746130f4520SKenneth D. Merry return; 747130f4520SKenneth D. Merry } 748130f4520SKenneth D. Merry 749130f4520SKenneth D. Merry /* 750696297adSAlexander Motin * If this is a write or a verify, we're all done. 751130f4520SKenneth D. Merry * If this is a read, we can now send the data to the user. 752130f4520SKenneth D. Merry */ 753696297adSAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 754696297adSAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 755130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 756130f4520SKenneth D. Merry ctl_complete_beio(beio); 757130f4520SKenneth D. Merry } else { 758f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 759f7241cceSAlexander Motin beio->beio_cont == NULL) 760f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 761130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 762130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 763130f4520SKenneth D. Merry #endif 764130f4520SKenneth D. Merry ctl_datamove(io); 765130f4520SKenneth D. Merry } 766130f4520SKenneth D. Merry } 767130f4520SKenneth D. Merry 768130f4520SKenneth D. Merry static void 769ef8daf3fSAlexander Motin ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 770ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 771ef8daf3fSAlexander Motin { 772ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 773ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 774ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 775ef8daf3fSAlexander Motin off_t roff, off; 776ef8daf3fSAlexander Motin int error, status; 777ef8daf3fSAlexander Motin 778ef8daf3fSAlexander Motin DPRINTF("entered\n"); 779ef8daf3fSAlexander Motin 780ef8daf3fSAlexander Motin off = roff = ((off_t)lbalen->lba) << be_lun->blocksize_shift; 781ef8daf3fSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 782ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, 783ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 784ef8daf3fSAlexander Motin if (error == 0 && off > roff) 785ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 786ef8daf3fSAlexander Motin else { 787ef8daf3fSAlexander Motin error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, 788ef8daf3fSAlexander Motin 0, curthread->td_ucred, curthread); 789ef8daf3fSAlexander Motin if (error == 0 && off > roff) 790ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 791ef8daf3fSAlexander Motin else { 792ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 793ef8daf3fSAlexander Motin off = be_lun->size_bytes; 794ef8daf3fSAlexander Motin } 795ef8daf3fSAlexander Motin } 796ef8daf3fSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 797ef8daf3fSAlexander Motin 798ef8daf3fSAlexander Motin off >>= be_lun->blocksize_shift; 799ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 800ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 801ef8daf3fSAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off - lbalen->lba), 802ef8daf3fSAlexander Motin data->descr[0].length); 803ef8daf3fSAlexander Motin data->descr[0].status = status; 804ef8daf3fSAlexander Motin 805ef8daf3fSAlexander Motin ctl_complete_beio(beio); 806ef8daf3fSAlexander Motin } 807ef8daf3fSAlexander Motin 80853c146deSAlexander Motin static uint64_t 80953c146deSAlexander Motin ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname) 81053c146deSAlexander Motin { 81153c146deSAlexander Motin struct vattr vattr; 81253c146deSAlexander Motin struct statfs statfs; 813b9b4269cSAlexander Motin uint64_t val; 81453c146deSAlexander Motin int error; 81553c146deSAlexander Motin 816b9b4269cSAlexander Motin val = UINT64_MAX; 81753c146deSAlexander Motin if (be_lun->vn == NULL) 818b9b4269cSAlexander Motin return (val); 819b9b4269cSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 82053c146deSAlexander Motin if (strcmp(attrname, "blocksused") == 0) { 82153c146deSAlexander Motin error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 822b9b4269cSAlexander Motin if (error == 0) 823b9b4269cSAlexander Motin val = vattr.va_bytes >> be_lun->blocksize_shift; 82453c146deSAlexander Motin } 825b9b4269cSAlexander Motin if (strcmp(attrname, "blocksavail") == 0 && 826b9b4269cSAlexander Motin (be_lun->vn->v_iflag & VI_DOOMED) == 0) { 82753c146deSAlexander Motin error = VFS_STATFS(be_lun->vn->v_mount, &statfs); 828b9b4269cSAlexander Motin if (error == 0) 829b9b4269cSAlexander Motin val = (statfs.f_bavail * statfs.f_bsize) >> 830b9b4269cSAlexander Motin be_lun->blocksize_shift; 83153c146deSAlexander Motin } 832b9b4269cSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 833b9b4269cSAlexander Motin return (val); 83453c146deSAlexander Motin } 83553c146deSAlexander Motin 836ef8daf3fSAlexander Motin static void 83767f586a8SAlexander Motin ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, 83867f586a8SAlexander Motin struct ctl_be_block_io *beio) 83967f586a8SAlexander Motin { 84067f586a8SAlexander Motin struct ctl_be_block_devdata *dev_data; 84167f586a8SAlexander Motin union ctl_io *io; 84267f586a8SAlexander Motin struct uio xuio; 84367f586a8SAlexander Motin struct iovec *xiovec; 84467f586a8SAlexander Motin int flags; 84567f586a8SAlexander Motin int error, i; 84667f586a8SAlexander Motin 84767f586a8SAlexander Motin DPRINTF("entered\n"); 84867f586a8SAlexander Motin 84967f586a8SAlexander Motin dev_data = &be_lun->backend.dev; 85067f586a8SAlexander Motin io = beio->io; 85155551d05SAlexander Motin flags = 0; 85255551d05SAlexander Motin if (ARGS(io)->flags & CTL_LLF_DPO) 85355551d05SAlexander Motin flags |= IO_DIRECT; 85455551d05SAlexander Motin if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 85555551d05SAlexander Motin flags |= IO_SYNC; 85667f586a8SAlexander Motin 85767f586a8SAlexander Motin bzero(&xuio, sizeof(xuio)); 85867f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 85967f586a8SAlexander Motin SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 86067f586a8SAlexander Motin xuio.uio_rw = UIO_READ; 86167f586a8SAlexander Motin } else { 86267f586a8SAlexander Motin SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 86367f586a8SAlexander Motin xuio.uio_rw = UIO_WRITE; 86467f586a8SAlexander Motin } 86567f586a8SAlexander Motin xuio.uio_offset = beio->io_offset; 86667f586a8SAlexander Motin xuio.uio_resid = beio->io_len; 86767f586a8SAlexander Motin xuio.uio_segflg = UIO_SYSSPACE; 86867f586a8SAlexander Motin xuio.uio_iov = beio->xiovecs; 86967f586a8SAlexander Motin xuio.uio_iovcnt = beio->num_segs; 87067f586a8SAlexander Motin xuio.uio_td = curthread; 87167f586a8SAlexander Motin 87267f586a8SAlexander Motin for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 87367f586a8SAlexander Motin xiovec->iov_base = beio->sg_segs[i].addr; 87467f586a8SAlexander Motin xiovec->iov_len = beio->sg_segs[i].len; 87567f586a8SAlexander Motin } 87667f586a8SAlexander Motin 87767f586a8SAlexander Motin binuptime(&beio->ds_t0); 87867f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 87967f586a8SAlexander Motin devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 88067f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 88167f586a8SAlexander Motin 88267f586a8SAlexander Motin if (beio->bio_cmd == BIO_READ) { 88355551d05SAlexander Motin error = (*dev_data->csw->d_read)(dev_data->cdev, &xuio, flags); 88467f586a8SAlexander Motin SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 88567f586a8SAlexander Motin } else { 88655551d05SAlexander Motin error = (*dev_data->csw->d_write)(dev_data->cdev, &xuio, flags); 88767f586a8SAlexander Motin SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 88867f586a8SAlexander Motin } 88967f586a8SAlexander Motin 89067f586a8SAlexander Motin mtx_lock(&be_lun->io_lock); 89167f586a8SAlexander Motin devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 89267f586a8SAlexander Motin beio->ds_tag_type, beio->ds_trans_type, 89367f586a8SAlexander Motin /*now*/ NULL, /*then*/&beio->ds_t0); 89467f586a8SAlexander Motin mtx_unlock(&be_lun->io_lock); 89567f586a8SAlexander Motin 89667f586a8SAlexander Motin /* 89767f586a8SAlexander Motin * If we got an error, set the sense data to "MEDIUM ERROR" and 89867f586a8SAlexander Motin * return the I/O to the user. 89967f586a8SAlexander Motin */ 90067f586a8SAlexander Motin if (error != 0) { 9010631de4aSAlexander Motin if (error == ENOSPC || error == EDQUOT) { 9024fc18ff9SAlexander Motin ctl_set_space_alloc_fail(&io->scsiio); 9034fc18ff9SAlexander Motin } else 90467f586a8SAlexander Motin ctl_set_medium_error(&io->scsiio); 90567f586a8SAlexander Motin ctl_complete_beio(beio); 90667f586a8SAlexander Motin return; 90767f586a8SAlexander Motin } 90867f586a8SAlexander Motin 90967f586a8SAlexander Motin /* 91067f586a8SAlexander Motin * If this is a write or a verify, we're all done. 91167f586a8SAlexander Motin * If this is a read, we can now send the data to the user. 91267f586a8SAlexander Motin */ 91367f586a8SAlexander Motin if ((beio->bio_cmd == BIO_WRITE) || 91467f586a8SAlexander Motin (ARGS(io)->flags & CTL_LLF_VERIFY)) { 91567f586a8SAlexander Motin ctl_set_success(&io->scsiio); 91667f586a8SAlexander Motin ctl_complete_beio(beio); 91767f586a8SAlexander Motin } else { 918f7241cceSAlexander Motin if ((ARGS(io)->flags & CTL_LLF_READ) && 919f7241cceSAlexander Motin beio->beio_cont == NULL) 920f7241cceSAlexander Motin ctl_set_success(&io->scsiio); 92167f586a8SAlexander Motin #ifdef CTL_TIME_IO 92267f586a8SAlexander Motin getbintime(&io->io_hdr.dma_start_bt); 92367f586a8SAlexander Motin #endif 92467f586a8SAlexander Motin ctl_datamove(io); 92567f586a8SAlexander Motin } 92667f586a8SAlexander Motin } 92767f586a8SAlexander Motin 92867f586a8SAlexander Motin static void 929ef8daf3fSAlexander Motin ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun, 930ef8daf3fSAlexander Motin struct ctl_be_block_io *beio) 931ef8daf3fSAlexander Motin { 932ef8daf3fSAlexander Motin struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 933ef8daf3fSAlexander Motin union ctl_io *io = beio->io; 934ef8daf3fSAlexander Motin struct ctl_lba_len_flags *lbalen = ARGS(io); 935ef8daf3fSAlexander Motin struct scsi_get_lba_status_data *data; 936ef8daf3fSAlexander Motin off_t roff, off; 937ef8daf3fSAlexander Motin int error, status; 938ef8daf3fSAlexander Motin 939ef8daf3fSAlexander Motin DPRINTF("entered\n"); 940ef8daf3fSAlexander Motin 941ef8daf3fSAlexander Motin off = roff = ((off_t)lbalen->lba) << be_lun->blocksize_shift; 942ef8daf3fSAlexander Motin error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKHOLE, 943ef8daf3fSAlexander Motin (caddr_t)&off, FREAD, curthread); 944ef8daf3fSAlexander Motin if (error == 0 && off > roff) 945ef8daf3fSAlexander Motin status = 0; /* mapped up to off */ 946ef8daf3fSAlexander Motin else { 947ef8daf3fSAlexander Motin error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKDATA, 948ef8daf3fSAlexander Motin (caddr_t)&off, FREAD, curthread); 949ef8daf3fSAlexander Motin if (error == 0 && off > roff) 950ef8daf3fSAlexander Motin status = 1; /* deallocated up to off */ 951ef8daf3fSAlexander Motin else { 952ef8daf3fSAlexander Motin status = 0; /* unknown up to the end */ 953ef8daf3fSAlexander Motin off = be_lun->size_bytes; 954ef8daf3fSAlexander Motin } 955ef8daf3fSAlexander Motin } 956ef8daf3fSAlexander Motin 957ef8daf3fSAlexander Motin off >>= be_lun->blocksize_shift; 958ef8daf3fSAlexander Motin data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 959ef8daf3fSAlexander Motin scsi_u64to8b(lbalen->lba, data->descr[0].addr); 960ef8daf3fSAlexander Motin scsi_ulto4b(MIN(UINT32_MAX, off - lbalen->lba), 961ef8daf3fSAlexander Motin data->descr[0].length); 962ef8daf3fSAlexander Motin data->descr[0].status = status; 963ef8daf3fSAlexander Motin 964ef8daf3fSAlexander Motin ctl_complete_beio(beio); 965ef8daf3fSAlexander Motin } 966ef8daf3fSAlexander Motin 967ef8daf3fSAlexander Motin static void 968130f4520SKenneth D. Merry ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 969130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 970130f4520SKenneth D. Merry { 971130f4520SKenneth D. Merry struct bio *bio; 972130f4520SKenneth D. Merry union ctl_io *io; 973130f4520SKenneth D. Merry struct ctl_be_block_devdata *dev_data; 974130f4520SKenneth D. Merry 975130f4520SKenneth D. Merry dev_data = &be_lun->backend.dev; 976130f4520SKenneth D. Merry io = beio->io; 977130f4520SKenneth D. Merry 978130f4520SKenneth D. Merry DPRINTF("entered\n"); 979130f4520SKenneth D. Merry 980130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 981130f4520SKenneth D. Merry bio = g_alloc_bio(); 982130f4520SKenneth D. Merry 983130f4520SKenneth D. Merry bio->bio_cmd = BIO_FLUSH; 984130f4520SKenneth D. Merry bio->bio_flags |= BIO_ORDERED; 985130f4520SKenneth D. Merry bio->bio_dev = dev_data->cdev; 986130f4520SKenneth D. Merry bio->bio_offset = 0; 987130f4520SKenneth D. Merry bio->bio_data = 0; 988130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 989130f4520SKenneth D. Merry bio->bio_caller1 = beio; 990130f4520SKenneth D. Merry bio->bio_pblkno = 0; 991130f4520SKenneth D. Merry 992130f4520SKenneth D. Merry /* 993130f4520SKenneth D. Merry * We don't need to acquire the LUN lock here, because we are only 994130f4520SKenneth D. Merry * sending one bio, and so there is no other context to synchronize 995130f4520SKenneth D. Merry * with. 996130f4520SKenneth D. Merry */ 997130f4520SKenneth D. Merry beio->num_bios_sent = 1; 998130f4520SKenneth D. Merry beio->send_complete = 1; 999130f4520SKenneth D. Merry 1000130f4520SKenneth D. Merry binuptime(&beio->ds_t0); 100175c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1002130f4520SKenneth D. Merry devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 100375c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1004130f4520SKenneth D. Merry 1005130f4520SKenneth D. Merry (*dev_data->csw->d_strategy)(bio); 1006130f4520SKenneth D. Merry } 1007130f4520SKenneth D. Merry 1008130f4520SKenneth D. Merry static void 1009ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun, 1010ee7f31c0SAlexander Motin struct ctl_be_block_io *beio, 1011ee7f31c0SAlexander Motin uint64_t off, uint64_t len, int last) 1012ee7f31c0SAlexander Motin { 1013ee7f31c0SAlexander Motin struct bio *bio; 1014ee7f31c0SAlexander Motin struct ctl_be_block_devdata *dev_data; 10158f5a226aSAlexander Motin uint64_t maxlen; 1016ee7f31c0SAlexander Motin 1017ee7f31c0SAlexander Motin dev_data = &be_lun->backend.dev; 10188f5a226aSAlexander Motin maxlen = LONG_MAX - (LONG_MAX % be_lun->blocksize); 1019ee7f31c0SAlexander Motin while (len > 0) { 1020ee7f31c0SAlexander Motin bio = g_alloc_bio(); 1021ee7f31c0SAlexander Motin bio->bio_cmd = BIO_DELETE; 1022ee7f31c0SAlexander Motin bio->bio_dev = dev_data->cdev; 1023ee7f31c0SAlexander Motin bio->bio_offset = off; 10248f5a226aSAlexander Motin bio->bio_length = MIN(len, maxlen); 1025ee7f31c0SAlexander Motin bio->bio_data = 0; 1026ee7f31c0SAlexander Motin bio->bio_done = ctl_be_block_biodone; 1027ee7f31c0SAlexander Motin bio->bio_caller1 = beio; 10288f5a226aSAlexander Motin bio->bio_pblkno = off / be_lun->blocksize; 1029ee7f31c0SAlexander Motin 1030ee7f31c0SAlexander Motin off += bio->bio_length; 1031ee7f31c0SAlexander Motin len -= bio->bio_length; 1032ee7f31c0SAlexander Motin 103375c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1034ee7f31c0SAlexander Motin beio->num_bios_sent++; 1035ee7f31c0SAlexander Motin if (last && len == 0) 1036ee7f31c0SAlexander Motin beio->send_complete = 1; 103775c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1038ee7f31c0SAlexander Motin 1039ee7f31c0SAlexander Motin (*dev_data->csw->d_strategy)(bio); 1040ee7f31c0SAlexander Motin } 1041ee7f31c0SAlexander Motin } 1042ee7f31c0SAlexander Motin 1043ee7f31c0SAlexander Motin static void 1044ee7f31c0SAlexander Motin ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 1045ee7f31c0SAlexander Motin struct ctl_be_block_io *beio) 1046ee7f31c0SAlexander Motin { 1047ee7f31c0SAlexander Motin union ctl_io *io; 1048ee7f31c0SAlexander Motin struct ctl_be_block_devdata *dev_data; 104966df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1050ee7f31c0SAlexander Motin struct scsi_unmap_desc *buf, *end; 1051ee7f31c0SAlexander Motin uint64_t len; 1052ee7f31c0SAlexander Motin 1053ee7f31c0SAlexander Motin dev_data = &be_lun->backend.dev; 1054ee7f31c0SAlexander Motin io = beio->io; 1055ee7f31c0SAlexander Motin 1056ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1057ee7f31c0SAlexander Motin 1058ee7f31c0SAlexander Motin binuptime(&beio->ds_t0); 105975c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 1060ee7f31c0SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 106175c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 1062ee7f31c0SAlexander Motin 1063ee7f31c0SAlexander Motin if (beio->io_offset == -1) { 1064ee7f31c0SAlexander Motin beio->io_len = 0; 106566df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 106666df9136SAlexander Motin buf = (struct scsi_unmap_desc *)ptrlen->ptr; 106766df9136SAlexander Motin end = buf + ptrlen->len / sizeof(*buf); 1068ee7f31c0SAlexander Motin for (; buf < end; buf++) { 1069ee7f31c0SAlexander Motin len = (uint64_t)scsi_4btoul(buf->length) * 1070ee7f31c0SAlexander Motin be_lun->blocksize; 1071ee7f31c0SAlexander Motin beio->io_len += len; 1072ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1073ee7f31c0SAlexander Motin scsi_8btou64(buf->lba) * be_lun->blocksize, len, 10747e0be022SAlexander Motin (end - buf < 2) ? TRUE : FALSE); 1075ee7f31c0SAlexander Motin } 1076ee7f31c0SAlexander Motin } else 1077ee7f31c0SAlexander Motin ctl_be_block_unmap_dev_range(be_lun, beio, 1078ee7f31c0SAlexander Motin beio->io_offset, beio->io_len, TRUE); 1079ee7f31c0SAlexander Motin } 1080ee7f31c0SAlexander Motin 1081ee7f31c0SAlexander Motin static void 1082130f4520SKenneth D. Merry ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 1083130f4520SKenneth D. Merry struct ctl_be_block_io *beio) 1084130f4520SKenneth D. Merry { 108575c7a1d3SAlexander Motin TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue); 1086130f4520SKenneth D. Merry int i; 1087130f4520SKenneth D. Merry struct bio *bio; 1088130f4520SKenneth D. Merry struct ctl_be_block_devdata *dev_data; 1089130f4520SKenneth D. Merry off_t cur_offset; 1090130f4520SKenneth D. Merry int max_iosize; 1091130f4520SKenneth D. Merry 1092130f4520SKenneth D. Merry DPRINTF("entered\n"); 1093130f4520SKenneth D. Merry 1094130f4520SKenneth D. Merry dev_data = &be_lun->backend.dev; 1095130f4520SKenneth D. Merry 1096130f4520SKenneth D. Merry /* 1097130f4520SKenneth D. Merry * We have to limit our I/O size to the maximum supported by the 1098130f4520SKenneth D. Merry * backend device. Hopefully it is MAXPHYS. If the driver doesn't 1099130f4520SKenneth D. Merry * set it properly, use DFLTPHYS. 1100130f4520SKenneth D. Merry */ 1101130f4520SKenneth D. Merry max_iosize = dev_data->cdev->si_iosize_max; 1102130f4520SKenneth D. Merry if (max_iosize < PAGE_SIZE) 1103130f4520SKenneth D. Merry max_iosize = DFLTPHYS; 1104130f4520SKenneth D. Merry 1105130f4520SKenneth D. Merry cur_offset = beio->io_offset; 1106130f4520SKenneth D. Merry for (i = 0; i < beio->num_segs; i++) { 1107130f4520SKenneth D. Merry size_t cur_size; 1108130f4520SKenneth D. Merry uint8_t *cur_ptr; 1109130f4520SKenneth D. Merry 1110130f4520SKenneth D. Merry cur_size = beio->sg_segs[i].len; 1111130f4520SKenneth D. Merry cur_ptr = beio->sg_segs[i].addr; 1112130f4520SKenneth D. Merry 1113130f4520SKenneth D. Merry while (cur_size > 0) { 1114130f4520SKenneth D. Merry /* This can't fail, it's a blocking allocation. */ 1115130f4520SKenneth D. Merry bio = g_alloc_bio(); 1116130f4520SKenneth D. Merry 1117130f4520SKenneth D. Merry KASSERT(bio != NULL, ("g_alloc_bio() failed!\n")); 1118130f4520SKenneth D. Merry 1119130f4520SKenneth D. Merry bio->bio_cmd = beio->bio_cmd; 1120130f4520SKenneth D. Merry bio->bio_dev = dev_data->cdev; 1121130f4520SKenneth D. Merry bio->bio_caller1 = beio; 1122130f4520SKenneth D. Merry bio->bio_length = min(cur_size, max_iosize); 1123130f4520SKenneth D. Merry bio->bio_offset = cur_offset; 1124130f4520SKenneth D. Merry bio->bio_data = cur_ptr; 1125130f4520SKenneth D. Merry bio->bio_done = ctl_be_block_biodone; 1126130f4520SKenneth D. Merry bio->bio_pblkno = cur_offset / be_lun->blocksize; 1127130f4520SKenneth D. Merry 1128130f4520SKenneth D. Merry cur_offset += bio->bio_length; 1129130f4520SKenneth D. Merry cur_ptr += bio->bio_length; 1130130f4520SKenneth D. Merry cur_size -= bio->bio_length; 1131130f4520SKenneth D. Merry 113275c7a1d3SAlexander Motin TAILQ_INSERT_TAIL(&queue, bio, bio_queue); 1133130f4520SKenneth D. Merry beio->num_bios_sent++; 1134130f4520SKenneth D. Merry } 1135130f4520SKenneth D. Merry } 113675c7a1d3SAlexander Motin binuptime(&beio->ds_t0); 113775c7a1d3SAlexander Motin mtx_lock(&be_lun->io_lock); 113875c7a1d3SAlexander Motin devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 113975c7a1d3SAlexander Motin beio->send_complete = 1; 114075c7a1d3SAlexander Motin mtx_unlock(&be_lun->io_lock); 114175c7a1d3SAlexander Motin 114275c7a1d3SAlexander Motin /* 114375c7a1d3SAlexander Motin * Fire off all allocated requests! 114475c7a1d3SAlexander Motin */ 114575c7a1d3SAlexander Motin while ((bio = TAILQ_FIRST(&queue)) != NULL) { 114675c7a1d3SAlexander Motin TAILQ_REMOVE(&queue, bio, bio_queue); 114775c7a1d3SAlexander Motin (*dev_data->csw->d_strategy)(bio); 114875c7a1d3SAlexander Motin } 1149130f4520SKenneth D. Merry } 1150130f4520SKenneth D. Merry 1151c3e7ba3eSAlexander Motin static uint64_t 1152c3e7ba3eSAlexander Motin ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname) 1153c3e7ba3eSAlexander Motin { 1154c3e7ba3eSAlexander Motin struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 1155c3e7ba3eSAlexander Motin struct diocgattr_arg arg; 1156c3e7ba3eSAlexander Motin int error; 1157c3e7ba3eSAlexander Motin 1158c3e7ba3eSAlexander Motin if (dev_data->csw == NULL || dev_data->csw->d_ioctl == NULL) 1159c3e7ba3eSAlexander Motin return (UINT64_MAX); 1160c3e7ba3eSAlexander Motin strlcpy(arg.name, attrname, sizeof(arg.name)); 1161c3e7ba3eSAlexander Motin arg.len = sizeof(arg.value.off); 1162c3e7ba3eSAlexander Motin error = dev_data->csw->d_ioctl(dev_data->cdev, 1163c3e7ba3eSAlexander Motin DIOCGATTR, (caddr_t)&arg, FREAD, curthread); 1164c3e7ba3eSAlexander Motin if (error != 0) 1165c3e7ba3eSAlexander Motin return (UINT64_MAX); 1166c3e7ba3eSAlexander Motin return (arg.value.off); 1167c3e7ba3eSAlexander Motin } 1168c3e7ba3eSAlexander Motin 1169130f4520SKenneth D. Merry static void 1170ee7f31c0SAlexander Motin ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio) 1171ee7f31c0SAlexander Motin { 1172ee7f31c0SAlexander Motin union ctl_io *io; 1173ee7f31c0SAlexander Motin 1174ee7f31c0SAlexander Motin io = beio->io; 1175ee7f31c0SAlexander Motin ctl_free_beio(beio); 1176ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1177ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1178ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1179ee7f31c0SAlexander Motin ctl_config_write_done(io); 1180ee7f31c0SAlexander Motin return; 1181ee7f31c0SAlexander Motin } 1182ee7f31c0SAlexander Motin 1183ee7f31c0SAlexander Motin ctl_be_block_config_write(io); 1184ee7f31c0SAlexander Motin } 1185ee7f31c0SAlexander Motin 1186ee7f31c0SAlexander Motin static void 1187ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun, 1188ee7f31c0SAlexander Motin union ctl_io *io) 1189ee7f31c0SAlexander Motin { 1190ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 1191ee7f31c0SAlexander Motin struct ctl_be_block_softc *softc; 119266df9136SAlexander Motin struct ctl_lba_len_flags *lbalen; 1193fee04ef7SAlexander Motin uint64_t len_left, lba; 1194fee04ef7SAlexander Motin uint32_t pb, pbo, adj; 1195ee7f31c0SAlexander Motin int i, seglen; 1196ee7f31c0SAlexander Motin uint8_t *buf, *end; 1197ee7f31c0SAlexander Motin 1198ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1199ee7f31c0SAlexander Motin 1200e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1201ee7f31c0SAlexander Motin softc = be_lun->softc; 120211b569f7SAlexander Motin lbalen = ARGS(beio->io); 1203ee7f31c0SAlexander Motin 120464c5167cSAlexander Motin if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || 12053406a2a0SAlexander Motin (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { 1206ee7f31c0SAlexander Motin ctl_free_beio(beio); 1207ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1208ee7f31c0SAlexander Motin /*sks_valid*/ 1, 1209ee7f31c0SAlexander Motin /*command*/ 1, 1210ee7f31c0SAlexander Motin /*field*/ 1, 1211ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1212ee7f31c0SAlexander Motin /*bit*/ 0); 1213ee7f31c0SAlexander Motin ctl_config_write_done(io); 1214ee7f31c0SAlexander Motin return; 1215ee7f31c0SAlexander Motin } 1216ee7f31c0SAlexander Motin 1217ee7f31c0SAlexander Motin switch (io->scsiio.tag_type) { 1218ee7f31c0SAlexander Motin case CTL_TAG_ORDERED: 1219ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1220ee7f31c0SAlexander Motin break; 1221ee7f31c0SAlexander Motin case CTL_TAG_HEAD_OF_QUEUE: 1222ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1223ee7f31c0SAlexander Motin break; 1224ee7f31c0SAlexander Motin case CTL_TAG_UNTAGGED: 1225ee7f31c0SAlexander Motin case CTL_TAG_SIMPLE: 1226ee7f31c0SAlexander Motin case CTL_TAG_ACA: 1227ee7f31c0SAlexander Motin default: 1228ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1229ee7f31c0SAlexander Motin break; 1230ee7f31c0SAlexander Motin } 1231ee7f31c0SAlexander Motin 12323406a2a0SAlexander Motin if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { 123366df9136SAlexander Motin beio->io_offset = lbalen->lba * be_lun->blocksize; 123466df9136SAlexander Motin beio->io_len = (uint64_t)lbalen->len * be_lun->blocksize; 1235ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1236ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 1237ee7f31c0SAlexander Motin 1238ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1239ee7f31c0SAlexander Motin return; 1240ee7f31c0SAlexander Motin } 1241ee7f31c0SAlexander Motin 1242ee7f31c0SAlexander Motin beio->bio_cmd = BIO_WRITE; 1243ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_WRITE; 1244ee7f31c0SAlexander Motin 1245ee7f31c0SAlexander Motin DPRINTF("WRITE SAME at LBA %jx len %u\n", 124666df9136SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len); 1247ee7f31c0SAlexander Motin 1248fee04ef7SAlexander Motin pb = be_lun->blocksize << be_lun->pblockexp; 1249fee04ef7SAlexander Motin if (be_lun->pblockoff > 0) 1250fee04ef7SAlexander Motin pbo = pb - be_lun->blocksize * be_lun->pblockoff; 1251fee04ef7SAlexander Motin else 1252fee04ef7SAlexander Motin pbo = 0; 125366df9136SAlexander Motin len_left = (uint64_t)lbalen->len * be_lun->blocksize; 1254ee7f31c0SAlexander Motin for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) { 1255ee7f31c0SAlexander Motin 1256ee7f31c0SAlexander Motin /* 1257ee7f31c0SAlexander Motin * Setup the S/G entry for this chunk. 1258ee7f31c0SAlexander Motin */ 125908a7cce5SAlexander Motin seglen = MIN(CTLBLK_MAX_SEG, len_left); 126093b8c96cSAlexander Motin if (pb > be_lun->blocksize) { 126193b8c96cSAlexander Motin adj = ((lbalen->lba + lba) * be_lun->blocksize + 126293b8c96cSAlexander Motin seglen - pbo) % pb; 126393b8c96cSAlexander Motin if (seglen > adj) 126493b8c96cSAlexander Motin seglen -= adj; 126593b8c96cSAlexander Motin else 126693b8c96cSAlexander Motin seglen -= seglen % be_lun->blocksize; 126793b8c96cSAlexander Motin } else 1268ee7f31c0SAlexander Motin seglen -= seglen % be_lun->blocksize; 1269ee7f31c0SAlexander Motin beio->sg_segs[i].len = seglen; 1270ee7f31c0SAlexander Motin beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1271ee7f31c0SAlexander Motin 1272ee7f31c0SAlexander Motin DPRINTF("segment %d addr %p len %zd\n", i, 1273ee7f31c0SAlexander Motin beio->sg_segs[i].addr, beio->sg_segs[i].len); 1274ee7f31c0SAlexander Motin 1275ee7f31c0SAlexander Motin beio->num_segs++; 1276ee7f31c0SAlexander Motin len_left -= seglen; 1277ee7f31c0SAlexander Motin 1278ee7f31c0SAlexander Motin buf = beio->sg_segs[i].addr; 1279ee7f31c0SAlexander Motin end = buf + seglen; 1280ee7f31c0SAlexander Motin for (; buf < end; buf += be_lun->blocksize) { 1281ee7f31c0SAlexander Motin memcpy(buf, io->scsiio.kern_data_ptr, be_lun->blocksize); 128266df9136SAlexander Motin if (lbalen->flags & SWS_LBDATA) 128366df9136SAlexander Motin scsi_ulto4b(lbalen->lba + lba, buf); 1284ee7f31c0SAlexander Motin lba++; 1285ee7f31c0SAlexander Motin } 1286ee7f31c0SAlexander Motin } 1287ee7f31c0SAlexander Motin 128866df9136SAlexander Motin beio->io_offset = lbalen->lba * be_lun->blocksize; 1289ee7f31c0SAlexander Motin beio->io_len = lba * be_lun->blocksize; 1290ee7f31c0SAlexander Motin 1291ee7f31c0SAlexander Motin /* We can not do all in one run. Correct and schedule rerun. */ 1292ee7f31c0SAlexander Motin if (len_left > 0) { 129366df9136SAlexander Motin lbalen->lba += lba; 129466df9136SAlexander Motin lbalen->len -= lba; 1295ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done_ws; 1296ee7f31c0SAlexander Motin } 1297ee7f31c0SAlexander Motin 1298ee7f31c0SAlexander Motin be_lun->dispatch(be_lun, beio); 1299ee7f31c0SAlexander Motin } 1300ee7f31c0SAlexander Motin 1301ee7f31c0SAlexander Motin static void 1302ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun, 1303ee7f31c0SAlexander Motin union ctl_io *io) 1304ee7f31c0SAlexander Motin { 1305ee7f31c0SAlexander Motin struct ctl_be_block_io *beio; 1306ee7f31c0SAlexander Motin struct ctl_be_block_softc *softc; 130766df9136SAlexander Motin struct ctl_ptr_len_flags *ptrlen; 1308ee7f31c0SAlexander Motin 1309ee7f31c0SAlexander Motin DPRINTF("entered\n"); 1310ee7f31c0SAlexander Motin 1311e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1312ee7f31c0SAlexander Motin softc = be_lun->softc; 131366df9136SAlexander Motin ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1314ee7f31c0SAlexander Motin 13153406a2a0SAlexander Motin if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { 1316ee7f31c0SAlexander Motin ctl_free_beio(beio); 1317ee7f31c0SAlexander Motin ctl_set_invalid_field(&io->scsiio, 1318ee7f31c0SAlexander Motin /*sks_valid*/ 0, 1319ee7f31c0SAlexander Motin /*command*/ 1, 1320ee7f31c0SAlexander Motin /*field*/ 0, 1321ee7f31c0SAlexander Motin /*bit_valid*/ 0, 1322ee7f31c0SAlexander Motin /*bit*/ 0); 1323ee7f31c0SAlexander Motin ctl_config_write_done(io); 1324ee7f31c0SAlexander Motin return; 1325ee7f31c0SAlexander Motin } 1326ee7f31c0SAlexander Motin 1327ee7f31c0SAlexander Motin switch (io->scsiio.tag_type) { 1328ee7f31c0SAlexander Motin case CTL_TAG_ORDERED: 1329ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1330ee7f31c0SAlexander Motin break; 1331ee7f31c0SAlexander Motin case CTL_TAG_HEAD_OF_QUEUE: 1332ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1333ee7f31c0SAlexander Motin break; 1334ee7f31c0SAlexander Motin case CTL_TAG_UNTAGGED: 1335ee7f31c0SAlexander Motin case CTL_TAG_SIMPLE: 1336ee7f31c0SAlexander Motin case CTL_TAG_ACA: 1337ee7f31c0SAlexander Motin default: 1338ee7f31c0SAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1339ee7f31c0SAlexander Motin break; 1340ee7f31c0SAlexander Motin } 1341ee7f31c0SAlexander Motin 1342ee7f31c0SAlexander Motin beio->io_len = 0; 1343ee7f31c0SAlexander Motin beio->io_offset = -1; 1344ee7f31c0SAlexander Motin 1345ee7f31c0SAlexander Motin beio->bio_cmd = BIO_DELETE; 1346ee7f31c0SAlexander Motin beio->ds_trans_type = DEVSTAT_FREE; 1347ee7f31c0SAlexander Motin 134866df9136SAlexander Motin DPRINTF("UNMAP\n"); 1349ee7f31c0SAlexander Motin 1350ee7f31c0SAlexander Motin be_lun->unmap(be_lun, beio); 1351ee7f31c0SAlexander Motin } 1352ee7f31c0SAlexander Motin 1353ee7f31c0SAlexander Motin static void 1354ef8daf3fSAlexander Motin ctl_be_block_cr_done(struct ctl_be_block_io *beio) 1355ef8daf3fSAlexander Motin { 1356ef8daf3fSAlexander Motin union ctl_io *io; 1357ef8daf3fSAlexander Motin 1358ef8daf3fSAlexander Motin io = beio->io; 1359ef8daf3fSAlexander Motin ctl_free_beio(beio); 1360ef8daf3fSAlexander Motin ctl_config_read_done(io); 1361ef8daf3fSAlexander Motin } 1362ef8daf3fSAlexander Motin 1363ef8daf3fSAlexander Motin static void 1364ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 1365ef8daf3fSAlexander Motin union ctl_io *io) 1366ef8daf3fSAlexander Motin { 1367ef8daf3fSAlexander Motin struct ctl_be_block_io *beio; 1368ef8daf3fSAlexander Motin struct ctl_be_block_softc *softc; 1369ef8daf3fSAlexander Motin 1370ef8daf3fSAlexander Motin DPRINTF("entered\n"); 1371ef8daf3fSAlexander Motin 1372ef8daf3fSAlexander Motin softc = be_lun->softc; 1373ef8daf3fSAlexander Motin beio = ctl_alloc_beio(softc); 1374ef8daf3fSAlexander Motin beio->io = io; 1375ef8daf3fSAlexander Motin beio->lun = be_lun; 1376ef8daf3fSAlexander Motin beio->beio_cont = ctl_be_block_cr_done; 1377ef8daf3fSAlexander Motin PRIV(io)->ptr = (void *)beio; 1378ef8daf3fSAlexander Motin 1379ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 1380ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: /* GET LBA STATUS */ 1381ef8daf3fSAlexander Motin beio->bio_cmd = -1; 1382ef8daf3fSAlexander Motin beio->ds_trans_type = DEVSTAT_NO_DATA; 1383ef8daf3fSAlexander Motin beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1384ef8daf3fSAlexander Motin beio->io_len = 0; 1385ef8daf3fSAlexander Motin if (be_lun->get_lba_status) 1386ef8daf3fSAlexander Motin be_lun->get_lba_status(be_lun, beio); 1387ef8daf3fSAlexander Motin else 1388ef8daf3fSAlexander Motin ctl_be_block_cr_done(beio); 1389ef8daf3fSAlexander Motin break; 1390ef8daf3fSAlexander Motin default: 1391ef8daf3fSAlexander Motin panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1392ef8daf3fSAlexander Motin break; 1393ef8daf3fSAlexander Motin } 1394ef8daf3fSAlexander Motin } 1395ef8daf3fSAlexander Motin 1396ef8daf3fSAlexander Motin static void 1397ee7f31c0SAlexander Motin ctl_be_block_cw_done(struct ctl_be_block_io *beio) 1398ee7f31c0SAlexander Motin { 1399ee7f31c0SAlexander Motin union ctl_io *io; 1400ee7f31c0SAlexander Motin 1401ee7f31c0SAlexander Motin io = beio->io; 1402ee7f31c0SAlexander Motin ctl_free_beio(beio); 1403ee7f31c0SAlexander Motin ctl_config_write_done(io); 1404ee7f31c0SAlexander Motin } 1405ee7f31c0SAlexander Motin 1406ee7f31c0SAlexander Motin static void 1407130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 1408130f4520SKenneth D. Merry union ctl_io *io) 1409130f4520SKenneth D. Merry { 1410130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1411130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1412130f4520SKenneth D. Merry 1413130f4520SKenneth D. Merry DPRINTF("entered\n"); 1414130f4520SKenneth D. Merry 1415130f4520SKenneth D. Merry softc = be_lun->softc; 1416130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1417130f4520SKenneth D. Merry beio->io = io; 1418130f4520SKenneth D. Merry beio->lun = be_lun; 1419ee7f31c0SAlexander Motin beio->beio_cont = ctl_be_block_cw_done; 1420e86a4142SAlexander Motin PRIV(io)->ptr = (void *)beio; 1421130f4520SKenneth D. Merry 1422130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 1423130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 1424130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 1425d2a0972dSEdward Tomasz Napierala beio->bio_cmd = BIO_FLUSH; 1426130f4520SKenneth D. Merry beio->ds_trans_type = DEVSTAT_NO_DATA; 1427130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1428130f4520SKenneth D. Merry beio->io_len = 0; 1429130f4520SKenneth D. Merry be_lun->lun_flush(be_lun, beio); 1430130f4520SKenneth D. Merry break; 1431ee7f31c0SAlexander Motin case WRITE_SAME_10: 1432ee7f31c0SAlexander Motin case WRITE_SAME_16: 1433ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_ws(be_lun, io); 1434ee7f31c0SAlexander Motin break; 1435ee7f31c0SAlexander Motin case UNMAP: 1436ee7f31c0SAlexander Motin ctl_be_block_cw_dispatch_unmap(be_lun, io); 1437ee7f31c0SAlexander Motin break; 1438130f4520SKenneth D. Merry default: 1439130f4520SKenneth D. Merry panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1440130f4520SKenneth D. Merry break; 1441130f4520SKenneth D. Merry } 1442130f4520SKenneth D. Merry } 1443130f4520SKenneth D. Merry 1444d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t"); 1445d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t"); 1446d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t"); 1447d9fae5abSAndriy Gapon SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t"); 1448130f4520SKenneth D. Merry 1449130f4520SKenneth D. Merry static void 145008a7cce5SAlexander Motin ctl_be_block_next(struct ctl_be_block_io *beio) 145108a7cce5SAlexander Motin { 145208a7cce5SAlexander Motin struct ctl_be_block_lun *be_lun; 145308a7cce5SAlexander Motin union ctl_io *io; 145408a7cce5SAlexander Motin 145508a7cce5SAlexander Motin io = beio->io; 145608a7cce5SAlexander Motin be_lun = beio->lun; 145708a7cce5SAlexander Motin ctl_free_beio(beio); 1458ead2f117SAlexander Motin if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1459ead2f117SAlexander Motin ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1460ead2f117SAlexander Motin (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 146111b569f7SAlexander Motin ctl_data_submit_done(io); 146208a7cce5SAlexander Motin return; 146308a7cce5SAlexander Motin } 146408a7cce5SAlexander Motin 146508a7cce5SAlexander Motin io->io_hdr.status &= ~CTL_STATUS_MASK; 146608a7cce5SAlexander Motin io->io_hdr.status |= CTL_STATUS_NONE; 146708a7cce5SAlexander Motin 146875c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 146908a7cce5SAlexander Motin /* 147008a7cce5SAlexander Motin * XXX KDM make sure that links is okay to use at this point. 147108a7cce5SAlexander Motin * Otherwise, we either need to add another field to ctl_io_hdr, 147208a7cce5SAlexander Motin * or deal with resource allocation here. 147308a7cce5SAlexander Motin */ 147408a7cce5SAlexander Motin STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 147575c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 147608a7cce5SAlexander Motin 147708a7cce5SAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 147808a7cce5SAlexander Motin } 147908a7cce5SAlexander Motin 148008a7cce5SAlexander Motin static void 1481130f4520SKenneth D. Merry ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 1482130f4520SKenneth D. Merry union ctl_io *io) 1483130f4520SKenneth D. Merry { 1484130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1485130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 148611b569f7SAlexander Motin struct ctl_lba_len_flags *lbalen; 1487e86a4142SAlexander Motin struct ctl_ptr_len_flags *bptrlen; 1488e86a4142SAlexander Motin uint64_t len_left, lbas; 1489130f4520SKenneth D. Merry int i; 1490130f4520SKenneth D. Merry 1491130f4520SKenneth D. Merry softc = be_lun->softc; 1492130f4520SKenneth D. Merry 1493130f4520SKenneth D. Merry DPRINTF("entered\n"); 1494130f4520SKenneth D. Merry 149511b569f7SAlexander Motin lbalen = ARGS(io); 149611b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1497130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0); 149811b569f7SAlexander Motin } else { 149911b569f7SAlexander Motin SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0); 1500130f4520SKenneth D. Merry } 1501130f4520SKenneth D. Merry 1502130f4520SKenneth D. Merry beio = ctl_alloc_beio(softc); 1503130f4520SKenneth D. Merry beio->io = io; 1504130f4520SKenneth D. Merry beio->lun = be_lun; 1505e86a4142SAlexander Motin bptrlen = PRIV(io); 1506e86a4142SAlexander Motin bptrlen->ptr = (void *)beio; 1507130f4520SKenneth D. Merry 1508130f4520SKenneth D. Merry switch (io->scsiio.tag_type) { 1509130f4520SKenneth D. Merry case CTL_TAG_ORDERED: 1510130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1511130f4520SKenneth D. Merry break; 1512130f4520SKenneth D. Merry case CTL_TAG_HEAD_OF_QUEUE: 1513130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1514130f4520SKenneth D. Merry break; 1515130f4520SKenneth D. Merry case CTL_TAG_UNTAGGED: 1516130f4520SKenneth D. Merry case CTL_TAG_SIMPLE: 1517130f4520SKenneth D. Merry case CTL_TAG_ACA: 1518130f4520SKenneth D. Merry default: 1519130f4520SKenneth D. Merry beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1520130f4520SKenneth D. Merry break; 1521130f4520SKenneth D. Merry } 1522130f4520SKenneth D. Merry 152311b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_WRITE) { 1524130f4520SKenneth D. Merry beio->bio_cmd = BIO_WRITE; 1525130f4520SKenneth D. Merry beio->ds_trans_type = DEVSTAT_WRITE; 152611b569f7SAlexander Motin } else { 152711b569f7SAlexander Motin beio->bio_cmd = BIO_READ; 152811b569f7SAlexander Motin beio->ds_trans_type = DEVSTAT_READ; 1529130f4520SKenneth D. Merry } 1530130f4520SKenneth D. Merry 153108a7cce5SAlexander Motin DPRINTF("%s at LBA %jx len %u @%ju\n", 1532130f4520SKenneth D. Merry (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", 1533e86a4142SAlexander Motin (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); 153411b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 153511b569f7SAlexander Motin lbas = CTLBLK_HALF_IO_SIZE; 153611b569f7SAlexander Motin else 153711b569f7SAlexander Motin lbas = CTLBLK_MAX_IO_SIZE; 153811b569f7SAlexander Motin lbas = MIN(lbalen->len - bptrlen->len, lbas / be_lun->blocksize); 1539e86a4142SAlexander Motin beio->io_offset = (lbalen->lba + bptrlen->len) * be_lun->blocksize; 1540e86a4142SAlexander Motin beio->io_len = lbas * be_lun->blocksize; 1541e86a4142SAlexander Motin bptrlen->len += lbas; 1542130f4520SKenneth D. Merry 154308a7cce5SAlexander Motin for (i = 0, len_left = beio->io_len; len_left > 0; i++) { 154408a7cce5SAlexander Motin KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)", 154508a7cce5SAlexander Motin i, CTLBLK_MAX_SEGS)); 1546130f4520SKenneth D. Merry 1547130f4520SKenneth D. Merry /* 1548130f4520SKenneth D. Merry * Setup the S/G entry for this chunk. 1549130f4520SKenneth D. Merry */ 155008a7cce5SAlexander Motin beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left); 1551130f4520SKenneth D. Merry beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1552130f4520SKenneth D. Merry 1553130f4520SKenneth D. Merry DPRINTF("segment %d addr %p len %zd\n", i, 1554130f4520SKenneth D. Merry beio->sg_segs[i].addr, beio->sg_segs[i].len); 1555130f4520SKenneth D. Merry 155611b569f7SAlexander Motin /* Set up second segment for compare operation. */ 155711b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) { 155811b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].len = 155911b569f7SAlexander Motin beio->sg_segs[i].len; 156011b569f7SAlexander Motin beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = 156111b569f7SAlexander Motin uma_zalloc(be_lun->lun_zone, M_WAITOK); 156211b569f7SAlexander Motin } 156311b569f7SAlexander Motin 1564130f4520SKenneth D. Merry beio->num_segs++; 1565130f4520SKenneth D. Merry len_left -= beio->sg_segs[i].len; 1566130f4520SKenneth D. Merry } 1567e86a4142SAlexander Motin if (bptrlen->len < lbalen->len) 156808a7cce5SAlexander Motin beio->beio_cont = ctl_be_block_next; 156908a7cce5SAlexander Motin io->scsiio.be_move_done = ctl_be_block_move_done; 157011b569f7SAlexander Motin /* For compare we have separate S/G lists for read and datamove. */ 157111b569f7SAlexander Motin if (lbalen->flags & CTL_LLF_COMPARE) 157211b569f7SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS]; 157311b569f7SAlexander Motin else 157408a7cce5SAlexander Motin io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs; 157508a7cce5SAlexander Motin io->scsiio.kern_data_len = beio->io_len; 157608a7cce5SAlexander Motin io->scsiio.kern_data_resid = 0; 157708a7cce5SAlexander Motin io->scsiio.kern_sg_entries = beio->num_segs; 157808a7cce5SAlexander Motin io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST; 1579130f4520SKenneth D. Merry 1580130f4520SKenneth D. Merry /* 1581130f4520SKenneth D. Merry * For the read case, we need to read the data into our buffers and 1582130f4520SKenneth D. Merry * then we can send it back to the user. For the write case, we 1583130f4520SKenneth D. Merry * need to get the data from the user first. 1584130f4520SKenneth D. Merry */ 1585130f4520SKenneth D. Merry if (beio->bio_cmd == BIO_READ) { 1586130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0); 1587130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1588130f4520SKenneth D. Merry } else { 1589130f4520SKenneth D. Merry SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0); 1590130f4520SKenneth D. Merry #ifdef CTL_TIME_IO 1591130f4520SKenneth D. Merry getbintime(&io->io_hdr.dma_start_bt); 1592130f4520SKenneth D. Merry #endif 1593130f4520SKenneth D. Merry ctl_datamove(io); 1594130f4520SKenneth D. Merry } 1595130f4520SKenneth D. Merry } 1596130f4520SKenneth D. Merry 1597130f4520SKenneth D. Merry static void 1598130f4520SKenneth D. Merry ctl_be_block_worker(void *context, int pending) 1599130f4520SKenneth D. Merry { 1600130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 1601130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1602130f4520SKenneth D. Merry union ctl_io *io; 1603130f4520SKenneth D. Merry 1604130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)context; 1605130f4520SKenneth D. Merry softc = be_lun->softc; 1606130f4520SKenneth D. Merry 1607130f4520SKenneth D. Merry DPRINTF("entered\n"); 1608130f4520SKenneth D. Merry 160975c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1610130f4520SKenneth D. Merry for (;;) { 1611130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); 1612130f4520SKenneth D. Merry if (io != NULL) { 1613130f4520SKenneth D. Merry struct ctl_be_block_io *beio; 1614130f4520SKenneth D. Merry 1615130f4520SKenneth D. Merry DPRINTF("datamove queue\n"); 1616130f4520SKenneth D. Merry 1617130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, 1618130f4520SKenneth D. Merry ctl_io_hdr, links); 1619130f4520SKenneth D. Merry 162075c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1621130f4520SKenneth D. Merry 1622e86a4142SAlexander Motin beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1623130f4520SKenneth D. Merry 1624130f4520SKenneth D. Merry be_lun->dispatch(be_lun, beio); 1625130f4520SKenneth D. Merry 162675c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1627130f4520SKenneth D. Merry continue; 1628130f4520SKenneth D. Merry } 1629130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); 1630130f4520SKenneth D. Merry if (io != NULL) { 1631130f4520SKenneth D. Merry DPRINTF("config write queue\n"); 1632130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, 1633130f4520SKenneth D. Merry ctl_io_hdr, links); 163475c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1635130f4520SKenneth D. Merry ctl_be_block_cw_dispatch(be_lun, io); 1636ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 1637ef8daf3fSAlexander Motin continue; 1638ef8daf3fSAlexander Motin } 1639ef8daf3fSAlexander Motin io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); 1640ef8daf3fSAlexander Motin if (io != NULL) { 1641ef8daf3fSAlexander Motin DPRINTF("config read queue\n"); 1642ef8daf3fSAlexander Motin STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, 1643ef8daf3fSAlexander Motin ctl_io_hdr, links); 1644ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 1645ef8daf3fSAlexander Motin ctl_be_block_cr_dispatch(be_lun, io); 164675c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1647130f4520SKenneth D. Merry continue; 1648130f4520SKenneth D. Merry } 1649130f4520SKenneth D. Merry io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); 1650130f4520SKenneth D. Merry if (io != NULL) { 1651130f4520SKenneth D. Merry DPRINTF("input queue\n"); 1652130f4520SKenneth D. Merry 1653130f4520SKenneth D. Merry STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, 1654130f4520SKenneth D. Merry ctl_io_hdr, links); 165575c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1656130f4520SKenneth D. Merry 1657130f4520SKenneth D. Merry /* 1658130f4520SKenneth D. Merry * We must drop the lock, since this routine and 1659130f4520SKenneth D. Merry * its children may sleep. 1660130f4520SKenneth D. Merry */ 1661130f4520SKenneth D. Merry ctl_be_block_dispatch(be_lun, io); 1662130f4520SKenneth D. Merry 166375c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1664130f4520SKenneth D. Merry continue; 1665130f4520SKenneth D. Merry } 1666130f4520SKenneth D. Merry 1667130f4520SKenneth D. Merry /* 1668130f4520SKenneth D. Merry * If we get here, there is no work left in the queues, so 1669130f4520SKenneth D. Merry * just break out and let the task queue go to sleep. 1670130f4520SKenneth D. Merry */ 1671130f4520SKenneth D. Merry break; 1672130f4520SKenneth D. Merry } 167375c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1674130f4520SKenneth D. Merry } 1675130f4520SKenneth D. Merry 1676130f4520SKenneth D. Merry /* 1677130f4520SKenneth D. Merry * Entry point from CTL to the backend for I/O. We queue everything to a 1678130f4520SKenneth D. Merry * work thread, so this just puts the I/O on a queue and wakes up the 1679130f4520SKenneth D. Merry * thread. 1680130f4520SKenneth D. Merry */ 1681130f4520SKenneth D. Merry static int 1682130f4520SKenneth D. Merry ctl_be_block_submit(union ctl_io *io) 1683130f4520SKenneth D. Merry { 1684130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 1685130f4520SKenneth D. Merry struct ctl_be_lun *ctl_be_lun; 1686130f4520SKenneth D. Merry 1687130f4520SKenneth D. Merry DPRINTF("entered\n"); 1688130f4520SKenneth D. Merry 1689130f4520SKenneth D. Merry ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 1690130f4520SKenneth D. Merry CTL_PRIV_BACKEND_LUN].ptr; 1691130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 1692130f4520SKenneth D. Merry 1693130f4520SKenneth D. Merry /* 1694130f4520SKenneth D. Merry * Make sure we only get SCSI I/O. 1695130f4520SKenneth D. Merry */ 1696130f4520SKenneth D. Merry KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type " 1697130f4520SKenneth D. Merry "%#x) encountered", io->io_hdr.io_type)); 1698130f4520SKenneth D. Merry 1699e86a4142SAlexander Motin PRIV(io)->len = 0; 1700e86a4142SAlexander Motin 170175c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 1702130f4520SKenneth D. Merry /* 1703130f4520SKenneth D. Merry * XXX KDM make sure that links is okay to use at this point. 1704130f4520SKenneth D. Merry * Otherwise, we either need to add another field to ctl_io_hdr, 1705130f4520SKenneth D. Merry * or deal with resource allocation here. 1706130f4520SKenneth D. Merry */ 1707130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 170875c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 1709130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1710130f4520SKenneth D. Merry 17119c71cd5aSAlexander Motin return (CTL_RETVAL_COMPLETE); 1712130f4520SKenneth D. Merry } 1713130f4520SKenneth D. Merry 1714130f4520SKenneth D. Merry static int 1715130f4520SKenneth D. Merry ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 1716130f4520SKenneth D. Merry int flag, struct thread *td) 1717130f4520SKenneth D. Merry { 1718130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 1719130f4520SKenneth D. Merry int error; 1720130f4520SKenneth D. Merry 1721130f4520SKenneth D. Merry softc = &backend_block_softc; 1722130f4520SKenneth D. Merry 1723130f4520SKenneth D. Merry error = 0; 1724130f4520SKenneth D. Merry 1725130f4520SKenneth D. Merry switch (cmd) { 1726130f4520SKenneth D. Merry case CTL_LUN_REQ: { 1727130f4520SKenneth D. Merry struct ctl_lun_req *lun_req; 1728130f4520SKenneth D. Merry 1729130f4520SKenneth D. Merry lun_req = (struct ctl_lun_req *)addr; 1730130f4520SKenneth D. Merry 1731130f4520SKenneth D. Merry switch (lun_req->reqtype) { 1732130f4520SKenneth D. Merry case CTL_LUNREQ_CREATE: 1733130f4520SKenneth D. Merry error = ctl_be_block_create(softc, lun_req); 1734130f4520SKenneth D. Merry break; 1735130f4520SKenneth D. Merry case CTL_LUNREQ_RM: 1736130f4520SKenneth D. Merry error = ctl_be_block_rm(softc, lun_req); 1737130f4520SKenneth D. Merry break; 173881177295SEdward Tomasz Napierala case CTL_LUNREQ_MODIFY: 173981177295SEdward Tomasz Napierala error = ctl_be_block_modify(softc, lun_req); 174081177295SEdward Tomasz Napierala break; 1741130f4520SKenneth D. Merry default: 1742130f4520SKenneth D. Merry lun_req->status = CTL_LUN_ERROR; 1743130f4520SKenneth D. Merry snprintf(lun_req->error_str, sizeof(lun_req->error_str), 174419720f41SAlexander Motin "invalid LUN request type %d", 1745130f4520SKenneth D. Merry lun_req->reqtype); 1746130f4520SKenneth D. Merry break; 1747130f4520SKenneth D. Merry } 1748130f4520SKenneth D. Merry break; 1749130f4520SKenneth D. Merry } 1750130f4520SKenneth D. Merry default: 1751130f4520SKenneth D. Merry error = ENOTTY; 1752130f4520SKenneth D. Merry break; 1753130f4520SKenneth D. Merry } 1754130f4520SKenneth D. Merry 1755130f4520SKenneth D. Merry return (error); 1756130f4520SKenneth D. Merry } 1757130f4520SKenneth D. Merry 1758130f4520SKenneth D. Merry static int 1759130f4520SKenneth D. Merry ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1760130f4520SKenneth D. Merry { 1761130f4520SKenneth D. Merry struct ctl_be_block_filedata *file_data; 1762130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 176334961f40SAlexander Motin char *value; 1764130f4520SKenneth D. Merry struct vattr vattr; 176534961f40SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos; 1766130f4520SKenneth D. Merry int error; 1767130f4520SKenneth D. Merry 1768130f4520SKenneth D. Merry error = 0; 1769130f4520SKenneth D. Merry file_data = &be_lun->backend.file; 177019720f41SAlexander Motin params = &be_lun->params; 1771130f4520SKenneth D. Merry 1772130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_FILE; 1773130f4520SKenneth D. Merry be_lun->dispatch = ctl_be_block_dispatch_file; 1774130f4520SKenneth D. Merry be_lun->lun_flush = ctl_be_block_flush_file; 1775ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_file; 177653c146deSAlexander Motin be_lun->getattr = ctl_be_block_getattr_file; 1777130f4520SKenneth D. Merry 1778130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 1779130f4520SKenneth D. Merry if (error != 0) { 1780130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1781130f4520SKenneth D. Merry "error calling VOP_GETATTR() for file %s", 1782130f4520SKenneth D. Merry be_lun->dev_path); 1783130f4520SKenneth D. Merry return (error); 1784130f4520SKenneth D. Merry } 1785130f4520SKenneth D. Merry 1786130f4520SKenneth D. Merry /* 1787130f4520SKenneth D. Merry * Verify that we have the ability to upgrade to exclusive 1788130f4520SKenneth D. Merry * access on this file so we can trap errors at open instead 1789130f4520SKenneth D. Merry * of reporting them during first access. 1790130f4520SKenneth D. Merry */ 1791130f4520SKenneth D. Merry if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) { 1792130f4520SKenneth D. Merry vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY); 1793130f4520SKenneth D. Merry if (be_lun->vn->v_iflag & VI_DOOMED) { 1794130f4520SKenneth D. Merry error = EBADF; 1795130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1796130f4520SKenneth D. Merry "error locking file %s", be_lun->dev_path); 1797130f4520SKenneth D. Merry return (error); 1798130f4520SKenneth D. Merry } 1799130f4520SKenneth D. Merry } 1800130f4520SKenneth D. Merry 1801130f4520SKenneth D. Merry 1802130f4520SKenneth D. Merry file_data->cred = crhold(curthread->td_ucred); 180381177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) 180481177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 180581177295SEdward Tomasz Napierala else 1806130f4520SKenneth D. Merry be_lun->size_bytes = vattr.va_size; 1807130f4520SKenneth D. Merry /* 1808130f4520SKenneth D. Merry * We set the multi thread flag for file operations because all 1809130f4520SKenneth D. Merry * filesystems (in theory) are capable of allowing multiple readers 1810130f4520SKenneth D. Merry * of a file at once. So we want to get the maximum possible 1811130f4520SKenneth D. Merry * concurrency. 1812130f4520SKenneth D. Merry */ 1813130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD; 1814130f4520SKenneth D. Merry 1815130f4520SKenneth D. Merry /* 181620a28d6cSAlexander Motin * For files we can use any logical block size. Prefer 512 bytes 181720a28d6cSAlexander Motin * for compatibility reasons. If file's vattr.va_blocksize 181820a28d6cSAlexander Motin * (preferred I/O block size) is bigger and multiple to chosen 181920a28d6cSAlexander Motin * logical block size -- report it as physical block size. 1820130f4520SKenneth D. Merry */ 1821130f4520SKenneth D. Merry if (params->blocksize_bytes != 0) 1822130f4520SKenneth D. Merry be_lun->blocksize = params->blocksize_bytes; 1823130f4520SKenneth D. Merry else 1824130f4520SKenneth D. Merry be_lun->blocksize = 512; 182534961f40SAlexander Motin 182634961f40SAlexander Motin us = ps = vattr.va_blocksize; 182734961f40SAlexander Motin uo = po = 0; 182834961f40SAlexander Motin 182934961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblocksize"); 183034961f40SAlexander Motin if (value != NULL) 183134961f40SAlexander Motin ctl_expand_number(value, &ps); 183234961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblockoffset"); 183334961f40SAlexander Motin if (value != NULL) 183434961f40SAlexander Motin ctl_expand_number(value, &po); 183534961f40SAlexander Motin pss = ps / be_lun->blocksize; 183634961f40SAlexander Motin pos = po / be_lun->blocksize; 183734961f40SAlexander Motin if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) && 183834961f40SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) { 183920a28d6cSAlexander Motin be_lun->pblockexp = fls(pss) - 1; 184034961f40SAlexander Motin be_lun->pblockoff = (pss - pos) % pss; 184134961f40SAlexander Motin } 184234961f40SAlexander Motin 184334961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublocksize"); 184434961f40SAlexander Motin if (value != NULL) 184534961f40SAlexander Motin ctl_expand_number(value, &us); 184634961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublockoffset"); 184734961f40SAlexander Motin if (value != NULL) 184834961f40SAlexander Motin ctl_expand_number(value, &uo); 184934961f40SAlexander Motin uss = us / be_lun->blocksize; 185034961f40SAlexander Motin uos = uo / be_lun->blocksize; 185134961f40SAlexander Motin if ((uss > 0) && (uss * be_lun->blocksize == us) && (uss >= uos) && 185234961f40SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * be_lun->blocksize == uo)) { 185334961f40SAlexander Motin be_lun->ublockexp = fls(uss) - 1; 185434961f40SAlexander Motin be_lun->ublockoff = (uss - uos) % uss; 185520a28d6cSAlexander Motin } 1856130f4520SKenneth D. Merry 1857130f4520SKenneth D. Merry /* 1858130f4520SKenneth D. Merry * Sanity check. The media size has to be at least one 1859130f4520SKenneth D. Merry * sector long. 1860130f4520SKenneth D. Merry */ 1861130f4520SKenneth D. Merry if (be_lun->size_bytes < be_lun->blocksize) { 1862130f4520SKenneth D. Merry error = EINVAL; 1863130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 1864130f4520SKenneth D. Merry "file %s size %ju < block size %u", be_lun->dev_path, 1865130f4520SKenneth D. Merry (uintmax_t)be_lun->size_bytes, be_lun->blocksize); 1866130f4520SKenneth D. Merry } 1867cb8727e2SAlexander Motin 1868cb8727e2SAlexander Motin be_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / be_lun->blocksize; 1869130f4520SKenneth D. Merry return (error); 1870130f4520SKenneth D. Merry } 1871130f4520SKenneth D. Merry 1872130f4520SKenneth D. Merry static int 1873130f4520SKenneth D. Merry ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1874130f4520SKenneth D. Merry { 1875130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 1876130f4520SKenneth D. Merry struct vattr vattr; 1877130f4520SKenneth D. Merry struct cdev *dev; 1878130f4520SKenneth D. Merry struct cdevsw *devsw; 187934961f40SAlexander Motin char *value; 1880fbc8d4ffSAlexander Motin int error, atomic, maxio, unmap; 188134961f40SAlexander Motin off_t ps, pss, po, pos, us, uss, uo, uos; 1882130f4520SKenneth D. Merry 188319720f41SAlexander Motin params = &be_lun->params; 1884130f4520SKenneth D. Merry 1885130f4520SKenneth D. Merry be_lun->dev_type = CTL_BE_BLOCK_DEV; 1886130f4520SKenneth D. Merry be_lun->backend.dev.cdev = be_lun->vn->v_rdev; 1887130f4520SKenneth D. Merry be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev, 1888130f4520SKenneth D. Merry &be_lun->backend.dev.dev_ref); 1889130f4520SKenneth D. Merry if (be_lun->backend.dev.csw == NULL) 1890130f4520SKenneth D. Merry panic("Unable to retrieve device switch"); 1891ef8daf3fSAlexander Motin if (strcmp(be_lun->backend.dev.csw->d_name, "zvol") == 0) { 189267f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_zvol; 1893ef8daf3fSAlexander Motin be_lun->get_lba_status = ctl_be_block_gls_zvol; 1894cb8727e2SAlexander Motin atomic = maxio = CTLBLK_MAX_IO_SIZE; 1895cb8727e2SAlexander Motin } else { 189667f586a8SAlexander Motin be_lun->dispatch = ctl_be_block_dispatch_dev; 1897cb8727e2SAlexander Motin atomic = 0; 1898cb8727e2SAlexander Motin maxio = be_lun->backend.dev.cdev->si_iosize_max; 1899cb8727e2SAlexander Motin if (maxio <= 0) 1900cb8727e2SAlexander Motin maxio = DFLTPHYS; 1901cb8727e2SAlexander Motin if (maxio > CTLBLK_MAX_IO_SIZE) 1902cb8727e2SAlexander Motin maxio = CTLBLK_MAX_IO_SIZE; 1903cb8727e2SAlexander Motin } 190467f586a8SAlexander Motin be_lun->lun_flush = ctl_be_block_flush_dev; 1905c3e7ba3eSAlexander Motin be_lun->getattr = ctl_be_block_getattr_dev; 1906130f4520SKenneth D. Merry 1907130f4520SKenneth D. Merry error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED); 1908130f4520SKenneth D. Merry if (error) { 1909130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 191019720f41SAlexander Motin "error getting vnode attributes for device %s", 191119720f41SAlexander Motin be_lun->dev_path); 1912130f4520SKenneth D. Merry return (error); 1913130f4520SKenneth D. Merry } 1914130f4520SKenneth D. Merry 1915130f4520SKenneth D. Merry dev = be_lun->vn->v_rdev; 1916130f4520SKenneth D. Merry devsw = dev->si_devsw; 1917130f4520SKenneth D. Merry if (!devsw->d_ioctl) { 1918130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 191919720f41SAlexander Motin "no d_ioctl for device %s!", 1920130f4520SKenneth D. Merry be_lun->dev_path); 1921130f4520SKenneth D. Merry return (ENODEV); 1922130f4520SKenneth D. Merry } 1923130f4520SKenneth D. Merry 1924130f4520SKenneth D. Merry error = devsw->d_ioctl(dev, DIOCGSECTORSIZE, 1925130f4520SKenneth D. Merry (caddr_t)&be_lun->blocksize, FREAD, 1926130f4520SKenneth D. Merry curthread); 1927130f4520SKenneth D. Merry if (error) { 1928130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 192919720f41SAlexander Motin "error %d returned for DIOCGSECTORSIZE ioctl " 193019720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 1931130f4520SKenneth D. Merry return (error); 1932130f4520SKenneth D. Merry } 1933130f4520SKenneth D. Merry 1934130f4520SKenneth D. Merry /* 1935130f4520SKenneth D. Merry * If the user has asked for a blocksize that is greater than the 1936130f4520SKenneth D. Merry * backing device's blocksize, we can do it only if the blocksize 1937130f4520SKenneth D. Merry * the user is asking for is an even multiple of the underlying 1938130f4520SKenneth D. Merry * device's blocksize. 1939130f4520SKenneth D. Merry */ 1940130f4520SKenneth D. Merry if ((params->blocksize_bytes != 0) 1941130f4520SKenneth D. Merry && (params->blocksize_bytes > be_lun->blocksize)) { 1942130f4520SKenneth D. Merry uint32_t bs_multiple, tmp_blocksize; 1943130f4520SKenneth D. Merry 1944130f4520SKenneth D. Merry bs_multiple = params->blocksize_bytes / be_lun->blocksize; 1945130f4520SKenneth D. Merry 1946130f4520SKenneth D. Merry tmp_blocksize = bs_multiple * be_lun->blocksize; 1947130f4520SKenneth D. Merry 1948130f4520SKenneth D. Merry if (tmp_blocksize == params->blocksize_bytes) { 1949130f4520SKenneth D. Merry be_lun->blocksize = params->blocksize_bytes; 1950130f4520SKenneth D. Merry } else { 1951130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 195219720f41SAlexander Motin "requested blocksize %u is not an even " 1953130f4520SKenneth D. Merry "multiple of backing device blocksize %u", 195419720f41SAlexander Motin params->blocksize_bytes, 1955130f4520SKenneth D. Merry be_lun->blocksize); 1956130f4520SKenneth D. Merry return (EINVAL); 1957130f4520SKenneth D. Merry 1958130f4520SKenneth D. Merry } 1959130f4520SKenneth D. Merry } else if ((params->blocksize_bytes != 0) 1960130f4520SKenneth D. Merry && (params->blocksize_bytes != be_lun->blocksize)) { 1961130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 196219720f41SAlexander Motin "requested blocksize %u < backing device " 196319720f41SAlexander Motin "blocksize %u", params->blocksize_bytes, 1964130f4520SKenneth D. Merry be_lun->blocksize); 1965130f4520SKenneth D. Merry return (EINVAL); 1966130f4520SKenneth D. Merry } 1967130f4520SKenneth D. Merry 1968130f4520SKenneth D. Merry error = devsw->d_ioctl(dev, DIOCGMEDIASIZE, 1969130f4520SKenneth D. Merry (caddr_t)&be_lun->size_bytes, FREAD, 1970130f4520SKenneth D. Merry curthread); 1971130f4520SKenneth D. Merry if (error) { 1972130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 197319720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE " 197419720f41SAlexander Motin " ioctl on %s!", error, 197581177295SEdward Tomasz Napierala be_lun->dev_path); 1976130f4520SKenneth D. Merry return (error); 1977130f4520SKenneth D. Merry } 1978130f4520SKenneth D. Merry 197981177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 198081177295SEdward Tomasz Napierala if (params->lun_size_bytes > be_lun->size_bytes) { 198181177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 198219720f41SAlexander Motin "requested LUN size %ju > backing device " 198319720f41SAlexander Motin "size %ju", 198481177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 198581177295SEdward Tomasz Napierala (uintmax_t)be_lun->size_bytes); 198681177295SEdward Tomasz Napierala return (EINVAL); 1987130f4520SKenneth D. Merry } 1988130f4520SKenneth D. Merry 198981177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 199081177295SEdward Tomasz Napierala } 199181177295SEdward Tomasz Napierala 1992f6012722SAlexander Motin error = devsw->d_ioctl(dev, DIOCGSTRIPESIZE, 1993f6012722SAlexander Motin (caddr_t)&ps, FREAD, curthread); 1994f6012722SAlexander Motin if (error) 1995f6012722SAlexander Motin ps = po = 0; 1996f6012722SAlexander Motin else { 1997f6012722SAlexander Motin error = devsw->d_ioctl(dev, DIOCGSTRIPEOFFSET, 1998f6012722SAlexander Motin (caddr_t)&po, FREAD, curthread); 1999f6012722SAlexander Motin if (error) 2000f6012722SAlexander Motin po = 0; 2001f6012722SAlexander Motin } 200234961f40SAlexander Motin us = ps; 200334961f40SAlexander Motin uo = po; 200434961f40SAlexander Motin 200534961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblocksize"); 200634961f40SAlexander Motin if (value != NULL) 200734961f40SAlexander Motin ctl_expand_number(value, &ps); 200834961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "pblockoffset"); 200934961f40SAlexander Motin if (value != NULL) 201034961f40SAlexander Motin ctl_expand_number(value, &po); 2011f6012722SAlexander Motin pss = ps / be_lun->blocksize; 2012f6012722SAlexander Motin pos = po / be_lun->blocksize; 2013f6012722SAlexander Motin if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) && 2014f6012722SAlexander Motin ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) { 2015f6012722SAlexander Motin be_lun->pblockexp = fls(pss) - 1; 2016f6012722SAlexander Motin be_lun->pblockoff = (pss - pos) % pss; 2017f6012722SAlexander Motin } 2018f6012722SAlexander Motin 201934961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublocksize"); 202034961f40SAlexander Motin if (value != NULL) 202134961f40SAlexander Motin ctl_expand_number(value, &us); 202234961f40SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "ublockoffset"); 202334961f40SAlexander Motin if (value != NULL) 202434961f40SAlexander Motin ctl_expand_number(value, &uo); 202534961f40SAlexander Motin uss = us / be_lun->blocksize; 202634961f40SAlexander Motin uos = uo / be_lun->blocksize; 202734961f40SAlexander Motin if ((uss > 0) && (uss * be_lun->blocksize == us) && (uss >= uos) && 202834961f40SAlexander Motin ((uss & (uss - 1)) == 0) && (uos * be_lun->blocksize == uo)) { 202934961f40SAlexander Motin be_lun->ublockexp = fls(uss) - 1; 203034961f40SAlexander Motin be_lun->ublockoff = (uss - uos) % uss; 203134961f40SAlexander Motin } 203234961f40SAlexander Motin 2033cb8727e2SAlexander Motin be_lun->atomicblock = atomic / be_lun->blocksize; 2034cb8727e2SAlexander Motin be_lun->opttxferlen = maxio / be_lun->blocksize; 2035fbc8d4ffSAlexander Motin 2036fbc8d4ffSAlexander Motin if (be_lun->dispatch == ctl_be_block_dispatch_zvol) { 2037fbc8d4ffSAlexander Motin unmap = 1; 2038fbc8d4ffSAlexander Motin } else { 2039fbc8d4ffSAlexander Motin struct diocgattr_arg arg; 2040fbc8d4ffSAlexander Motin 2041fbc8d4ffSAlexander Motin strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); 2042fbc8d4ffSAlexander Motin arg.len = sizeof(arg.value.i); 2043fbc8d4ffSAlexander Motin error = devsw->d_ioctl(dev, DIOCGATTR, 2044fbc8d4ffSAlexander Motin (caddr_t)&arg, FREAD, curthread); 2045fbc8d4ffSAlexander Motin unmap = (error == 0) ? arg.value.i : 0; 2046fbc8d4ffSAlexander Motin } 2047fbc8d4ffSAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "unmap"); 2048fbc8d4ffSAlexander Motin if (value != NULL) 2049fbc8d4ffSAlexander Motin unmap = (strcmp(value, "on") == 0); 2050fbc8d4ffSAlexander Motin if (unmap) 2051fbc8d4ffSAlexander Motin be_lun->unmap = ctl_be_block_unmap_dev; 2052fbc8d4ffSAlexander Motin 205381177295SEdward Tomasz Napierala return (0); 205481177295SEdward Tomasz Napierala } 2055130f4520SKenneth D. Merry 2056130f4520SKenneth D. Merry static int 2057130f4520SKenneth D. Merry ctl_be_block_close(struct ctl_be_block_lun *be_lun) 2058130f4520SKenneth D. Merry { 2059130f4520SKenneth D. Merry DROP_GIANT(); 2060130f4520SKenneth D. Merry if (be_lun->vn) { 2061130f4520SKenneth D. Merry int flags = FREAD | FWRITE; 2062130f4520SKenneth D. Merry 2063130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2064130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2065130f4520SKenneth D. Merry if (be_lun->backend.dev.csw) { 2066130f4520SKenneth D. Merry dev_relthread(be_lun->backend.dev.cdev, 2067130f4520SKenneth D. Merry be_lun->backend.dev.dev_ref); 2068130f4520SKenneth D. Merry be_lun->backend.dev.csw = NULL; 2069130f4520SKenneth D. Merry be_lun->backend.dev.cdev = NULL; 2070130f4520SKenneth D. Merry } 2071130f4520SKenneth D. Merry break; 2072130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2073130f4520SKenneth D. Merry break; 2074130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2075025a2301SEdward Tomasz Napierala break; 2076130f4520SKenneth D. Merry default: 2077130f4520SKenneth D. Merry panic("Unexpected backend type."); 2078130f4520SKenneth D. Merry break; 2079130f4520SKenneth D. Merry } 2080130f4520SKenneth D. Merry 2081130f4520SKenneth D. Merry (void)vn_close(be_lun->vn, flags, NOCRED, curthread); 2082130f4520SKenneth D. Merry be_lun->vn = NULL; 2083130f4520SKenneth D. Merry 2084130f4520SKenneth D. Merry switch (be_lun->dev_type) { 2085130f4520SKenneth D. Merry case CTL_BE_BLOCK_DEV: 2086130f4520SKenneth D. Merry break; 2087130f4520SKenneth D. Merry case CTL_BE_BLOCK_FILE: 2088130f4520SKenneth D. Merry if (be_lun->backend.file.cred != NULL) { 2089130f4520SKenneth D. Merry crfree(be_lun->backend.file.cred); 2090130f4520SKenneth D. Merry be_lun->backend.file.cred = NULL; 2091130f4520SKenneth D. Merry } 2092130f4520SKenneth D. Merry break; 2093130f4520SKenneth D. Merry case CTL_BE_BLOCK_NONE: 2094025a2301SEdward Tomasz Napierala break; 2095130f4520SKenneth D. Merry default: 2096130f4520SKenneth D. Merry panic("Unexpected backend type."); 2097130f4520SKenneth D. Merry break; 2098130f4520SKenneth D. Merry } 209919720f41SAlexander Motin be_lun->dev_type = CTL_BE_BLOCK_NONE; 2100130f4520SKenneth D. Merry } 2101130f4520SKenneth D. Merry PICKUP_GIANT(); 2102130f4520SKenneth D. Merry 2103130f4520SKenneth D. Merry return (0); 2104130f4520SKenneth D. Merry } 2105130f4520SKenneth D. Merry 2106130f4520SKenneth D. Merry static int 2107130f4520SKenneth D. Merry ctl_be_block_open(struct ctl_be_block_softc *softc, 2108130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 2109130f4520SKenneth D. Merry { 2110130f4520SKenneth D. Merry struct nameidata nd; 2111130f4520SKenneth D. Merry int flags; 2112130f4520SKenneth D. Merry int error; 2113130f4520SKenneth D. Merry 2114130f4520SKenneth D. Merry /* 2115130f4520SKenneth D. Merry * XXX KDM allow a read-only option? 2116130f4520SKenneth D. Merry */ 2117130f4520SKenneth D. Merry flags = FREAD | FWRITE; 2118130f4520SKenneth D. Merry error = 0; 2119130f4520SKenneth D. Merry 2120130f4520SKenneth D. Merry if (rootvnode == NULL) { 2121130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 212219720f41SAlexander Motin "Root filesystem is not mounted"); 2123130f4520SKenneth D. Merry return (1); 2124130f4520SKenneth D. Merry } 2125130f4520SKenneth D. Merry 2126*8a08cec1SMateusz Guzik pwd_ensure_dirs(); 2127130f4520SKenneth D. Merry 2128130f4520SKenneth D. Merry again: 2129130f4520SKenneth D. Merry NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread); 2130130f4520SKenneth D. Merry error = vn_open(&nd, &flags, 0, NULL); 2131130f4520SKenneth D. Merry if (error) { 2132130f4520SKenneth D. Merry /* 2133130f4520SKenneth D. Merry * This is the only reasonable guess we can make as far as 2134130f4520SKenneth D. Merry * path if the user doesn't give us a fully qualified path. 2135130f4520SKenneth D. Merry * If they want to specify a file, they need to specify the 2136130f4520SKenneth D. Merry * full path. 2137130f4520SKenneth D. Merry */ 2138130f4520SKenneth D. Merry if (be_lun->dev_path[0] != '/') { 2139130f4520SKenneth D. Merry char *dev_path = "/dev/"; 2140130f4520SKenneth D. Merry char *dev_name; 2141130f4520SKenneth D. Merry 2142130f4520SKenneth D. Merry /* Try adding device path at beginning of name */ 2143130f4520SKenneth D. Merry dev_name = malloc(strlen(be_lun->dev_path) 2144130f4520SKenneth D. Merry + strlen(dev_path) + 1, 2145130f4520SKenneth D. Merry M_CTLBLK, M_WAITOK); 2146130f4520SKenneth D. Merry if (dev_name) { 2147130f4520SKenneth D. Merry sprintf(dev_name, "%s%s", dev_path, 2148130f4520SKenneth D. Merry be_lun->dev_path); 2149130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 2150130f4520SKenneth D. Merry be_lun->dev_path = dev_name; 2151130f4520SKenneth D. Merry goto again; 2152130f4520SKenneth D. Merry } 2153130f4520SKenneth D. Merry } 2154130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 215519720f41SAlexander Motin "error opening %s: %d", be_lun->dev_path, error); 2156130f4520SKenneth D. Merry return (error); 2157130f4520SKenneth D. Merry } 2158130f4520SKenneth D. Merry 2159130f4520SKenneth D. Merry NDFREE(&nd, NDF_ONLY_PNBUF); 2160130f4520SKenneth D. Merry 2161130f4520SKenneth D. Merry be_lun->vn = nd.ni_vp; 2162130f4520SKenneth D. Merry 2163130f4520SKenneth D. Merry /* We only support disks and files. */ 2164130f4520SKenneth D. Merry if (vn_isdisk(be_lun->vn, &error)) { 2165130f4520SKenneth D. Merry error = ctl_be_block_open_dev(be_lun, req); 2166130f4520SKenneth D. Merry } else if (be_lun->vn->v_type == VREG) { 2167130f4520SKenneth D. Merry error = ctl_be_block_open_file(be_lun, req); 2168130f4520SKenneth D. Merry } else { 2169130f4520SKenneth D. Merry error = EINVAL; 2170130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 2171025a2301SEdward Tomasz Napierala "%s is not a disk or plain file", be_lun->dev_path); 2172130f4520SKenneth D. Merry } 2173130f4520SKenneth D. Merry VOP_UNLOCK(be_lun->vn, 0); 2174130f4520SKenneth D. Merry 2175130f4520SKenneth D. Merry if (error != 0) { 2176130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 2177130f4520SKenneth D. Merry return (error); 2178130f4520SKenneth D. Merry } 2179130f4520SKenneth D. Merry 2180130f4520SKenneth D. Merry be_lun->blocksize_shift = fls(be_lun->blocksize) - 1; 2181130f4520SKenneth D. Merry be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift; 2182130f4520SKenneth D. Merry 2183130f4520SKenneth D. Merry return (0); 2184130f4520SKenneth D. Merry } 2185130f4520SKenneth D. Merry 2186130f4520SKenneth D. Merry static int 2187130f4520SKenneth D. Merry ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2188130f4520SKenneth D. Merry { 2189130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2190130f4520SKenneth D. Merry struct ctl_lun_create_params *params; 219157a5db13SAlexander Motin char num_thread_str[16]; 2192130f4520SKenneth D. Merry char tmpstr[32]; 219357a5db13SAlexander Motin char *value; 2194fbc8d4ffSAlexander Motin int retval, num_threads; 219557a5db13SAlexander Motin int tmp_num_threads; 2196130f4520SKenneth D. Merry 2197130f4520SKenneth D. Merry params = &req->reqdata.create; 2198130f4520SKenneth D. Merry retval = 0; 219919720f41SAlexander Motin req->status = CTL_LUN_OK; 2200130f4520SKenneth D. Merry 2201130f4520SKenneth D. Merry num_threads = cbb_num_threads; 2202130f4520SKenneth D. Merry 2203130f4520SKenneth D. Merry be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK); 2204130f4520SKenneth D. Merry 220519720f41SAlexander Motin be_lun->params = req->reqdata.create; 2206130f4520SKenneth D. Merry be_lun->softc = softc; 2207130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->input_queue); 2208ef8daf3fSAlexander Motin STAILQ_INIT(&be_lun->config_read_queue); 2209130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->config_write_queue); 2210130f4520SKenneth D. Merry STAILQ_INIT(&be_lun->datamove_queue); 2211130f4520SKenneth D. Merry sprintf(be_lun->lunname, "cblk%d", softc->num_luns); 221275c7a1d3SAlexander Motin mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF); 221375c7a1d3SAlexander Motin mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF); 221443fb3a65SAlexander Motin ctl_init_opts(&be_lun->ctl_be_lun.options, 221543fb3a65SAlexander Motin req->num_be_args, req->kern_be_args); 2216130f4520SKenneth D. Merry 221708a7cce5SAlexander Motin be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG, 2218eeb94054SAlexander Motin NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0); 2219130f4520SKenneth D. Merry 2220130f4520SKenneth D. Merry if (be_lun->lun_zone == NULL) { 2221130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 222219720f41SAlexander Motin "error allocating UMA zone"); 2223130f4520SKenneth D. Merry goto bailout_error; 2224130f4520SKenneth D. Merry } 2225130f4520SKenneth D. Merry 2226130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_DEV_TYPE) 2227130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type = params->device_type; 2228130f4520SKenneth D. Merry else 2229130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type = T_DIRECT; 2230130f4520SKenneth D. Merry 2231130f4520SKenneth D. Merry if (be_lun->ctl_be_lun.lun_type == T_DIRECT) { 223243fb3a65SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "file"); 2233eb3687a6SAlexander Motin if (value == NULL) { 2234130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 223519720f41SAlexander Motin "no file argument specified"); 2236130f4520SKenneth D. Merry goto bailout_error; 2237130f4520SKenneth D. Merry } 2238eb3687a6SAlexander Motin be_lun->dev_path = strdup(value, M_CTLBLK); 223919720f41SAlexander Motin be_lun->blocksize = 512; 224019720f41SAlexander Motin be_lun->blocksize_shift = fls(be_lun->blocksize) - 1; 2241130f4520SKenneth D. Merry 2242130f4520SKenneth D. Merry retval = ctl_be_block_open(softc, be_lun, req); 2243130f4520SKenneth D. Merry if (retval != 0) { 2244130f4520SKenneth D. Merry retval = 0; 224519720f41SAlexander Motin req->status = CTL_LUN_WARNING; 2246130f4520SKenneth D. Merry } 2247130f4520SKenneth D. Merry } else { 2248130f4520SKenneth D. Merry /* 2249130f4520SKenneth D. Merry * For processor devices, we don't have any size. 2250130f4520SKenneth D. Merry */ 2251130f4520SKenneth D. Merry be_lun->blocksize = 0; 2252f6012722SAlexander Motin be_lun->pblockexp = 0; 2253f6012722SAlexander Motin be_lun->pblockoff = 0; 225434961f40SAlexander Motin be_lun->ublockexp = 0; 225534961f40SAlexander Motin be_lun->ublockoff = 0; 2256130f4520SKenneth D. Merry be_lun->size_blocks = 0; 2257130f4520SKenneth D. Merry be_lun->size_bytes = 0; 2258130f4520SKenneth D. Merry be_lun->ctl_be_lun.maxlba = 0; 2259130f4520SKenneth D. Merry 2260130f4520SKenneth D. Merry /* 2261130f4520SKenneth D. Merry * Default to just 1 thread for processor devices. 2262130f4520SKenneth D. Merry */ 2263130f4520SKenneth D. Merry num_threads = 1; 2264130f4520SKenneth D. Merry } 2265130f4520SKenneth D. Merry 2266130f4520SKenneth D. Merry /* 2267130f4520SKenneth D. Merry * XXX This searching loop might be refactored to be combined with 2268130f4520SKenneth D. Merry * the loop above, 2269130f4520SKenneth D. Merry */ 227043fb3a65SAlexander Motin value = ctl_get_opt(&be_lun->ctl_be_lun.options, "num_threads"); 227157a5db13SAlexander Motin if (value != NULL) { 227257a5db13SAlexander Motin tmp_num_threads = strtol(value, NULL, 0); 2273130f4520SKenneth D. Merry 2274130f4520SKenneth D. Merry /* 2275130f4520SKenneth D. Merry * We don't let the user specify less than one 2276130f4520SKenneth D. Merry * thread, but hope he's clueful enough not to 2277130f4520SKenneth D. Merry * specify 1000 threads. 2278130f4520SKenneth D. Merry */ 2279130f4520SKenneth D. Merry if (tmp_num_threads < 1) { 2280130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 228119720f41SAlexander Motin "invalid number of threads %s", 228219720f41SAlexander Motin num_thread_str); 2283130f4520SKenneth D. Merry goto bailout_error; 2284130f4520SKenneth D. Merry } 2285130f4520SKenneth D. Merry num_threads = tmp_num_threads; 228657a5db13SAlexander Motin } 2287130f4520SKenneth D. Merry 2288130f4520SKenneth D. Merry be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED; 2289130f4520SKenneth D. Merry be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY; 229019720f41SAlexander Motin if (be_lun->vn == NULL) 229119720f41SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_OFFLINE; 2292fbc8d4ffSAlexander Motin if (be_lun->unmap != NULL) 2293ee7f31c0SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP; 2294bfbfc4a3SAlexander Motin if (be_lun->dispatch != ctl_be_block_dispatch_dev) 2295bfbfc4a3SAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_SERSEQ_READ; 2296130f4520SKenneth D. Merry be_lun->ctl_be_lun.be_lun = be_lun; 229719720f41SAlexander Motin be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 229819720f41SAlexander Motin 0 : (be_lun->size_blocks - 1); 2299130f4520SKenneth D. Merry be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 2300f6012722SAlexander Motin be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 2301f6012722SAlexander Motin be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 230234961f40SAlexander Motin be_lun->ctl_be_lun.ublockexp = be_lun->ublockexp; 230334961f40SAlexander Motin be_lun->ctl_be_lun.ublockoff = be_lun->ublockoff; 2304cb8727e2SAlexander Motin be_lun->ctl_be_lun.atomicblock = be_lun->atomicblock; 2305cb8727e2SAlexander Motin be_lun->ctl_be_lun.opttxferlen = be_lun->opttxferlen; 2306130f4520SKenneth D. Merry /* Tell the user the blocksize we ended up using */ 230719720f41SAlexander Motin params->lun_size_bytes = be_lun->size_bytes; 2308130f4520SKenneth D. Merry params->blocksize_bytes = be_lun->blocksize; 2309130f4520SKenneth D. Merry if (params->flags & CTL_LUN_FLAG_ID_REQ) { 2310130f4520SKenneth D. Merry be_lun->ctl_be_lun.req_lun_id = params->req_lun_id; 2311130f4520SKenneth D. Merry be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ; 2312130f4520SKenneth D. Merry } else 2313130f4520SKenneth D. Merry be_lun->ctl_be_lun.req_lun_id = 0; 2314130f4520SKenneth D. Merry 2315130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown; 2316130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_config_status = 2317130f4520SKenneth D. Merry ctl_be_block_lun_config_status; 2318130f4520SKenneth D. Merry be_lun->ctl_be_lun.be = &ctl_be_block_driver; 2319130f4520SKenneth D. Merry 2320130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { 2321130f4520SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d", 2322130f4520SKenneth D. Merry softc->num_luns); 2323130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr, 2324e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.serial_num), 2325130f4520SKenneth D. Merry sizeof(tmpstr))); 2326130f4520SKenneth D. Merry 2327130f4520SKenneth D. Merry /* Tell the user what we used for a serial number */ 2328130f4520SKenneth D. Merry strncpy((char *)params->serial_num, tmpstr, 2329e7038eb7SAlexander Motin MIN(sizeof(params->serial_num), sizeof(tmpstr))); 2330130f4520SKenneth D. Merry } else { 2331130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.serial_num, 2332130f4520SKenneth D. Merry params->serial_num, 2333e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.serial_num), 2334130f4520SKenneth D. Merry sizeof(params->serial_num))); 2335130f4520SKenneth D. Merry } 2336130f4520SKenneth D. Merry if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { 2337130f4520SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns); 2338130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr, 2339e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.device_id), 2340130f4520SKenneth D. Merry sizeof(tmpstr))); 2341130f4520SKenneth D. Merry 2342130f4520SKenneth D. Merry /* Tell the user what we used for a device ID */ 2343130f4520SKenneth D. Merry strncpy((char *)params->device_id, tmpstr, 2344e7038eb7SAlexander Motin MIN(sizeof(params->device_id), sizeof(tmpstr))); 2345130f4520SKenneth D. Merry } else { 2346130f4520SKenneth D. Merry strncpy((char *)be_lun->ctl_be_lun.device_id, 2347130f4520SKenneth D. Merry params->device_id, 2348e7038eb7SAlexander Motin MIN(sizeof(be_lun->ctl_be_lun.device_id), 2349130f4520SKenneth D. Merry sizeof(params->device_id))); 2350130f4520SKenneth D. Merry } 2351130f4520SKenneth D. Merry 2352130f4520SKenneth D. Merry TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); 2353130f4520SKenneth D. Merry 2354130f4520SKenneth D. Merry be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK, 2355130f4520SKenneth D. Merry taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); 2356130f4520SKenneth D. Merry 2357130f4520SKenneth D. Merry if (be_lun->io_taskqueue == NULL) { 2358130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 235919720f41SAlexander Motin "unable to create taskqueue"); 2360130f4520SKenneth D. Merry goto bailout_error; 2361130f4520SKenneth D. Merry } 2362130f4520SKenneth D. Merry 2363130f4520SKenneth D. Merry /* 2364130f4520SKenneth D. Merry * Note that we start the same number of threads by default for 2365130f4520SKenneth D. Merry * both the file case and the block device case. For the file 2366130f4520SKenneth D. Merry * case, we need multiple threads to allow concurrency, because the 2367130f4520SKenneth D. Merry * vnode interface is designed to be a blocking interface. For the 2368130f4520SKenneth D. Merry * block device case, ZFS zvols at least will block the caller's 2369130f4520SKenneth D. Merry * context in many instances, and so we need multiple threads to 2370130f4520SKenneth D. Merry * overcome that problem. Other block devices don't need as many 2371130f4520SKenneth D. Merry * threads, but they shouldn't cause too many problems. 2372130f4520SKenneth D. Merry * 2373130f4520SKenneth D. Merry * If the user wants to just have a single thread for a block 2374130f4520SKenneth D. Merry * device, he can specify that when the LUN is created, or change 2375130f4520SKenneth D. Merry * the tunable/sysctl to alter the default number of threads. 2376130f4520SKenneth D. Merry */ 2377130f4520SKenneth D. Merry retval = taskqueue_start_threads(&be_lun->io_taskqueue, 2378130f4520SKenneth D. Merry /*num threads*/num_threads, 2379130f4520SKenneth D. Merry /*priority*/PWAIT, 2380130f4520SKenneth D. Merry /*thread name*/ 2381130f4520SKenneth D. Merry "%s taskq", be_lun->lunname); 2382130f4520SKenneth D. Merry 2383130f4520SKenneth D. Merry if (retval != 0) 2384130f4520SKenneth D. Merry goto bailout_error; 2385130f4520SKenneth D. Merry 2386130f4520SKenneth D. Merry be_lun->num_threads = num_threads; 2387130f4520SKenneth D. Merry 2388130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2389130f4520SKenneth D. Merry softc->num_luns++; 2390130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links); 2391130f4520SKenneth D. Merry 2392130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2393130f4520SKenneth D. Merry 2394130f4520SKenneth D. Merry retval = ctl_add_lun(&be_lun->ctl_be_lun); 2395130f4520SKenneth D. Merry if (retval != 0) { 2396130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2397130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2398130f4520SKenneth D. Merry links); 2399130f4520SKenneth D. Merry softc->num_luns--; 2400130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2401130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 240219720f41SAlexander Motin "ctl_add_lun() returned error %d, see dmesg for " 240319720f41SAlexander Motin "details", retval); 2404130f4520SKenneth D. Merry retval = 0; 2405130f4520SKenneth D. Merry goto bailout_error; 2406130f4520SKenneth D. Merry } 2407130f4520SKenneth D. Merry 2408130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2409130f4520SKenneth D. Merry 2410130f4520SKenneth D. Merry /* 2411130f4520SKenneth D. Merry * Tell the config_status routine that we're waiting so it won't 2412130f4520SKenneth D. Merry * clean up the LUN in the event of an error. 2413130f4520SKenneth D. Merry */ 2414130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2415130f4520SKenneth D. Merry 2416130f4520SKenneth D. Merry while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { 2417130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2418130f4520SKenneth D. Merry if (retval == EINTR) 2419130f4520SKenneth D. Merry break; 2420130f4520SKenneth D. Merry } 2421130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2422130f4520SKenneth D. Merry 2423130f4520SKenneth D. Merry if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) { 2424130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 242519720f41SAlexander Motin "LUN configuration error, see dmesg for details"); 2426130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2427130f4520SKenneth D. Merry links); 2428130f4520SKenneth D. Merry softc->num_luns--; 2429130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2430130f4520SKenneth D. Merry goto bailout_error; 2431130f4520SKenneth D. Merry } else { 2432130f4520SKenneth D. Merry params->req_lun_id = be_lun->ctl_be_lun.lun_id; 2433130f4520SKenneth D. Merry } 2434130f4520SKenneth D. Merry 2435130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2436130f4520SKenneth D. Merry 2437130f4520SKenneth D. Merry be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id, 2438130f4520SKenneth D. Merry be_lun->blocksize, 2439130f4520SKenneth D. Merry DEVSTAT_ALL_SUPPORTED, 2440130f4520SKenneth D. Merry be_lun->ctl_be_lun.lun_type 2441130f4520SKenneth D. Merry | DEVSTAT_TYPE_IF_OTHER, 2442130f4520SKenneth D. Merry DEVSTAT_PRIORITY_OTHER); 2443130f4520SKenneth D. Merry 2444130f4520SKenneth D. Merry return (retval); 2445130f4520SKenneth D. Merry 2446130f4520SKenneth D. Merry bailout_error: 2447130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2448130f4520SKenneth D. Merry 24499e005bbcSAlexander Motin if (be_lun->io_taskqueue != NULL) 24509e005bbcSAlexander Motin taskqueue_free(be_lun->io_taskqueue); 2451130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 24529e005bbcSAlexander Motin if (be_lun->dev_path != NULL) 2453130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 24549e005bbcSAlexander Motin if (be_lun->lun_zone != NULL) 24559e005bbcSAlexander Motin uma_zdestroy(be_lun->lun_zone); 245643fb3a65SAlexander Motin ctl_free_opts(&be_lun->ctl_be_lun.options); 245775c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 245875c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2459130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2460130f4520SKenneth D. Merry 2461130f4520SKenneth D. Merry return (retval); 2462130f4520SKenneth D. Merry } 2463130f4520SKenneth D. Merry 2464130f4520SKenneth D. Merry static int 2465130f4520SKenneth D. Merry ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2466130f4520SKenneth D. Merry { 2467130f4520SKenneth D. Merry struct ctl_lun_rm_params *params; 2468130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2469130f4520SKenneth D. Merry int retval; 2470130f4520SKenneth D. Merry 2471130f4520SKenneth D. Merry params = &req->reqdata.rm; 2472130f4520SKenneth D. Merry 2473130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2474130f4520SKenneth D. Merry 2475130f4520SKenneth D. Merry be_lun = NULL; 2476130f4520SKenneth D. Merry 2477130f4520SKenneth D. Merry STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 2478130f4520SKenneth D. Merry if (be_lun->ctl_be_lun.lun_id == params->lun_id) 2479130f4520SKenneth D. Merry break; 2480130f4520SKenneth D. Merry } 2481130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2482130f4520SKenneth D. Merry 2483130f4520SKenneth D. Merry if (be_lun == NULL) { 2484130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 248519720f41SAlexander Motin "LUN %u is not managed by the block backend", 248619720f41SAlexander Motin params->lun_id); 2487130f4520SKenneth D. Merry goto bailout_error; 2488130f4520SKenneth D. Merry } 2489130f4520SKenneth D. Merry 2490130f4520SKenneth D. Merry retval = ctl_disable_lun(&be_lun->ctl_be_lun); 2491130f4520SKenneth D. Merry 2492130f4520SKenneth D. Merry if (retval != 0) { 2493130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 249419720f41SAlexander Motin "error %d returned from ctl_disable_lun() for " 249519720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2496130f4520SKenneth D. Merry goto bailout_error; 2497130f4520SKenneth D. Merry 2498130f4520SKenneth D. Merry } 2499130f4520SKenneth D. Merry 2500130f4520SKenneth D. Merry retval = ctl_invalidate_lun(&be_lun->ctl_be_lun); 2501130f4520SKenneth D. Merry if (retval != 0) { 2502130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 250319720f41SAlexander Motin "error %d returned from ctl_invalidate_lun() for " 250419720f41SAlexander Motin "LUN %d", retval, params->lun_id); 2505130f4520SKenneth D. Merry goto bailout_error; 2506130f4520SKenneth D. Merry } 2507130f4520SKenneth D. Merry 2508130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2509130f4520SKenneth D. Merry 2510130f4520SKenneth D. Merry be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2511130f4520SKenneth D. Merry 2512130f4520SKenneth D. Merry while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2513130f4520SKenneth D. Merry retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2514130f4520SKenneth D. Merry if (retval == EINTR) 2515130f4520SKenneth D. Merry break; 2516130f4520SKenneth D. Merry } 2517130f4520SKenneth D. Merry 2518130f4520SKenneth D. Merry be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2519130f4520SKenneth D. Merry 2520130f4520SKenneth D. Merry if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2521130f4520SKenneth D. Merry snprintf(req->error_str, sizeof(req->error_str), 252219720f41SAlexander Motin "interrupted waiting for LUN to be freed"); 2523130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2524130f4520SKenneth D. Merry goto bailout_error; 2525130f4520SKenneth D. Merry } 2526130f4520SKenneth D. Merry 2527130f4520SKenneth D. Merry STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links); 2528130f4520SKenneth D. Merry 2529130f4520SKenneth D. Merry softc->num_luns--; 2530130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2531130f4520SKenneth D. Merry 2532130f4520SKenneth D. Merry taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task); 2533130f4520SKenneth D. Merry 2534130f4520SKenneth D. Merry taskqueue_free(be_lun->io_taskqueue); 2535130f4520SKenneth D. Merry 2536130f4520SKenneth D. Merry ctl_be_block_close(be_lun); 2537130f4520SKenneth D. Merry 2538130f4520SKenneth D. Merry if (be_lun->disk_stats != NULL) 2539130f4520SKenneth D. Merry devstat_remove_entry(be_lun->disk_stats); 2540130f4520SKenneth D. Merry 2541130f4520SKenneth D. Merry uma_zdestroy(be_lun->lun_zone); 2542130f4520SKenneth D. Merry 254343fb3a65SAlexander Motin ctl_free_opts(&be_lun->ctl_be_lun.options); 2544130f4520SKenneth D. Merry free(be_lun->dev_path, M_CTLBLK); 254575c7a1d3SAlexander Motin mtx_destroy(&be_lun->queue_lock); 254675c7a1d3SAlexander Motin mtx_destroy(&be_lun->io_lock); 2547130f4520SKenneth D. Merry free(be_lun, M_CTLBLK); 2548130f4520SKenneth D. Merry 2549130f4520SKenneth D. Merry req->status = CTL_LUN_OK; 2550130f4520SKenneth D. Merry 2551130f4520SKenneth D. Merry return (0); 2552130f4520SKenneth D. Merry 2553130f4520SKenneth D. Merry bailout_error: 2554130f4520SKenneth D. Merry 2555130f4520SKenneth D. Merry req->status = CTL_LUN_ERROR; 2556130f4520SKenneth D. Merry 2557130f4520SKenneth D. Merry return (0); 2558130f4520SKenneth D. Merry } 2559130f4520SKenneth D. Merry 256081177295SEdward Tomasz Napierala static int 256181177295SEdward Tomasz Napierala ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 256281177295SEdward Tomasz Napierala struct ctl_lun_req *req) 256381177295SEdward Tomasz Napierala { 256481177295SEdward Tomasz Napierala struct vattr vattr; 256581177295SEdward Tomasz Napierala int error; 256619720f41SAlexander Motin struct ctl_lun_create_params *params = &be_lun->params; 256781177295SEdward Tomasz Napierala 256881177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 256981177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 257081177295SEdward Tomasz Napierala } else { 257171d8e97eSAlexander Motin vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 257281177295SEdward Tomasz Napierala error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 257371d8e97eSAlexander Motin VOP_UNLOCK(be_lun->vn, 0); 257481177295SEdward Tomasz Napierala if (error != 0) { 257581177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 257681177295SEdward Tomasz Napierala "error calling VOP_GETATTR() for file %s", 257781177295SEdward Tomasz Napierala be_lun->dev_path); 257881177295SEdward Tomasz Napierala return (error); 257981177295SEdward Tomasz Napierala } 258081177295SEdward Tomasz Napierala 258181177295SEdward Tomasz Napierala be_lun->size_bytes = vattr.va_size; 258281177295SEdward Tomasz Napierala } 258381177295SEdward Tomasz Napierala 258481177295SEdward Tomasz Napierala return (0); 258581177295SEdward Tomasz Napierala } 258681177295SEdward Tomasz Napierala 258781177295SEdward Tomasz Napierala static int 258881177295SEdward Tomasz Napierala ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 258981177295SEdward Tomasz Napierala struct ctl_lun_req *req) 259081177295SEdward Tomasz Napierala { 259171d8e97eSAlexander Motin struct ctl_be_block_devdata *dev_data; 259281177295SEdward Tomasz Napierala int error; 259319720f41SAlexander Motin struct ctl_lun_create_params *params = &be_lun->params; 259481177295SEdward Tomasz Napierala uint64_t size_bytes; 259581177295SEdward Tomasz Napierala 259671d8e97eSAlexander Motin dev_data = &be_lun->backend.dev; 259771d8e97eSAlexander Motin if (!dev_data->csw->d_ioctl) { 259881177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 259919720f41SAlexander Motin "no d_ioctl for device %s!", be_lun->dev_path); 260081177295SEdward Tomasz Napierala return (ENODEV); 260181177295SEdward Tomasz Napierala } 260281177295SEdward Tomasz Napierala 260371d8e97eSAlexander Motin error = dev_data->csw->d_ioctl(dev_data->cdev, DIOCGMEDIASIZE, 260481177295SEdward Tomasz Napierala (caddr_t)&size_bytes, FREAD, 260581177295SEdward Tomasz Napierala curthread); 260681177295SEdward Tomasz Napierala if (error) { 260781177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 260819720f41SAlexander Motin "error %d returned for DIOCGMEDIASIZE ioctl " 260919720f41SAlexander Motin "on %s!", error, be_lun->dev_path); 261081177295SEdward Tomasz Napierala return (error); 261181177295SEdward Tomasz Napierala } 261281177295SEdward Tomasz Napierala 261381177295SEdward Tomasz Napierala if (params->lun_size_bytes != 0) { 261481177295SEdward Tomasz Napierala if (params->lun_size_bytes > size_bytes) { 261581177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 261619720f41SAlexander Motin "requested LUN size %ju > backing device " 261719720f41SAlexander Motin "size %ju", 261881177295SEdward Tomasz Napierala (uintmax_t)params->lun_size_bytes, 261981177295SEdward Tomasz Napierala (uintmax_t)size_bytes); 262081177295SEdward Tomasz Napierala return (EINVAL); 262181177295SEdward Tomasz Napierala } 262281177295SEdward Tomasz Napierala 262381177295SEdward Tomasz Napierala be_lun->size_bytes = params->lun_size_bytes; 262481177295SEdward Tomasz Napierala } else { 262581177295SEdward Tomasz Napierala be_lun->size_bytes = size_bytes; 262681177295SEdward Tomasz Napierala } 262781177295SEdward Tomasz Napierala 262881177295SEdward Tomasz Napierala return (0); 262981177295SEdward Tomasz Napierala } 263081177295SEdward Tomasz Napierala 263181177295SEdward Tomasz Napierala static int 263281177295SEdward Tomasz Napierala ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 263381177295SEdward Tomasz Napierala { 263481177295SEdward Tomasz Napierala struct ctl_lun_modify_params *params; 263581177295SEdward Tomasz Napierala struct ctl_be_block_lun *be_lun; 263671d8e97eSAlexander Motin uint64_t oldsize; 26375050aa86SKonstantin Belousov int error; 263881177295SEdward Tomasz Napierala 263981177295SEdward Tomasz Napierala params = &req->reqdata.modify; 264081177295SEdward Tomasz Napierala 264181177295SEdward Tomasz Napierala mtx_lock(&softc->lock); 264281177295SEdward Tomasz Napierala be_lun = NULL; 264381177295SEdward Tomasz Napierala STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 264481177295SEdward Tomasz Napierala if (be_lun->ctl_be_lun.lun_id == params->lun_id) 264581177295SEdward Tomasz Napierala break; 264681177295SEdward Tomasz Napierala } 264781177295SEdward Tomasz Napierala mtx_unlock(&softc->lock); 264881177295SEdward Tomasz Napierala 264981177295SEdward Tomasz Napierala if (be_lun == NULL) { 265081177295SEdward Tomasz Napierala snprintf(req->error_str, sizeof(req->error_str), 265119720f41SAlexander Motin "LUN %u is not managed by the block backend", 265219720f41SAlexander Motin params->lun_id); 265381177295SEdward Tomasz Napierala goto bailout_error; 265481177295SEdward Tomasz Napierala } 265581177295SEdward Tomasz Napierala 265619720f41SAlexander Motin be_lun->params.lun_size_bytes = params->lun_size_bytes; 265781177295SEdward Tomasz Napierala 26583f829b0cSAlexander Motin oldsize = be_lun->size_bytes; 265919720f41SAlexander Motin if (be_lun->vn == NULL) 266019720f41SAlexander Motin error = ctl_be_block_open(softc, be_lun, req); 2661b9b4269cSAlexander Motin else if (vn_isdisk(be_lun->vn, &error)) 2662b9b4269cSAlexander Motin error = ctl_be_block_modify_dev(be_lun, req); 266319720f41SAlexander Motin else if (be_lun->vn->v_type == VREG) 266481177295SEdward Tomasz Napierala error = ctl_be_block_modify_file(be_lun, req); 266581177295SEdward Tomasz Napierala else 2666b9b4269cSAlexander Motin error = EINVAL; 266781177295SEdward Tomasz Napierala 26683f829b0cSAlexander Motin if (error == 0 && be_lun->size_bytes != oldsize) { 266971d8e97eSAlexander Motin be_lun->size_blocks = be_lun->size_bytes >> 267071d8e97eSAlexander Motin be_lun->blocksize_shift; 267181177295SEdward Tomasz Napierala 267281177295SEdward Tomasz Napierala /* 267381177295SEdward Tomasz Napierala * The maximum LBA is the size - 1. 267481177295SEdward Tomasz Napierala * 267581177295SEdward Tomasz Napierala * XXX: Note that this field is being updated without locking, 267681177295SEdward Tomasz Napierala * which might cause problems on 32-bit architectures. 267781177295SEdward Tomasz Napierala */ 2678fbc8d4ffSAlexander Motin if (be_lun->unmap != NULL) 2679fbc8d4ffSAlexander Motin be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP; 268019720f41SAlexander Motin be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 268119720f41SAlexander Motin 0 : (be_lun->size_blocks - 1); 268219720f41SAlexander Motin be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 268319720f41SAlexander Motin be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 268419720f41SAlexander Motin be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 268534961f40SAlexander Motin be_lun->ctl_be_lun.ublockexp = be_lun->ublockexp; 268634961f40SAlexander Motin be_lun->ctl_be_lun.ublockoff = be_lun->ublockoff; 2687cb8727e2SAlexander Motin be_lun->ctl_be_lun.atomicblock = be_lun->atomicblock; 2688cb8727e2SAlexander Motin be_lun->ctl_be_lun.opttxferlen = be_lun->opttxferlen; 268981177295SEdward Tomasz Napierala ctl_lun_capacity_changed(&be_lun->ctl_be_lun); 269019720f41SAlexander Motin if (oldsize == 0 && be_lun->size_blocks != 0) 269119720f41SAlexander Motin ctl_lun_online(&be_lun->ctl_be_lun); 269271d8e97eSAlexander Motin } 269381177295SEdward Tomasz Napierala 269481177295SEdward Tomasz Napierala /* Tell the user the exact size we ended up using */ 269581177295SEdward Tomasz Napierala params->lun_size_bytes = be_lun->size_bytes; 269681177295SEdward Tomasz Napierala 269719720f41SAlexander Motin req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; 269881177295SEdward Tomasz Napierala 269981177295SEdward Tomasz Napierala return (0); 270081177295SEdward Tomasz Napierala 270181177295SEdward Tomasz Napierala bailout_error: 270281177295SEdward Tomasz Napierala req->status = CTL_LUN_ERROR; 270381177295SEdward Tomasz Napierala 270481177295SEdward Tomasz Napierala return (0); 270581177295SEdward Tomasz Napierala } 270681177295SEdward Tomasz Napierala 2707130f4520SKenneth D. Merry static void 2708130f4520SKenneth D. Merry ctl_be_block_lun_shutdown(void *be_lun) 2709130f4520SKenneth D. Merry { 2710130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2711130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2712130f4520SKenneth D. Merry 2713130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2714130f4520SKenneth D. Merry 2715130f4520SKenneth D. Merry softc = lun->softc; 2716130f4520SKenneth D. Merry 2717130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2718130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; 2719130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2720130f4520SKenneth D. Merry wakeup(lun); 2721130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2722130f4520SKenneth D. Merry 2723130f4520SKenneth D. Merry } 2724130f4520SKenneth D. Merry 2725130f4520SKenneth D. Merry static void 2726130f4520SKenneth D. Merry ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status) 2727130f4520SKenneth D. Merry { 2728130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2729130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2730130f4520SKenneth D. Merry 2731130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2732130f4520SKenneth D. Merry softc = lun->softc; 2733130f4520SKenneth D. Merry 2734130f4520SKenneth D. Merry if (status == CTL_LUN_CONFIG_OK) { 2735130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2736130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2737130f4520SKenneth D. Merry if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2738130f4520SKenneth D. Merry wakeup(lun); 2739130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2740130f4520SKenneth D. Merry 2741130f4520SKenneth D. Merry /* 2742130f4520SKenneth D. Merry * We successfully added the LUN, attempt to enable it. 2743130f4520SKenneth D. Merry */ 2744130f4520SKenneth D. Merry if (ctl_enable_lun(&lun->ctl_be_lun) != 0) { 2745130f4520SKenneth D. Merry printf("%s: ctl_enable_lun() failed!\n", __func__); 2746130f4520SKenneth D. Merry if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) { 2747130f4520SKenneth D. Merry printf("%s: ctl_invalidate_lun() failed!\n", 2748130f4520SKenneth D. Merry __func__); 2749130f4520SKenneth D. Merry } 2750130f4520SKenneth D. Merry } 2751130f4520SKenneth D. Merry 2752130f4520SKenneth D. Merry return; 2753130f4520SKenneth D. Merry } 2754130f4520SKenneth D. Merry 2755130f4520SKenneth D. Merry 2756130f4520SKenneth D. Merry mtx_lock(&softc->lock); 2757130f4520SKenneth D. Merry lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2758130f4520SKenneth D. Merry lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR; 2759130f4520SKenneth D. Merry wakeup(lun); 2760130f4520SKenneth D. Merry mtx_unlock(&softc->lock); 2761130f4520SKenneth D. Merry } 2762130f4520SKenneth D. Merry 2763130f4520SKenneth D. Merry 2764130f4520SKenneth D. Merry static int 2765130f4520SKenneth D. Merry ctl_be_block_config_write(union ctl_io *io) 2766130f4520SKenneth D. Merry { 2767130f4520SKenneth D. Merry struct ctl_be_block_lun *be_lun; 2768130f4520SKenneth D. Merry struct ctl_be_lun *ctl_be_lun; 2769130f4520SKenneth D. Merry int retval; 2770130f4520SKenneth D. Merry 2771130f4520SKenneth D. Merry retval = 0; 2772130f4520SKenneth D. Merry 2773130f4520SKenneth D. Merry DPRINTF("entered\n"); 2774130f4520SKenneth D. Merry 2775130f4520SKenneth D. Merry ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2776130f4520SKenneth D. Merry CTL_PRIV_BACKEND_LUN].ptr; 2777130f4520SKenneth D. Merry be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2778130f4520SKenneth D. Merry 2779130f4520SKenneth D. Merry switch (io->scsiio.cdb[0]) { 2780130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE: 2781130f4520SKenneth D. Merry case SYNCHRONIZE_CACHE_16: 2782ee7f31c0SAlexander Motin case WRITE_SAME_10: 2783ee7f31c0SAlexander Motin case WRITE_SAME_16: 2784ee7f31c0SAlexander Motin case UNMAP: 2785130f4520SKenneth D. Merry /* 2786130f4520SKenneth D. Merry * The upper level CTL code will filter out any CDBs with 2787130f4520SKenneth D. Merry * the immediate bit set and return the proper error. 2788130f4520SKenneth D. Merry * 2789130f4520SKenneth D. Merry * We don't really need to worry about what LBA range the 2790130f4520SKenneth D. Merry * user asked to be synced out. When they issue a sync 2791130f4520SKenneth D. Merry * cache command, we'll sync out the whole thing. 2792130f4520SKenneth D. Merry */ 279375c7a1d3SAlexander Motin mtx_lock(&be_lun->queue_lock); 2794130f4520SKenneth D. Merry STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, 2795130f4520SKenneth D. Merry links); 279675c7a1d3SAlexander Motin mtx_unlock(&be_lun->queue_lock); 2797130f4520SKenneth D. Merry taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 2798130f4520SKenneth D. Merry break; 2799130f4520SKenneth D. Merry case START_STOP_UNIT: { 2800130f4520SKenneth D. Merry struct scsi_start_stop_unit *cdb; 2801130f4520SKenneth D. Merry 2802130f4520SKenneth D. Merry cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; 2803130f4520SKenneth D. Merry 2804130f4520SKenneth D. Merry if (cdb->how & SSS_START) 2805130f4520SKenneth D. Merry retval = ctl_start_lun(ctl_be_lun); 2806130f4520SKenneth D. Merry else { 2807130f4520SKenneth D. Merry retval = ctl_stop_lun(ctl_be_lun); 2808130f4520SKenneth D. Merry /* 2809130f4520SKenneth D. Merry * XXX KDM Copan-specific offline behavior. 2810130f4520SKenneth D. Merry * Figure out a reasonable way to port this? 2811130f4520SKenneth D. Merry */ 2812130f4520SKenneth D. Merry #ifdef NEEDTOPORT 2813130f4520SKenneth D. Merry if ((retval == 0) 2814130f4520SKenneth D. Merry && (cdb->byte2 & SSS_ONOFFLINE)) 2815130f4520SKenneth D. Merry retval = ctl_lun_offline(ctl_be_lun); 2816130f4520SKenneth D. Merry #endif 2817130f4520SKenneth D. Merry } 2818130f4520SKenneth D. Merry 2819130f4520SKenneth D. Merry /* 2820130f4520SKenneth D. Merry * In general, the above routines should not fail. They 2821130f4520SKenneth D. Merry * just set state for the LUN. So we've got something 2822130f4520SKenneth D. Merry * pretty wrong here if we can't start or stop the LUN. 2823130f4520SKenneth D. Merry */ 2824130f4520SKenneth D. Merry if (retval != 0) { 2825130f4520SKenneth D. Merry ctl_set_internal_failure(&io->scsiio, 2826130f4520SKenneth D. Merry /*sks_valid*/ 1, 2827130f4520SKenneth D. Merry /*retry_count*/ 0xf051); 2828130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2829130f4520SKenneth D. Merry } else { 2830130f4520SKenneth D. Merry ctl_set_success(&io->scsiio); 2831130f4520SKenneth D. Merry } 2832130f4520SKenneth D. Merry ctl_config_write_done(io); 2833130f4520SKenneth D. Merry break; 2834130f4520SKenneth D. Merry } 2835130f4520SKenneth D. Merry default: 2836130f4520SKenneth D. Merry ctl_set_invalid_opcode(&io->scsiio); 2837130f4520SKenneth D. Merry ctl_config_write_done(io); 2838130f4520SKenneth D. Merry retval = CTL_RETVAL_COMPLETE; 2839130f4520SKenneth D. Merry break; 2840130f4520SKenneth D. Merry } 2841130f4520SKenneth D. Merry 2842130f4520SKenneth D. Merry return (retval); 2843130f4520SKenneth D. Merry } 2844130f4520SKenneth D. Merry 2845130f4520SKenneth D. Merry static int 2846130f4520SKenneth D. Merry ctl_be_block_config_read(union ctl_io *io) 2847130f4520SKenneth D. Merry { 2848ef8daf3fSAlexander Motin struct ctl_be_block_lun *be_lun; 2849ef8daf3fSAlexander Motin struct ctl_be_lun *ctl_be_lun; 2850ef8daf3fSAlexander Motin int retval = 0; 2851ef8daf3fSAlexander Motin 2852ef8daf3fSAlexander Motin DPRINTF("entered\n"); 2853ef8daf3fSAlexander Motin 2854ef8daf3fSAlexander Motin ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2855ef8daf3fSAlexander Motin CTL_PRIV_BACKEND_LUN].ptr; 2856ef8daf3fSAlexander Motin be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2857ef8daf3fSAlexander Motin 2858ef8daf3fSAlexander Motin switch (io->scsiio.cdb[0]) { 2859ef8daf3fSAlexander Motin case SERVICE_ACTION_IN: 2860ef8daf3fSAlexander Motin if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { 2861ef8daf3fSAlexander Motin mtx_lock(&be_lun->queue_lock); 2862ef8daf3fSAlexander Motin STAILQ_INSERT_TAIL(&be_lun->config_read_queue, 2863ef8daf3fSAlexander Motin &io->io_hdr, links); 2864ef8daf3fSAlexander Motin mtx_unlock(&be_lun->queue_lock); 2865ef8daf3fSAlexander Motin taskqueue_enqueue(be_lun->io_taskqueue, 2866ef8daf3fSAlexander Motin &be_lun->io_task); 2867ef8daf3fSAlexander Motin retval = CTL_RETVAL_QUEUED; 2868ef8daf3fSAlexander Motin break; 2869ef8daf3fSAlexander Motin } 2870ef8daf3fSAlexander Motin ctl_set_invalid_field(&io->scsiio, 2871ef8daf3fSAlexander Motin /*sks_valid*/ 1, 2872ef8daf3fSAlexander Motin /*command*/ 1, 2873ef8daf3fSAlexander Motin /*field*/ 1, 2874ef8daf3fSAlexander Motin /*bit_valid*/ 1, 2875ef8daf3fSAlexander Motin /*bit*/ 4); 2876ef8daf3fSAlexander Motin ctl_config_read_done(io); 2877ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2878ef8daf3fSAlexander Motin break; 2879ef8daf3fSAlexander Motin default: 2880ef8daf3fSAlexander Motin ctl_set_invalid_opcode(&io->scsiio); 2881ef8daf3fSAlexander Motin ctl_config_read_done(io); 2882ef8daf3fSAlexander Motin retval = CTL_RETVAL_COMPLETE; 2883ef8daf3fSAlexander Motin break; 2884ef8daf3fSAlexander Motin } 2885ef8daf3fSAlexander Motin 2886ef8daf3fSAlexander Motin return (retval); 2887130f4520SKenneth D. Merry } 2888130f4520SKenneth D. Merry 2889130f4520SKenneth D. Merry static int 2890130f4520SKenneth D. Merry ctl_be_block_lun_info(void *be_lun, struct sbuf *sb) 2891130f4520SKenneth D. Merry { 2892130f4520SKenneth D. Merry struct ctl_be_block_lun *lun; 2893130f4520SKenneth D. Merry int retval; 2894130f4520SKenneth D. Merry 2895130f4520SKenneth D. Merry lun = (struct ctl_be_block_lun *)be_lun; 2896130f4520SKenneth D. Merry retval = 0; 2897130f4520SKenneth D. Merry 28982cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "\t<num_threads>"); 2899130f4520SKenneth D. Merry 2900130f4520SKenneth D. Merry if (retval != 0) 2901130f4520SKenneth D. Merry goto bailout; 2902130f4520SKenneth D. Merry 2903130f4520SKenneth D. Merry retval = sbuf_printf(sb, "%d", lun->num_threads); 2904130f4520SKenneth D. Merry 2905130f4520SKenneth D. Merry if (retval != 0) 2906130f4520SKenneth D. Merry goto bailout; 2907130f4520SKenneth D. Merry 29082cfbcb9bSAlexander Motin retval = sbuf_printf(sb, "</num_threads>\n"); 2909130f4520SKenneth D. Merry 2910130f4520SKenneth D. Merry bailout: 2911130f4520SKenneth D. Merry 2912130f4520SKenneth D. Merry return (retval); 2913130f4520SKenneth D. Merry } 2914130f4520SKenneth D. Merry 2915c3e7ba3eSAlexander Motin static uint64_t 2916c3e7ba3eSAlexander Motin ctl_be_block_lun_attr(void *be_lun, const char *attrname) 2917c3e7ba3eSAlexander Motin { 2918c3e7ba3eSAlexander Motin struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun; 2919c3e7ba3eSAlexander Motin 2920c3e7ba3eSAlexander Motin if (lun->getattr == NULL) 2921c3e7ba3eSAlexander Motin return (UINT64_MAX); 2922c3e7ba3eSAlexander Motin return (lun->getattr(lun, attrname)); 2923c3e7ba3eSAlexander Motin } 2924c3e7ba3eSAlexander Motin 2925130f4520SKenneth D. Merry int 2926130f4520SKenneth D. Merry ctl_be_block_init(void) 2927130f4520SKenneth D. Merry { 2928130f4520SKenneth D. Merry struct ctl_be_block_softc *softc; 2929130f4520SKenneth D. Merry int retval; 2930130f4520SKenneth D. Merry 2931130f4520SKenneth D. Merry softc = &backend_block_softc; 2932130f4520SKenneth D. Merry retval = 0; 2933130f4520SKenneth D. Merry 293475c7a1d3SAlexander Motin mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); 2935a0e36aeeSEdward Tomasz Napierala beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), 2936a0e36aeeSEdward Tomasz Napierala NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 2937130f4520SKenneth D. Merry STAILQ_INIT(&softc->disk_list); 2938130f4520SKenneth D. Merry STAILQ_INIT(&softc->lun_list); 2939130f4520SKenneth D. Merry 2940130f4520SKenneth D. Merry return (retval); 2941130f4520SKenneth D. Merry } 2942