/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO SCSI devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" static int vtscsi_modevent(module_t, int, void *); static int vtscsi_probe(device_t); static int vtscsi_attach(device_t); static int vtscsi_detach(device_t); static int vtscsi_suspend(device_t); static int vtscsi_resume(device_t); static int vtscsi_negotiate_features(struct vtscsi_softc *); static int vtscsi_setup_features(struct vtscsi_softc *); static void vtscsi_read_config(struct vtscsi_softc *, struct virtio_scsi_config *); static int vtscsi_maximum_segments(struct vtscsi_softc *, int); static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); static void vtscsi_check_sizes(struct vtscsi_softc *); static void vtscsi_write_device_config(struct vtscsi_softc *); static int vtscsi_reinit(struct vtscsi_softc *); static int vtscsi_alloc_cam(struct vtscsi_softc *); static int vtscsi_register_cam(struct vtscsi_softc *); static void vtscsi_free_cam(struct vtscsi_softc *); static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); static int vtscsi_register_async(struct vtscsi_softc *); static void vtscsi_deregister_async(struct vtscsi_softc *); static void vtscsi_cam_action(struct cam_sim *, union ccb *); static void vtscsi_cam_poll(struct cam_sim *); static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, union ccb *); static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, struct cam_sim *, union ccb *); static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, struct sglist *, struct ccb_scsiio *); static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, struct vtscsi_request *, int *, int *); static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_timedout_scsi_cmd(void *); static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, struct vtscsi_request *, struct sglist *, int, int, int); static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, struct vtscsi_request *); static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *, struct ccb_scsiio *, struct virtio_scsi_cmd_req *); static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *, struct ccb_hdr *, uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *); static void vtscsi_freeze_simq(struct vtscsi_softc *, int); static int vtscsi_thaw_simq(struct vtscsi_softc *, int); static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, lun_id_t); static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, lun_id_t); static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); static void vtscsi_handle_event(struct vtscsi_softc *, struct virtio_scsi_event *); static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, struct virtio_scsi_event *); static int vtscsi_init_event_vq(struct vtscsi_softc *); static void vtscsi_reinit_event_vq(struct vtscsi_softc *); static void vtscsi_drain_event_vq(struct vtscsi_softc *); static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); static void vtscsi_complete_vqs(struct vtscsi_softc *); static void vtscsi_drain_vqs(struct vtscsi_softc *); static void vtscsi_cancel_request(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); static void vtscsi_stop(struct vtscsi_softc *); static int vtscsi_reset_bus(struct vtscsi_softc *); static void vtscsi_init_request(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_alloc_requests(struct vtscsi_softc *); static void vtscsi_free_requests(struct vtscsi_softc *); static void vtscsi_enqueue_request(struct vtscsi_softc *, struct vtscsi_request *); static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); static void vtscsi_complete_request(struct vtscsi_request *); static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); static void vtscsi_control_vq_intr(void *); static void vtscsi_event_vq_intr(void *); static void vtscsi_request_vq_intr(void *); static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); static void vtscsi_get_tunables(struct vtscsi_softc *); static void vtscsi_setup_sysctl(struct vtscsi_softc *); static void vtscsi_printf_req(struct vtscsi_request *, const char *, const char *, ...); #define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0) #define vtscsi_htog16(_sc, _val) virtio_htog16(vtscsi_modern(_sc), _val) #define vtscsi_htog32(_sc, _val) virtio_htog32(vtscsi_modern(_sc), _val) #define vtscsi_htog64(_sc, _val) virtio_htog64(vtscsi_modern(_sc), _val) #define vtscsi_gtoh16(_sc, _val) virtio_gtoh16(vtscsi_modern(_sc), _val) #define vtscsi_gtoh32(_sc, _val) virtio_gtoh32(vtscsi_modern(_sc), _val) #define vtscsi_gtoh64(_sc, _val) virtio_gtoh64(vtscsi_modern(_sc), _val) /* Global tunables. */ /* * The current QEMU VirtIO SCSI implementation does not cancel in-flight * IO during virtio_stop(). So in-flight requests still complete after the * device reset. We would have to wait for all the in-flight IO to complete, * which defeats the typical purpose of a bus reset. We could simulate the * bus reset with either I_T_NEXUS_RESET of all the targets, or with * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the * control virtqueue). But this isn't very useful if things really go off * the rails, so default to disabled for now. */ static int vtscsi_bus_reset_disable = 1; TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); static struct virtio_feature_desc vtscsi_feature_desc[] = { { VIRTIO_SCSI_F_INOUT, "InOut" }, { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, { VIRTIO_SCSI_F_CHANGE, "ChangeEvent" }, { VIRTIO_SCSI_F_T10_PI, "T10PI" }, { 0, NULL } }; static device_method_t vtscsi_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtscsi_probe), DEVMETHOD(device_attach, vtscsi_attach), DEVMETHOD(device_detach, vtscsi_detach), DEVMETHOD(device_suspend, vtscsi_suspend), DEVMETHOD(device_resume, vtscsi_resume), DEVMETHOD_END }; static driver_t vtscsi_driver = { "vtscsi", vtscsi_methods, sizeof(struct vtscsi_softc) }; VIRTIO_DRIVER_MODULE(virtio_scsi, vtscsi_driver, vtscsi_modevent, NULL); MODULE_VERSION(virtio_scsi, 1); MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_scsi, VIRTIO_ID_SCSI, "VirtIO SCSI Adapter"); static int vtscsi_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static int vtscsi_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_scsi)); } static int vtscsi_attach(device_t dev) { struct vtscsi_softc *sc; struct virtio_scsi_config scsicfg; int error; sc = device_get_softc(dev); sc->vtscsi_dev = dev; virtio_set_feature_desc(dev, vtscsi_feature_desc); VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); TAILQ_INIT(&sc->vtscsi_req_free); vtscsi_get_tunables(sc); vtscsi_setup_sysctl(sc); error = vtscsi_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } vtscsi_read_config(sc, &scsicfg); sc->vtscsi_max_channel = scsicfg.max_channel; sc->vtscsi_max_target = scsicfg.max_target; sc->vtscsi_max_lun = scsicfg.max_lun; sc->vtscsi_event_buf_size = scsicfg.event_info_size; vtscsi_write_device_config(sc); sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); if (sc->vtscsi_sglist == NULL) { error = ENOMEM; device_printf(dev, "cannot allocate sglist\n"); goto fail; } error = vtscsi_alloc_virtqueues(sc); if (error) { device_printf(dev, "cannot allocate virtqueues\n"); goto fail; } vtscsi_check_sizes(sc); error = vtscsi_init_event_vq(sc); if (error) { device_printf(dev, "cannot populate the eventvq\n"); goto fail; } error = vtscsi_alloc_requests(sc); if (error) { device_printf(dev, "cannot allocate requests\n"); goto fail; } error = vtscsi_alloc_cam(sc); if (error) { device_printf(dev, "cannot allocate CAM structures\n"); goto fail; } error = virtio_setup_intr(dev, INTR_TYPE_CAM); if (error) { device_printf(dev, "cannot setup virtqueue interrupts\n"); goto fail; } vtscsi_enable_vqs_intr(sc); /* * Register with CAM after interrupts are enabled so we will get * notified of the probe responses. */ error = vtscsi_register_cam(sc); if (error) { device_printf(dev, "cannot register with CAM\n"); goto fail; } fail: if (error) vtscsi_detach(dev); return (error); } static int vtscsi_detach(device_t dev) { struct vtscsi_softc *sc; sc = device_get_softc(dev); VTSCSI_LOCK(sc); sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; if (device_is_attached(dev)) vtscsi_stop(sc); VTSCSI_UNLOCK(sc); vtscsi_complete_vqs(sc); vtscsi_drain_vqs(sc); vtscsi_free_cam(sc); vtscsi_free_requests(sc); if (sc->vtscsi_sglist != NULL) { sglist_free(sc->vtscsi_sglist); sc->vtscsi_sglist = NULL; } VTSCSI_LOCK_DESTROY(sc); return (0); } static int vtscsi_suspend(device_t dev) { return (0); } static int vtscsi_resume(device_t dev) { return (0); } static int vtscsi_negotiate_features(struct vtscsi_softc *sc) { device_t dev; uint64_t features; dev = sc->vtscsi_dev; features = VTSCSI_FEATURES; sc->vtscsi_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtscsi_setup_features(struct vtscsi_softc *sc) { device_t dev; int error; dev = sc->vtscsi_dev; error = vtscsi_negotiate_features(sc); if (error) return (error); if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; return (0); } #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ virtio_read_device_config(_dev, \ offsetof(struct virtio_scsi_config, _field), \ &(_cfg)->_field, sizeof((_cfg)->_field)) \ static void vtscsi_read_config(struct vtscsi_softc *sc, struct virtio_scsi_config *scsicfg) { device_t dev; dev = sc->vtscsi_dev; bzero(scsicfg, sizeof(struct virtio_scsi_config)); VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); VTSCSI_GET_CONFIG(dev, max_target, scsicfg); VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); } #undef VTSCSI_GET_CONFIG static int vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) { int nsegs; nsegs = VTSCSI_MIN_SEGMENTS; if (seg_max > 0) { nsegs += MIN(seg_max, maxphys / PAGE_SIZE + 1); if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); } else nsegs += 1; return (nsegs); } static int vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) { device_t dev; struct vq_alloc_info vq_info[3]; int nvqs; dev = sc->vtscsi_dev; nvqs = 3; VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, "%s request", device_get_nameunit(dev)); return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); } static void vtscsi_check_sizes(struct vtscsi_softc *sc) { int rqsize; if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) { /* * Ensure the assertions in virtqueue_enqueue(), * even if the hypervisor reports a bad seg_max. */ rqsize = virtqueue_size(sc->vtscsi_request_vq); if (sc->vtscsi_max_nsegs > rqsize) { device_printf(sc->vtscsi_dev, "clamping seg_max (%d %d)\n", sc->vtscsi_max_nsegs, rqsize); sc->vtscsi_max_nsegs = rqsize; } } } static void vtscsi_write_device_config(struct vtscsi_softc *sc) { virtio_write_dev_config_4(sc->vtscsi_dev, offsetof(struct virtio_scsi_config, sense_size), VIRTIO_SCSI_SENSE_SIZE); /* * This is the size in the virtio_scsi_cmd_req structure. Note * this value (32) is larger than the maximum CAM CDB size (16). */ virtio_write_dev_config_4(sc->vtscsi_dev, offsetof(struct virtio_scsi_config, cdb_size), VIRTIO_SCSI_CDB_SIZE); } static int vtscsi_reinit(struct vtscsi_softc *sc) { device_t dev; int error; dev = sc->vtscsi_dev; error = virtio_reinit(dev, sc->vtscsi_features); if (error == 0) { vtscsi_write_device_config(sc); virtio_reinit_complete(dev); vtscsi_reinit_event_vq(sc); vtscsi_enable_vqs_intr(sc); } vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); return (error); } static int vtscsi_alloc_cam(struct vtscsi_softc *sc) { device_t dev; struct cam_devq *devq; int openings; dev = sc->vtscsi_dev; openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; devq = cam_simq_alloc(openings); if (devq == NULL) { device_printf(dev, "cannot allocate SIM queue\n"); return (ENOMEM); } sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, openings, devq); if (sc->vtscsi_sim == NULL) { cam_simq_free(devq); device_printf(dev, "cannot allocate SIM\n"); return (ENOMEM); } return (0); } static int vtscsi_register_cam(struct vtscsi_softc *sc) { device_t dev; int registered, error; dev = sc->vtscsi_dev; registered = 0; VTSCSI_LOCK(sc); if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) { error = ENOMEM; device_printf(dev, "cannot register XPT bus\n"); goto fail; } registered = 1; if (xpt_create_path(&sc->vtscsi_path, NULL, cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { error = ENOMEM; device_printf(dev, "cannot create bus path\n"); goto fail; } if (vtscsi_register_async(sc) != CAM_REQ_CMP) { error = EIO; device_printf(dev, "cannot register async callback\n"); goto fail; } VTSCSI_UNLOCK(sc); return (0); fail: if (sc->vtscsi_path != NULL) { xpt_free_path(sc->vtscsi_path); sc->vtscsi_path = NULL; } if (registered != 0) xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); VTSCSI_UNLOCK(sc); return (error); } static void vtscsi_free_cam(struct vtscsi_softc *sc) { VTSCSI_LOCK(sc); if (sc->vtscsi_path != NULL) { vtscsi_deregister_async(sc); xpt_free_path(sc->vtscsi_path); sc->vtscsi_path = NULL; xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); } if (sc->vtscsi_sim != NULL) { cam_sim_free(sc->vtscsi_sim, 1); sc->vtscsi_sim = NULL; } VTSCSI_UNLOCK(sc); } static void vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) { struct cam_sim *sim; struct vtscsi_softc *sc; sim = cb_arg; sc = cam_sim_softc(sim); vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); /* * TODO Once QEMU supports event reporting, we should * (un)subscribe to events here. */ switch (code) { case AC_FOUND_DEVICE: break; case AC_LOST_DEVICE: break; } } static int vtscsi_register_async(struct vtscsi_softc *sc) { struct ccb_setasync csa; memset(&csa, 0, sizeof(csa)); xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; csa.callback = vtscsi_cam_async; csa.callback_arg = sc->vtscsi_sim; xpt_action((union ccb *) &csa); return (csa.ccb_h.status); } static void vtscsi_deregister_async(struct vtscsi_softc *sc) { struct ccb_setasync csa; memset(&csa, 0, sizeof(csa)); xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = 0; csa.callback = vtscsi_cam_async; csa.callback_arg = sc->vtscsi_sim; xpt_action((union ccb *) &csa); } static void vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) { struct vtscsi_softc *sc; struct ccb_hdr *ccbh; sc = cam_sim_softc(sim); ccbh = &ccb->ccb_h; VTSCSI_LOCK_OWNED(sc); if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { /* * The VTSCSI_MTX is briefly dropped between setting * VTSCSI_FLAG_DETACH and deregistering with CAM, so * drop any CCBs that come in during that window. */ ccbh->status = CAM_NO_HBA; xpt_done(ccb); return; } switch (ccbh->func_code) { case XPT_SCSI_IO: vtscsi_cam_scsi_io(sc, sim, ccb); break; case XPT_SET_TRAN_SETTINGS: ccbh->status = CAM_FUNC_NOTAVAIL; xpt_done(ccb); break; case XPT_GET_TRAN_SETTINGS: vtscsi_cam_get_tran_settings(sc, ccb); break; case XPT_RESET_BUS: vtscsi_cam_reset_bus(sc, ccb); break; case XPT_RESET_DEV: vtscsi_cam_reset_dev(sc, ccb); break; case XPT_ABORT: vtscsi_cam_abort(sc, ccb); break; case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, 1); xpt_done(ccb); break; case XPT_PATH_INQ: vtscsi_cam_path_inquiry(sc, sim, ccb); break; default: vtscsi_dprintf(sc, VTSCSI_ERROR, "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); ccbh->status = CAM_REQ_INVALID; xpt_done(ccb); break; } } static void vtscsi_cam_poll(struct cam_sim *sim) { struct vtscsi_softc *sc; sc = cam_sim_softc(sim); vtscsi_complete_vqs_locked(sc); } static void vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, union ccb *ccb) { struct ccb_hdr *ccbh; struct ccb_scsiio *csio; int error; ccbh = &ccb->ccb_h; csio = &ccb->csio; if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { error = EINVAL; ccbh->status = CAM_REQ_INVALID; goto done; } if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { error = EINVAL; ccbh->status = CAM_REQ_INVALID; goto done; } error = vtscsi_start_scsi_cmd(sc, ccb); done: if (error) { vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); xpt_done(ccb); } } static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) { struct ccb_trans_settings *cts; struct ccb_trans_settings_scsi *scsi; cts = &ccb->cts; scsi = &cts->proto_specific.scsi; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_SPC3; cts->transport = XPORT_SAS; cts->transport_version = 0; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static void vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) { int error; error = vtscsi_reset_bus(sc); if (error == 0) ccb->ccb_h.status = CAM_REQ_CMP; else ccb->ccb_h.status = CAM_REQ_CMP_ERR; vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", error, ccb, ccb->ccb_h.status); xpt_done(ccb); } static void vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) { struct ccb_hdr *ccbh; struct vtscsi_request *req; int error; ccbh = &ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = EAGAIN; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); goto fail; } req->vsr_ccb = ccb; error = vtscsi_execute_reset_dev_cmd(sc, req); if (error == 0) return; vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", error, req, ccb); if (error == EAGAIN) ccbh->status = CAM_RESRC_UNAVAIL; else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); } static void vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) { struct vtscsi_request *req; struct ccb_hdr *ccbh; int error; ccbh = &ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = EAGAIN; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); goto fail; } req->vsr_ccb = ccb; error = vtscsi_execute_abort_task_cmd(sc, req); if (error == 0) return; vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", error, req, ccb); if (error == EAGAIN) ccbh->status = CAM_RESRC_UNAVAIL; else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); } static void vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, union ccb *ccb) { device_t dev; struct ccb_pathinq *cpi; dev = sc->vtscsi_dev; cpi = &ccb->cpi; vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); cpi->version_num = 1; cpi->hba_inquiry = PI_TAG_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; if (vtscsi_bus_reset_disable != 0) cpi->hba_misc |= PIM_NOBUSRESET; cpi->hba_eng_cnt = 0; cpi->max_target = sc->vtscsi_max_target; cpi->max_lun = sc->vtscsi_max_lun; cpi->initiator_id = cpi->max_target + 1; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = 300000; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC3; cpi->transport = XPORT_SAS; cpi->transport_version = 0; cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * PAGE_SIZE; cpi->hba_vendor = virtio_get_vendor(dev); cpi->hba_device = virtio_get_device(dev); cpi->hba_subvendor = virtio_get_subvendor(dev); cpi->hba_subdevice = virtio_get_subdevice(dev); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, struct ccb_scsiio *csio) { struct ccb_hdr *ccbh; struct bus_dma_segment *dseg; int i, error; ccbh = &csio->ccb_h; error = 0; switch ((ccbh->flags & CAM_DATA_MASK)) { case CAM_DATA_VADDR: error = sglist_append(sg, csio->data_ptr, csio->dxfer_len); break; case CAM_DATA_PADDR: error = sglist_append_phys(sg, (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len); break; case CAM_DATA_SG: for (i = 0; i < csio->sglist_cnt && error == 0; i++) { dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; error = sglist_append(sg, (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len); } break; case CAM_DATA_SG_PADDR: for (i = 0; i < csio->sglist_cnt && error == 0; i++) { dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; error = sglist_append_phys(sg, (vm_paddr_t) dseg->ds_addr, dseg->ds_len); } break; case CAM_DATA_BIO: error = sglist_append_bio(sg, (struct bio *) csio->data_ptr); break; default: error = EINVAL; break; } return (error); } static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, int *readable, int *writable) { struct sglist *sg; struct ccb_hdr *ccbh; struct ccb_scsiio *csio; struct virtio_scsi_cmd_req *cmd_req; struct virtio_scsi_cmd_resp *cmd_resp; int error; sg = sc->vtscsi_sglist; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_req = &req->vsr_cmd_req; cmd_resp = &req->vsr_cmd_resp; sglist_reset(sg); sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { error = vtscsi_sg_append_scsi_buf(sc, sg, csio); /* At least one segment must be left for the response. */ if (error || sg->sg_nseg == sg->sg_maxseg) goto fail; } *readable = sg->sg_nseg; sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { error = vtscsi_sg_append_scsi_buf(sc, sg, csio); if (error) goto fail; } *writable = sg->sg_nseg - *readable; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " "writable=%d\n", req, ccbh, *readable, *writable); return (0); fail: /* * This should never happen unless maxio was incorrectly set. */ vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " "nseg=%d maxseg=%d\n", error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); return (EFBIG); } static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct virtqueue *vq; struct ccb_scsiio *csio; struct ccb_hdr *ccbh; struct virtio_scsi_cmd_req *cmd_req; struct virtio_scsi_cmd_resp *cmd_resp; int readable, writable, error; sg = sc->vtscsi_sglist; vq = sc->vtscsi_request_vq; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_req = &req->vsr_cmd_req; cmd_resp = &req->vsr_cmd_resp; vtscsi_init_scsi_cmd_req(sc, csio, cmd_req); error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); if (error) return (error); req->vsr_complete = vtscsi_complete_scsi_cmd; cmd_resp->response = -1; error = virtqueue_enqueue(vq, req, sg, readable, writable); if (error) { vtscsi_dprintf(sc, VTSCSI_ERROR, "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); ccbh->status = CAM_REQUEUE_REQ; vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); return (error); } ccbh->status |= CAM_SIM_QUEUED; ccbh->ccbh_vtscsi_req = req; virtqueue_notify(vq); if (ccbh->timeout != CAM_TIME_INFINITY) { req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout, 0, vtscsi_timedout_scsi_cmd, req, 0); } vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", req, ccbh); return (0); } static int vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) { struct vtscsi_request *req; int error; req = vtscsi_dequeue_request(sc); if (req == NULL) { ccb->ccb_h.status = CAM_REQUEUE_REQ; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); return (ENOBUFS); } req->vsr_ccb = ccb; error = vtscsi_execute_scsi_cmd(sc, req); if (error) vtscsi_enqueue_request(sc, req); return (error); } static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct virtio_scsi_ctrl_tmf_resp *tmf_resp; struct vtscsi_request *to_req; uint8_t response; tmf_resp = &req->vsr_tmf_resp; response = tmf_resp->response; to_req = req->vsr_timedout_req; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", req, to_req, response); vtscsi_enqueue_request(sc, req); /* * The timedout request could have completed between when the * abort task was sent and when the host processed it. */ if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) return; /* The timedout request was successfully aborted. */ if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) return; /* Don't bother if the device is going away. */ if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) return; /* The timedout request will be aborted by the reset. */ if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) return; vtscsi_reset_bus(sc); } static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *to_req) { struct sglist *sg; struct ccb_hdr *to_ccbh; struct vtscsi_request *req; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; int error; sg = sc->vtscsi_sglist; to_ccbh = &to_req->vsr_ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = ENOBUFS; goto fail; } tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, (uintptr_t) to_ccbh, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_timedout_req = to_req; req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); if (error == 0) return (0); vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); return (error); } static void vtscsi_timedout_scsi_cmd(void *xreq) { struct vtscsi_softc *sc; struct vtscsi_request *to_req; to_req = xreq; sc = to_req->vsr_softc; vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", to_req, to_req->vsr_ccb, to_req->vsr_state); /* Don't bother if the device is going away. */ if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) return; /* * Bail if the request is not in use. We likely raced when * stopping the callout handler or it has already been aborted. */ if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) return; /* * Complete the request queue in case the timedout request is * actually just pending. */ vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) return; sc->vtscsi_stats.scsi_cmd_timeouts++; to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) return; vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); vtscsi_reset_bus(sc); } static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) { cam_status status; switch (cmd_resp->response) { case VIRTIO_SCSI_S_OK: status = CAM_REQ_CMP; break; case VIRTIO_SCSI_S_OVERRUN: status = CAM_DATA_RUN_ERR; break; case VIRTIO_SCSI_S_ABORTED: status = CAM_REQ_ABORTED; break; case VIRTIO_SCSI_S_BAD_TARGET: status = CAM_SEL_TIMEOUT; break; case VIRTIO_SCSI_S_RESET: status = CAM_SCSI_BUS_RESET; break; case VIRTIO_SCSI_S_BUSY: status = CAM_SCSI_BUSY; break; case VIRTIO_SCSI_S_TRANSPORT_FAILURE: case VIRTIO_SCSI_S_TARGET_FAILURE: case VIRTIO_SCSI_S_NEXUS_FAILURE: status = CAM_SCSI_IT_NEXUS_LOST; break; default: /* VIRTIO_SCSI_S_FAILURE */ status = CAM_REQ_CMP_ERR; break; } return (status); } static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) { uint32_t resp_sense_length; cam_status status; csio->scsi_status = cmd_resp->status; csio->resid = vtscsi_htog32(sc, cmd_resp->resid); if (csio->scsi_status == SCSI_STATUS_OK) status = CAM_REQ_CMP; else status = CAM_SCSI_STATUS_ERROR; resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len); if (resp_sense_length > 0) { status |= CAM_AUTOSNS_VALID; if (resp_sense_length < csio->sense_len) csio->sense_resid = csio->sense_len - resp_sense_length; else csio->sense_resid = 0; memcpy(&csio->sense_data, cmd_resp->sense, csio->sense_len - csio->sense_resid); } vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", csio, csio->scsi_status, csio->resid, csio->sense_resid); return (status); } static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct ccb_hdr *ccbh; struct ccb_scsiio *csio; struct virtio_scsi_cmd_resp *cmd_resp; cam_status status; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_resp = &req->vsr_cmd_resp; KASSERT(ccbh->ccbh_vtscsi_req == req, ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) callout_stop(&req->vsr_callout); status = vtscsi_scsi_cmd_cam_status(cmd_resp); if (status == CAM_REQ_ABORTED) { if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) status = CAM_CMD_TIMEOUT; } else if (status == CAM_REQ_CMP) status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { status |= CAM_DEV_QFRZN; xpt_freeze_devq(ccbh->path, 1); } if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) status |= CAM_RELEASE_SIMQ; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", req, ccbh, status); ccbh->status = status; xpt_done(req->vsr_ccb); vtscsi_enqueue_request(sc, req); } static void vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) { /* XXX We probably shouldn't poll forever. */ req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; do vtscsi_complete_vq(sc, sc->vtscsi_control_vq); while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; } static int vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, struct sglist *sg, int readable, int writable, int flag) { struct virtqueue *vq; int error; vq = sc->vtscsi_control_vq; MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); error = virtqueue_enqueue(vq, req, sg, readable, writable); if (error) { /* * Return EAGAIN when the virtqueue does not have enough * descriptors available. */ if (error == ENOSPC || error == EMSGSIZE) error = EAGAIN; return (error); } virtqueue_notify(vq); if (flag == VTSCSI_EXECUTE_POLL) vtscsi_poll_ctrl_req(sc, req); return (0); } static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; ccb = req->vsr_ccb; ccbh = &ccb->ccb_h; tmf_resp = &req->vsr_tmf_resp; switch (tmf_resp->response) { case VIRTIO_SCSI_S_FUNCTION_COMPLETE: ccbh->status = CAM_REQ_CMP; break; case VIRTIO_SCSI_S_FUNCTION_REJECTED: ccbh->status = CAM_UA_ABORT; break; default: ccbh->status = CAM_REQ_CMP_ERR; break; } xpt_done(ccb); vtscsi_enqueue_request(sc, req); } static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct ccb_abort *cab; struct ccb_hdr *ccbh; struct ccb_hdr *abort_ccbh; struct vtscsi_request *abort_req; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; int error; sg = sc->vtscsi_sglist; cab = &req->vsr_ccb->cab; ccbh = &cab->ccb_h; tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; /* CCB header and request that's to be aborted. */ abort_ccbh = &cab->abort_ccb->ccb_h; abort_req = abort_ccbh->ccbh_vtscsi_req; if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { error = EINVAL; goto fail; } /* Only attempt to abort requests that could be in-flight. */ if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { error = EALREADY; goto fail; } abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) callout_stop(&abort_req->vsr_callout); vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, (uintptr_t) abort_ccbh, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_complete = vtscsi_complete_abort_task_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); fail: vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " "abort_req=%p\n", error, req, abort_ccbh, abort_req); return (error); } static void vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; ccb = req->vsr_ccb; ccbh = &ccb->ccb_h; tmf_resp = &req->vsr_tmf_resp; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", req, ccb, tmf_resp->response); if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { ccbh->status = CAM_REQ_CMP; vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, ccbh->target_lun); } else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); vtscsi_enqueue_request(sc, req); } static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct ccb_resetdev *crd; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; uint32_t subtype; int error; sg = sc->vtscsi_sglist; crd = &req->vsr_ccb->crd; ccbh = &crd->ccb_h; tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; if (ccbh->target_lun == CAM_LUN_WILDCARD) subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; else subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_complete = vtscsi_complete_reset_dev_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", error, req, ccbh); return (error); } static void vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) { *target_id = lun[1]; *lun_id = (lun[2] << 8) | lun[3]; } static void vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) { lun[0] = 1; lun[1] = ccbh->target_id; lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); lun[3] = ccbh->target_lun & 0xFF; } static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *sc, struct ccb_scsiio *csio, struct virtio_scsi_cmd_req *cmd_req) { uint8_t attr; switch (csio->tag_action) { case MSG_HEAD_OF_Q_TAG: attr = VIRTIO_SCSI_S_HEAD; break; case MSG_ORDERED_Q_TAG: attr = VIRTIO_SCSI_S_ORDERED; break; case MSG_ACA_TASK: attr = VIRTIO_SCSI_S_ACA; break; default: /* MSG_SIMPLE_Q_TAG */ attr = VIRTIO_SCSI_S_SIMPLE; break; } vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio); cmd_req->task_attr = attr; memcpy(cmd_req->cdb, csio->ccb_h.flags & CAM_CDB_POINTER ? csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, csio->cdb_len); } static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *sc, struct ccb_hdr *ccbh, uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) { vtscsi_set_request_lun(ccbh, tmf_req->lun); tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF); tmf_req->subtype = vtscsi_gtoh32(sc, subtype); tmf_req->tag = vtscsi_gtoh64(sc, tag); } static void vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) { int frozen; frozen = sc->vtscsi_frozen; if (reason & VTSCSI_REQUEST && (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; if (reason & VTSCSI_REQUEST_VQ && (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; /* Freeze the SIMQ if transitioned to frozen. */ if (frozen == 0 && sc->vtscsi_frozen != 0) { vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); xpt_freeze_simq(sc->vtscsi_sim, 1); } } static int vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) { int thawed; if (sc->vtscsi_frozen == 0 || reason == 0) return (0); if (reason & VTSCSI_REQUEST && sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; if (reason & VTSCSI_REQUEST_VQ && sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; thawed = sc->vtscsi_frozen == 0; if (thawed != 0) vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); return (thawed); } static void vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, target_id_t target_id, lun_id_t lun_id) { struct cam_path *path; /* Use the wildcard path from our softc for bus announcements. */ if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { xpt_async(ac_code, sc->vtscsi_path, NULL); return; } if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), target_id, lun_id) != CAM_REQ_CMP) { vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); return; } xpt_async(ac_code, path, NULL); xpt_free_path(path); } static void vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, lun_id_t lun_id) { union ccb *ccb; cam_status status; ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); return; } status = xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->vtscsi_sim), target_id, lun_id); if (status != CAM_REQ_CMP) { xpt_free_ccb(ccb); return; } xpt_rescan(ccb); } static void vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) { vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); } static void vtscsi_transport_reset_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { target_id_t target_id; lun_id_t lun_id; vtscsi_get_request_lun(event->lun, &target_id, &lun_id); switch (event->reason) { case VIRTIO_SCSI_EVT_RESET_RESCAN: case VIRTIO_SCSI_EVT_RESET_REMOVED: vtscsi_execute_rescan(sc, target_id, lun_id); break; default: device_printf(sc->vtscsi_dev, "unhandled transport event reason: %d\n", event->reason); break; } } static void vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { int error __diagused; if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { switch (event->event) { case VIRTIO_SCSI_T_TRANSPORT_RESET: vtscsi_transport_reset_event(sc, event); break; default: device_printf(sc->vtscsi_dev, "unhandled event: %d\n", event->event); break; } } else vtscsi_execute_rescan_bus(sc); /* * This should always be successful since the buffer * was just dequeued. */ error = vtscsi_enqueue_event_buf(sc, event); KASSERT(error == 0, ("cannot requeue event buffer: %d", error)); } static int vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { struct sglist *sg; struct virtqueue *vq; int size, error; sg = sc->vtscsi_sglist; vq = sc->vtscsi_event_vq; size = sc->vtscsi_event_buf_size; bzero(event, size); sglist_reset(sg); error = sglist_append(sg, event, size); if (error) return (error); error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); if (error) return (error); virtqueue_notify(vq); return (0); } static int vtscsi_init_event_vq(struct vtscsi_softc *sc) { struct virtio_scsi_event *event; int i, size, error; /* * The first release of QEMU with VirtIO SCSI support would crash * when attempting to notify the event virtqueue. This was fixed * when hotplug support was added. */ if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) size = sc->vtscsi_event_buf_size; else size = 0; if (size < sizeof(struct virtio_scsi_event)) return (0); for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { event = &sc->vtscsi_event_bufs[i]; error = vtscsi_enqueue_event_buf(sc, event); if (error) break; } /* * Even just one buffer is enough. Missed events are * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. */ if (i > 0) error = 0; return (error); } static void vtscsi_reinit_event_vq(struct vtscsi_softc *sc) { struct virtio_scsi_event *event; int i, error; if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) return; for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { event = &sc->vtscsi_event_bufs[i]; error = vtscsi_enqueue_event_buf(sc, event); if (error) break; } KASSERT(i > 0, ("cannot reinit event vq: %d", error)); } static void vtscsi_drain_event_vq(struct vtscsi_softc *sc) { struct virtqueue *vq; int last; vq = sc->vtscsi_event_vq; last = 0; while (virtqueue_drain(vq, &last) != NULL) ; KASSERT(virtqueue_empty(vq), ("eventvq not empty")); } static void vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) { VTSCSI_LOCK_OWNED(sc); if (sc->vtscsi_request_vq != NULL) vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (sc->vtscsi_control_vq != NULL) vtscsi_complete_vq(sc, sc->vtscsi_control_vq); } static void vtscsi_complete_vqs(struct vtscsi_softc *sc) { VTSCSI_LOCK(sc); vtscsi_complete_vqs_locked(sc); VTSCSI_UNLOCK(sc); } static void vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; int detach; ccb = req->vsr_ccb; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); /* * The callout must be drained when detaching since the request is * about to be freed. The VTSCSI_MTX must not be held for this in * case the callout is pending because there is a deadlock potential. * Otherwise, the virtqueue is being drained because of a bus reset * so we only need to attempt to stop the callouts. */ detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; if (detach != 0) VTSCSI_LOCK_NOTOWNED(sc); else VTSCSI_LOCK_OWNED(sc); if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { if (detach != 0) callout_drain(&req->vsr_callout); else callout_stop(&req->vsr_callout); } if (ccb != NULL) { if (detach != 0) { VTSCSI_LOCK(sc); ccb->ccb_h.status = CAM_NO_HBA; } else ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); if (detach != 0) VTSCSI_UNLOCK(sc); } vtscsi_enqueue_request(sc, req); } static void vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) { struct vtscsi_request *req; int last; last = 0; vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); while ((req = virtqueue_drain(vq, &last)) != NULL) vtscsi_cancel_request(sc, req); KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); } static void vtscsi_drain_vqs(struct vtscsi_softc *sc) { if (sc->vtscsi_control_vq != NULL) vtscsi_drain_vq(sc, sc->vtscsi_control_vq); if (sc->vtscsi_request_vq != NULL) vtscsi_drain_vq(sc, sc->vtscsi_request_vq); if (sc->vtscsi_event_vq != NULL) vtscsi_drain_event_vq(sc); } static void vtscsi_stop(struct vtscsi_softc *sc) { vtscsi_disable_vqs_intr(sc); virtio_stop(sc->vtscsi_dev); } static int vtscsi_reset_bus(struct vtscsi_softc *sc) { int error; VTSCSI_LOCK_OWNED(sc); if (vtscsi_bus_reset_disable != 0) { device_printf(sc->vtscsi_dev, "bus reset disabled\n"); return (0); } sc->vtscsi_flags |= VTSCSI_FLAG_RESET; /* * vtscsi_stop() will cause the in-flight requests to be canceled. * Those requests are then completed here so CAM will retry them * after the reset is complete. */ vtscsi_stop(sc); vtscsi_complete_vqs_locked(sc); /* Rid the virtqueues of any remaining requests. */ vtscsi_drain_vqs(sc); /* * Any resource shortage that froze the SIMQ cannot persist across * a bus reset so ensure it gets thawed here. */ if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) xpt_release_simq(sc->vtscsi_sim, 0); error = vtscsi_reinit(sc); if (error) { device_printf(sc->vtscsi_dev, "reinitialization failed, stopping device...\n"); vtscsi_stop(sc); } else vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; return (error); } static void vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { #ifdef INVARIANTS int req_nsegs, resp_nsegs; req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); KASSERT(req_nsegs == 1, ("request crossed page boundary")); KASSERT(resp_nsegs == 1, ("response crossed page boundary")); #endif req->vsr_softc = sc; callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0); } static int vtscsi_alloc_requests(struct vtscsi_softc *sc) { struct vtscsi_request *req; int i, nreqs; /* * Commands destined for either the request or control queues come * from the same SIM queue. Use the size of the request virtqueue * as it (should) be much more frequently used. Some additional * requests are allocated for internal (TMF) use. */ nreqs = virtqueue_size(sc->vtscsi_request_vq); if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) nreqs /= VTSCSI_MIN_SEGMENTS; nreqs += VTSCSI_RESERVED_REQUESTS; for (i = 0; i < nreqs; i++) { req = malloc(sizeof(struct vtscsi_request), M_DEVBUF, M_NOWAIT); if (req == NULL) return (ENOMEM); vtscsi_init_request(sc, req); sc->vtscsi_nrequests++; vtscsi_enqueue_request(sc, req); } return (0); } static void vtscsi_free_requests(struct vtscsi_softc *sc) { struct vtscsi_request *req; while ((req = vtscsi_dequeue_request(sc)) != NULL) { KASSERT(callout_active(&req->vsr_callout) == 0, ("request callout still active")); sc->vtscsi_nrequests--; free(req, M_DEVBUF); } KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", sc->vtscsi_nrequests)); } static void vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { KASSERT(req->vsr_softc == sc, ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); /* A request is available so the SIMQ could be released. */ if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) xpt_release_simq(sc->vtscsi_sim, 1); req->vsr_ccb = NULL; req->vsr_complete = NULL; req->vsr_ptr0 = NULL; req->vsr_state = VTSCSI_REQ_STATE_FREE; req->vsr_flags = 0; bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); /* * We insert at the tail of the queue in order to make it * very unlikely a request will be reused if we race with * stopping its callout handler. */ TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); } static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *sc) { struct vtscsi_request *req; req = TAILQ_FIRST(&sc->vtscsi_req_free); if (req != NULL) { req->vsr_state = VTSCSI_REQ_STATE_INUSE; TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); } else sc->vtscsi_stats.dequeue_no_requests++; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); return (req); } static void vtscsi_complete_request(struct vtscsi_request *req) { if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; if (req->vsr_complete != NULL) req->vsr_complete(req->vsr_softc, req); } static void vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) { struct vtscsi_request *req; VTSCSI_LOCK_OWNED(sc); while ((req = virtqueue_dequeue(vq, NULL)) != NULL) vtscsi_complete_request(req); } static void vtscsi_control_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; sc = xsc; vq = sc->vtscsi_control_vq; again: VTSCSI_LOCK(sc); vtscsi_complete_vq(sc, sc->vtscsi_control_vq); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_event_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; struct virtio_scsi_event *event; sc = xsc; vq = sc->vtscsi_event_vq; again: VTSCSI_LOCK(sc); while ((event = virtqueue_dequeue(vq, NULL)) != NULL) vtscsi_handle_event(sc, event); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_request_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; sc = xsc; vq = sc->vtscsi_request_vq; again: VTSCSI_LOCK(sc); vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) { virtqueue_disable_intr(sc->vtscsi_control_vq); virtqueue_disable_intr(sc->vtscsi_event_vq); virtqueue_disable_intr(sc->vtscsi_request_vq); } static void vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) { virtqueue_enable_intr(sc->vtscsi_control_vq); virtqueue_enable_intr(sc->vtscsi_event_vq); virtqueue_enable_intr(sc->vtscsi_request_vq); } static void vtscsi_get_tunables(struct vtscsi_softc *sc) { char tmpstr[64]; TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", device_get_unit(sc->vtscsi_dev)); TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); } static void vtscsi_setup_sysctl(struct vtscsi_softc *sc) { device_t dev; struct vtscsi_statistics *stats; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = sc->vtscsi_dev; stats = &sc->vtscsi_stats; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", CTLFLAG_RW, &sc->vtscsi_debug, 0, "Debug level"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", CTLFLAG_RD, &stats->scsi_cmd_timeouts, "SCSI command timeouts"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", CTLFLAG_RD, &stats->dequeue_no_requests, "No available requests to dequeue"); } static void vtscsi_printf_req(struct vtscsi_request *req, const char *func, const char *fmt, ...) { struct vtscsi_softc *sc; union ccb *ccb; struct sbuf sb; va_list ap; char str[192]; char path_str[64]; if (req == NULL) return; sc = req->vsr_softc; ccb = req->vsr_ccb; va_start(ap, fmt); sbuf_new(&sb, str, sizeof(str), 0); if (ccb == NULL) { sbuf_printf(&sb, "(noperiph:%s%d:%u): ", cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), cam_sim_bus(sc->vtscsi_sim)); } else { xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); sbuf_cat(&sb, path_str); if (ccb->ccb_h.func_code == XPT_SCSI_IO) { scsi_command_string(&ccb->csio, &sb); sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); } } sbuf_vprintf(&sb, fmt, ap); va_end(ap); sbuf_finish(&sb); printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, sbuf_data(&sb)); }