/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2012, Alexey Zaytsev */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtiovar.h" #include "virtioreg.h" /* Feature bits */ #define VIRTIO_BLK_F_BARRIER (1<<0) #define VIRTIO_BLK_F_SIZE_MAX (1<<1) #define VIRTIO_BLK_F_SEG_MAX (1<<2) #define VIRTIO_BLK_F_GEOMETRY (1<<4) #define VIRTIO_BLK_F_RO (1<<5) #define VIRTIO_BLK_F_BLK_SIZE (1<<6) #define VIRTIO_BLK_F_SCSI (1<<7) #define VIRTIO_BLK_F_FLUSH (1<<9) #define VIRTIO_BLK_F_TOPOLOGY (1<<10) /* Configuration registers */ #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */ #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */ #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */ #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */ #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */ #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */ #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */ #define VIRTIO_BLK_CONFIG_TOPO_PBEXP 24 /* 8bit */ #define VIRTIO_BLK_CONFIG_TOPO_ALIGN 25 /* 8bit */ #define VIRTIO_BLK_CONFIG_TOPO_MIN_SZ 26 /* 16bit */ #define VIRTIO_BLK_CONFIG_TOPO_OPT_SZ 28 /* 32bit */ /* Command */ #define VIRTIO_BLK_T_IN 0 #define VIRTIO_BLK_T_OUT 1 #define VIRTIO_BLK_T_SCSI_CMD 2 #define VIRTIO_BLK_T_SCSI_CMD_OUT 3 #define VIRTIO_BLK_T_FLUSH 4 #define VIRTIO_BLK_T_FLUSH_OUT 5 #define VIRTIO_BLK_T_GET_ID 8 #define VIRTIO_BLK_T_BARRIER 0x80000000 #define VIRTIO_BLK_ID_BYTES 20 /* devid */ /* Statuses */ #define VIRTIO_BLK_S_OK 0 #define VIRTIO_BLK_S_IOERR 1 #define VIRTIO_BLK_S_UNSUPP 2 #define DEF_MAXINDIRECT (128) #define DEF_MAXSECTOR (4096) #define VIOBLK_POISON 0xdead0001dead0001 /* * Static Variables. */ static char vioblk_ident[] = "VirtIO block driver"; /* Request header structure */ struct vioblk_req_hdr { uint32_t type; /* VIRTIO_BLK_T_* */ uint32_t ioprio; uint64_t sector; }; struct vioblk_req { struct vioblk_req_hdr hdr; uint8_t status; uint8_t unused[3]; unsigned int ndmac; ddi_dma_handle_t dmah; ddi_dma_handle_t bd_dmah; ddi_dma_cookie_t dmac; bd_xfer_t *xfer; }; struct vioblk_stats { struct kstat_named sts_rw_outofmemory; struct kstat_named sts_rw_badoffset; struct kstat_named sts_rw_queuemax; struct kstat_named sts_rw_cookiesmax; struct kstat_named sts_rw_cacheflush; struct kstat_named sts_intr_queuemax; struct kstat_named sts_intr_total; struct kstat_named sts_io_errors; struct kstat_named sts_unsupp_errors; struct kstat_named sts_nxio_errors; }; struct vioblk_lstats { uint64_t rw_cacheflush; uint64_t intr_total; unsigned int rw_cookiesmax; unsigned int intr_queuemax; unsigned int io_errors; unsigned int unsupp_errors; unsigned int nxio_errors; }; struct vioblk_softc { dev_info_t *sc_dev; /* mirrors virtio_softc->sc_dev */ struct virtio_softc sc_virtio; struct virtqueue *sc_vq; bd_handle_t bd_h; struct vioblk_req *sc_reqs; struct vioblk_stats *ks_data; kstat_t *sc_intrstat; uint64_t sc_capacity; uint64_t sc_nblks; struct vioblk_lstats sc_stats; short sc_blkflags; boolean_t sc_in_poll_mode; boolean_t sc_readonly; int sc_blk_size; int sc_pblk_size; int sc_seg_max; int sc_seg_size_max; kmutex_t lock_devid; kcondvar_t cv_devid; char devid[VIRTIO_BLK_ID_BYTES + 1]; }; static int vioblk_get_id(struct vioblk_softc *sc); static int vioblk_read(void *arg, bd_xfer_t *xfer); static int vioblk_write(void *arg, bd_xfer_t *xfer); static int vioblk_flush(void *arg, bd_xfer_t *xfer); static void vioblk_driveinfo(void *arg, bd_drive_t *drive); static int vioblk_mediainfo(void *arg, bd_media_t *media); static int vioblk_devid_init(void *, dev_info_t *, ddi_devid_t *); uint_t vioblk_int_handler(caddr_t arg1, caddr_t arg2); static bd_ops_t vioblk_ops = { BD_OPS_VERSION_0, vioblk_driveinfo, vioblk_mediainfo, vioblk_devid_init, vioblk_flush, vioblk_read, vioblk_write, }; static int vioblk_quiesce(dev_info_t *); static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t); static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t); static struct dev_ops vioblk_dev_ops = { DEVO_REV, 0, ddi_no_info, nulldev, /* identify */ nulldev, /* probe */ vioblk_attach, /* attach */ vioblk_detach, /* detach */ nodev, /* reset */ NULL, /* cb_ops */ NULL, /* bus_ops */ NULL, /* power */ vioblk_quiesce /* quiesce */ }; /* Standard Module linkage initialization for a Streams driver */ extern struct mod_ops mod_driverops; static struct modldrv modldrv = { &mod_driverops, /* Type of module. This one is a driver */ vioblk_ident, /* short description */ &vioblk_dev_ops /* driver specific ops */ }; static struct modlinkage modlinkage = { MODREV_1, { (void *)&modldrv, NULL, }, }; ddi_device_acc_attr_t vioblk_attr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, /* virtio is always native byte order */ DDI_STORECACHING_OK_ACC, DDI_DEFAULT_ACC }; /* DMA attr for the header/status blocks. */ static ddi_dma_attr_t vioblk_req_dma_attr = { DMA_ATTR_V0, /* dma_attr version */ 0, /* dma_attr_addr_lo */ 0xFFFFFFFFFFFFFFFFull, /* dma_attr_addr_hi */ 0x00000000FFFFFFFFull, /* dma_attr_count_max */ 1, /* dma_attr_align */ 1, /* dma_attr_burstsizes */ 1, /* dma_attr_minxfer */ 0xFFFFFFFFull, /* dma_attr_maxxfer */ 0xFFFFFFFFFFFFFFFFull, /* dma_attr_seg */ 1, /* dma_attr_sgllen */ 1, /* dma_attr_granular */ 0, /* dma_attr_flags */ }; /* DMA attr for the data blocks. */ static ddi_dma_attr_t vioblk_bd_dma_attr = { DMA_ATTR_V0, /* dma_attr version */ 0, /* dma_attr_addr_lo */ 0xFFFFFFFFFFFFFFFFull, /* dma_attr_addr_hi */ 0x00000000FFFFFFFFull, /* dma_attr_count_max */ 1, /* dma_attr_align */ 1, /* dma_attr_burstsizes */ 1, /* dma_attr_minxfer */ 0, /* dma_attr_maxxfer, set in attach */ 0xFFFFFFFFFFFFFFFFull, /* dma_attr_seg */ 0, /* dma_attr_sgllen, set in attach */ 1, /* dma_attr_granular */ 0, /* dma_attr_flags */ }; static int vioblk_rw(struct vioblk_softc *sc, bd_xfer_t *xfer, int type, uint32_t len) { struct vioblk_req *req; struct vq_entry *ve_hdr; int total_cookies, write; write = (type == VIRTIO_BLK_T_OUT || type == VIRTIO_BLK_T_FLUSH_OUT) ? 1 : 0; total_cookies = 2; if ((xfer->x_blkno + xfer->x_nblks) > sc->sc_nblks) { sc->ks_data->sts_rw_badoffset.value.ui64++; return (EINVAL); } /* allocate top entry */ ve_hdr = vq_alloc_entry(sc->sc_vq); if (!ve_hdr) { sc->ks_data->sts_rw_outofmemory.value.ui64++; return (ENOMEM); } /* getting request */ req = &sc->sc_reqs[ve_hdr->qe_index]; req->hdr.type = type; req->hdr.ioprio = 0; req->hdr.sector = xfer->x_blkno; req->xfer = xfer; /* Header */ virtio_ve_add_indirect_buf(ve_hdr, req->dmac.dmac_laddress, sizeof (struct vioblk_req_hdr), B_TRUE); /* Payload */ if (len > 0) { virtio_ve_add_cookie(ve_hdr, xfer->x_dmah, xfer->x_dmac, xfer->x_ndmac, write ? B_TRUE : B_FALSE); total_cookies += xfer->x_ndmac; } /* Status */ virtio_ve_add_indirect_buf(ve_hdr, req->dmac.dmac_laddress + sizeof (struct vioblk_req_hdr), sizeof (uint8_t), B_FALSE); /* sending the whole chain to the device */ virtio_push_chain(ve_hdr, B_TRUE); if (sc->sc_stats.rw_cookiesmax < total_cookies) sc->sc_stats.rw_cookiesmax = total_cookies; return (DDI_SUCCESS); } /* * Now in polling mode. Interrupts are off, so we * 1) poll for the already queued requests to complete. * 2) push our request. * 3) wait for our request to complete. */ static int vioblk_rw_poll(struct vioblk_softc *sc, bd_xfer_t *xfer, int type, uint32_t len) { clock_t tmout; int ret; ASSERT(xfer->x_flags & BD_XFER_POLL); /* Prevent a hard hang. */ tmout = drv_usectohz(30000000); /* Poll for an empty queue */ while (vq_num_used(sc->sc_vq)) { /* Check if any pending requests completed. */ ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL); if (ret != DDI_INTR_CLAIMED) { drv_usecwait(10); tmout -= 10; return (ETIMEDOUT); } } ret = vioblk_rw(sc, xfer, type, len); if (ret) return (ret); tmout = drv_usectohz(30000000); /* Poll for an empty queue again. */ while (vq_num_used(sc->sc_vq)) { /* Check if any pending requests completed. */ ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL); if (ret != DDI_INTR_CLAIMED) { drv_usecwait(10); tmout -= 10; return (ETIMEDOUT); } } return (DDI_SUCCESS); } static int vioblk_read(void *arg, bd_xfer_t *xfer) { int ret; struct vioblk_softc *sc = (void *)arg; if (xfer->x_flags & BD_XFER_POLL) { if (!sc->sc_in_poll_mode) { virtio_stop_vq_intr(sc->sc_vq); sc->sc_in_poll_mode = 1; } ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_IN, xfer->x_nblks * DEV_BSIZE); } else { if (sc->sc_in_poll_mode) { virtio_start_vq_intr(sc->sc_vq); sc->sc_in_poll_mode = 0; } ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_IN, xfer->x_nblks * DEV_BSIZE); } return (ret); } static int vioblk_write(void *arg, bd_xfer_t *xfer) { int ret; struct vioblk_softc *sc = (void *)arg; if (xfer->x_flags & BD_XFER_POLL) { if (!sc->sc_in_poll_mode) { virtio_stop_vq_intr(sc->sc_vq); sc->sc_in_poll_mode = 1; } ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_OUT, xfer->x_nblks * DEV_BSIZE); } else { if (sc->sc_in_poll_mode) { virtio_start_vq_intr(sc->sc_vq); sc->sc_in_poll_mode = 0; } ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_OUT, xfer->x_nblks * DEV_BSIZE); } return (ret); } static int vioblk_flush(void *arg, bd_xfer_t *xfer) { int ret; struct vioblk_softc *sc = (void *)arg; ASSERT((xfer->x_flags & BD_XFER_POLL) == 0); ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_FLUSH_OUT, xfer->x_nblks * DEV_BSIZE); if (!ret) sc->sc_stats.rw_cacheflush++; return (ret); } static void vioblk_driveinfo(void *arg, bd_drive_t *drive) { struct vioblk_softc *sc = (void *)arg; drive->d_qsize = sc->sc_vq->vq_num; drive->d_removable = B_FALSE; drive->d_hotpluggable = B_TRUE; drive->d_target = 0; drive->d_lun = 0; drive->d_vendor = "Virtio"; drive->d_vendor_len = strlen(drive->d_vendor); drive->d_product = "Block Device"; drive->d_product_len = strlen(drive->d_product); (void) vioblk_get_id(sc); drive->d_serial = sc->devid; drive->d_serial_len = strlen(drive->d_serial); drive->d_revision = "0000"; drive->d_revision_len = strlen(drive->d_revision); } static int vioblk_mediainfo(void *arg, bd_media_t *media) { struct vioblk_softc *sc = (void *)arg; media->m_nblks = sc->sc_nblks; media->m_blksize = sc->sc_blk_size; media->m_readonly = sc->sc_readonly; media->m_pblksize = sc->sc_pblk_size; return (0); } static int vioblk_get_id(struct vioblk_softc *sc) { clock_t deadline; int ret; bd_xfer_t xfer; deadline = ddi_get_lbolt() + (clock_t)drv_usectohz(3 * 1000000); (void) memset(&xfer, 0, sizeof (bd_xfer_t)); xfer.x_nblks = 1; ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_bd_dma_attr, DDI_DMA_SLEEP, NULL, &xfer.x_dmah); if (ret != DDI_SUCCESS) goto out_alloc; ret = ddi_dma_addr_bind_handle(xfer.x_dmah, NULL, (caddr_t)&sc->devid, VIRTIO_BLK_ID_BYTES, DDI_DMA_READ | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &xfer.x_dmac, &xfer.x_ndmac); if (ret != DDI_DMA_MAPPED) { ret = DDI_FAILURE; goto out_map; } mutex_enter(&sc->lock_devid); ret = vioblk_rw(sc, &xfer, VIRTIO_BLK_T_GET_ID, VIRTIO_BLK_ID_BYTES); if (ret) { mutex_exit(&sc->lock_devid); goto out_rw; } /* wait for reply */ ret = cv_timedwait(&sc->cv_devid, &sc->lock_devid, deadline); mutex_exit(&sc->lock_devid); (void) ddi_dma_unbind_handle(xfer.x_dmah); ddi_dma_free_handle(&xfer.x_dmah); /* timeout */ if (ret < 0) { dev_err(sc->sc_dev, CE_WARN, "Cannot get devid from the device"); return (DDI_FAILURE); } return (0); out_rw: (void) ddi_dma_unbind_handle(xfer.x_dmah); out_map: ddi_dma_free_handle(&xfer.x_dmah); out_alloc: return (ret); } static int vioblk_devid_init(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) { struct vioblk_softc *sc = (void *)arg; int ret; ret = vioblk_get_id(sc); if (ret != DDI_SUCCESS) return (ret); ret = ddi_devid_init(devinfo, DEVID_ATA_SERIAL, VIRTIO_BLK_ID_BYTES, sc->devid, devid); if (ret != DDI_SUCCESS) { dev_err(devinfo, CE_WARN, "Cannot build devid from the device"); return (ret); } dev_debug(sc->sc_dev, CE_NOTE, "devid %x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x", sc->devid[0], sc->devid[1], sc->devid[2], sc->devid[3], sc->devid[4], sc->devid[5], sc->devid[6], sc->devid[7], sc->devid[8], sc->devid[9], sc->devid[10], sc->devid[11], sc->devid[12], sc->devid[13], sc->devid[14], sc->devid[15], sc->devid[16], sc->devid[17], sc->devid[18], sc->devid[19]); return (0); } static void vioblk_show_features(struct vioblk_softc *sc, const char *prefix, uint32_t features) { char buf[512]; char *bufp = buf; char *bufend = buf + sizeof (buf); /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, prefix); /* LINTED E_PTRDIFF_OVERFLOW */ bufp += virtio_show_features(features, bufp, bufend - bufp); /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "Vioblk ( "); if (features & VIRTIO_BLK_F_BARRIER) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "BARRIER "); if (features & VIRTIO_BLK_F_SIZE_MAX) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "SIZE_MAX "); if (features & VIRTIO_BLK_F_SEG_MAX) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "SEG_MAX "); if (features & VIRTIO_BLK_F_GEOMETRY) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "GEOMETRY "); if (features & VIRTIO_BLK_F_RO) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "RO "); if (features & VIRTIO_BLK_F_BLK_SIZE) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "BLK_SIZE "); if (features & VIRTIO_BLK_F_SCSI) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "SCSI "); if (features & VIRTIO_BLK_F_FLUSH) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "FLUSH "); if (features & VIRTIO_BLK_F_TOPOLOGY) /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, "TOPOLOGY "); /* LINTED E_PTRDIFF_OVERFLOW */ bufp += snprintf(bufp, bufend - bufp, ")"); *bufp = '\0'; dev_debug(sc->sc_dev, CE_NOTE, "%s", buf); } static int vioblk_dev_features(struct vioblk_softc *sc) { uint32_t host_features; host_features = virtio_negotiate_features(&sc->sc_virtio, VIRTIO_BLK_F_RO | VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_TOPOLOGY | VIRTIO_BLK_F_SEG_MAX | VIRTIO_BLK_F_SIZE_MAX | VIRTIO_F_RING_INDIRECT_DESC); vioblk_show_features(sc, "Host features: ", host_features); vioblk_show_features(sc, "Negotiated features: ", sc->sc_virtio.sc_features); if (!(sc->sc_virtio.sc_features & VIRTIO_F_RING_INDIRECT_DESC)) { dev_err(sc->sc_dev, CE_NOTE, "Host does not support RING_INDIRECT_DESC, bye."); return (DDI_FAILURE); } return (DDI_SUCCESS); } /* ARGSUSED */ uint_t vioblk_int_handler(caddr_t arg1, caddr_t arg2) { struct virtio_softc *vsc = (void *)arg1; struct vioblk_softc *sc = container_of(vsc, struct vioblk_softc, sc_virtio); struct vq_entry *ve; uint32_t len; int i = 0, error; while ((ve = virtio_pull_chain(sc->sc_vq, &len))) { struct vioblk_req *req = &sc->sc_reqs[ve->qe_index]; bd_xfer_t *xfer = req->xfer; uint8_t status = req->status; uint32_t type = req->hdr.type; if (req->xfer == (void *)VIOBLK_POISON) { dev_err(sc->sc_dev, CE_WARN, "Poisoned descriptor!"); virtio_free_chain(ve); return (DDI_INTR_CLAIMED); } req->xfer = (void *) VIOBLK_POISON; /* Note: blkdev tears down the payload mapping for us. */ virtio_free_chain(ve); /* returning payload back to blkdev */ switch (status) { case VIRTIO_BLK_S_OK: error = 0; break; case VIRTIO_BLK_S_IOERR: error = EIO; sc->sc_stats.io_errors++; break; case VIRTIO_BLK_S_UNSUPP: sc->sc_stats.unsupp_errors++; error = ENOTTY; break; default: sc->sc_stats.nxio_errors++; error = ENXIO; break; } if (type == VIRTIO_BLK_T_GET_ID) { /* notify devid_init */ mutex_enter(&sc->lock_devid); cv_broadcast(&sc->cv_devid); mutex_exit(&sc->lock_devid); } else bd_xfer_done(xfer, error); i++; } /* update stats */ if (sc->sc_stats.intr_queuemax < i) sc->sc_stats.intr_queuemax = i; sc->sc_stats.intr_total++; return (DDI_INTR_CLAIMED); } /* ARGSUSED */ uint_t vioblk_config_handler(caddr_t arg1, caddr_t arg2) { return (DDI_INTR_CLAIMED); } static int vioblk_register_ints(struct vioblk_softc *sc) { int ret; struct virtio_int_handler vioblk_conf_h = { vioblk_config_handler }; struct virtio_int_handler vioblk_vq_h[] = { { vioblk_int_handler }, { NULL }, }; ret = virtio_register_ints(&sc->sc_virtio, &vioblk_conf_h, vioblk_vq_h); return (ret); } static void vioblk_free_reqs(struct vioblk_softc *sc) { int i, qsize; qsize = sc->sc_vq->vq_num; for (i = 0; i < qsize; i++) { struct vioblk_req *req = &sc->sc_reqs[i]; if (req->ndmac) (void) ddi_dma_unbind_handle(req->dmah); if (req->dmah) ddi_dma_free_handle(&req->dmah); } kmem_free(sc->sc_reqs, sizeof (struct vioblk_req) * qsize); } static int vioblk_alloc_reqs(struct vioblk_softc *sc) { int i, qsize; int ret; qsize = sc->sc_vq->vq_num; sc->sc_reqs = kmem_zalloc(sizeof (struct vioblk_req) * qsize, KM_SLEEP); for (i = 0; i < qsize; i++) { struct vioblk_req *req = &sc->sc_reqs[i]; ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_req_dma_attr, DDI_DMA_SLEEP, NULL, &req->dmah); if (ret != DDI_SUCCESS) { dev_err(sc->sc_dev, CE_WARN, "Can't allocate dma handle for req " "buffer %d", i); goto exit; } ret = ddi_dma_addr_bind_handle(req->dmah, NULL, (caddr_t)&req->hdr, sizeof (struct vioblk_req_hdr) + sizeof (uint8_t), DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &req->dmac, &req->ndmac); if (ret != DDI_DMA_MAPPED) { dev_err(sc->sc_dev, CE_WARN, "Can't bind req buffer %d", i); goto exit; } } return (0); exit: vioblk_free_reqs(sc); return (ENOMEM); } static int vioblk_ksupdate(kstat_t *ksp, int rw) { struct vioblk_softc *sc = ksp->ks_private; if (rw == KSTAT_WRITE) return (EACCES); sc->ks_data->sts_rw_cookiesmax.value.ui32 = sc->sc_stats.rw_cookiesmax; sc->ks_data->sts_intr_queuemax.value.ui32 = sc->sc_stats.intr_queuemax; sc->ks_data->sts_unsupp_errors.value.ui32 = sc->sc_stats.unsupp_errors; sc->ks_data->sts_nxio_errors.value.ui32 = sc->sc_stats.nxio_errors; sc->ks_data->sts_io_errors.value.ui32 = sc->sc_stats.io_errors; sc->ks_data->sts_rw_cacheflush.value.ui64 = sc->sc_stats.rw_cacheflush; sc->ks_data->sts_intr_total.value.ui64 = sc->sc_stats.intr_total; return (0); } static int vioblk_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) { int ret = DDI_SUCCESS; int instance; struct vioblk_softc *sc; struct virtio_softc *vsc; struct vioblk_stats *ks_data; instance = ddi_get_instance(devinfo); switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: case DDI_PM_RESUME: dev_err(devinfo, CE_WARN, "resume not supported yet"); ret = DDI_FAILURE; goto exit; default: dev_err(devinfo, CE_WARN, "cmd 0x%x not recognized", cmd); ret = DDI_FAILURE; goto exit; } sc = kmem_zalloc(sizeof (struct vioblk_softc), KM_SLEEP); ddi_set_driver_private(devinfo, sc); vsc = &sc->sc_virtio; /* Duplicate for faster access / less typing */ sc->sc_dev = devinfo; vsc->sc_dev = devinfo; cv_init(&sc->cv_devid, NULL, CV_DRIVER, NULL); mutex_init(&sc->lock_devid, NULL, MUTEX_DRIVER, NULL); /* * Initialize interrupt kstat. This should not normally fail, since * we don't use a persistent stat. We do it this way to avoid having * to test for it at run time on the hot path. */ sc->sc_intrstat = kstat_create("vioblk", instance, "intrs", "controller", KSTAT_TYPE_NAMED, sizeof (struct vioblk_stats) / sizeof (kstat_named_t), KSTAT_FLAG_PERSISTENT); if (sc->sc_intrstat == NULL) { dev_err(devinfo, CE_WARN, "kstat_create failed"); goto exit_intrstat; } ks_data = (struct vioblk_stats *)sc->sc_intrstat->ks_data; kstat_named_init(&ks_data->sts_rw_outofmemory, "total_rw_outofmemory", KSTAT_DATA_UINT64); kstat_named_init(&ks_data->sts_rw_badoffset, "total_rw_badoffset", KSTAT_DATA_UINT64); kstat_named_init(&ks_data->sts_intr_total, "total_intr", KSTAT_DATA_UINT64); kstat_named_init(&ks_data->sts_io_errors, "total_io_errors", KSTAT_DATA_UINT32); kstat_named_init(&ks_data->sts_unsupp_errors, "total_unsupp_errors", KSTAT_DATA_UINT32); kstat_named_init(&ks_data->sts_nxio_errors, "total_nxio_errors", KSTAT_DATA_UINT32); kstat_named_init(&ks_data->sts_rw_cacheflush, "total_rw_cacheflush", KSTAT_DATA_UINT64); kstat_named_init(&ks_data->sts_rw_cookiesmax, "max_rw_cookies", KSTAT_DATA_UINT32); kstat_named_init(&ks_data->sts_intr_queuemax, "max_intr_queue", KSTAT_DATA_UINT32); sc->ks_data = ks_data; sc->sc_intrstat->ks_private = sc; sc->sc_intrstat->ks_update = vioblk_ksupdate; kstat_install(sc->sc_intrstat); /* map BAR0 */ ret = ddi_regs_map_setup(devinfo, 1, (caddr_t *)&sc->sc_virtio.sc_io_addr, 0, 0, &vioblk_attr, &sc->sc_virtio.sc_ioh); if (ret != DDI_SUCCESS) { dev_err(devinfo, CE_WARN, "unable to map bar0: [%d]", ret); goto exit_map; } virtio_device_reset(&sc->sc_virtio); virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_ACK); virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); if (vioblk_register_ints(sc)) { dev_err(devinfo, CE_WARN, "Unable to add interrupt"); goto exit_int; } ret = vioblk_dev_features(sc); if (ret) goto exit_features; if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_RO) sc->sc_readonly = B_TRUE; else sc->sc_readonly = B_FALSE; sc->sc_capacity = virtio_read_device_config_8(&sc->sc_virtio, VIRTIO_BLK_CONFIG_CAPACITY); sc->sc_nblks = sc->sc_capacity; sc->sc_blk_size = DEV_BSIZE; if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_BLK_SIZE) { sc->sc_blk_size = virtio_read_device_config_4(&sc->sc_virtio, VIRTIO_BLK_CONFIG_BLK_SIZE); } sc->sc_pblk_size = sc->sc_blk_size; if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_TOPOLOGY) { sc->sc_pblk_size <<= virtio_read_device_config_1(&sc->sc_virtio, VIRTIO_BLK_CONFIG_TOPO_PBEXP); } /* Flushing is not supported. */ if (!(sc->sc_virtio.sc_features & VIRTIO_BLK_F_FLUSH)) { vioblk_ops.o_sync_cache = NULL; } sc->sc_seg_max = DEF_MAXINDIRECT; /* The max number of segments (cookies) in a request */ if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SEG_MAX) { sc->sc_seg_max = virtio_read_device_config_4(&sc->sc_virtio, VIRTIO_BLK_CONFIG_SEG_MAX); /* That's what Linux does. */ if (!sc->sc_seg_max) sc->sc_seg_max = 1; /* * SEG_MAX corresponds to the number of _data_ * blocks in a request */ sc->sc_seg_max += 2; } /* 2 descriptors taken for header/status */ vioblk_bd_dma_attr.dma_attr_sgllen = sc->sc_seg_max - 2; /* The maximum size for a cookie in a request. */ sc->sc_seg_size_max = DEF_MAXSECTOR; if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SIZE_MAX) { sc->sc_seg_size_max = virtio_read_device_config_4( &sc->sc_virtio, VIRTIO_BLK_CONFIG_SIZE_MAX); } /* The maximum request size */ vioblk_bd_dma_attr.dma_attr_maxxfer = vioblk_bd_dma_attr.dma_attr_sgllen * sc->sc_seg_size_max; dev_debug(devinfo, CE_NOTE, "nblks=%" PRIu64 " blksize=%d (%d) num_seg=%d, " "seg_size=%d, maxxfer=%" PRIu64, sc->sc_nblks, sc->sc_blk_size, sc->sc_pblk_size, vioblk_bd_dma_attr.dma_attr_sgllen, sc->sc_seg_size_max, vioblk_bd_dma_attr.dma_attr_maxxfer); sc->sc_vq = virtio_alloc_vq(&sc->sc_virtio, 0, 0, sc->sc_seg_max, "I/O request"); if (sc->sc_vq == NULL) { goto exit_alloc1; } ret = vioblk_alloc_reqs(sc); if (ret) { goto exit_alloc2; } sc->bd_h = bd_alloc_handle(sc, &vioblk_ops, &vioblk_bd_dma_attr, KM_SLEEP); virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); virtio_start_vq_intr(sc->sc_vq); ret = virtio_enable_ints(&sc->sc_virtio); if (ret) goto exit_enable_ints; ret = bd_attach_handle(devinfo, sc->bd_h); if (ret != DDI_SUCCESS) { dev_err(devinfo, CE_WARN, "Failed to attach blkdev"); goto exit_attach_bd; } return (DDI_SUCCESS); exit_attach_bd: /* * There is no virtio_disable_ints(), it's done in virtio_release_ints. * If they ever get split, don't forget to add a call here. */ exit_enable_ints: virtio_stop_vq_intr(sc->sc_vq); bd_free_handle(sc->bd_h); vioblk_free_reqs(sc); exit_alloc2: virtio_free_vq(sc->sc_vq); exit_alloc1: exit_features: virtio_release_ints(&sc->sc_virtio); exit_int: virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); ddi_regs_map_free(&sc->sc_virtio.sc_ioh); exit_map: kstat_delete(sc->sc_intrstat); exit_intrstat: mutex_destroy(&sc->lock_devid); cv_destroy(&sc->cv_devid); kmem_free(sc, sizeof (struct vioblk_softc)); exit: return (ret); } static int vioblk_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) { struct vioblk_softc *sc = ddi_get_driver_private(devinfo); switch (cmd) { case DDI_DETACH: break; case DDI_PM_SUSPEND: cmn_err(CE_WARN, "suspend not supported yet"); return (DDI_FAILURE); default: cmn_err(CE_WARN, "cmd 0x%x unrecognized", cmd); return (DDI_FAILURE); } (void) bd_detach_handle(sc->bd_h); virtio_stop_vq_intr(sc->sc_vq); virtio_release_ints(&sc->sc_virtio); vioblk_free_reqs(sc); virtio_free_vq(sc->sc_vq); virtio_device_reset(&sc->sc_virtio); ddi_regs_map_free(&sc->sc_virtio.sc_ioh); kstat_delete(sc->sc_intrstat); kmem_free(sc, sizeof (struct vioblk_softc)); return (DDI_SUCCESS); } static int vioblk_quiesce(dev_info_t *devinfo) { struct vioblk_softc *sc = ddi_get_driver_private(devinfo); virtio_stop_vq_intr(sc->sc_vq); virtio_device_reset(&sc->sc_virtio); return (DDI_SUCCESS); } int _init(void) { int rv; bd_mod_init(&vioblk_dev_ops); if ((rv = mod_install(&modlinkage)) != 0) { bd_mod_fini(&vioblk_dev_ops); } return (rv); } int _fini(void) { int rv; if ((rv = mod_remove(&modlinkage)) == 0) { bd_mod_fini(&vioblk_dev_ops); } return (rv); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); }