11c5bc425SAlexey Zaytsev /* 21c5bc425SAlexey Zaytsev * CDDL HEADER START 31c5bc425SAlexey Zaytsev * 41c5bc425SAlexey Zaytsev * The contents of this file are subject to the terms of the 51c5bc425SAlexey Zaytsev * Common Development and Distribution License (the "License"). 61c5bc425SAlexey Zaytsev * You may not use this file except in compliance with the License. 71c5bc425SAlexey Zaytsev * 81c5bc425SAlexey Zaytsev * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91c5bc425SAlexey Zaytsev * or http://www.opensolaris.org/os/licensing. 101c5bc425SAlexey Zaytsev * See the License for the specific language governing permissions 111c5bc425SAlexey Zaytsev * and limitations under the License. 121c5bc425SAlexey Zaytsev * 131c5bc425SAlexey Zaytsev * When distributing Covered Code, include this CDDL HEADER in each 141c5bc425SAlexey Zaytsev * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151c5bc425SAlexey Zaytsev * If applicable, add the following below this CDDL HEADER, with the 161c5bc425SAlexey Zaytsev * fields enclosed by brackets "[]" replaced with your own identifying 171c5bc425SAlexey Zaytsev * information: Portions Copyright [yyyy] [name of copyright owner] 181c5bc425SAlexey Zaytsev * 191c5bc425SAlexey Zaytsev * CDDL HEADER END 201c5bc425SAlexey Zaytsev */ 211c5bc425SAlexey Zaytsev 221c5bc425SAlexey Zaytsev /* 23510a6847SHans Rosenfeld * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved. 241c5bc425SAlexey Zaytsev * Copyright (c) 2012, Alexey Zaytsev <alexey.zaytsev@gmail.com> 251a5ae140SJason King * Copyright 2020 Joyent Inc. 264d95620bSPaul Winder * Copyright 2019 Western Digital Corporation. 27*3decf168SPatrick Mooney * Copyright 2020 Oxide Computer Company 281c5bc425SAlexey Zaytsev */ 291c5bc425SAlexey Zaytsev 30f8296c60SJoshua M. Clulow /* 31f8296c60SJoshua M. Clulow * VIRTIO BLOCK DRIVER 32f8296c60SJoshua M. Clulow * 33f8296c60SJoshua M. Clulow * This driver provides support for Virtio Block devices. Each driver instance 34f8296c60SJoshua M. Clulow * attaches to a single underlying block device. 35f8296c60SJoshua M. Clulow * 36f8296c60SJoshua M. Clulow * REQUEST CHAIN LAYOUT 37f8296c60SJoshua M. Clulow * 38f8296c60SJoshua M. Clulow * Every request chain sent to the I/O queue has the following structure. Each 39f8296c60SJoshua M. Clulow * box in the diagram represents a descriptor entry (i.e., a DMA cookie) within 40f8296c60SJoshua M. Clulow * the chain: 41f8296c60SJoshua M. Clulow * 42f8296c60SJoshua M. Clulow * +-0-----------------------------------------+ 43f8296c60SJoshua M. Clulow * | struct virtio_blk_hdr |-----------------------\ 44f8296c60SJoshua M. Clulow * | (written by driver, read by device) | | 45f8296c60SJoshua M. Clulow * +-1-----------------------------------------+ | 46f8296c60SJoshua M. Clulow * | optional data payload |--\ | 47f8296c60SJoshua M. Clulow * | (written by driver for write requests, | | | 48f8296c60SJoshua M. Clulow * | or by device for read requests) | | | 49f8296c60SJoshua M. Clulow * +-2-----------------------------------------+ | | 50f8296c60SJoshua M. Clulow * | ,~` : |-cookies loaned | 51f8296c60SJoshua M. Clulow * |/ : ,~`| | from blkdev | 52f8296c60SJoshua M. Clulow * : / | | | 53f8296c60SJoshua M. Clulow * +-(N - 1)-----------------------------------+ | | 54f8296c60SJoshua M. Clulow * | ... end of data payload. | | | 55f8296c60SJoshua M. Clulow * | | | | 56f8296c60SJoshua M. Clulow * | |--/ | 57f8296c60SJoshua M. Clulow * +-N-----------------------------------------+ | 58f8296c60SJoshua M. Clulow * | status byte | | 59f8296c60SJoshua M. Clulow * | (written by device, read by driver) |--------------------\ | 60f8296c60SJoshua M. Clulow * +-------------------------------------------+ | | 61f8296c60SJoshua M. Clulow * | | 62f8296c60SJoshua M. Clulow * The memory for the header and status bytes (i.e., 0 and N above) | | 63f8296c60SJoshua M. Clulow * is allocated as a single chunk by vioblk_alloc_reqs(): | | 64f8296c60SJoshua M. Clulow * | | 65f8296c60SJoshua M. Clulow * +-------------------------------------------+ | | 66f8296c60SJoshua M. Clulow * | struct virtio_blk_hdr |<----------------------/ 67f8296c60SJoshua M. Clulow * +-------------------------------------------+ | 68f8296c60SJoshua M. Clulow * | status byte |<-------------------/ 69f8296c60SJoshua M. Clulow * +-------------------------------------------+ 70f8296c60SJoshua M. Clulow */ 711c5bc425SAlexey Zaytsev 721c5bc425SAlexey Zaytsev #include <sys/modctl.h> 731c5bc425SAlexey Zaytsev #include <sys/blkdev.h> 741c5bc425SAlexey Zaytsev #include <sys/types.h> 751c5bc425SAlexey Zaytsev #include <sys/errno.h> 761c5bc425SAlexey Zaytsev #include <sys/param.h> 771c5bc425SAlexey Zaytsev #include <sys/stropts.h> 781c5bc425SAlexey Zaytsev #include <sys/stream.h> 791c5bc425SAlexey Zaytsev #include <sys/strsubr.h> 801c5bc425SAlexey Zaytsev #include <sys/kmem.h> 811c5bc425SAlexey Zaytsev #include <sys/conf.h> 821c5bc425SAlexey Zaytsev #include <sys/devops.h> 831c5bc425SAlexey Zaytsev #include <sys/ksynch.h> 841c5bc425SAlexey Zaytsev #include <sys/stat.h> 851c5bc425SAlexey Zaytsev #include <sys/modctl.h> 861c5bc425SAlexey Zaytsev #include <sys/debug.h> 871c5bc425SAlexey Zaytsev #include <sys/pci.h> 8894c3dad2SToomas Soome #include <sys/containerof.h> 89f8296c60SJoshua M. Clulow #include <sys/ctype.h> 90f8296c60SJoshua M. Clulow #include <sys/sysmacros.h> 911a5ae140SJason King #include <sys/dkioc_free_util.h> 921c5bc425SAlexey Zaytsev 93f8296c60SJoshua M. Clulow #include "virtio.h" 94f8296c60SJoshua M. Clulow #include "vioblk.h" 951c5bc425SAlexey Zaytsev 96f8296c60SJoshua M. Clulow static void vioblk_get_id(vioblk_t *); 97f8296c60SJoshua M. Clulow uint_t vioblk_int_handler(caddr_t, caddr_t); 98f8296c60SJoshua M. Clulow static uint_t vioblk_poll(vioblk_t *); 991c5bc425SAlexey Zaytsev static int vioblk_quiesce(dev_info_t *); 1001c5bc425SAlexey Zaytsev static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t); 1011c5bc425SAlexey Zaytsev static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t); 1021c5bc425SAlexey Zaytsev 103f8296c60SJoshua M. Clulow 1041c5bc425SAlexey Zaytsev static struct dev_ops vioblk_dev_ops = { 105f8296c60SJoshua M. Clulow .devo_rev = DEVO_REV, 106f8296c60SJoshua M. Clulow .devo_refcnt = 0, 107f8296c60SJoshua M. Clulow 108f8296c60SJoshua M. Clulow .devo_attach = vioblk_attach, 109f8296c60SJoshua M. Clulow .devo_detach = vioblk_detach, 110f8296c60SJoshua M. Clulow .devo_quiesce = vioblk_quiesce, 111f8296c60SJoshua M. Clulow 112f8296c60SJoshua M. Clulow .devo_getinfo = ddi_no_info, 113f8296c60SJoshua M. Clulow .devo_identify = nulldev, 114f8296c60SJoshua M. Clulow .devo_probe = nulldev, 115f8296c60SJoshua M. Clulow .devo_reset = nodev, 116f8296c60SJoshua M. Clulow .devo_cb_ops = NULL, 117f8296c60SJoshua M. Clulow .devo_bus_ops = NULL, 118f8296c60SJoshua M. Clulow .devo_power = NULL, 119f8296c60SJoshua M. Clulow }; 120f8296c60SJoshua M. Clulow 121f8296c60SJoshua M. Clulow static struct modldrv vioblk_modldrv = { 122f8296c60SJoshua M. Clulow .drv_modops = &mod_driverops, 123f8296c60SJoshua M. Clulow .drv_linkinfo = "VIRTIO block driver", 124f8296c60SJoshua M. Clulow .drv_dev_ops = &vioblk_dev_ops 125f8296c60SJoshua M. Clulow }; 126f8296c60SJoshua M. Clulow 127f8296c60SJoshua M. Clulow static struct modlinkage vioblk_modlinkage = { 128f8296c60SJoshua M. Clulow .ml_rev = MODREV_1, 129f8296c60SJoshua M. Clulow .ml_linkage = { &vioblk_modldrv, NULL } 130f8296c60SJoshua M. Clulow }; 131f8296c60SJoshua M. Clulow 132f8296c60SJoshua M. Clulow /* 133f8296c60SJoshua M. Clulow * DMA attribute template for header and status blocks. We also make a 134f8296c60SJoshua M. Clulow * per-instance copy of this template with negotiated sizes from the device for 135f8296c60SJoshua M. Clulow * blkdev. 136f8296c60SJoshua M. Clulow */ 137f8296c60SJoshua M. Clulow static const ddi_dma_attr_t vioblk_dma_attr = { 138f8296c60SJoshua M. Clulow .dma_attr_version = DMA_ATTR_V0, 139f8296c60SJoshua M. Clulow .dma_attr_addr_lo = 0x0000000000000000, 140f8296c60SJoshua M. Clulow .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFF, 141f8296c60SJoshua M. Clulow .dma_attr_count_max = 0x00000000FFFFFFFF, 142f8296c60SJoshua M. Clulow .dma_attr_align = 1, 143f8296c60SJoshua M. Clulow .dma_attr_burstsizes = 1, 144f8296c60SJoshua M. Clulow .dma_attr_minxfer = 1, 145f8296c60SJoshua M. Clulow .dma_attr_maxxfer = 0x00000000FFFFFFFF, 146f8296c60SJoshua M. Clulow .dma_attr_seg = 0x00000000FFFFFFFF, 147f8296c60SJoshua M. Clulow .dma_attr_sgllen = 1, 148f8296c60SJoshua M. Clulow .dma_attr_granular = 1, 149f8296c60SJoshua M. Clulow .dma_attr_flags = 0 1501c5bc425SAlexey Zaytsev }; 1511c5bc425SAlexey Zaytsev 152f8296c60SJoshua M. Clulow static vioblk_req_t * 153f8296c60SJoshua M. Clulow vioblk_req_alloc(vioblk_t *vib) 1541c5bc425SAlexey Zaytsev { 155f8296c60SJoshua M. Clulow vioblk_req_t *vbr; 1561c5bc425SAlexey Zaytsev 157f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 1581c5bc425SAlexey Zaytsev 159f8296c60SJoshua M. Clulow if ((vbr = list_remove_head(&vib->vib_reqs)) == NULL) { 160f8296c60SJoshua M. Clulow return (NULL); 161f8296c60SJoshua M. Clulow } 162f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc++; 1631c5bc425SAlexey Zaytsev 164f8296c60SJoshua M. Clulow VERIFY0(vbr->vbr_status); 165f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_ALLOCATED; 166f8296c60SJoshua M. Clulow 167*3decf168SPatrick Mooney VERIFY3P(vbr->vbr_chain, !=, NULL); 168f8296c60SJoshua M. Clulow VERIFY3P(vbr->vbr_xfer, ==, NULL); 169f8296c60SJoshua M. Clulow VERIFY3S(vbr->vbr_error, ==, 0); 170f8296c60SJoshua M. Clulow 171f8296c60SJoshua M. Clulow return (vbr); 172f8296c60SJoshua M. Clulow } 173f8296c60SJoshua M. Clulow 174f8296c60SJoshua M. Clulow static void 175f8296c60SJoshua M. Clulow vioblk_req_free(vioblk_t *vib, vioblk_req_t *vbr) 176f8296c60SJoshua M. Clulow { 177f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 178f8296c60SJoshua M. Clulow 179f8296c60SJoshua M. Clulow /* 180f8296c60SJoshua M. Clulow * Check that this request was allocated, then zero the status field to 181f8296c60SJoshua M. Clulow * clear all status bits. 182f8296c60SJoshua M. Clulow */ 183f8296c60SJoshua M. Clulow VERIFY(vbr->vbr_status & VIOBLK_REQSTAT_ALLOCATED); 184f8296c60SJoshua M. Clulow vbr->vbr_status = 0; 185f8296c60SJoshua M. Clulow 186f8296c60SJoshua M. Clulow vbr->vbr_xfer = NULL; 187f8296c60SJoshua M. Clulow vbr->vbr_error = 0; 188f8296c60SJoshua M. Clulow vbr->vbr_type = 0; 189*3decf168SPatrick Mooney virtio_chain_clear(vbr->vbr_chain); 190f8296c60SJoshua M. Clulow 191f8296c60SJoshua M. Clulow list_insert_head(&vib->vib_reqs, vbr); 192f8296c60SJoshua M. Clulow 193f8296c60SJoshua M. Clulow VERIFY3U(vib->vib_nreqs_alloc, >, 0); 194f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc--; 195f8296c60SJoshua M. Clulow } 196f8296c60SJoshua M. Clulow 197f8296c60SJoshua M. Clulow static void 198f8296c60SJoshua M. Clulow vioblk_complete(vioblk_t *vib, vioblk_req_t *vbr) 199f8296c60SJoshua M. Clulow { 200f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 201f8296c60SJoshua M. Clulow 202f8296c60SJoshua M. Clulow VERIFY(!(vbr->vbr_status & VIOBLK_REQSTAT_COMPLETE)); 203f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_COMPLETE; 204f8296c60SJoshua M. Clulow 205f8296c60SJoshua M. Clulow if (vbr->vbr_type == VIRTIO_BLK_T_FLUSH) { 206f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_cacheflush.value.ui64++; 207f8296c60SJoshua M. Clulow } 208f8296c60SJoshua M. Clulow 209f8296c60SJoshua M. Clulow if (vbr->vbr_xfer != NULL) { 210f8296c60SJoshua M. Clulow /* 211f8296c60SJoshua M. Clulow * This is a blkdev framework request. 212f8296c60SJoshua M. Clulow */ 213f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 214f8296c60SJoshua M. Clulow bd_xfer_done(vbr->vbr_xfer, vbr->vbr_error); 215f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 216f8296c60SJoshua M. Clulow vbr->vbr_xfer = NULL; 217f8296c60SJoshua M. Clulow } 218f8296c60SJoshua M. Clulow } 219f8296c60SJoshua M. Clulow 220*3decf168SPatrick Mooney static vioblk_req_t * 221f8296c60SJoshua M. Clulow vioblk_common_start(vioblk_t *vib, int type, uint64_t sector, 222f8296c60SJoshua M. Clulow boolean_t polled) 223f8296c60SJoshua M. Clulow { 224f8296c60SJoshua M. Clulow vioblk_req_t *vbr = NULL; 225f8296c60SJoshua M. Clulow 226f8296c60SJoshua M. Clulow if ((vbr = vioblk_req_alloc(vib)) == NULL) { 227f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_outofmemory.value.ui64++; 228f8296c60SJoshua M. Clulow return (NULL); 229f8296c60SJoshua M. Clulow } 230f8296c60SJoshua M. Clulow vbr->vbr_type = type; 231f8296c60SJoshua M. Clulow 232f8296c60SJoshua M. Clulow if (polled) { 233f8296c60SJoshua M. Clulow /* 234f8296c60SJoshua M. Clulow * Mark this command as polled so that we can wait on it 235f8296c60SJoshua M. Clulow * ourselves. 236f8296c60SJoshua M. Clulow */ 237f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_POLLED; 238f8296c60SJoshua M. Clulow } 239f8296c60SJoshua M. Clulow 240f8296c60SJoshua M. Clulow struct vioblk_req_hdr vbh; 241f8296c60SJoshua M. Clulow vbh.vbh_type = type; 242f8296c60SJoshua M. Clulow vbh.vbh_ioprio = 0; 243c5c712a8SToomas Soome vbh.vbh_sector = (sector * vib->vib_blk_size) / DEV_BSIZE; 244f8296c60SJoshua M. Clulow bcopy(&vbh, virtio_dma_va(vbr->vbr_dma, 0), sizeof (vbh)); 245f8296c60SJoshua M. Clulow 246f8296c60SJoshua M. Clulow /* 247f8296c60SJoshua M. Clulow * Put the header in the first descriptor. See the block comment at 248f8296c60SJoshua M. Clulow * the top of the file for more details on the chain layout. 249f8296c60SJoshua M. Clulow */ 250*3decf168SPatrick Mooney if (virtio_chain_append(vbr->vbr_chain, 251*3decf168SPatrick Mooney virtio_dma_cookie_pa(vbr->vbr_dma, 0), 252f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr), VIRTIO_DIR_DEVICE_READS) != 253f8296c60SJoshua M. Clulow DDI_SUCCESS) { 254f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr); 255f8296c60SJoshua M. Clulow return (NULL); 256f8296c60SJoshua M. Clulow } 2571c5bc425SAlexey Zaytsev 258*3decf168SPatrick Mooney return (vbr); 259*3decf168SPatrick Mooney } 260*3decf168SPatrick Mooney 2611c5bc425SAlexey Zaytsev static int 262*3decf168SPatrick Mooney vioblk_common_submit(vioblk_t *vib, vioblk_req_t *vbr) 2631c5bc425SAlexey Zaytsev { 264*3decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain; 265f8296c60SJoshua M. Clulow int r; 2661c5bc425SAlexey Zaytsev 267f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 2681c5bc425SAlexey Zaytsev 269f8296c60SJoshua M. Clulow /* 270f8296c60SJoshua M. Clulow * The device will write the status byte into this last descriptor. 271f8296c60SJoshua M. Clulow * See the block comment at the top of the file for more details on the 272f8296c60SJoshua M. Clulow * chain layout. 273f8296c60SJoshua M. Clulow */ 274f8296c60SJoshua M. Clulow if (virtio_chain_append(vic, virtio_dma_cookie_pa(vbr->vbr_dma, 0) + 275f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr), sizeof (uint8_t), 276f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS) { 277*3decf168SPatrick Mooney vioblk_req_free(vib, vbr); 278*3decf168SPatrick Mooney return (ENOMEM); 2791c5bc425SAlexey Zaytsev } 2801c5bc425SAlexey Zaytsev 281f8296c60SJoshua M. Clulow virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORDEV); 282f8296c60SJoshua M. Clulow virtio_chain_submit(vic, B_TRUE); 2831c5bc425SAlexey Zaytsev 284f8296c60SJoshua M. Clulow if (!(vbr->vbr_status & VIOBLK_REQSTAT_POLLED)) { 285f8296c60SJoshua M. Clulow /* 286f8296c60SJoshua M. Clulow * This is not a polled request. Our request will be freed and 287f8296c60SJoshua M. Clulow * the caller notified later in vioblk_poll(). 288f8296c60SJoshua M. Clulow */ 289f8296c60SJoshua M. Clulow return (0); 2901c5bc425SAlexey Zaytsev } 2911c5bc425SAlexey Zaytsev 2921c5bc425SAlexey Zaytsev /* 293f8296c60SJoshua M. Clulow * This is a polled request. We need to block here and wait for the 294f8296c60SJoshua M. Clulow * device to complete request processing. 2951c5bc425SAlexey Zaytsev */ 296f8296c60SJoshua M. Clulow while (!(vbr->vbr_status & VIOBLK_REQSTAT_POLL_COMPLETE)) { 297f8296c60SJoshua M. Clulow if (ddi_in_panic()) { 298f8296c60SJoshua M. Clulow /* 299f8296c60SJoshua M. Clulow * When panicking, interrupts are disabled. We must 300f8296c60SJoshua M. Clulow * poll the queue manually. 301f8296c60SJoshua M. Clulow */ 3021c5bc425SAlexey Zaytsev drv_usecwait(10); 303f8296c60SJoshua M. Clulow (void) vioblk_poll(vib); 304f8296c60SJoshua M. Clulow continue; 3051c5bc425SAlexey Zaytsev } 3061c5bc425SAlexey Zaytsev 307f8296c60SJoshua M. Clulow /* 308f8296c60SJoshua M. Clulow * When not panicking, the device will interrupt on command 309f8296c60SJoshua M. Clulow * completion and vioblk_poll() will be called to wake us up. 310f8296c60SJoshua M. Clulow */ 311f8296c60SJoshua M. Clulow cv_wait(&vib->vib_cv, &vib->vib_mutex); 3121c5bc425SAlexey Zaytsev } 3131c5bc425SAlexey Zaytsev 314f8296c60SJoshua M. Clulow vioblk_complete(vib, vbr); 315f8296c60SJoshua M. Clulow r = vbr->vbr_error; 316f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr); 317f8296c60SJoshua M. Clulow return (r); 3181c5bc425SAlexey Zaytsev } 3191c5bc425SAlexey Zaytsev 3201c5bc425SAlexey Zaytsev static int 321f8296c60SJoshua M. Clulow vioblk_internal(vioblk_t *vib, int type, virtio_dma_t *dma, 322f8296c60SJoshua M. Clulow uint64_t sector, virtio_direction_t dir) 3231c5bc425SAlexey Zaytsev { 324f8296c60SJoshua M. Clulow vioblk_req_t *vbr; 3251c5bc425SAlexey Zaytsev 326f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 327f8296c60SJoshua M. Clulow 328f8296c60SJoshua M. Clulow /* 329f8296c60SJoshua M. Clulow * Allocate a polled request. 330f8296c60SJoshua M. Clulow */ 331*3decf168SPatrick Mooney if ((vbr = vioblk_common_start(vib, type, sector, B_TRUE)) == NULL) { 332f8296c60SJoshua M. Clulow return (ENOMEM); 333f8296c60SJoshua M. Clulow } 334f8296c60SJoshua M. Clulow 335f8296c60SJoshua M. Clulow /* 336f8296c60SJoshua M. Clulow * If there is a request payload, it goes between the header and the 337f8296c60SJoshua M. Clulow * status byte. See the block comment at the top of the file for more 338f8296c60SJoshua M. Clulow * detail on the chain layout. 339f8296c60SJoshua M. Clulow */ 340f8296c60SJoshua M. Clulow if (dma != NULL) { 341*3decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain; 342f8296c60SJoshua M. Clulow for (uint_t n = 0; n < virtio_dma_ncookies(dma); n++) { 343f8296c60SJoshua M. Clulow if (virtio_chain_append(vic, 344f8296c60SJoshua M. Clulow virtio_dma_cookie_pa(dma, n), 345f8296c60SJoshua M. Clulow virtio_dma_cookie_size(dma, n), dir) != 346f8296c60SJoshua M. Clulow DDI_SUCCESS) { 347f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr); 348*3decf168SPatrick Mooney return (ENOMEM); 349*3decf168SPatrick Mooney } 350*3decf168SPatrick Mooney } 351*3decf168SPatrick Mooney } 352*3decf168SPatrick Mooney 353*3decf168SPatrick Mooney return (vioblk_common_submit(vib, vbr)); 354f8296c60SJoshua M. Clulow } 355f8296c60SJoshua M. Clulow 356f8296c60SJoshua M. Clulow static int 3571a5ae140SJason King vioblk_map_discard(vioblk_t *vib, virtio_chain_t *vic, const bd_xfer_t *xfer) 3581a5ae140SJason King { 3591a5ae140SJason King const dkioc_free_list_t *dfl = xfer->x_dfl; 3601a5ae140SJason King const dkioc_free_list_ext_t *exts = dfl->dfl_exts; 3611a5ae140SJason King virtio_dma_t *dma = NULL; 3621a5ae140SJason King struct vioblk_discard_write_zeroes *wzp = NULL; 3631a5ae140SJason King 3641a5ae140SJason King dma = virtio_dma_alloc(vib->vib_virtio, 3651a5ae140SJason King dfl->dfl_num_exts * sizeof (*wzp), &vioblk_dma_attr, 3661a5ae140SJason King DDI_DMA_CONSISTENT | DDI_DMA_WRITE, KM_SLEEP); 3671a5ae140SJason King if (dma == NULL) 3681a5ae140SJason King return (ENOMEM); 3691a5ae140SJason King 3701a5ae140SJason King wzp = virtio_dma_va(dma, 0); 3711a5ae140SJason King 3721a5ae140SJason King for (uint64_t i = 0; i < dfl->dfl_num_exts; i++, exts++, wzp++) { 3731a5ae140SJason King uint64_t start = dfl->dfl_offset + exts->dfle_start; 3741a5ae140SJason King 3751a5ae140SJason King const struct vioblk_discard_write_zeroes vdwz = { 3761a5ae140SJason King .vdwz_sector = start >> DEV_BSHIFT, 3771a5ae140SJason King .vdwz_num_sectors = exts->dfle_length >> DEV_BSHIFT, 3781a5ae140SJason King .vdwz_flags = 0 3791a5ae140SJason King }; 3801a5ae140SJason King 3811a5ae140SJason King bcopy(&vdwz, wzp, sizeof (*wzp)); 3821a5ae140SJason King } 3831a5ae140SJason King 3841a5ae140SJason King if (virtio_chain_append(vic, 3851a5ae140SJason King virtio_dma_cookie_pa(dma, 0), 3861a5ae140SJason King virtio_dma_cookie_size(dma, 0), 3871a5ae140SJason King VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) { 3881a5ae140SJason King virtio_dma_free(dma); 3891a5ae140SJason King return (ENOMEM); 3901a5ae140SJason King } 3911a5ae140SJason King 3921a5ae140SJason King return (0); 3931a5ae140SJason King } 3941a5ae140SJason King 3951a5ae140SJason King static int 396f8296c60SJoshua M. Clulow vioblk_request(vioblk_t *vib, bd_xfer_t *xfer, int type) 397f8296c60SJoshua M. Clulow { 398f8296c60SJoshua M. Clulow vioblk_req_t *vbr = NULL; 399f8296c60SJoshua M. Clulow uint_t total_cookies = 2; 400f8296c60SJoshua M. Clulow boolean_t polled = (xfer->x_flags & BD_XFER_POLL) != 0; 401f8296c60SJoshua M. Clulow 402f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 403f8296c60SJoshua M. Clulow 404f8296c60SJoshua M. Clulow /* 405f8296c60SJoshua M. Clulow * Ensure that this request falls within the advertised size of the 406f8296c60SJoshua M. Clulow * block device. Be careful to avoid overflow. 407f8296c60SJoshua M. Clulow */ 408f8296c60SJoshua M. Clulow if (xfer->x_nblks > SIZE_MAX - xfer->x_blkno || 409f8296c60SJoshua M. Clulow (xfer->x_blkno + xfer->x_nblks) > vib->vib_nblks) { 410f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_badoffset.value.ui64++; 411f8296c60SJoshua M. Clulow return (EINVAL); 412f8296c60SJoshua M. Clulow } 413f8296c60SJoshua M. Clulow 414*3decf168SPatrick Mooney if ((vbr = vioblk_common_start(vib, type, xfer->x_blkno, polled)) == 415f8296c60SJoshua M. Clulow NULL) { 416f8296c60SJoshua M. Clulow return (ENOMEM); 417f8296c60SJoshua M. Clulow } 418f8296c60SJoshua M. Clulow vbr->vbr_xfer = xfer; 419f8296c60SJoshua M. Clulow 420f8296c60SJoshua M. Clulow /* 421f8296c60SJoshua M. Clulow * If there is a request payload, it goes between the header and the 422f8296c60SJoshua M. Clulow * status byte. See the block comment at the top of the file for more 423f8296c60SJoshua M. Clulow * detail on the chain layout. 424f8296c60SJoshua M. Clulow */ 425f8296c60SJoshua M. Clulow if ((type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_OUT) && 426f8296c60SJoshua M. Clulow xfer->x_nblks > 0) { 427f8296c60SJoshua M. Clulow virtio_direction_t dir = (type == VIRTIO_BLK_T_OUT) ? 428f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_READS : VIRTIO_DIR_DEVICE_WRITES; 429*3decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain; 430f8296c60SJoshua M. Clulow 431f8296c60SJoshua M. Clulow for (uint_t n = 0; n < xfer->x_ndmac; n++) { 432f8296c60SJoshua M. Clulow ddi_dma_cookie_t dmac; 433f8296c60SJoshua M. Clulow 434f8296c60SJoshua M. Clulow if (n == 0) { 435f8296c60SJoshua M. Clulow /* 436f8296c60SJoshua M. Clulow * The first cookie is in the blkdev request. 437f8296c60SJoshua M. Clulow */ 438f8296c60SJoshua M. Clulow dmac = xfer->x_dmac; 4391c5bc425SAlexey Zaytsev } else { 440f8296c60SJoshua M. Clulow ddi_dma_nextcookie(xfer->x_dmah, &dmac); 4411c5bc425SAlexey Zaytsev } 4421c5bc425SAlexey Zaytsev 443f8296c60SJoshua M. Clulow if (virtio_chain_append(vic, dmac.dmac_laddress, 444f8296c60SJoshua M. Clulow dmac.dmac_size, dir) != DDI_SUCCESS) { 445*3decf168SPatrick Mooney vioblk_req_free(vib, vbr); 446*3decf168SPatrick Mooney return (ENOMEM); 447f8296c60SJoshua M. Clulow } 4481c5bc425SAlexey Zaytsev } 4491c5bc425SAlexey Zaytsev 450f8296c60SJoshua M. Clulow total_cookies += xfer->x_ndmac; 451f8296c60SJoshua M. Clulow 452f8296c60SJoshua M. Clulow } else if (xfer->x_nblks > 0) { 453f8296c60SJoshua M. Clulow dev_err(vib->vib_dip, CE_PANIC, 454f8296c60SJoshua M. Clulow "request of type %d had payload length of %lu blocks", type, 455f8296c60SJoshua M. Clulow xfer->x_nblks); 4561a5ae140SJason King } else if (type == VIRTIO_BLK_T_DISCARD) { 457*3decf168SPatrick Mooney int r = vioblk_map_discard(vib, vbr->vbr_chain, xfer); 4581a5ae140SJason King if (r != 0) { 459*3decf168SPatrick Mooney vioblk_req_free(vib, vbr); 460*3decf168SPatrick Mooney return (r); 4611a5ae140SJason King } 462f8296c60SJoshua M. Clulow } 463f8296c60SJoshua M. Clulow 464f8296c60SJoshua M. Clulow if (vib->vib_stats->vbs_rw_cookiesmax.value.ui32 < total_cookies) { 465f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_cookiesmax.value.ui32 = total_cookies; 466f8296c60SJoshua M. Clulow } 467f8296c60SJoshua M. Clulow 468*3decf168SPatrick Mooney return (vioblk_common_submit(vib, vbr)); 4691c5bc425SAlexey Zaytsev } 4701c5bc425SAlexey Zaytsev 4711c5bc425SAlexey Zaytsev static int 472f8296c60SJoshua M. Clulow vioblk_bd_read(void *arg, bd_xfer_t *xfer) 4731c5bc425SAlexey Zaytsev { 474f8296c60SJoshua M. Clulow vioblk_t *vib = arg; 475f8296c60SJoshua M. Clulow int r; 4761c5bc425SAlexey Zaytsev 477f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 478f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_IN); 479f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 4801c5bc425SAlexey Zaytsev 481f8296c60SJoshua M. Clulow return (r); 4821c5bc425SAlexey Zaytsev } 4831c5bc425SAlexey Zaytsev 4841c5bc425SAlexey Zaytsev static int 485f8296c60SJoshua M. Clulow vioblk_bd_write(void *arg, bd_xfer_t *xfer) 4861c5bc425SAlexey Zaytsev { 487f8296c60SJoshua M. Clulow vioblk_t *vib = arg; 488f8296c60SJoshua M. Clulow int r; 4891c5bc425SAlexey Zaytsev 490f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 491f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_OUT); 492f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 4931c5bc425SAlexey Zaytsev 494f8296c60SJoshua M. Clulow return (r); 4951c5bc425SAlexey Zaytsev } 4961c5bc425SAlexey Zaytsev 497f8296c60SJoshua M. Clulow static int 498f8296c60SJoshua M. Clulow vioblk_bd_flush(void *arg, bd_xfer_t *xfer) 499f8296c60SJoshua M. Clulow { 500f8296c60SJoshua M. Clulow vioblk_t *vib = arg; 501f8296c60SJoshua M. Clulow int r; 502f8296c60SJoshua M. Clulow 503f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 504f8296c60SJoshua M. Clulow if (!virtio_feature_present(vib->vib_virtio, VIRTIO_BLK_F_FLUSH)) { 505f8296c60SJoshua M. Clulow /* 506f8296c60SJoshua M. Clulow * We don't really expect to get here, because if we did not 507f8296c60SJoshua M. Clulow * negotiate the flush feature we would not have installed this 508f8296c60SJoshua M. Clulow * function in the blkdev ops vector. 509f8296c60SJoshua M. Clulow */ 510f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 511f8296c60SJoshua M. Clulow return (ENOTSUP); 512f8296c60SJoshua M. Clulow } 513f8296c60SJoshua M. Clulow 514f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_FLUSH); 515f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 516f8296c60SJoshua M. Clulow 517f8296c60SJoshua M. Clulow return (r); 518f8296c60SJoshua M. Clulow } 5191c5bc425SAlexey Zaytsev 5201c5bc425SAlexey Zaytsev static void 521f8296c60SJoshua M. Clulow vioblk_bd_driveinfo(void *arg, bd_drive_t *drive) 5221c5bc425SAlexey Zaytsev { 523f8296c60SJoshua M. Clulow vioblk_t *vib = arg; 5241c5bc425SAlexey Zaytsev 525f8296c60SJoshua M. Clulow drive->d_qsize = vib->vib_reqs_capacity; 5261c5bc425SAlexey Zaytsev drive->d_removable = B_FALSE; 5271c5bc425SAlexey Zaytsev drive->d_hotpluggable = B_TRUE; 5281c5bc425SAlexey Zaytsev drive->d_target = 0; 5291c5bc425SAlexey Zaytsev drive->d_lun = 0; 530510a6847SHans Rosenfeld 531510a6847SHans Rosenfeld drive->d_vendor = "Virtio"; 532510a6847SHans Rosenfeld drive->d_vendor_len = strlen(drive->d_vendor); 533510a6847SHans Rosenfeld 534510a6847SHans Rosenfeld drive->d_product = "Block Device"; 535510a6847SHans Rosenfeld drive->d_product_len = strlen(drive->d_product); 536510a6847SHans Rosenfeld 537f8296c60SJoshua M. Clulow drive->d_serial = vib->vib_devid; 538510a6847SHans Rosenfeld drive->d_serial_len = strlen(drive->d_serial); 539510a6847SHans Rosenfeld 540510a6847SHans Rosenfeld drive->d_revision = "0000"; 541510a6847SHans Rosenfeld drive->d_revision_len = strlen(drive->d_revision); 5421a5ae140SJason King 5431a5ae140SJason King if (vib->vib_can_discard) { 5441a5ae140SJason King drive->d_free_align = vib->vib_discard_sector_align; 5451a5ae140SJason King drive->d_max_free_seg = vib->vib_max_discard_seg; 5461a5ae140SJason King drive->d_max_free_blks = vib->vib_max_discard_sectors; 5471a5ae140SJason King /* 5481a5ae140SJason King * The virtio 1.1 spec doesn't specify a per segment sector 5491a5ae140SJason King * limit for discards -- only a limit on the total sectors in 5501a5ae140SJason King * a discard request. Therefore, we assume a vioblk device must 5511a5ae140SJason King * be able to accept a single segment of vib_max_discard_sectors 5521a5ae140SJason King * (when it supports discard requests) and use 5531a5ae140SJason King * vib_max_discard_sectors both for the overall limit for 5541a5ae140SJason King * a discard request, but also as the limit for a single 5551a5ae140SJason King * segment. blkdev will ensure we are never called with 5561a5ae140SJason King * a dkioc_free_list_t that violates either limit. 5571a5ae140SJason King */ 5581a5ae140SJason King drive->d_max_free_seg_blks = vib->vib_max_discard_sectors; 5591a5ae140SJason King } 5601c5bc425SAlexey Zaytsev } 5611c5bc425SAlexey Zaytsev 5621c5bc425SAlexey Zaytsev static int 563f8296c60SJoshua M. Clulow vioblk_bd_mediainfo(void *arg, bd_media_t *media) 5641c5bc425SAlexey Zaytsev { 565f8296c60SJoshua M. Clulow vioblk_t *vib = (void *)arg; 5661c5bc425SAlexey Zaytsev 567f8296c60SJoshua M. Clulow /* 568f8296c60SJoshua M. Clulow * The device protocol is specified in terms of 512 byte logical 569f8296c60SJoshua M. Clulow * blocks, regardless of the recommended I/O size which might be 570f8296c60SJoshua M. Clulow * larger. 571f8296c60SJoshua M. Clulow */ 572f8296c60SJoshua M. Clulow media->m_nblks = vib->vib_nblks; 573c5c712a8SToomas Soome media->m_blksize = vib->vib_blk_size; 5741c5bc425SAlexey Zaytsev 575f8296c60SJoshua M. Clulow media->m_readonly = vib->vib_readonly; 576f8296c60SJoshua M. Clulow media->m_pblksize = vib->vib_pblk_size; 5771c5bc425SAlexey Zaytsev return (0); 5781c5bc425SAlexey Zaytsev } 5791c5bc425SAlexey Zaytsev 5801c5bc425SAlexey Zaytsev static void 581f8296c60SJoshua M. Clulow vioblk_get_id(vioblk_t *vib) 5821c5bc425SAlexey Zaytsev { 583f8296c60SJoshua M. Clulow virtio_dma_t *dma; 584f8296c60SJoshua M. Clulow int r; 5851c5bc425SAlexey Zaytsev 586f8296c60SJoshua M. Clulow if ((dma = virtio_dma_alloc(vib->vib_virtio, VIRTIO_BLK_ID_BYTES, 587f8296c60SJoshua M. Clulow &vioblk_dma_attr, DDI_DMA_CONSISTENT | DDI_DMA_READ, 588f8296c60SJoshua M. Clulow KM_SLEEP)) == NULL) { 589f8296c60SJoshua M. Clulow return; 590f8296c60SJoshua M. Clulow } 5911c5bc425SAlexey Zaytsev 592f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 593f8296c60SJoshua M. Clulow if ((r = vioblk_internal(vib, VIRTIO_BLK_T_GET_ID, dma, 0, 594f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_WRITES)) == 0) { 595f8296c60SJoshua M. Clulow const char *b = virtio_dma_va(dma, 0); 596f8296c60SJoshua M. Clulow uint_t pos = 0; 5971c5bc425SAlexey Zaytsev 598f8296c60SJoshua M. Clulow /* 599f8296c60SJoshua M. Clulow * Save the entire response for debugging purposes. 600f8296c60SJoshua M. Clulow */ 601f8296c60SJoshua M. Clulow bcopy(virtio_dma_va(dma, 0), vib->vib_rawid, 602f8296c60SJoshua M. Clulow VIRTIO_BLK_ID_BYTES); 6031c5bc425SAlexey Zaytsev 604f8296c60SJoshua M. Clulow /* 605f8296c60SJoshua M. Clulow * Process the returned ID. 606f8296c60SJoshua M. Clulow */ 607f8296c60SJoshua M. Clulow bzero(vib->vib_devid, sizeof (vib->vib_devid)); 608f8296c60SJoshua M. Clulow for (uint_t n = 0; n < VIRTIO_BLK_ID_BYTES; n++) { 609f8296c60SJoshua M. Clulow if (isalnum(b[n]) || b[n] == '-' || b[n] == '_') { 610f8296c60SJoshua M. Clulow /* 611f8296c60SJoshua M. Clulow * Accept a subset of printable ASCII 612f8296c60SJoshua M. Clulow * characters. 613f8296c60SJoshua M. Clulow */ 614f8296c60SJoshua M. Clulow vib->vib_devid[pos++] = b[n]; 615f8296c60SJoshua M. Clulow } else { 616f8296c60SJoshua M. Clulow /* 617f8296c60SJoshua M. Clulow * Stop processing at the first sign of 618f8296c60SJoshua M. Clulow * trouble. 619f8296c60SJoshua M. Clulow */ 620f8296c60SJoshua M. Clulow break; 621f8296c60SJoshua M. Clulow } 622f8296c60SJoshua M. Clulow } 6231c5bc425SAlexey Zaytsev 624f8296c60SJoshua M. Clulow vib->vib_devid_fetched = B_TRUE; 625f8296c60SJoshua M. Clulow } 626f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 6271c5bc425SAlexey Zaytsev 628f8296c60SJoshua M. Clulow virtio_dma_free(dma); 6291c5bc425SAlexey Zaytsev } 6301c5bc425SAlexey Zaytsev 6311c5bc425SAlexey Zaytsev static int 632f8296c60SJoshua M. Clulow vioblk_bd_devid(void *arg, dev_info_t *dip, ddi_devid_t *devid) 6331c5bc425SAlexey Zaytsev { 634f8296c60SJoshua M. Clulow vioblk_t *vib = arg; 635f8296c60SJoshua M. Clulow size_t len; 6361c5bc425SAlexey Zaytsev 637f8296c60SJoshua M. Clulow if ((len = strlen(vib->vib_devid)) == 0) { 638f8296c60SJoshua M. Clulow /* 639f8296c60SJoshua M. Clulow * The device has no ID. 640f8296c60SJoshua M. Clulow */ 6411c5bc425SAlexey Zaytsev return (DDI_FAILURE); 6421c5bc425SAlexey Zaytsev } 6431c5bc425SAlexey Zaytsev 644f8296c60SJoshua M. Clulow return (ddi_devid_init(dip, DEVID_ATA_SERIAL, len, vib->vib_devid, 645f8296c60SJoshua M. Clulow devid)); 6461c5bc425SAlexey Zaytsev } 6471c5bc425SAlexey Zaytsev 6481a5ae140SJason King static int 6491a5ae140SJason King vioblk_bd_free_space(void *arg, bd_xfer_t *xfer) 6501a5ae140SJason King { 6511a5ae140SJason King vioblk_t *vib = arg; 6521a5ae140SJason King int r = 0; 6531a5ae140SJason King 6541a5ae140SJason King /* 6551a5ae140SJason King * Since vib_can_discard is write once (and set during attach), 6561a5ae140SJason King * we can check if it's enabled without taking the mutex. 6571a5ae140SJason King */ 6581a5ae140SJason King if (!vib->vib_can_discard) { 6591a5ae140SJason King return (ENOTSUP); 6601a5ae140SJason King } 6611a5ae140SJason King 6621a5ae140SJason King mutex_enter(&vib->vib_mutex); 6631a5ae140SJason King r = vioblk_request(vib, xfer, VIRTIO_BLK_T_DISCARD); 6641a5ae140SJason King mutex_exit(&vib->vib_mutex); 6651a5ae140SJason King 6661a5ae140SJason King return (r); 6671a5ae140SJason King } 6681a5ae140SJason King 669f8296c60SJoshua M. Clulow /* 670f8296c60SJoshua M. Clulow * As the device completes processing of a request, it returns the chain for 671f8296c60SJoshua M. Clulow * that request to our I/O queue. This routine is called in two contexts: 672f8296c60SJoshua M. Clulow * - from the interrupt handler, in response to notification from the device 673f8296c60SJoshua M. Clulow * - synchronously in line with request processing when panicking 674f8296c60SJoshua M. Clulow */ 675f8296c60SJoshua M. Clulow static uint_t 676f8296c60SJoshua M. Clulow vioblk_poll(vioblk_t *vib) 6771c5bc425SAlexey Zaytsev { 678f8296c60SJoshua M. Clulow virtio_chain_t *vic; 679f8296c60SJoshua M. Clulow uint_t count = 0; 680f8296c60SJoshua M. Clulow boolean_t wakeup = B_FALSE; 6811c5bc425SAlexey Zaytsev 682f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex)); 6831c5bc425SAlexey Zaytsev 684f8296c60SJoshua M. Clulow while ((vic = virtio_queue_poll(vib->vib_vq)) != NULL) { 685f8296c60SJoshua M. Clulow vioblk_req_t *vbr = virtio_chain_data(vic); 686f8296c60SJoshua M. Clulow uint8_t status; 6871c5bc425SAlexey Zaytsev 688f8296c60SJoshua M. Clulow virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORCPU); 6891c5bc425SAlexey Zaytsev 690f8296c60SJoshua M. Clulow bcopy(virtio_dma_va(vbr->vbr_dma, 691f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr)), &status, sizeof (status)); 6921c5bc425SAlexey Zaytsev 6931c5bc425SAlexey Zaytsev switch (status) { 6941c5bc425SAlexey Zaytsev case VIRTIO_BLK_S_OK: 695f8296c60SJoshua M. Clulow vbr->vbr_error = 0; 6961c5bc425SAlexey Zaytsev break; 6971c5bc425SAlexey Zaytsev case VIRTIO_BLK_S_IOERR: 698f8296c60SJoshua M. Clulow vbr->vbr_error = EIO; 699f8296c60SJoshua M. Clulow vib->vib_stats->vbs_io_errors.value.ui64++; 7001c5bc425SAlexey Zaytsev break; 7011c5bc425SAlexey Zaytsev case VIRTIO_BLK_S_UNSUPP: 702f8296c60SJoshua M. Clulow vbr->vbr_error = ENOTTY; 703f8296c60SJoshua M. Clulow vib->vib_stats->vbs_unsupp_errors.value.ui64++; 7041c5bc425SAlexey Zaytsev break; 7051c5bc425SAlexey Zaytsev default: 706f8296c60SJoshua M. Clulow vbr->vbr_error = ENXIO; 707f8296c60SJoshua M. Clulow vib->vib_stats->vbs_nxio_errors.value.ui64++; 7081c5bc425SAlexey Zaytsev break; 7091c5bc425SAlexey Zaytsev } 7101c5bc425SAlexey Zaytsev 711f8296c60SJoshua M. Clulow count++; 7121c5bc425SAlexey Zaytsev 713f8296c60SJoshua M. Clulow if (vbr->vbr_status & VIOBLK_REQSTAT_POLLED) { 714f8296c60SJoshua M. Clulow /* 715f8296c60SJoshua M. Clulow * This request must not be freed as it is being held 716f8296c60SJoshua M. Clulow * by a call to vioblk_common_submit(). 717f8296c60SJoshua M. Clulow */ 718f8296c60SJoshua M. Clulow VERIFY(!(vbr->vbr_status & 719f8296c60SJoshua M. Clulow VIOBLK_REQSTAT_POLL_COMPLETE)); 720f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_POLL_COMPLETE; 721f8296c60SJoshua M. Clulow wakeup = B_TRUE; 722f8296c60SJoshua M. Clulow continue; 7231c5bc425SAlexey Zaytsev } 7241c5bc425SAlexey Zaytsev 725f8296c60SJoshua M. Clulow vioblk_complete(vib, vbr); 7261c5bc425SAlexey Zaytsev 727f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr); 7281c5bc425SAlexey Zaytsev } 7291c5bc425SAlexey Zaytsev 730f8296c60SJoshua M. Clulow if (wakeup) { 731f8296c60SJoshua M. Clulow /* 732f8296c60SJoshua M. Clulow * Signal anybody waiting for polled command completion. 733f8296c60SJoshua M. Clulow */ 734f8296c60SJoshua M. Clulow cv_broadcast(&vib->vib_cv); 735f8296c60SJoshua M. Clulow } 736f8296c60SJoshua M. Clulow 737f8296c60SJoshua M. Clulow return (count); 738f8296c60SJoshua M. Clulow } 739f8296c60SJoshua M. Clulow 7401c5bc425SAlexey Zaytsev uint_t 741f8296c60SJoshua M. Clulow vioblk_int_handler(caddr_t arg0, caddr_t arg1) 7421c5bc425SAlexey Zaytsev { 743f8296c60SJoshua M. Clulow vioblk_t *vib = (vioblk_t *)arg0; 744f8296c60SJoshua M. Clulow uint_t count; 745f8296c60SJoshua M. Clulow 746f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 747f8296c60SJoshua M. Clulow if ((count = vioblk_poll(vib)) > 748f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_queuemax.value.ui32) { 749f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_queuemax.value.ui32 = count; 7501c5bc425SAlexey Zaytsev } 7511c5bc425SAlexey Zaytsev 752f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_total.value.ui64++; 753f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 7541c5bc425SAlexey Zaytsev 755f8296c60SJoshua M. Clulow return (DDI_INTR_CLAIMED); 7561c5bc425SAlexey Zaytsev } 7571c5bc425SAlexey Zaytsev 7581c5bc425SAlexey Zaytsev static void 759f8296c60SJoshua M. Clulow vioblk_free_reqs(vioblk_t *vib) 7601c5bc425SAlexey Zaytsev { 761f8296c60SJoshua M. Clulow VERIFY3U(vib->vib_nreqs_alloc, ==, 0); 7621c5bc425SAlexey Zaytsev 763f8296c60SJoshua M. Clulow for (uint_t i = 0; i < vib->vib_reqs_capacity; i++) { 764f8296c60SJoshua M. Clulow struct vioblk_req *vbr = &vib->vib_reqs_mem[i]; 7651c5bc425SAlexey Zaytsev 766f8296c60SJoshua M. Clulow VERIFY(list_link_active(&vbr->vbr_link)); 767f8296c60SJoshua M. Clulow list_remove(&vib->vib_reqs, vbr); 7681c5bc425SAlexey Zaytsev 769f8296c60SJoshua M. Clulow VERIFY0(vbr->vbr_status); 7701c5bc425SAlexey Zaytsev 771*3decf168SPatrick Mooney if (vbr->vbr_chain != NULL) { 772*3decf168SPatrick Mooney virtio_chain_free(vbr->vbr_chain); 773*3decf168SPatrick Mooney vbr->vbr_chain = NULL; 774*3decf168SPatrick Mooney } 775f8296c60SJoshua M. Clulow if (vbr->vbr_dma != NULL) { 776f8296c60SJoshua M. Clulow virtio_dma_free(vbr->vbr_dma); 777f8296c60SJoshua M. Clulow vbr->vbr_dma = NULL; 7781c5bc425SAlexey Zaytsev } 779f8296c60SJoshua M. Clulow } 780f8296c60SJoshua M. Clulow VERIFY(list_is_empty(&vib->vib_reqs)); 7811c5bc425SAlexey Zaytsev 782f8296c60SJoshua M. Clulow if (vib->vib_reqs_mem != NULL) { 783f8296c60SJoshua M. Clulow kmem_free(vib->vib_reqs_mem, 784f8296c60SJoshua M. Clulow sizeof (struct vioblk_req) * vib->vib_reqs_capacity); 785f8296c60SJoshua M. Clulow vib->vib_reqs_mem = NULL; 786f8296c60SJoshua M. Clulow vib->vib_reqs_capacity = 0; 787f8296c60SJoshua M. Clulow } 7881c5bc425SAlexey Zaytsev } 7891c5bc425SAlexey Zaytsev 7901c5bc425SAlexey Zaytsev static int 791f8296c60SJoshua M. Clulow vioblk_alloc_reqs(vioblk_t *vib) 7921c5bc425SAlexey Zaytsev { 793f8296c60SJoshua M. Clulow vib->vib_reqs_capacity = MIN(virtio_queue_size(vib->vib_vq), 794f8296c60SJoshua M. Clulow VIRTIO_BLK_REQ_BUFS); 795f8296c60SJoshua M. Clulow vib->vib_reqs_mem = kmem_zalloc( 796f8296c60SJoshua M. Clulow sizeof (struct vioblk_req) * vib->vib_reqs_capacity, KM_SLEEP); 797f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc = 0; 7981c5bc425SAlexey Zaytsev 799f8296c60SJoshua M. Clulow for (uint_t i = 0; i < vib->vib_reqs_capacity; i++) { 800f8296c60SJoshua M. Clulow list_insert_tail(&vib->vib_reqs, &vib->vib_reqs_mem[i]); 8011c5bc425SAlexey Zaytsev } 8021c5bc425SAlexey Zaytsev 803f8296c60SJoshua M. Clulow for (vioblk_req_t *vbr = list_head(&vib->vib_reqs); vbr != NULL; 804f8296c60SJoshua M. Clulow vbr = list_next(&vib->vib_reqs, vbr)) { 805f8296c60SJoshua M. Clulow if ((vbr->vbr_dma = virtio_dma_alloc(vib->vib_virtio, 8061c5bc425SAlexey Zaytsev sizeof (struct vioblk_req_hdr) + sizeof (uint8_t), 807f8296c60SJoshua M. Clulow &vioblk_dma_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 808f8296c60SJoshua M. Clulow KM_SLEEP)) == NULL) { 809f8296c60SJoshua M. Clulow goto fail; 8101c5bc425SAlexey Zaytsev } 811*3decf168SPatrick Mooney vbr->vbr_chain = virtio_chain_alloc(vib->vib_vq, KM_SLEEP); 812*3decf168SPatrick Mooney if (vbr->vbr_chain == NULL) { 813*3decf168SPatrick Mooney goto fail; 814*3decf168SPatrick Mooney } 815*3decf168SPatrick Mooney virtio_chain_data_set(vbr->vbr_chain, vbr); 8161c5bc425SAlexey Zaytsev } 8171c5bc425SAlexey Zaytsev 8181c5bc425SAlexey Zaytsev return (0); 8191c5bc425SAlexey Zaytsev 820f8296c60SJoshua M. Clulow fail: 821f8296c60SJoshua M. Clulow vioblk_free_reqs(vib); 8221c5bc425SAlexey Zaytsev return (ENOMEM); 8231c5bc425SAlexey Zaytsev } 8241c5bc425SAlexey Zaytsev 8251c5bc425SAlexey Zaytsev static int 826f8296c60SJoshua M. Clulow vioblk_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 8271c5bc425SAlexey Zaytsev { 828f8296c60SJoshua M. Clulow int instance = ddi_get_instance(dip); 829f8296c60SJoshua M. Clulow vioblk_t *vib; 830f8296c60SJoshua M. Clulow virtio_t *vio; 831f8296c60SJoshua M. Clulow boolean_t did_mutex = B_FALSE; 8321c5bc425SAlexey Zaytsev 833f8296c60SJoshua M. Clulow if (cmd != DDI_ATTACH) { 834d48defc5SHans Rosenfeld return (DDI_FAILURE); 8351c5bc425SAlexey Zaytsev } 8361c5bc425SAlexey Zaytsev 837f8296c60SJoshua M. Clulow if ((vio = virtio_init(dip, VIRTIO_BLK_WANTED_FEATURES, B_TRUE)) == 838f8296c60SJoshua M. Clulow NULL) { 839f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "failed to start Virtio init"); 840f8296c60SJoshua M. Clulow return (DDI_FAILURE); 841f8296c60SJoshua M. Clulow } 8421c5bc425SAlexey Zaytsev 843f8296c60SJoshua M. Clulow vib = kmem_zalloc(sizeof (*vib), KM_SLEEP); 844f8296c60SJoshua M. Clulow vib->vib_dip = dip; 845f8296c60SJoshua M. Clulow vib->vib_virtio = vio; 846f8296c60SJoshua M. Clulow ddi_set_driver_private(dip, vib); 847f8296c60SJoshua M. Clulow list_create(&vib->vib_reqs, sizeof (vioblk_req_t), 848f8296c60SJoshua M. Clulow offsetof(vioblk_req_t, vbr_link)); 8491c5bc425SAlexey Zaytsev 8501c5bc425SAlexey Zaytsev /* 851f8296c60SJoshua M. Clulow * Determine how many scatter-gather entries we can use in a single 852f8296c60SJoshua M. Clulow * request. 8531c5bc425SAlexey Zaytsev */ 854f8296c60SJoshua M. Clulow vib->vib_seg_max = VIRTIO_BLK_DEFAULT_MAX_SEG; 855f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_SEG_MAX)) { 856f8296c60SJoshua M. Clulow vib->vib_seg_max = virtio_dev_get32(vio, 8571c5bc425SAlexey Zaytsev VIRTIO_BLK_CONFIG_SEG_MAX); 8581c5bc425SAlexey Zaytsev 859f8296c60SJoshua M. Clulow if (vib->vib_seg_max == 0 || vib->vib_seg_max == PCI_EINVAL32) { 860f8296c60SJoshua M. Clulow /* 861f8296c60SJoshua M. Clulow * We need to be able to use at least one data segment, 862f8296c60SJoshua M. Clulow * so we'll assume that this device is just poorly 863f8296c60SJoshua M. Clulow * implemented and try for one. 864f8296c60SJoshua M. Clulow */ 865f8296c60SJoshua M. Clulow vib->vib_seg_max = 1; 866f8296c60SJoshua M. Clulow } 867f8296c60SJoshua M. Clulow } 8681c5bc425SAlexey Zaytsev 8691a5ae140SJason King if (virtio_feature_present(vio, VIRTIO_BLK_F_DISCARD)) { 8701a5ae140SJason King vib->vib_max_discard_sectors = virtio_dev_get32(vio, 8711a5ae140SJason King VIRTIO_BLK_CONFIG_MAX_DISCARD_SECT); 8721a5ae140SJason King vib->vib_max_discard_seg = virtio_dev_get32(vio, 8731a5ae140SJason King VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG); 8741a5ae140SJason King vib->vib_discard_sector_align = virtio_dev_get32(vio, 8751a5ae140SJason King VIRTIO_BLK_CONFIG_DISCARD_ALIGN); 8761a5ae140SJason King 8771a5ae140SJason King if (vib->vib_max_discard_sectors == 0 || 8781a5ae140SJason King vib->vib_max_discard_seg == 0 || 8791a5ae140SJason King vib->vib_discard_sector_align == 0) { 8801a5ae140SJason King vib->vib_can_discard = B_FALSE; 8811a5ae140SJason King 8821a5ae140SJason King /* 8831a5ae140SJason King * The hypervisor shouldn't be giving us bad values. 8841a5ae140SJason King * If it is, it's probably worth notifying the 8851a5ae140SJason King * operator. 8861a5ae140SJason King */ 8871a5ae140SJason King dev_err(dip, CE_NOTE, 8881a5ae140SJason King "Host is advertising DISCARD support but with bad" 8891a5ae140SJason King "parameters: max_discard_sectors=%u, " 8901a5ae140SJason King "max_discard_segments=%u, discard_sector_align=%u", 8911a5ae140SJason King vib->vib_max_discard_sectors, 8921a5ae140SJason King vib->vib_max_discard_seg, 8931a5ae140SJason King vib->vib_discard_sector_align); 8941a5ae140SJason King } else { 8951a5ae140SJason King vib->vib_can_discard = B_TRUE; 8961a5ae140SJason King } 8971a5ae140SJason King } 8981a5ae140SJason King 8991c5bc425SAlexey Zaytsev /* 900f8296c60SJoshua M. Clulow * When allocating the request queue, we include two additional 901f8296c60SJoshua M. Clulow * descriptors (beyond those required for request data) to account for 902f8296c60SJoshua M. Clulow * the header and the status byte. 9031c5bc425SAlexey Zaytsev */ 904f8296c60SJoshua M. Clulow if ((vib->vib_vq = virtio_queue_alloc(vio, VIRTIO_BLK_VIRTQ_IO, "io", 905f8296c60SJoshua M. Clulow vioblk_int_handler, vib, B_FALSE, vib->vib_seg_max + 2)) == NULL) { 906f8296c60SJoshua M. Clulow goto fail; 9071c5bc425SAlexey Zaytsev } 9081c5bc425SAlexey Zaytsev 909f8296c60SJoshua M. Clulow if (virtio_init_complete(vio, 0) != DDI_SUCCESS) { 910f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "failed to complete Virtio init"); 911f8296c60SJoshua M. Clulow goto fail; 9121c5bc425SAlexey Zaytsev } 9131c5bc425SAlexey Zaytsev 914f8296c60SJoshua M. Clulow cv_init(&vib->vib_cv, NULL, CV_DRIVER, NULL); 915f8296c60SJoshua M. Clulow mutex_init(&vib->vib_mutex, NULL, MUTEX_DRIVER, virtio_intr_pri(vio)); 916f8296c60SJoshua M. Clulow did_mutex = B_TRUE; 917f8296c60SJoshua M. Clulow 918f8296c60SJoshua M. Clulow if ((vib->vib_kstat = kstat_create("vioblk", instance, 919f8296c60SJoshua M. Clulow "statistics", "controller", KSTAT_TYPE_NAMED, 920f8296c60SJoshua M. Clulow sizeof (struct vioblk_stats) / sizeof (kstat_named_t), 921f8296c60SJoshua M. Clulow KSTAT_FLAG_PERSISTENT)) == NULL) { 922f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "kstat_create failed"); 923f8296c60SJoshua M. Clulow goto fail; 924f8296c60SJoshua M. Clulow } 925f8296c60SJoshua M. Clulow vib->vib_stats = (vioblk_stats_t *)vib->vib_kstat->ks_data; 926f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_outofmemory, 927f8296c60SJoshua M. Clulow "total_rw_outofmemory", KSTAT_DATA_UINT64); 928f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_badoffset, 929f8296c60SJoshua M. Clulow "total_rw_badoffset", KSTAT_DATA_UINT64); 930f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_intr_total, 931f8296c60SJoshua M. Clulow "total_intr", KSTAT_DATA_UINT64); 932f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_io_errors, 933f8296c60SJoshua M. Clulow "total_io_errors", KSTAT_DATA_UINT64); 934f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_unsupp_errors, 935f8296c60SJoshua M. Clulow "total_unsupp_errors", KSTAT_DATA_UINT64); 936f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_nxio_errors, 937f8296c60SJoshua M. Clulow "total_nxio_errors", KSTAT_DATA_UINT64); 938f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_cacheflush, 939f8296c60SJoshua M. Clulow "total_rw_cacheflush", KSTAT_DATA_UINT64); 940f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_cookiesmax, 941f8296c60SJoshua M. Clulow "max_rw_cookies", KSTAT_DATA_UINT32); 942f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_intr_queuemax, 943f8296c60SJoshua M. Clulow "max_intr_queue", KSTAT_DATA_UINT32); 944f8296c60SJoshua M. Clulow kstat_install(vib->vib_kstat); 945f8296c60SJoshua M. Clulow 946f8296c60SJoshua M. Clulow vib->vib_readonly = virtio_feature_present(vio, VIRTIO_BLK_F_RO); 947f8296c60SJoshua M. Clulow if ((vib->vib_nblks = virtio_dev_get64(vio, 948f8296c60SJoshua M. Clulow VIRTIO_BLK_CONFIG_CAPACITY)) == UINT64_MAX) { 949f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "invalid capacity"); 950f8296c60SJoshua M. Clulow goto fail; 9511c5bc425SAlexey Zaytsev } 9521c5bc425SAlexey Zaytsev 953f8296c60SJoshua M. Clulow /* 954f8296c60SJoshua M. Clulow * Determine the optimal logical block size recommended by the device. 955f8296c60SJoshua M. Clulow * This size is advisory; the protocol always deals in 512 byte blocks. 956f8296c60SJoshua M. Clulow */ 957f8296c60SJoshua M. Clulow vib->vib_blk_size = DEV_BSIZE; 958f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_BLK_SIZE)) { 959f8296c60SJoshua M. Clulow uint32_t v = virtio_dev_get32(vio, VIRTIO_BLK_CONFIG_BLK_SIZE); 9601c5bc425SAlexey Zaytsev 961f8296c60SJoshua M. Clulow if (v != 0 && v != PCI_EINVAL32) { 962f8296c60SJoshua M. Clulow vib->vib_blk_size = v; 963f8296c60SJoshua M. Clulow } 964f8296c60SJoshua M. Clulow } 9651c5bc425SAlexey Zaytsev 966f8296c60SJoshua M. Clulow /* 967c5c712a8SToomas Soome * Device capacity is always in 512-byte units, convert to 968c5c712a8SToomas Soome * native blocks. 969c5c712a8SToomas Soome */ 970c5c712a8SToomas Soome vib->vib_nblks = (vib->vib_nblks * DEV_BSIZE) / vib->vib_blk_size; 971c5c712a8SToomas Soome 972c5c712a8SToomas Soome /* 973f8296c60SJoshua M. Clulow * The device may also provide an advisory physical block size. 974f8296c60SJoshua M. Clulow */ 975f8296c60SJoshua M. Clulow vib->vib_pblk_size = vib->vib_blk_size; 976f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_TOPOLOGY)) { 977f8296c60SJoshua M. Clulow uint8_t v = virtio_dev_get8(vio, VIRTIO_BLK_CONFIG_TOPO_PBEXP); 9781c5bc425SAlexey Zaytsev 979f8296c60SJoshua M. Clulow if (v != PCI_EINVAL8) { 980f8296c60SJoshua M. Clulow vib->vib_pblk_size <<= v; 981f8296c60SJoshua M. Clulow } 982f8296c60SJoshua M. Clulow } 9831c5bc425SAlexey Zaytsev 984f8296c60SJoshua M. Clulow /* 985f8296c60SJoshua M. Clulow * The maximum size for a cookie in a request. 986f8296c60SJoshua M. Clulow */ 987f8296c60SJoshua M. Clulow vib->vib_seg_size_max = VIRTIO_BLK_DEFAULT_MAX_SIZE; 988f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_SIZE_MAX)) { 989f8296c60SJoshua M. Clulow uint32_t v = virtio_dev_get32(vio, VIRTIO_BLK_CONFIG_SIZE_MAX); 990f8296c60SJoshua M. Clulow 991f8296c60SJoshua M. Clulow if (v != 0 && v != PCI_EINVAL32) { 992f8296c60SJoshua M. Clulow vib->vib_seg_size_max = v; 993f8296c60SJoshua M. Clulow } 994f8296c60SJoshua M. Clulow } 995f8296c60SJoshua M. Clulow 996f8296c60SJoshua M. Clulow /* 997f8296c60SJoshua M. Clulow * Set up the DMA attributes for blkdev to use for request data. The 998f8296c60SJoshua M. Clulow * specification is not extremely clear about whether DMA-related 999f8296c60SJoshua M. Clulow * parameters include or exclude the header and status descriptors. 1000f8296c60SJoshua M. Clulow * For now, we assume they cover only the request data and not the 1001f8296c60SJoshua M. Clulow * headers. 1002f8296c60SJoshua M. Clulow */ 1003f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr = vioblk_dma_attr; 1004f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_sgllen = vib->vib_seg_max; 1005f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_count_max = vib->vib_seg_size_max; 1006f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_maxxfer = vib->vib_seg_max * 1007f8296c60SJoshua M. Clulow vib->vib_seg_size_max; 1008f8296c60SJoshua M. Clulow 1009f8296c60SJoshua M. Clulow if (vioblk_alloc_reqs(vib) != 0) { 1010f8296c60SJoshua M. Clulow goto fail; 1011f8296c60SJoshua M. Clulow } 1012f8296c60SJoshua M. Clulow 1013f8296c60SJoshua M. Clulow /* 1014f8296c60SJoshua M. Clulow * The blkdev framework does not provide a way to specify that the 1015f8296c60SJoshua M. Clulow * device does not support write cache flushing, except by omitting the 1016f8296c60SJoshua M. Clulow * "o_sync_cache" member from the ops vector. As "bd_alloc_handle()" 1017f8296c60SJoshua M. Clulow * makes a copy of the ops vector, we can safely assemble one on the 1018f8296c60SJoshua M. Clulow * stack based on negotiated features. 10191a5ae140SJason King * 10201a5ae140SJason King * Similarly, the blkdev framework does not provide a way to indicate 10211a5ae140SJason King * if a device supports an TRIM/UNMAP/DISCARD type operation except 10221a5ae140SJason King * by omitting the "o_free_space" member from the ops vector. 1023f8296c60SJoshua M. Clulow */ 1024f8296c60SJoshua M. Clulow bd_ops_t vioblk_bd_ops = { 10254d95620bSPaul Winder .o_version = BD_OPS_CURRENT_VERSION, 1026f8296c60SJoshua M. Clulow .o_drive_info = vioblk_bd_driveinfo, 1027f8296c60SJoshua M. Clulow .o_media_info = vioblk_bd_mediainfo, 1028f8296c60SJoshua M. Clulow .o_devid_init = vioblk_bd_devid, 1029f8296c60SJoshua M. Clulow .o_sync_cache = vioblk_bd_flush, 1030f8296c60SJoshua M. Clulow .o_read = vioblk_bd_read, 1031f8296c60SJoshua M. Clulow .o_write = vioblk_bd_write, 10321a5ae140SJason King .o_free_space = vioblk_bd_free_space, 1033f8296c60SJoshua M. Clulow }; 1034f8296c60SJoshua M. Clulow if (!virtio_feature_present(vio, VIRTIO_BLK_F_FLUSH)) { 1035f8296c60SJoshua M. Clulow vioblk_bd_ops.o_sync_cache = NULL; 1036f8296c60SJoshua M. Clulow } 10371a5ae140SJason King if (!vib->vib_can_discard) { 10381a5ae140SJason King vioblk_bd_ops.o_free_space = NULL; 10391a5ae140SJason King } 1040f8296c60SJoshua M. Clulow 1041f8296c60SJoshua M. Clulow vib->vib_bd_h = bd_alloc_handle(vib, &vioblk_bd_ops, 1042f8296c60SJoshua M. Clulow &vib->vib_bd_dma_attr, KM_SLEEP); 1043f8296c60SJoshua M. Clulow 1044f8296c60SJoshua M. Clulow /* 1045f8296c60SJoshua M. Clulow * Enable interrupts now so that we can request the device identity. 1046f8296c60SJoshua M. Clulow */ 1047f8296c60SJoshua M. Clulow if (virtio_interrupts_enable(vio) != DDI_SUCCESS) { 1048f8296c60SJoshua M. Clulow goto fail; 1049f8296c60SJoshua M. Clulow } 1050f8296c60SJoshua M. Clulow 1051f8296c60SJoshua M. Clulow vioblk_get_id(vib); 1052f8296c60SJoshua M. Clulow 1053f8296c60SJoshua M. Clulow if (bd_attach_handle(dip, vib->vib_bd_h) != DDI_SUCCESS) { 1054f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "Failed to attach blkdev"); 1055f8296c60SJoshua M. Clulow goto fail; 10561c5bc425SAlexey Zaytsev } 10571c5bc425SAlexey Zaytsev 10581c5bc425SAlexey Zaytsev return (DDI_SUCCESS); 10591c5bc425SAlexey Zaytsev 1060f8296c60SJoshua M. Clulow fail: 1061f8296c60SJoshua M. Clulow if (vib->vib_bd_h != NULL) { 1062f8296c60SJoshua M. Clulow (void) bd_detach_handle(vib->vib_bd_h); 1063f8296c60SJoshua M. Clulow bd_free_handle(vib->vib_bd_h); 1064f8296c60SJoshua M. Clulow } 1065f8296c60SJoshua M. Clulow if (vio != NULL) { 1066f8296c60SJoshua M. Clulow (void) virtio_fini(vio, B_TRUE); 1067f8296c60SJoshua M. Clulow } 1068f8296c60SJoshua M. Clulow if (did_mutex) { 1069f8296c60SJoshua M. Clulow mutex_destroy(&vib->vib_mutex); 1070f8296c60SJoshua M. Clulow cv_destroy(&vib->vib_cv); 1071f8296c60SJoshua M. Clulow } 1072f8296c60SJoshua M. Clulow if (vib->vib_kstat != NULL) { 1073f8296c60SJoshua M. Clulow kstat_delete(vib->vib_kstat); 1074f8296c60SJoshua M. Clulow } 1075f8296c60SJoshua M. Clulow vioblk_free_reqs(vib); 1076f8296c60SJoshua M. Clulow kmem_free(vib, sizeof (*vib)); 1077d48defc5SHans Rosenfeld return (DDI_FAILURE); 10781c5bc425SAlexey Zaytsev } 10791c5bc425SAlexey Zaytsev 10801c5bc425SAlexey Zaytsev static int 1081f8296c60SJoshua M. Clulow vioblk_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 10821c5bc425SAlexey Zaytsev { 1083f8296c60SJoshua M. Clulow vioblk_t *vib = ddi_get_driver_private(dip); 10841c5bc425SAlexey Zaytsev 1085f8296c60SJoshua M. Clulow if (cmd != DDI_DETACH) { 10861c5bc425SAlexey Zaytsev return (DDI_FAILURE); 10871c5bc425SAlexey Zaytsev } 10881c5bc425SAlexey Zaytsev 1089f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex); 1090f8296c60SJoshua M. Clulow if (vib->vib_nreqs_alloc > 0) { 1091f8296c60SJoshua M. Clulow /* 1092f8296c60SJoshua M. Clulow * Cannot detach while there are still outstanding requests. 1093f8296c60SJoshua M. Clulow */ 1094f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 1095f8296c60SJoshua M. Clulow return (DDI_FAILURE); 1096f8296c60SJoshua M. Clulow } 1097f8296c60SJoshua M. Clulow 1098f8296c60SJoshua M. Clulow if (bd_detach_handle(vib->vib_bd_h) != DDI_SUCCESS) { 1099f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 1100f8296c60SJoshua M. Clulow return (DDI_FAILURE); 1101f8296c60SJoshua M. Clulow } 1102f8296c60SJoshua M. Clulow 1103f8296c60SJoshua M. Clulow /* 1104f8296c60SJoshua M. Clulow * Tear down the Virtio framework before freeing the rest of the 1105f8296c60SJoshua M. Clulow * resources. This will ensure the interrupt handlers are no longer 1106f8296c60SJoshua M. Clulow * running. 1107f8296c60SJoshua M. Clulow */ 1108f8296c60SJoshua M. Clulow virtio_fini(vib->vib_virtio, B_FALSE); 1109f8296c60SJoshua M. Clulow 1110f8296c60SJoshua M. Clulow vioblk_free_reqs(vib); 1111f8296c60SJoshua M. Clulow kstat_delete(vib->vib_kstat); 1112f8296c60SJoshua M. Clulow 1113f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex); 1114f8296c60SJoshua M. Clulow mutex_destroy(&vib->vib_mutex); 1115f8296c60SJoshua M. Clulow 1116f8296c60SJoshua M. Clulow kmem_free(vib, sizeof (*vib)); 11171c5bc425SAlexey Zaytsev 11181c5bc425SAlexey Zaytsev return (DDI_SUCCESS); 11191c5bc425SAlexey Zaytsev } 11201c5bc425SAlexey Zaytsev 11211c5bc425SAlexey Zaytsev static int 1122f8296c60SJoshua M. Clulow vioblk_quiesce(dev_info_t *dip) 11231c5bc425SAlexey Zaytsev { 1124f8296c60SJoshua M. Clulow vioblk_t *vib; 11251c5bc425SAlexey Zaytsev 1126f8296c60SJoshua M. Clulow if ((vib = ddi_get_driver_private(dip)) == NULL) { 1127f8296c60SJoshua M. Clulow return (DDI_FAILURE); 1128f8296c60SJoshua M. Clulow } 11291c5bc425SAlexey Zaytsev 1130f8296c60SJoshua M. Clulow return (virtio_quiesce(vib->vib_virtio)); 11311c5bc425SAlexey Zaytsev } 11321c5bc425SAlexey Zaytsev 11331c5bc425SAlexey Zaytsev int 11341c5bc425SAlexey Zaytsev _init(void) 11351c5bc425SAlexey Zaytsev { 11361c5bc425SAlexey Zaytsev int rv; 11371c5bc425SAlexey Zaytsev 11381c5bc425SAlexey Zaytsev bd_mod_init(&vioblk_dev_ops); 11391c5bc425SAlexey Zaytsev 1140f8296c60SJoshua M. Clulow if ((rv = mod_install(&vioblk_modlinkage)) != 0) { 11411c5bc425SAlexey Zaytsev bd_mod_fini(&vioblk_dev_ops); 11421c5bc425SAlexey Zaytsev } 11431c5bc425SAlexey Zaytsev 11441c5bc425SAlexey Zaytsev return (rv); 11451c5bc425SAlexey Zaytsev } 11461c5bc425SAlexey Zaytsev 11471c5bc425SAlexey Zaytsev int 11481c5bc425SAlexey Zaytsev _fini(void) 11491c5bc425SAlexey Zaytsev { 11501c5bc425SAlexey Zaytsev int rv; 11511c5bc425SAlexey Zaytsev 1152f8296c60SJoshua M. Clulow if ((rv = mod_remove(&vioblk_modlinkage)) == 0) { 11531c5bc425SAlexey Zaytsev bd_mod_fini(&vioblk_dev_ops); 11541c5bc425SAlexey Zaytsev } 11551c5bc425SAlexey Zaytsev 11561c5bc425SAlexey Zaytsev return (rv); 11571c5bc425SAlexey Zaytsev } 11581c5bc425SAlexey Zaytsev 11591c5bc425SAlexey Zaytsev int 11601c5bc425SAlexey Zaytsev _info(struct modinfo *modinfop) 11611c5bc425SAlexey Zaytsev { 1162f8296c60SJoshua M. Clulow return (mod_info(&vioblk_modlinkage, modinfop)); 11631c5bc425SAlexey Zaytsev } 1164