14c87aefeSPatrick Mooney /*- 24c87aefeSPatrick Mooney * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 34c87aefeSPatrick Mooney * 44c87aefeSPatrick Mooney * Copyright (c) 2017 Shunsuke Mie 54c87aefeSPatrick Mooney * Copyright (c) 2018 Leon Dang 64c87aefeSPatrick Mooney * 7*84659b24SMichael Zeller * Function crc16 Copyright (c) 2017, Fedor Uporov 8*84659b24SMichael Zeller * Obtained from function ext2_crc16() in sys/fs/ext2fs/ext2_csum.c 9*84659b24SMichael Zeller * 104c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without 114c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions 124c87aefeSPatrick Mooney * are met: 134c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright 144c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer. 154c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright 164c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the 174c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution. 184c87aefeSPatrick Mooney * 194c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 204c87aefeSPatrick Mooney * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 214c87aefeSPatrick Mooney * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 224c87aefeSPatrick Mooney * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 234c87aefeSPatrick Mooney * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 244c87aefeSPatrick Mooney * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 254c87aefeSPatrick Mooney * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 264c87aefeSPatrick Mooney * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 274c87aefeSPatrick Mooney * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 284c87aefeSPatrick Mooney * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 294c87aefeSPatrick Mooney * SUCH DAMAGE. 304c87aefeSPatrick Mooney */ 314c87aefeSPatrick Mooney 324c87aefeSPatrick Mooney /* 334c87aefeSPatrick Mooney * bhyve PCIe-NVMe device emulation. 344c87aefeSPatrick Mooney * 354c87aefeSPatrick Mooney * options: 36*84659b24SMichael Zeller * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=# 374c87aefeSPatrick Mooney * 384c87aefeSPatrick Mooney * accepted devpath: 394c87aefeSPatrick Mooney * /dev/blockdev 404c87aefeSPatrick Mooney * /path/to/image 414c87aefeSPatrick Mooney * ram=size_in_MiB 424c87aefeSPatrick Mooney * 434c87aefeSPatrick Mooney * maxq = max number of queues 444c87aefeSPatrick Mooney * qsz = max elements in each queue 454c87aefeSPatrick Mooney * ioslots = max number of concurrent io requests 464c87aefeSPatrick Mooney * sectsz = sector size (defaults to blockif sector size) 474c87aefeSPatrick Mooney * ser = serial number (20-chars max) 48*84659b24SMichael Zeller * eui64 = IEEE Extended Unique Identifier (8 byte value) 494c87aefeSPatrick Mooney * 504c87aefeSPatrick Mooney */ 514c87aefeSPatrick Mooney 524c87aefeSPatrick Mooney /* TODO: 534c87aefeSPatrick Mooney - create async event for smart and log 544c87aefeSPatrick Mooney - intr coalesce 554c87aefeSPatrick Mooney */ 564c87aefeSPatrick Mooney 574c87aefeSPatrick Mooney #include <sys/cdefs.h> 584c87aefeSPatrick Mooney __FBSDID("$FreeBSD$"); 594c87aefeSPatrick Mooney 604c87aefeSPatrick Mooney #include <sys/types.h> 61*84659b24SMichael Zeller #include <net/ieee_oui.h> 62*84659b24SMichael Zeller #ifndef __FreeBSD__ 63*84659b24SMichael Zeller #include <endian.h> 64*84659b24SMichael Zeller #endif 654c87aefeSPatrick Mooney 664c87aefeSPatrick Mooney #include <assert.h> 674c87aefeSPatrick Mooney #include <pthread.h> 684c87aefeSPatrick Mooney #include <semaphore.h> 694c87aefeSPatrick Mooney #include <stdbool.h> 704c87aefeSPatrick Mooney #include <stddef.h> 714c87aefeSPatrick Mooney #include <stdint.h> 724c87aefeSPatrick Mooney #include <stdio.h> 734c87aefeSPatrick Mooney #include <stdlib.h> 744c87aefeSPatrick Mooney #include <string.h> 754c87aefeSPatrick Mooney 764c87aefeSPatrick Mooney #include <machine/atomic.h> 774c87aefeSPatrick Mooney #include <machine/vmm.h> 784c87aefeSPatrick Mooney #include <vmmapi.h> 794c87aefeSPatrick Mooney 804c87aefeSPatrick Mooney #include <dev/nvme/nvme.h> 814c87aefeSPatrick Mooney 824c87aefeSPatrick Mooney #include "bhyverun.h" 834c87aefeSPatrick Mooney #include "block_if.h" 844c87aefeSPatrick Mooney #include "pci_emul.h" 854c87aefeSPatrick Mooney 864c87aefeSPatrick Mooney 874c87aefeSPatrick Mooney static int nvme_debug = 0; 884c87aefeSPatrick Mooney #define DPRINTF(params) if (nvme_debug) printf params 894c87aefeSPatrick Mooney #define WPRINTF(params) printf params 904c87aefeSPatrick Mooney 914c87aefeSPatrick Mooney /* defaults; can be overridden */ 924c87aefeSPatrick Mooney #define NVME_MSIX_BAR 4 934c87aefeSPatrick Mooney 944c87aefeSPatrick Mooney #define NVME_IOSLOTS 8 954c87aefeSPatrick Mooney 964c87aefeSPatrick Mooney /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ 974c87aefeSPatrick Mooney #define NVME_MMIO_SPACE_MIN (1 << 14) 984c87aefeSPatrick Mooney 994c87aefeSPatrick Mooney #define NVME_QUEUES 16 1004c87aefeSPatrick Mooney #define NVME_MAX_QENTRIES 2048 1014c87aefeSPatrick Mooney 1024c87aefeSPatrick Mooney #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) 1034c87aefeSPatrick Mooney #define NVME_MAX_BLOCKIOVS 512 1044c87aefeSPatrick Mooney 105*84659b24SMichael Zeller /* This is a synthetic status code to indicate there is no status */ 106*84659b24SMichael Zeller #define NVME_NO_STATUS 0xffff 107*84659b24SMichael Zeller #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS) 108*84659b24SMichael Zeller 1094c87aefeSPatrick Mooney /* helpers */ 1104c87aefeSPatrick Mooney 1114c87aefeSPatrick Mooney /* Convert a zero-based value into a one-based value */ 1124c87aefeSPatrick Mooney #define ONE_BASED(zero) ((zero) + 1) 1134c87aefeSPatrick Mooney /* Convert a one-based value into a zero-based value */ 1144c87aefeSPatrick Mooney #define ZERO_BASED(one) ((one) - 1) 1154c87aefeSPatrick Mooney 1164c87aefeSPatrick Mooney /* Encode number of SQ's and CQ's for Set/Get Features */ 1174c87aefeSPatrick Mooney #define NVME_FEATURE_NUM_QUEUES(sc) \ 1184c87aefeSPatrick Mooney (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ 1194c87aefeSPatrick Mooney (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16; 1204c87aefeSPatrick Mooney 1214c87aefeSPatrick Mooney #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) 1224c87aefeSPatrick Mooney 1234c87aefeSPatrick Mooney enum nvme_controller_register_offsets { 1244c87aefeSPatrick Mooney NVME_CR_CAP_LOW = 0x00, 1254c87aefeSPatrick Mooney NVME_CR_CAP_HI = 0x04, 1264c87aefeSPatrick Mooney NVME_CR_VS = 0x08, 1274c87aefeSPatrick Mooney NVME_CR_INTMS = 0x0c, 1284c87aefeSPatrick Mooney NVME_CR_INTMC = 0x10, 1294c87aefeSPatrick Mooney NVME_CR_CC = 0x14, 1304c87aefeSPatrick Mooney NVME_CR_CSTS = 0x1c, 1314c87aefeSPatrick Mooney NVME_CR_NSSR = 0x20, 1324c87aefeSPatrick Mooney NVME_CR_AQA = 0x24, 1334c87aefeSPatrick Mooney NVME_CR_ASQ_LOW = 0x28, 1344c87aefeSPatrick Mooney NVME_CR_ASQ_HI = 0x2c, 1354c87aefeSPatrick Mooney NVME_CR_ACQ_LOW = 0x30, 1364c87aefeSPatrick Mooney NVME_CR_ACQ_HI = 0x34, 1374c87aefeSPatrick Mooney }; 1384c87aefeSPatrick Mooney 1394c87aefeSPatrick Mooney enum nvme_cmd_cdw11 { 1404c87aefeSPatrick Mooney NVME_CMD_CDW11_PC = 0x0001, 1414c87aefeSPatrick Mooney NVME_CMD_CDW11_IEN = 0x0002, 1424c87aefeSPatrick Mooney NVME_CMD_CDW11_IV = 0xFFFF0000, 1434c87aefeSPatrick Mooney }; 1444c87aefeSPatrick Mooney 1454c87aefeSPatrick Mooney #define NVME_CQ_INTEN 0x01 1464c87aefeSPatrick Mooney #define NVME_CQ_INTCOAL 0x02 1474c87aefeSPatrick Mooney 1484c87aefeSPatrick Mooney struct nvme_completion_queue { 1494c87aefeSPatrick Mooney struct nvme_completion *qbase; 1504c87aefeSPatrick Mooney uint32_t size; 1514c87aefeSPatrick Mooney uint16_t tail; /* nvme progress */ 1524c87aefeSPatrick Mooney uint16_t head; /* guest progress */ 1534c87aefeSPatrick Mooney uint16_t intr_vec; 1544c87aefeSPatrick Mooney uint32_t intr_en; 1554c87aefeSPatrick Mooney pthread_mutex_t mtx; 1564c87aefeSPatrick Mooney }; 1574c87aefeSPatrick Mooney 1584c87aefeSPatrick Mooney struct nvme_submission_queue { 1594c87aefeSPatrick Mooney struct nvme_command *qbase; 1604c87aefeSPatrick Mooney uint32_t size; 1614c87aefeSPatrick Mooney uint16_t head; /* nvme progress */ 1624c87aefeSPatrick Mooney uint16_t tail; /* guest progress */ 1634c87aefeSPatrick Mooney uint16_t cqid; /* completion queue id */ 1644c87aefeSPatrick Mooney int busy; /* queue is being processed */ 1654c87aefeSPatrick Mooney int qpriority; 1664c87aefeSPatrick Mooney }; 1674c87aefeSPatrick Mooney 1684c87aefeSPatrick Mooney enum nvme_storage_type { 1694c87aefeSPatrick Mooney NVME_STOR_BLOCKIF = 0, 1704c87aefeSPatrick Mooney NVME_STOR_RAM = 1, 1714c87aefeSPatrick Mooney }; 1724c87aefeSPatrick Mooney 1734c87aefeSPatrick Mooney struct pci_nvme_blockstore { 1744c87aefeSPatrick Mooney enum nvme_storage_type type; 1754c87aefeSPatrick Mooney void *ctx; 1764c87aefeSPatrick Mooney uint64_t size; 1774c87aefeSPatrick Mooney uint32_t sectsz; 1784c87aefeSPatrick Mooney uint32_t sectsz_bits; 179*84659b24SMichael Zeller uint64_t eui64; 1804c87aefeSPatrick Mooney }; 1814c87aefeSPatrick Mooney 1824c87aefeSPatrick Mooney struct pci_nvme_ioreq { 1834c87aefeSPatrick Mooney struct pci_nvme_softc *sc; 1844c87aefeSPatrick Mooney struct pci_nvme_ioreq *next; 1854c87aefeSPatrick Mooney struct nvme_submission_queue *nvme_sq; 1864c87aefeSPatrick Mooney uint16_t sqid; 1874c87aefeSPatrick Mooney 1884c87aefeSPatrick Mooney /* command information */ 1894c87aefeSPatrick Mooney uint16_t opc; 1904c87aefeSPatrick Mooney uint16_t cid; 1914c87aefeSPatrick Mooney uint32_t nsid; 1924c87aefeSPatrick Mooney 1934c87aefeSPatrick Mooney uint64_t prev_gpaddr; 1944c87aefeSPatrick Mooney size_t prev_size; 1954c87aefeSPatrick Mooney 1964c87aefeSPatrick Mooney /* 1974c87aefeSPatrick Mooney * lock if all iovs consumed (big IO); 1984c87aefeSPatrick Mooney * complete transaction before continuing 1994c87aefeSPatrick Mooney */ 2004c87aefeSPatrick Mooney pthread_mutex_t mtx; 2014c87aefeSPatrick Mooney pthread_cond_t cv; 2024c87aefeSPatrick Mooney 2034c87aefeSPatrick Mooney struct blockif_req io_req; 2044c87aefeSPatrick Mooney 2054c87aefeSPatrick Mooney /* pad to fit up to 512 page descriptors from guest IO request */ 2064c87aefeSPatrick Mooney struct iovec iovpadding[NVME_MAX_BLOCKIOVS-BLOCKIF_IOV_MAX]; 2074c87aefeSPatrick Mooney }; 2084c87aefeSPatrick Mooney 2094c87aefeSPatrick Mooney struct pci_nvme_softc { 2104c87aefeSPatrick Mooney struct pci_devinst *nsc_pi; 2114c87aefeSPatrick Mooney 2124c87aefeSPatrick Mooney pthread_mutex_t mtx; 2134c87aefeSPatrick Mooney 2144c87aefeSPatrick Mooney struct nvme_registers regs; 2154c87aefeSPatrick Mooney 2164c87aefeSPatrick Mooney struct nvme_namespace_data nsdata; 2174c87aefeSPatrick Mooney struct nvme_controller_data ctrldata; 2184c87aefeSPatrick Mooney struct nvme_error_information_entry err_log; 2194c87aefeSPatrick Mooney struct nvme_health_information_page health_log; 2204c87aefeSPatrick Mooney struct nvme_firmware_page fw_log; 2214c87aefeSPatrick Mooney 2224c87aefeSPatrick Mooney struct pci_nvme_blockstore nvstore; 2234c87aefeSPatrick Mooney 2244c87aefeSPatrick Mooney uint16_t max_qentries; /* max entries per queue */ 2254c87aefeSPatrick Mooney uint32_t max_queues; /* max number of IO SQ's or CQ's */ 2264c87aefeSPatrick Mooney uint32_t num_cqueues; 2274c87aefeSPatrick Mooney uint32_t num_squeues; 2284c87aefeSPatrick Mooney 2294c87aefeSPatrick Mooney struct pci_nvme_ioreq *ioreqs; 2304c87aefeSPatrick Mooney struct pci_nvme_ioreq *ioreqs_free; /* free list of ioreqs */ 2314c87aefeSPatrick Mooney uint32_t pending_ios; 2324c87aefeSPatrick Mooney uint32_t ioslots; 2334c87aefeSPatrick Mooney sem_t iosemlock; 2344c87aefeSPatrick Mooney 2354c87aefeSPatrick Mooney /* 2364c87aefeSPatrick Mooney * Memory mapped Submission and Completion queues 2374c87aefeSPatrick Mooney * Each array includes both Admin and IO queues 2384c87aefeSPatrick Mooney */ 2394c87aefeSPatrick Mooney struct nvme_completion_queue *compl_queues; 2404c87aefeSPatrick Mooney struct nvme_submission_queue *submit_queues; 2414c87aefeSPatrick Mooney 2424c87aefeSPatrick Mooney /* controller features */ 2434c87aefeSPatrick Mooney uint32_t intr_coales_aggr_time; /* 0x08: uS to delay intr */ 2444c87aefeSPatrick Mooney uint32_t intr_coales_aggr_thresh; /* 0x08: compl-Q entries */ 2454c87aefeSPatrick Mooney uint32_t async_ev_config; /* 0x0B: async event config */ 2464c87aefeSPatrick Mooney }; 2474c87aefeSPatrick Mooney 2484c87aefeSPatrick Mooney 2494c87aefeSPatrick Mooney static void pci_nvme_io_partial(struct blockif_req *br, int err); 2504c87aefeSPatrick Mooney 2514c87aefeSPatrick Mooney /* Controller Configuration utils */ 2524c87aefeSPatrick Mooney #define NVME_CC_GET_EN(cc) \ 2534c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK) 2544c87aefeSPatrick Mooney #define NVME_CC_GET_CSS(cc) \ 2554c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK) 2564c87aefeSPatrick Mooney #define NVME_CC_GET_SHN(cc) \ 2574c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK) 2584c87aefeSPatrick Mooney #define NVME_CC_GET_IOSQES(cc) \ 2594c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK) 2604c87aefeSPatrick Mooney #define NVME_CC_GET_IOCQES(cc) \ 2614c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK) 2624c87aefeSPatrick Mooney 2634c87aefeSPatrick Mooney #define NVME_CC_WRITE_MASK \ 2644c87aefeSPatrick Mooney ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \ 2654c87aefeSPatrick Mooney (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \ 2664c87aefeSPatrick Mooney (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT)) 2674c87aefeSPatrick Mooney 2684c87aefeSPatrick Mooney #define NVME_CC_NEN_WRITE_MASK \ 2694c87aefeSPatrick Mooney ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \ 2704c87aefeSPatrick Mooney (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \ 2714c87aefeSPatrick Mooney (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT)) 2724c87aefeSPatrick Mooney 2734c87aefeSPatrick Mooney /* Controller Status utils */ 2744c87aefeSPatrick Mooney #define NVME_CSTS_GET_RDY(sts) \ 2754c87aefeSPatrick Mooney ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK) 2764c87aefeSPatrick Mooney 2774c87aefeSPatrick Mooney #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT) 2784c87aefeSPatrick Mooney 2794c87aefeSPatrick Mooney /* Completion Queue status word utils */ 2804c87aefeSPatrick Mooney #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT) 2814c87aefeSPatrick Mooney #define NVME_STATUS_MASK \ 2824c87aefeSPatrick Mooney ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\ 2834c87aefeSPatrick Mooney (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT)) 2844c87aefeSPatrick Mooney 2854c87aefeSPatrick Mooney static __inline void 2864c87aefeSPatrick Mooney cpywithpad(char *dst, size_t dst_size, const char *src, char pad) 2874c87aefeSPatrick Mooney { 2884c87aefeSPatrick Mooney size_t len; 2894c87aefeSPatrick Mooney 2904c87aefeSPatrick Mooney len = strnlen(src, dst_size); 2914c87aefeSPatrick Mooney memset(dst, pad, dst_size); 2924c87aefeSPatrick Mooney memcpy(dst, src, len); 2934c87aefeSPatrick Mooney } 2944c87aefeSPatrick Mooney 2954c87aefeSPatrick Mooney static __inline void 2964c87aefeSPatrick Mooney pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) 2974c87aefeSPatrick Mooney { 2984c87aefeSPatrick Mooney 2994c87aefeSPatrick Mooney *status &= ~NVME_STATUS_MASK; 3004c87aefeSPatrick Mooney *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT | 3014c87aefeSPatrick Mooney (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; 3024c87aefeSPatrick Mooney } 3034c87aefeSPatrick Mooney 3044c87aefeSPatrick Mooney static __inline void 3054c87aefeSPatrick Mooney pci_nvme_status_genc(uint16_t *status, uint16_t code) 3064c87aefeSPatrick Mooney { 3074c87aefeSPatrick Mooney 3084c87aefeSPatrick Mooney pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); 3094c87aefeSPatrick Mooney } 3104c87aefeSPatrick Mooney 3114c87aefeSPatrick Mooney static __inline void 3124c87aefeSPatrick Mooney pci_nvme_toggle_phase(uint16_t *status, int prev) 3134c87aefeSPatrick Mooney { 3144c87aefeSPatrick Mooney 3154c87aefeSPatrick Mooney if (prev) 3164c87aefeSPatrick Mooney *status &= ~NVME_STATUS_P; 3174c87aefeSPatrick Mooney else 3184c87aefeSPatrick Mooney *status |= NVME_STATUS_P; 3194c87aefeSPatrick Mooney } 3204c87aefeSPatrick Mooney 3214c87aefeSPatrick Mooney static void 3224c87aefeSPatrick Mooney pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) 3234c87aefeSPatrick Mooney { 3244c87aefeSPatrick Mooney struct nvme_controller_data *cd = &sc->ctrldata; 3254c87aefeSPatrick Mooney 3264c87aefeSPatrick Mooney cd->vid = 0xFB5D; 3274c87aefeSPatrick Mooney cd->ssvid = 0x0000; 3284c87aefeSPatrick Mooney 3294c87aefeSPatrick Mooney cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); 3304c87aefeSPatrick Mooney cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); 3314c87aefeSPatrick Mooney 3324c87aefeSPatrick Mooney /* Num of submission commands that we can handle at a time (2^rab) */ 3334c87aefeSPatrick Mooney cd->rab = 4; 3344c87aefeSPatrick Mooney 3354c87aefeSPatrick Mooney /* FreeBSD OUI */ 3364c87aefeSPatrick Mooney cd->ieee[0] = 0x58; 3374c87aefeSPatrick Mooney cd->ieee[1] = 0x9c; 3384c87aefeSPatrick Mooney cd->ieee[2] = 0xfc; 3394c87aefeSPatrick Mooney 3404c87aefeSPatrick Mooney cd->mic = 0; 3414c87aefeSPatrick Mooney 3424c87aefeSPatrick Mooney cd->mdts = 9; /* max data transfer size (2^mdts * CAP.MPSMIN) */ 3434c87aefeSPatrick Mooney 3444c87aefeSPatrick Mooney cd->ver = 0x00010300; 3454c87aefeSPatrick Mooney 3464c87aefeSPatrick Mooney cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT; 3474c87aefeSPatrick Mooney cd->acl = 2; 3484c87aefeSPatrick Mooney cd->aerl = 4; 3494c87aefeSPatrick Mooney 3504c87aefeSPatrick Mooney cd->lpa = 0; /* TODO: support some simple things like SMART */ 3514c87aefeSPatrick Mooney cd->elpe = 0; /* max error log page entries */ 3524c87aefeSPatrick Mooney cd->npss = 1; /* number of power states support */ 3534c87aefeSPatrick Mooney 3544c87aefeSPatrick Mooney /* Warning Composite Temperature Threshold */ 3554c87aefeSPatrick Mooney cd->wctemp = 0x0157; 3564c87aefeSPatrick Mooney 3574c87aefeSPatrick Mooney cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) | 3584c87aefeSPatrick Mooney (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT); 3594c87aefeSPatrick Mooney cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) | 3604c87aefeSPatrick Mooney (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT); 3614c87aefeSPatrick Mooney cd->nn = 1; /* number of namespaces */ 3624c87aefeSPatrick Mooney 3634c87aefeSPatrick Mooney cd->fna = 0x03; 3644c87aefeSPatrick Mooney 3654c87aefeSPatrick Mooney cd->power_state[0].mp = 10; 3664c87aefeSPatrick Mooney } 3674c87aefeSPatrick Mooney 368*84659b24SMichael Zeller /* 369*84659b24SMichael Zeller * Calculate the CRC-16 of the given buffer 370*84659b24SMichael Zeller * See copyright attribution at top of file 371*84659b24SMichael Zeller */ 372*84659b24SMichael Zeller static uint16_t 373*84659b24SMichael Zeller crc16(uint16_t crc, const void *buffer, unsigned int len) 3744c87aefeSPatrick Mooney { 375*84659b24SMichael Zeller const unsigned char *cp = buffer; 376*84659b24SMichael Zeller /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ 377*84659b24SMichael Zeller static uint16_t const crc16_table[256] = { 378*84659b24SMichael Zeller 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 379*84659b24SMichael Zeller 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 380*84659b24SMichael Zeller 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 381*84659b24SMichael Zeller 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 382*84659b24SMichael Zeller 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 383*84659b24SMichael Zeller 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 384*84659b24SMichael Zeller 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 385*84659b24SMichael Zeller 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 386*84659b24SMichael Zeller 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 387*84659b24SMichael Zeller 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 388*84659b24SMichael Zeller 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 389*84659b24SMichael Zeller 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 390*84659b24SMichael Zeller 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 391*84659b24SMichael Zeller 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 392*84659b24SMichael Zeller 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 393*84659b24SMichael Zeller 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 394*84659b24SMichael Zeller 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 395*84659b24SMichael Zeller 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 396*84659b24SMichael Zeller 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 397*84659b24SMichael Zeller 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 398*84659b24SMichael Zeller 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 399*84659b24SMichael Zeller 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 400*84659b24SMichael Zeller 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 401*84659b24SMichael Zeller 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 402*84659b24SMichael Zeller 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 403*84659b24SMichael Zeller 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 404*84659b24SMichael Zeller 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 405*84659b24SMichael Zeller 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 406*84659b24SMichael Zeller 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 407*84659b24SMichael Zeller 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 408*84659b24SMichael Zeller 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 409*84659b24SMichael Zeller 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 410*84659b24SMichael Zeller }; 4114c87aefeSPatrick Mooney 412*84659b24SMichael Zeller while (len--) 413*84659b24SMichael Zeller crc = (((crc >> 8) & 0xffU) ^ 414*84659b24SMichael Zeller crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; 415*84659b24SMichael Zeller return crc; 416*84659b24SMichael Zeller } 417*84659b24SMichael Zeller 418*84659b24SMichael Zeller static void 419*84659b24SMichael Zeller pci_nvme_init_nsdata(struct pci_nvme_softc *sc, 420*84659b24SMichael Zeller struct nvme_namespace_data *nd, uint32_t nsid, 421*84659b24SMichael Zeller uint64_t eui64) 422*84659b24SMichael Zeller { 4234c87aefeSPatrick Mooney 4244c87aefeSPatrick Mooney nd->nsze = sc->nvstore.size / sc->nvstore.sectsz; 4254c87aefeSPatrick Mooney nd->ncap = nd->nsze; 4264c87aefeSPatrick Mooney nd->nuse = nd->nsze; 4274c87aefeSPatrick Mooney 4284c87aefeSPatrick Mooney /* Get LBA and backstore information from backing store */ 4294c87aefeSPatrick Mooney nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ 430*84659b24SMichael Zeller nd->flbas = 0; 431*84659b24SMichael Zeller 432*84659b24SMichael Zeller /* Create an EUI-64 if user did not provide one */ 433*84659b24SMichael Zeller if (eui64 == 0) { 434*84659b24SMichael Zeller char *data = NULL; 435*84659b24SMichael Zeller 436*84659b24SMichael Zeller asprintf(&data, "%s%u%u%u", vmname, sc->nsc_pi->pi_bus, 437*84659b24SMichael Zeller sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 438*84659b24SMichael Zeller 439*84659b24SMichael Zeller if (data != NULL) { 440*84659b24SMichael Zeller eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data)); 441*84659b24SMichael Zeller free(data); 442*84659b24SMichael Zeller } 443*84659b24SMichael Zeller eui64 = (eui64 << 16) | (nsid & 0xffff); 444*84659b24SMichael Zeller } 445*84659b24SMichael Zeller be64enc(nd->eui64, eui64); 446*84659b24SMichael Zeller 4474c87aefeSPatrick Mooney /* LBA data-sz = 2^lbads */ 4484c87aefeSPatrick Mooney nd->lbaf[0] = sc->nvstore.sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT; 4494c87aefeSPatrick Mooney } 4504c87aefeSPatrick Mooney 4514c87aefeSPatrick Mooney static void 4524c87aefeSPatrick Mooney pci_nvme_init_logpages(struct pci_nvme_softc *sc) 4534c87aefeSPatrick Mooney { 4544c87aefeSPatrick Mooney 4554c87aefeSPatrick Mooney memset(&sc->err_log, 0, sizeof(sc->err_log)); 4564c87aefeSPatrick Mooney memset(&sc->health_log, 0, sizeof(sc->health_log)); 4574c87aefeSPatrick Mooney memset(&sc->fw_log, 0, sizeof(sc->fw_log)); 4584c87aefeSPatrick Mooney } 4594c87aefeSPatrick Mooney 4604c87aefeSPatrick Mooney static void 4614c87aefeSPatrick Mooney pci_nvme_reset_locked(struct pci_nvme_softc *sc) 4624c87aefeSPatrick Mooney { 4634c87aefeSPatrick Mooney DPRINTF(("%s\r\n", __func__)); 4644c87aefeSPatrick Mooney 4654c87aefeSPatrick Mooney sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | 4664c87aefeSPatrick Mooney (1 << NVME_CAP_LO_REG_CQR_SHIFT) | 4674c87aefeSPatrick Mooney (60 << NVME_CAP_LO_REG_TO_SHIFT); 4684c87aefeSPatrick Mooney 4694c87aefeSPatrick Mooney sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT; 4704c87aefeSPatrick Mooney 4714c87aefeSPatrick Mooney sc->regs.vs = 0x00010300; /* NVMe v1.3 */ 4724c87aefeSPatrick Mooney 4734c87aefeSPatrick Mooney sc->regs.cc = 0; 4744c87aefeSPatrick Mooney sc->regs.csts = 0; 4754c87aefeSPatrick Mooney 4764c87aefeSPatrick Mooney sc->num_cqueues = sc->num_squeues = sc->max_queues; 4774c87aefeSPatrick Mooney if (sc->submit_queues != NULL) { 4784c87aefeSPatrick Mooney for (int i = 0; i < sc->num_squeues + 1; i++) { 4794c87aefeSPatrick Mooney /* 4804c87aefeSPatrick Mooney * The Admin Submission Queue is at index 0. 4814c87aefeSPatrick Mooney * It must not be changed at reset otherwise the 4824c87aefeSPatrick Mooney * emulation will be out of sync with the guest. 4834c87aefeSPatrick Mooney */ 4844c87aefeSPatrick Mooney if (i != 0) { 4854c87aefeSPatrick Mooney sc->submit_queues[i].qbase = NULL; 4864c87aefeSPatrick Mooney sc->submit_queues[i].size = 0; 4874c87aefeSPatrick Mooney sc->submit_queues[i].cqid = 0; 4884c87aefeSPatrick Mooney } 4894c87aefeSPatrick Mooney sc->submit_queues[i].tail = 0; 4904c87aefeSPatrick Mooney sc->submit_queues[i].head = 0; 4914c87aefeSPatrick Mooney sc->submit_queues[i].busy = 0; 4924c87aefeSPatrick Mooney } 4934c87aefeSPatrick Mooney } else 4944c87aefeSPatrick Mooney sc->submit_queues = calloc(sc->num_squeues + 1, 4954c87aefeSPatrick Mooney sizeof(struct nvme_submission_queue)); 4964c87aefeSPatrick Mooney 4974c87aefeSPatrick Mooney if (sc->compl_queues != NULL) { 4984c87aefeSPatrick Mooney for (int i = 0; i < sc->num_cqueues + 1; i++) { 4994c87aefeSPatrick Mooney /* See Admin Submission Queue note above */ 5004c87aefeSPatrick Mooney if (i != 0) { 5014c87aefeSPatrick Mooney sc->compl_queues[i].qbase = NULL; 5024c87aefeSPatrick Mooney sc->compl_queues[i].size = 0; 5034c87aefeSPatrick Mooney } 5044c87aefeSPatrick Mooney 5054c87aefeSPatrick Mooney sc->compl_queues[i].tail = 0; 5064c87aefeSPatrick Mooney sc->compl_queues[i].head = 0; 5074c87aefeSPatrick Mooney } 5084c87aefeSPatrick Mooney } else { 5094c87aefeSPatrick Mooney sc->compl_queues = calloc(sc->num_cqueues + 1, 5104c87aefeSPatrick Mooney sizeof(struct nvme_completion_queue)); 5114c87aefeSPatrick Mooney 5124c87aefeSPatrick Mooney for (int i = 0; i < sc->num_cqueues + 1; i++) 5134c87aefeSPatrick Mooney pthread_mutex_init(&sc->compl_queues[i].mtx, NULL); 5144c87aefeSPatrick Mooney } 5154c87aefeSPatrick Mooney } 5164c87aefeSPatrick Mooney 5174c87aefeSPatrick Mooney static void 5184c87aefeSPatrick Mooney pci_nvme_reset(struct pci_nvme_softc *sc) 5194c87aefeSPatrick Mooney { 5204c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 5214c87aefeSPatrick Mooney pci_nvme_reset_locked(sc); 5224c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 5234c87aefeSPatrick Mooney } 5244c87aefeSPatrick Mooney 5254c87aefeSPatrick Mooney static void 5264c87aefeSPatrick Mooney pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc) 5274c87aefeSPatrick Mooney { 5284c87aefeSPatrick Mooney uint16_t acqs, asqs; 5294c87aefeSPatrick Mooney 5304c87aefeSPatrick Mooney DPRINTF(("%s\r\n", __func__)); 5314c87aefeSPatrick Mooney 5324c87aefeSPatrick Mooney asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1; 5334c87aefeSPatrick Mooney sc->submit_queues[0].size = asqs; 5344c87aefeSPatrick Mooney sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq, 5354c87aefeSPatrick Mooney sizeof(struct nvme_command) * asqs); 5364c87aefeSPatrick Mooney 5374c87aefeSPatrick Mooney DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p\r\n", 5384c87aefeSPatrick Mooney __func__, sc->regs.asq, sc->submit_queues[0].qbase)); 5394c87aefeSPatrick Mooney 5404c87aefeSPatrick Mooney acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & 5414c87aefeSPatrick Mooney NVME_AQA_REG_ACQS_MASK) + 1; 5424c87aefeSPatrick Mooney sc->compl_queues[0].size = acqs; 5434c87aefeSPatrick Mooney sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq, 5444c87aefeSPatrick Mooney sizeof(struct nvme_completion) * acqs); 5454c87aefeSPatrick Mooney DPRINTF(("%s mapping Admin-CQ guest 0x%lx, host: %p\r\n", 5464c87aefeSPatrick Mooney __func__, sc->regs.acq, sc->compl_queues[0].qbase)); 5474c87aefeSPatrick Mooney } 5484c87aefeSPatrick Mooney 5494c87aefeSPatrick Mooney static int 5504c87aefeSPatrick Mooney nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *src, 5514c87aefeSPatrick Mooney size_t len) 5524c87aefeSPatrick Mooney { 5534c87aefeSPatrick Mooney uint8_t *dst; 5544c87aefeSPatrick Mooney size_t bytes; 5554c87aefeSPatrick Mooney 5564c87aefeSPatrick Mooney if (len > (8 * 1024)) { 5574c87aefeSPatrick Mooney return (-1); 5584c87aefeSPatrick Mooney } 5594c87aefeSPatrick Mooney 5604c87aefeSPatrick Mooney /* Copy from the start of prp1 to the end of the physical page */ 5614c87aefeSPatrick Mooney bytes = PAGE_SIZE - (prp1 & PAGE_MASK); 5624c87aefeSPatrick Mooney bytes = MIN(bytes, len); 5634c87aefeSPatrick Mooney 5644c87aefeSPatrick Mooney dst = vm_map_gpa(ctx, prp1, bytes); 5654c87aefeSPatrick Mooney if (dst == NULL) { 5664c87aefeSPatrick Mooney return (-1); 5674c87aefeSPatrick Mooney } 5684c87aefeSPatrick Mooney 5694c87aefeSPatrick Mooney memcpy(dst, src, bytes); 5704c87aefeSPatrick Mooney 5714c87aefeSPatrick Mooney src += bytes; 5724c87aefeSPatrick Mooney 5734c87aefeSPatrick Mooney len -= bytes; 5744c87aefeSPatrick Mooney if (len == 0) { 5754c87aefeSPatrick Mooney return (0); 5764c87aefeSPatrick Mooney } 5774c87aefeSPatrick Mooney 5784c87aefeSPatrick Mooney len = MIN(len, PAGE_SIZE); 5794c87aefeSPatrick Mooney 5804c87aefeSPatrick Mooney dst = vm_map_gpa(ctx, prp2, len); 5814c87aefeSPatrick Mooney if (dst == NULL) { 5824c87aefeSPatrick Mooney return (-1); 5834c87aefeSPatrick Mooney } 5844c87aefeSPatrick Mooney 5854c87aefeSPatrick Mooney memcpy(dst, src, len); 5864c87aefeSPatrick Mooney 5874c87aefeSPatrick Mooney return (0); 5884c87aefeSPatrick Mooney } 5894c87aefeSPatrick Mooney 5904c87aefeSPatrick Mooney static int 5914c87aefeSPatrick Mooney nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 5924c87aefeSPatrick Mooney struct nvme_completion* compl) 5934c87aefeSPatrick Mooney { 5944c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 5954c87aefeSPatrick Mooney 5964c87aefeSPatrick Mooney DPRINTF(("%s DELETE_IO_SQ %u\r\n", __func__, qid)); 5974c87aefeSPatrick Mooney if (qid == 0 || qid > sc->num_squeues) { 5984c87aefeSPatrick Mooney WPRINTF(("%s NOT PERMITTED queue id %u / num_squeues %u\r\n", 5994c87aefeSPatrick Mooney __func__, qid, sc->num_squeues)); 6004c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 6014c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 6024c87aefeSPatrick Mooney return (1); 6034c87aefeSPatrick Mooney } 6044c87aefeSPatrick Mooney 6054c87aefeSPatrick Mooney sc->submit_queues[qid].qbase = NULL; 6064c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 6074c87aefeSPatrick Mooney return (1); 6084c87aefeSPatrick Mooney } 6094c87aefeSPatrick Mooney 6104c87aefeSPatrick Mooney static int 6114c87aefeSPatrick Mooney nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 6124c87aefeSPatrick Mooney struct nvme_completion* compl) 6134c87aefeSPatrick Mooney { 6144c87aefeSPatrick Mooney if (command->cdw11 & NVME_CMD_CDW11_PC) { 6154c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 6164c87aefeSPatrick Mooney struct nvme_submission_queue *nsq; 6174c87aefeSPatrick Mooney 6184c87aefeSPatrick Mooney if ((qid == 0) || (qid > sc->num_squeues)) { 6194c87aefeSPatrick Mooney WPRINTF(("%s queue index %u > num_squeues %u\r\n", 6204c87aefeSPatrick Mooney __func__, qid, sc->num_squeues)); 6214c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, 6224c87aefeSPatrick Mooney NVME_SCT_COMMAND_SPECIFIC, 6234c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 6244c87aefeSPatrick Mooney return (1); 6254c87aefeSPatrick Mooney } 6264c87aefeSPatrick Mooney 6274c87aefeSPatrick Mooney nsq = &sc->submit_queues[qid]; 6284c87aefeSPatrick Mooney nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 6294c87aefeSPatrick Mooney 6304c87aefeSPatrick Mooney nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 6314c87aefeSPatrick Mooney sizeof(struct nvme_command) * (size_t)nsq->size); 6324c87aefeSPatrick Mooney nsq->cqid = (command->cdw11 >> 16) & 0xffff; 6334c87aefeSPatrick Mooney nsq->qpriority = (command->cdw11 >> 1) & 0x03; 6344c87aefeSPatrick Mooney 6354c87aefeSPatrick Mooney DPRINTF(("%s sq %u size %u gaddr %p cqid %u\r\n", __func__, 6364c87aefeSPatrick Mooney qid, nsq->size, nsq->qbase, nsq->cqid)); 6374c87aefeSPatrick Mooney 6384c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 6394c87aefeSPatrick Mooney 6404c87aefeSPatrick Mooney DPRINTF(("%s completed creating IOSQ qid %u\r\n", 6414c87aefeSPatrick Mooney __func__, qid)); 6424c87aefeSPatrick Mooney } else { 6434c87aefeSPatrick Mooney /* 6444c87aefeSPatrick Mooney * Guest sent non-cont submission queue request. 6454c87aefeSPatrick Mooney * This setting is unsupported by this emulation. 6464c87aefeSPatrick Mooney */ 6474c87aefeSPatrick Mooney WPRINTF(("%s unsupported non-contig (list-based) " 6484c87aefeSPatrick Mooney "create i/o submission queue\r\n", __func__)); 6494c87aefeSPatrick Mooney 6504c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 6514c87aefeSPatrick Mooney } 6524c87aefeSPatrick Mooney return (1); 6534c87aefeSPatrick Mooney } 6544c87aefeSPatrick Mooney 6554c87aefeSPatrick Mooney static int 6564c87aefeSPatrick Mooney nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 6574c87aefeSPatrick Mooney struct nvme_completion* compl) 6584c87aefeSPatrick Mooney { 6594c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 6604c87aefeSPatrick Mooney 6614c87aefeSPatrick Mooney DPRINTF(("%s DELETE_IO_CQ %u\r\n", __func__, qid)); 6624c87aefeSPatrick Mooney if (qid == 0 || qid > sc->num_cqueues) { 6634c87aefeSPatrick Mooney WPRINTF(("%s queue index %u / num_cqueues %u\r\n", 6644c87aefeSPatrick Mooney __func__, qid, sc->num_cqueues)); 6654c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 6664c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 6674c87aefeSPatrick Mooney return (1); 6684c87aefeSPatrick Mooney } 6694c87aefeSPatrick Mooney 6704c87aefeSPatrick Mooney sc->compl_queues[qid].qbase = NULL; 6714c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 6724c87aefeSPatrick Mooney return (1); 6734c87aefeSPatrick Mooney } 6744c87aefeSPatrick Mooney 6754c87aefeSPatrick Mooney static int 6764c87aefeSPatrick Mooney nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 6774c87aefeSPatrick Mooney struct nvme_completion* compl) 6784c87aefeSPatrick Mooney { 6794c87aefeSPatrick Mooney if (command->cdw11 & NVME_CMD_CDW11_PC) { 6804c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 6814c87aefeSPatrick Mooney struct nvme_completion_queue *ncq; 6824c87aefeSPatrick Mooney 6834c87aefeSPatrick Mooney if ((qid == 0) || (qid > sc->num_cqueues)) { 6844c87aefeSPatrick Mooney WPRINTF(("%s queue index %u > num_cqueues %u\r\n", 6854c87aefeSPatrick Mooney __func__, qid, sc->num_cqueues)); 6864c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, 6874c87aefeSPatrick Mooney NVME_SCT_COMMAND_SPECIFIC, 6884c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 6894c87aefeSPatrick Mooney return (1); 6904c87aefeSPatrick Mooney } 6914c87aefeSPatrick Mooney 6924c87aefeSPatrick Mooney ncq = &sc->compl_queues[qid]; 6934c87aefeSPatrick Mooney ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; 6944c87aefeSPatrick Mooney ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; 6954c87aefeSPatrick Mooney ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 6964c87aefeSPatrick Mooney 6974c87aefeSPatrick Mooney ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 6984c87aefeSPatrick Mooney command->prp1, 6994c87aefeSPatrick Mooney sizeof(struct nvme_command) * (size_t)ncq->size); 7004c87aefeSPatrick Mooney 7014c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 7024c87aefeSPatrick Mooney } else { 7034c87aefeSPatrick Mooney /* 7044c87aefeSPatrick Mooney * Non-contig completion queue unsupported. 7054c87aefeSPatrick Mooney */ 7064c87aefeSPatrick Mooney WPRINTF(("%s unsupported non-contig (list-based) " 7074c87aefeSPatrick Mooney "create i/o completion queue\r\n", 7084c87aefeSPatrick Mooney __func__)); 7094c87aefeSPatrick Mooney 7104c87aefeSPatrick Mooney /* 0x12 = Invalid Use of Controller Memory Buffer */ 7114c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 0x12); 7124c87aefeSPatrick Mooney } 7134c87aefeSPatrick Mooney 7144c87aefeSPatrick Mooney return (1); 7154c87aefeSPatrick Mooney } 7164c87aefeSPatrick Mooney 7174c87aefeSPatrick Mooney static int 7184c87aefeSPatrick Mooney nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, 7194c87aefeSPatrick Mooney struct nvme_completion* compl) 7204c87aefeSPatrick Mooney { 7214c87aefeSPatrick Mooney uint32_t logsize = (1 + ((command->cdw10 >> 16) & 0xFFF)) * 2; 7224c87aefeSPatrick Mooney uint8_t logpage = command->cdw10 & 0xFF; 7234c87aefeSPatrick Mooney 7244c87aefeSPatrick Mooney DPRINTF(("%s log page %u len %u\r\n", __func__, logpage, logsize)); 7254c87aefeSPatrick Mooney 7264c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 7274c87aefeSPatrick Mooney 7284c87aefeSPatrick Mooney switch (logpage) { 7294c87aefeSPatrick Mooney case NVME_LOG_ERROR: 7304c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 7314c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->err_log, logsize); 7324c87aefeSPatrick Mooney break; 7334c87aefeSPatrick Mooney case NVME_LOG_HEALTH_INFORMATION: 7344c87aefeSPatrick Mooney /* TODO: present some smart info */ 7354c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 7364c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->health_log, logsize); 7374c87aefeSPatrick Mooney break; 7384c87aefeSPatrick Mooney case NVME_LOG_FIRMWARE_SLOT: 7394c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 7404c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->fw_log, logsize); 7414c87aefeSPatrick Mooney break; 7424c87aefeSPatrick Mooney default: 7434c87aefeSPatrick Mooney WPRINTF(("%s get log page %x command not supported\r\n", 7444c87aefeSPatrick Mooney __func__, logpage)); 7454c87aefeSPatrick Mooney 7464c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 7474c87aefeSPatrick Mooney NVME_SC_INVALID_LOG_PAGE); 7484c87aefeSPatrick Mooney } 7494c87aefeSPatrick Mooney 7504c87aefeSPatrick Mooney return (1); 7514c87aefeSPatrick Mooney } 7524c87aefeSPatrick Mooney 7534c87aefeSPatrick Mooney static int 7544c87aefeSPatrick Mooney nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, 7554c87aefeSPatrick Mooney struct nvme_completion* compl) 7564c87aefeSPatrick Mooney { 7574c87aefeSPatrick Mooney void *dest; 7584c87aefeSPatrick Mooney 7594c87aefeSPatrick Mooney DPRINTF(("%s identify 0x%x nsid 0x%x\r\n", __func__, 7604c87aefeSPatrick Mooney command->cdw10 & 0xFF, command->nsid)); 7614c87aefeSPatrick Mooney 7624c87aefeSPatrick Mooney switch (command->cdw10 & 0xFF) { 7634c87aefeSPatrick Mooney case 0x00: /* return Identify Namespace data structure */ 7644c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 7654c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata)); 7664c87aefeSPatrick Mooney break; 7674c87aefeSPatrick Mooney case 0x01: /* return Identify Controller data structure */ 7684c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 7694c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->ctrldata, 7704c87aefeSPatrick Mooney sizeof(sc->ctrldata)); 7714c87aefeSPatrick Mooney break; 7724c87aefeSPatrick Mooney case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ 7734c87aefeSPatrick Mooney dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 7744c87aefeSPatrick Mooney sizeof(uint32_t) * 1024); 7754c87aefeSPatrick Mooney ((uint32_t *)dest)[0] = 1; 7764c87aefeSPatrick Mooney ((uint32_t *)dest)[1] = 0; 7774c87aefeSPatrick Mooney break; 7784c87aefeSPatrick Mooney case 0x11: 7794c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 7804c87aefeSPatrick Mooney NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 7814c87aefeSPatrick Mooney return (1); 7824c87aefeSPatrick Mooney case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ 7834c87aefeSPatrick Mooney case 0x10: 7844c87aefeSPatrick Mooney case 0x12: 7854c87aefeSPatrick Mooney case 0x13: 7864c87aefeSPatrick Mooney case 0x14: 7874c87aefeSPatrick Mooney case 0x15: 7884c87aefeSPatrick Mooney default: 7894c87aefeSPatrick Mooney DPRINTF(("%s unsupported identify command requested 0x%x\r\n", 7904c87aefeSPatrick Mooney __func__, command->cdw10 & 0xFF)); 7914c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 7924c87aefeSPatrick Mooney return (1); 7934c87aefeSPatrick Mooney } 7944c87aefeSPatrick Mooney 7954c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 7964c87aefeSPatrick Mooney return (1); 7974c87aefeSPatrick Mooney } 7984c87aefeSPatrick Mooney 7994c87aefeSPatrick Mooney static int 8004c87aefeSPatrick Mooney nvme_set_feature_queues(struct pci_nvme_softc* sc, struct nvme_command* command, 8014c87aefeSPatrick Mooney struct nvme_completion* compl) 8024c87aefeSPatrick Mooney { 8034c87aefeSPatrick Mooney uint16_t nqr; /* Number of Queues Requested */ 8044c87aefeSPatrick Mooney 8054c87aefeSPatrick Mooney nqr = command->cdw11 & 0xFFFF; 8064c87aefeSPatrick Mooney if (nqr == 0xffff) { 8074c87aefeSPatrick Mooney WPRINTF(("%s: Illegal NSQR value %#x\n", __func__, nqr)); 8084c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 8094c87aefeSPatrick Mooney return (-1); 8104c87aefeSPatrick Mooney } 8114c87aefeSPatrick Mooney 8124c87aefeSPatrick Mooney sc->num_squeues = ONE_BASED(nqr); 8134c87aefeSPatrick Mooney if (sc->num_squeues > sc->max_queues) { 8144c87aefeSPatrick Mooney DPRINTF(("NSQR=%u is greater than max %u\n", sc->num_squeues, 8154c87aefeSPatrick Mooney sc->max_queues)); 8164c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues; 8174c87aefeSPatrick Mooney } 8184c87aefeSPatrick Mooney 8194c87aefeSPatrick Mooney nqr = (command->cdw11 >> 16) & 0xFFFF; 8204c87aefeSPatrick Mooney if (nqr == 0xffff) { 8214c87aefeSPatrick Mooney WPRINTF(("%s: Illegal NCQR value %#x\n", __func__, nqr)); 8224c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 8234c87aefeSPatrick Mooney return (-1); 8244c87aefeSPatrick Mooney } 8254c87aefeSPatrick Mooney 8264c87aefeSPatrick Mooney sc->num_cqueues = ONE_BASED(nqr); 8274c87aefeSPatrick Mooney if (sc->num_cqueues > sc->max_queues) { 8284c87aefeSPatrick Mooney DPRINTF(("NCQR=%u is greater than max %u\n", sc->num_cqueues, 8294c87aefeSPatrick Mooney sc->max_queues)); 8304c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues; 8314c87aefeSPatrick Mooney } 8324c87aefeSPatrick Mooney 8334c87aefeSPatrick Mooney compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 8344c87aefeSPatrick Mooney 8354c87aefeSPatrick Mooney return (0); 8364c87aefeSPatrick Mooney } 8374c87aefeSPatrick Mooney 8384c87aefeSPatrick Mooney static int 8394c87aefeSPatrick Mooney nvme_opc_set_features(struct pci_nvme_softc* sc, struct nvme_command* command, 8404c87aefeSPatrick Mooney struct nvme_completion* compl) 8414c87aefeSPatrick Mooney { 8424c87aefeSPatrick Mooney int feature = command->cdw10 & 0xFF; 8434c87aefeSPatrick Mooney uint32_t iv; 8444c87aefeSPatrick Mooney 8454c87aefeSPatrick Mooney DPRINTF(("%s feature 0x%x\r\n", __func__, feature)); 8464c87aefeSPatrick Mooney compl->cdw0 = 0; 8474c87aefeSPatrick Mooney 8484c87aefeSPatrick Mooney switch (feature) { 8494c87aefeSPatrick Mooney case NVME_FEAT_ARBITRATION: 8504c87aefeSPatrick Mooney DPRINTF((" arbitration 0x%x\r\n", command->cdw11)); 8514c87aefeSPatrick Mooney break; 8524c87aefeSPatrick Mooney case NVME_FEAT_POWER_MANAGEMENT: 8534c87aefeSPatrick Mooney DPRINTF((" power management 0x%x\r\n", command->cdw11)); 8544c87aefeSPatrick Mooney break; 8554c87aefeSPatrick Mooney case NVME_FEAT_LBA_RANGE_TYPE: 8564c87aefeSPatrick Mooney DPRINTF((" lba range 0x%x\r\n", command->cdw11)); 8574c87aefeSPatrick Mooney break; 8584c87aefeSPatrick Mooney case NVME_FEAT_TEMPERATURE_THRESHOLD: 8594c87aefeSPatrick Mooney DPRINTF((" temperature threshold 0x%x\r\n", command->cdw11)); 8604c87aefeSPatrick Mooney break; 8614c87aefeSPatrick Mooney case NVME_FEAT_ERROR_RECOVERY: 8624c87aefeSPatrick Mooney DPRINTF((" error recovery 0x%x\r\n", command->cdw11)); 8634c87aefeSPatrick Mooney break; 8644c87aefeSPatrick Mooney case NVME_FEAT_VOLATILE_WRITE_CACHE: 8654c87aefeSPatrick Mooney DPRINTF((" volatile write cache 0x%x\r\n", command->cdw11)); 8664c87aefeSPatrick Mooney break; 8674c87aefeSPatrick Mooney case NVME_FEAT_NUMBER_OF_QUEUES: 8684c87aefeSPatrick Mooney nvme_set_feature_queues(sc, command, compl); 8694c87aefeSPatrick Mooney break; 8704c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_COALESCING: 8714c87aefeSPatrick Mooney DPRINTF((" interrupt coalescing 0x%x\r\n", command->cdw11)); 8724c87aefeSPatrick Mooney 8734c87aefeSPatrick Mooney /* in uS */ 8744c87aefeSPatrick Mooney sc->intr_coales_aggr_time = ((command->cdw11 >> 8) & 0xFF)*100; 8754c87aefeSPatrick Mooney 8764c87aefeSPatrick Mooney sc->intr_coales_aggr_thresh = command->cdw11 & 0xFF; 8774c87aefeSPatrick Mooney break; 8784c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 8794c87aefeSPatrick Mooney iv = command->cdw11 & 0xFFFF; 8804c87aefeSPatrick Mooney 8814c87aefeSPatrick Mooney DPRINTF((" interrupt vector configuration 0x%x\r\n", 8824c87aefeSPatrick Mooney command->cdw11)); 8834c87aefeSPatrick Mooney 8844c87aefeSPatrick Mooney for (uint32_t i = 0; i < sc->num_cqueues + 1; i++) { 8854c87aefeSPatrick Mooney if (sc->compl_queues[i].intr_vec == iv) { 8864c87aefeSPatrick Mooney if (command->cdw11 & (1 << 16)) 8874c87aefeSPatrick Mooney sc->compl_queues[i].intr_en |= 8884c87aefeSPatrick Mooney NVME_CQ_INTCOAL; 8894c87aefeSPatrick Mooney else 8904c87aefeSPatrick Mooney sc->compl_queues[i].intr_en &= 8914c87aefeSPatrick Mooney ~NVME_CQ_INTCOAL; 8924c87aefeSPatrick Mooney } 8934c87aefeSPatrick Mooney } 8944c87aefeSPatrick Mooney break; 8954c87aefeSPatrick Mooney case NVME_FEAT_WRITE_ATOMICITY: 8964c87aefeSPatrick Mooney DPRINTF((" write atomicity 0x%x\r\n", command->cdw11)); 8974c87aefeSPatrick Mooney break; 8984c87aefeSPatrick Mooney case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 8994c87aefeSPatrick Mooney DPRINTF((" async event configuration 0x%x\r\n", 9004c87aefeSPatrick Mooney command->cdw11)); 9014c87aefeSPatrick Mooney sc->async_ev_config = command->cdw11; 9024c87aefeSPatrick Mooney break; 9034c87aefeSPatrick Mooney case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 9044c87aefeSPatrick Mooney DPRINTF((" software progress marker 0x%x\r\n", 9054c87aefeSPatrick Mooney command->cdw11)); 9064c87aefeSPatrick Mooney break; 9074c87aefeSPatrick Mooney case 0x0C: 9084c87aefeSPatrick Mooney DPRINTF((" autonomous power state transition 0x%x\r\n", 9094c87aefeSPatrick Mooney command->cdw11)); 9104c87aefeSPatrick Mooney break; 9114c87aefeSPatrick Mooney default: 9124c87aefeSPatrick Mooney WPRINTF(("%s invalid feature\r\n", __func__)); 9134c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 9144c87aefeSPatrick Mooney return (1); 9154c87aefeSPatrick Mooney } 9164c87aefeSPatrick Mooney 9174c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 9184c87aefeSPatrick Mooney return (1); 9194c87aefeSPatrick Mooney } 9204c87aefeSPatrick Mooney 9214c87aefeSPatrick Mooney static int 9224c87aefeSPatrick Mooney nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, 9234c87aefeSPatrick Mooney struct nvme_completion* compl) 9244c87aefeSPatrick Mooney { 9254c87aefeSPatrick Mooney int feature = command->cdw10 & 0xFF; 9264c87aefeSPatrick Mooney 9274c87aefeSPatrick Mooney DPRINTF(("%s feature 0x%x\r\n", __func__, feature)); 9284c87aefeSPatrick Mooney 9294c87aefeSPatrick Mooney compl->cdw0 = 0; 9304c87aefeSPatrick Mooney 9314c87aefeSPatrick Mooney switch (feature) { 9324c87aefeSPatrick Mooney case NVME_FEAT_ARBITRATION: 9334c87aefeSPatrick Mooney DPRINTF((" arbitration\r\n")); 9344c87aefeSPatrick Mooney break; 9354c87aefeSPatrick Mooney case NVME_FEAT_POWER_MANAGEMENT: 9364c87aefeSPatrick Mooney DPRINTF((" power management\r\n")); 9374c87aefeSPatrick Mooney break; 9384c87aefeSPatrick Mooney case NVME_FEAT_LBA_RANGE_TYPE: 9394c87aefeSPatrick Mooney DPRINTF((" lba range\r\n")); 9404c87aefeSPatrick Mooney break; 9414c87aefeSPatrick Mooney case NVME_FEAT_TEMPERATURE_THRESHOLD: 9424c87aefeSPatrick Mooney DPRINTF((" temperature threshold\r\n")); 9434c87aefeSPatrick Mooney switch ((command->cdw11 >> 20) & 0x3) { 9444c87aefeSPatrick Mooney case 0: 9454c87aefeSPatrick Mooney /* Over temp threshold */ 9464c87aefeSPatrick Mooney compl->cdw0 = 0xFFFF; 9474c87aefeSPatrick Mooney break; 9484c87aefeSPatrick Mooney case 1: 9494c87aefeSPatrick Mooney /* Under temp threshold */ 9504c87aefeSPatrick Mooney compl->cdw0 = 0; 9514c87aefeSPatrick Mooney break; 9524c87aefeSPatrick Mooney default: 9534c87aefeSPatrick Mooney WPRINTF((" invalid threshold type select\r\n")); 9544c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 9554c87aefeSPatrick Mooney NVME_SC_INVALID_FIELD); 9564c87aefeSPatrick Mooney return (1); 9574c87aefeSPatrick Mooney } 9584c87aefeSPatrick Mooney break; 9594c87aefeSPatrick Mooney case NVME_FEAT_ERROR_RECOVERY: 9604c87aefeSPatrick Mooney DPRINTF((" error recovery\r\n")); 9614c87aefeSPatrick Mooney break; 9624c87aefeSPatrick Mooney case NVME_FEAT_VOLATILE_WRITE_CACHE: 9634c87aefeSPatrick Mooney DPRINTF((" volatile write cache\r\n")); 9644c87aefeSPatrick Mooney break; 9654c87aefeSPatrick Mooney case NVME_FEAT_NUMBER_OF_QUEUES: 9664c87aefeSPatrick Mooney compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 9674c87aefeSPatrick Mooney 9684c87aefeSPatrick Mooney DPRINTF((" number of queues (submit %u, completion %u)\r\n", 9694c87aefeSPatrick Mooney compl->cdw0 & 0xFFFF, 9704c87aefeSPatrick Mooney (compl->cdw0 >> 16) & 0xFFFF)); 9714c87aefeSPatrick Mooney 9724c87aefeSPatrick Mooney break; 9734c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_COALESCING: 9744c87aefeSPatrick Mooney DPRINTF((" interrupt coalescing\r\n")); 9754c87aefeSPatrick Mooney break; 9764c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 9774c87aefeSPatrick Mooney DPRINTF((" interrupt vector configuration\r\n")); 9784c87aefeSPatrick Mooney break; 9794c87aefeSPatrick Mooney case NVME_FEAT_WRITE_ATOMICITY: 9804c87aefeSPatrick Mooney DPRINTF((" write atomicity\r\n")); 9814c87aefeSPatrick Mooney break; 9824c87aefeSPatrick Mooney case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 9834c87aefeSPatrick Mooney DPRINTF((" async event configuration\r\n")); 9844c87aefeSPatrick Mooney sc->async_ev_config = command->cdw11; 9854c87aefeSPatrick Mooney break; 9864c87aefeSPatrick Mooney case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 9874c87aefeSPatrick Mooney DPRINTF((" software progress marker\r\n")); 9884c87aefeSPatrick Mooney break; 9894c87aefeSPatrick Mooney case 0x0C: 9904c87aefeSPatrick Mooney DPRINTF((" autonomous power state transition\r\n")); 9914c87aefeSPatrick Mooney break; 9924c87aefeSPatrick Mooney default: 9934c87aefeSPatrick Mooney WPRINTF(("%s invalid feature 0x%x\r\n", __func__, feature)); 9944c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 9954c87aefeSPatrick Mooney return (1); 9964c87aefeSPatrick Mooney } 9974c87aefeSPatrick Mooney 9984c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 9994c87aefeSPatrick Mooney return (1); 10004c87aefeSPatrick Mooney } 10014c87aefeSPatrick Mooney 10024c87aefeSPatrick Mooney static int 10034c87aefeSPatrick Mooney nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command, 10044c87aefeSPatrick Mooney struct nvme_completion* compl) 10054c87aefeSPatrick Mooney { 10064c87aefeSPatrick Mooney DPRINTF(("%s submission queue %u, command ID 0x%x\r\n", __func__, 10074c87aefeSPatrick Mooney command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF)); 10084c87aefeSPatrick Mooney 10094c87aefeSPatrick Mooney /* TODO: search for the command ID and abort it */ 10104c87aefeSPatrick Mooney 10114c87aefeSPatrick Mooney compl->cdw0 = 1; 10124c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 10134c87aefeSPatrick Mooney return (1); 10144c87aefeSPatrick Mooney } 10154c87aefeSPatrick Mooney 10164c87aefeSPatrick Mooney #ifdef __FreeBSD__ 10174c87aefeSPatrick Mooney static int 10184c87aefeSPatrick Mooney nvme_opc_async_event_req(struct pci_nvme_softc* sc, 10194c87aefeSPatrick Mooney struct nvme_command* command, struct nvme_completion* compl) 10204c87aefeSPatrick Mooney { 10214c87aefeSPatrick Mooney DPRINTF(("%s async event request 0x%x\r\n", __func__, command->cdw11)); 10224c87aefeSPatrick Mooney 10234c87aefeSPatrick Mooney /* 10244c87aefeSPatrick Mooney * TODO: raise events when they happen based on the Set Features cmd. 10254c87aefeSPatrick Mooney * These events happen async, so only set completion successful if 10264c87aefeSPatrick Mooney * there is an event reflective of the request to get event. 10274c87aefeSPatrick Mooney */ 10284c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 10294c87aefeSPatrick Mooney NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); 10304c87aefeSPatrick Mooney return (0); 10314c87aefeSPatrick Mooney } 10324c87aefeSPatrick Mooney #else 10334c87aefeSPatrick Mooney /* This is kept behind an ifdef while it's unused to appease the compiler. */ 10344c87aefeSPatrick Mooney #endif /* __FreeBSD__ */ 10354c87aefeSPatrick Mooney 10364c87aefeSPatrick Mooney static void 10374c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) 10384c87aefeSPatrick Mooney { 10394c87aefeSPatrick Mooney struct nvme_completion compl; 10404c87aefeSPatrick Mooney struct nvme_command *cmd; 10414c87aefeSPatrick Mooney struct nvme_submission_queue *sq; 10424c87aefeSPatrick Mooney struct nvme_completion_queue *cq; 10434c87aefeSPatrick Mooney int do_intr = 0; 10444c87aefeSPatrick Mooney uint16_t sqhead; 10454c87aefeSPatrick Mooney 10464c87aefeSPatrick Mooney DPRINTF(("%s index %u\r\n", __func__, (uint32_t)value)); 10474c87aefeSPatrick Mooney 10484c87aefeSPatrick Mooney sq = &sc->submit_queues[0]; 10494c87aefeSPatrick Mooney 10504c87aefeSPatrick Mooney sqhead = atomic_load_acq_short(&sq->head); 10514c87aefeSPatrick Mooney 10524c87aefeSPatrick Mooney if (atomic_testandset_int(&sq->busy, 1)) { 10534c87aefeSPatrick Mooney DPRINTF(("%s SQ busy, head %u, tail %u\r\n", 10544c87aefeSPatrick Mooney __func__, sqhead, sq->tail)); 10554c87aefeSPatrick Mooney return; 10564c87aefeSPatrick Mooney } 10574c87aefeSPatrick Mooney 10584c87aefeSPatrick Mooney DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail)); 10594c87aefeSPatrick Mooney 10604c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) { 10614c87aefeSPatrick Mooney cmd = &(sq->qbase)[sqhead]; 1062*84659b24SMichael Zeller compl.cdw0 = 0; 10634c87aefeSPatrick Mooney compl.status = 0; 10644c87aefeSPatrick Mooney 10654c87aefeSPatrick Mooney switch (cmd->opc) { 10664c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_SQ: 10674c87aefeSPatrick Mooney DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__)); 10684c87aefeSPatrick Mooney do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl); 10694c87aefeSPatrick Mooney break; 10704c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_SQ: 10714c87aefeSPatrick Mooney DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__)); 10724c87aefeSPatrick Mooney do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl); 10734c87aefeSPatrick Mooney break; 10744c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_CQ: 10754c87aefeSPatrick Mooney DPRINTF(("%s command DELETE_IO_CQ\r\n", __func__)); 10764c87aefeSPatrick Mooney do_intr |= nvme_opc_delete_io_cq(sc, cmd, &compl); 10774c87aefeSPatrick Mooney break; 10784c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_CQ: 10794c87aefeSPatrick Mooney DPRINTF(("%s command CREATE_IO_CQ\r\n", __func__)); 10804c87aefeSPatrick Mooney do_intr |= nvme_opc_create_io_cq(sc, cmd, &compl); 10814c87aefeSPatrick Mooney break; 10824c87aefeSPatrick Mooney case NVME_OPC_GET_LOG_PAGE: 10834c87aefeSPatrick Mooney DPRINTF(("%s command GET_LOG_PAGE\r\n", __func__)); 10844c87aefeSPatrick Mooney do_intr |= nvme_opc_get_log_page(sc, cmd, &compl); 10854c87aefeSPatrick Mooney break; 10864c87aefeSPatrick Mooney case NVME_OPC_IDENTIFY: 10874c87aefeSPatrick Mooney DPRINTF(("%s command IDENTIFY\r\n", __func__)); 10884c87aefeSPatrick Mooney do_intr |= nvme_opc_identify(sc, cmd, &compl); 10894c87aefeSPatrick Mooney break; 10904c87aefeSPatrick Mooney case NVME_OPC_ABORT: 10914c87aefeSPatrick Mooney DPRINTF(("%s command ABORT\r\n", __func__)); 10924c87aefeSPatrick Mooney do_intr |= nvme_opc_abort(sc, cmd, &compl); 10934c87aefeSPatrick Mooney break; 10944c87aefeSPatrick Mooney case NVME_OPC_SET_FEATURES: 10954c87aefeSPatrick Mooney DPRINTF(("%s command SET_FEATURES\r\n", __func__)); 10964c87aefeSPatrick Mooney do_intr |= nvme_opc_set_features(sc, cmd, &compl); 10974c87aefeSPatrick Mooney break; 10984c87aefeSPatrick Mooney case NVME_OPC_GET_FEATURES: 10994c87aefeSPatrick Mooney DPRINTF(("%s command GET_FEATURES\r\n", __func__)); 11004c87aefeSPatrick Mooney do_intr |= nvme_opc_get_features(sc, cmd, &compl); 11014c87aefeSPatrick Mooney break; 11024c87aefeSPatrick Mooney case NVME_OPC_ASYNC_EVENT_REQUEST: 11034c87aefeSPatrick Mooney DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__)); 11044c87aefeSPatrick Mooney /* XXX dont care, unhandled for now 11054c87aefeSPatrick Mooney do_intr |= nvme_opc_async_event_req(sc, cmd, &compl); 11064c87aefeSPatrick Mooney */ 1107*84659b24SMichael Zeller compl.status = NVME_NO_STATUS; 11084c87aefeSPatrick Mooney break; 11094c87aefeSPatrick Mooney default: 11104c87aefeSPatrick Mooney WPRINTF(("0x%x command is not implemented\r\n", 11114c87aefeSPatrick Mooney cmd->opc)); 1112*84659b24SMichael Zeller pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); 1113*84659b24SMichael Zeller do_intr |= 1; 11144c87aefeSPatrick Mooney } 11154c87aefeSPatrick Mooney 1116*84659b24SMichael Zeller if (NVME_COMPLETION_VALID(compl)) { 11174c87aefeSPatrick Mooney struct nvme_completion *cp; 11184c87aefeSPatrick Mooney int phase; 11194c87aefeSPatrick Mooney 11204c87aefeSPatrick Mooney cq = &sc->compl_queues[0]; 11214c87aefeSPatrick Mooney 11224c87aefeSPatrick Mooney cp = &(cq->qbase)[cq->tail]; 11234c87aefeSPatrick Mooney cp->cdw0 = compl.cdw0; 11244c87aefeSPatrick Mooney cp->sqid = 0; 11254c87aefeSPatrick Mooney cp->sqhd = sqhead; 11264c87aefeSPatrick Mooney cp->cid = cmd->cid; 11274c87aefeSPatrick Mooney 11284c87aefeSPatrick Mooney phase = NVME_STATUS_GET_P(cp->status); 11294c87aefeSPatrick Mooney cp->status = compl.status; 11304c87aefeSPatrick Mooney pci_nvme_toggle_phase(&cp->status, phase); 11314c87aefeSPatrick Mooney 11324c87aefeSPatrick Mooney cq->tail = (cq->tail + 1) % cq->size; 11334c87aefeSPatrick Mooney } 11344c87aefeSPatrick Mooney sqhead = (sqhead + 1) % sq->size; 11354c87aefeSPatrick Mooney } 11364c87aefeSPatrick Mooney 11374c87aefeSPatrick Mooney DPRINTF(("setting sqhead %u\r\n", sqhead)); 11384c87aefeSPatrick Mooney atomic_store_short(&sq->head, sqhead); 11394c87aefeSPatrick Mooney atomic_store_int(&sq->busy, 0); 11404c87aefeSPatrick Mooney 11414c87aefeSPatrick Mooney if (do_intr) 11424c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, 0); 11434c87aefeSPatrick Mooney 11444c87aefeSPatrick Mooney } 11454c87aefeSPatrick Mooney 11464c87aefeSPatrick Mooney static int 11474c87aefeSPatrick Mooney pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req, 11484c87aefeSPatrick Mooney uint64_t gpaddr, size_t size, int do_write, uint64_t lba) 11494c87aefeSPatrick Mooney { 11504c87aefeSPatrick Mooney int iovidx; 11514c87aefeSPatrick Mooney 11524c87aefeSPatrick Mooney if (req != NULL) { 11534c87aefeSPatrick Mooney /* concatenate contig block-iovs to minimize number of iovs */ 11544c87aefeSPatrick Mooney if ((req->prev_gpaddr + req->prev_size) == gpaddr) { 11554c87aefeSPatrick Mooney iovidx = req->io_req.br_iovcnt - 1; 11564c87aefeSPatrick Mooney 11574c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_base = 11584c87aefeSPatrick Mooney paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 11594c87aefeSPatrick Mooney req->prev_gpaddr, size); 11604c87aefeSPatrick Mooney 11614c87aefeSPatrick Mooney req->prev_size += size; 11624c87aefeSPatrick Mooney req->io_req.br_resid += size; 11634c87aefeSPatrick Mooney 11644c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_len = req->prev_size; 11654c87aefeSPatrick Mooney } else { 11664c87aefeSPatrick Mooney pthread_mutex_lock(&req->mtx); 11674c87aefeSPatrick Mooney 11684c87aefeSPatrick Mooney iovidx = req->io_req.br_iovcnt; 11694c87aefeSPatrick Mooney if (iovidx == NVME_MAX_BLOCKIOVS) { 11704c87aefeSPatrick Mooney int err = 0; 11714c87aefeSPatrick Mooney 11724c87aefeSPatrick Mooney DPRINTF(("large I/O, doing partial req\r\n")); 11734c87aefeSPatrick Mooney 11744c87aefeSPatrick Mooney iovidx = 0; 11754c87aefeSPatrick Mooney req->io_req.br_iovcnt = 0; 11764c87aefeSPatrick Mooney 11774c87aefeSPatrick Mooney req->io_req.br_callback = pci_nvme_io_partial; 11784c87aefeSPatrick Mooney 11794c87aefeSPatrick Mooney if (!do_write) 11804c87aefeSPatrick Mooney err = blockif_read(sc->nvstore.ctx, 11814c87aefeSPatrick Mooney &req->io_req); 11824c87aefeSPatrick Mooney else 11834c87aefeSPatrick Mooney err = blockif_write(sc->nvstore.ctx, 11844c87aefeSPatrick Mooney &req->io_req); 11854c87aefeSPatrick Mooney 11864c87aefeSPatrick Mooney /* wait until req completes before cont */ 11874c87aefeSPatrick Mooney if (err == 0) 11884c87aefeSPatrick Mooney pthread_cond_wait(&req->cv, &req->mtx); 11894c87aefeSPatrick Mooney } 11904c87aefeSPatrick Mooney if (iovidx == 0) { 11914c87aefeSPatrick Mooney req->io_req.br_offset = lba; 11924c87aefeSPatrick Mooney req->io_req.br_resid = 0; 11934c87aefeSPatrick Mooney req->io_req.br_param = req; 11944c87aefeSPatrick Mooney } 11954c87aefeSPatrick Mooney 11964c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_base = 11974c87aefeSPatrick Mooney paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 11984c87aefeSPatrick Mooney gpaddr, size); 11994c87aefeSPatrick Mooney 12004c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_len = size; 12014c87aefeSPatrick Mooney 12024c87aefeSPatrick Mooney req->prev_gpaddr = gpaddr; 12034c87aefeSPatrick Mooney req->prev_size = size; 12044c87aefeSPatrick Mooney req->io_req.br_resid += size; 12054c87aefeSPatrick Mooney 12064c87aefeSPatrick Mooney req->io_req.br_iovcnt++; 12074c87aefeSPatrick Mooney 12084c87aefeSPatrick Mooney pthread_mutex_unlock(&req->mtx); 12094c87aefeSPatrick Mooney } 12104c87aefeSPatrick Mooney } else { 12114c87aefeSPatrick Mooney /* RAM buffer: read/write directly */ 12124c87aefeSPatrick Mooney void *p = sc->nvstore.ctx; 12134c87aefeSPatrick Mooney void *gptr; 12144c87aefeSPatrick Mooney 12154c87aefeSPatrick Mooney if ((lba + size) > sc->nvstore.size) { 12164c87aefeSPatrick Mooney WPRINTF(("%s write would overflow RAM\r\n", __func__)); 12174c87aefeSPatrick Mooney return (-1); 12184c87aefeSPatrick Mooney } 12194c87aefeSPatrick Mooney 12204c87aefeSPatrick Mooney p = (void *)((uintptr_t)p + (uintptr_t)lba); 12214c87aefeSPatrick Mooney gptr = paddr_guest2host(sc->nsc_pi->pi_vmctx, gpaddr, size); 12224c87aefeSPatrick Mooney if (do_write) 12234c87aefeSPatrick Mooney memcpy(p, gptr, size); 12244c87aefeSPatrick Mooney else 12254c87aefeSPatrick Mooney memcpy(gptr, p, size); 12264c87aefeSPatrick Mooney } 12274c87aefeSPatrick Mooney return (0); 12284c87aefeSPatrick Mooney } 12294c87aefeSPatrick Mooney 12304c87aefeSPatrick Mooney static void 12314c87aefeSPatrick Mooney pci_nvme_set_completion(struct pci_nvme_softc *sc, 12324c87aefeSPatrick Mooney struct nvme_submission_queue *sq, int sqid, uint16_t cid, 12334c87aefeSPatrick Mooney uint32_t cdw0, uint16_t status, int ignore_busy) 12344c87aefeSPatrick Mooney { 12354c87aefeSPatrick Mooney struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; 12364c87aefeSPatrick Mooney struct nvme_completion *compl; 12374c87aefeSPatrick Mooney int do_intr = 0; 12384c87aefeSPatrick Mooney int phase; 12394c87aefeSPatrick Mooney 12404c87aefeSPatrick Mooney DPRINTF(("%s sqid %d cqid %u cid %u status: 0x%x 0x%x\r\n", 12414c87aefeSPatrick Mooney __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), 12424c87aefeSPatrick Mooney NVME_STATUS_GET_SC(status))); 12434c87aefeSPatrick Mooney 12444c87aefeSPatrick Mooney pthread_mutex_lock(&cq->mtx); 12454c87aefeSPatrick Mooney 12464c87aefeSPatrick Mooney assert(cq->qbase != NULL); 12474c87aefeSPatrick Mooney 12484c87aefeSPatrick Mooney compl = &cq->qbase[cq->tail]; 12494c87aefeSPatrick Mooney 12504c87aefeSPatrick Mooney compl->sqhd = atomic_load_acq_short(&sq->head); 12514c87aefeSPatrick Mooney compl->sqid = sqid; 12524c87aefeSPatrick Mooney compl->cid = cid; 12534c87aefeSPatrick Mooney 12544c87aefeSPatrick Mooney // toggle phase 12554c87aefeSPatrick Mooney phase = NVME_STATUS_GET_P(compl->status); 12564c87aefeSPatrick Mooney compl->status = status; 12574c87aefeSPatrick Mooney pci_nvme_toggle_phase(&compl->status, phase); 12584c87aefeSPatrick Mooney 12594c87aefeSPatrick Mooney cq->tail = (cq->tail + 1) % cq->size; 12604c87aefeSPatrick Mooney 12614c87aefeSPatrick Mooney if (cq->intr_en & NVME_CQ_INTEN) 12624c87aefeSPatrick Mooney do_intr = 1; 12634c87aefeSPatrick Mooney 12644c87aefeSPatrick Mooney pthread_mutex_unlock(&cq->mtx); 12654c87aefeSPatrick Mooney 12664c87aefeSPatrick Mooney if (ignore_busy || !atomic_load_acq_int(&sq->busy)) 12674c87aefeSPatrick Mooney if (do_intr) 12684c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, cq->intr_vec); 12694c87aefeSPatrick Mooney } 12704c87aefeSPatrick Mooney 12714c87aefeSPatrick Mooney static void 12724c87aefeSPatrick Mooney pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) 12734c87aefeSPatrick Mooney { 12744c87aefeSPatrick Mooney req->sc = NULL; 12754c87aefeSPatrick Mooney req->nvme_sq = NULL; 12764c87aefeSPatrick Mooney req->sqid = 0; 12774c87aefeSPatrick Mooney 12784c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 12794c87aefeSPatrick Mooney 12804c87aefeSPatrick Mooney req->next = sc->ioreqs_free; 12814c87aefeSPatrick Mooney sc->ioreqs_free = req; 12824c87aefeSPatrick Mooney sc->pending_ios--; 12834c87aefeSPatrick Mooney 12844c87aefeSPatrick Mooney /* when no more IO pending, can set to ready if device reset/enabled */ 12854c87aefeSPatrick Mooney if (sc->pending_ios == 0 && 12864c87aefeSPatrick Mooney NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) 12874c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY; 12884c87aefeSPatrick Mooney 12894c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 12904c87aefeSPatrick Mooney 12914c87aefeSPatrick Mooney sem_post(&sc->iosemlock); 12924c87aefeSPatrick Mooney } 12934c87aefeSPatrick Mooney 12944c87aefeSPatrick Mooney static struct pci_nvme_ioreq * 12954c87aefeSPatrick Mooney pci_nvme_get_ioreq(struct pci_nvme_softc *sc) 12964c87aefeSPatrick Mooney { 12974c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = NULL;; 12984c87aefeSPatrick Mooney 12994c87aefeSPatrick Mooney sem_wait(&sc->iosemlock); 13004c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 13014c87aefeSPatrick Mooney 13024c87aefeSPatrick Mooney req = sc->ioreqs_free; 13034c87aefeSPatrick Mooney assert(req != NULL); 13044c87aefeSPatrick Mooney 13054c87aefeSPatrick Mooney sc->ioreqs_free = req->next; 13064c87aefeSPatrick Mooney 13074c87aefeSPatrick Mooney req->next = NULL; 13084c87aefeSPatrick Mooney req->sc = sc; 13094c87aefeSPatrick Mooney 13104c87aefeSPatrick Mooney sc->pending_ios++; 13114c87aefeSPatrick Mooney 13124c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 13134c87aefeSPatrick Mooney 13144c87aefeSPatrick Mooney req->io_req.br_iovcnt = 0; 13154c87aefeSPatrick Mooney req->io_req.br_offset = 0; 13164c87aefeSPatrick Mooney req->io_req.br_resid = 0; 13174c87aefeSPatrick Mooney req->io_req.br_param = req; 13184c87aefeSPatrick Mooney req->prev_gpaddr = 0; 13194c87aefeSPatrick Mooney req->prev_size = 0; 13204c87aefeSPatrick Mooney 13214c87aefeSPatrick Mooney return req; 13224c87aefeSPatrick Mooney } 13234c87aefeSPatrick Mooney 13244c87aefeSPatrick Mooney static void 13254c87aefeSPatrick Mooney pci_nvme_io_done(struct blockif_req *br, int err) 13264c87aefeSPatrick Mooney { 13274c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param; 13284c87aefeSPatrick Mooney struct nvme_submission_queue *sq = req->nvme_sq; 13294c87aefeSPatrick Mooney uint16_t code, status = 0; 13304c87aefeSPatrick Mooney 13314c87aefeSPatrick Mooney DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err))); 13324c87aefeSPatrick Mooney 13334c87aefeSPatrick Mooney /* TODO return correct error */ 13344c87aefeSPatrick Mooney code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; 13354c87aefeSPatrick Mooney pci_nvme_status_genc(&status, code); 13364c87aefeSPatrick Mooney 13374c87aefeSPatrick Mooney pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0); 13384c87aefeSPatrick Mooney pci_nvme_release_ioreq(req->sc, req); 13394c87aefeSPatrick Mooney } 13404c87aefeSPatrick Mooney 13414c87aefeSPatrick Mooney static void 13424c87aefeSPatrick Mooney pci_nvme_io_partial(struct blockif_req *br, int err) 13434c87aefeSPatrick Mooney { 13444c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param; 13454c87aefeSPatrick Mooney 13464c87aefeSPatrick Mooney DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err))); 13474c87aefeSPatrick Mooney 13484c87aefeSPatrick Mooney pthread_cond_signal(&req->cv); 13494c87aefeSPatrick Mooney } 13504c87aefeSPatrick Mooney 13514c87aefeSPatrick Mooney 13524c87aefeSPatrick Mooney static void 13534c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) 13544c87aefeSPatrick Mooney { 13554c87aefeSPatrick Mooney struct nvme_submission_queue *sq; 13564c87aefeSPatrick Mooney uint16_t status = 0; 13574c87aefeSPatrick Mooney uint16_t sqhead; 13584c87aefeSPatrick Mooney int err; 13594c87aefeSPatrick Mooney 13604c87aefeSPatrick Mooney /* handle all submissions up to sq->tail index */ 13614c87aefeSPatrick Mooney sq = &sc->submit_queues[idx]; 13624c87aefeSPatrick Mooney 13634c87aefeSPatrick Mooney if (atomic_testandset_int(&sq->busy, 1)) { 13644c87aefeSPatrick Mooney DPRINTF(("%s sqid %u busy\r\n", __func__, idx)); 13654c87aefeSPatrick Mooney return; 13664c87aefeSPatrick Mooney } 13674c87aefeSPatrick Mooney 13684c87aefeSPatrick Mooney sqhead = atomic_load_acq_short(&sq->head); 13694c87aefeSPatrick Mooney 13704c87aefeSPatrick Mooney DPRINTF(("nvme_handle_io qid %u head %u tail %u cmdlist %p\r\n", 13714c87aefeSPatrick Mooney idx, sqhead, sq->tail, sq->qbase)); 13724c87aefeSPatrick Mooney 13734c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) { 13744c87aefeSPatrick Mooney struct nvme_command *cmd; 13754c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = NULL; 13764c87aefeSPatrick Mooney uint64_t lba; 13774c87aefeSPatrick Mooney uint64_t nblocks, bytes, size, cpsz; 13784c87aefeSPatrick Mooney 13794c87aefeSPatrick Mooney /* TODO: support scatter gather list handling */ 13804c87aefeSPatrick Mooney 13814c87aefeSPatrick Mooney cmd = &sq->qbase[sqhead]; 13824c87aefeSPatrick Mooney sqhead = (sqhead + 1) % sq->size; 13834c87aefeSPatrick Mooney 13844c87aefeSPatrick Mooney lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; 13854c87aefeSPatrick Mooney 13864c87aefeSPatrick Mooney if (cmd->opc == NVME_OPC_FLUSH) { 13874c87aefeSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 13884c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 13894c87aefeSPatrick Mooney status, 1); 13904c87aefeSPatrick Mooney 13914c87aefeSPatrick Mooney continue; 13924c87aefeSPatrick Mooney } else if (cmd->opc == 0x08) { 13934c87aefeSPatrick Mooney /* TODO: write zeroes */ 13944c87aefeSPatrick Mooney WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n", 13954c87aefeSPatrick Mooney __func__, lba, cmd->cdw12 & 0xFFFF)); 13964c87aefeSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 13974c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 13984c87aefeSPatrick Mooney status, 1); 13994c87aefeSPatrick Mooney 14004c87aefeSPatrick Mooney continue; 14014c87aefeSPatrick Mooney } 14024c87aefeSPatrick Mooney 14034c87aefeSPatrick Mooney nblocks = (cmd->cdw12 & 0xFFFF) + 1; 14044c87aefeSPatrick Mooney 14054c87aefeSPatrick Mooney bytes = nblocks * sc->nvstore.sectsz; 14064c87aefeSPatrick Mooney 14074c87aefeSPatrick Mooney if (sc->nvstore.type == NVME_STOR_BLOCKIF) { 14084c87aefeSPatrick Mooney req = pci_nvme_get_ioreq(sc); 14094c87aefeSPatrick Mooney req->nvme_sq = sq; 14104c87aefeSPatrick Mooney req->sqid = idx; 14114c87aefeSPatrick Mooney } 14124c87aefeSPatrick Mooney 14134c87aefeSPatrick Mooney /* 14144c87aefeSPatrick Mooney * If data starts mid-page and flows into the next page, then 14154c87aefeSPatrick Mooney * increase page count 14164c87aefeSPatrick Mooney */ 14174c87aefeSPatrick Mooney 14184c87aefeSPatrick Mooney DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu " 14194c87aefeSPatrick Mooney "(%lu-bytes)\r\n", 14204c87aefeSPatrick Mooney sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size, 14214c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE ? 14224c87aefeSPatrick Mooney "WRITE" : "READ", 14234c87aefeSPatrick Mooney lba, nblocks, bytes)); 14244c87aefeSPatrick Mooney 14254c87aefeSPatrick Mooney cmd->prp1 &= ~(0x03UL); 14264c87aefeSPatrick Mooney cmd->prp2 &= ~(0x03UL); 14274c87aefeSPatrick Mooney 14284c87aefeSPatrick Mooney DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2)); 14294c87aefeSPatrick Mooney 14304c87aefeSPatrick Mooney size = bytes; 14314c87aefeSPatrick Mooney lba *= sc->nvstore.sectsz; 14324c87aefeSPatrick Mooney 14334c87aefeSPatrick Mooney cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE); 14344c87aefeSPatrick Mooney 14354c87aefeSPatrick Mooney if (cpsz > bytes) 14364c87aefeSPatrick Mooney cpsz = bytes; 14374c87aefeSPatrick Mooney 14384c87aefeSPatrick Mooney if (req != NULL) { 14394c87aefeSPatrick Mooney req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) | 14404c87aefeSPatrick Mooney cmd->cdw10; 14414c87aefeSPatrick Mooney req->opc = cmd->opc; 14424c87aefeSPatrick Mooney req->cid = cmd->cid; 14434c87aefeSPatrick Mooney req->nsid = cmd->nsid; 14444c87aefeSPatrick Mooney } 14454c87aefeSPatrick Mooney 14464c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz, 14474c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, lba); 14484c87aefeSPatrick Mooney lba += cpsz; 14494c87aefeSPatrick Mooney size -= cpsz; 14504c87aefeSPatrick Mooney 14514c87aefeSPatrick Mooney if (size == 0) 14524c87aefeSPatrick Mooney goto iodone; 14534c87aefeSPatrick Mooney 14544c87aefeSPatrick Mooney if (size <= PAGE_SIZE) { 14554c87aefeSPatrick Mooney /* prp2 is second (and final) page in transfer */ 14564c87aefeSPatrick Mooney 14574c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, cmd->prp2, 14584c87aefeSPatrick Mooney size, 14594c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, 14604c87aefeSPatrick Mooney lba); 14614c87aefeSPatrick Mooney } else { 14624c87aefeSPatrick Mooney uint64_t *prp_list; 14634c87aefeSPatrick Mooney int i; 14644c87aefeSPatrick Mooney 14654c87aefeSPatrick Mooney /* prp2 is pointer to a physical region page list */ 14664c87aefeSPatrick Mooney prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx, 14674c87aefeSPatrick Mooney cmd->prp2, PAGE_SIZE); 14684c87aefeSPatrick Mooney 14694c87aefeSPatrick Mooney i = 0; 14704c87aefeSPatrick Mooney while (size != 0) { 14714c87aefeSPatrick Mooney cpsz = MIN(size, PAGE_SIZE); 14724c87aefeSPatrick Mooney 14734c87aefeSPatrick Mooney /* 14744c87aefeSPatrick Mooney * Move to linked physical region page list 14754c87aefeSPatrick Mooney * in last item. 14764c87aefeSPatrick Mooney */ 14774c87aefeSPatrick Mooney if (i == (NVME_PRP2_ITEMS-1) && 14784c87aefeSPatrick Mooney size > PAGE_SIZE) { 14794c87aefeSPatrick Mooney assert((prp_list[i] & (PAGE_SIZE-1)) == 0); 14804c87aefeSPatrick Mooney prp_list = paddr_guest2host( 14814c87aefeSPatrick Mooney sc->nsc_pi->pi_vmctx, 14824c87aefeSPatrick Mooney prp_list[i], PAGE_SIZE); 14834c87aefeSPatrick Mooney i = 0; 14844c87aefeSPatrick Mooney } 14854c87aefeSPatrick Mooney if (prp_list[i] == 0) { 14864c87aefeSPatrick Mooney WPRINTF(("PRP2[%d] = 0 !!!\r\n", i)); 14874c87aefeSPatrick Mooney err = 1; 14884c87aefeSPatrick Mooney break; 14894c87aefeSPatrick Mooney } 14904c87aefeSPatrick Mooney 14914c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, 14924c87aefeSPatrick Mooney prp_list[i], cpsz, 14934c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, lba); 14944c87aefeSPatrick Mooney if (err) 14954c87aefeSPatrick Mooney break; 14964c87aefeSPatrick Mooney 14974c87aefeSPatrick Mooney lba += cpsz; 14984c87aefeSPatrick Mooney size -= cpsz; 14994c87aefeSPatrick Mooney i++; 15004c87aefeSPatrick Mooney } 15014c87aefeSPatrick Mooney } 15024c87aefeSPatrick Mooney 15034c87aefeSPatrick Mooney iodone: 15044c87aefeSPatrick Mooney if (sc->nvstore.type == NVME_STOR_RAM) { 15054c87aefeSPatrick Mooney uint16_t code, status = 0; 15064c87aefeSPatrick Mooney 15074c87aefeSPatrick Mooney code = err ? NVME_SC_LBA_OUT_OF_RANGE : 15084c87aefeSPatrick Mooney NVME_SC_SUCCESS; 15094c87aefeSPatrick Mooney pci_nvme_status_genc(&status, code); 15104c87aefeSPatrick Mooney 15114c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 15124c87aefeSPatrick Mooney status, 1); 15134c87aefeSPatrick Mooney 15144c87aefeSPatrick Mooney continue; 15154c87aefeSPatrick Mooney } 15164c87aefeSPatrick Mooney 15174c87aefeSPatrick Mooney 15184c87aefeSPatrick Mooney if (err) 15194c87aefeSPatrick Mooney goto do_error; 15204c87aefeSPatrick Mooney 15214c87aefeSPatrick Mooney req->io_req.br_callback = pci_nvme_io_done; 15224c87aefeSPatrick Mooney 15234c87aefeSPatrick Mooney err = 0; 15244c87aefeSPatrick Mooney switch (cmd->opc) { 15254c87aefeSPatrick Mooney case NVME_OPC_READ: 15264c87aefeSPatrick Mooney err = blockif_read(sc->nvstore.ctx, &req->io_req); 15274c87aefeSPatrick Mooney break; 15284c87aefeSPatrick Mooney case NVME_OPC_WRITE: 15294c87aefeSPatrick Mooney err = blockif_write(sc->nvstore.ctx, &req->io_req); 15304c87aefeSPatrick Mooney break; 15314c87aefeSPatrick Mooney default: 15324c87aefeSPatrick Mooney WPRINTF(("%s unhandled io command 0x%x\r\n", 15334c87aefeSPatrick Mooney __func__, cmd->opc)); 15344c87aefeSPatrick Mooney err = 1; 15354c87aefeSPatrick Mooney } 15364c87aefeSPatrick Mooney 15374c87aefeSPatrick Mooney do_error: 15384c87aefeSPatrick Mooney if (err) { 15394c87aefeSPatrick Mooney uint16_t status = 0; 15404c87aefeSPatrick Mooney 15414c87aefeSPatrick Mooney pci_nvme_status_genc(&status, 15424c87aefeSPatrick Mooney NVME_SC_DATA_TRANSFER_ERROR); 15434c87aefeSPatrick Mooney 15444c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 15454c87aefeSPatrick Mooney status, 1); 15464c87aefeSPatrick Mooney pci_nvme_release_ioreq(sc, req); 15474c87aefeSPatrick Mooney } 15484c87aefeSPatrick Mooney } 15494c87aefeSPatrick Mooney 15504c87aefeSPatrick Mooney atomic_store_short(&sq->head, sqhead); 15514c87aefeSPatrick Mooney atomic_store_int(&sq->busy, 0); 15524c87aefeSPatrick Mooney } 15534c87aefeSPatrick Mooney 15544c87aefeSPatrick Mooney static void 15554c87aefeSPatrick Mooney pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc, 15564c87aefeSPatrick Mooney uint64_t idx, int is_sq, uint64_t value) 15574c87aefeSPatrick Mooney { 15584c87aefeSPatrick Mooney DPRINTF(("nvme doorbell %lu, %s, val 0x%lx\r\n", 15594c87aefeSPatrick Mooney idx, is_sq ? "SQ" : "CQ", value & 0xFFFF)); 15604c87aefeSPatrick Mooney 15614c87aefeSPatrick Mooney if (is_sq) { 15624c87aefeSPatrick Mooney atomic_store_short(&sc->submit_queues[idx].tail, 15634c87aefeSPatrick Mooney (uint16_t)value); 15644c87aefeSPatrick Mooney 15654c87aefeSPatrick Mooney if (idx == 0) { 15664c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(sc, value); 15674c87aefeSPatrick Mooney } else { 15684c87aefeSPatrick Mooney /* submission queue; handle new entries in SQ */ 15694c87aefeSPatrick Mooney if (idx > sc->num_squeues) { 15704c87aefeSPatrick Mooney WPRINTF(("%s SQ index %lu overflow from " 15714c87aefeSPatrick Mooney "guest (max %u)\r\n", 15724c87aefeSPatrick Mooney __func__, idx, sc->num_squeues)); 15734c87aefeSPatrick Mooney return; 15744c87aefeSPatrick Mooney } 15754c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(sc, (uint16_t)idx); 15764c87aefeSPatrick Mooney } 15774c87aefeSPatrick Mooney } else { 15784c87aefeSPatrick Mooney if (idx > sc->num_cqueues) { 15794c87aefeSPatrick Mooney WPRINTF(("%s queue index %lu overflow from " 15804c87aefeSPatrick Mooney "guest (max %u)\r\n", 15814c87aefeSPatrick Mooney __func__, idx, sc->num_cqueues)); 15824c87aefeSPatrick Mooney return; 15834c87aefeSPatrick Mooney } 15844c87aefeSPatrick Mooney 15854c87aefeSPatrick Mooney sc->compl_queues[idx].head = (uint16_t)value; 15864c87aefeSPatrick Mooney } 15874c87aefeSPatrick Mooney } 15884c87aefeSPatrick Mooney 15894c87aefeSPatrick Mooney static void 15904c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) 15914c87aefeSPatrick Mooney { 15924c87aefeSPatrick Mooney const char *s = iswrite ? "WRITE" : "READ"; 15934c87aefeSPatrick Mooney 15944c87aefeSPatrick Mooney switch (offset) { 15954c87aefeSPatrick Mooney case NVME_CR_CAP_LOW: 15964c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CAP_LOW\r\n", func, s)); 15974c87aefeSPatrick Mooney break; 15984c87aefeSPatrick Mooney case NVME_CR_CAP_HI: 15994c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CAP_HI\r\n", func, s)); 16004c87aefeSPatrick Mooney break; 16014c87aefeSPatrick Mooney case NVME_CR_VS: 16024c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_VS\r\n", func, s)); 16034c87aefeSPatrick Mooney break; 16044c87aefeSPatrick Mooney case NVME_CR_INTMS: 16054c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_INTMS\r\n", func, s)); 16064c87aefeSPatrick Mooney break; 16074c87aefeSPatrick Mooney case NVME_CR_INTMC: 16084c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_INTMC\r\n", func, s)); 16094c87aefeSPatrick Mooney break; 16104c87aefeSPatrick Mooney case NVME_CR_CC: 16114c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CC\r\n", func, s)); 16124c87aefeSPatrick Mooney break; 16134c87aefeSPatrick Mooney case NVME_CR_CSTS: 16144c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CSTS\r\n", func, s)); 16154c87aefeSPatrick Mooney break; 16164c87aefeSPatrick Mooney case NVME_CR_NSSR: 16174c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_NSSR\r\n", func, s)); 16184c87aefeSPatrick Mooney break; 16194c87aefeSPatrick Mooney case NVME_CR_AQA: 16204c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_AQA\r\n", func, s)); 16214c87aefeSPatrick Mooney break; 16224c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW: 16234c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ASQ_LOW\r\n", func, s)); 16244c87aefeSPatrick Mooney break; 16254c87aefeSPatrick Mooney case NVME_CR_ASQ_HI: 16264c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ASQ_HI\r\n", func, s)); 16274c87aefeSPatrick Mooney break; 16284c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW: 16294c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ACQ_LOW\r\n", func, s)); 16304c87aefeSPatrick Mooney break; 16314c87aefeSPatrick Mooney case NVME_CR_ACQ_HI: 16324c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ACQ_HI\r\n", func, s)); 16334c87aefeSPatrick Mooney break; 16344c87aefeSPatrick Mooney default: 16354c87aefeSPatrick Mooney DPRINTF(("unknown nvme bar-0 offset 0x%lx\r\n", offset)); 16364c87aefeSPatrick Mooney } 16374c87aefeSPatrick Mooney 16384c87aefeSPatrick Mooney } 16394c87aefeSPatrick Mooney 16404c87aefeSPatrick Mooney static void 16414c87aefeSPatrick Mooney pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc, 16424c87aefeSPatrick Mooney uint64_t offset, int size, uint64_t value) 16434c87aefeSPatrick Mooney { 16444c87aefeSPatrick Mooney uint32_t ccreg; 16454c87aefeSPatrick Mooney 16464c87aefeSPatrick Mooney if (offset >= NVME_DOORBELL_OFFSET) { 16474c87aefeSPatrick Mooney uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; 16484c87aefeSPatrick Mooney uint64_t idx = belloffset / 8; /* door bell size = 2*int */ 16494c87aefeSPatrick Mooney int is_sq = (belloffset % 8) < 4; 16504c87aefeSPatrick Mooney 16514c87aefeSPatrick Mooney if (belloffset > ((sc->max_queues+1) * 8 - 4)) { 16524c87aefeSPatrick Mooney WPRINTF(("guest attempted an overflow write offset " 16534c87aefeSPatrick Mooney "0x%lx, val 0x%lx in %s", 16544c87aefeSPatrick Mooney offset, value, __func__)); 16554c87aefeSPatrick Mooney return; 16564c87aefeSPatrick Mooney } 16574c87aefeSPatrick Mooney 16584c87aefeSPatrick Mooney pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value); 16594c87aefeSPatrick Mooney return; 16604c87aefeSPatrick Mooney } 16614c87aefeSPatrick Mooney 16624c87aefeSPatrick Mooney DPRINTF(("nvme-write offset 0x%lx, size %d, value 0x%lx\r\n", 16634c87aefeSPatrick Mooney offset, size, value)); 16644c87aefeSPatrick Mooney 16654c87aefeSPatrick Mooney if (size != 4) { 16664c87aefeSPatrick Mooney WPRINTF(("guest wrote invalid size %d (offset 0x%lx, " 16674c87aefeSPatrick Mooney "val 0x%lx) to bar0 in %s", 16684c87aefeSPatrick Mooney size, offset, value, __func__)); 16694c87aefeSPatrick Mooney /* TODO: shutdown device */ 16704c87aefeSPatrick Mooney return; 16714c87aefeSPatrick Mooney } 16724c87aefeSPatrick Mooney 16734c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 1); 16744c87aefeSPatrick Mooney 16754c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 16764c87aefeSPatrick Mooney 16774c87aefeSPatrick Mooney switch (offset) { 16784c87aefeSPatrick Mooney case NVME_CR_CAP_LOW: 16794c87aefeSPatrick Mooney case NVME_CR_CAP_HI: 16804c87aefeSPatrick Mooney /* readonly */ 16814c87aefeSPatrick Mooney break; 16824c87aefeSPatrick Mooney case NVME_CR_VS: 16834c87aefeSPatrick Mooney /* readonly */ 16844c87aefeSPatrick Mooney break; 16854c87aefeSPatrick Mooney case NVME_CR_INTMS: 16864c87aefeSPatrick Mooney /* MSI-X, so ignore */ 16874c87aefeSPatrick Mooney break; 16884c87aefeSPatrick Mooney case NVME_CR_INTMC: 16894c87aefeSPatrick Mooney /* MSI-X, so ignore */ 16904c87aefeSPatrick Mooney break; 16914c87aefeSPatrick Mooney case NVME_CR_CC: 16924c87aefeSPatrick Mooney ccreg = (uint32_t)value; 16934c87aefeSPatrick Mooney 16944c87aefeSPatrick Mooney DPRINTF(("%s NVME_CR_CC en %x css %x shn %x iosqes %u " 16954c87aefeSPatrick Mooney "iocqes %u\r\n", 16964c87aefeSPatrick Mooney __func__, 16974c87aefeSPatrick Mooney NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), 16984c87aefeSPatrick Mooney NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), 16994c87aefeSPatrick Mooney NVME_CC_GET_IOCQES(ccreg))); 17004c87aefeSPatrick Mooney 17014c87aefeSPatrick Mooney if (NVME_CC_GET_SHN(ccreg)) { 17024c87aefeSPatrick Mooney /* perform shutdown - flush out data to backend */ 17034c87aefeSPatrick Mooney sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK << 17044c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT); 17054c87aefeSPatrick Mooney sc->regs.csts |= NVME_SHST_COMPLETE << 17064c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT; 17074c87aefeSPatrick Mooney } 17084c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { 17094c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0) 17104c87aefeSPatrick Mooney /* transition 1-> causes controller reset */ 17114c87aefeSPatrick Mooney pci_nvme_reset_locked(sc); 17124c87aefeSPatrick Mooney else 17134c87aefeSPatrick Mooney pci_nvme_init_controller(ctx, sc); 17144c87aefeSPatrick Mooney } 17154c87aefeSPatrick Mooney 17164c87aefeSPatrick Mooney /* Insert the iocqes, iosqes and en bits from the write */ 17174c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_WRITE_MASK; 17184c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; 17194c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0) { 17204c87aefeSPatrick Mooney /* Insert the ams, mps and css bit fields */ 17214c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; 17224c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; 17234c87aefeSPatrick Mooney sc->regs.csts &= ~NVME_CSTS_RDY; 17244c87aefeSPatrick Mooney } else if (sc->pending_ios == 0) { 17254c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY; 17264c87aefeSPatrick Mooney } 17274c87aefeSPatrick Mooney break; 17284c87aefeSPatrick Mooney case NVME_CR_CSTS: 17294c87aefeSPatrick Mooney break; 17304c87aefeSPatrick Mooney case NVME_CR_NSSR: 17314c87aefeSPatrick Mooney /* ignore writes; don't support subsystem reset */ 17324c87aefeSPatrick Mooney break; 17334c87aefeSPatrick Mooney case NVME_CR_AQA: 17344c87aefeSPatrick Mooney sc->regs.aqa = (uint32_t)value; 17354c87aefeSPatrick Mooney break; 17364c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW: 17374c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | 17384c87aefeSPatrick Mooney (0xFFFFF000 & value); 17394c87aefeSPatrick Mooney break; 17404c87aefeSPatrick Mooney case NVME_CR_ASQ_HI: 17414c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | 17424c87aefeSPatrick Mooney (value << 32); 17434c87aefeSPatrick Mooney break; 17444c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW: 17454c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | 17464c87aefeSPatrick Mooney (0xFFFFF000 & value); 17474c87aefeSPatrick Mooney break; 17484c87aefeSPatrick Mooney case NVME_CR_ACQ_HI: 17494c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | 17504c87aefeSPatrick Mooney (value << 32); 17514c87aefeSPatrick Mooney break; 17524c87aefeSPatrick Mooney default: 17534c87aefeSPatrick Mooney DPRINTF(("%s unknown offset 0x%lx, value 0x%lx size %d\r\n", 17544c87aefeSPatrick Mooney __func__, offset, value, size)); 17554c87aefeSPatrick Mooney } 17564c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 17574c87aefeSPatrick Mooney } 17584c87aefeSPatrick Mooney 17594c87aefeSPatrick Mooney static void 17604c87aefeSPatrick Mooney pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, 17614c87aefeSPatrick Mooney int baridx, uint64_t offset, int size, uint64_t value) 17624c87aefeSPatrick Mooney { 17634c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg; 17644c87aefeSPatrick Mooney 17654c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) || 17664c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) { 17674c87aefeSPatrick Mooney DPRINTF(("nvme-write baridx %d, msix: off 0x%lx, size %d, " 17684c87aefeSPatrick Mooney " value 0x%lx\r\n", baridx, offset, size, value)); 17694c87aefeSPatrick Mooney 17704c87aefeSPatrick Mooney pci_emul_msix_twrite(pi, offset, size, value); 17714c87aefeSPatrick Mooney return; 17724c87aefeSPatrick Mooney } 17734c87aefeSPatrick Mooney 17744c87aefeSPatrick Mooney switch (baridx) { 17754c87aefeSPatrick Mooney case 0: 17764c87aefeSPatrick Mooney pci_nvme_write_bar_0(ctx, sc, offset, size, value); 17774c87aefeSPatrick Mooney break; 17784c87aefeSPatrick Mooney 17794c87aefeSPatrick Mooney default: 17804c87aefeSPatrick Mooney DPRINTF(("%s unknown baridx %d, val 0x%lx\r\n", 17814c87aefeSPatrick Mooney __func__, baridx, value)); 17824c87aefeSPatrick Mooney } 17834c87aefeSPatrick Mooney } 17844c87aefeSPatrick Mooney 17854c87aefeSPatrick Mooney static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, 17864c87aefeSPatrick Mooney uint64_t offset, int size) 17874c87aefeSPatrick Mooney { 17884c87aefeSPatrick Mooney uint64_t value; 17894c87aefeSPatrick Mooney 17904c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 0); 17914c87aefeSPatrick Mooney 17924c87aefeSPatrick Mooney if (offset < NVME_DOORBELL_OFFSET) { 17934c87aefeSPatrick Mooney void *p = &(sc->regs); 17944c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 17954c87aefeSPatrick Mooney memcpy(&value, (void *)((uintptr_t)p + offset), size); 17964c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 17974c87aefeSPatrick Mooney } else { 17984c87aefeSPatrick Mooney value = 0; 17994c87aefeSPatrick Mooney WPRINTF(("pci_nvme: read invalid offset %ld\r\n", offset)); 18004c87aefeSPatrick Mooney } 18014c87aefeSPatrick Mooney 18024c87aefeSPatrick Mooney switch (size) { 18034c87aefeSPatrick Mooney case 1: 18044c87aefeSPatrick Mooney value &= 0xFF; 18054c87aefeSPatrick Mooney break; 18064c87aefeSPatrick Mooney case 2: 18074c87aefeSPatrick Mooney value &= 0xFFFF; 18084c87aefeSPatrick Mooney break; 18094c87aefeSPatrick Mooney case 4: 18104c87aefeSPatrick Mooney value &= 0xFFFFFFFF; 18114c87aefeSPatrick Mooney break; 18124c87aefeSPatrick Mooney } 18134c87aefeSPatrick Mooney 18144c87aefeSPatrick Mooney DPRINTF((" nvme-read offset 0x%lx, size %d -> value 0x%x\r\n", 18154c87aefeSPatrick Mooney offset, size, (uint32_t)value)); 18164c87aefeSPatrick Mooney 18174c87aefeSPatrick Mooney return (value); 18184c87aefeSPatrick Mooney } 18194c87aefeSPatrick Mooney 18204c87aefeSPatrick Mooney 18214c87aefeSPatrick Mooney 18224c87aefeSPatrick Mooney static uint64_t 18234c87aefeSPatrick Mooney pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, 18244c87aefeSPatrick Mooney uint64_t offset, int size) 18254c87aefeSPatrick Mooney { 18264c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg; 18274c87aefeSPatrick Mooney 18284c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) || 18294c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) { 18304c87aefeSPatrick Mooney DPRINTF(("nvme-read bar: %d, msix: regoff 0x%lx, size %d\r\n", 18314c87aefeSPatrick Mooney baridx, offset, size)); 18324c87aefeSPatrick Mooney 18334c87aefeSPatrick Mooney return pci_emul_msix_tread(pi, offset, size); 18344c87aefeSPatrick Mooney } 18354c87aefeSPatrick Mooney 18364c87aefeSPatrick Mooney switch (baridx) { 18374c87aefeSPatrick Mooney case 0: 18384c87aefeSPatrick Mooney return pci_nvme_read_bar_0(sc, offset, size); 18394c87aefeSPatrick Mooney 18404c87aefeSPatrick Mooney default: 18414c87aefeSPatrick Mooney DPRINTF(("unknown bar %d, 0x%lx\r\n", baridx, offset)); 18424c87aefeSPatrick Mooney } 18434c87aefeSPatrick Mooney 18444c87aefeSPatrick Mooney return (0); 18454c87aefeSPatrick Mooney } 18464c87aefeSPatrick Mooney 18474c87aefeSPatrick Mooney 18484c87aefeSPatrick Mooney static int 18494c87aefeSPatrick Mooney pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts) 18504c87aefeSPatrick Mooney { 18514c87aefeSPatrick Mooney char bident[sizeof("XX:X:X")]; 18524c87aefeSPatrick Mooney char *uopt, *xopts, *config; 18534c87aefeSPatrick Mooney uint32_t sectsz; 18544c87aefeSPatrick Mooney int optidx; 18554c87aefeSPatrick Mooney 18564c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES; 18574c87aefeSPatrick Mooney sc->max_qentries = NVME_MAX_QENTRIES; 18584c87aefeSPatrick Mooney sc->ioslots = NVME_IOSLOTS; 18594c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues; 18604c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues; 18614c87aefeSPatrick Mooney sectsz = 0; 18624c87aefeSPatrick Mooney 18634c87aefeSPatrick Mooney uopt = strdup(opts); 18644c87aefeSPatrick Mooney optidx = 0; 18654c87aefeSPatrick Mooney snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), 18664c87aefeSPatrick Mooney "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 18674c87aefeSPatrick Mooney for (xopts = strtok(uopt, ","); 18684c87aefeSPatrick Mooney xopts != NULL; 18694c87aefeSPatrick Mooney xopts = strtok(NULL, ",")) { 18704c87aefeSPatrick Mooney 18714c87aefeSPatrick Mooney if ((config = strchr(xopts, '=')) != NULL) 18724c87aefeSPatrick Mooney *config++ = '\0'; 18734c87aefeSPatrick Mooney 18744c87aefeSPatrick Mooney if (!strcmp("maxq", xopts)) { 18754c87aefeSPatrick Mooney sc->max_queues = atoi(config); 18764c87aefeSPatrick Mooney } else if (!strcmp("qsz", xopts)) { 18774c87aefeSPatrick Mooney sc->max_qentries = atoi(config); 18784c87aefeSPatrick Mooney } else if (!strcmp("ioslots", xopts)) { 18794c87aefeSPatrick Mooney sc->ioslots = atoi(config); 18804c87aefeSPatrick Mooney } else if (!strcmp("sectsz", xopts)) { 18814c87aefeSPatrick Mooney sectsz = atoi(config); 18824c87aefeSPatrick Mooney } else if (!strcmp("ser", xopts)) { 18834c87aefeSPatrick Mooney /* 18844c87aefeSPatrick Mooney * This field indicates the Product Serial Number in 18854c87aefeSPatrick Mooney * 7-bit ASCII, unused bytes should be space characters. 18864c87aefeSPatrick Mooney * Ref: NVMe v1.3c. 18874c87aefeSPatrick Mooney */ 18884c87aefeSPatrick Mooney cpywithpad((char *)sc->ctrldata.sn, 18894c87aefeSPatrick Mooney sizeof(sc->ctrldata.sn), config, ' '); 18904c87aefeSPatrick Mooney } else if (!strcmp("ram", xopts)) { 18914c87aefeSPatrick Mooney uint64_t sz = strtoull(&xopts[4], NULL, 10); 18924c87aefeSPatrick Mooney 18934c87aefeSPatrick Mooney sc->nvstore.type = NVME_STOR_RAM; 18944c87aefeSPatrick Mooney sc->nvstore.size = sz * 1024 * 1024; 18954c87aefeSPatrick Mooney sc->nvstore.ctx = calloc(1, sc->nvstore.size); 18964c87aefeSPatrick Mooney sc->nvstore.sectsz = 4096; 18974c87aefeSPatrick Mooney sc->nvstore.sectsz_bits = 12; 18984c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL) { 18994c87aefeSPatrick Mooney perror("Unable to allocate RAM"); 19004c87aefeSPatrick Mooney free(uopt); 19014c87aefeSPatrick Mooney return (-1); 19024c87aefeSPatrick Mooney } 1903*84659b24SMichael Zeller } else if (!strcmp("eui64", xopts)) { 1904*84659b24SMichael Zeller sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0)); 19054c87aefeSPatrick Mooney } else if (optidx == 0) { 19064c87aefeSPatrick Mooney snprintf(bident, sizeof(bident), "%d:%d", 19074c87aefeSPatrick Mooney sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 19084c87aefeSPatrick Mooney sc->nvstore.ctx = blockif_open(xopts, bident); 19094c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL) { 19104c87aefeSPatrick Mooney perror("Could not open backing file"); 19114c87aefeSPatrick Mooney free(uopt); 19124c87aefeSPatrick Mooney return (-1); 19134c87aefeSPatrick Mooney } 19144c87aefeSPatrick Mooney sc->nvstore.type = NVME_STOR_BLOCKIF; 19154c87aefeSPatrick Mooney sc->nvstore.size = blockif_size(sc->nvstore.ctx); 19164c87aefeSPatrick Mooney } else { 19174c87aefeSPatrick Mooney fprintf(stderr, "Invalid option %s\n", xopts); 19184c87aefeSPatrick Mooney free(uopt); 19194c87aefeSPatrick Mooney return (-1); 19204c87aefeSPatrick Mooney } 19214c87aefeSPatrick Mooney 19224c87aefeSPatrick Mooney optidx++; 19234c87aefeSPatrick Mooney } 19244c87aefeSPatrick Mooney free(uopt); 19254c87aefeSPatrick Mooney 19264c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) { 19274c87aefeSPatrick Mooney fprintf(stderr, "backing store not specified\n"); 19284c87aefeSPatrick Mooney return (-1); 19294c87aefeSPatrick Mooney } 19304c87aefeSPatrick Mooney if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) 19314c87aefeSPatrick Mooney sc->nvstore.sectsz = sectsz; 19324c87aefeSPatrick Mooney else if (sc->nvstore.type != NVME_STOR_RAM) 19334c87aefeSPatrick Mooney sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); 19344c87aefeSPatrick Mooney for (sc->nvstore.sectsz_bits = 9; 19354c87aefeSPatrick Mooney (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; 19364c87aefeSPatrick Mooney sc->nvstore.sectsz_bits++); 19374c87aefeSPatrick Mooney 19384c87aefeSPatrick Mooney if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) 19394c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES; 19404c87aefeSPatrick Mooney 19414c87aefeSPatrick Mooney if (sc->max_qentries <= 0) { 19424c87aefeSPatrick Mooney fprintf(stderr, "Invalid qsz option\n"); 19434c87aefeSPatrick Mooney return (-1); 19444c87aefeSPatrick Mooney } 19454c87aefeSPatrick Mooney if (sc->ioslots <= 0) { 19464c87aefeSPatrick Mooney fprintf(stderr, "Invalid ioslots option\n"); 19474c87aefeSPatrick Mooney return (-1); 19484c87aefeSPatrick Mooney } 19494c87aefeSPatrick Mooney 19504c87aefeSPatrick Mooney return (0); 19514c87aefeSPatrick Mooney } 19524c87aefeSPatrick Mooney 19534c87aefeSPatrick Mooney static int 19544c87aefeSPatrick Mooney pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) 19554c87aefeSPatrick Mooney { 19564c87aefeSPatrick Mooney struct pci_nvme_softc *sc; 19574c87aefeSPatrick Mooney uint32_t pci_membar_sz; 19584c87aefeSPatrick Mooney int error; 19594c87aefeSPatrick Mooney 19604c87aefeSPatrick Mooney error = 0; 19614c87aefeSPatrick Mooney 19624c87aefeSPatrick Mooney sc = calloc(1, sizeof(struct pci_nvme_softc)); 19634c87aefeSPatrick Mooney pi->pi_arg = sc; 19644c87aefeSPatrick Mooney sc->nsc_pi = pi; 19654c87aefeSPatrick Mooney 19664c87aefeSPatrick Mooney error = pci_nvme_parse_opts(sc, opts); 19674c87aefeSPatrick Mooney if (error < 0) 19684c87aefeSPatrick Mooney goto done; 19694c87aefeSPatrick Mooney else 19704c87aefeSPatrick Mooney error = 0; 19714c87aefeSPatrick Mooney 19724c87aefeSPatrick Mooney sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); 19734c87aefeSPatrick Mooney for (int i = 0; i < sc->ioslots; i++) { 19744c87aefeSPatrick Mooney if (i < (sc->ioslots-1)) 19754c87aefeSPatrick Mooney sc->ioreqs[i].next = &sc->ioreqs[i+1]; 19764c87aefeSPatrick Mooney pthread_mutex_init(&sc->ioreqs[i].mtx, NULL); 19774c87aefeSPatrick Mooney pthread_cond_init(&sc->ioreqs[i].cv, NULL); 19784c87aefeSPatrick Mooney } 19794c87aefeSPatrick Mooney sc->ioreqs_free = sc->ioreqs; 19804c87aefeSPatrick Mooney sc->intr_coales_aggr_thresh = 1; 19814c87aefeSPatrick Mooney 19824c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); 19834c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); 19844c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 19854c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); 19864c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_PROGIF, 19874c87aefeSPatrick Mooney PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); 19884c87aefeSPatrick Mooney 19894c87aefeSPatrick Mooney /* 19904c87aefeSPatrick Mooney * Allocate size of NVMe registers + doorbell space for all queues. 19914c87aefeSPatrick Mooney * 19924c87aefeSPatrick Mooney * The specification requires a minimum memory I/O window size of 16K. 19934c87aefeSPatrick Mooney * The Windows driver will refuse to start a device with a smaller 19944c87aefeSPatrick Mooney * window. 19954c87aefeSPatrick Mooney */ 19964c87aefeSPatrick Mooney pci_membar_sz = sizeof(struct nvme_registers) + 19974c87aefeSPatrick Mooney 2 * sizeof(uint32_t) * (sc->max_queues + 1); 19984c87aefeSPatrick Mooney pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); 19994c87aefeSPatrick Mooney 20004c87aefeSPatrick Mooney DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz)); 20014c87aefeSPatrick Mooney 20024c87aefeSPatrick Mooney error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); 20034c87aefeSPatrick Mooney if (error) { 20044c87aefeSPatrick Mooney WPRINTF(("%s pci alloc mem bar failed\r\n", __func__)); 20054c87aefeSPatrick Mooney goto done; 20064c87aefeSPatrick Mooney } 20074c87aefeSPatrick Mooney 20084c87aefeSPatrick Mooney error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); 20094c87aefeSPatrick Mooney if (error) { 20104c87aefeSPatrick Mooney WPRINTF(("%s pci add msixcap failed\r\n", __func__)); 20114c87aefeSPatrick Mooney goto done; 20124c87aefeSPatrick Mooney } 20134c87aefeSPatrick Mooney 2014*84659b24SMichael Zeller error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP); 2015*84659b24SMichael Zeller if (error) { 2016*84659b24SMichael Zeller WPRINTF(("%s pci add Express capability failed\r\n", __func__)); 2017*84659b24SMichael Zeller goto done; 2018*84659b24SMichael Zeller } 2019*84659b24SMichael Zeller 20204c87aefeSPatrick Mooney pthread_mutex_init(&sc->mtx, NULL); 20214c87aefeSPatrick Mooney sem_init(&sc->iosemlock, 0, sc->ioslots); 20224c87aefeSPatrick Mooney 20234c87aefeSPatrick Mooney pci_nvme_reset(sc); 20244c87aefeSPatrick Mooney pci_nvme_init_ctrldata(sc); 2025*84659b24SMichael Zeller pci_nvme_init_nsdata(sc, &sc->nsdata, 1, sc->nvstore.eui64); 20264c87aefeSPatrick Mooney pci_nvme_init_logpages(sc); 20274c87aefeSPatrick Mooney 20284c87aefeSPatrick Mooney pci_lintr_request(pi); 20294c87aefeSPatrick Mooney 20304c87aefeSPatrick Mooney done: 20314c87aefeSPatrick Mooney return (error); 20324c87aefeSPatrick Mooney } 20334c87aefeSPatrick Mooney 20344c87aefeSPatrick Mooney 20354c87aefeSPatrick Mooney struct pci_devemu pci_de_nvme = { 20364c87aefeSPatrick Mooney .pe_emu = "nvme", 20374c87aefeSPatrick Mooney .pe_init = pci_nvme_init, 20384c87aefeSPatrick Mooney .pe_barwrite = pci_nvme_write, 20394c87aefeSPatrick Mooney .pe_barread = pci_nvme_read 20404c87aefeSPatrick Mooney }; 20414c87aefeSPatrick Mooney PCI_EMUL_SET(pci_de_nvme); 2042