1*4c87aefeSPatrick Mooney /*- 2*4c87aefeSPatrick Mooney * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3*4c87aefeSPatrick Mooney * 4*4c87aefeSPatrick Mooney * Copyright (c) 2017 Shunsuke Mie 5*4c87aefeSPatrick Mooney * Copyright (c) 2018 Leon Dang 6*4c87aefeSPatrick Mooney * 7*4c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without 8*4c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions 9*4c87aefeSPatrick Mooney * are met: 10*4c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright 11*4c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer. 12*4c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright 13*4c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the 14*4c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution. 15*4c87aefeSPatrick Mooney * 16*4c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17*4c87aefeSPatrick Mooney * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18*4c87aefeSPatrick Mooney * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19*4c87aefeSPatrick Mooney * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20*4c87aefeSPatrick Mooney * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21*4c87aefeSPatrick Mooney * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22*4c87aefeSPatrick Mooney * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23*4c87aefeSPatrick Mooney * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24*4c87aefeSPatrick Mooney * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25*4c87aefeSPatrick Mooney * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26*4c87aefeSPatrick Mooney * SUCH DAMAGE. 27*4c87aefeSPatrick Mooney */ 28*4c87aefeSPatrick Mooney 29*4c87aefeSPatrick Mooney /* 30*4c87aefeSPatrick Mooney * bhyve PCIe-NVMe device emulation. 31*4c87aefeSPatrick Mooney * 32*4c87aefeSPatrick Mooney * options: 33*4c87aefeSPatrick Mooney * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z 34*4c87aefeSPatrick Mooney * 35*4c87aefeSPatrick Mooney * accepted devpath: 36*4c87aefeSPatrick Mooney * /dev/blockdev 37*4c87aefeSPatrick Mooney * /path/to/image 38*4c87aefeSPatrick Mooney * ram=size_in_MiB 39*4c87aefeSPatrick Mooney * 40*4c87aefeSPatrick Mooney * maxq = max number of queues 41*4c87aefeSPatrick Mooney * qsz = max elements in each queue 42*4c87aefeSPatrick Mooney * ioslots = max number of concurrent io requests 43*4c87aefeSPatrick Mooney * sectsz = sector size (defaults to blockif sector size) 44*4c87aefeSPatrick Mooney * ser = serial number (20-chars max) 45*4c87aefeSPatrick Mooney * 46*4c87aefeSPatrick Mooney */ 47*4c87aefeSPatrick Mooney 48*4c87aefeSPatrick Mooney /* TODO: 49*4c87aefeSPatrick Mooney - create async event for smart and log 50*4c87aefeSPatrick Mooney - intr coalesce 51*4c87aefeSPatrick Mooney */ 52*4c87aefeSPatrick Mooney 53*4c87aefeSPatrick Mooney #include <sys/cdefs.h> 54*4c87aefeSPatrick Mooney __FBSDID("$FreeBSD$"); 55*4c87aefeSPatrick Mooney 56*4c87aefeSPatrick Mooney #include <sys/types.h> 57*4c87aefeSPatrick Mooney 58*4c87aefeSPatrick Mooney #include <assert.h> 59*4c87aefeSPatrick Mooney #include <pthread.h> 60*4c87aefeSPatrick Mooney #include <semaphore.h> 61*4c87aefeSPatrick Mooney #include <stdbool.h> 62*4c87aefeSPatrick Mooney #include <stddef.h> 63*4c87aefeSPatrick Mooney #include <stdint.h> 64*4c87aefeSPatrick Mooney #include <stdio.h> 65*4c87aefeSPatrick Mooney #include <stdlib.h> 66*4c87aefeSPatrick Mooney #include <string.h> 67*4c87aefeSPatrick Mooney 68*4c87aefeSPatrick Mooney #include <machine/atomic.h> 69*4c87aefeSPatrick Mooney #include <machine/vmm.h> 70*4c87aefeSPatrick Mooney #include <vmmapi.h> 71*4c87aefeSPatrick Mooney 72*4c87aefeSPatrick Mooney #include <dev/nvme/nvme.h> 73*4c87aefeSPatrick Mooney 74*4c87aefeSPatrick Mooney #include "bhyverun.h" 75*4c87aefeSPatrick Mooney #include "block_if.h" 76*4c87aefeSPatrick Mooney #include "pci_emul.h" 77*4c87aefeSPatrick Mooney 78*4c87aefeSPatrick Mooney 79*4c87aefeSPatrick Mooney static int nvme_debug = 0; 80*4c87aefeSPatrick Mooney #define DPRINTF(params) if (nvme_debug) printf params 81*4c87aefeSPatrick Mooney #define WPRINTF(params) printf params 82*4c87aefeSPatrick Mooney 83*4c87aefeSPatrick Mooney /* defaults; can be overridden */ 84*4c87aefeSPatrick Mooney #define NVME_MSIX_BAR 4 85*4c87aefeSPatrick Mooney 86*4c87aefeSPatrick Mooney #define NVME_IOSLOTS 8 87*4c87aefeSPatrick Mooney 88*4c87aefeSPatrick Mooney /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ 89*4c87aefeSPatrick Mooney #define NVME_MMIO_SPACE_MIN (1 << 14) 90*4c87aefeSPatrick Mooney 91*4c87aefeSPatrick Mooney #define NVME_QUEUES 16 92*4c87aefeSPatrick Mooney #define NVME_MAX_QENTRIES 2048 93*4c87aefeSPatrick Mooney 94*4c87aefeSPatrick Mooney #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) 95*4c87aefeSPatrick Mooney #define NVME_MAX_BLOCKIOVS 512 96*4c87aefeSPatrick Mooney 97*4c87aefeSPatrick Mooney /* helpers */ 98*4c87aefeSPatrick Mooney 99*4c87aefeSPatrick Mooney /* Convert a zero-based value into a one-based value */ 100*4c87aefeSPatrick Mooney #define ONE_BASED(zero) ((zero) + 1) 101*4c87aefeSPatrick Mooney /* Convert a one-based value into a zero-based value */ 102*4c87aefeSPatrick Mooney #define ZERO_BASED(one) ((one) - 1) 103*4c87aefeSPatrick Mooney 104*4c87aefeSPatrick Mooney /* Encode number of SQ's and CQ's for Set/Get Features */ 105*4c87aefeSPatrick Mooney #define NVME_FEATURE_NUM_QUEUES(sc) \ 106*4c87aefeSPatrick Mooney (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ 107*4c87aefeSPatrick Mooney (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16; 108*4c87aefeSPatrick Mooney 109*4c87aefeSPatrick Mooney #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) 110*4c87aefeSPatrick Mooney 111*4c87aefeSPatrick Mooney enum nvme_controller_register_offsets { 112*4c87aefeSPatrick Mooney NVME_CR_CAP_LOW = 0x00, 113*4c87aefeSPatrick Mooney NVME_CR_CAP_HI = 0x04, 114*4c87aefeSPatrick Mooney NVME_CR_VS = 0x08, 115*4c87aefeSPatrick Mooney NVME_CR_INTMS = 0x0c, 116*4c87aefeSPatrick Mooney NVME_CR_INTMC = 0x10, 117*4c87aefeSPatrick Mooney NVME_CR_CC = 0x14, 118*4c87aefeSPatrick Mooney NVME_CR_CSTS = 0x1c, 119*4c87aefeSPatrick Mooney NVME_CR_NSSR = 0x20, 120*4c87aefeSPatrick Mooney NVME_CR_AQA = 0x24, 121*4c87aefeSPatrick Mooney NVME_CR_ASQ_LOW = 0x28, 122*4c87aefeSPatrick Mooney NVME_CR_ASQ_HI = 0x2c, 123*4c87aefeSPatrick Mooney NVME_CR_ACQ_LOW = 0x30, 124*4c87aefeSPatrick Mooney NVME_CR_ACQ_HI = 0x34, 125*4c87aefeSPatrick Mooney }; 126*4c87aefeSPatrick Mooney 127*4c87aefeSPatrick Mooney enum nvme_cmd_cdw11 { 128*4c87aefeSPatrick Mooney NVME_CMD_CDW11_PC = 0x0001, 129*4c87aefeSPatrick Mooney NVME_CMD_CDW11_IEN = 0x0002, 130*4c87aefeSPatrick Mooney NVME_CMD_CDW11_IV = 0xFFFF0000, 131*4c87aefeSPatrick Mooney }; 132*4c87aefeSPatrick Mooney 133*4c87aefeSPatrick Mooney #define NVME_CQ_INTEN 0x01 134*4c87aefeSPatrick Mooney #define NVME_CQ_INTCOAL 0x02 135*4c87aefeSPatrick Mooney 136*4c87aefeSPatrick Mooney struct nvme_completion_queue { 137*4c87aefeSPatrick Mooney struct nvme_completion *qbase; 138*4c87aefeSPatrick Mooney uint32_t size; 139*4c87aefeSPatrick Mooney uint16_t tail; /* nvme progress */ 140*4c87aefeSPatrick Mooney uint16_t head; /* guest progress */ 141*4c87aefeSPatrick Mooney uint16_t intr_vec; 142*4c87aefeSPatrick Mooney uint32_t intr_en; 143*4c87aefeSPatrick Mooney pthread_mutex_t mtx; 144*4c87aefeSPatrick Mooney }; 145*4c87aefeSPatrick Mooney 146*4c87aefeSPatrick Mooney struct nvme_submission_queue { 147*4c87aefeSPatrick Mooney struct nvme_command *qbase; 148*4c87aefeSPatrick Mooney uint32_t size; 149*4c87aefeSPatrick Mooney uint16_t head; /* nvme progress */ 150*4c87aefeSPatrick Mooney uint16_t tail; /* guest progress */ 151*4c87aefeSPatrick Mooney uint16_t cqid; /* completion queue id */ 152*4c87aefeSPatrick Mooney int busy; /* queue is being processed */ 153*4c87aefeSPatrick Mooney int qpriority; 154*4c87aefeSPatrick Mooney }; 155*4c87aefeSPatrick Mooney 156*4c87aefeSPatrick Mooney enum nvme_storage_type { 157*4c87aefeSPatrick Mooney NVME_STOR_BLOCKIF = 0, 158*4c87aefeSPatrick Mooney NVME_STOR_RAM = 1, 159*4c87aefeSPatrick Mooney }; 160*4c87aefeSPatrick Mooney 161*4c87aefeSPatrick Mooney struct pci_nvme_blockstore { 162*4c87aefeSPatrick Mooney enum nvme_storage_type type; 163*4c87aefeSPatrick Mooney void *ctx; 164*4c87aefeSPatrick Mooney uint64_t size; 165*4c87aefeSPatrick Mooney uint32_t sectsz; 166*4c87aefeSPatrick Mooney uint32_t sectsz_bits; 167*4c87aefeSPatrick Mooney }; 168*4c87aefeSPatrick Mooney 169*4c87aefeSPatrick Mooney struct pci_nvme_ioreq { 170*4c87aefeSPatrick Mooney struct pci_nvme_softc *sc; 171*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *next; 172*4c87aefeSPatrick Mooney struct nvme_submission_queue *nvme_sq; 173*4c87aefeSPatrick Mooney uint16_t sqid; 174*4c87aefeSPatrick Mooney 175*4c87aefeSPatrick Mooney /* command information */ 176*4c87aefeSPatrick Mooney uint16_t opc; 177*4c87aefeSPatrick Mooney uint16_t cid; 178*4c87aefeSPatrick Mooney uint32_t nsid; 179*4c87aefeSPatrick Mooney 180*4c87aefeSPatrick Mooney uint64_t prev_gpaddr; 181*4c87aefeSPatrick Mooney size_t prev_size; 182*4c87aefeSPatrick Mooney 183*4c87aefeSPatrick Mooney /* 184*4c87aefeSPatrick Mooney * lock if all iovs consumed (big IO); 185*4c87aefeSPatrick Mooney * complete transaction before continuing 186*4c87aefeSPatrick Mooney */ 187*4c87aefeSPatrick Mooney pthread_mutex_t mtx; 188*4c87aefeSPatrick Mooney pthread_cond_t cv; 189*4c87aefeSPatrick Mooney 190*4c87aefeSPatrick Mooney struct blockif_req io_req; 191*4c87aefeSPatrick Mooney 192*4c87aefeSPatrick Mooney /* pad to fit up to 512 page descriptors from guest IO request */ 193*4c87aefeSPatrick Mooney struct iovec iovpadding[NVME_MAX_BLOCKIOVS-BLOCKIF_IOV_MAX]; 194*4c87aefeSPatrick Mooney }; 195*4c87aefeSPatrick Mooney 196*4c87aefeSPatrick Mooney struct pci_nvme_softc { 197*4c87aefeSPatrick Mooney struct pci_devinst *nsc_pi; 198*4c87aefeSPatrick Mooney 199*4c87aefeSPatrick Mooney pthread_mutex_t mtx; 200*4c87aefeSPatrick Mooney 201*4c87aefeSPatrick Mooney struct nvme_registers regs; 202*4c87aefeSPatrick Mooney 203*4c87aefeSPatrick Mooney struct nvme_namespace_data nsdata; 204*4c87aefeSPatrick Mooney struct nvme_controller_data ctrldata; 205*4c87aefeSPatrick Mooney struct nvme_error_information_entry err_log; 206*4c87aefeSPatrick Mooney struct nvme_health_information_page health_log; 207*4c87aefeSPatrick Mooney struct nvme_firmware_page fw_log; 208*4c87aefeSPatrick Mooney 209*4c87aefeSPatrick Mooney struct pci_nvme_blockstore nvstore; 210*4c87aefeSPatrick Mooney 211*4c87aefeSPatrick Mooney uint16_t max_qentries; /* max entries per queue */ 212*4c87aefeSPatrick Mooney uint32_t max_queues; /* max number of IO SQ's or CQ's */ 213*4c87aefeSPatrick Mooney uint32_t num_cqueues; 214*4c87aefeSPatrick Mooney uint32_t num_squeues; 215*4c87aefeSPatrick Mooney 216*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *ioreqs; 217*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *ioreqs_free; /* free list of ioreqs */ 218*4c87aefeSPatrick Mooney uint32_t pending_ios; 219*4c87aefeSPatrick Mooney uint32_t ioslots; 220*4c87aefeSPatrick Mooney sem_t iosemlock; 221*4c87aefeSPatrick Mooney 222*4c87aefeSPatrick Mooney /* 223*4c87aefeSPatrick Mooney * Memory mapped Submission and Completion queues 224*4c87aefeSPatrick Mooney * Each array includes both Admin and IO queues 225*4c87aefeSPatrick Mooney */ 226*4c87aefeSPatrick Mooney struct nvme_completion_queue *compl_queues; 227*4c87aefeSPatrick Mooney struct nvme_submission_queue *submit_queues; 228*4c87aefeSPatrick Mooney 229*4c87aefeSPatrick Mooney /* controller features */ 230*4c87aefeSPatrick Mooney uint32_t intr_coales_aggr_time; /* 0x08: uS to delay intr */ 231*4c87aefeSPatrick Mooney uint32_t intr_coales_aggr_thresh; /* 0x08: compl-Q entries */ 232*4c87aefeSPatrick Mooney uint32_t async_ev_config; /* 0x0B: async event config */ 233*4c87aefeSPatrick Mooney }; 234*4c87aefeSPatrick Mooney 235*4c87aefeSPatrick Mooney 236*4c87aefeSPatrick Mooney static void pci_nvme_io_partial(struct blockif_req *br, int err); 237*4c87aefeSPatrick Mooney 238*4c87aefeSPatrick Mooney /* Controller Configuration utils */ 239*4c87aefeSPatrick Mooney #define NVME_CC_GET_EN(cc) \ 240*4c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK) 241*4c87aefeSPatrick Mooney #define NVME_CC_GET_CSS(cc) \ 242*4c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK) 243*4c87aefeSPatrick Mooney #define NVME_CC_GET_SHN(cc) \ 244*4c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK) 245*4c87aefeSPatrick Mooney #define NVME_CC_GET_IOSQES(cc) \ 246*4c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK) 247*4c87aefeSPatrick Mooney #define NVME_CC_GET_IOCQES(cc) \ 248*4c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK) 249*4c87aefeSPatrick Mooney 250*4c87aefeSPatrick Mooney #define NVME_CC_WRITE_MASK \ 251*4c87aefeSPatrick Mooney ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \ 252*4c87aefeSPatrick Mooney (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \ 253*4c87aefeSPatrick Mooney (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT)) 254*4c87aefeSPatrick Mooney 255*4c87aefeSPatrick Mooney #define NVME_CC_NEN_WRITE_MASK \ 256*4c87aefeSPatrick Mooney ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \ 257*4c87aefeSPatrick Mooney (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \ 258*4c87aefeSPatrick Mooney (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT)) 259*4c87aefeSPatrick Mooney 260*4c87aefeSPatrick Mooney /* Controller Status utils */ 261*4c87aefeSPatrick Mooney #define NVME_CSTS_GET_RDY(sts) \ 262*4c87aefeSPatrick Mooney ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK) 263*4c87aefeSPatrick Mooney 264*4c87aefeSPatrick Mooney #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT) 265*4c87aefeSPatrick Mooney 266*4c87aefeSPatrick Mooney /* Completion Queue status word utils */ 267*4c87aefeSPatrick Mooney #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT) 268*4c87aefeSPatrick Mooney #define NVME_STATUS_MASK \ 269*4c87aefeSPatrick Mooney ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\ 270*4c87aefeSPatrick Mooney (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT)) 271*4c87aefeSPatrick Mooney 272*4c87aefeSPatrick Mooney static __inline void 273*4c87aefeSPatrick Mooney cpywithpad(char *dst, size_t dst_size, const char *src, char pad) 274*4c87aefeSPatrick Mooney { 275*4c87aefeSPatrick Mooney size_t len; 276*4c87aefeSPatrick Mooney 277*4c87aefeSPatrick Mooney len = strnlen(src, dst_size); 278*4c87aefeSPatrick Mooney memset(dst, pad, dst_size); 279*4c87aefeSPatrick Mooney memcpy(dst, src, len); 280*4c87aefeSPatrick Mooney } 281*4c87aefeSPatrick Mooney 282*4c87aefeSPatrick Mooney static __inline void 283*4c87aefeSPatrick Mooney pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) 284*4c87aefeSPatrick Mooney { 285*4c87aefeSPatrick Mooney 286*4c87aefeSPatrick Mooney *status &= ~NVME_STATUS_MASK; 287*4c87aefeSPatrick Mooney *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT | 288*4c87aefeSPatrick Mooney (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; 289*4c87aefeSPatrick Mooney } 290*4c87aefeSPatrick Mooney 291*4c87aefeSPatrick Mooney static __inline void 292*4c87aefeSPatrick Mooney pci_nvme_status_genc(uint16_t *status, uint16_t code) 293*4c87aefeSPatrick Mooney { 294*4c87aefeSPatrick Mooney 295*4c87aefeSPatrick Mooney pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); 296*4c87aefeSPatrick Mooney } 297*4c87aefeSPatrick Mooney 298*4c87aefeSPatrick Mooney static __inline void 299*4c87aefeSPatrick Mooney pci_nvme_toggle_phase(uint16_t *status, int prev) 300*4c87aefeSPatrick Mooney { 301*4c87aefeSPatrick Mooney 302*4c87aefeSPatrick Mooney if (prev) 303*4c87aefeSPatrick Mooney *status &= ~NVME_STATUS_P; 304*4c87aefeSPatrick Mooney else 305*4c87aefeSPatrick Mooney *status |= NVME_STATUS_P; 306*4c87aefeSPatrick Mooney } 307*4c87aefeSPatrick Mooney 308*4c87aefeSPatrick Mooney static void 309*4c87aefeSPatrick Mooney pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) 310*4c87aefeSPatrick Mooney { 311*4c87aefeSPatrick Mooney struct nvme_controller_data *cd = &sc->ctrldata; 312*4c87aefeSPatrick Mooney 313*4c87aefeSPatrick Mooney cd->vid = 0xFB5D; 314*4c87aefeSPatrick Mooney cd->ssvid = 0x0000; 315*4c87aefeSPatrick Mooney 316*4c87aefeSPatrick Mooney cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); 317*4c87aefeSPatrick Mooney cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); 318*4c87aefeSPatrick Mooney 319*4c87aefeSPatrick Mooney /* Num of submission commands that we can handle at a time (2^rab) */ 320*4c87aefeSPatrick Mooney cd->rab = 4; 321*4c87aefeSPatrick Mooney 322*4c87aefeSPatrick Mooney /* FreeBSD OUI */ 323*4c87aefeSPatrick Mooney cd->ieee[0] = 0x58; 324*4c87aefeSPatrick Mooney cd->ieee[1] = 0x9c; 325*4c87aefeSPatrick Mooney cd->ieee[2] = 0xfc; 326*4c87aefeSPatrick Mooney 327*4c87aefeSPatrick Mooney cd->mic = 0; 328*4c87aefeSPatrick Mooney 329*4c87aefeSPatrick Mooney cd->mdts = 9; /* max data transfer size (2^mdts * CAP.MPSMIN) */ 330*4c87aefeSPatrick Mooney 331*4c87aefeSPatrick Mooney cd->ver = 0x00010300; 332*4c87aefeSPatrick Mooney 333*4c87aefeSPatrick Mooney cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT; 334*4c87aefeSPatrick Mooney cd->acl = 2; 335*4c87aefeSPatrick Mooney cd->aerl = 4; 336*4c87aefeSPatrick Mooney 337*4c87aefeSPatrick Mooney cd->lpa = 0; /* TODO: support some simple things like SMART */ 338*4c87aefeSPatrick Mooney cd->elpe = 0; /* max error log page entries */ 339*4c87aefeSPatrick Mooney cd->npss = 1; /* number of power states support */ 340*4c87aefeSPatrick Mooney 341*4c87aefeSPatrick Mooney /* Warning Composite Temperature Threshold */ 342*4c87aefeSPatrick Mooney cd->wctemp = 0x0157; 343*4c87aefeSPatrick Mooney 344*4c87aefeSPatrick Mooney cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) | 345*4c87aefeSPatrick Mooney (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT); 346*4c87aefeSPatrick Mooney cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) | 347*4c87aefeSPatrick Mooney (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT); 348*4c87aefeSPatrick Mooney cd->nn = 1; /* number of namespaces */ 349*4c87aefeSPatrick Mooney 350*4c87aefeSPatrick Mooney cd->fna = 0x03; 351*4c87aefeSPatrick Mooney 352*4c87aefeSPatrick Mooney cd->power_state[0].mp = 10; 353*4c87aefeSPatrick Mooney } 354*4c87aefeSPatrick Mooney 355*4c87aefeSPatrick Mooney static void 356*4c87aefeSPatrick Mooney pci_nvme_init_nsdata(struct pci_nvme_softc *sc) 357*4c87aefeSPatrick Mooney { 358*4c87aefeSPatrick Mooney struct nvme_namespace_data *nd; 359*4c87aefeSPatrick Mooney 360*4c87aefeSPatrick Mooney nd = &sc->nsdata; 361*4c87aefeSPatrick Mooney 362*4c87aefeSPatrick Mooney nd->nsze = sc->nvstore.size / sc->nvstore.sectsz; 363*4c87aefeSPatrick Mooney nd->ncap = nd->nsze; 364*4c87aefeSPatrick Mooney nd->nuse = nd->nsze; 365*4c87aefeSPatrick Mooney 366*4c87aefeSPatrick Mooney /* Get LBA and backstore information from backing store */ 367*4c87aefeSPatrick Mooney nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ 368*4c87aefeSPatrick Mooney /* LBA data-sz = 2^lbads */ 369*4c87aefeSPatrick Mooney nd->lbaf[0] = sc->nvstore.sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT; 370*4c87aefeSPatrick Mooney 371*4c87aefeSPatrick Mooney nd->flbas = 0; 372*4c87aefeSPatrick Mooney } 373*4c87aefeSPatrick Mooney 374*4c87aefeSPatrick Mooney static void 375*4c87aefeSPatrick Mooney pci_nvme_init_logpages(struct pci_nvme_softc *sc) 376*4c87aefeSPatrick Mooney { 377*4c87aefeSPatrick Mooney 378*4c87aefeSPatrick Mooney memset(&sc->err_log, 0, sizeof(sc->err_log)); 379*4c87aefeSPatrick Mooney memset(&sc->health_log, 0, sizeof(sc->health_log)); 380*4c87aefeSPatrick Mooney memset(&sc->fw_log, 0, sizeof(sc->fw_log)); 381*4c87aefeSPatrick Mooney } 382*4c87aefeSPatrick Mooney 383*4c87aefeSPatrick Mooney static void 384*4c87aefeSPatrick Mooney pci_nvme_reset_locked(struct pci_nvme_softc *sc) 385*4c87aefeSPatrick Mooney { 386*4c87aefeSPatrick Mooney DPRINTF(("%s\r\n", __func__)); 387*4c87aefeSPatrick Mooney 388*4c87aefeSPatrick Mooney sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | 389*4c87aefeSPatrick Mooney (1 << NVME_CAP_LO_REG_CQR_SHIFT) | 390*4c87aefeSPatrick Mooney (60 << NVME_CAP_LO_REG_TO_SHIFT); 391*4c87aefeSPatrick Mooney 392*4c87aefeSPatrick Mooney sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT; 393*4c87aefeSPatrick Mooney 394*4c87aefeSPatrick Mooney sc->regs.vs = 0x00010300; /* NVMe v1.3 */ 395*4c87aefeSPatrick Mooney 396*4c87aefeSPatrick Mooney sc->regs.cc = 0; 397*4c87aefeSPatrick Mooney sc->regs.csts = 0; 398*4c87aefeSPatrick Mooney 399*4c87aefeSPatrick Mooney sc->num_cqueues = sc->num_squeues = sc->max_queues; 400*4c87aefeSPatrick Mooney if (sc->submit_queues != NULL) { 401*4c87aefeSPatrick Mooney for (int i = 0; i < sc->num_squeues + 1; i++) { 402*4c87aefeSPatrick Mooney /* 403*4c87aefeSPatrick Mooney * The Admin Submission Queue is at index 0. 404*4c87aefeSPatrick Mooney * It must not be changed at reset otherwise the 405*4c87aefeSPatrick Mooney * emulation will be out of sync with the guest. 406*4c87aefeSPatrick Mooney */ 407*4c87aefeSPatrick Mooney if (i != 0) { 408*4c87aefeSPatrick Mooney sc->submit_queues[i].qbase = NULL; 409*4c87aefeSPatrick Mooney sc->submit_queues[i].size = 0; 410*4c87aefeSPatrick Mooney sc->submit_queues[i].cqid = 0; 411*4c87aefeSPatrick Mooney } 412*4c87aefeSPatrick Mooney sc->submit_queues[i].tail = 0; 413*4c87aefeSPatrick Mooney sc->submit_queues[i].head = 0; 414*4c87aefeSPatrick Mooney sc->submit_queues[i].busy = 0; 415*4c87aefeSPatrick Mooney } 416*4c87aefeSPatrick Mooney } else 417*4c87aefeSPatrick Mooney sc->submit_queues = calloc(sc->num_squeues + 1, 418*4c87aefeSPatrick Mooney sizeof(struct nvme_submission_queue)); 419*4c87aefeSPatrick Mooney 420*4c87aefeSPatrick Mooney if (sc->compl_queues != NULL) { 421*4c87aefeSPatrick Mooney for (int i = 0; i < sc->num_cqueues + 1; i++) { 422*4c87aefeSPatrick Mooney /* See Admin Submission Queue note above */ 423*4c87aefeSPatrick Mooney if (i != 0) { 424*4c87aefeSPatrick Mooney sc->compl_queues[i].qbase = NULL; 425*4c87aefeSPatrick Mooney sc->compl_queues[i].size = 0; 426*4c87aefeSPatrick Mooney } 427*4c87aefeSPatrick Mooney 428*4c87aefeSPatrick Mooney sc->compl_queues[i].tail = 0; 429*4c87aefeSPatrick Mooney sc->compl_queues[i].head = 0; 430*4c87aefeSPatrick Mooney } 431*4c87aefeSPatrick Mooney } else { 432*4c87aefeSPatrick Mooney sc->compl_queues = calloc(sc->num_cqueues + 1, 433*4c87aefeSPatrick Mooney sizeof(struct nvme_completion_queue)); 434*4c87aefeSPatrick Mooney 435*4c87aefeSPatrick Mooney for (int i = 0; i < sc->num_cqueues + 1; i++) 436*4c87aefeSPatrick Mooney pthread_mutex_init(&sc->compl_queues[i].mtx, NULL); 437*4c87aefeSPatrick Mooney } 438*4c87aefeSPatrick Mooney } 439*4c87aefeSPatrick Mooney 440*4c87aefeSPatrick Mooney static void 441*4c87aefeSPatrick Mooney pci_nvme_reset(struct pci_nvme_softc *sc) 442*4c87aefeSPatrick Mooney { 443*4c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 444*4c87aefeSPatrick Mooney pci_nvme_reset_locked(sc); 445*4c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 446*4c87aefeSPatrick Mooney } 447*4c87aefeSPatrick Mooney 448*4c87aefeSPatrick Mooney static void 449*4c87aefeSPatrick Mooney pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc) 450*4c87aefeSPatrick Mooney { 451*4c87aefeSPatrick Mooney uint16_t acqs, asqs; 452*4c87aefeSPatrick Mooney 453*4c87aefeSPatrick Mooney DPRINTF(("%s\r\n", __func__)); 454*4c87aefeSPatrick Mooney 455*4c87aefeSPatrick Mooney asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1; 456*4c87aefeSPatrick Mooney sc->submit_queues[0].size = asqs; 457*4c87aefeSPatrick Mooney sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq, 458*4c87aefeSPatrick Mooney sizeof(struct nvme_command) * asqs); 459*4c87aefeSPatrick Mooney 460*4c87aefeSPatrick Mooney DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p\r\n", 461*4c87aefeSPatrick Mooney __func__, sc->regs.asq, sc->submit_queues[0].qbase)); 462*4c87aefeSPatrick Mooney 463*4c87aefeSPatrick Mooney acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & 464*4c87aefeSPatrick Mooney NVME_AQA_REG_ACQS_MASK) + 1; 465*4c87aefeSPatrick Mooney sc->compl_queues[0].size = acqs; 466*4c87aefeSPatrick Mooney sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq, 467*4c87aefeSPatrick Mooney sizeof(struct nvme_completion) * acqs); 468*4c87aefeSPatrick Mooney DPRINTF(("%s mapping Admin-CQ guest 0x%lx, host: %p\r\n", 469*4c87aefeSPatrick Mooney __func__, sc->regs.acq, sc->compl_queues[0].qbase)); 470*4c87aefeSPatrick Mooney } 471*4c87aefeSPatrick Mooney 472*4c87aefeSPatrick Mooney static int 473*4c87aefeSPatrick Mooney nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *src, 474*4c87aefeSPatrick Mooney size_t len) 475*4c87aefeSPatrick Mooney { 476*4c87aefeSPatrick Mooney uint8_t *dst; 477*4c87aefeSPatrick Mooney size_t bytes; 478*4c87aefeSPatrick Mooney 479*4c87aefeSPatrick Mooney if (len > (8 * 1024)) { 480*4c87aefeSPatrick Mooney return (-1); 481*4c87aefeSPatrick Mooney } 482*4c87aefeSPatrick Mooney 483*4c87aefeSPatrick Mooney /* Copy from the start of prp1 to the end of the physical page */ 484*4c87aefeSPatrick Mooney bytes = PAGE_SIZE - (prp1 & PAGE_MASK); 485*4c87aefeSPatrick Mooney bytes = MIN(bytes, len); 486*4c87aefeSPatrick Mooney 487*4c87aefeSPatrick Mooney dst = vm_map_gpa(ctx, prp1, bytes); 488*4c87aefeSPatrick Mooney if (dst == NULL) { 489*4c87aefeSPatrick Mooney return (-1); 490*4c87aefeSPatrick Mooney } 491*4c87aefeSPatrick Mooney 492*4c87aefeSPatrick Mooney memcpy(dst, src, bytes); 493*4c87aefeSPatrick Mooney 494*4c87aefeSPatrick Mooney src += bytes; 495*4c87aefeSPatrick Mooney 496*4c87aefeSPatrick Mooney len -= bytes; 497*4c87aefeSPatrick Mooney if (len == 0) { 498*4c87aefeSPatrick Mooney return (0); 499*4c87aefeSPatrick Mooney } 500*4c87aefeSPatrick Mooney 501*4c87aefeSPatrick Mooney len = MIN(len, PAGE_SIZE); 502*4c87aefeSPatrick Mooney 503*4c87aefeSPatrick Mooney dst = vm_map_gpa(ctx, prp2, len); 504*4c87aefeSPatrick Mooney if (dst == NULL) { 505*4c87aefeSPatrick Mooney return (-1); 506*4c87aefeSPatrick Mooney } 507*4c87aefeSPatrick Mooney 508*4c87aefeSPatrick Mooney memcpy(dst, src, len); 509*4c87aefeSPatrick Mooney 510*4c87aefeSPatrick Mooney return (0); 511*4c87aefeSPatrick Mooney } 512*4c87aefeSPatrick Mooney 513*4c87aefeSPatrick Mooney static int 514*4c87aefeSPatrick Mooney nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 515*4c87aefeSPatrick Mooney struct nvme_completion* compl) 516*4c87aefeSPatrick Mooney { 517*4c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 518*4c87aefeSPatrick Mooney 519*4c87aefeSPatrick Mooney DPRINTF(("%s DELETE_IO_SQ %u\r\n", __func__, qid)); 520*4c87aefeSPatrick Mooney if (qid == 0 || qid > sc->num_squeues) { 521*4c87aefeSPatrick Mooney WPRINTF(("%s NOT PERMITTED queue id %u / num_squeues %u\r\n", 522*4c87aefeSPatrick Mooney __func__, qid, sc->num_squeues)); 523*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 524*4c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 525*4c87aefeSPatrick Mooney return (1); 526*4c87aefeSPatrick Mooney } 527*4c87aefeSPatrick Mooney 528*4c87aefeSPatrick Mooney sc->submit_queues[qid].qbase = NULL; 529*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 530*4c87aefeSPatrick Mooney return (1); 531*4c87aefeSPatrick Mooney } 532*4c87aefeSPatrick Mooney 533*4c87aefeSPatrick Mooney static int 534*4c87aefeSPatrick Mooney nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 535*4c87aefeSPatrick Mooney struct nvme_completion* compl) 536*4c87aefeSPatrick Mooney { 537*4c87aefeSPatrick Mooney if (command->cdw11 & NVME_CMD_CDW11_PC) { 538*4c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 539*4c87aefeSPatrick Mooney struct nvme_submission_queue *nsq; 540*4c87aefeSPatrick Mooney 541*4c87aefeSPatrick Mooney if ((qid == 0) || (qid > sc->num_squeues)) { 542*4c87aefeSPatrick Mooney WPRINTF(("%s queue index %u > num_squeues %u\r\n", 543*4c87aefeSPatrick Mooney __func__, qid, sc->num_squeues)); 544*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, 545*4c87aefeSPatrick Mooney NVME_SCT_COMMAND_SPECIFIC, 546*4c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 547*4c87aefeSPatrick Mooney return (1); 548*4c87aefeSPatrick Mooney } 549*4c87aefeSPatrick Mooney 550*4c87aefeSPatrick Mooney nsq = &sc->submit_queues[qid]; 551*4c87aefeSPatrick Mooney nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 552*4c87aefeSPatrick Mooney 553*4c87aefeSPatrick Mooney nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 554*4c87aefeSPatrick Mooney sizeof(struct nvme_command) * (size_t)nsq->size); 555*4c87aefeSPatrick Mooney nsq->cqid = (command->cdw11 >> 16) & 0xffff; 556*4c87aefeSPatrick Mooney nsq->qpriority = (command->cdw11 >> 1) & 0x03; 557*4c87aefeSPatrick Mooney 558*4c87aefeSPatrick Mooney DPRINTF(("%s sq %u size %u gaddr %p cqid %u\r\n", __func__, 559*4c87aefeSPatrick Mooney qid, nsq->size, nsq->qbase, nsq->cqid)); 560*4c87aefeSPatrick Mooney 561*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 562*4c87aefeSPatrick Mooney 563*4c87aefeSPatrick Mooney DPRINTF(("%s completed creating IOSQ qid %u\r\n", 564*4c87aefeSPatrick Mooney __func__, qid)); 565*4c87aefeSPatrick Mooney } else { 566*4c87aefeSPatrick Mooney /* 567*4c87aefeSPatrick Mooney * Guest sent non-cont submission queue request. 568*4c87aefeSPatrick Mooney * This setting is unsupported by this emulation. 569*4c87aefeSPatrick Mooney */ 570*4c87aefeSPatrick Mooney WPRINTF(("%s unsupported non-contig (list-based) " 571*4c87aefeSPatrick Mooney "create i/o submission queue\r\n", __func__)); 572*4c87aefeSPatrick Mooney 573*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 574*4c87aefeSPatrick Mooney } 575*4c87aefeSPatrick Mooney return (1); 576*4c87aefeSPatrick Mooney } 577*4c87aefeSPatrick Mooney 578*4c87aefeSPatrick Mooney static int 579*4c87aefeSPatrick Mooney nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 580*4c87aefeSPatrick Mooney struct nvme_completion* compl) 581*4c87aefeSPatrick Mooney { 582*4c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 583*4c87aefeSPatrick Mooney 584*4c87aefeSPatrick Mooney DPRINTF(("%s DELETE_IO_CQ %u\r\n", __func__, qid)); 585*4c87aefeSPatrick Mooney if (qid == 0 || qid > sc->num_cqueues) { 586*4c87aefeSPatrick Mooney WPRINTF(("%s queue index %u / num_cqueues %u\r\n", 587*4c87aefeSPatrick Mooney __func__, qid, sc->num_cqueues)); 588*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 589*4c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 590*4c87aefeSPatrick Mooney return (1); 591*4c87aefeSPatrick Mooney } 592*4c87aefeSPatrick Mooney 593*4c87aefeSPatrick Mooney sc->compl_queues[qid].qbase = NULL; 594*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 595*4c87aefeSPatrick Mooney return (1); 596*4c87aefeSPatrick Mooney } 597*4c87aefeSPatrick Mooney 598*4c87aefeSPatrick Mooney static int 599*4c87aefeSPatrick Mooney nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 600*4c87aefeSPatrick Mooney struct nvme_completion* compl) 601*4c87aefeSPatrick Mooney { 602*4c87aefeSPatrick Mooney if (command->cdw11 & NVME_CMD_CDW11_PC) { 603*4c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff; 604*4c87aefeSPatrick Mooney struct nvme_completion_queue *ncq; 605*4c87aefeSPatrick Mooney 606*4c87aefeSPatrick Mooney if ((qid == 0) || (qid > sc->num_cqueues)) { 607*4c87aefeSPatrick Mooney WPRINTF(("%s queue index %u > num_cqueues %u\r\n", 608*4c87aefeSPatrick Mooney __func__, qid, sc->num_cqueues)); 609*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, 610*4c87aefeSPatrick Mooney NVME_SCT_COMMAND_SPECIFIC, 611*4c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER); 612*4c87aefeSPatrick Mooney return (1); 613*4c87aefeSPatrick Mooney } 614*4c87aefeSPatrick Mooney 615*4c87aefeSPatrick Mooney ncq = &sc->compl_queues[qid]; 616*4c87aefeSPatrick Mooney ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; 617*4c87aefeSPatrick Mooney ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; 618*4c87aefeSPatrick Mooney ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 619*4c87aefeSPatrick Mooney 620*4c87aefeSPatrick Mooney ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 621*4c87aefeSPatrick Mooney command->prp1, 622*4c87aefeSPatrick Mooney sizeof(struct nvme_command) * (size_t)ncq->size); 623*4c87aefeSPatrick Mooney 624*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 625*4c87aefeSPatrick Mooney } else { 626*4c87aefeSPatrick Mooney /* 627*4c87aefeSPatrick Mooney * Non-contig completion queue unsupported. 628*4c87aefeSPatrick Mooney */ 629*4c87aefeSPatrick Mooney WPRINTF(("%s unsupported non-contig (list-based) " 630*4c87aefeSPatrick Mooney "create i/o completion queue\r\n", 631*4c87aefeSPatrick Mooney __func__)); 632*4c87aefeSPatrick Mooney 633*4c87aefeSPatrick Mooney /* 0x12 = Invalid Use of Controller Memory Buffer */ 634*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 0x12); 635*4c87aefeSPatrick Mooney } 636*4c87aefeSPatrick Mooney 637*4c87aefeSPatrick Mooney return (1); 638*4c87aefeSPatrick Mooney } 639*4c87aefeSPatrick Mooney 640*4c87aefeSPatrick Mooney static int 641*4c87aefeSPatrick Mooney nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, 642*4c87aefeSPatrick Mooney struct nvme_completion* compl) 643*4c87aefeSPatrick Mooney { 644*4c87aefeSPatrick Mooney uint32_t logsize = (1 + ((command->cdw10 >> 16) & 0xFFF)) * 2; 645*4c87aefeSPatrick Mooney uint8_t logpage = command->cdw10 & 0xFF; 646*4c87aefeSPatrick Mooney 647*4c87aefeSPatrick Mooney DPRINTF(("%s log page %u len %u\r\n", __func__, logpage, logsize)); 648*4c87aefeSPatrick Mooney 649*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 650*4c87aefeSPatrick Mooney 651*4c87aefeSPatrick Mooney switch (logpage) { 652*4c87aefeSPatrick Mooney case NVME_LOG_ERROR: 653*4c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 654*4c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->err_log, logsize); 655*4c87aefeSPatrick Mooney break; 656*4c87aefeSPatrick Mooney case NVME_LOG_HEALTH_INFORMATION: 657*4c87aefeSPatrick Mooney /* TODO: present some smart info */ 658*4c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 659*4c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->health_log, logsize); 660*4c87aefeSPatrick Mooney break; 661*4c87aefeSPatrick Mooney case NVME_LOG_FIRMWARE_SLOT: 662*4c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 663*4c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->fw_log, logsize); 664*4c87aefeSPatrick Mooney break; 665*4c87aefeSPatrick Mooney default: 666*4c87aefeSPatrick Mooney WPRINTF(("%s get log page %x command not supported\r\n", 667*4c87aefeSPatrick Mooney __func__, logpage)); 668*4c87aefeSPatrick Mooney 669*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 670*4c87aefeSPatrick Mooney NVME_SC_INVALID_LOG_PAGE); 671*4c87aefeSPatrick Mooney } 672*4c87aefeSPatrick Mooney 673*4c87aefeSPatrick Mooney return (1); 674*4c87aefeSPatrick Mooney } 675*4c87aefeSPatrick Mooney 676*4c87aefeSPatrick Mooney static int 677*4c87aefeSPatrick Mooney nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, 678*4c87aefeSPatrick Mooney struct nvme_completion* compl) 679*4c87aefeSPatrick Mooney { 680*4c87aefeSPatrick Mooney void *dest; 681*4c87aefeSPatrick Mooney 682*4c87aefeSPatrick Mooney DPRINTF(("%s identify 0x%x nsid 0x%x\r\n", __func__, 683*4c87aefeSPatrick Mooney command->cdw10 & 0xFF, command->nsid)); 684*4c87aefeSPatrick Mooney 685*4c87aefeSPatrick Mooney switch (command->cdw10 & 0xFF) { 686*4c87aefeSPatrick Mooney case 0x00: /* return Identify Namespace data structure */ 687*4c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 688*4c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata)); 689*4c87aefeSPatrick Mooney break; 690*4c87aefeSPatrick Mooney case 0x01: /* return Identify Controller data structure */ 691*4c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 692*4c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->ctrldata, 693*4c87aefeSPatrick Mooney sizeof(sc->ctrldata)); 694*4c87aefeSPatrick Mooney break; 695*4c87aefeSPatrick Mooney case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ 696*4c87aefeSPatrick Mooney dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 697*4c87aefeSPatrick Mooney sizeof(uint32_t) * 1024); 698*4c87aefeSPatrick Mooney ((uint32_t *)dest)[0] = 1; 699*4c87aefeSPatrick Mooney ((uint32_t *)dest)[1] = 0; 700*4c87aefeSPatrick Mooney break; 701*4c87aefeSPatrick Mooney case 0x11: 702*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 703*4c87aefeSPatrick Mooney NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 704*4c87aefeSPatrick Mooney return (1); 705*4c87aefeSPatrick Mooney case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ 706*4c87aefeSPatrick Mooney case 0x10: 707*4c87aefeSPatrick Mooney case 0x12: 708*4c87aefeSPatrick Mooney case 0x13: 709*4c87aefeSPatrick Mooney case 0x14: 710*4c87aefeSPatrick Mooney case 0x15: 711*4c87aefeSPatrick Mooney default: 712*4c87aefeSPatrick Mooney DPRINTF(("%s unsupported identify command requested 0x%x\r\n", 713*4c87aefeSPatrick Mooney __func__, command->cdw10 & 0xFF)); 714*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 715*4c87aefeSPatrick Mooney return (1); 716*4c87aefeSPatrick Mooney } 717*4c87aefeSPatrick Mooney 718*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 719*4c87aefeSPatrick Mooney return (1); 720*4c87aefeSPatrick Mooney } 721*4c87aefeSPatrick Mooney 722*4c87aefeSPatrick Mooney static int 723*4c87aefeSPatrick Mooney nvme_set_feature_queues(struct pci_nvme_softc* sc, struct nvme_command* command, 724*4c87aefeSPatrick Mooney struct nvme_completion* compl) 725*4c87aefeSPatrick Mooney { 726*4c87aefeSPatrick Mooney uint16_t nqr; /* Number of Queues Requested */ 727*4c87aefeSPatrick Mooney 728*4c87aefeSPatrick Mooney nqr = command->cdw11 & 0xFFFF; 729*4c87aefeSPatrick Mooney if (nqr == 0xffff) { 730*4c87aefeSPatrick Mooney WPRINTF(("%s: Illegal NSQR value %#x\n", __func__, nqr)); 731*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 732*4c87aefeSPatrick Mooney return (-1); 733*4c87aefeSPatrick Mooney } 734*4c87aefeSPatrick Mooney 735*4c87aefeSPatrick Mooney sc->num_squeues = ONE_BASED(nqr); 736*4c87aefeSPatrick Mooney if (sc->num_squeues > sc->max_queues) { 737*4c87aefeSPatrick Mooney DPRINTF(("NSQR=%u is greater than max %u\n", sc->num_squeues, 738*4c87aefeSPatrick Mooney sc->max_queues)); 739*4c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues; 740*4c87aefeSPatrick Mooney } 741*4c87aefeSPatrick Mooney 742*4c87aefeSPatrick Mooney nqr = (command->cdw11 >> 16) & 0xFFFF; 743*4c87aefeSPatrick Mooney if (nqr == 0xffff) { 744*4c87aefeSPatrick Mooney WPRINTF(("%s: Illegal NCQR value %#x\n", __func__, nqr)); 745*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 746*4c87aefeSPatrick Mooney return (-1); 747*4c87aefeSPatrick Mooney } 748*4c87aefeSPatrick Mooney 749*4c87aefeSPatrick Mooney sc->num_cqueues = ONE_BASED(nqr); 750*4c87aefeSPatrick Mooney if (sc->num_cqueues > sc->max_queues) { 751*4c87aefeSPatrick Mooney DPRINTF(("NCQR=%u is greater than max %u\n", sc->num_cqueues, 752*4c87aefeSPatrick Mooney sc->max_queues)); 753*4c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues; 754*4c87aefeSPatrick Mooney } 755*4c87aefeSPatrick Mooney 756*4c87aefeSPatrick Mooney compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 757*4c87aefeSPatrick Mooney 758*4c87aefeSPatrick Mooney return (0); 759*4c87aefeSPatrick Mooney } 760*4c87aefeSPatrick Mooney 761*4c87aefeSPatrick Mooney static int 762*4c87aefeSPatrick Mooney nvme_opc_set_features(struct pci_nvme_softc* sc, struct nvme_command* command, 763*4c87aefeSPatrick Mooney struct nvme_completion* compl) 764*4c87aefeSPatrick Mooney { 765*4c87aefeSPatrick Mooney int feature = command->cdw10 & 0xFF; 766*4c87aefeSPatrick Mooney uint32_t iv; 767*4c87aefeSPatrick Mooney 768*4c87aefeSPatrick Mooney DPRINTF(("%s feature 0x%x\r\n", __func__, feature)); 769*4c87aefeSPatrick Mooney compl->cdw0 = 0; 770*4c87aefeSPatrick Mooney 771*4c87aefeSPatrick Mooney switch (feature) { 772*4c87aefeSPatrick Mooney case NVME_FEAT_ARBITRATION: 773*4c87aefeSPatrick Mooney DPRINTF((" arbitration 0x%x\r\n", command->cdw11)); 774*4c87aefeSPatrick Mooney break; 775*4c87aefeSPatrick Mooney case NVME_FEAT_POWER_MANAGEMENT: 776*4c87aefeSPatrick Mooney DPRINTF((" power management 0x%x\r\n", command->cdw11)); 777*4c87aefeSPatrick Mooney break; 778*4c87aefeSPatrick Mooney case NVME_FEAT_LBA_RANGE_TYPE: 779*4c87aefeSPatrick Mooney DPRINTF((" lba range 0x%x\r\n", command->cdw11)); 780*4c87aefeSPatrick Mooney break; 781*4c87aefeSPatrick Mooney case NVME_FEAT_TEMPERATURE_THRESHOLD: 782*4c87aefeSPatrick Mooney DPRINTF((" temperature threshold 0x%x\r\n", command->cdw11)); 783*4c87aefeSPatrick Mooney break; 784*4c87aefeSPatrick Mooney case NVME_FEAT_ERROR_RECOVERY: 785*4c87aefeSPatrick Mooney DPRINTF((" error recovery 0x%x\r\n", command->cdw11)); 786*4c87aefeSPatrick Mooney break; 787*4c87aefeSPatrick Mooney case NVME_FEAT_VOLATILE_WRITE_CACHE: 788*4c87aefeSPatrick Mooney DPRINTF((" volatile write cache 0x%x\r\n", command->cdw11)); 789*4c87aefeSPatrick Mooney break; 790*4c87aefeSPatrick Mooney case NVME_FEAT_NUMBER_OF_QUEUES: 791*4c87aefeSPatrick Mooney nvme_set_feature_queues(sc, command, compl); 792*4c87aefeSPatrick Mooney break; 793*4c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_COALESCING: 794*4c87aefeSPatrick Mooney DPRINTF((" interrupt coalescing 0x%x\r\n", command->cdw11)); 795*4c87aefeSPatrick Mooney 796*4c87aefeSPatrick Mooney /* in uS */ 797*4c87aefeSPatrick Mooney sc->intr_coales_aggr_time = ((command->cdw11 >> 8) & 0xFF)*100; 798*4c87aefeSPatrick Mooney 799*4c87aefeSPatrick Mooney sc->intr_coales_aggr_thresh = command->cdw11 & 0xFF; 800*4c87aefeSPatrick Mooney break; 801*4c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 802*4c87aefeSPatrick Mooney iv = command->cdw11 & 0xFFFF; 803*4c87aefeSPatrick Mooney 804*4c87aefeSPatrick Mooney DPRINTF((" interrupt vector configuration 0x%x\r\n", 805*4c87aefeSPatrick Mooney command->cdw11)); 806*4c87aefeSPatrick Mooney 807*4c87aefeSPatrick Mooney for (uint32_t i = 0; i < sc->num_cqueues + 1; i++) { 808*4c87aefeSPatrick Mooney if (sc->compl_queues[i].intr_vec == iv) { 809*4c87aefeSPatrick Mooney if (command->cdw11 & (1 << 16)) 810*4c87aefeSPatrick Mooney sc->compl_queues[i].intr_en |= 811*4c87aefeSPatrick Mooney NVME_CQ_INTCOAL; 812*4c87aefeSPatrick Mooney else 813*4c87aefeSPatrick Mooney sc->compl_queues[i].intr_en &= 814*4c87aefeSPatrick Mooney ~NVME_CQ_INTCOAL; 815*4c87aefeSPatrick Mooney } 816*4c87aefeSPatrick Mooney } 817*4c87aefeSPatrick Mooney break; 818*4c87aefeSPatrick Mooney case NVME_FEAT_WRITE_ATOMICITY: 819*4c87aefeSPatrick Mooney DPRINTF((" write atomicity 0x%x\r\n", command->cdw11)); 820*4c87aefeSPatrick Mooney break; 821*4c87aefeSPatrick Mooney case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 822*4c87aefeSPatrick Mooney DPRINTF((" async event configuration 0x%x\r\n", 823*4c87aefeSPatrick Mooney command->cdw11)); 824*4c87aefeSPatrick Mooney sc->async_ev_config = command->cdw11; 825*4c87aefeSPatrick Mooney break; 826*4c87aefeSPatrick Mooney case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 827*4c87aefeSPatrick Mooney DPRINTF((" software progress marker 0x%x\r\n", 828*4c87aefeSPatrick Mooney command->cdw11)); 829*4c87aefeSPatrick Mooney break; 830*4c87aefeSPatrick Mooney case 0x0C: 831*4c87aefeSPatrick Mooney DPRINTF((" autonomous power state transition 0x%x\r\n", 832*4c87aefeSPatrick Mooney command->cdw11)); 833*4c87aefeSPatrick Mooney break; 834*4c87aefeSPatrick Mooney default: 835*4c87aefeSPatrick Mooney WPRINTF(("%s invalid feature\r\n", __func__)); 836*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 837*4c87aefeSPatrick Mooney return (1); 838*4c87aefeSPatrick Mooney } 839*4c87aefeSPatrick Mooney 840*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 841*4c87aefeSPatrick Mooney return (1); 842*4c87aefeSPatrick Mooney } 843*4c87aefeSPatrick Mooney 844*4c87aefeSPatrick Mooney static int 845*4c87aefeSPatrick Mooney nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, 846*4c87aefeSPatrick Mooney struct nvme_completion* compl) 847*4c87aefeSPatrick Mooney { 848*4c87aefeSPatrick Mooney int feature = command->cdw10 & 0xFF; 849*4c87aefeSPatrick Mooney 850*4c87aefeSPatrick Mooney DPRINTF(("%s feature 0x%x\r\n", __func__, feature)); 851*4c87aefeSPatrick Mooney 852*4c87aefeSPatrick Mooney compl->cdw0 = 0; 853*4c87aefeSPatrick Mooney 854*4c87aefeSPatrick Mooney switch (feature) { 855*4c87aefeSPatrick Mooney case NVME_FEAT_ARBITRATION: 856*4c87aefeSPatrick Mooney DPRINTF((" arbitration\r\n")); 857*4c87aefeSPatrick Mooney break; 858*4c87aefeSPatrick Mooney case NVME_FEAT_POWER_MANAGEMENT: 859*4c87aefeSPatrick Mooney DPRINTF((" power management\r\n")); 860*4c87aefeSPatrick Mooney break; 861*4c87aefeSPatrick Mooney case NVME_FEAT_LBA_RANGE_TYPE: 862*4c87aefeSPatrick Mooney DPRINTF((" lba range\r\n")); 863*4c87aefeSPatrick Mooney break; 864*4c87aefeSPatrick Mooney case NVME_FEAT_TEMPERATURE_THRESHOLD: 865*4c87aefeSPatrick Mooney DPRINTF((" temperature threshold\r\n")); 866*4c87aefeSPatrick Mooney switch ((command->cdw11 >> 20) & 0x3) { 867*4c87aefeSPatrick Mooney case 0: 868*4c87aefeSPatrick Mooney /* Over temp threshold */ 869*4c87aefeSPatrick Mooney compl->cdw0 = 0xFFFF; 870*4c87aefeSPatrick Mooney break; 871*4c87aefeSPatrick Mooney case 1: 872*4c87aefeSPatrick Mooney /* Under temp threshold */ 873*4c87aefeSPatrick Mooney compl->cdw0 = 0; 874*4c87aefeSPatrick Mooney break; 875*4c87aefeSPatrick Mooney default: 876*4c87aefeSPatrick Mooney WPRINTF((" invalid threshold type select\r\n")); 877*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, 878*4c87aefeSPatrick Mooney NVME_SC_INVALID_FIELD); 879*4c87aefeSPatrick Mooney return (1); 880*4c87aefeSPatrick Mooney } 881*4c87aefeSPatrick Mooney break; 882*4c87aefeSPatrick Mooney case NVME_FEAT_ERROR_RECOVERY: 883*4c87aefeSPatrick Mooney DPRINTF((" error recovery\r\n")); 884*4c87aefeSPatrick Mooney break; 885*4c87aefeSPatrick Mooney case NVME_FEAT_VOLATILE_WRITE_CACHE: 886*4c87aefeSPatrick Mooney DPRINTF((" volatile write cache\r\n")); 887*4c87aefeSPatrick Mooney break; 888*4c87aefeSPatrick Mooney case NVME_FEAT_NUMBER_OF_QUEUES: 889*4c87aefeSPatrick Mooney compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 890*4c87aefeSPatrick Mooney 891*4c87aefeSPatrick Mooney DPRINTF((" number of queues (submit %u, completion %u)\r\n", 892*4c87aefeSPatrick Mooney compl->cdw0 & 0xFFFF, 893*4c87aefeSPatrick Mooney (compl->cdw0 >> 16) & 0xFFFF)); 894*4c87aefeSPatrick Mooney 895*4c87aefeSPatrick Mooney break; 896*4c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_COALESCING: 897*4c87aefeSPatrick Mooney DPRINTF((" interrupt coalescing\r\n")); 898*4c87aefeSPatrick Mooney break; 899*4c87aefeSPatrick Mooney case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 900*4c87aefeSPatrick Mooney DPRINTF((" interrupt vector configuration\r\n")); 901*4c87aefeSPatrick Mooney break; 902*4c87aefeSPatrick Mooney case NVME_FEAT_WRITE_ATOMICITY: 903*4c87aefeSPatrick Mooney DPRINTF((" write atomicity\r\n")); 904*4c87aefeSPatrick Mooney break; 905*4c87aefeSPatrick Mooney case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 906*4c87aefeSPatrick Mooney DPRINTF((" async event configuration\r\n")); 907*4c87aefeSPatrick Mooney sc->async_ev_config = command->cdw11; 908*4c87aefeSPatrick Mooney break; 909*4c87aefeSPatrick Mooney case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 910*4c87aefeSPatrick Mooney DPRINTF((" software progress marker\r\n")); 911*4c87aefeSPatrick Mooney break; 912*4c87aefeSPatrick Mooney case 0x0C: 913*4c87aefeSPatrick Mooney DPRINTF((" autonomous power state transition\r\n")); 914*4c87aefeSPatrick Mooney break; 915*4c87aefeSPatrick Mooney default: 916*4c87aefeSPatrick Mooney WPRINTF(("%s invalid feature 0x%x\r\n", __func__, feature)); 917*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 918*4c87aefeSPatrick Mooney return (1); 919*4c87aefeSPatrick Mooney } 920*4c87aefeSPatrick Mooney 921*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 922*4c87aefeSPatrick Mooney return (1); 923*4c87aefeSPatrick Mooney } 924*4c87aefeSPatrick Mooney 925*4c87aefeSPatrick Mooney static int 926*4c87aefeSPatrick Mooney nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command, 927*4c87aefeSPatrick Mooney struct nvme_completion* compl) 928*4c87aefeSPatrick Mooney { 929*4c87aefeSPatrick Mooney DPRINTF(("%s submission queue %u, command ID 0x%x\r\n", __func__, 930*4c87aefeSPatrick Mooney command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF)); 931*4c87aefeSPatrick Mooney 932*4c87aefeSPatrick Mooney /* TODO: search for the command ID and abort it */ 933*4c87aefeSPatrick Mooney 934*4c87aefeSPatrick Mooney compl->cdw0 = 1; 935*4c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 936*4c87aefeSPatrick Mooney return (1); 937*4c87aefeSPatrick Mooney } 938*4c87aefeSPatrick Mooney 939*4c87aefeSPatrick Mooney #ifdef __FreeBSD__ 940*4c87aefeSPatrick Mooney static int 941*4c87aefeSPatrick Mooney nvme_opc_async_event_req(struct pci_nvme_softc* sc, 942*4c87aefeSPatrick Mooney struct nvme_command* command, struct nvme_completion* compl) 943*4c87aefeSPatrick Mooney { 944*4c87aefeSPatrick Mooney DPRINTF(("%s async event request 0x%x\r\n", __func__, command->cdw11)); 945*4c87aefeSPatrick Mooney 946*4c87aefeSPatrick Mooney /* 947*4c87aefeSPatrick Mooney * TODO: raise events when they happen based on the Set Features cmd. 948*4c87aefeSPatrick Mooney * These events happen async, so only set completion successful if 949*4c87aefeSPatrick Mooney * there is an event reflective of the request to get event. 950*4c87aefeSPatrick Mooney */ 951*4c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 952*4c87aefeSPatrick Mooney NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); 953*4c87aefeSPatrick Mooney return (0); 954*4c87aefeSPatrick Mooney } 955*4c87aefeSPatrick Mooney #else 956*4c87aefeSPatrick Mooney /* This is kept behind an ifdef while it's unused to appease the compiler. */ 957*4c87aefeSPatrick Mooney #endif /* __FreeBSD__ */ 958*4c87aefeSPatrick Mooney 959*4c87aefeSPatrick Mooney static void 960*4c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) 961*4c87aefeSPatrick Mooney { 962*4c87aefeSPatrick Mooney struct nvme_completion compl; 963*4c87aefeSPatrick Mooney struct nvme_command *cmd; 964*4c87aefeSPatrick Mooney struct nvme_submission_queue *sq; 965*4c87aefeSPatrick Mooney struct nvme_completion_queue *cq; 966*4c87aefeSPatrick Mooney int do_intr = 0; 967*4c87aefeSPatrick Mooney uint16_t sqhead; 968*4c87aefeSPatrick Mooney 969*4c87aefeSPatrick Mooney DPRINTF(("%s index %u\r\n", __func__, (uint32_t)value)); 970*4c87aefeSPatrick Mooney 971*4c87aefeSPatrick Mooney sq = &sc->submit_queues[0]; 972*4c87aefeSPatrick Mooney 973*4c87aefeSPatrick Mooney sqhead = atomic_load_acq_short(&sq->head); 974*4c87aefeSPatrick Mooney 975*4c87aefeSPatrick Mooney if (atomic_testandset_int(&sq->busy, 1)) { 976*4c87aefeSPatrick Mooney DPRINTF(("%s SQ busy, head %u, tail %u\r\n", 977*4c87aefeSPatrick Mooney __func__, sqhead, sq->tail)); 978*4c87aefeSPatrick Mooney return; 979*4c87aefeSPatrick Mooney } 980*4c87aefeSPatrick Mooney 981*4c87aefeSPatrick Mooney DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail)); 982*4c87aefeSPatrick Mooney 983*4c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) { 984*4c87aefeSPatrick Mooney cmd = &(sq->qbase)[sqhead]; 985*4c87aefeSPatrick Mooney compl.status = 0; 986*4c87aefeSPatrick Mooney 987*4c87aefeSPatrick Mooney switch (cmd->opc) { 988*4c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_SQ: 989*4c87aefeSPatrick Mooney DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__)); 990*4c87aefeSPatrick Mooney do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl); 991*4c87aefeSPatrick Mooney break; 992*4c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_SQ: 993*4c87aefeSPatrick Mooney DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__)); 994*4c87aefeSPatrick Mooney do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl); 995*4c87aefeSPatrick Mooney break; 996*4c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_CQ: 997*4c87aefeSPatrick Mooney DPRINTF(("%s command DELETE_IO_CQ\r\n", __func__)); 998*4c87aefeSPatrick Mooney do_intr |= nvme_opc_delete_io_cq(sc, cmd, &compl); 999*4c87aefeSPatrick Mooney break; 1000*4c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_CQ: 1001*4c87aefeSPatrick Mooney DPRINTF(("%s command CREATE_IO_CQ\r\n", __func__)); 1002*4c87aefeSPatrick Mooney do_intr |= nvme_opc_create_io_cq(sc, cmd, &compl); 1003*4c87aefeSPatrick Mooney break; 1004*4c87aefeSPatrick Mooney case NVME_OPC_GET_LOG_PAGE: 1005*4c87aefeSPatrick Mooney DPRINTF(("%s command GET_LOG_PAGE\r\n", __func__)); 1006*4c87aefeSPatrick Mooney do_intr |= nvme_opc_get_log_page(sc, cmd, &compl); 1007*4c87aefeSPatrick Mooney break; 1008*4c87aefeSPatrick Mooney case NVME_OPC_IDENTIFY: 1009*4c87aefeSPatrick Mooney DPRINTF(("%s command IDENTIFY\r\n", __func__)); 1010*4c87aefeSPatrick Mooney do_intr |= nvme_opc_identify(sc, cmd, &compl); 1011*4c87aefeSPatrick Mooney break; 1012*4c87aefeSPatrick Mooney case NVME_OPC_ABORT: 1013*4c87aefeSPatrick Mooney DPRINTF(("%s command ABORT\r\n", __func__)); 1014*4c87aefeSPatrick Mooney do_intr |= nvme_opc_abort(sc, cmd, &compl); 1015*4c87aefeSPatrick Mooney break; 1016*4c87aefeSPatrick Mooney case NVME_OPC_SET_FEATURES: 1017*4c87aefeSPatrick Mooney DPRINTF(("%s command SET_FEATURES\r\n", __func__)); 1018*4c87aefeSPatrick Mooney do_intr |= nvme_opc_set_features(sc, cmd, &compl); 1019*4c87aefeSPatrick Mooney break; 1020*4c87aefeSPatrick Mooney case NVME_OPC_GET_FEATURES: 1021*4c87aefeSPatrick Mooney DPRINTF(("%s command GET_FEATURES\r\n", __func__)); 1022*4c87aefeSPatrick Mooney do_intr |= nvme_opc_get_features(sc, cmd, &compl); 1023*4c87aefeSPatrick Mooney break; 1024*4c87aefeSPatrick Mooney case NVME_OPC_ASYNC_EVENT_REQUEST: 1025*4c87aefeSPatrick Mooney DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__)); 1026*4c87aefeSPatrick Mooney /* XXX dont care, unhandled for now 1027*4c87aefeSPatrick Mooney do_intr |= nvme_opc_async_event_req(sc, cmd, &compl); 1028*4c87aefeSPatrick Mooney */ 1029*4c87aefeSPatrick Mooney break; 1030*4c87aefeSPatrick Mooney default: 1031*4c87aefeSPatrick Mooney WPRINTF(("0x%x command is not implemented\r\n", 1032*4c87aefeSPatrick Mooney cmd->opc)); 1033*4c87aefeSPatrick Mooney } 1034*4c87aefeSPatrick Mooney 1035*4c87aefeSPatrick Mooney /* for now skip async event generation */ 1036*4c87aefeSPatrick Mooney if (cmd->opc != NVME_OPC_ASYNC_EVENT_REQUEST) { 1037*4c87aefeSPatrick Mooney struct nvme_completion *cp; 1038*4c87aefeSPatrick Mooney int phase; 1039*4c87aefeSPatrick Mooney 1040*4c87aefeSPatrick Mooney cq = &sc->compl_queues[0]; 1041*4c87aefeSPatrick Mooney 1042*4c87aefeSPatrick Mooney cp = &(cq->qbase)[cq->tail]; 1043*4c87aefeSPatrick Mooney cp->cdw0 = compl.cdw0; 1044*4c87aefeSPatrick Mooney cp->sqid = 0; 1045*4c87aefeSPatrick Mooney cp->sqhd = sqhead; 1046*4c87aefeSPatrick Mooney cp->cid = cmd->cid; 1047*4c87aefeSPatrick Mooney 1048*4c87aefeSPatrick Mooney phase = NVME_STATUS_GET_P(cp->status); 1049*4c87aefeSPatrick Mooney cp->status = compl.status; 1050*4c87aefeSPatrick Mooney pci_nvme_toggle_phase(&cp->status, phase); 1051*4c87aefeSPatrick Mooney 1052*4c87aefeSPatrick Mooney cq->tail = (cq->tail + 1) % cq->size; 1053*4c87aefeSPatrick Mooney } 1054*4c87aefeSPatrick Mooney sqhead = (sqhead + 1) % sq->size; 1055*4c87aefeSPatrick Mooney } 1056*4c87aefeSPatrick Mooney 1057*4c87aefeSPatrick Mooney DPRINTF(("setting sqhead %u\r\n", sqhead)); 1058*4c87aefeSPatrick Mooney atomic_store_short(&sq->head, sqhead); 1059*4c87aefeSPatrick Mooney atomic_store_int(&sq->busy, 0); 1060*4c87aefeSPatrick Mooney 1061*4c87aefeSPatrick Mooney if (do_intr) 1062*4c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, 0); 1063*4c87aefeSPatrick Mooney 1064*4c87aefeSPatrick Mooney } 1065*4c87aefeSPatrick Mooney 1066*4c87aefeSPatrick Mooney static int 1067*4c87aefeSPatrick Mooney pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req, 1068*4c87aefeSPatrick Mooney uint64_t gpaddr, size_t size, int do_write, uint64_t lba) 1069*4c87aefeSPatrick Mooney { 1070*4c87aefeSPatrick Mooney int iovidx; 1071*4c87aefeSPatrick Mooney 1072*4c87aefeSPatrick Mooney if (req != NULL) { 1073*4c87aefeSPatrick Mooney /* concatenate contig block-iovs to minimize number of iovs */ 1074*4c87aefeSPatrick Mooney if ((req->prev_gpaddr + req->prev_size) == gpaddr) { 1075*4c87aefeSPatrick Mooney iovidx = req->io_req.br_iovcnt - 1; 1076*4c87aefeSPatrick Mooney 1077*4c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_base = 1078*4c87aefeSPatrick Mooney paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 1079*4c87aefeSPatrick Mooney req->prev_gpaddr, size); 1080*4c87aefeSPatrick Mooney 1081*4c87aefeSPatrick Mooney req->prev_size += size; 1082*4c87aefeSPatrick Mooney req->io_req.br_resid += size; 1083*4c87aefeSPatrick Mooney 1084*4c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_len = req->prev_size; 1085*4c87aefeSPatrick Mooney } else { 1086*4c87aefeSPatrick Mooney pthread_mutex_lock(&req->mtx); 1087*4c87aefeSPatrick Mooney 1088*4c87aefeSPatrick Mooney iovidx = req->io_req.br_iovcnt; 1089*4c87aefeSPatrick Mooney if (iovidx == NVME_MAX_BLOCKIOVS) { 1090*4c87aefeSPatrick Mooney int err = 0; 1091*4c87aefeSPatrick Mooney 1092*4c87aefeSPatrick Mooney DPRINTF(("large I/O, doing partial req\r\n")); 1093*4c87aefeSPatrick Mooney 1094*4c87aefeSPatrick Mooney iovidx = 0; 1095*4c87aefeSPatrick Mooney req->io_req.br_iovcnt = 0; 1096*4c87aefeSPatrick Mooney 1097*4c87aefeSPatrick Mooney req->io_req.br_callback = pci_nvme_io_partial; 1098*4c87aefeSPatrick Mooney 1099*4c87aefeSPatrick Mooney if (!do_write) 1100*4c87aefeSPatrick Mooney err = blockif_read(sc->nvstore.ctx, 1101*4c87aefeSPatrick Mooney &req->io_req); 1102*4c87aefeSPatrick Mooney else 1103*4c87aefeSPatrick Mooney err = blockif_write(sc->nvstore.ctx, 1104*4c87aefeSPatrick Mooney &req->io_req); 1105*4c87aefeSPatrick Mooney 1106*4c87aefeSPatrick Mooney /* wait until req completes before cont */ 1107*4c87aefeSPatrick Mooney if (err == 0) 1108*4c87aefeSPatrick Mooney pthread_cond_wait(&req->cv, &req->mtx); 1109*4c87aefeSPatrick Mooney } 1110*4c87aefeSPatrick Mooney if (iovidx == 0) { 1111*4c87aefeSPatrick Mooney req->io_req.br_offset = lba; 1112*4c87aefeSPatrick Mooney req->io_req.br_resid = 0; 1113*4c87aefeSPatrick Mooney req->io_req.br_param = req; 1114*4c87aefeSPatrick Mooney } 1115*4c87aefeSPatrick Mooney 1116*4c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_base = 1117*4c87aefeSPatrick Mooney paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 1118*4c87aefeSPatrick Mooney gpaddr, size); 1119*4c87aefeSPatrick Mooney 1120*4c87aefeSPatrick Mooney req->io_req.br_iov[iovidx].iov_len = size; 1121*4c87aefeSPatrick Mooney 1122*4c87aefeSPatrick Mooney req->prev_gpaddr = gpaddr; 1123*4c87aefeSPatrick Mooney req->prev_size = size; 1124*4c87aefeSPatrick Mooney req->io_req.br_resid += size; 1125*4c87aefeSPatrick Mooney 1126*4c87aefeSPatrick Mooney req->io_req.br_iovcnt++; 1127*4c87aefeSPatrick Mooney 1128*4c87aefeSPatrick Mooney pthread_mutex_unlock(&req->mtx); 1129*4c87aefeSPatrick Mooney } 1130*4c87aefeSPatrick Mooney } else { 1131*4c87aefeSPatrick Mooney /* RAM buffer: read/write directly */ 1132*4c87aefeSPatrick Mooney void *p = sc->nvstore.ctx; 1133*4c87aefeSPatrick Mooney void *gptr; 1134*4c87aefeSPatrick Mooney 1135*4c87aefeSPatrick Mooney if ((lba + size) > sc->nvstore.size) { 1136*4c87aefeSPatrick Mooney WPRINTF(("%s write would overflow RAM\r\n", __func__)); 1137*4c87aefeSPatrick Mooney return (-1); 1138*4c87aefeSPatrick Mooney } 1139*4c87aefeSPatrick Mooney 1140*4c87aefeSPatrick Mooney p = (void *)((uintptr_t)p + (uintptr_t)lba); 1141*4c87aefeSPatrick Mooney gptr = paddr_guest2host(sc->nsc_pi->pi_vmctx, gpaddr, size); 1142*4c87aefeSPatrick Mooney if (do_write) 1143*4c87aefeSPatrick Mooney memcpy(p, gptr, size); 1144*4c87aefeSPatrick Mooney else 1145*4c87aefeSPatrick Mooney memcpy(gptr, p, size); 1146*4c87aefeSPatrick Mooney } 1147*4c87aefeSPatrick Mooney return (0); 1148*4c87aefeSPatrick Mooney } 1149*4c87aefeSPatrick Mooney 1150*4c87aefeSPatrick Mooney static void 1151*4c87aefeSPatrick Mooney pci_nvme_set_completion(struct pci_nvme_softc *sc, 1152*4c87aefeSPatrick Mooney struct nvme_submission_queue *sq, int sqid, uint16_t cid, 1153*4c87aefeSPatrick Mooney uint32_t cdw0, uint16_t status, int ignore_busy) 1154*4c87aefeSPatrick Mooney { 1155*4c87aefeSPatrick Mooney struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; 1156*4c87aefeSPatrick Mooney struct nvme_completion *compl; 1157*4c87aefeSPatrick Mooney int do_intr = 0; 1158*4c87aefeSPatrick Mooney int phase; 1159*4c87aefeSPatrick Mooney 1160*4c87aefeSPatrick Mooney DPRINTF(("%s sqid %d cqid %u cid %u status: 0x%x 0x%x\r\n", 1161*4c87aefeSPatrick Mooney __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), 1162*4c87aefeSPatrick Mooney NVME_STATUS_GET_SC(status))); 1163*4c87aefeSPatrick Mooney 1164*4c87aefeSPatrick Mooney pthread_mutex_lock(&cq->mtx); 1165*4c87aefeSPatrick Mooney 1166*4c87aefeSPatrick Mooney assert(cq->qbase != NULL); 1167*4c87aefeSPatrick Mooney 1168*4c87aefeSPatrick Mooney compl = &cq->qbase[cq->tail]; 1169*4c87aefeSPatrick Mooney 1170*4c87aefeSPatrick Mooney compl->sqhd = atomic_load_acq_short(&sq->head); 1171*4c87aefeSPatrick Mooney compl->sqid = sqid; 1172*4c87aefeSPatrick Mooney compl->cid = cid; 1173*4c87aefeSPatrick Mooney 1174*4c87aefeSPatrick Mooney // toggle phase 1175*4c87aefeSPatrick Mooney phase = NVME_STATUS_GET_P(compl->status); 1176*4c87aefeSPatrick Mooney compl->status = status; 1177*4c87aefeSPatrick Mooney pci_nvme_toggle_phase(&compl->status, phase); 1178*4c87aefeSPatrick Mooney 1179*4c87aefeSPatrick Mooney cq->tail = (cq->tail + 1) % cq->size; 1180*4c87aefeSPatrick Mooney 1181*4c87aefeSPatrick Mooney if (cq->intr_en & NVME_CQ_INTEN) 1182*4c87aefeSPatrick Mooney do_intr = 1; 1183*4c87aefeSPatrick Mooney 1184*4c87aefeSPatrick Mooney pthread_mutex_unlock(&cq->mtx); 1185*4c87aefeSPatrick Mooney 1186*4c87aefeSPatrick Mooney if (ignore_busy || !atomic_load_acq_int(&sq->busy)) 1187*4c87aefeSPatrick Mooney if (do_intr) 1188*4c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, cq->intr_vec); 1189*4c87aefeSPatrick Mooney } 1190*4c87aefeSPatrick Mooney 1191*4c87aefeSPatrick Mooney static void 1192*4c87aefeSPatrick Mooney pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) 1193*4c87aefeSPatrick Mooney { 1194*4c87aefeSPatrick Mooney req->sc = NULL; 1195*4c87aefeSPatrick Mooney req->nvme_sq = NULL; 1196*4c87aefeSPatrick Mooney req->sqid = 0; 1197*4c87aefeSPatrick Mooney 1198*4c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 1199*4c87aefeSPatrick Mooney 1200*4c87aefeSPatrick Mooney req->next = sc->ioreqs_free; 1201*4c87aefeSPatrick Mooney sc->ioreqs_free = req; 1202*4c87aefeSPatrick Mooney sc->pending_ios--; 1203*4c87aefeSPatrick Mooney 1204*4c87aefeSPatrick Mooney /* when no more IO pending, can set to ready if device reset/enabled */ 1205*4c87aefeSPatrick Mooney if (sc->pending_ios == 0 && 1206*4c87aefeSPatrick Mooney NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) 1207*4c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY; 1208*4c87aefeSPatrick Mooney 1209*4c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 1210*4c87aefeSPatrick Mooney 1211*4c87aefeSPatrick Mooney sem_post(&sc->iosemlock); 1212*4c87aefeSPatrick Mooney } 1213*4c87aefeSPatrick Mooney 1214*4c87aefeSPatrick Mooney static struct pci_nvme_ioreq * 1215*4c87aefeSPatrick Mooney pci_nvme_get_ioreq(struct pci_nvme_softc *sc) 1216*4c87aefeSPatrick Mooney { 1217*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = NULL;; 1218*4c87aefeSPatrick Mooney 1219*4c87aefeSPatrick Mooney sem_wait(&sc->iosemlock); 1220*4c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 1221*4c87aefeSPatrick Mooney 1222*4c87aefeSPatrick Mooney req = sc->ioreqs_free; 1223*4c87aefeSPatrick Mooney assert(req != NULL); 1224*4c87aefeSPatrick Mooney 1225*4c87aefeSPatrick Mooney sc->ioreqs_free = req->next; 1226*4c87aefeSPatrick Mooney 1227*4c87aefeSPatrick Mooney req->next = NULL; 1228*4c87aefeSPatrick Mooney req->sc = sc; 1229*4c87aefeSPatrick Mooney 1230*4c87aefeSPatrick Mooney sc->pending_ios++; 1231*4c87aefeSPatrick Mooney 1232*4c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 1233*4c87aefeSPatrick Mooney 1234*4c87aefeSPatrick Mooney req->io_req.br_iovcnt = 0; 1235*4c87aefeSPatrick Mooney req->io_req.br_offset = 0; 1236*4c87aefeSPatrick Mooney req->io_req.br_resid = 0; 1237*4c87aefeSPatrick Mooney req->io_req.br_param = req; 1238*4c87aefeSPatrick Mooney req->prev_gpaddr = 0; 1239*4c87aefeSPatrick Mooney req->prev_size = 0; 1240*4c87aefeSPatrick Mooney 1241*4c87aefeSPatrick Mooney return req; 1242*4c87aefeSPatrick Mooney } 1243*4c87aefeSPatrick Mooney 1244*4c87aefeSPatrick Mooney static void 1245*4c87aefeSPatrick Mooney pci_nvme_io_done(struct blockif_req *br, int err) 1246*4c87aefeSPatrick Mooney { 1247*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param; 1248*4c87aefeSPatrick Mooney struct nvme_submission_queue *sq = req->nvme_sq; 1249*4c87aefeSPatrick Mooney uint16_t code, status = 0; 1250*4c87aefeSPatrick Mooney 1251*4c87aefeSPatrick Mooney DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err))); 1252*4c87aefeSPatrick Mooney 1253*4c87aefeSPatrick Mooney /* TODO return correct error */ 1254*4c87aefeSPatrick Mooney code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; 1255*4c87aefeSPatrick Mooney pci_nvme_status_genc(&status, code); 1256*4c87aefeSPatrick Mooney 1257*4c87aefeSPatrick Mooney pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0); 1258*4c87aefeSPatrick Mooney pci_nvme_release_ioreq(req->sc, req); 1259*4c87aefeSPatrick Mooney } 1260*4c87aefeSPatrick Mooney 1261*4c87aefeSPatrick Mooney static void 1262*4c87aefeSPatrick Mooney pci_nvme_io_partial(struct blockif_req *br, int err) 1263*4c87aefeSPatrick Mooney { 1264*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param; 1265*4c87aefeSPatrick Mooney 1266*4c87aefeSPatrick Mooney DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err))); 1267*4c87aefeSPatrick Mooney 1268*4c87aefeSPatrick Mooney pthread_cond_signal(&req->cv); 1269*4c87aefeSPatrick Mooney } 1270*4c87aefeSPatrick Mooney 1271*4c87aefeSPatrick Mooney 1272*4c87aefeSPatrick Mooney static void 1273*4c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) 1274*4c87aefeSPatrick Mooney { 1275*4c87aefeSPatrick Mooney struct nvme_submission_queue *sq; 1276*4c87aefeSPatrick Mooney uint16_t status = 0; 1277*4c87aefeSPatrick Mooney uint16_t sqhead; 1278*4c87aefeSPatrick Mooney int err; 1279*4c87aefeSPatrick Mooney 1280*4c87aefeSPatrick Mooney /* handle all submissions up to sq->tail index */ 1281*4c87aefeSPatrick Mooney sq = &sc->submit_queues[idx]; 1282*4c87aefeSPatrick Mooney 1283*4c87aefeSPatrick Mooney if (atomic_testandset_int(&sq->busy, 1)) { 1284*4c87aefeSPatrick Mooney DPRINTF(("%s sqid %u busy\r\n", __func__, idx)); 1285*4c87aefeSPatrick Mooney return; 1286*4c87aefeSPatrick Mooney } 1287*4c87aefeSPatrick Mooney 1288*4c87aefeSPatrick Mooney sqhead = atomic_load_acq_short(&sq->head); 1289*4c87aefeSPatrick Mooney 1290*4c87aefeSPatrick Mooney DPRINTF(("nvme_handle_io qid %u head %u tail %u cmdlist %p\r\n", 1291*4c87aefeSPatrick Mooney idx, sqhead, sq->tail, sq->qbase)); 1292*4c87aefeSPatrick Mooney 1293*4c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) { 1294*4c87aefeSPatrick Mooney struct nvme_command *cmd; 1295*4c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = NULL; 1296*4c87aefeSPatrick Mooney uint64_t lba; 1297*4c87aefeSPatrick Mooney uint64_t nblocks, bytes, size, cpsz; 1298*4c87aefeSPatrick Mooney 1299*4c87aefeSPatrick Mooney /* TODO: support scatter gather list handling */ 1300*4c87aefeSPatrick Mooney 1301*4c87aefeSPatrick Mooney cmd = &sq->qbase[sqhead]; 1302*4c87aefeSPatrick Mooney sqhead = (sqhead + 1) % sq->size; 1303*4c87aefeSPatrick Mooney 1304*4c87aefeSPatrick Mooney lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; 1305*4c87aefeSPatrick Mooney 1306*4c87aefeSPatrick Mooney if (cmd->opc == NVME_OPC_FLUSH) { 1307*4c87aefeSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 1308*4c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 1309*4c87aefeSPatrick Mooney status, 1); 1310*4c87aefeSPatrick Mooney 1311*4c87aefeSPatrick Mooney continue; 1312*4c87aefeSPatrick Mooney } else if (cmd->opc == 0x08) { 1313*4c87aefeSPatrick Mooney /* TODO: write zeroes */ 1314*4c87aefeSPatrick Mooney WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n", 1315*4c87aefeSPatrick Mooney __func__, lba, cmd->cdw12 & 0xFFFF)); 1316*4c87aefeSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 1317*4c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 1318*4c87aefeSPatrick Mooney status, 1); 1319*4c87aefeSPatrick Mooney 1320*4c87aefeSPatrick Mooney continue; 1321*4c87aefeSPatrick Mooney } 1322*4c87aefeSPatrick Mooney 1323*4c87aefeSPatrick Mooney nblocks = (cmd->cdw12 & 0xFFFF) + 1; 1324*4c87aefeSPatrick Mooney 1325*4c87aefeSPatrick Mooney bytes = nblocks * sc->nvstore.sectsz; 1326*4c87aefeSPatrick Mooney 1327*4c87aefeSPatrick Mooney if (sc->nvstore.type == NVME_STOR_BLOCKIF) { 1328*4c87aefeSPatrick Mooney req = pci_nvme_get_ioreq(sc); 1329*4c87aefeSPatrick Mooney req->nvme_sq = sq; 1330*4c87aefeSPatrick Mooney req->sqid = idx; 1331*4c87aefeSPatrick Mooney } 1332*4c87aefeSPatrick Mooney 1333*4c87aefeSPatrick Mooney /* 1334*4c87aefeSPatrick Mooney * If data starts mid-page and flows into the next page, then 1335*4c87aefeSPatrick Mooney * increase page count 1336*4c87aefeSPatrick Mooney */ 1337*4c87aefeSPatrick Mooney 1338*4c87aefeSPatrick Mooney DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu " 1339*4c87aefeSPatrick Mooney "(%lu-bytes)\r\n", 1340*4c87aefeSPatrick Mooney sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size, 1341*4c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE ? 1342*4c87aefeSPatrick Mooney "WRITE" : "READ", 1343*4c87aefeSPatrick Mooney lba, nblocks, bytes)); 1344*4c87aefeSPatrick Mooney 1345*4c87aefeSPatrick Mooney cmd->prp1 &= ~(0x03UL); 1346*4c87aefeSPatrick Mooney cmd->prp2 &= ~(0x03UL); 1347*4c87aefeSPatrick Mooney 1348*4c87aefeSPatrick Mooney DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2)); 1349*4c87aefeSPatrick Mooney 1350*4c87aefeSPatrick Mooney size = bytes; 1351*4c87aefeSPatrick Mooney lba *= sc->nvstore.sectsz; 1352*4c87aefeSPatrick Mooney 1353*4c87aefeSPatrick Mooney cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE); 1354*4c87aefeSPatrick Mooney 1355*4c87aefeSPatrick Mooney if (cpsz > bytes) 1356*4c87aefeSPatrick Mooney cpsz = bytes; 1357*4c87aefeSPatrick Mooney 1358*4c87aefeSPatrick Mooney if (req != NULL) { 1359*4c87aefeSPatrick Mooney req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) | 1360*4c87aefeSPatrick Mooney cmd->cdw10; 1361*4c87aefeSPatrick Mooney req->opc = cmd->opc; 1362*4c87aefeSPatrick Mooney req->cid = cmd->cid; 1363*4c87aefeSPatrick Mooney req->nsid = cmd->nsid; 1364*4c87aefeSPatrick Mooney } 1365*4c87aefeSPatrick Mooney 1366*4c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz, 1367*4c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, lba); 1368*4c87aefeSPatrick Mooney lba += cpsz; 1369*4c87aefeSPatrick Mooney size -= cpsz; 1370*4c87aefeSPatrick Mooney 1371*4c87aefeSPatrick Mooney if (size == 0) 1372*4c87aefeSPatrick Mooney goto iodone; 1373*4c87aefeSPatrick Mooney 1374*4c87aefeSPatrick Mooney if (size <= PAGE_SIZE) { 1375*4c87aefeSPatrick Mooney /* prp2 is second (and final) page in transfer */ 1376*4c87aefeSPatrick Mooney 1377*4c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, cmd->prp2, 1378*4c87aefeSPatrick Mooney size, 1379*4c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, 1380*4c87aefeSPatrick Mooney lba); 1381*4c87aefeSPatrick Mooney } else { 1382*4c87aefeSPatrick Mooney uint64_t *prp_list; 1383*4c87aefeSPatrick Mooney int i; 1384*4c87aefeSPatrick Mooney 1385*4c87aefeSPatrick Mooney /* prp2 is pointer to a physical region page list */ 1386*4c87aefeSPatrick Mooney prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx, 1387*4c87aefeSPatrick Mooney cmd->prp2, PAGE_SIZE); 1388*4c87aefeSPatrick Mooney 1389*4c87aefeSPatrick Mooney i = 0; 1390*4c87aefeSPatrick Mooney while (size != 0) { 1391*4c87aefeSPatrick Mooney cpsz = MIN(size, PAGE_SIZE); 1392*4c87aefeSPatrick Mooney 1393*4c87aefeSPatrick Mooney /* 1394*4c87aefeSPatrick Mooney * Move to linked physical region page list 1395*4c87aefeSPatrick Mooney * in last item. 1396*4c87aefeSPatrick Mooney */ 1397*4c87aefeSPatrick Mooney if (i == (NVME_PRP2_ITEMS-1) && 1398*4c87aefeSPatrick Mooney size > PAGE_SIZE) { 1399*4c87aefeSPatrick Mooney assert((prp_list[i] & (PAGE_SIZE-1)) == 0); 1400*4c87aefeSPatrick Mooney prp_list = paddr_guest2host( 1401*4c87aefeSPatrick Mooney sc->nsc_pi->pi_vmctx, 1402*4c87aefeSPatrick Mooney prp_list[i], PAGE_SIZE); 1403*4c87aefeSPatrick Mooney i = 0; 1404*4c87aefeSPatrick Mooney } 1405*4c87aefeSPatrick Mooney if (prp_list[i] == 0) { 1406*4c87aefeSPatrick Mooney WPRINTF(("PRP2[%d] = 0 !!!\r\n", i)); 1407*4c87aefeSPatrick Mooney err = 1; 1408*4c87aefeSPatrick Mooney break; 1409*4c87aefeSPatrick Mooney } 1410*4c87aefeSPatrick Mooney 1411*4c87aefeSPatrick Mooney err = pci_nvme_append_iov_req(sc, req, 1412*4c87aefeSPatrick Mooney prp_list[i], cpsz, 1413*4c87aefeSPatrick Mooney cmd->opc == NVME_OPC_WRITE, lba); 1414*4c87aefeSPatrick Mooney if (err) 1415*4c87aefeSPatrick Mooney break; 1416*4c87aefeSPatrick Mooney 1417*4c87aefeSPatrick Mooney lba += cpsz; 1418*4c87aefeSPatrick Mooney size -= cpsz; 1419*4c87aefeSPatrick Mooney i++; 1420*4c87aefeSPatrick Mooney } 1421*4c87aefeSPatrick Mooney } 1422*4c87aefeSPatrick Mooney 1423*4c87aefeSPatrick Mooney iodone: 1424*4c87aefeSPatrick Mooney if (sc->nvstore.type == NVME_STOR_RAM) { 1425*4c87aefeSPatrick Mooney uint16_t code, status = 0; 1426*4c87aefeSPatrick Mooney 1427*4c87aefeSPatrick Mooney code = err ? NVME_SC_LBA_OUT_OF_RANGE : 1428*4c87aefeSPatrick Mooney NVME_SC_SUCCESS; 1429*4c87aefeSPatrick Mooney pci_nvme_status_genc(&status, code); 1430*4c87aefeSPatrick Mooney 1431*4c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 1432*4c87aefeSPatrick Mooney status, 1); 1433*4c87aefeSPatrick Mooney 1434*4c87aefeSPatrick Mooney continue; 1435*4c87aefeSPatrick Mooney } 1436*4c87aefeSPatrick Mooney 1437*4c87aefeSPatrick Mooney 1438*4c87aefeSPatrick Mooney if (err) 1439*4c87aefeSPatrick Mooney goto do_error; 1440*4c87aefeSPatrick Mooney 1441*4c87aefeSPatrick Mooney req->io_req.br_callback = pci_nvme_io_done; 1442*4c87aefeSPatrick Mooney 1443*4c87aefeSPatrick Mooney err = 0; 1444*4c87aefeSPatrick Mooney switch (cmd->opc) { 1445*4c87aefeSPatrick Mooney case NVME_OPC_READ: 1446*4c87aefeSPatrick Mooney err = blockif_read(sc->nvstore.ctx, &req->io_req); 1447*4c87aefeSPatrick Mooney break; 1448*4c87aefeSPatrick Mooney case NVME_OPC_WRITE: 1449*4c87aefeSPatrick Mooney err = blockif_write(sc->nvstore.ctx, &req->io_req); 1450*4c87aefeSPatrick Mooney break; 1451*4c87aefeSPatrick Mooney default: 1452*4c87aefeSPatrick Mooney WPRINTF(("%s unhandled io command 0x%x\r\n", 1453*4c87aefeSPatrick Mooney __func__, cmd->opc)); 1454*4c87aefeSPatrick Mooney err = 1; 1455*4c87aefeSPatrick Mooney } 1456*4c87aefeSPatrick Mooney 1457*4c87aefeSPatrick Mooney do_error: 1458*4c87aefeSPatrick Mooney if (err) { 1459*4c87aefeSPatrick Mooney uint16_t status = 0; 1460*4c87aefeSPatrick Mooney 1461*4c87aefeSPatrick Mooney pci_nvme_status_genc(&status, 1462*4c87aefeSPatrick Mooney NVME_SC_DATA_TRANSFER_ERROR); 1463*4c87aefeSPatrick Mooney 1464*4c87aefeSPatrick Mooney pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 1465*4c87aefeSPatrick Mooney status, 1); 1466*4c87aefeSPatrick Mooney pci_nvme_release_ioreq(sc, req); 1467*4c87aefeSPatrick Mooney } 1468*4c87aefeSPatrick Mooney } 1469*4c87aefeSPatrick Mooney 1470*4c87aefeSPatrick Mooney atomic_store_short(&sq->head, sqhead); 1471*4c87aefeSPatrick Mooney atomic_store_int(&sq->busy, 0); 1472*4c87aefeSPatrick Mooney } 1473*4c87aefeSPatrick Mooney 1474*4c87aefeSPatrick Mooney static void 1475*4c87aefeSPatrick Mooney pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc, 1476*4c87aefeSPatrick Mooney uint64_t idx, int is_sq, uint64_t value) 1477*4c87aefeSPatrick Mooney { 1478*4c87aefeSPatrick Mooney DPRINTF(("nvme doorbell %lu, %s, val 0x%lx\r\n", 1479*4c87aefeSPatrick Mooney idx, is_sq ? "SQ" : "CQ", value & 0xFFFF)); 1480*4c87aefeSPatrick Mooney 1481*4c87aefeSPatrick Mooney if (is_sq) { 1482*4c87aefeSPatrick Mooney atomic_store_short(&sc->submit_queues[idx].tail, 1483*4c87aefeSPatrick Mooney (uint16_t)value); 1484*4c87aefeSPatrick Mooney 1485*4c87aefeSPatrick Mooney if (idx == 0) { 1486*4c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(sc, value); 1487*4c87aefeSPatrick Mooney } else { 1488*4c87aefeSPatrick Mooney /* submission queue; handle new entries in SQ */ 1489*4c87aefeSPatrick Mooney if (idx > sc->num_squeues) { 1490*4c87aefeSPatrick Mooney WPRINTF(("%s SQ index %lu overflow from " 1491*4c87aefeSPatrick Mooney "guest (max %u)\r\n", 1492*4c87aefeSPatrick Mooney __func__, idx, sc->num_squeues)); 1493*4c87aefeSPatrick Mooney return; 1494*4c87aefeSPatrick Mooney } 1495*4c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(sc, (uint16_t)idx); 1496*4c87aefeSPatrick Mooney } 1497*4c87aefeSPatrick Mooney } else { 1498*4c87aefeSPatrick Mooney if (idx > sc->num_cqueues) { 1499*4c87aefeSPatrick Mooney WPRINTF(("%s queue index %lu overflow from " 1500*4c87aefeSPatrick Mooney "guest (max %u)\r\n", 1501*4c87aefeSPatrick Mooney __func__, idx, sc->num_cqueues)); 1502*4c87aefeSPatrick Mooney return; 1503*4c87aefeSPatrick Mooney } 1504*4c87aefeSPatrick Mooney 1505*4c87aefeSPatrick Mooney sc->compl_queues[idx].head = (uint16_t)value; 1506*4c87aefeSPatrick Mooney } 1507*4c87aefeSPatrick Mooney } 1508*4c87aefeSPatrick Mooney 1509*4c87aefeSPatrick Mooney static void 1510*4c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) 1511*4c87aefeSPatrick Mooney { 1512*4c87aefeSPatrick Mooney const char *s = iswrite ? "WRITE" : "READ"; 1513*4c87aefeSPatrick Mooney 1514*4c87aefeSPatrick Mooney switch (offset) { 1515*4c87aefeSPatrick Mooney case NVME_CR_CAP_LOW: 1516*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CAP_LOW\r\n", func, s)); 1517*4c87aefeSPatrick Mooney break; 1518*4c87aefeSPatrick Mooney case NVME_CR_CAP_HI: 1519*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CAP_HI\r\n", func, s)); 1520*4c87aefeSPatrick Mooney break; 1521*4c87aefeSPatrick Mooney case NVME_CR_VS: 1522*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_VS\r\n", func, s)); 1523*4c87aefeSPatrick Mooney break; 1524*4c87aefeSPatrick Mooney case NVME_CR_INTMS: 1525*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_INTMS\r\n", func, s)); 1526*4c87aefeSPatrick Mooney break; 1527*4c87aefeSPatrick Mooney case NVME_CR_INTMC: 1528*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_INTMC\r\n", func, s)); 1529*4c87aefeSPatrick Mooney break; 1530*4c87aefeSPatrick Mooney case NVME_CR_CC: 1531*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CC\r\n", func, s)); 1532*4c87aefeSPatrick Mooney break; 1533*4c87aefeSPatrick Mooney case NVME_CR_CSTS: 1534*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_CSTS\r\n", func, s)); 1535*4c87aefeSPatrick Mooney break; 1536*4c87aefeSPatrick Mooney case NVME_CR_NSSR: 1537*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_NSSR\r\n", func, s)); 1538*4c87aefeSPatrick Mooney break; 1539*4c87aefeSPatrick Mooney case NVME_CR_AQA: 1540*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_AQA\r\n", func, s)); 1541*4c87aefeSPatrick Mooney break; 1542*4c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW: 1543*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ASQ_LOW\r\n", func, s)); 1544*4c87aefeSPatrick Mooney break; 1545*4c87aefeSPatrick Mooney case NVME_CR_ASQ_HI: 1546*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ASQ_HI\r\n", func, s)); 1547*4c87aefeSPatrick Mooney break; 1548*4c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW: 1549*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ACQ_LOW\r\n", func, s)); 1550*4c87aefeSPatrick Mooney break; 1551*4c87aefeSPatrick Mooney case NVME_CR_ACQ_HI: 1552*4c87aefeSPatrick Mooney DPRINTF(("%s %s NVME_CR_ACQ_HI\r\n", func, s)); 1553*4c87aefeSPatrick Mooney break; 1554*4c87aefeSPatrick Mooney default: 1555*4c87aefeSPatrick Mooney DPRINTF(("unknown nvme bar-0 offset 0x%lx\r\n", offset)); 1556*4c87aefeSPatrick Mooney } 1557*4c87aefeSPatrick Mooney 1558*4c87aefeSPatrick Mooney } 1559*4c87aefeSPatrick Mooney 1560*4c87aefeSPatrick Mooney static void 1561*4c87aefeSPatrick Mooney pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc, 1562*4c87aefeSPatrick Mooney uint64_t offset, int size, uint64_t value) 1563*4c87aefeSPatrick Mooney { 1564*4c87aefeSPatrick Mooney uint32_t ccreg; 1565*4c87aefeSPatrick Mooney 1566*4c87aefeSPatrick Mooney if (offset >= NVME_DOORBELL_OFFSET) { 1567*4c87aefeSPatrick Mooney uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; 1568*4c87aefeSPatrick Mooney uint64_t idx = belloffset / 8; /* door bell size = 2*int */ 1569*4c87aefeSPatrick Mooney int is_sq = (belloffset % 8) < 4; 1570*4c87aefeSPatrick Mooney 1571*4c87aefeSPatrick Mooney if (belloffset > ((sc->max_queues+1) * 8 - 4)) { 1572*4c87aefeSPatrick Mooney WPRINTF(("guest attempted an overflow write offset " 1573*4c87aefeSPatrick Mooney "0x%lx, val 0x%lx in %s", 1574*4c87aefeSPatrick Mooney offset, value, __func__)); 1575*4c87aefeSPatrick Mooney return; 1576*4c87aefeSPatrick Mooney } 1577*4c87aefeSPatrick Mooney 1578*4c87aefeSPatrick Mooney pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value); 1579*4c87aefeSPatrick Mooney return; 1580*4c87aefeSPatrick Mooney } 1581*4c87aefeSPatrick Mooney 1582*4c87aefeSPatrick Mooney DPRINTF(("nvme-write offset 0x%lx, size %d, value 0x%lx\r\n", 1583*4c87aefeSPatrick Mooney offset, size, value)); 1584*4c87aefeSPatrick Mooney 1585*4c87aefeSPatrick Mooney if (size != 4) { 1586*4c87aefeSPatrick Mooney WPRINTF(("guest wrote invalid size %d (offset 0x%lx, " 1587*4c87aefeSPatrick Mooney "val 0x%lx) to bar0 in %s", 1588*4c87aefeSPatrick Mooney size, offset, value, __func__)); 1589*4c87aefeSPatrick Mooney /* TODO: shutdown device */ 1590*4c87aefeSPatrick Mooney return; 1591*4c87aefeSPatrick Mooney } 1592*4c87aefeSPatrick Mooney 1593*4c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 1); 1594*4c87aefeSPatrick Mooney 1595*4c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 1596*4c87aefeSPatrick Mooney 1597*4c87aefeSPatrick Mooney switch (offset) { 1598*4c87aefeSPatrick Mooney case NVME_CR_CAP_LOW: 1599*4c87aefeSPatrick Mooney case NVME_CR_CAP_HI: 1600*4c87aefeSPatrick Mooney /* readonly */ 1601*4c87aefeSPatrick Mooney break; 1602*4c87aefeSPatrick Mooney case NVME_CR_VS: 1603*4c87aefeSPatrick Mooney /* readonly */ 1604*4c87aefeSPatrick Mooney break; 1605*4c87aefeSPatrick Mooney case NVME_CR_INTMS: 1606*4c87aefeSPatrick Mooney /* MSI-X, so ignore */ 1607*4c87aefeSPatrick Mooney break; 1608*4c87aefeSPatrick Mooney case NVME_CR_INTMC: 1609*4c87aefeSPatrick Mooney /* MSI-X, so ignore */ 1610*4c87aefeSPatrick Mooney break; 1611*4c87aefeSPatrick Mooney case NVME_CR_CC: 1612*4c87aefeSPatrick Mooney ccreg = (uint32_t)value; 1613*4c87aefeSPatrick Mooney 1614*4c87aefeSPatrick Mooney DPRINTF(("%s NVME_CR_CC en %x css %x shn %x iosqes %u " 1615*4c87aefeSPatrick Mooney "iocqes %u\r\n", 1616*4c87aefeSPatrick Mooney __func__, 1617*4c87aefeSPatrick Mooney NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), 1618*4c87aefeSPatrick Mooney NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), 1619*4c87aefeSPatrick Mooney NVME_CC_GET_IOCQES(ccreg))); 1620*4c87aefeSPatrick Mooney 1621*4c87aefeSPatrick Mooney if (NVME_CC_GET_SHN(ccreg)) { 1622*4c87aefeSPatrick Mooney /* perform shutdown - flush out data to backend */ 1623*4c87aefeSPatrick Mooney sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK << 1624*4c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT); 1625*4c87aefeSPatrick Mooney sc->regs.csts |= NVME_SHST_COMPLETE << 1626*4c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT; 1627*4c87aefeSPatrick Mooney } 1628*4c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { 1629*4c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0) 1630*4c87aefeSPatrick Mooney /* transition 1-> causes controller reset */ 1631*4c87aefeSPatrick Mooney pci_nvme_reset_locked(sc); 1632*4c87aefeSPatrick Mooney else 1633*4c87aefeSPatrick Mooney pci_nvme_init_controller(ctx, sc); 1634*4c87aefeSPatrick Mooney } 1635*4c87aefeSPatrick Mooney 1636*4c87aefeSPatrick Mooney /* Insert the iocqes, iosqes and en bits from the write */ 1637*4c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_WRITE_MASK; 1638*4c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; 1639*4c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0) { 1640*4c87aefeSPatrick Mooney /* Insert the ams, mps and css bit fields */ 1641*4c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; 1642*4c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; 1643*4c87aefeSPatrick Mooney sc->regs.csts &= ~NVME_CSTS_RDY; 1644*4c87aefeSPatrick Mooney } else if (sc->pending_ios == 0) { 1645*4c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY; 1646*4c87aefeSPatrick Mooney } 1647*4c87aefeSPatrick Mooney break; 1648*4c87aefeSPatrick Mooney case NVME_CR_CSTS: 1649*4c87aefeSPatrick Mooney break; 1650*4c87aefeSPatrick Mooney case NVME_CR_NSSR: 1651*4c87aefeSPatrick Mooney /* ignore writes; don't support subsystem reset */ 1652*4c87aefeSPatrick Mooney break; 1653*4c87aefeSPatrick Mooney case NVME_CR_AQA: 1654*4c87aefeSPatrick Mooney sc->regs.aqa = (uint32_t)value; 1655*4c87aefeSPatrick Mooney break; 1656*4c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW: 1657*4c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | 1658*4c87aefeSPatrick Mooney (0xFFFFF000 & value); 1659*4c87aefeSPatrick Mooney break; 1660*4c87aefeSPatrick Mooney case NVME_CR_ASQ_HI: 1661*4c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | 1662*4c87aefeSPatrick Mooney (value << 32); 1663*4c87aefeSPatrick Mooney break; 1664*4c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW: 1665*4c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | 1666*4c87aefeSPatrick Mooney (0xFFFFF000 & value); 1667*4c87aefeSPatrick Mooney break; 1668*4c87aefeSPatrick Mooney case NVME_CR_ACQ_HI: 1669*4c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | 1670*4c87aefeSPatrick Mooney (value << 32); 1671*4c87aefeSPatrick Mooney break; 1672*4c87aefeSPatrick Mooney default: 1673*4c87aefeSPatrick Mooney DPRINTF(("%s unknown offset 0x%lx, value 0x%lx size %d\r\n", 1674*4c87aefeSPatrick Mooney __func__, offset, value, size)); 1675*4c87aefeSPatrick Mooney } 1676*4c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 1677*4c87aefeSPatrick Mooney } 1678*4c87aefeSPatrick Mooney 1679*4c87aefeSPatrick Mooney static void 1680*4c87aefeSPatrick Mooney pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, 1681*4c87aefeSPatrick Mooney int baridx, uint64_t offset, int size, uint64_t value) 1682*4c87aefeSPatrick Mooney { 1683*4c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg; 1684*4c87aefeSPatrick Mooney 1685*4c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) || 1686*4c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) { 1687*4c87aefeSPatrick Mooney DPRINTF(("nvme-write baridx %d, msix: off 0x%lx, size %d, " 1688*4c87aefeSPatrick Mooney " value 0x%lx\r\n", baridx, offset, size, value)); 1689*4c87aefeSPatrick Mooney 1690*4c87aefeSPatrick Mooney pci_emul_msix_twrite(pi, offset, size, value); 1691*4c87aefeSPatrick Mooney return; 1692*4c87aefeSPatrick Mooney } 1693*4c87aefeSPatrick Mooney 1694*4c87aefeSPatrick Mooney switch (baridx) { 1695*4c87aefeSPatrick Mooney case 0: 1696*4c87aefeSPatrick Mooney pci_nvme_write_bar_0(ctx, sc, offset, size, value); 1697*4c87aefeSPatrick Mooney break; 1698*4c87aefeSPatrick Mooney 1699*4c87aefeSPatrick Mooney default: 1700*4c87aefeSPatrick Mooney DPRINTF(("%s unknown baridx %d, val 0x%lx\r\n", 1701*4c87aefeSPatrick Mooney __func__, baridx, value)); 1702*4c87aefeSPatrick Mooney } 1703*4c87aefeSPatrick Mooney } 1704*4c87aefeSPatrick Mooney 1705*4c87aefeSPatrick Mooney static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, 1706*4c87aefeSPatrick Mooney uint64_t offset, int size) 1707*4c87aefeSPatrick Mooney { 1708*4c87aefeSPatrick Mooney uint64_t value; 1709*4c87aefeSPatrick Mooney 1710*4c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 0); 1711*4c87aefeSPatrick Mooney 1712*4c87aefeSPatrick Mooney if (offset < NVME_DOORBELL_OFFSET) { 1713*4c87aefeSPatrick Mooney void *p = &(sc->regs); 1714*4c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx); 1715*4c87aefeSPatrick Mooney memcpy(&value, (void *)((uintptr_t)p + offset), size); 1716*4c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx); 1717*4c87aefeSPatrick Mooney } else { 1718*4c87aefeSPatrick Mooney value = 0; 1719*4c87aefeSPatrick Mooney WPRINTF(("pci_nvme: read invalid offset %ld\r\n", offset)); 1720*4c87aefeSPatrick Mooney } 1721*4c87aefeSPatrick Mooney 1722*4c87aefeSPatrick Mooney switch (size) { 1723*4c87aefeSPatrick Mooney case 1: 1724*4c87aefeSPatrick Mooney value &= 0xFF; 1725*4c87aefeSPatrick Mooney break; 1726*4c87aefeSPatrick Mooney case 2: 1727*4c87aefeSPatrick Mooney value &= 0xFFFF; 1728*4c87aefeSPatrick Mooney break; 1729*4c87aefeSPatrick Mooney case 4: 1730*4c87aefeSPatrick Mooney value &= 0xFFFFFFFF; 1731*4c87aefeSPatrick Mooney break; 1732*4c87aefeSPatrick Mooney } 1733*4c87aefeSPatrick Mooney 1734*4c87aefeSPatrick Mooney DPRINTF((" nvme-read offset 0x%lx, size %d -> value 0x%x\r\n", 1735*4c87aefeSPatrick Mooney offset, size, (uint32_t)value)); 1736*4c87aefeSPatrick Mooney 1737*4c87aefeSPatrick Mooney return (value); 1738*4c87aefeSPatrick Mooney } 1739*4c87aefeSPatrick Mooney 1740*4c87aefeSPatrick Mooney 1741*4c87aefeSPatrick Mooney 1742*4c87aefeSPatrick Mooney static uint64_t 1743*4c87aefeSPatrick Mooney pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, 1744*4c87aefeSPatrick Mooney uint64_t offset, int size) 1745*4c87aefeSPatrick Mooney { 1746*4c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg; 1747*4c87aefeSPatrick Mooney 1748*4c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) || 1749*4c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) { 1750*4c87aefeSPatrick Mooney DPRINTF(("nvme-read bar: %d, msix: regoff 0x%lx, size %d\r\n", 1751*4c87aefeSPatrick Mooney baridx, offset, size)); 1752*4c87aefeSPatrick Mooney 1753*4c87aefeSPatrick Mooney return pci_emul_msix_tread(pi, offset, size); 1754*4c87aefeSPatrick Mooney } 1755*4c87aefeSPatrick Mooney 1756*4c87aefeSPatrick Mooney switch (baridx) { 1757*4c87aefeSPatrick Mooney case 0: 1758*4c87aefeSPatrick Mooney return pci_nvme_read_bar_0(sc, offset, size); 1759*4c87aefeSPatrick Mooney 1760*4c87aefeSPatrick Mooney default: 1761*4c87aefeSPatrick Mooney DPRINTF(("unknown bar %d, 0x%lx\r\n", baridx, offset)); 1762*4c87aefeSPatrick Mooney } 1763*4c87aefeSPatrick Mooney 1764*4c87aefeSPatrick Mooney return (0); 1765*4c87aefeSPatrick Mooney } 1766*4c87aefeSPatrick Mooney 1767*4c87aefeSPatrick Mooney 1768*4c87aefeSPatrick Mooney static int 1769*4c87aefeSPatrick Mooney pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts) 1770*4c87aefeSPatrick Mooney { 1771*4c87aefeSPatrick Mooney char bident[sizeof("XX:X:X")]; 1772*4c87aefeSPatrick Mooney char *uopt, *xopts, *config; 1773*4c87aefeSPatrick Mooney uint32_t sectsz; 1774*4c87aefeSPatrick Mooney int optidx; 1775*4c87aefeSPatrick Mooney 1776*4c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES; 1777*4c87aefeSPatrick Mooney sc->max_qentries = NVME_MAX_QENTRIES; 1778*4c87aefeSPatrick Mooney sc->ioslots = NVME_IOSLOTS; 1779*4c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues; 1780*4c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues; 1781*4c87aefeSPatrick Mooney sectsz = 0; 1782*4c87aefeSPatrick Mooney 1783*4c87aefeSPatrick Mooney uopt = strdup(opts); 1784*4c87aefeSPatrick Mooney optidx = 0; 1785*4c87aefeSPatrick Mooney snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), 1786*4c87aefeSPatrick Mooney "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 1787*4c87aefeSPatrick Mooney for (xopts = strtok(uopt, ","); 1788*4c87aefeSPatrick Mooney xopts != NULL; 1789*4c87aefeSPatrick Mooney xopts = strtok(NULL, ",")) { 1790*4c87aefeSPatrick Mooney 1791*4c87aefeSPatrick Mooney if ((config = strchr(xopts, '=')) != NULL) 1792*4c87aefeSPatrick Mooney *config++ = '\0'; 1793*4c87aefeSPatrick Mooney 1794*4c87aefeSPatrick Mooney if (!strcmp("maxq", xopts)) { 1795*4c87aefeSPatrick Mooney sc->max_queues = atoi(config); 1796*4c87aefeSPatrick Mooney } else if (!strcmp("qsz", xopts)) { 1797*4c87aefeSPatrick Mooney sc->max_qentries = atoi(config); 1798*4c87aefeSPatrick Mooney } else if (!strcmp("ioslots", xopts)) { 1799*4c87aefeSPatrick Mooney sc->ioslots = atoi(config); 1800*4c87aefeSPatrick Mooney } else if (!strcmp("sectsz", xopts)) { 1801*4c87aefeSPatrick Mooney sectsz = atoi(config); 1802*4c87aefeSPatrick Mooney } else if (!strcmp("ser", xopts)) { 1803*4c87aefeSPatrick Mooney /* 1804*4c87aefeSPatrick Mooney * This field indicates the Product Serial Number in 1805*4c87aefeSPatrick Mooney * 7-bit ASCII, unused bytes should be space characters. 1806*4c87aefeSPatrick Mooney * Ref: NVMe v1.3c. 1807*4c87aefeSPatrick Mooney */ 1808*4c87aefeSPatrick Mooney cpywithpad((char *)sc->ctrldata.sn, 1809*4c87aefeSPatrick Mooney sizeof(sc->ctrldata.sn), config, ' '); 1810*4c87aefeSPatrick Mooney } else if (!strcmp("ram", xopts)) { 1811*4c87aefeSPatrick Mooney uint64_t sz = strtoull(&xopts[4], NULL, 10); 1812*4c87aefeSPatrick Mooney 1813*4c87aefeSPatrick Mooney sc->nvstore.type = NVME_STOR_RAM; 1814*4c87aefeSPatrick Mooney sc->nvstore.size = sz * 1024 * 1024; 1815*4c87aefeSPatrick Mooney sc->nvstore.ctx = calloc(1, sc->nvstore.size); 1816*4c87aefeSPatrick Mooney sc->nvstore.sectsz = 4096; 1817*4c87aefeSPatrick Mooney sc->nvstore.sectsz_bits = 12; 1818*4c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL) { 1819*4c87aefeSPatrick Mooney perror("Unable to allocate RAM"); 1820*4c87aefeSPatrick Mooney free(uopt); 1821*4c87aefeSPatrick Mooney return (-1); 1822*4c87aefeSPatrick Mooney } 1823*4c87aefeSPatrick Mooney } else if (optidx == 0) { 1824*4c87aefeSPatrick Mooney snprintf(bident, sizeof(bident), "%d:%d", 1825*4c87aefeSPatrick Mooney sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 1826*4c87aefeSPatrick Mooney sc->nvstore.ctx = blockif_open(xopts, bident); 1827*4c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL) { 1828*4c87aefeSPatrick Mooney perror("Could not open backing file"); 1829*4c87aefeSPatrick Mooney free(uopt); 1830*4c87aefeSPatrick Mooney return (-1); 1831*4c87aefeSPatrick Mooney } 1832*4c87aefeSPatrick Mooney sc->nvstore.type = NVME_STOR_BLOCKIF; 1833*4c87aefeSPatrick Mooney sc->nvstore.size = blockif_size(sc->nvstore.ctx); 1834*4c87aefeSPatrick Mooney } else { 1835*4c87aefeSPatrick Mooney fprintf(stderr, "Invalid option %s\n", xopts); 1836*4c87aefeSPatrick Mooney free(uopt); 1837*4c87aefeSPatrick Mooney return (-1); 1838*4c87aefeSPatrick Mooney } 1839*4c87aefeSPatrick Mooney 1840*4c87aefeSPatrick Mooney optidx++; 1841*4c87aefeSPatrick Mooney } 1842*4c87aefeSPatrick Mooney free(uopt); 1843*4c87aefeSPatrick Mooney 1844*4c87aefeSPatrick Mooney if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) { 1845*4c87aefeSPatrick Mooney fprintf(stderr, "backing store not specified\n"); 1846*4c87aefeSPatrick Mooney return (-1); 1847*4c87aefeSPatrick Mooney } 1848*4c87aefeSPatrick Mooney if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) 1849*4c87aefeSPatrick Mooney sc->nvstore.sectsz = sectsz; 1850*4c87aefeSPatrick Mooney else if (sc->nvstore.type != NVME_STOR_RAM) 1851*4c87aefeSPatrick Mooney sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); 1852*4c87aefeSPatrick Mooney for (sc->nvstore.sectsz_bits = 9; 1853*4c87aefeSPatrick Mooney (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; 1854*4c87aefeSPatrick Mooney sc->nvstore.sectsz_bits++); 1855*4c87aefeSPatrick Mooney 1856*4c87aefeSPatrick Mooney if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) 1857*4c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES; 1858*4c87aefeSPatrick Mooney 1859*4c87aefeSPatrick Mooney if (sc->max_qentries <= 0) { 1860*4c87aefeSPatrick Mooney fprintf(stderr, "Invalid qsz option\n"); 1861*4c87aefeSPatrick Mooney return (-1); 1862*4c87aefeSPatrick Mooney } 1863*4c87aefeSPatrick Mooney if (sc->ioslots <= 0) { 1864*4c87aefeSPatrick Mooney fprintf(stderr, "Invalid ioslots option\n"); 1865*4c87aefeSPatrick Mooney return (-1); 1866*4c87aefeSPatrick Mooney } 1867*4c87aefeSPatrick Mooney 1868*4c87aefeSPatrick Mooney return (0); 1869*4c87aefeSPatrick Mooney } 1870*4c87aefeSPatrick Mooney 1871*4c87aefeSPatrick Mooney static int 1872*4c87aefeSPatrick Mooney pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) 1873*4c87aefeSPatrick Mooney { 1874*4c87aefeSPatrick Mooney struct pci_nvme_softc *sc; 1875*4c87aefeSPatrick Mooney uint32_t pci_membar_sz; 1876*4c87aefeSPatrick Mooney int error; 1877*4c87aefeSPatrick Mooney 1878*4c87aefeSPatrick Mooney error = 0; 1879*4c87aefeSPatrick Mooney 1880*4c87aefeSPatrick Mooney sc = calloc(1, sizeof(struct pci_nvme_softc)); 1881*4c87aefeSPatrick Mooney pi->pi_arg = sc; 1882*4c87aefeSPatrick Mooney sc->nsc_pi = pi; 1883*4c87aefeSPatrick Mooney 1884*4c87aefeSPatrick Mooney error = pci_nvme_parse_opts(sc, opts); 1885*4c87aefeSPatrick Mooney if (error < 0) 1886*4c87aefeSPatrick Mooney goto done; 1887*4c87aefeSPatrick Mooney else 1888*4c87aefeSPatrick Mooney error = 0; 1889*4c87aefeSPatrick Mooney 1890*4c87aefeSPatrick Mooney sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); 1891*4c87aefeSPatrick Mooney for (int i = 0; i < sc->ioslots; i++) { 1892*4c87aefeSPatrick Mooney if (i < (sc->ioslots-1)) 1893*4c87aefeSPatrick Mooney sc->ioreqs[i].next = &sc->ioreqs[i+1]; 1894*4c87aefeSPatrick Mooney pthread_mutex_init(&sc->ioreqs[i].mtx, NULL); 1895*4c87aefeSPatrick Mooney pthread_cond_init(&sc->ioreqs[i].cv, NULL); 1896*4c87aefeSPatrick Mooney } 1897*4c87aefeSPatrick Mooney sc->ioreqs_free = sc->ioreqs; 1898*4c87aefeSPatrick Mooney sc->intr_coales_aggr_thresh = 1; 1899*4c87aefeSPatrick Mooney 1900*4c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); 1901*4c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); 1902*4c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 1903*4c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); 1904*4c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_PROGIF, 1905*4c87aefeSPatrick Mooney PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); 1906*4c87aefeSPatrick Mooney 1907*4c87aefeSPatrick Mooney /* 1908*4c87aefeSPatrick Mooney * Allocate size of NVMe registers + doorbell space for all queues. 1909*4c87aefeSPatrick Mooney * 1910*4c87aefeSPatrick Mooney * The specification requires a minimum memory I/O window size of 16K. 1911*4c87aefeSPatrick Mooney * The Windows driver will refuse to start a device with a smaller 1912*4c87aefeSPatrick Mooney * window. 1913*4c87aefeSPatrick Mooney */ 1914*4c87aefeSPatrick Mooney pci_membar_sz = sizeof(struct nvme_registers) + 1915*4c87aefeSPatrick Mooney 2 * sizeof(uint32_t) * (sc->max_queues + 1); 1916*4c87aefeSPatrick Mooney pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); 1917*4c87aefeSPatrick Mooney 1918*4c87aefeSPatrick Mooney DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz)); 1919*4c87aefeSPatrick Mooney 1920*4c87aefeSPatrick Mooney error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); 1921*4c87aefeSPatrick Mooney if (error) { 1922*4c87aefeSPatrick Mooney WPRINTF(("%s pci alloc mem bar failed\r\n", __func__)); 1923*4c87aefeSPatrick Mooney goto done; 1924*4c87aefeSPatrick Mooney } 1925*4c87aefeSPatrick Mooney 1926*4c87aefeSPatrick Mooney error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); 1927*4c87aefeSPatrick Mooney if (error) { 1928*4c87aefeSPatrick Mooney WPRINTF(("%s pci add msixcap failed\r\n", __func__)); 1929*4c87aefeSPatrick Mooney goto done; 1930*4c87aefeSPatrick Mooney } 1931*4c87aefeSPatrick Mooney 1932*4c87aefeSPatrick Mooney pthread_mutex_init(&sc->mtx, NULL); 1933*4c87aefeSPatrick Mooney sem_init(&sc->iosemlock, 0, sc->ioslots); 1934*4c87aefeSPatrick Mooney 1935*4c87aefeSPatrick Mooney pci_nvme_reset(sc); 1936*4c87aefeSPatrick Mooney pci_nvme_init_ctrldata(sc); 1937*4c87aefeSPatrick Mooney pci_nvme_init_nsdata(sc); 1938*4c87aefeSPatrick Mooney pci_nvme_init_logpages(sc); 1939*4c87aefeSPatrick Mooney 1940*4c87aefeSPatrick Mooney pci_lintr_request(pi); 1941*4c87aefeSPatrick Mooney 1942*4c87aefeSPatrick Mooney done: 1943*4c87aefeSPatrick Mooney return (error); 1944*4c87aefeSPatrick Mooney } 1945*4c87aefeSPatrick Mooney 1946*4c87aefeSPatrick Mooney 1947*4c87aefeSPatrick Mooney struct pci_devemu pci_de_nvme = { 1948*4c87aefeSPatrick Mooney .pe_emu = "nvme", 1949*4c87aefeSPatrick Mooney .pe_init = pci_nvme_init, 1950*4c87aefeSPatrick Mooney .pe_barwrite = pci_nvme_write, 1951*4c87aefeSPatrick Mooney .pe_barread = pci_nvme_read 1952*4c87aefeSPatrick Mooney }; 1953*4c87aefeSPatrick Mooney PCI_EMUL_SET(pci_de_nvme); 1954