1bb0ec6b3SJim Harris /*- 2bb0ec6b3SJim Harris * Copyright (C) 2012 Intel Corporation 3bb0ec6b3SJim Harris * All rights reserved. 4bb0ec6b3SJim Harris * 5bb0ec6b3SJim Harris * Redistribution and use in source and binary forms, with or without 6bb0ec6b3SJim Harris * modification, are permitted provided that the following conditions 7bb0ec6b3SJim Harris * are met: 8bb0ec6b3SJim Harris * 1. Redistributions of source code must retain the above copyright 9bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer. 10bb0ec6b3SJim Harris * 2. Redistributions in binary form must reproduce the above copyright 11bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer in the 12bb0ec6b3SJim Harris * documentation and/or other materials provided with the distribution. 13bb0ec6b3SJim Harris * 14bb0ec6b3SJim Harris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15bb0ec6b3SJim Harris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16bb0ec6b3SJim Harris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17bb0ec6b3SJim Harris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18bb0ec6b3SJim Harris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19bb0ec6b3SJim Harris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20bb0ec6b3SJim Harris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21bb0ec6b3SJim Harris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22bb0ec6b3SJim Harris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23bb0ec6b3SJim Harris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24bb0ec6b3SJim Harris * SUCH DAMAGE. 25bb0ec6b3SJim Harris */ 26bb0ec6b3SJim Harris 27bb0ec6b3SJim Harris #include <sys/cdefs.h> 28bb0ec6b3SJim Harris __FBSDID("$FreeBSD$"); 29bb0ec6b3SJim Harris 30bb0ec6b3SJim Harris #include <sys/param.h> 31bb0ec6b3SJim Harris #include <sys/bus.h> 32bb0ec6b3SJim Harris #include <sys/conf.h> 33bb0ec6b3SJim Harris #include <sys/ioccom.h> 34bb0ec6b3SJim Harris #include <sys/smp.h> 35bb0ec6b3SJim Harris 36bb0ec6b3SJim Harris #include <dev/pci/pcireg.h> 37bb0ec6b3SJim Harris #include <dev/pci/pcivar.h> 38bb0ec6b3SJim Harris 39bb0ec6b3SJim Harris #include "nvme_private.h" 40bb0ec6b3SJim Harris 41bb0ec6b3SJim Harris static void 42bb0ec6b3SJim Harris nvme_ctrlr_cb(void *arg, const struct nvme_completion *status) 43bb0ec6b3SJim Harris { 44bb0ec6b3SJim Harris struct nvme_completion *cpl = arg; 45bb0ec6b3SJim Harris struct mtx *mtx; 46bb0ec6b3SJim Harris 47bb0ec6b3SJim Harris /* 48bb0ec6b3SJim Harris * Copy status into the argument passed by the caller, so that 49bb0ec6b3SJim Harris * the caller can check the status to determine if the 50bb0ec6b3SJim Harris * the request passed or failed. 51bb0ec6b3SJim Harris */ 52bb0ec6b3SJim Harris memcpy(cpl, status, sizeof(*cpl)); 53bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, cpl); 54bb0ec6b3SJim Harris mtx_lock(mtx); 55bb0ec6b3SJim Harris wakeup(cpl); 56bb0ec6b3SJim Harris mtx_unlock(mtx); 57bb0ec6b3SJim Harris } 58bb0ec6b3SJim Harris 59bb0ec6b3SJim Harris static int 60bb0ec6b3SJim Harris nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr) 61bb0ec6b3SJim Harris { 62bb0ec6b3SJim Harris 63bb0ec6b3SJim Harris /* Chatham puts the NVMe MMRs behind BAR 2/3, not BAR 0/1. */ 64bb0ec6b3SJim Harris if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 65bb0ec6b3SJim Harris ctrlr->resource_id = PCIR_BAR(2); 66bb0ec6b3SJim Harris else 67bb0ec6b3SJim Harris ctrlr->resource_id = PCIR_BAR(0); 68bb0ec6b3SJim Harris 69bb0ec6b3SJim Harris ctrlr->resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY, 70bb0ec6b3SJim Harris &ctrlr->resource_id, 0, ~0, 1, RF_ACTIVE); 71bb0ec6b3SJim Harris 72bb0ec6b3SJim Harris if(ctrlr->resource == NULL) { 73bb0ec6b3SJim Harris device_printf(ctrlr->dev, "unable to allocate pci resource\n"); 74bb0ec6b3SJim Harris return (ENOMEM); 75bb0ec6b3SJim Harris } 76bb0ec6b3SJim Harris 77bb0ec6b3SJim Harris ctrlr->bus_tag = rman_get_bustag(ctrlr->resource); 78bb0ec6b3SJim Harris ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource); 79bb0ec6b3SJim Harris ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle; 80bb0ec6b3SJim Harris 81bb0ec6b3SJim Harris return (0); 82bb0ec6b3SJim Harris } 83bb0ec6b3SJim Harris 84bb0ec6b3SJim Harris #ifdef CHATHAM2 85bb0ec6b3SJim Harris static int 86bb0ec6b3SJim Harris nvme_ctrlr_allocate_chatham_bar(struct nvme_controller *ctrlr) 87bb0ec6b3SJim Harris { 88bb0ec6b3SJim Harris 89bb0ec6b3SJim Harris ctrlr->chatham_resource_id = PCIR_BAR(CHATHAM_CONTROL_BAR); 90bb0ec6b3SJim Harris ctrlr->chatham_resource = bus_alloc_resource(ctrlr->dev, 91bb0ec6b3SJim Harris SYS_RES_MEMORY, &ctrlr->chatham_resource_id, 0, ~0, 1, 92bb0ec6b3SJim Harris RF_ACTIVE); 93bb0ec6b3SJim Harris 94bb0ec6b3SJim Harris if(ctrlr->chatham_resource == NULL) { 95bb0ec6b3SJim Harris device_printf(ctrlr->dev, "unable to alloc pci resource\n"); 96bb0ec6b3SJim Harris return (ENOMEM); 97bb0ec6b3SJim Harris } 98bb0ec6b3SJim Harris 99bb0ec6b3SJim Harris ctrlr->chatham_bus_tag = rman_get_bustag(ctrlr->chatham_resource); 100bb0ec6b3SJim Harris ctrlr->chatham_bus_handle = 101bb0ec6b3SJim Harris rman_get_bushandle(ctrlr->chatham_resource); 102bb0ec6b3SJim Harris 103bb0ec6b3SJim Harris return (0); 104bb0ec6b3SJim Harris } 105bb0ec6b3SJim Harris 106bb0ec6b3SJim Harris static void 107bb0ec6b3SJim Harris nvme_ctrlr_setup_chatham(struct nvme_controller *ctrlr) 108bb0ec6b3SJim Harris { 109bb0ec6b3SJim Harris uint64_t reg1, reg2, reg3; 110bb0ec6b3SJim Harris uint64_t temp1, temp2; 111bb0ec6b3SJim Harris uint32_t temp3; 112bb0ec6b3SJim Harris uint32_t use_flash_timings = 0; 113bb0ec6b3SJim Harris 114bb0ec6b3SJim Harris DELAY(10000); 115bb0ec6b3SJim Harris 116bb0ec6b3SJim Harris temp3 = chatham_read_4(ctrlr, 0x8080); 117bb0ec6b3SJim Harris 118bb0ec6b3SJim Harris device_printf(ctrlr->dev, "Chatham version: 0x%x\n", temp3); 119bb0ec6b3SJim Harris 120bb0ec6b3SJim Harris ctrlr->chatham_lbas = chatham_read_4(ctrlr, 0x8068) - 0x110; 121bb0ec6b3SJim Harris ctrlr->chatham_size = ctrlr->chatham_lbas * 512; 122bb0ec6b3SJim Harris 123bb0ec6b3SJim Harris device_printf(ctrlr->dev, "Chatham size: %lld\n", 124bb0ec6b3SJim Harris (long long)ctrlr->chatham_size); 125bb0ec6b3SJim Harris 126bb0ec6b3SJim Harris reg1 = reg2 = reg3 = ctrlr->chatham_size - 1; 127bb0ec6b3SJim Harris 128bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.use_flash_timings", &use_flash_timings); 129bb0ec6b3SJim Harris if (use_flash_timings) { 130bb0ec6b3SJim Harris device_printf(ctrlr->dev, "Chatham: using flash timings\n"); 131bb0ec6b3SJim Harris temp1 = 0x00001b58000007d0LL; 132bb0ec6b3SJim Harris temp2 = 0x000000cb00000131LL; 133bb0ec6b3SJim Harris } else { 134bb0ec6b3SJim Harris device_printf(ctrlr->dev, "Chatham: using DDR timings\n"); 135bb0ec6b3SJim Harris temp1 = temp2 = 0x0LL; 136bb0ec6b3SJim Harris } 137bb0ec6b3SJim Harris 138bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8000, reg1); 139bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8008, reg2); 140bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8010, reg3); 141bb0ec6b3SJim Harris 142bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8020, temp1); 143bb0ec6b3SJim Harris temp3 = chatham_read_4(ctrlr, 0x8020); 144bb0ec6b3SJim Harris 145bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8028, temp2); 146bb0ec6b3SJim Harris temp3 = chatham_read_4(ctrlr, 0x8028); 147bb0ec6b3SJim Harris 148bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8030, temp1); 149bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8038, temp2); 150bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8040, temp1); 151bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8048, temp2); 152bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8050, temp1); 153bb0ec6b3SJim Harris chatham_write_8(ctrlr, 0x8058, temp2); 154bb0ec6b3SJim Harris 155bb0ec6b3SJim Harris DELAY(10000); 156bb0ec6b3SJim Harris } 157bb0ec6b3SJim Harris 158bb0ec6b3SJim Harris static void 159bb0ec6b3SJim Harris nvme_chatham_populate_cdata(struct nvme_controller *ctrlr) 160bb0ec6b3SJim Harris { 161bb0ec6b3SJim Harris struct nvme_controller_data *cdata; 162bb0ec6b3SJim Harris 163bb0ec6b3SJim Harris cdata = &ctrlr->cdata; 164bb0ec6b3SJim Harris 165bb0ec6b3SJim Harris cdata->vid = 0x8086; 166bb0ec6b3SJim Harris cdata->ssvid = 0x2011; 167bb0ec6b3SJim Harris 168bb0ec6b3SJim Harris /* 169bb0ec6b3SJim Harris * Chatham2 puts garbage data in these fields when we 170bb0ec6b3SJim Harris * invoke IDENTIFY_CONTROLLER, so we need to re-zero 171bb0ec6b3SJim Harris * the fields before calling bcopy(). 172bb0ec6b3SJim Harris */ 173bb0ec6b3SJim Harris memset(cdata->sn, 0, sizeof(cdata->sn)); 174bb0ec6b3SJim Harris memcpy(cdata->sn, "2012", strlen("2012")); 175bb0ec6b3SJim Harris memset(cdata->mn, 0, sizeof(cdata->mn)); 176bb0ec6b3SJim Harris memcpy(cdata->mn, "CHATHAM2", strlen("CHATHAM2")); 177bb0ec6b3SJim Harris memset(cdata->fr, 0, sizeof(cdata->fr)); 178bb0ec6b3SJim Harris memcpy(cdata->fr, "0", strlen("0")); 179bb0ec6b3SJim Harris cdata->rab = 8; 180bb0ec6b3SJim Harris cdata->aerl = 3; 181bb0ec6b3SJim Harris cdata->lpa.ns_smart = 1; 182bb0ec6b3SJim Harris cdata->sqes.min = 6; 183bb0ec6b3SJim Harris cdata->sqes.max = 6; 184bb0ec6b3SJim Harris cdata->sqes.min = 4; 185bb0ec6b3SJim Harris cdata->sqes.max = 4; 186bb0ec6b3SJim Harris cdata->nn = 1; 187bb0ec6b3SJim Harris 188bb0ec6b3SJim Harris /* Chatham2 doesn't support DSM command */ 189bb0ec6b3SJim Harris cdata->oncs.dsm = 0; 190bb0ec6b3SJim Harris 191bb0ec6b3SJim Harris cdata->vwc.present = 1; 192bb0ec6b3SJim Harris } 193bb0ec6b3SJim Harris #endif /* CHATHAM2 */ 194bb0ec6b3SJim Harris 195bb0ec6b3SJim Harris static void 196bb0ec6b3SJim Harris nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr) 197bb0ec6b3SJim Harris { 198bb0ec6b3SJim Harris struct nvme_qpair *qpair; 199bb0ec6b3SJim Harris uint32_t num_entries; 200bb0ec6b3SJim Harris 201bb0ec6b3SJim Harris qpair = &ctrlr->adminq; 202bb0ec6b3SJim Harris 203bb0ec6b3SJim Harris num_entries = NVME_ADMIN_ENTRIES; 204bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries); 205bb0ec6b3SJim Harris /* 206bb0ec6b3SJim Harris * If admin_entries was overridden to an invalid value, revert it 207bb0ec6b3SJim Harris * back to our default value. 208bb0ec6b3SJim Harris */ 209bb0ec6b3SJim Harris if (num_entries < NVME_MIN_ADMIN_ENTRIES || 210bb0ec6b3SJim Harris num_entries > NVME_MAX_ADMIN_ENTRIES) { 211bb0ec6b3SJim Harris printf("nvme: invalid hw.nvme.admin_entries=%d specified\n", 212bb0ec6b3SJim Harris num_entries); 213bb0ec6b3SJim Harris num_entries = NVME_ADMIN_ENTRIES; 214bb0ec6b3SJim Harris } 215bb0ec6b3SJim Harris 216bb0ec6b3SJim Harris /* 217bb0ec6b3SJim Harris * The admin queue's max xfer size is treated differently than the 218bb0ec6b3SJim Harris * max I/O xfer size. 16KB is sufficient here - maybe even less? 219bb0ec6b3SJim Harris */ 220bb0ec6b3SJim Harris nvme_qpair_construct(qpair, 0, 0, num_entries, 16*1024, ctrlr); 221bb0ec6b3SJim Harris } 222bb0ec6b3SJim Harris 223bb0ec6b3SJim Harris static int 224bb0ec6b3SJim Harris nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr) 225bb0ec6b3SJim Harris { 226bb0ec6b3SJim Harris struct nvme_qpair *qpair; 227bb0ec6b3SJim Harris union cap_lo_register cap_lo; 228bb0ec6b3SJim Harris int i, num_entries; 229bb0ec6b3SJim Harris 230bb0ec6b3SJim Harris num_entries = NVME_IO_ENTRIES; 231bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); 232bb0ec6b3SJim Harris 233bb0ec6b3SJim Harris num_entries = max(num_entries, NVME_MIN_IO_ENTRIES); 234bb0ec6b3SJim Harris 235bb0ec6b3SJim Harris /* 236bb0ec6b3SJim Harris * NVMe spec sets a hard limit of 64K max entries, but 237bb0ec6b3SJim Harris * devices may specify a smaller limit, so we need to check 238bb0ec6b3SJim Harris * the MQES field in the capabilities register. 239bb0ec6b3SJim Harris */ 240bb0ec6b3SJim Harris cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo); 241bb0ec6b3SJim Harris num_entries = min(num_entries, cap_lo.bits.mqes+1); 242bb0ec6b3SJim Harris 243bb0ec6b3SJim Harris ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; 244bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.max_xfer_size", &ctrlr->max_xfer_size); 245bb0ec6b3SJim Harris /* 246bb0ec6b3SJim Harris * Check that tunable doesn't specify a size greater than what our 247bb0ec6b3SJim Harris * driver supports, and is an even PAGE_SIZE multiple. 248bb0ec6b3SJim Harris */ 249bb0ec6b3SJim Harris if (ctrlr->max_xfer_size > NVME_MAX_XFER_SIZE || 250bb0ec6b3SJim Harris ctrlr->max_xfer_size % PAGE_SIZE) 251bb0ec6b3SJim Harris ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; 252bb0ec6b3SJim Harris 253bb0ec6b3SJim Harris ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair), 254bb0ec6b3SJim Harris M_NVME, M_ZERO | M_NOWAIT); 255bb0ec6b3SJim Harris 256bb0ec6b3SJim Harris if (ctrlr->ioq == NULL) 257bb0ec6b3SJim Harris return (ENOMEM); 258bb0ec6b3SJim Harris 259bb0ec6b3SJim Harris for (i = 0; i < ctrlr->num_io_queues; i++) { 260bb0ec6b3SJim Harris qpair = &ctrlr->ioq[i]; 261bb0ec6b3SJim Harris 262bb0ec6b3SJim Harris /* 263bb0ec6b3SJim Harris * Admin queue has ID=0. IO queues start at ID=1 - 264bb0ec6b3SJim Harris * hence the 'i+1' here. 265bb0ec6b3SJim Harris * 266bb0ec6b3SJim Harris * For I/O queues, use the controller-wide max_xfer_size 267bb0ec6b3SJim Harris * calculated in nvme_attach(). 268bb0ec6b3SJim Harris */ 269bb0ec6b3SJim Harris nvme_qpair_construct(qpair, 270bb0ec6b3SJim Harris i+1, /* qpair ID */ 271bb0ec6b3SJim Harris ctrlr->msix_enabled ? i+1 : 0, /* vector */ 272bb0ec6b3SJim Harris num_entries, 273bb0ec6b3SJim Harris ctrlr->max_xfer_size, 274bb0ec6b3SJim Harris ctrlr); 275bb0ec6b3SJim Harris 276bb0ec6b3SJim Harris if (ctrlr->per_cpu_io_queues) 277bb0ec6b3SJim Harris bus_bind_intr(ctrlr->dev, qpair->res, i); 278bb0ec6b3SJim Harris } 279bb0ec6b3SJim Harris 280bb0ec6b3SJim Harris return (0); 281bb0ec6b3SJim Harris } 282bb0ec6b3SJim Harris 283bb0ec6b3SJim Harris static int 284bb0ec6b3SJim Harris nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr) 285bb0ec6b3SJim Harris { 286bb0ec6b3SJim Harris int ms_waited; 287bb0ec6b3SJim Harris union cc_register cc; 288bb0ec6b3SJim Harris union csts_register csts; 289bb0ec6b3SJim Harris 290bb0ec6b3SJim Harris cc.raw = nvme_mmio_read_4(ctrlr, cc); 291bb0ec6b3SJim Harris csts.raw = nvme_mmio_read_4(ctrlr, csts); 292bb0ec6b3SJim Harris 293bb0ec6b3SJim Harris if (!cc.bits.en) { 294bb0ec6b3SJim Harris device_printf(ctrlr->dev, "%s called with cc.en = 0\n", 295bb0ec6b3SJim Harris __func__); 296bb0ec6b3SJim Harris return (ENXIO); 297bb0ec6b3SJim Harris } 298bb0ec6b3SJim Harris 299bb0ec6b3SJim Harris ms_waited = 0; 300bb0ec6b3SJim Harris 301bb0ec6b3SJim Harris while (!csts.bits.rdy) { 302bb0ec6b3SJim Harris DELAY(1000); 303bb0ec6b3SJim Harris if (ms_waited++ > ctrlr->ready_timeout_in_ms) { 304bb0ec6b3SJim Harris device_printf(ctrlr->dev, "controller did not become " 305bb0ec6b3SJim Harris "ready within %d ms\n", ctrlr->ready_timeout_in_ms); 306bb0ec6b3SJim Harris return (ENXIO); 307bb0ec6b3SJim Harris } 308bb0ec6b3SJim Harris csts.raw = nvme_mmio_read_4(ctrlr, csts); 309bb0ec6b3SJim Harris } 310bb0ec6b3SJim Harris 311bb0ec6b3SJim Harris return (0); 312bb0ec6b3SJim Harris } 313bb0ec6b3SJim Harris 314bb0ec6b3SJim Harris static void 315bb0ec6b3SJim Harris nvme_ctrlr_disable(struct nvme_controller *ctrlr) 316bb0ec6b3SJim Harris { 317bb0ec6b3SJim Harris union cc_register cc; 318bb0ec6b3SJim Harris union csts_register csts; 319bb0ec6b3SJim Harris 320bb0ec6b3SJim Harris cc.raw = nvme_mmio_read_4(ctrlr, cc); 321bb0ec6b3SJim Harris csts.raw = nvme_mmio_read_4(ctrlr, csts); 322bb0ec6b3SJim Harris 323bb0ec6b3SJim Harris if (cc.bits.en == 1 && csts.bits.rdy == 0) 324bb0ec6b3SJim Harris nvme_ctrlr_wait_for_ready(ctrlr); 325bb0ec6b3SJim Harris 326bb0ec6b3SJim Harris cc.bits.en = 0; 327bb0ec6b3SJim Harris nvme_mmio_write_4(ctrlr, cc, cc.raw); 328bb0ec6b3SJim Harris DELAY(5000); 329bb0ec6b3SJim Harris } 330bb0ec6b3SJim Harris 331bb0ec6b3SJim Harris static int 332bb0ec6b3SJim Harris nvme_ctrlr_enable(struct nvme_controller *ctrlr) 333bb0ec6b3SJim Harris { 334bb0ec6b3SJim Harris union cc_register cc; 335bb0ec6b3SJim Harris union csts_register csts; 336bb0ec6b3SJim Harris union aqa_register aqa; 337bb0ec6b3SJim Harris 338bb0ec6b3SJim Harris cc.raw = nvme_mmio_read_4(ctrlr, cc); 339bb0ec6b3SJim Harris csts.raw = nvme_mmio_read_4(ctrlr, csts); 340bb0ec6b3SJim Harris 341bb0ec6b3SJim Harris if (cc.bits.en == 1) { 342bb0ec6b3SJim Harris if (csts.bits.rdy == 1) 343bb0ec6b3SJim Harris return (0); 344bb0ec6b3SJim Harris else 345bb0ec6b3SJim Harris return (nvme_ctrlr_wait_for_ready(ctrlr)); 346bb0ec6b3SJim Harris } 347bb0ec6b3SJim Harris 348bb0ec6b3SJim Harris nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr); 349bb0ec6b3SJim Harris DELAY(5000); 350bb0ec6b3SJim Harris nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr); 351bb0ec6b3SJim Harris DELAY(5000); 352bb0ec6b3SJim Harris 353bb0ec6b3SJim Harris aqa.raw = 0; 354bb0ec6b3SJim Harris /* acqs and asqs are 0-based. */ 355bb0ec6b3SJim Harris aqa.bits.acqs = ctrlr->adminq.num_entries-1; 356bb0ec6b3SJim Harris aqa.bits.asqs = ctrlr->adminq.num_entries-1; 357bb0ec6b3SJim Harris nvme_mmio_write_4(ctrlr, aqa, aqa.raw); 358bb0ec6b3SJim Harris DELAY(5000); 359bb0ec6b3SJim Harris 360bb0ec6b3SJim Harris cc.bits.en = 1; 361bb0ec6b3SJim Harris cc.bits.css = 0; 362bb0ec6b3SJim Harris cc.bits.ams = 0; 363bb0ec6b3SJim Harris cc.bits.shn = 0; 364bb0ec6b3SJim Harris cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ 365bb0ec6b3SJim Harris cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ 366bb0ec6b3SJim Harris 367bb0ec6b3SJim Harris /* This evaluates to 0, which is according to spec. */ 368bb0ec6b3SJim Harris cc.bits.mps = (PAGE_SIZE >> 13); 369bb0ec6b3SJim Harris 370bb0ec6b3SJim Harris nvme_mmio_write_4(ctrlr, cc, cc.raw); 371bb0ec6b3SJim Harris DELAY(5000); 372bb0ec6b3SJim Harris 373bb0ec6b3SJim Harris return (nvme_ctrlr_wait_for_ready(ctrlr)); 374bb0ec6b3SJim Harris } 375bb0ec6b3SJim Harris 376bb0ec6b3SJim Harris int 377bb0ec6b3SJim Harris nvme_ctrlr_reset(struct nvme_controller *ctrlr) 378bb0ec6b3SJim Harris { 379bb0ec6b3SJim Harris 380bb0ec6b3SJim Harris nvme_ctrlr_disable(ctrlr); 381bb0ec6b3SJim Harris return (nvme_ctrlr_enable(ctrlr)); 382bb0ec6b3SJim Harris } 383bb0ec6b3SJim Harris 3848a382371SJim Harris /* 3858a382371SJim Harris * Disable this code for now, since Chatham doesn't support 3868a382371SJim Harris * AERs so I have no good way to test them. 3878a382371SJim Harris */ 3888a382371SJim Harris #if 0 389bb0ec6b3SJim Harris static void 390bb0ec6b3SJim Harris nvme_async_event_cb(void *arg, const struct nvme_completion *status) 391bb0ec6b3SJim Harris { 392bb0ec6b3SJim Harris struct nvme_controller *ctrlr = arg; 393bb0ec6b3SJim Harris 394bb0ec6b3SJim Harris printf("Asynchronous event occurred.\n"); 395bb0ec6b3SJim Harris 396bb0ec6b3SJim Harris /* TODO: decode async event type based on status */ 397bb0ec6b3SJim Harris /* TODO: check status for any error bits */ 398bb0ec6b3SJim Harris 399bb0ec6b3SJim Harris /* 400bb0ec6b3SJim Harris * Repost an asynchronous event request so that it can be 401bb0ec6b3SJim Harris * used again by the controller. 402bb0ec6b3SJim Harris */ 403bb0ec6b3SJim Harris nvme_ctrlr_cmd_asynchronous_event_request(ctrlr, nvme_async_event_cb, 404bb0ec6b3SJim Harris ctrlr); 405bb0ec6b3SJim Harris } 4068a382371SJim Harris #endif 407bb0ec6b3SJim Harris 408bb0ec6b3SJim Harris static int 409bb0ec6b3SJim Harris nvme_ctrlr_identify(struct nvme_controller *ctrlr) 410bb0ec6b3SJim Harris { 411bb0ec6b3SJim Harris struct mtx *mtx; 412bb0ec6b3SJim Harris struct nvme_completion cpl; 413bb0ec6b3SJim Harris int status; 414bb0ec6b3SJim Harris 415bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, &cpl); 416bb0ec6b3SJim Harris 417bb0ec6b3SJim Harris mtx_lock(mtx); 418bb0ec6b3SJim Harris nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, 419bb0ec6b3SJim Harris nvme_ctrlr_cb, &cpl); 420bb0ec6b3SJim Harris status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5); 421bb0ec6b3SJim Harris mtx_unlock(mtx); 422bb0ec6b3SJim Harris if ((status != 0) || cpl.sf_sc || cpl.sf_sct) { 423bb0ec6b3SJim Harris printf("nvme_identify_controller failed!\n"); 424bb0ec6b3SJim Harris return (ENXIO); 425bb0ec6b3SJim Harris } 426bb0ec6b3SJim Harris 427bb0ec6b3SJim Harris #ifdef CHATHAM2 428bb0ec6b3SJim Harris if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 429bb0ec6b3SJim Harris nvme_chatham_populate_cdata(ctrlr); 430bb0ec6b3SJim Harris #endif 431bb0ec6b3SJim Harris 432bb0ec6b3SJim Harris return (0); 433bb0ec6b3SJim Harris } 434bb0ec6b3SJim Harris 435bb0ec6b3SJim Harris static int 436bb0ec6b3SJim Harris nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr) 437bb0ec6b3SJim Harris { 438bb0ec6b3SJim Harris struct mtx *mtx; 439bb0ec6b3SJim Harris struct nvme_completion cpl; 440bb0ec6b3SJim Harris int cq_allocated, sq_allocated, status; 441bb0ec6b3SJim Harris 442bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, &cpl); 443bb0ec6b3SJim Harris 444bb0ec6b3SJim Harris mtx_lock(mtx); 445bb0ec6b3SJim Harris nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues, 446bb0ec6b3SJim Harris nvme_ctrlr_cb, &cpl); 447bb0ec6b3SJim Harris status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5); 448bb0ec6b3SJim Harris mtx_unlock(mtx); 449bb0ec6b3SJim Harris if ((status != 0) || cpl.sf_sc || cpl.sf_sct) { 450bb0ec6b3SJim Harris printf("nvme_set_num_queues failed!\n"); 451bb0ec6b3SJim Harris return (ENXIO); 452bb0ec6b3SJim Harris } 453bb0ec6b3SJim Harris 454bb0ec6b3SJim Harris /* 455bb0ec6b3SJim Harris * Data in cdw0 is 0-based. 456bb0ec6b3SJim Harris * Lower 16-bits indicate number of submission queues allocated. 457bb0ec6b3SJim Harris * Upper 16-bits indicate number of completion queues allocated. 458bb0ec6b3SJim Harris */ 459bb0ec6b3SJim Harris sq_allocated = (cpl.cdw0 & 0xFFFF) + 1; 460bb0ec6b3SJim Harris cq_allocated = (cpl.cdw0 >> 16) + 1; 461bb0ec6b3SJim Harris 462bb0ec6b3SJim Harris /* 463bb0ec6b3SJim Harris * Check that the controller was able to allocate the number of 464bb0ec6b3SJim Harris * queues we requested. If not, revert to one IO queue. 465bb0ec6b3SJim Harris */ 466bb0ec6b3SJim Harris if (sq_allocated < ctrlr->num_io_queues || 467bb0ec6b3SJim Harris cq_allocated < ctrlr->num_io_queues) { 468bb0ec6b3SJim Harris ctrlr->num_io_queues = 1; 469bb0ec6b3SJim Harris ctrlr->per_cpu_io_queues = 0; 470bb0ec6b3SJim Harris 471bb0ec6b3SJim Harris /* TODO: destroy extra queues that were created 472bb0ec6b3SJim Harris * previously but now found to be not needed. 473bb0ec6b3SJim Harris */ 474bb0ec6b3SJim Harris } 475bb0ec6b3SJim Harris 476bb0ec6b3SJim Harris return (0); 477bb0ec6b3SJim Harris } 478bb0ec6b3SJim Harris 479bb0ec6b3SJim Harris static int 480bb0ec6b3SJim Harris nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr) 481bb0ec6b3SJim Harris { 482bb0ec6b3SJim Harris struct mtx *mtx; 483bb0ec6b3SJim Harris struct nvme_qpair *qpair; 484bb0ec6b3SJim Harris struct nvme_completion cpl; 485bb0ec6b3SJim Harris int i, status; 486bb0ec6b3SJim Harris 487bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, &cpl); 488bb0ec6b3SJim Harris 489bb0ec6b3SJim Harris for (i = 0; i < ctrlr->num_io_queues; i++) { 490bb0ec6b3SJim Harris qpair = &ctrlr->ioq[i]; 491bb0ec6b3SJim Harris 492bb0ec6b3SJim Harris mtx_lock(mtx); 493bb0ec6b3SJim Harris nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector, 494bb0ec6b3SJim Harris nvme_ctrlr_cb, &cpl); 495bb0ec6b3SJim Harris status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5); 496bb0ec6b3SJim Harris mtx_unlock(mtx); 497bb0ec6b3SJim Harris if ((status != 0) || cpl.sf_sc || cpl.sf_sct) { 498bb0ec6b3SJim Harris printf("nvme_create_io_cq failed!\n"); 499bb0ec6b3SJim Harris return (ENXIO); 500bb0ec6b3SJim Harris } 501bb0ec6b3SJim Harris 502bb0ec6b3SJim Harris mtx_lock(mtx); 503bb0ec6b3SJim Harris nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, 504bb0ec6b3SJim Harris nvme_ctrlr_cb, &cpl); 505bb0ec6b3SJim Harris status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5); 506bb0ec6b3SJim Harris mtx_unlock(mtx); 507bb0ec6b3SJim Harris if ((status != 0) || cpl.sf_sc || cpl.sf_sct) { 508bb0ec6b3SJim Harris printf("nvme_create_io_sq failed!\n"); 509bb0ec6b3SJim Harris return (ENXIO); 510bb0ec6b3SJim Harris } 511bb0ec6b3SJim Harris } 512bb0ec6b3SJim Harris 513bb0ec6b3SJim Harris return (0); 514bb0ec6b3SJim Harris } 515bb0ec6b3SJim Harris 516bb0ec6b3SJim Harris static int 517bb0ec6b3SJim Harris nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) 518bb0ec6b3SJim Harris { 519bb0ec6b3SJim Harris struct nvme_namespace *ns; 520bb0ec6b3SJim Harris int i, status; 521bb0ec6b3SJim Harris 522bb0ec6b3SJim Harris for (i = 0; i < ctrlr->cdata.nn; i++) { 523bb0ec6b3SJim Harris ns = &ctrlr->ns[i]; 524bb0ec6b3SJim Harris status = nvme_ns_construct(ns, i+1, ctrlr); 525bb0ec6b3SJim Harris if (status != 0) 526bb0ec6b3SJim Harris return (status); 527bb0ec6b3SJim Harris } 528bb0ec6b3SJim Harris 529bb0ec6b3SJim Harris return (0); 530bb0ec6b3SJim Harris } 531bb0ec6b3SJim Harris 532bb0ec6b3SJim Harris static void 533bb0ec6b3SJim Harris nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr) 534bb0ec6b3SJim Harris { 535bb0ec6b3SJim Harris union nvme_critical_warning_state state; 536bb0ec6b3SJim Harris uint8_t num_async_events; 537bb0ec6b3SJim Harris 538bb0ec6b3SJim Harris state.raw = 0xFF; 539bb0ec6b3SJim Harris state.bits.reserved = 0; 540bb0ec6b3SJim Harris nvme_ctrlr_cmd_set_asynchronous_event_config(ctrlr, state, NULL, NULL); 541bb0ec6b3SJim Harris 542bb0ec6b3SJim Harris /* aerl is a zero-based value, so we need to add 1 here. */ 543bb0ec6b3SJim Harris num_async_events = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1)); 544bb0ec6b3SJim Harris 545bb0ec6b3SJim Harris /* 546bb0ec6b3SJim Harris * Disable this code for now, since Chatham doesn't support 547bb0ec6b3SJim Harris * AERs so I have no good way to test them. 548bb0ec6b3SJim Harris */ 549bb0ec6b3SJim Harris #if 0 550bb0ec6b3SJim Harris for (int i = 0; i < num_async_events; i++) 551bb0ec6b3SJim Harris nvme_ctrlr_cmd_asynchronous_event_request(ctrlr, 552bb0ec6b3SJim Harris nvme_async_event_cb, ctrlr); 553bb0ec6b3SJim Harris #endif 554bb0ec6b3SJim Harris } 555bb0ec6b3SJim Harris 556bb0ec6b3SJim Harris static void 557bb0ec6b3SJim Harris nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr) 558bb0ec6b3SJim Harris { 559bb0ec6b3SJim Harris 560bb0ec6b3SJim Harris ctrlr->int_coal_time = 0; 561bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.int_coal_time", 562bb0ec6b3SJim Harris &ctrlr->int_coal_time); 563bb0ec6b3SJim Harris 564bb0ec6b3SJim Harris ctrlr->int_coal_threshold = 0; 565bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold", 566bb0ec6b3SJim Harris &ctrlr->int_coal_threshold); 567bb0ec6b3SJim Harris 568bb0ec6b3SJim Harris nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time, 569bb0ec6b3SJim Harris ctrlr->int_coal_threshold, NULL, NULL); 570bb0ec6b3SJim Harris } 571bb0ec6b3SJim Harris 572bb0ec6b3SJim Harris void 573bb0ec6b3SJim Harris nvme_ctrlr_start(void *ctrlr_arg) 574bb0ec6b3SJim Harris { 575bb0ec6b3SJim Harris struct nvme_controller *ctrlr = ctrlr_arg; 576bb0ec6b3SJim Harris 577bb0ec6b3SJim Harris if (nvme_ctrlr_identify(ctrlr) != 0) 578bb0ec6b3SJim Harris goto err; 579bb0ec6b3SJim Harris 580bb0ec6b3SJim Harris if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) 581bb0ec6b3SJim Harris goto err; 582bb0ec6b3SJim Harris 583bb0ec6b3SJim Harris if (nvme_ctrlr_create_qpairs(ctrlr) != 0) 584bb0ec6b3SJim Harris goto err; 585bb0ec6b3SJim Harris 586bb0ec6b3SJim Harris if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) 587bb0ec6b3SJim Harris goto err; 588bb0ec6b3SJim Harris 589bb0ec6b3SJim Harris nvme_ctrlr_configure_aer(ctrlr); 590bb0ec6b3SJim Harris nvme_ctrlr_configure_int_coalescing(ctrlr); 591bb0ec6b3SJim Harris 592bb0ec6b3SJim Harris ctrlr->is_started = TRUE; 593bb0ec6b3SJim Harris 594bb0ec6b3SJim Harris err: 595bb0ec6b3SJim Harris 596bb0ec6b3SJim Harris /* 597bb0ec6b3SJim Harris * Initialize sysctls, even if controller failed to start, to 598bb0ec6b3SJim Harris * assist with debugging admin queue pair. 599bb0ec6b3SJim Harris */ 600bb0ec6b3SJim Harris nvme_sysctl_initialize_ctrlr(ctrlr); 601bb0ec6b3SJim Harris config_intrhook_disestablish(&ctrlr->config_hook); 602bb0ec6b3SJim Harris } 603bb0ec6b3SJim Harris 604bb0ec6b3SJim Harris static void 605bb0ec6b3SJim Harris nvme_ctrlr_intx_task(void *arg, int pending) 606bb0ec6b3SJim Harris { 607bb0ec6b3SJim Harris struct nvme_controller *ctrlr = arg; 608bb0ec6b3SJim Harris 609bb0ec6b3SJim Harris nvme_qpair_process_completions(&ctrlr->adminq); 610bb0ec6b3SJim Harris 611bb0ec6b3SJim Harris if (ctrlr->ioq[0].cpl) 612bb0ec6b3SJim Harris nvme_qpair_process_completions(&ctrlr->ioq[0]); 613bb0ec6b3SJim Harris 614bb0ec6b3SJim Harris nvme_mmio_write_4(ctrlr, intmc, 1); 615bb0ec6b3SJim Harris } 616bb0ec6b3SJim Harris 617bb0ec6b3SJim Harris static void 618bb0ec6b3SJim Harris nvme_ctrlr_intx_handler(void *arg) 619bb0ec6b3SJim Harris { 620bb0ec6b3SJim Harris struct nvme_controller *ctrlr = arg; 621bb0ec6b3SJim Harris 622bb0ec6b3SJim Harris nvme_mmio_write_4(ctrlr, intms, 1); 623bb0ec6b3SJim Harris taskqueue_enqueue_fast(ctrlr->taskqueue, &ctrlr->task); 624bb0ec6b3SJim Harris } 625bb0ec6b3SJim Harris 626bb0ec6b3SJim Harris static int 627bb0ec6b3SJim Harris nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr) 628bb0ec6b3SJim Harris { 629bb0ec6b3SJim Harris 630bb0ec6b3SJim Harris ctrlr->num_io_queues = 1; 631bb0ec6b3SJim Harris ctrlr->per_cpu_io_queues = 0; 632bb0ec6b3SJim Harris ctrlr->rid = 0; 633bb0ec6b3SJim Harris ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, 634bb0ec6b3SJim Harris &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); 635bb0ec6b3SJim Harris 636bb0ec6b3SJim Harris if (ctrlr->res == NULL) { 637bb0ec6b3SJim Harris device_printf(ctrlr->dev, "unable to allocate shared IRQ\n"); 638bb0ec6b3SJim Harris return (ENOMEM); 639bb0ec6b3SJim Harris } 640bb0ec6b3SJim Harris 641bb0ec6b3SJim Harris bus_setup_intr(ctrlr->dev, ctrlr->res, 642bb0ec6b3SJim Harris INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler, 643bb0ec6b3SJim Harris ctrlr, &ctrlr->tag); 644bb0ec6b3SJim Harris 645bb0ec6b3SJim Harris if (ctrlr->tag == NULL) { 646bb0ec6b3SJim Harris device_printf(ctrlr->dev, 647bb0ec6b3SJim Harris "unable to setup legacy interrupt handler\n"); 648bb0ec6b3SJim Harris return (ENOMEM); 649bb0ec6b3SJim Harris } 650bb0ec6b3SJim Harris 651bb0ec6b3SJim Harris TASK_INIT(&ctrlr->task, 0, nvme_ctrlr_intx_task, ctrlr); 652bb0ec6b3SJim Harris ctrlr->taskqueue = taskqueue_create_fast("nvme_taskq", M_NOWAIT, 653bb0ec6b3SJim Harris taskqueue_thread_enqueue, &ctrlr->taskqueue); 654bb0ec6b3SJim Harris taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_NET, 655bb0ec6b3SJim Harris "%s intx taskq", device_get_nameunit(ctrlr->dev)); 656bb0ec6b3SJim Harris 657bb0ec6b3SJim Harris return (0); 658bb0ec6b3SJim Harris } 659bb0ec6b3SJim Harris 660bb0ec6b3SJim Harris static int 661bb0ec6b3SJim Harris nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag, 662bb0ec6b3SJim Harris struct thread *td) 663bb0ec6b3SJim Harris { 664bb0ec6b3SJim Harris struct nvme_controller *ctrlr; 665bb0ec6b3SJim Harris struct nvme_completion cpl; 666bb0ec6b3SJim Harris struct mtx *mtx; 667bb0ec6b3SJim Harris 668bb0ec6b3SJim Harris ctrlr = cdev->si_drv1; 669bb0ec6b3SJim Harris 670bb0ec6b3SJim Harris switch (cmd) { 671bb0ec6b3SJim Harris case NVME_IDENTIFY_CONTROLLER: 672bb0ec6b3SJim Harris #ifdef CHATHAM2 673bb0ec6b3SJim Harris /* 674bb0ec6b3SJim Harris * Don't refresh data on Chatham, since Chatham returns 675bb0ec6b3SJim Harris * garbage on IDENTIFY anyways. 676bb0ec6b3SJim Harris */ 677bb0ec6b3SJim Harris if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) { 678bb0ec6b3SJim Harris memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata)); 679bb0ec6b3SJim Harris break; 680bb0ec6b3SJim Harris } 681bb0ec6b3SJim Harris #endif 682bb0ec6b3SJim Harris /* Refresh data before returning to user. */ 683bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, &cpl); 684bb0ec6b3SJim Harris mtx_lock(mtx); 685bb0ec6b3SJim Harris nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, 686bb0ec6b3SJim Harris nvme_ctrlr_cb, &cpl); 687bb0ec6b3SJim Harris msleep(&cpl, mtx, PRIBIO, "nvme_ioctl", 0); 688bb0ec6b3SJim Harris mtx_unlock(mtx); 689bb0ec6b3SJim Harris if (cpl.sf_sc || cpl.sf_sct) 690bb0ec6b3SJim Harris return (ENXIO); 691bb0ec6b3SJim Harris memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata)); 692bb0ec6b3SJim Harris break; 693bb0ec6b3SJim Harris default: 694bb0ec6b3SJim Harris return (ENOTTY); 695bb0ec6b3SJim Harris } 696bb0ec6b3SJim Harris 697bb0ec6b3SJim Harris return (0); 698bb0ec6b3SJim Harris } 699bb0ec6b3SJim Harris 700bb0ec6b3SJim Harris static struct cdevsw nvme_ctrlr_cdevsw = { 701bb0ec6b3SJim Harris .d_version = D_VERSION, 702bb0ec6b3SJim Harris .d_flags = 0, 703bb0ec6b3SJim Harris .d_ioctl = nvme_ctrlr_ioctl 704bb0ec6b3SJim Harris }; 705bb0ec6b3SJim Harris 706bb0ec6b3SJim Harris int 707bb0ec6b3SJim Harris nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev) 708bb0ec6b3SJim Harris { 709bb0ec6b3SJim Harris union cap_lo_register cap_lo; 710bb0ec6b3SJim Harris union cap_hi_register cap_hi; 711bb0ec6b3SJim Harris int num_vectors, per_cpu_io_queues, status = 0; 712bb0ec6b3SJim Harris 713bb0ec6b3SJim Harris ctrlr->dev = dev; 714bb0ec6b3SJim Harris ctrlr->is_started = FALSE; 715bb0ec6b3SJim Harris 716bb0ec6b3SJim Harris status = nvme_ctrlr_allocate_bar(ctrlr); 717bb0ec6b3SJim Harris 718bb0ec6b3SJim Harris if (status != 0) 719bb0ec6b3SJim Harris return (status); 720bb0ec6b3SJim Harris 721bb0ec6b3SJim Harris #ifdef CHATHAM2 722bb0ec6b3SJim Harris if (pci_get_devid(dev) == CHATHAM_PCI_ID) { 723bb0ec6b3SJim Harris status = nvme_ctrlr_allocate_chatham_bar(ctrlr); 724bb0ec6b3SJim Harris if (status != 0) 725bb0ec6b3SJim Harris return (status); 726bb0ec6b3SJim Harris nvme_ctrlr_setup_chatham(ctrlr); 727bb0ec6b3SJim Harris } 728bb0ec6b3SJim Harris #endif 729bb0ec6b3SJim Harris 730bb0ec6b3SJim Harris /* 731bb0ec6b3SJim Harris * Software emulators may set the doorbell stride to something 732bb0ec6b3SJim Harris * other than zero, but this driver is not set up to handle that. 733bb0ec6b3SJim Harris */ 734bb0ec6b3SJim Harris cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi); 735bb0ec6b3SJim Harris if (cap_hi.bits.dstrd != 0) 736bb0ec6b3SJim Harris return (ENXIO); 737bb0ec6b3SJim Harris 738bb0ec6b3SJim Harris /* Get ready timeout value from controller, in units of 500ms. */ 739bb0ec6b3SJim Harris cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo); 740bb0ec6b3SJim Harris ctrlr->ready_timeout_in_ms = cap_lo.bits.to * 500; 741bb0ec6b3SJim Harris 742bb0ec6b3SJim Harris per_cpu_io_queues = 1; 743bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues); 744bb0ec6b3SJim Harris ctrlr->per_cpu_io_queues = per_cpu_io_queues ? TRUE : FALSE; 745bb0ec6b3SJim Harris 746bb0ec6b3SJim Harris if (ctrlr->per_cpu_io_queues) 747bb0ec6b3SJim Harris ctrlr->num_io_queues = mp_ncpus; 748bb0ec6b3SJim Harris else 749bb0ec6b3SJim Harris ctrlr->num_io_queues = 1; 750bb0ec6b3SJim Harris 751bb0ec6b3SJim Harris ctrlr->force_intx = 0; 752bb0ec6b3SJim Harris TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx); 753bb0ec6b3SJim Harris 754bb0ec6b3SJim Harris ctrlr->msix_enabled = 1; 755bb0ec6b3SJim Harris 756bb0ec6b3SJim Harris if (ctrlr->force_intx) { 757bb0ec6b3SJim Harris ctrlr->msix_enabled = 0; 758bb0ec6b3SJim Harris goto intx; 759bb0ec6b3SJim Harris } 760bb0ec6b3SJim Harris 761bb0ec6b3SJim Harris /* One vector per IO queue, plus one vector for admin queue. */ 762bb0ec6b3SJim Harris num_vectors = ctrlr->num_io_queues + 1; 763bb0ec6b3SJim Harris 764bb0ec6b3SJim Harris if (pci_msix_count(dev) < num_vectors) { 765bb0ec6b3SJim Harris ctrlr->msix_enabled = 0; 766bb0ec6b3SJim Harris goto intx; 767bb0ec6b3SJim Harris } 768bb0ec6b3SJim Harris 769bb0ec6b3SJim Harris if (pci_alloc_msix(dev, &num_vectors) != 0) 770bb0ec6b3SJim Harris ctrlr->msix_enabled = 0; 771bb0ec6b3SJim Harris 772bb0ec6b3SJim Harris intx: 773bb0ec6b3SJim Harris 774bb0ec6b3SJim Harris if (!ctrlr->msix_enabled) 775bb0ec6b3SJim Harris nvme_ctrlr_configure_intx(ctrlr); 776bb0ec6b3SJim Harris 777bb0ec6b3SJim Harris nvme_ctrlr_construct_admin_qpair(ctrlr); 778bb0ec6b3SJim Harris 779bb0ec6b3SJim Harris status = nvme_ctrlr_construct_io_qpairs(ctrlr); 780bb0ec6b3SJim Harris 781bb0ec6b3SJim Harris if (status != 0) 782bb0ec6b3SJim Harris return (status); 783bb0ec6b3SJim Harris 784bb0ec6b3SJim Harris ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 785bb0ec6b3SJim Harris "nvme%d", device_get_unit(dev)); 786bb0ec6b3SJim Harris 787bb0ec6b3SJim Harris if (ctrlr->cdev == NULL) 788bb0ec6b3SJim Harris return (ENXIO); 789bb0ec6b3SJim Harris 790bb0ec6b3SJim Harris ctrlr->cdev->si_drv1 = (void *)ctrlr; 791bb0ec6b3SJim Harris 792bb0ec6b3SJim Harris return (0); 793bb0ec6b3SJim Harris } 794*d281e8fbSJim Harris 795*d281e8fbSJim Harris void 796*d281e8fbSJim Harris nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, 797*d281e8fbSJim Harris struct nvme_request *req) 798*d281e8fbSJim Harris { 799*d281e8fbSJim Harris struct nvme_qpair *qpair; 800*d281e8fbSJim Harris struct nvme_tracker *tr; 801*d281e8fbSJim Harris int err; 802*d281e8fbSJim Harris 803*d281e8fbSJim Harris qpair = &ctrlr->adminq; 804*d281e8fbSJim Harris 805*d281e8fbSJim Harris tr = nvme_qpair_allocate_tracker(qpair); 806*d281e8fbSJim Harris 807*d281e8fbSJim Harris tr->req = req; 808*d281e8fbSJim Harris 809*d281e8fbSJim Harris if (req->payload_size > 0) { 810*d281e8fbSJim Harris err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, 811*d281e8fbSJim Harris req->payload, req->payload_size, 812*d281e8fbSJim Harris nvme_payload_map, tr, 0); 813*d281e8fbSJim Harris if (err != 0) 814*d281e8fbSJim Harris panic("bus_dmamap_load returned non-zero!\n"); 815*d281e8fbSJim Harris } else 816*d281e8fbSJim Harris nvme_qpair_submit_cmd(tr->qpair, tr); 817*d281e8fbSJim Harris } 818*d281e8fbSJim Harris 819*d281e8fbSJim Harris void 820*d281e8fbSJim Harris nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, 821*d281e8fbSJim Harris struct nvme_request *req) 822*d281e8fbSJim Harris { 823*d281e8fbSJim Harris struct nvme_qpair *qpair; 824*d281e8fbSJim Harris struct nvme_tracker *tr; 825*d281e8fbSJim Harris int err; 826*d281e8fbSJim Harris 827*d281e8fbSJim Harris if (ctrlr->per_cpu_io_queues) 828*d281e8fbSJim Harris qpair = &ctrlr->ioq[curcpu]; 829*d281e8fbSJim Harris else 830*d281e8fbSJim Harris qpair = &ctrlr->ioq[0]; 831*d281e8fbSJim Harris 832*d281e8fbSJim Harris tr = nvme_qpair_allocate_tracker(qpair); 833*d281e8fbSJim Harris 834*d281e8fbSJim Harris tr->req = req; 835*d281e8fbSJim Harris 836*d281e8fbSJim Harris if (req->payload_size > 0) { 837*d281e8fbSJim Harris err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, 838*d281e8fbSJim Harris req->payload, req->payload_size, 839*d281e8fbSJim Harris nvme_payload_map, tr, 0); 840*d281e8fbSJim Harris if (err != 0) 841*d281e8fbSJim Harris panic("bus_dmamap_load returned non-zero!\n"); 842*d281e8fbSJim Harris } else 843*d281e8fbSJim Harris nvme_qpair_submit_cmd(tr->qpair, tr); 844*d281e8fbSJim Harris } 845