1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 33 #include <dev/pci/pcivar.h> 34 35 #include "nvme_private.h" 36 37 static boolean_t 38 nvme_completion_check_retry(const struct nvme_completion *cpl) 39 { 40 /* 41 * TODO: spec is not clear how commands that are aborted due 42 * to TLER will be marked. So for now, it seems 43 * NAMESPACE_NOT_READY is the only case where we should 44 * look at the DNR bit. 45 */ 46 switch (cpl->sf_sct) { 47 case NVME_SCT_GENERIC: 48 switch (cpl->sf_sc) { 49 case NVME_SC_NAMESPACE_NOT_READY: 50 if (cpl->sf_dnr) 51 return (0); 52 else 53 return (1); 54 case NVME_SC_INVALID_OPCODE: 55 case NVME_SC_INVALID_FIELD: 56 case NVME_SC_COMMAND_ID_CONFLICT: 57 case NVME_SC_DATA_TRANSFER_ERROR: 58 case NVME_SC_ABORTED_POWER_LOSS: 59 case NVME_SC_INTERNAL_DEVICE_ERROR: 60 case NVME_SC_ABORTED_BY_REQUEST: 61 case NVME_SC_ABORTED_SQ_DELETION: 62 case NVME_SC_ABORTED_FAILED_FUSED: 63 case NVME_SC_ABORTED_MISSING_FUSED: 64 case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 65 case NVME_SC_COMMAND_SEQUENCE_ERROR: 66 case NVME_SC_LBA_OUT_OF_RANGE: 67 case NVME_SC_CAPACITY_EXCEEDED: 68 default: 69 return (0); 70 } 71 case NVME_SCT_COMMAND_SPECIFIC: 72 case NVME_SCT_MEDIA_ERROR: 73 case NVME_SCT_VENDOR_SPECIFIC: 74 default: 75 return (0); 76 } 77 } 78 79 static void 80 nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 81 uint16_t cid) 82 { 83 84 bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); 85 bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); 86 87 bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, 88 sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); 89 90 callout_init_mtx(&tr->timer, &qpair->lock, 0); 91 tr->cid = cid; 92 tr->qpair = qpair; 93 } 94 95 void 96 nvme_qpair_process_completions(struct nvme_qpair *qpair) 97 { 98 struct nvme_tracker *tr; 99 struct nvme_request *req; 100 struct nvme_completion *cpl; 101 boolean_t retry, error; 102 103 qpair->num_intr_handler_calls++; 104 105 while (1) { 106 cpl = &qpair->cpl[qpair->cq_head]; 107 108 if (cpl->p != qpair->phase) 109 break; 110 111 tr = qpair->act_tr[cpl->cid]; 112 req = tr->req; 113 114 KASSERT(tr, 115 ("completion queue has entries but no active trackers\n")); 116 117 error = cpl->sf_sc || cpl->sf_sct; 118 retry = error && nvme_completion_check_retry(cpl); 119 120 if (error) { 121 nvme_dump_completion(cpl); 122 nvme_dump_command(&tr->req->cmd); 123 } 124 125 qpair->act_tr[cpl->cid] = NULL; 126 127 KASSERT(cpl->cid == req->cmd.cid, 128 ("cpl cid does not match cmd cid\n")); 129 130 if (req->cb_fn && !retry) 131 req->cb_fn(req->cb_arg, cpl); 132 133 qpair->sq_head = cpl->sqhd; 134 135 mtx_lock(&qpair->lock); 136 callout_stop(&tr->timer); 137 138 if (retry) 139 nvme_qpair_submit_cmd(qpair, tr); 140 else { 141 if (req->payload_size > 0 || req->uio != NULL) 142 bus_dmamap_unload(qpair->dma_tag, 143 tr->payload_dma_map); 144 145 nvme_free_request(req); 146 147 SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); 148 149 if (!STAILQ_EMPTY(&qpair->queued_req)) { 150 req = STAILQ_FIRST(&qpair->queued_req); 151 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 152 nvme_qpair_submit_request(qpair, req); 153 } 154 } 155 156 mtx_unlock(&qpair->lock); 157 158 if (++qpair->cq_head == qpair->num_entries) { 159 qpair->cq_head = 0; 160 qpair->phase = !qpair->phase; 161 } 162 163 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, 164 qpair->cq_head); 165 } 166 } 167 168 static void 169 nvme_qpair_msix_handler(void *arg) 170 { 171 struct nvme_qpair *qpair = arg; 172 173 nvme_qpair_process_completions(qpair); 174 } 175 176 void 177 nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 178 uint16_t vector, uint32_t num_entries, uint32_t num_trackers, 179 uint32_t max_xfer_size, struct nvme_controller *ctrlr) 180 { 181 struct nvme_tracker *tr; 182 uint32_t i; 183 184 qpair->id = id; 185 qpair->vector = vector; 186 qpair->num_entries = num_entries; 187 #ifdef CHATHAM2 188 /* 189 * Chatham prototype board starts having issues at higher queue 190 * depths. So use a conservative estimate here of no more than 64 191 * outstanding I/O per queue at any one point. 192 */ 193 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 194 num_trackers = min(num_trackers, 64); 195 #endif 196 qpair->num_trackers = num_trackers; 197 qpair->max_xfer_size = max_xfer_size; 198 qpair->ctrlr = ctrlr; 199 200 /* 201 * First time through the completion queue, HW will set phase 202 * bit on completions to 1. So set this to 1 here, indicating 203 * we're looking for a 1 to know which entries have completed. 204 * we'll toggle the bit each time when the completion queue 205 * rolls over. 206 */ 207 qpair->phase = 1; 208 209 if (ctrlr->msix_enabled) { 210 211 /* 212 * MSI-X vector resource IDs start at 1, so we add one to 213 * the queue's vector to get the corresponding rid to use. 214 */ 215 qpair->rid = vector + 1; 216 217 qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, 218 &qpair->rid, RF_ACTIVE); 219 220 bus_setup_intr(ctrlr->dev, qpair->res, 221 INTR_TYPE_MISC | INTR_MPSAFE, NULL, 222 nvme_qpair_msix_handler, qpair, &qpair->tag); 223 } 224 225 mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); 226 227 bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 228 sizeof(uint64_t), PAGE_SIZE, BUS_SPACE_MAXADDR, 229 BUS_SPACE_MAXADDR, NULL, NULL, qpair->max_xfer_size, 230 (qpair->max_xfer_size/PAGE_SIZE)+1, PAGE_SIZE, 0, 231 NULL, NULL, &qpair->dma_tag); 232 233 qpair->num_cmds = 0; 234 qpair->num_intr_handler_calls = 0; 235 qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; 236 237 /* TODO: error checking on contigmalloc, bus_dmamap_load calls */ 238 qpair->cmd = contigmalloc(qpair->num_entries * 239 sizeof(struct nvme_command), M_NVME, M_ZERO | M_NOWAIT, 240 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 241 qpair->cpl = contigmalloc(qpair->num_entries * 242 sizeof(struct nvme_completion), M_NVME, M_ZERO | M_NOWAIT, 243 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 244 245 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map); 246 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map); 247 248 bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map, 249 qpair->cmd, qpair->num_entries * sizeof(struct nvme_command), 250 nvme_single_map, &qpair->cmd_bus_addr, 0); 251 bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map, 252 qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion), 253 nvme_single_map, &qpair->cpl_bus_addr, 0); 254 255 qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); 256 qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); 257 258 SLIST_INIT(&qpair->free_tr); 259 STAILQ_INIT(&qpair->queued_req); 260 261 for (i = 0; i < qpair->num_trackers; i++) { 262 tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT); 263 264 if (tr == NULL) { 265 printf("warning: nvme tracker malloc failed\n"); 266 break; 267 } 268 269 nvme_qpair_construct_tracker(qpair, tr, i); 270 SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); 271 } 272 273 qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries, 274 M_NVME, M_ZERO | M_NOWAIT); 275 } 276 277 static void 278 nvme_qpair_destroy(struct nvme_qpair *qpair) 279 { 280 struct nvme_tracker *tr; 281 282 if (qpair->tag) 283 bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); 284 285 if (qpair->res) 286 bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, 287 rman_get_rid(qpair->res), qpair->res); 288 289 if (qpair->dma_tag) 290 bus_dma_tag_destroy(qpair->dma_tag); 291 292 if (qpair->act_tr) 293 free(qpair->act_tr, M_NVME); 294 295 while (!SLIST_EMPTY(&qpair->free_tr)) { 296 tr = SLIST_FIRST(&qpair->free_tr); 297 SLIST_REMOVE_HEAD(&qpair->free_tr, slist); 298 bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map); 299 bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map); 300 free(tr, M_NVME); 301 } 302 } 303 304 void 305 nvme_admin_qpair_destroy(struct nvme_qpair *qpair) 306 { 307 308 /* 309 * For NVMe, you don't send delete queue commands for the admin 310 * queue, so we just need to unload and free the cmd and cpl memory. 311 */ 312 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map); 313 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map); 314 315 contigfree(qpair->cmd, 316 qpair->num_entries * sizeof(struct nvme_command), M_NVME); 317 318 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map); 319 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map); 320 contigfree(qpair->cpl, 321 qpair->num_entries * sizeof(struct nvme_completion), M_NVME); 322 323 nvme_qpair_destroy(qpair); 324 } 325 326 static void 327 nvme_free_cmd_ring(void *arg, const struct nvme_completion *status) 328 { 329 struct nvme_qpair *qpair; 330 331 qpair = (struct nvme_qpair *)arg; 332 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map); 333 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map); 334 contigfree(qpair->cmd, 335 qpair->num_entries * sizeof(struct nvme_command), M_NVME); 336 qpair->cmd = NULL; 337 } 338 339 static void 340 nvme_free_cpl_ring(void *arg, const struct nvme_completion *status) 341 { 342 struct nvme_qpair *qpair; 343 344 qpair = (struct nvme_qpair *)arg; 345 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map); 346 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map); 347 contigfree(qpair->cpl, 348 qpair->num_entries * sizeof(struct nvme_completion), M_NVME); 349 qpair->cpl = NULL; 350 } 351 352 void 353 nvme_io_qpair_destroy(struct nvme_qpair *qpair) 354 { 355 struct nvme_controller *ctrlr = qpair->ctrlr; 356 357 if (qpair->num_entries > 0) { 358 359 nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair, nvme_free_cmd_ring, 360 qpair); 361 /* Spin until free_cmd_ring sets qpair->cmd to NULL. */ 362 while (qpair->cmd) 363 DELAY(5); 364 365 nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair, nvme_free_cpl_ring, 366 qpair); 367 /* Spin until free_cpl_ring sets qpair->cmd to NULL. */ 368 while (qpair->cpl) 369 DELAY(5); 370 371 nvme_qpair_destroy(qpair); 372 } 373 } 374 375 static void 376 nvme_timeout(void *arg) 377 { 378 /* 379 * TODO: Add explicit abort operation here, once nvme(4) supports 380 * abort commands. 381 */ 382 } 383 384 void 385 nvme_qpair_submit_cmd(struct nvme_qpair *qpair, struct nvme_tracker *tr) 386 { 387 struct nvme_request *req; 388 389 req = tr->req; 390 req->cmd.cid = tr->cid; 391 qpair->act_tr[tr->cid] = tr; 392 393 #if __FreeBSD_version >= 800030 394 callout_reset_curcpu(&tr->timer, NVME_TIMEOUT_IN_SEC * hz, 395 nvme_timeout, tr); 396 #else 397 callout_reset(&tr->timer, NVME_TIMEOUT_IN_SEC * hz, nvme_timeout, tr); 398 #endif 399 400 /* Copy the command from the tracker to the submission queue. */ 401 memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); 402 403 if (++qpair->sq_tail == qpair->num_entries) 404 qpair->sq_tail = 0; 405 406 wmb(); 407 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, 408 qpair->sq_tail); 409 410 qpair->num_cmds++; 411 } 412 413 void 414 nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 415 { 416 struct nvme_tracker *tr; 417 int err; 418 419 mtx_lock(&qpair->lock); 420 421 tr = SLIST_FIRST(&qpair->free_tr); 422 423 if (tr == NULL) { 424 /* 425 * No tracker is available. Put the request on the qpair's 426 * request queue to be processed when a tracker frees up 427 * via a command completion. 428 */ 429 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 430 goto ret; 431 } 432 433 SLIST_REMOVE_HEAD(&qpair->free_tr, slist); 434 tr->req = req; 435 436 if (req->uio == NULL) { 437 if (req->payload_size > 0) { 438 err = bus_dmamap_load(tr->qpair->dma_tag, 439 tr->payload_dma_map, req->payload, 440 req->payload_size, 441 nvme_payload_map, tr, 0); 442 if (err != 0) 443 panic("bus_dmamap_load returned non-zero!\n"); 444 } else 445 nvme_qpair_submit_cmd(tr->qpair, tr); 446 } else { 447 err = bus_dmamap_load_uio(tr->qpair->dma_tag, 448 tr->payload_dma_map, req->uio, 449 nvme_payload_map_uio, tr, 0); 450 if (err != 0) 451 panic("bus_dmamap_load returned non-zero!\n"); 452 } 453 454 ret: 455 mtx_unlock(&qpair->lock); 456 } 457