xref: /freebsd/sys/dev/nvme/nvme_qpair.c (revision 9268022b74279434ed6300244e3f977e56a8ceb5)
1 /*-
2  * Copyright (C) 2012-2014 Intel Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 
33 #include <dev/pci/pcivar.h>
34 
35 #include "nvme_private.h"
36 
37 static void	_nvme_qpair_submit_request(struct nvme_qpair *qpair,
38 					   struct nvme_request *req);
39 
40 struct nvme_opcode_string {
41 
42 	uint16_t	opc;
43 	const char *	str;
44 };
45 
46 static struct nvme_opcode_string admin_opcode[] = {
47 	{ NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
48 	{ NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
49 	{ NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
50 	{ NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
51 	{ NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
52 	{ NVME_OPC_IDENTIFY, "IDENTIFY" },
53 	{ NVME_OPC_ABORT, "ABORT" },
54 	{ NVME_OPC_SET_FEATURES, "SET FEATURES" },
55 	{ NVME_OPC_GET_FEATURES, "GET FEATURES" },
56 	{ NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
57 	{ NVME_OPC_FIRMWARE_ACTIVATE, "FIRMWARE ACTIVATE" },
58 	{ NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
59 	{ NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
60 	{ NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
61 	{ NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
62 	{ 0xFFFF, "ADMIN COMMAND" }
63 };
64 
65 static struct nvme_opcode_string io_opcode[] = {
66 	{ NVME_OPC_FLUSH, "FLUSH" },
67 	{ NVME_OPC_WRITE, "WRITE" },
68 	{ NVME_OPC_READ, "READ" },
69 	{ NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
70 	{ NVME_OPC_COMPARE, "COMPARE" },
71 	{ NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
72 	{ 0xFFFF, "IO COMMAND" }
73 };
74 
75 static const char *
76 get_admin_opcode_string(uint16_t opc)
77 {
78 	struct nvme_opcode_string *entry;
79 
80 	entry = admin_opcode;
81 
82 	while (entry->opc != 0xFFFF) {
83 		if (entry->opc == opc)
84 			return (entry->str);
85 		entry++;
86 	}
87 	return (entry->str);
88 }
89 
90 static const char *
91 get_io_opcode_string(uint16_t opc)
92 {
93 	struct nvme_opcode_string *entry;
94 
95 	entry = io_opcode;
96 
97 	while (entry->opc != 0xFFFF) {
98 		if (entry->opc == opc)
99 			return (entry->str);
100 		entry++;
101 	}
102 	return (entry->str);
103 }
104 
105 
106 static void
107 nvme_admin_qpair_print_command(struct nvme_qpair *qpair,
108     struct nvme_command *cmd)
109 {
110 
111 	nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x "
112 	    "cdw10:%08x cdw11:%08x\n",
113 	    get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid,
114 	    cmd->nsid, cmd->cdw10, cmd->cdw11);
115 }
116 
117 static void
118 nvme_io_qpair_print_command(struct nvme_qpair *qpair,
119     struct nvme_command *cmd)
120 {
121 
122 	switch (cmd->opc) {
123 	case NVME_OPC_WRITE:
124 	case NVME_OPC_READ:
125 	case NVME_OPC_WRITE_UNCORRECTABLE:
126 	case NVME_OPC_COMPARE:
127 		nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d "
128 		    "lba:%llu len:%d\n",
129 		    get_io_opcode_string(cmd->opc), qpair->id, cmd->cid,
130 		    cmd->nsid,
131 		    ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
132 		    (cmd->cdw12 & 0xFFFF) + 1);
133 		break;
134 	case NVME_OPC_FLUSH:
135 	case NVME_OPC_DATASET_MANAGEMENT:
136 		nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n",
137 		    get_io_opcode_string(cmd->opc), qpair->id, cmd->cid,
138 		    cmd->nsid);
139 		break;
140 	default:
141 		nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n",
142 		    get_io_opcode_string(cmd->opc), cmd->opc, qpair->id,
143 		    cmd->cid, cmd->nsid);
144 		break;
145 	}
146 }
147 
148 static void
149 nvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd)
150 {
151 	if (qpair->id == 0)
152 		nvme_admin_qpair_print_command(qpair, cmd);
153 	else
154 		nvme_io_qpair_print_command(qpair, cmd);
155 }
156 
157 struct nvme_status_string {
158 
159 	uint16_t	sc;
160 	const char *	str;
161 };
162 
163 static struct nvme_status_string generic_status[] = {
164 	{ NVME_SC_SUCCESS, "SUCCESS" },
165 	{ NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
166 	{ NVME_SC_INVALID_FIELD, "INVALID_FIELD" },
167 	{ NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
168 	{ NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
169 	{ NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
170 	{ NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
171 	{ NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
172 	{ NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
173 	{ NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
174 	{ NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
175 	{ NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
176 	{ NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
177 	{ NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
178 	{ NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
179 	{ NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
180 	{ 0xFFFF, "GENERIC" }
181 };
182 
183 static struct nvme_status_string command_specific_status[] = {
184 	{ NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
185 	{ NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
186 	{ NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" },
187 	{ NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
188 	{ NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" },
189 	{ NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
190 	{ NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
191 	{ NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
192 	{ NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
193 	{ NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
194 	{ NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" },
195 	{ NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
196 	{ NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
197 	{ NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" },
198 	{ 0xFFFF, "COMMAND SPECIFIC" }
199 };
200 
201 static struct nvme_status_string media_error_status[] = {
202 	{ NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
203 	{ NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
204 	{ NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
205 	{ NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
206 	{ NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
207 	{ NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
208 	{ NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
209 	{ 0xFFFF, "MEDIA ERROR" }
210 };
211 
212 static const char *
213 get_status_string(uint16_t sct, uint16_t sc)
214 {
215 	struct nvme_status_string *entry;
216 
217 	switch (sct) {
218 	case NVME_SCT_GENERIC:
219 		entry = generic_status;
220 		break;
221 	case NVME_SCT_COMMAND_SPECIFIC:
222 		entry = command_specific_status;
223 		break;
224 	case NVME_SCT_MEDIA_ERROR:
225 		entry = media_error_status;
226 		break;
227 	case NVME_SCT_VENDOR_SPECIFIC:
228 		return ("VENDOR SPECIFIC");
229 	default:
230 		return ("RESERVED");
231 	}
232 
233 	while (entry->sc != 0xFFFF) {
234 		if (entry->sc == sc)
235 			return (entry->str);
236 		entry++;
237 	}
238 	return (entry->str);
239 }
240 
241 static void
242 nvme_qpair_print_completion(struct nvme_qpair *qpair,
243     struct nvme_completion *cpl)
244 {
245 	nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n",
246 	    get_status_string(cpl->status.sct, cpl->status.sc),
247 	    cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0);
248 }
249 
250 static boolean_t
251 nvme_completion_is_retry(const struct nvme_completion *cpl)
252 {
253 	/*
254 	 * TODO: spec is not clear how commands that are aborted due
255 	 *  to TLER will be marked.  So for now, it seems
256 	 *  NAMESPACE_NOT_READY is the only case where we should
257 	 *  look at the DNR bit.
258 	 */
259 	switch (cpl->status.sct) {
260 	case NVME_SCT_GENERIC:
261 		switch (cpl->status.sc) {
262 		case NVME_SC_ABORTED_BY_REQUEST:
263 		case NVME_SC_NAMESPACE_NOT_READY:
264 			if (cpl->status.dnr)
265 				return (0);
266 			else
267 				return (1);
268 		case NVME_SC_INVALID_OPCODE:
269 		case NVME_SC_INVALID_FIELD:
270 		case NVME_SC_COMMAND_ID_CONFLICT:
271 		case NVME_SC_DATA_TRANSFER_ERROR:
272 		case NVME_SC_ABORTED_POWER_LOSS:
273 		case NVME_SC_INTERNAL_DEVICE_ERROR:
274 		case NVME_SC_ABORTED_SQ_DELETION:
275 		case NVME_SC_ABORTED_FAILED_FUSED:
276 		case NVME_SC_ABORTED_MISSING_FUSED:
277 		case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
278 		case NVME_SC_COMMAND_SEQUENCE_ERROR:
279 		case NVME_SC_LBA_OUT_OF_RANGE:
280 		case NVME_SC_CAPACITY_EXCEEDED:
281 		default:
282 			return (0);
283 		}
284 	case NVME_SCT_COMMAND_SPECIFIC:
285 	case NVME_SCT_MEDIA_ERROR:
286 	case NVME_SCT_VENDOR_SPECIFIC:
287 	default:
288 		return (0);
289 	}
290 }
291 
292 static void
293 nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
294     uint16_t cid)
295 {
296 
297 	bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map);
298 	bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map);
299 
300 	bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp,
301 	    sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0);
302 
303 	callout_init(&tr->timer, 1);
304 	tr->cid = cid;
305 	tr->qpair = qpair;
306 }
307 
308 static void
309 nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
310     struct nvme_completion *cpl, boolean_t print_on_error)
311 {
312 	struct nvme_request	*req;
313 	boolean_t		retry, error;
314 
315 	req = tr->req;
316 	error = nvme_completion_is_error(cpl);
317 	retry = error && nvme_completion_is_retry(cpl) &&
318 	   req->retries < nvme_retry_count;
319 
320 	if (error && print_on_error) {
321 		nvme_qpair_print_command(qpair, &req->cmd);
322 		nvme_qpair_print_completion(qpair, cpl);
323 	}
324 
325 	qpair->act_tr[cpl->cid] = NULL;
326 
327 	KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n"));
328 
329 	if (req->cb_fn && !retry)
330 		req->cb_fn(req->cb_arg, cpl);
331 
332 	mtx_lock(&qpair->lock);
333 	callout_stop(&tr->timer);
334 
335 	if (retry) {
336 		req->retries++;
337 		nvme_qpair_submit_tracker(qpair, tr);
338 	} else {
339 		if (req->type != NVME_REQUEST_NULL)
340 			bus_dmamap_unload(qpair->dma_tag,
341 			    tr->payload_dma_map);
342 
343 		nvme_free_request(req);
344 		tr->req = NULL;
345 
346 		TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq);
347 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
348 
349 		/*
350 		 * If the controller is in the middle of resetting, don't
351 		 *  try to submit queued requests here - let the reset logic
352 		 *  handle that instead.
353 		 */
354 		if (!STAILQ_EMPTY(&qpair->queued_req) &&
355 		    !qpair->ctrlr->is_resetting) {
356 			req = STAILQ_FIRST(&qpair->queued_req);
357 			STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
358 			_nvme_qpair_submit_request(qpair, req);
359 		}
360 	}
361 
362 	mtx_unlock(&qpair->lock);
363 }
364 
365 static void
366 nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
367     struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
368     boolean_t print_on_error)
369 {
370 	struct nvme_completion	cpl;
371 
372 	memset(&cpl, 0, sizeof(cpl));
373 	cpl.sqid = qpair->id;
374 	cpl.cid = tr->cid;
375 	cpl.status.sct = sct;
376 	cpl.status.sc = sc;
377 	cpl.status.dnr = dnr;
378 	nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
379 }
380 
381 void
382 nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
383     struct nvme_request *req, uint32_t sct, uint32_t sc,
384     boolean_t print_on_error)
385 {
386 	struct nvme_completion	cpl;
387 	boolean_t		error;
388 
389 	memset(&cpl, 0, sizeof(cpl));
390 	cpl.sqid = qpair->id;
391 	cpl.status.sct = sct;
392 	cpl.status.sc = sc;
393 
394 	error = nvme_completion_is_error(&cpl);
395 
396 	if (error && print_on_error) {
397 		nvme_qpair_print_command(qpair, &req->cmd);
398 		nvme_qpair_print_completion(qpair, &cpl);
399 	}
400 
401 	if (req->cb_fn)
402 		req->cb_fn(req->cb_arg, &cpl);
403 
404 	nvme_free_request(req);
405 }
406 
407 void
408 nvme_qpair_process_completions(struct nvme_qpair *qpair)
409 {
410 	struct nvme_tracker	*tr;
411 	struct nvme_completion	*cpl;
412 
413 	qpair->num_intr_handler_calls++;
414 
415 	if (!qpair->is_enabled)
416 		/*
417 		 * qpair is not enabled, likely because a controller reset is
418 		 *  is in progress.  Ignore the interrupt - any I/O that was
419 		 *  associated with this interrupt will get retried when the
420 		 *  reset is complete.
421 		 */
422 		return;
423 
424 	while (1) {
425 		cpl = &qpair->cpl[qpair->cq_head];
426 
427 		if (cpl->status.p != qpair->phase)
428 			break;
429 
430 		tr = qpair->act_tr[cpl->cid];
431 
432 		if (tr != NULL) {
433 			nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE);
434 			qpair->sq_head = cpl->sqhd;
435 		} else {
436 			nvme_printf(qpair->ctrlr,
437 			    "cpl does not map to outstanding cmd\n");
438 			nvme_dump_completion(cpl);
439 			KASSERT(0, ("received completion for unknown cmd\n"));
440 		}
441 
442 		if (++qpair->cq_head == qpair->num_entries) {
443 			qpair->cq_head = 0;
444 			qpair->phase = !qpair->phase;
445 		}
446 
447 		nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
448 		    qpair->cq_head);
449 	}
450 }
451 
452 static void
453 nvme_qpair_msix_handler(void *arg)
454 {
455 	struct nvme_qpair *qpair = arg;
456 
457 	nvme_qpair_process_completions(qpair);
458 }
459 
460 void
461 nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
462     uint16_t vector, uint32_t num_entries, uint32_t num_trackers,
463     struct nvme_controller *ctrlr)
464 {
465 	struct nvme_tracker	*tr;
466 	uint32_t		i;
467 
468 	qpair->id = id;
469 	qpair->vector = vector;
470 	qpair->num_entries = num_entries;
471 #ifdef CHATHAM2
472 	/*
473 	 * Chatham prototype board starts having issues at higher queue
474 	 *  depths.  So use a conservative estimate here of no more than 64
475 	 *  outstanding I/O per queue at any one point.
476 	 */
477 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
478 		num_trackers = min(num_trackers, 64);
479 #endif
480 	qpair->num_trackers = num_trackers;
481 	qpair->ctrlr = ctrlr;
482 
483 	if (ctrlr->msix_enabled) {
484 
485 		/*
486 		 * MSI-X vector resource IDs start at 1, so we add one to
487 		 *  the queue's vector to get the corresponding rid to use.
488 		 */
489 		qpair->rid = vector + 1;
490 		qpair->res = ctrlr->msi_res[vector];
491 
492 		bus_setup_intr(ctrlr->dev, qpair->res,
493 		    INTR_TYPE_MISC | INTR_MPSAFE, NULL,
494 		    nvme_qpair_msix_handler, qpair, &qpair->tag);
495 	}
496 
497 	mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
498 
499 	/* Note: NVMe PRP format is restricted to 4-byte alignment. */
500 	bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
501 	    4, PAGE_SIZE, BUS_SPACE_MAXADDR,
502 	    BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE,
503 	    (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0,
504 	    NULL, NULL, &qpair->dma_tag);
505 
506 	qpair->num_cmds = 0;
507 	qpair->num_intr_handler_calls = 0;
508 
509 	qpair->cmd = contigmalloc(qpair->num_entries *
510 	    sizeof(struct nvme_command), M_NVME, M_ZERO,
511 	    0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
512 	qpair->cpl = contigmalloc(qpair->num_entries *
513 	    sizeof(struct nvme_completion), M_NVME, M_ZERO,
514 	    0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
515 
516 	bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map);
517 	bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map);
518 
519 	bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map,
520 	    qpair->cmd, qpair->num_entries * sizeof(struct nvme_command),
521 	    nvme_single_map, &qpair->cmd_bus_addr, 0);
522 	bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map,
523 	    qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion),
524 	    nvme_single_map, &qpair->cpl_bus_addr, 0);
525 
526 	qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
527 	qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
528 
529 	TAILQ_INIT(&qpair->free_tr);
530 	TAILQ_INIT(&qpair->outstanding_tr);
531 	STAILQ_INIT(&qpair->queued_req);
532 
533 	for (i = 0; i < qpair->num_trackers; i++) {
534 		tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_WAITOK);
535 		nvme_qpair_construct_tracker(qpair, tr, i);
536 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
537 	}
538 
539 	qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries,
540 	    M_NVME, M_ZERO | M_WAITOK);
541 }
542 
543 static void
544 nvme_qpair_destroy(struct nvme_qpair *qpair)
545 {
546 	struct nvme_tracker	*tr;
547 
548 	if (qpair->tag)
549 		bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag);
550 
551 	if (qpair->res)
552 		bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ,
553 		    rman_get_rid(qpair->res), qpair->res);
554 
555 	if (qpair->cmd) {
556 		bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map);
557 		bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map);
558 		contigfree(qpair->cmd,
559 		    qpair->num_entries * sizeof(struct nvme_command), M_NVME);
560 	}
561 
562 	if (qpair->cpl) {
563 		bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map);
564 		bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map);
565 		contigfree(qpair->cpl,
566 		    qpair->num_entries * sizeof(struct nvme_completion),
567 		    M_NVME);
568 	}
569 
570 	if (qpair->dma_tag)
571 		bus_dma_tag_destroy(qpair->dma_tag);
572 
573 	if (qpair->act_tr)
574 		free(qpair->act_tr, M_NVME);
575 
576 	while (!TAILQ_EMPTY(&qpair->free_tr)) {
577 		tr = TAILQ_FIRST(&qpair->free_tr);
578 		TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
579 		bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map);
580 		bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map);
581 		free(tr, M_NVME);
582 	}
583 }
584 
585 static void
586 nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
587 {
588 	struct nvme_tracker	*tr;
589 
590 	tr = TAILQ_FIRST(&qpair->outstanding_tr);
591 	while (tr != NULL) {
592 		if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
593 			nvme_qpair_manual_complete_tracker(qpair, tr,
594 			    NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0,
595 			    FALSE);
596 			tr = TAILQ_FIRST(&qpair->outstanding_tr);
597 		} else {
598 			tr = TAILQ_NEXT(tr, tailq);
599 		}
600 	}
601 }
602 
603 void
604 nvme_admin_qpair_destroy(struct nvme_qpair *qpair)
605 {
606 
607 	nvme_admin_qpair_abort_aers(qpair);
608 	nvme_qpair_destroy(qpair);
609 }
610 
611 void
612 nvme_io_qpair_destroy(struct nvme_qpair *qpair)
613 {
614 
615 	nvme_qpair_destroy(qpair);
616 }
617 
618 static void
619 nvme_abort_complete(void *arg, const struct nvme_completion *status)
620 {
621 	struct nvme_tracker	*tr = arg;
622 
623 	/*
624 	 * If cdw0 == 1, the controller was not able to abort the command
625 	 *  we requested.  We still need to check the active tracker array,
626 	 *  to cover race where I/O timed out at same time controller was
627 	 *  completing the I/O.
628 	 */
629 	if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) {
630 		/*
631 		 * An I/O has timed out, and the controller was unable to
632 		 *  abort it for some reason.  Construct a fake completion
633 		 *  status, and then complete the I/O's tracker manually.
634 		 */
635 		nvme_printf(tr->qpair->ctrlr,
636 		    "abort command failed, aborting command manually\n");
637 		nvme_qpair_manual_complete_tracker(tr->qpair, tr,
638 		    NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
639 	}
640 }
641 
642 static void
643 nvme_timeout(void *arg)
644 {
645 	struct nvme_tracker	*tr = arg;
646 	struct nvme_qpair	*qpair = tr->qpair;
647 	struct nvme_controller	*ctrlr = qpair->ctrlr;
648 	union csts_register	csts;
649 
650 	/* Read csts to get value of cfs - controller fatal status. */
651 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
652 
653 	if (ctrlr->enable_aborts && csts.bits.cfs == 0) {
654 		/*
655 		 * If aborts are enabled, only use them if the controller is
656 		 *  not reporting fatal status.
657 		 */
658 		nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,
659 		    nvme_abort_complete, tr);
660 	} else
661 		nvme_ctrlr_reset(ctrlr);
662 }
663 
664 void
665 nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
666 {
667 	struct nvme_request	*req;
668 	struct nvme_controller	*ctrlr;
669 
670 	mtx_assert(&qpair->lock, MA_OWNED);
671 
672 	req = tr->req;
673 	req->cmd.cid = tr->cid;
674 	qpair->act_tr[tr->cid] = tr;
675 	ctrlr = qpair->ctrlr;
676 
677 	if (req->timeout)
678 #if __FreeBSD_version >= 800030
679 		callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz,
680 		    nvme_timeout, tr);
681 #else
682 		callout_reset(&tr->timer, ctrlr->timeout_period * hz,
683 		    nvme_timeout, tr);
684 #endif
685 
686 	/* Copy the command from the tracker to the submission queue. */
687 	memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
688 
689 	if (++qpair->sq_tail == qpair->num_entries)
690 		qpair->sq_tail = 0;
691 
692 	wmb();
693 	nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
694 	    qpair->sq_tail);
695 
696 	qpair->num_cmds++;
697 }
698 
699 static void
700 nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
701 {
702 	struct nvme_tracker 	*tr = arg;
703 	uint32_t		cur_nseg;
704 
705 	/*
706 	 * If the mapping operation failed, return immediately.  The caller
707 	 *  is responsible for detecting the error status and failing the
708 	 *  tracker manually.
709 	 */
710 	if (error != 0)
711 		return;
712 
713 	/*
714 	 * Note that we specified PAGE_SIZE for alignment and max
715 	 *  segment size when creating the bus dma tags.  So here
716 	 *  we can safely just transfer each segment to its
717 	 *  associated PRP entry.
718 	 */
719 	tr->req->cmd.prp1 = seg[0].ds_addr;
720 
721 	if (nseg == 2) {
722 		tr->req->cmd.prp2 = seg[1].ds_addr;
723 	} else if (nseg > 2) {
724 		cur_nseg = 1;
725 		tr->req->cmd.prp2 = (uint64_t)tr->prp_bus_addr;
726 		while (cur_nseg < nseg) {
727 			tr->prp[cur_nseg-1] =
728 			    (uint64_t)seg[cur_nseg].ds_addr;
729 			cur_nseg++;
730 		}
731 	}
732 
733 	nvme_qpair_submit_tracker(tr->qpair, tr);
734 }
735 
736 static void
737 _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
738 {
739 	struct nvme_tracker	*tr;
740 	int			err = 0;
741 
742 	mtx_assert(&qpair->lock, MA_OWNED);
743 
744 	tr = TAILQ_FIRST(&qpair->free_tr);
745 	req->qpair = qpair;
746 
747 	if (tr == NULL || !qpair->is_enabled) {
748 		/*
749 		 * No tracker is available, or the qpair is disabled due to
750 		 *  an in-progress controller-level reset or controller
751 		 *  failure.
752 		 */
753 
754 		if (qpair->ctrlr->is_failed) {
755 			/*
756 			 * The controller has failed.  Post the request to a
757 			 *  task where it will be aborted, so that we do not
758 			 *  invoke the request's callback in the context
759 			 *  of the submission.
760 			 */
761 			nvme_ctrlr_post_failed_request(qpair->ctrlr, req);
762 		} else {
763 			/*
764 			 * Put the request on the qpair's request queue to be
765 			 *  processed when a tracker frees up via a command
766 			 *  completion or when the controller reset is
767 			 *  completed.
768 			 */
769 			STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
770 		}
771 		return;
772 	}
773 
774 	TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
775 	TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
776 	tr->req = req;
777 
778 	switch (req->type) {
779 	case NVME_REQUEST_VADDR:
780 		KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size,
781 		    ("payload_size (%d) exceeds max_xfer_size (%d)\n",
782 		    req->payload_size, qpair->ctrlr->max_xfer_size));
783 		err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map,
784 		    req->u.payload, req->payload_size, nvme_payload_map, tr, 0);
785 		if (err != 0)
786 			nvme_printf(qpair->ctrlr,
787 			    "bus_dmamap_load returned 0x%x!\n", err);
788 		break;
789 	case NVME_REQUEST_NULL:
790 		nvme_qpair_submit_tracker(tr->qpair, tr);
791 		break;
792 #ifdef NVME_UNMAPPED_BIO_SUPPORT
793 	case NVME_REQUEST_BIO:
794 		KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size,
795 		    ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n",
796 		    (intmax_t)req->u.bio->bio_bcount,
797 		    qpair->ctrlr->max_xfer_size));
798 		err = bus_dmamap_load_bio(tr->qpair->dma_tag,
799 		    tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0);
800 		if (err != 0)
801 			nvme_printf(qpair->ctrlr,
802 			    "bus_dmamap_load_bio returned 0x%x!\n", err);
803 		break;
804 #endif
805 	default:
806 		panic("unknown nvme request type 0x%x\n", req->type);
807 		break;
808 	}
809 
810 	if (err != 0) {
811 		/*
812 		 * The dmamap operation failed, so we manually fail the
813 		 *  tracker here with DATA_TRANSFER_ERROR status.
814 		 *
815 		 * nvme_qpair_manual_complete_tracker must not be called
816 		 *  with the qpair lock held.
817 		 */
818 		mtx_unlock(&qpair->lock);
819 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
820 		    NVME_SC_DATA_TRANSFER_ERROR, 1 /* do not retry */, TRUE);
821 		mtx_lock(&qpair->lock);
822 	}
823 }
824 
825 void
826 nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
827 {
828 
829 	mtx_lock(&qpair->lock);
830 	_nvme_qpair_submit_request(qpair, req);
831 	mtx_unlock(&qpair->lock);
832 }
833 
834 static void
835 nvme_qpair_enable(struct nvme_qpair *qpair)
836 {
837 
838 	qpair->is_enabled = TRUE;
839 }
840 
841 void
842 nvme_qpair_reset(struct nvme_qpair *qpair)
843 {
844 
845 	qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;
846 
847 	/*
848 	 * First time through the completion queue, HW will set phase
849 	 *  bit on completions to 1.  So set this to 1 here, indicating
850 	 *  we're looking for a 1 to know which entries have completed.
851 	 *  we'll toggle the bit each time when the completion queue
852 	 *  rolls over.
853 	 */
854 	qpair->phase = 1;
855 
856 	memset(qpair->cmd, 0,
857 	    qpair->num_entries * sizeof(struct nvme_command));
858 	memset(qpair->cpl, 0,
859 	    qpair->num_entries * sizeof(struct nvme_completion));
860 }
861 
862 void
863 nvme_admin_qpair_enable(struct nvme_qpair *qpair)
864 {
865 	struct nvme_tracker		*tr;
866 	struct nvme_tracker		*tr_temp;
867 
868 	/*
869 	 * Manually abort each outstanding admin command.  Do not retry
870 	 *  admin commands found here, since they will be left over from
871 	 *  a controller reset and its likely the context in which the
872 	 *  command was issued no longer applies.
873 	 */
874 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
875 		nvme_printf(qpair->ctrlr,
876 		    "aborting outstanding admin command\n");
877 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
878 		    NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE);
879 	}
880 
881 	nvme_qpair_enable(qpair);
882 }
883 
884 void
885 nvme_io_qpair_enable(struct nvme_qpair *qpair)
886 {
887 	STAILQ_HEAD(, nvme_request)	temp;
888 	struct nvme_tracker		*tr;
889 	struct nvme_tracker		*tr_temp;
890 	struct nvme_request		*req;
891 
892 	/*
893 	 * Manually abort each outstanding I/O.  This normally results in a
894 	 *  retry, unless the retry count on the associated request has
895 	 *  reached its limit.
896 	 */
897 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
898 		nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n");
899 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
900 		    NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
901 	}
902 
903 	mtx_lock(&qpair->lock);
904 
905 	nvme_qpair_enable(qpair);
906 
907 	STAILQ_INIT(&temp);
908 	STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);
909 
910 	while (!STAILQ_EMPTY(&temp)) {
911 		req = STAILQ_FIRST(&temp);
912 		STAILQ_REMOVE_HEAD(&temp, stailq);
913 		nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n");
914 		nvme_qpair_print_command(qpair, &req->cmd);
915 		_nvme_qpair_submit_request(qpair, req);
916 	}
917 
918 	mtx_unlock(&qpair->lock);
919 }
920 
921 static void
922 nvme_qpair_disable(struct nvme_qpair *qpair)
923 {
924 	struct nvme_tracker *tr;
925 
926 	qpair->is_enabled = FALSE;
927 	mtx_lock(&qpair->lock);
928 	TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq)
929 		callout_stop(&tr->timer);
930 	mtx_unlock(&qpair->lock);
931 }
932 
933 void
934 nvme_admin_qpair_disable(struct nvme_qpair *qpair)
935 {
936 
937 	nvme_qpair_disable(qpair);
938 	nvme_admin_qpair_abort_aers(qpair);
939 }
940 
941 void
942 nvme_io_qpair_disable(struct nvme_qpair *qpair)
943 {
944 
945 	nvme_qpair_disable(qpair);
946 }
947 
948 void
949 nvme_qpair_fail(struct nvme_qpair *qpair)
950 {
951 	struct nvme_tracker		*tr;
952 	struct nvme_request		*req;
953 
954 	mtx_lock(&qpair->lock);
955 
956 	while (!STAILQ_EMPTY(&qpair->queued_req)) {
957 		req = STAILQ_FIRST(&qpair->queued_req);
958 		STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
959 		nvme_printf(qpair->ctrlr, "failing queued i/o\n");
960 		mtx_unlock(&qpair->lock);
961 		nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
962 		    NVME_SC_ABORTED_BY_REQUEST, TRUE);
963 		mtx_lock(&qpair->lock);
964 	}
965 
966 	/* Manually abort each outstanding I/O. */
967 	while (!TAILQ_EMPTY(&qpair->outstanding_tr)) {
968 		tr = TAILQ_FIRST(&qpair->outstanding_tr);
969 		/*
970 		 * Do not remove the tracker.  The abort_tracker path will
971 		 *  do that for us.
972 		 */
973 		nvme_printf(qpair->ctrlr, "failing outstanding i/o\n");
974 		mtx_unlock(&qpair->lock);
975 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
976 		    NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE);
977 		mtx_lock(&qpair->lock);
978 	}
979 
980 	mtx_unlock(&qpair->lock);
981 }
982 
983