xref: /freebsd/sys/dev/nvme/nvme_ctrlr.c (revision 6af6a52ee47bbe1b6148006bd33377f419f9643a)
1bb0ec6b3SJim Harris /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
450dea2daSJim Harris  * Copyright (C) 2012-2016 Intel Corporation
5bb0ec6b3SJim Harris  * All rights reserved.
6bb0ec6b3SJim Harris  *
7bb0ec6b3SJim Harris  * Redistribution and use in source and binary forms, with or without
8bb0ec6b3SJim Harris  * modification, are permitted provided that the following conditions
9bb0ec6b3SJim Harris  * are met:
10bb0ec6b3SJim Harris  * 1. Redistributions of source code must retain the above copyright
11bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer.
12bb0ec6b3SJim Harris  * 2. Redistributions in binary form must reproduce the above copyright
13bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer in the
14bb0ec6b3SJim Harris  *    documentation and/or other materials provided with the distribution.
15bb0ec6b3SJim Harris  *
16bb0ec6b3SJim Harris  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17bb0ec6b3SJim Harris  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18bb0ec6b3SJim Harris  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19bb0ec6b3SJim Harris  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20bb0ec6b3SJim Harris  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21bb0ec6b3SJim Harris  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22bb0ec6b3SJim Harris  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23bb0ec6b3SJim Harris  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24bb0ec6b3SJim Harris  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25bb0ec6b3SJim Harris  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26bb0ec6b3SJim Harris  * SUCH DAMAGE.
27bb0ec6b3SJim Harris  */
28bb0ec6b3SJim Harris 
29bb0ec6b3SJim Harris #include <sys/cdefs.h>
30bb0ec6b3SJim Harris __FBSDID("$FreeBSD$");
31bb0ec6b3SJim Harris 
32f24c011bSWarner Losh #include "opt_cam.h"
334b3da659SWarner Losh #include "opt_nvme.h"
34f24c011bSWarner Losh 
35bb0ec6b3SJim Harris #include <sys/param.h>
367c3f19d7SJim Harris #include <sys/systm.h>
377c3f19d7SJim Harris #include <sys/buf.h>
38bb0ec6b3SJim Harris #include <sys/bus.h>
39bb0ec6b3SJim Harris #include <sys/conf.h>
40bb0ec6b3SJim Harris #include <sys/ioccom.h>
417c3f19d7SJim Harris #include <sys/proc.h>
42bb0ec6b3SJim Harris #include <sys/smp.h>
437c3f19d7SJim Harris #include <sys/uio.h>
44244b8053SWarner Losh #include <sys/sbuf.h>
450d787e9bSWojciech Macek #include <sys/endian.h>
46244b8053SWarner Losh #include <machine/stdarg.h>
471eab19cbSAlexander Motin #include <vm/vm.h>
48bb0ec6b3SJim Harris 
49bb0ec6b3SJim Harris #include "nvme_private.h"
50bb0ec6b3SJim Harris 
510d787e9bSWojciech Macek #define B4_CHK_RDY_DELAY_MS	2300		/* work around controller bug */
52ce1ec9c1SWarner Losh 
530a0b08ccSJim Harris static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
540a0b08ccSJim Harris 						struct nvme_async_event_request *aer);
55bb0ec6b3SJim Harris 
56244b8053SWarner Losh static void
57d5fca1dcSWarner Losh nvme_ctrlr_barrier(struct nvme_controller *ctrlr, int flags)
58d5fca1dcSWarner Losh {
59d5fca1dcSWarner Losh 	bus_barrier(ctrlr->resource, 0, rman_get_size(ctrlr->resource), flags);
60d5fca1dcSWarner Losh }
61d5fca1dcSWarner Losh 
62d5fca1dcSWarner Losh static void
63244b8053SWarner Losh nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)
64244b8053SWarner Losh {
65244b8053SWarner Losh 	struct sbuf sb;
66244b8053SWarner Losh 	va_list ap;
67244b8053SWarner Losh 	int error;
68244b8053SWarner Losh 
694e6a434bSWarner Losh 	if (sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND | SBUF_NOWAIT) == NULL)
704e6a434bSWarner Losh 		return;
71244b8053SWarner Losh 	sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));
72244b8053SWarner Losh 	va_start(ap, msg);
73244b8053SWarner Losh 	sbuf_vprintf(&sb, msg, ap);
74244b8053SWarner Losh 	va_end(ap);
75244b8053SWarner Losh 	error = sbuf_finish(&sb);
76244b8053SWarner Losh 	if (error == 0)
77244b8053SWarner Losh 		printf("%s\n", sbuf_data(&sb));
78244b8053SWarner Losh 
79244b8053SWarner Losh 	sbuf_clear(&sb);
80244b8053SWarner Losh 	sbuf_printf(&sb, "name=\"%s\" reason=\"", device_get_nameunit(ctrlr->dev));
81244b8053SWarner Losh 	va_start(ap, msg);
82244b8053SWarner Losh 	sbuf_vprintf(&sb, msg, ap);
83244b8053SWarner Losh 	va_end(ap);
84244b8053SWarner Losh 	sbuf_printf(&sb, "\"");
85244b8053SWarner Losh 	error = sbuf_finish(&sb);
86244b8053SWarner Losh 	if (error == 0)
87244b8053SWarner Losh 		devctl_notify("nvme", "controller", type, sbuf_data(&sb));
88244b8053SWarner Losh 	sbuf_delete(&sb);
89244b8053SWarner Losh }
90244b8053SWarner Losh 
91a965389bSScott Long static int
92bb0ec6b3SJim Harris nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
93bb0ec6b3SJim Harris {
94bb0ec6b3SJim Harris 	struct nvme_qpair	*qpair;
95bb0ec6b3SJim Harris 	uint32_t		num_entries;
96a965389bSScott Long 	int			error;
97bb0ec6b3SJim Harris 
98bb0ec6b3SJim Harris 	qpair = &ctrlr->adminq;
991eab19cbSAlexander Motin 	qpair->id = 0;
1001eab19cbSAlexander Motin 	qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
1011eab19cbSAlexander Motin 	qpair->domain = ctrlr->domain;
102bb0ec6b3SJim Harris 
103bb0ec6b3SJim Harris 	num_entries = NVME_ADMIN_ENTRIES;
104bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
105bb0ec6b3SJim Harris 	/*
106bb0ec6b3SJim Harris 	 * If admin_entries was overridden to an invalid value, revert it
107bb0ec6b3SJim Harris 	 *  back to our default value.
108bb0ec6b3SJim Harris 	 */
109bb0ec6b3SJim Harris 	if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
110bb0ec6b3SJim Harris 	    num_entries > NVME_MAX_ADMIN_ENTRIES) {
111547d523eSJim Harris 		nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
112547d523eSJim Harris 		    "specified\n", num_entries);
113bb0ec6b3SJim Harris 		num_entries = NVME_ADMIN_ENTRIES;
114bb0ec6b3SJim Harris 	}
115bb0ec6b3SJim Harris 
116bb0ec6b3SJim Harris 	/*
117bb0ec6b3SJim Harris 	 * The admin queue's max xfer size is treated differently than the
118bb0ec6b3SJim Harris 	 *  max I/O xfer size.  16KB is sufficient here - maybe even less?
119bb0ec6b3SJim Harris 	 */
1201eab19cbSAlexander Motin 	error = nvme_qpair_construct(qpair, num_entries, NVME_ADMIN_TRACKERS,
12121b6da58SJim Harris 	     ctrlr);
122a965389bSScott Long 	return (error);
123bb0ec6b3SJim Harris }
124bb0ec6b3SJim Harris 
1251eab19cbSAlexander Motin #define QP(ctrlr, c)	((c) * (ctrlr)->num_io_queues / mp_ncpus)
1261eab19cbSAlexander Motin 
127bb0ec6b3SJim Harris static int
128bb0ec6b3SJim Harris nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
129bb0ec6b3SJim Harris {
130bb0ec6b3SJim Harris 	struct nvme_qpair	*qpair;
1310d787e9bSWojciech Macek 	uint32_t		cap_lo;
1320d787e9bSWojciech Macek 	uint16_t		mqes;
1331eab19cbSAlexander Motin 	int			c, error, i, n;
1341eab19cbSAlexander Motin 	int			num_entries, num_trackers, max_entries;
135bb0ec6b3SJim Harris 
136bb0ec6b3SJim Harris 	/*
137f93b7f95SWarner Losh 	 * NVMe spec sets a hard limit of 64K max entries, but devices may
138f93b7f95SWarner Losh 	 * specify a smaller limit, so we need to check the MQES field in the
139f93b7f95SWarner Losh 	 * capabilities register. We have to cap the number of entries to the
140f93b7f95SWarner Losh 	 * current stride allows for in BAR 0/1, otherwise the remainder entries
141f93b7f95SWarner Losh 	 * are inaccessable. MQES should reflect this, and this is just a
142f93b7f95SWarner Losh 	 * fail-safe.
143bb0ec6b3SJim Harris 	 */
144f93b7f95SWarner Losh 	max_entries =
145f93b7f95SWarner Losh 	    (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
146f93b7f95SWarner Losh 	    (1 << (ctrlr->dstrd + 1));
147f93b7f95SWarner Losh 	num_entries = NVME_IO_ENTRIES;
148f93b7f95SWarner Losh 	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
1490d787e9bSWojciech Macek 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
15062d2cf18SWarner Losh 	mqes = NVME_CAP_LO_MQES(cap_lo);
1510d787e9bSWojciech Macek 	num_entries = min(num_entries, mqes + 1);
152f93b7f95SWarner Losh 	num_entries = min(num_entries, max_entries);
153bb0ec6b3SJim Harris 
15421b6da58SJim Harris 	num_trackers = NVME_IO_TRACKERS;
15521b6da58SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
15621b6da58SJim Harris 
15721b6da58SJim Harris 	num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
15821b6da58SJim Harris 	num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
15921b6da58SJim Harris 	/*
160f93b7f95SWarner Losh 	 * No need to have more trackers than entries in the submit queue.  Note
161f93b7f95SWarner Losh 	 * also that for a queue size of N, we can only have (N-1) commands
162f93b7f95SWarner Losh 	 * outstanding, hence the "-1" here.
16321b6da58SJim Harris 	 */
16421b6da58SJim Harris 	num_trackers = min(num_trackers, (num_entries-1));
16521b6da58SJim Harris 
1662b647da7SJim Harris 	/*
167c02565f9SWarner Losh 	 * Our best estimate for the maximum number of I/Os that we should
1684d547561SWarner Losh 	 * normally have in flight at one time. This should be viewed as a hint,
1694d547561SWarner Losh 	 * not a hard limit and will need to be revisited when the upper layers
170c02565f9SWarner Losh 	 * of the storage system grows multi-queue support.
171c02565f9SWarner Losh 	 */
1725fff95ccSWarner Losh 	ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;
173c02565f9SWarner Losh 
174bb0ec6b3SJim Harris 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
175237d2019SJim Harris 	    M_NVME, M_ZERO | M_WAITOK);
176bb0ec6b3SJim Harris 
1771eab19cbSAlexander Motin 	for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {
178bb0ec6b3SJim Harris 		qpair = &ctrlr->ioq[i];
179bb0ec6b3SJim Harris 
180bb0ec6b3SJim Harris 		/*
181bb0ec6b3SJim Harris 		 * Admin queue has ID=0. IO queues start at ID=1 -
182bb0ec6b3SJim Harris 		 *  hence the 'i+1' here.
1831eab19cbSAlexander Motin 		 */
1841eab19cbSAlexander Motin 		qpair->id = i + 1;
1851eab19cbSAlexander Motin 		if (ctrlr->num_io_queues > 1) {
1861eab19cbSAlexander Motin 			/* Find number of CPUs served by this queue. */
1871eab19cbSAlexander Motin 			for (n = 1; QP(ctrlr, c + n) == i; n++)
1881eab19cbSAlexander Motin 				;
1891eab19cbSAlexander Motin 			/* Shuffle multiple NVMe devices between CPUs. */
1901eab19cbSAlexander Motin 			qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;
1911eab19cbSAlexander Motin 			qpair->domain = pcpu_find(qpair->cpu)->pc_domain;
1921eab19cbSAlexander Motin 		} else {
1931eab19cbSAlexander Motin 			qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
1941eab19cbSAlexander Motin 			qpair->domain = ctrlr->domain;
1951eab19cbSAlexander Motin 		}
1961eab19cbSAlexander Motin 
1971eab19cbSAlexander Motin 		/*
198bb0ec6b3SJim Harris 		 * For I/O queues, use the controller-wide max_xfer_size
199bb0ec6b3SJim Harris 		 *  calculated in nvme_attach().
200bb0ec6b3SJim Harris 		 */
2011eab19cbSAlexander Motin 		error = nvme_qpair_construct(qpair, num_entries, num_trackers,
202bb0ec6b3SJim Harris 		    ctrlr);
203a965389bSScott Long 		if (error)
204a965389bSScott Long 			return (error);
205bb0ec6b3SJim Harris 
2062b647da7SJim Harris 		/*
2072b647da7SJim Harris 		 * Do not bother binding interrupts if we only have one I/O
2082b647da7SJim Harris 		 *  interrupt thread for this controller.
2092b647da7SJim Harris 		 */
210c75ad8ceSJim Harris 		if (ctrlr->num_io_queues > 1)
2111eab19cbSAlexander Motin 			bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);
212bb0ec6b3SJim Harris 	}
213bb0ec6b3SJim Harris 
214bb0ec6b3SJim Harris 	return (0);
215bb0ec6b3SJim Harris }
216bb0ec6b3SJim Harris 
217232e2edbSJim Harris static void
218232e2edbSJim Harris nvme_ctrlr_fail(struct nvme_controller *ctrlr)
219232e2edbSJim Harris {
220232e2edbSJim Harris 	int i;
221232e2edbSJim Harris 
2227588c6ccSWarner Losh 	ctrlr->is_failed = true;
22371a28181SAlexander Motin 	nvme_admin_qpair_disable(&ctrlr->adminq);
224232e2edbSJim Harris 	nvme_qpair_fail(&ctrlr->adminq);
225824073fbSWarner Losh 	if (ctrlr->ioq != NULL) {
22671a28181SAlexander Motin 		for (i = 0; i < ctrlr->num_io_queues; i++) {
22771a28181SAlexander Motin 			nvme_io_qpair_disable(&ctrlr->ioq[i]);
228232e2edbSJim Harris 			nvme_qpair_fail(&ctrlr->ioq[i]);
229824073fbSWarner Losh 		}
23071a28181SAlexander Motin 	}
231232e2edbSJim Harris 	nvme_notify_fail_consumers(ctrlr);
232232e2edbSJim Harris }
233232e2edbSJim Harris 
234232e2edbSJim Harris void
235232e2edbSJim Harris nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
236232e2edbSJim Harris     struct nvme_request *req)
237232e2edbSJim Harris {
238232e2edbSJim Harris 
239a90b8104SJim Harris 	mtx_lock(&ctrlr->lock);
240232e2edbSJim Harris 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
241a90b8104SJim Harris 	mtx_unlock(&ctrlr->lock);
242502dc84aSWarner Losh 	if (!ctrlr->is_dying)
243232e2edbSJim Harris 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
244232e2edbSJim Harris }
245232e2edbSJim Harris 
246232e2edbSJim Harris static void
247232e2edbSJim Harris nvme_ctrlr_fail_req_task(void *arg, int pending)
248232e2edbSJim Harris {
249232e2edbSJim Harris 	struct nvme_controller	*ctrlr = arg;
250232e2edbSJim Harris 	struct nvme_request	*req;
251232e2edbSJim Harris 
252a90b8104SJim Harris 	mtx_lock(&ctrlr->lock);
253c252f637SAlexander Motin 	while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {
254232e2edbSJim Harris 		STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
255c252f637SAlexander Motin 		mtx_unlock(&ctrlr->lock);
256232e2edbSJim Harris 		nvme_qpair_manual_complete_request(req->qpair, req,
2572ffd6fceSWarner Losh 		    NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST);
258c252f637SAlexander Motin 		mtx_lock(&ctrlr->lock);
259232e2edbSJim Harris 	}
260a90b8104SJim Harris 	mtx_unlock(&ctrlr->lock);
261232e2edbSJim Harris }
262232e2edbSJim Harris 
26383581511SWarner Losh /*
26483581511SWarner Losh  * Wait for RDY to change.
26583581511SWarner Losh  *
26683581511SWarner Losh  * Starts sleeping for 1us and geometrically increases it the longer we wait,
26783581511SWarner Losh  * capped at 1ms.
26883581511SWarner Losh  */
269bb0ec6b3SJim Harris static int
270cbdec09cSJim Harris nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
271bb0ec6b3SJim Harris {
27226259f6aSWarner Losh 	int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);
27383581511SWarner Losh 	sbintime_t delta_t = SBT_1US;
2740d787e9bSWojciech Macek 	uint32_t csts;
275bb0ec6b3SJim Harris 
27671a28181SAlexander Motin 	while (1) {
27771a28181SAlexander Motin 		csts = nvme_mmio_read_4(ctrlr, csts);
2789600aa31SWarner Losh 		if (csts == NVME_GONE)		/* Hot unplug. */
27971a28181SAlexander Motin 			return (ENXIO);
28071a28181SAlexander Motin 		if (((csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK)
28171a28181SAlexander Motin 		    == desired_val)
28271a28181SAlexander Motin 			break;
2834fbbe523SAlexander Motin 		if (timeout - ticks < 0) {
284cbdec09cSJim Harris 			nvme_printf(ctrlr, "controller ready did not become %d "
285cbdec09cSJim Harris 			    "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
286bb0ec6b3SJim Harris 			return (ENXIO);
287bb0ec6b3SJim Harris 		}
28883581511SWarner Losh 
28983581511SWarner Losh 		pause_sbt("nvmerdy", delta_t, 0, C_PREL(1));
29083581511SWarner Losh 		delta_t = min(SBT_1MS, delta_t * 3 / 2);
291bb0ec6b3SJim Harris 	}
292bb0ec6b3SJim Harris 
293bb0ec6b3SJim Harris 	return (0);
294bb0ec6b3SJim Harris }
295bb0ec6b3SJim Harris 
296ce1ec9c1SWarner Losh static int
297bb0ec6b3SJim Harris nvme_ctrlr_disable(struct nvme_controller *ctrlr)
298bb0ec6b3SJim Harris {
2990d787e9bSWojciech Macek 	uint32_t cc;
3000d787e9bSWojciech Macek 	uint32_t csts;
3010d787e9bSWojciech Macek 	uint8_t  en, rdy;
302ce1ec9c1SWarner Losh 	int err;
303bb0ec6b3SJim Harris 
3040d787e9bSWojciech Macek 	cc = nvme_mmio_read_4(ctrlr, cc);
3050d787e9bSWojciech Macek 	csts = nvme_mmio_read_4(ctrlr, csts);
3060d787e9bSWojciech Macek 
3070d787e9bSWojciech Macek 	en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
3080d787e9bSWojciech Macek 	rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
309bb0ec6b3SJim Harris 
310ce1ec9c1SWarner Losh 	/*
311ce1ec9c1SWarner Losh 	 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
312ce1ec9c1SWarner Losh 	 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
313ce1ec9c1SWarner Losh 	 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
314ce1ec9c1SWarner Losh 	 * isn't the desired value. Short circuit if we're already disabled.
315ce1ec9c1SWarner Losh 	 */
316a245627aSWarner Losh 	if (en == 0) {
317a245627aSWarner Losh 		/* Wait for RDY == 0 or timeout & fail */
318a245627aSWarner Losh 		if (rdy == 0)
319a245627aSWarner Losh 			return (0);
320a245627aSWarner Losh 		return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
321a245627aSWarner Losh 	}
3220d787e9bSWojciech Macek 	if (rdy == 0) {
323a245627aSWarner Losh 		/* EN == 1, wait for  RDY == 1 or timeout & fail */
324ce1ec9c1SWarner Losh 		err = nvme_ctrlr_wait_for_ready(ctrlr, 1);
325ce1ec9c1SWarner Losh 		if (err != 0)
326ce1ec9c1SWarner Losh 			return (err);
327ce1ec9c1SWarner Losh 	}
328bb0ec6b3SJim Harris 
3290d787e9bSWojciech Macek 	cc &= ~NVME_CC_REG_EN_MASK;
3300d787e9bSWojciech Macek 	nvme_mmio_write_4(ctrlr, cc, cc);
33177054a89SWarner Losh 
332ce1ec9c1SWarner Losh 	/*
33377054a89SWarner Losh 	 * A few drives have firmware bugs that freeze the drive if we access
33477054a89SWarner Losh 	 * the mmio too soon after we disable.
335ce1ec9c1SWarner Losh 	 */
336989c7f0bSWarner Losh 	if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)
33726259f6aSWarner Losh 		pause("nvmeR", MSEC_2_TICKS(B4_CHK_RDY_DELAY_MS));
338ce1ec9c1SWarner Losh 	return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
339bb0ec6b3SJim Harris }
340bb0ec6b3SJim Harris 
341bb0ec6b3SJim Harris static int
342bb0ec6b3SJim Harris nvme_ctrlr_enable(struct nvme_controller *ctrlr)
343bb0ec6b3SJim Harris {
3440d787e9bSWojciech Macek 	uint32_t	cc;
3450d787e9bSWojciech Macek 	uint32_t	csts;
3460d787e9bSWojciech Macek 	uint32_t	aqa;
3470d787e9bSWojciech Macek 	uint32_t	qsize;
3480d787e9bSWojciech Macek 	uint8_t		en, rdy;
349ce1ec9c1SWarner Losh 	int		err;
350bb0ec6b3SJim Harris 
3510d787e9bSWojciech Macek 	cc = nvme_mmio_read_4(ctrlr, cc);
3520d787e9bSWojciech Macek 	csts = nvme_mmio_read_4(ctrlr, csts);
3530d787e9bSWojciech Macek 
3540d787e9bSWojciech Macek 	en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
3550d787e9bSWojciech Macek 	rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
356bb0ec6b3SJim Harris 
357ce1ec9c1SWarner Losh 	/*
358ce1ec9c1SWarner Losh 	 * See note in nvme_ctrlr_disable. Short circuit if we're already enabled.
359ce1ec9c1SWarner Losh 	 */
3600d787e9bSWojciech Macek 	if (en == 1) {
3610d787e9bSWojciech Macek 		if (rdy == 1)
362bb0ec6b3SJim Harris 			return (0);
363cbdec09cSJim Harris 		return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
364a245627aSWarner Losh 	}
365a245627aSWarner Losh 
366a245627aSWarner Losh 	/* EN == 0 already wait for RDY == 0 or timeout & fail */
367ce1ec9c1SWarner Losh 	err = nvme_ctrlr_wait_for_ready(ctrlr, 0);
368ce1ec9c1SWarner Losh 	if (err != 0)
369ce1ec9c1SWarner Losh 		return (err);
370bb0ec6b3SJim Harris 
371bb0ec6b3SJim Harris 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
372bb0ec6b3SJim Harris 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
373bb0ec6b3SJim Harris 
374bb0ec6b3SJim Harris 	/* acqs and asqs are 0-based. */
3750d787e9bSWojciech Macek 	qsize = ctrlr->adminq.num_entries - 1;
3760d787e9bSWojciech Macek 
3770d787e9bSWojciech Macek 	aqa = 0;
3780d787e9bSWojciech Macek 	aqa = (qsize & NVME_AQA_REG_ACQS_MASK) << NVME_AQA_REG_ACQS_SHIFT;
3790d787e9bSWojciech Macek 	aqa |= (qsize & NVME_AQA_REG_ASQS_MASK) << NVME_AQA_REG_ASQS_SHIFT;
3800d787e9bSWojciech Macek 	nvme_mmio_write_4(ctrlr, aqa, aqa);
381bb0ec6b3SJim Harris 
3820d787e9bSWojciech Macek 	/* Initialization values for CC */
3830d787e9bSWojciech Macek 	cc = 0;
3840d787e9bSWojciech Macek 	cc |= 1 << NVME_CC_REG_EN_SHIFT;
3850d787e9bSWojciech Macek 	cc |= 0 << NVME_CC_REG_CSS_SHIFT;
3860d787e9bSWojciech Macek 	cc |= 0 << NVME_CC_REG_AMS_SHIFT;
3870d787e9bSWojciech Macek 	cc |= 0 << NVME_CC_REG_SHN_SHIFT;
3880d787e9bSWojciech Macek 	cc |= 6 << NVME_CC_REG_IOSQES_SHIFT; /* SQ entry size == 64 == 2^6 */
3890d787e9bSWojciech Macek 	cc |= 4 << NVME_CC_REG_IOCQES_SHIFT; /* CQ entry size == 16 == 2^4 */
390bb0ec6b3SJim Harris 
391bb0ec6b3SJim Harris 	/* This evaluates to 0, which is according to spec. */
392a70b5660SWarner Losh 	cc |= (PAGE_SHIFT - NVME_BASE_SHIFT) << NVME_CC_REG_MPS_SHIFT;
393bb0ec6b3SJim Harris 
394d5fca1dcSWarner Losh 	nvme_ctrlr_barrier(ctrlr, BUS_SPACE_BARRIER_WRITE);
3950d787e9bSWojciech Macek 	nvme_mmio_write_4(ctrlr, cc, cc);
396bb0ec6b3SJim Harris 
397cbdec09cSJim Harris 	return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
398bb0ec6b3SJim Harris }
399bb0ec6b3SJim Harris 
4004d547561SWarner Losh static void
4014d547561SWarner Losh nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)
402bb0ec6b3SJim Harris {
4034d547561SWarner Losh 	int i;
404b846efd7SJim Harris 
405b846efd7SJim Harris 	nvme_admin_qpair_disable(&ctrlr->adminq);
4062b647da7SJim Harris 	/*
4072b647da7SJim Harris 	 * I/O queues are not allocated before the initial HW
4082b647da7SJim Harris 	 *  reset, so do not try to disable them.  Use is_initialized
4092b647da7SJim Harris 	 *  to determine if this is the initial HW reset.
4102b647da7SJim Harris 	 */
4112b647da7SJim Harris 	if (ctrlr->is_initialized) {
412b846efd7SJim Harris 		for (i = 0; i < ctrlr->num_io_queues; i++)
413b846efd7SJim Harris 			nvme_io_qpair_disable(&ctrlr->ioq[i]);
4142b647da7SJim Harris 	}
4154d547561SWarner Losh }
4164d547561SWarner Losh 
417dd2516fcSWarner Losh static int
4184d547561SWarner Losh nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
4194d547561SWarner Losh {
4204d547561SWarner Losh 	int err;
4214d547561SWarner Losh 
422bad42df9SColin Percival 	TSENTER();
423b846efd7SJim Harris 
424e5e26e4aSWarner Losh 	nvme_ctrlr_disable_qpairs(ctrlr);
425bb0ec6b3SJim Harris 
426ce1ec9c1SWarner Losh 	err = nvme_ctrlr_disable(ctrlr);
427ce1ec9c1SWarner Losh 	if (err != 0)
428ce1ec9c1SWarner Losh 		return err;
429e5e26e4aSWarner Losh 
430bad42df9SColin Percival 	err = nvme_ctrlr_enable(ctrlr);
431bad42df9SColin Percival 	TSEXIT();
432bad42df9SColin Percival 	return (err);
433bb0ec6b3SJim Harris }
434bb0ec6b3SJim Harris 
435b846efd7SJim Harris void
436b846efd7SJim Harris nvme_ctrlr_reset(struct nvme_controller *ctrlr)
437b846efd7SJim Harris {
438f37c22a3SJim Harris 	int cmpset;
439f37c22a3SJim Harris 
440f37c22a3SJim Harris 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
441f37c22a3SJim Harris 
442232e2edbSJim Harris 	if (cmpset == 0 || ctrlr->is_failed)
443232e2edbSJim Harris 		/*
444232e2edbSJim Harris 		 * Controller is already resetting or has failed.  Return
445232e2edbSJim Harris 		 *  immediately since there is no need to kick off another
446232e2edbSJim Harris 		 *  reset in these cases.
447232e2edbSJim Harris 		 */
448f37c22a3SJim Harris 		return;
449b846efd7SJim Harris 
450502dc84aSWarner Losh 	if (!ctrlr->is_dying)
45148ce3178SJim Harris 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
452b846efd7SJim Harris }
453b846efd7SJim Harris 
454bb0ec6b3SJim Harris static int
455bb0ec6b3SJim Harris nvme_ctrlr_identify(struct nvme_controller *ctrlr)
456bb0ec6b3SJim Harris {
457955910a9SJim Harris 	struct nvme_completion_poll_status	status;
458bb0ec6b3SJim Harris 
45929077eb4SWarner Losh 	status.done = 0;
460bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
461955910a9SJim Harris 	    nvme_completion_poll_cb, &status);
462ab0681aaSWarner Losh 	nvme_completion_poll(&status);
463955910a9SJim Harris 	if (nvme_completion_is_error(&status.cpl)) {
464547d523eSJim Harris 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
465bb0ec6b3SJim Harris 		return (ENXIO);
466bb0ec6b3SJim Harris 	}
467bb0ec6b3SJim Harris 
4680d787e9bSWojciech Macek 	/* Convert data to host endian */
4690d787e9bSWojciech Macek 	nvme_controller_data_swapbytes(&ctrlr->cdata);
4700d787e9bSWojciech Macek 
47102e33484SJim Harris 	/*
47202e33484SJim Harris 	 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
47302e33484SJim Harris 	 *  controller supports.
47402e33484SJim Harris 	 */
47502e33484SJim Harris 	if (ctrlr->cdata.mdts > 0)
47602e33484SJim Harris 		ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
47702e33484SJim Harris 		    ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
47802e33484SJim Harris 
479bb0ec6b3SJim Harris 	return (0);
480bb0ec6b3SJim Harris }
481bb0ec6b3SJim Harris 
482bb0ec6b3SJim Harris static int
483bb0ec6b3SJim Harris nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
484bb0ec6b3SJim Harris {
485955910a9SJim Harris 	struct nvme_completion_poll_status	status;
4862b647da7SJim Harris 	int					cq_allocated, sq_allocated;
487bb0ec6b3SJim Harris 
48829077eb4SWarner Losh 	status.done = 0;
489bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
490955910a9SJim Harris 	    nvme_completion_poll_cb, &status);
491ab0681aaSWarner Losh 	nvme_completion_poll(&status);
492955910a9SJim Harris 	if (nvme_completion_is_error(&status.cpl)) {
493824073fbSWarner Losh 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
494bb0ec6b3SJim Harris 		return (ENXIO);
495bb0ec6b3SJim Harris 	}
496bb0ec6b3SJim Harris 
497bb0ec6b3SJim Harris 	/*
498bb0ec6b3SJim Harris 	 * Data in cdw0 is 0-based.
499bb0ec6b3SJim Harris 	 * Lower 16-bits indicate number of submission queues allocated.
500bb0ec6b3SJim Harris 	 * Upper 16-bits indicate number of completion queues allocated.
501bb0ec6b3SJim Harris 	 */
502955910a9SJim Harris 	sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
503955910a9SJim Harris 	cq_allocated = (status.cpl.cdw0 >> 16) + 1;
504bb0ec6b3SJim Harris 
505bb0ec6b3SJim Harris 	/*
5062b647da7SJim Harris 	 * Controller may allocate more queues than we requested,
5072b647da7SJim Harris 	 *  so use the minimum of the number requested and what was
5082b647da7SJim Harris 	 *  actually allocated.
509bb0ec6b3SJim Harris 	 */
5102b647da7SJim Harris 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);
5112b647da7SJim Harris 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);
5121eab19cbSAlexander Motin 	if (ctrlr->num_io_queues > vm_ndomains)
5131eab19cbSAlexander Motin 		ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;
514bb0ec6b3SJim Harris 
515bb0ec6b3SJim Harris 	return (0);
516bb0ec6b3SJim Harris }
517bb0ec6b3SJim Harris 
518bb0ec6b3SJim Harris static int
519bb0ec6b3SJim Harris nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
520bb0ec6b3SJim Harris {
521955910a9SJim Harris 	struct nvme_completion_poll_status	status;
522bb0ec6b3SJim Harris 	struct nvme_qpair			*qpair;
523955910a9SJim Harris 	int					i;
524bb0ec6b3SJim Harris 
525bb0ec6b3SJim Harris 	for (i = 0; i < ctrlr->num_io_queues; i++) {
526bb0ec6b3SJim Harris 		qpair = &ctrlr->ioq[i];
527bb0ec6b3SJim Harris 
52829077eb4SWarner Losh 		status.done = 0;
5291eab19cbSAlexander Motin 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,
530955910a9SJim Harris 		    nvme_completion_poll_cb, &status);
531ab0681aaSWarner Losh 		nvme_completion_poll(&status);
532955910a9SJim Harris 		if (nvme_completion_is_error(&status.cpl)) {
533547d523eSJim Harris 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
534bb0ec6b3SJim Harris 			return (ENXIO);
535bb0ec6b3SJim Harris 		}
536bb0ec6b3SJim Harris 
53729077eb4SWarner Losh 		status.done = 0;
538ead7e103SAlexander Motin 		nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,
539955910a9SJim Harris 		    nvme_completion_poll_cb, &status);
540ab0681aaSWarner Losh 		nvme_completion_poll(&status);
541955910a9SJim Harris 		if (nvme_completion_is_error(&status.cpl)) {
542547d523eSJim Harris 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
543bb0ec6b3SJim Harris 			return (ENXIO);
544bb0ec6b3SJim Harris 		}
545bb0ec6b3SJim Harris 	}
546bb0ec6b3SJim Harris 
547bb0ec6b3SJim Harris 	return (0);
548bb0ec6b3SJim Harris }
549bb0ec6b3SJim Harris 
550bb0ec6b3SJim Harris static int
5514d547561SWarner Losh nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)
5528b1e6ebeSWarner Losh {
5538b1e6ebeSWarner Losh 	struct nvme_completion_poll_status	status;
5549835d216SWarner Losh 	struct nvme_qpair			*qpair;
5559835d216SWarner Losh 
5569835d216SWarner Losh 	for (int i = 0; i < ctrlr->num_io_queues; i++) {
5579835d216SWarner Losh 		qpair = &ctrlr->ioq[i];
5588b1e6ebeSWarner Losh 
5598b1e6ebeSWarner Losh 		status.done = 0;
5605d7fd8f7SWarner Losh 		nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
5618b1e6ebeSWarner Losh 		    nvme_completion_poll_cb, &status);
562ab0681aaSWarner Losh 		nvme_completion_poll(&status);
5638b1e6ebeSWarner Losh 		if (nvme_completion_is_error(&status.cpl)) {
5645d7fd8f7SWarner Losh 			nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
5658b1e6ebeSWarner Losh 			return (ENXIO);
5668b1e6ebeSWarner Losh 		}
5678b1e6ebeSWarner Losh 
5688b1e6ebeSWarner Losh 		status.done = 0;
5698b1e6ebeSWarner Losh 		nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
5708b1e6ebeSWarner Losh 		    nvme_completion_poll_cb, &status);
571ab0681aaSWarner Losh 		nvme_completion_poll(&status);
5728b1e6ebeSWarner Losh 		if (nvme_completion_is_error(&status.cpl)) {
5735d7fd8f7SWarner Losh 			nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
5748b1e6ebeSWarner Losh 			return (ENXIO);
5758b1e6ebeSWarner Losh 		}
5769835d216SWarner Losh 	}
5778b1e6ebeSWarner Losh 
5788b1e6ebeSWarner Losh 	return (0);
5798b1e6ebeSWarner Losh }
5808b1e6ebeSWarner Losh 
5818b1e6ebeSWarner Losh static int
582bb0ec6b3SJim Harris nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
583bb0ec6b3SJim Harris {
584bb0ec6b3SJim Harris 	struct nvme_namespace	*ns;
585696c9502SWarner Losh 	uint32_t 		i;
586bb0ec6b3SJim Harris 
587a8a18dd5SWarner Losh 	for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
588bb0ec6b3SJim Harris 		ns = &ctrlr->ns[i];
589a8a18dd5SWarner Losh 		nvme_ns_construct(ns, i+1, ctrlr);
590bb0ec6b3SJim Harris 	}
591bb0ec6b3SJim Harris 
592bb0ec6b3SJim Harris 	return (0);
593bb0ec6b3SJim Harris }
594bb0ec6b3SJim Harris 
5957588c6ccSWarner Losh static bool
5962868353aSJim Harris is_log_page_id_valid(uint8_t page_id)
5972868353aSJim Harris {
5982868353aSJim Harris 
5992868353aSJim Harris 	switch (page_id) {
6002868353aSJim Harris 	case NVME_LOG_ERROR:
6012868353aSJim Harris 	case NVME_LOG_HEALTH_INFORMATION:
6022868353aSJim Harris 	case NVME_LOG_FIRMWARE_SLOT:
603f439e3a4SAlexander Motin 	case NVME_LOG_CHANGED_NAMESPACE:
6046c99d132SAlexander Motin 	case NVME_LOG_COMMAND_EFFECT:
6056c99d132SAlexander Motin 	case NVME_LOG_RES_NOTIFICATION:
6066c99d132SAlexander Motin 	case NVME_LOG_SANITIZE_STATUS:
6077588c6ccSWarner Losh 		return (true);
6082868353aSJim Harris 	}
6092868353aSJim Harris 
6107588c6ccSWarner Losh 	return (false);
6112868353aSJim Harris }
6122868353aSJim Harris 
6132868353aSJim Harris static uint32_t
6142868353aSJim Harris nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
6152868353aSJim Harris {
6162868353aSJim Harris 	uint32_t	log_page_size;
6172868353aSJim Harris 
6182868353aSJim Harris 	switch (page_id) {
6192868353aSJim Harris 	case NVME_LOG_ERROR:
6202868353aSJim Harris 		log_page_size = min(
6212868353aSJim Harris 		    sizeof(struct nvme_error_information_entry) *
6220d787e9bSWojciech Macek 		    (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);
6232868353aSJim Harris 		break;
6242868353aSJim Harris 	case NVME_LOG_HEALTH_INFORMATION:
6252868353aSJim Harris 		log_page_size = sizeof(struct nvme_health_information_page);
6262868353aSJim Harris 		break;
6272868353aSJim Harris 	case NVME_LOG_FIRMWARE_SLOT:
6282868353aSJim Harris 		log_page_size = sizeof(struct nvme_firmware_page);
6292868353aSJim Harris 		break;
630f439e3a4SAlexander Motin 	case NVME_LOG_CHANGED_NAMESPACE:
631f439e3a4SAlexander Motin 		log_page_size = sizeof(struct nvme_ns_list);
632f439e3a4SAlexander Motin 		break;
6336c99d132SAlexander Motin 	case NVME_LOG_COMMAND_EFFECT:
6346c99d132SAlexander Motin 		log_page_size = sizeof(struct nvme_command_effects_page);
6356c99d132SAlexander Motin 		break;
6366c99d132SAlexander Motin 	case NVME_LOG_RES_NOTIFICATION:
6376c99d132SAlexander Motin 		log_page_size = sizeof(struct nvme_res_notification_page);
6386c99d132SAlexander Motin 		break;
6396c99d132SAlexander Motin 	case NVME_LOG_SANITIZE_STATUS:
6406c99d132SAlexander Motin 		log_page_size = sizeof(struct nvme_sanitize_status_page);
6416c99d132SAlexander Motin 		break;
6422868353aSJim Harris 	default:
6432868353aSJim Harris 		log_page_size = 0;
6442868353aSJim Harris 		break;
6452868353aSJim Harris 	}
6462868353aSJim Harris 
6472868353aSJim Harris 	return (log_page_size);
6482868353aSJim Harris }
6492868353aSJim Harris 
6502868353aSJim Harris static void
651bb2f67fdSJim Harris nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
6520d787e9bSWojciech Macek     uint8_t state)
653bb2f67fdSJim Harris {
654bb2f67fdSJim Harris 
6550d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE)
656244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
657244b8053SWarner Losh 		    "available spare space below threshold");
658bb2f67fdSJim Harris 
6590d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_TEMPERATURE)
660244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
661244b8053SWarner Losh 		    "temperature above threshold");
662bb2f67fdSJim Harris 
6630d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_DEVICE_RELIABILITY)
664244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
665244b8053SWarner Losh 		    "device reliability degraded");
666bb2f67fdSJim Harris 
6670d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_READ_ONLY)
668244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
669244b8053SWarner Losh 		    "media placed in read only mode");
670bb2f67fdSJim Harris 
6710d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP)
672244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
673244b8053SWarner Losh 		    "volatile memory backup device failed");
674bb2f67fdSJim Harris 
6750d787e9bSWojciech Macek 	if (state & NVME_CRIT_WARN_ST_RESERVED_MASK)
676244b8053SWarner Losh 		nvme_ctrlr_devctl_log(ctrlr, "critical",
677244b8053SWarner Losh 		    "unknown critical warning(s): state = 0x%02x", state);
678bb2f67fdSJim Harris }
679bb2f67fdSJim Harris 
680bb2f67fdSJim Harris static void
6812868353aSJim Harris nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl)
6822868353aSJim Harris {
6832868353aSJim Harris 	struct nvme_async_event_request		*aer = arg;
684bb2f67fdSJim Harris 	struct nvme_health_information_page	*health_info;
685f439e3a4SAlexander Motin 	struct nvme_ns_list			*nsl;
6860d787e9bSWojciech Macek 	struct nvme_error_information_entry	*err;
6870d787e9bSWojciech Macek 	int i;
6882868353aSJim Harris 
6890d7e13ecSJim Harris 	/*
6900d7e13ecSJim Harris 	 * If the log page fetch for some reason completed with an error,
6910d7e13ecSJim Harris 	 *  don't pass log page data to the consumers.  In practice, this case
6920d7e13ecSJim Harris 	 *  should never happen.
6930d7e13ecSJim Harris 	 */
6940d7e13ecSJim Harris 	if (nvme_completion_is_error(cpl))
6950d7e13ecSJim Harris 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
6960d7e13ecSJim Harris 		    aer->log_page_id, NULL, 0);
697bb2f67fdSJim Harris 	else {
6980d787e9bSWojciech Macek 		/* Convert data to host endian */
6990d787e9bSWojciech Macek 		switch (aer->log_page_id) {
7000d787e9bSWojciech Macek 		case NVME_LOG_ERROR:
7010d787e9bSWojciech Macek 			err = (struct nvme_error_information_entry *)aer->log_page_buffer;
7020d787e9bSWojciech Macek 			for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)
7030d787e9bSWojciech Macek 				nvme_error_information_entry_swapbytes(err++);
7040d787e9bSWojciech Macek 			break;
7050d787e9bSWojciech Macek 		case NVME_LOG_HEALTH_INFORMATION:
7060d787e9bSWojciech Macek 			nvme_health_information_page_swapbytes(
7070d787e9bSWojciech Macek 			    (struct nvme_health_information_page *)aer->log_page_buffer);
7080d787e9bSWojciech Macek 			break;
7090d787e9bSWojciech Macek 		case NVME_LOG_FIRMWARE_SLOT:
7100d787e9bSWojciech Macek 			nvme_firmware_page_swapbytes(
7110d787e9bSWojciech Macek 			    (struct nvme_firmware_page *)aer->log_page_buffer);
7120d787e9bSWojciech Macek 			break;
713f439e3a4SAlexander Motin 		case NVME_LOG_CHANGED_NAMESPACE:
714f439e3a4SAlexander Motin 			nvme_ns_list_swapbytes(
715f439e3a4SAlexander Motin 			    (struct nvme_ns_list *)aer->log_page_buffer);
716f439e3a4SAlexander Motin 			break;
7176c99d132SAlexander Motin 		case NVME_LOG_COMMAND_EFFECT:
7186c99d132SAlexander Motin 			nvme_command_effects_page_swapbytes(
7196c99d132SAlexander Motin 			    (struct nvme_command_effects_page *)aer->log_page_buffer);
7206c99d132SAlexander Motin 			break;
7216c99d132SAlexander Motin 		case NVME_LOG_RES_NOTIFICATION:
7226c99d132SAlexander Motin 			nvme_res_notification_page_swapbytes(
7236c99d132SAlexander Motin 			    (struct nvme_res_notification_page *)aer->log_page_buffer);
7246c99d132SAlexander Motin 			break;
7256c99d132SAlexander Motin 		case NVME_LOG_SANITIZE_STATUS:
7266c99d132SAlexander Motin 			nvme_sanitize_status_page_swapbytes(
7276c99d132SAlexander Motin 			    (struct nvme_sanitize_status_page *)aer->log_page_buffer);
7286c99d132SAlexander Motin 			break;
7290d787e9bSWojciech Macek 		case INTEL_LOG_TEMP_STATS:
7300d787e9bSWojciech Macek 			intel_log_temp_stats_swapbytes(
7310d787e9bSWojciech Macek 			    (struct intel_log_temp_stats *)aer->log_page_buffer);
7320d787e9bSWojciech Macek 			break;
7330d787e9bSWojciech Macek 		default:
7340d787e9bSWojciech Macek 			break;
7350d787e9bSWojciech Macek 		}
7360d787e9bSWojciech Macek 
737bb2f67fdSJim Harris 		if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
738bb2f67fdSJim Harris 			health_info = (struct nvme_health_information_page *)
739bb2f67fdSJim Harris 			    aer->log_page_buffer;
740bb2f67fdSJim Harris 			nvme_ctrlr_log_critical_warnings(aer->ctrlr,
741bb2f67fdSJim Harris 			    health_info->critical_warning);
742bb2f67fdSJim Harris 			/*
743bb2f67fdSJim Harris 			 * Critical warnings reported through the
744bb2f67fdSJim Harris 			 *  SMART/health log page are persistent, so
745bb2f67fdSJim Harris 			 *  clear the associated bits in the async event
746bb2f67fdSJim Harris 			 *  config so that we do not receive repeated
747bb2f67fdSJim Harris 			 *  notifications for the same event.
748bb2f67fdSJim Harris 			 */
7490d787e9bSWojciech Macek 			aer->ctrlr->async_event_config &=
7500d787e9bSWojciech Macek 			    ~health_info->critical_warning;
751bb2f67fdSJim Harris 			nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
752bb2f67fdSJim Harris 			    aer->ctrlr->async_event_config, NULL, NULL);
753f439e3a4SAlexander Motin 		} else if (aer->log_page_id == NVME_LOG_CHANGED_NAMESPACE &&
754f439e3a4SAlexander Motin 		    !nvme_use_nvd) {
755f439e3a4SAlexander Motin 			nsl = (struct nvme_ns_list *)aer->log_page_buffer;
756f439e3a4SAlexander Motin 			for (i = 0; i < nitems(nsl->ns) && nsl->ns[i] != 0; i++) {
757f439e3a4SAlexander Motin 				if (nsl->ns[i] > NVME_MAX_NAMESPACES)
758f439e3a4SAlexander Motin 					break;
759f439e3a4SAlexander Motin 				nvme_notify_ns(aer->ctrlr, nsl->ns[i]);
760f439e3a4SAlexander Motin 			}
761bb2f67fdSJim Harris 		}
762bb2f67fdSJim Harris 
7630d7e13ecSJim Harris 		/*
7640d7e13ecSJim Harris 		 * Pass the cpl data from the original async event completion,
7650d7e13ecSJim Harris 		 *  not the log page fetch.
7660d7e13ecSJim Harris 		 */
7670d7e13ecSJim Harris 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
7680d7e13ecSJim Harris 		    aer->log_page_id, aer->log_page_buffer, aer->log_page_size);
769bb2f67fdSJim Harris 	}
7702868353aSJim Harris 
7712868353aSJim Harris 	/*
7722868353aSJim Harris 	 * Repost another asynchronous event request to replace the one
7732868353aSJim Harris 	 *  that just completed.
7742868353aSJim Harris 	 */
7752868353aSJim Harris 	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
7762868353aSJim Harris }
7772868353aSJim Harris 
778bb0ec6b3SJim Harris static void
7790a0b08ccSJim Harris nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
7800a0b08ccSJim Harris {
7810a0b08ccSJim Harris 	struct nvme_async_event_request	*aer = arg;
7820a0b08ccSJim Harris 
783ec526ea9SJim Harris 	if (nvme_completion_is_error(cpl)) {
7840a0b08ccSJim Harris 		/*
785ec526ea9SJim Harris 		 *  Do not retry failed async event requests.  This avoids
786ec526ea9SJim Harris 		 *  infinite loops where a new async event request is submitted
787ec526ea9SJim Harris 		 *  to replace the one just failed, only to fail again and
788ec526ea9SJim Harris 		 *  perpetuate the loop.
7890a0b08ccSJim Harris 		 */
7900a0b08ccSJim Harris 		return;
7910a0b08ccSJim Harris 	}
7920a0b08ccSJim Harris 
7932868353aSJim Harris 	/* Associated log page is in bits 23:16 of completion entry dw0. */
7940d7e13ecSJim Harris 	aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
7952868353aSJim Harris 
796f439e3a4SAlexander Motin 	nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"
797a6d222ebSAlexander Motin 	    " page 0x%02x)\n", (cpl->cdw0 & 0x07), (cpl->cdw0 & 0xFF00) >> 8,
798547d523eSJim Harris 	    aer->log_page_id);
799547d523eSJim Harris 
8000d7e13ecSJim Harris 	if (is_log_page_id_valid(aer->log_page_id)) {
8012868353aSJim Harris 		aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,
8020d7e13ecSJim Harris 		    aer->log_page_id);
8032868353aSJim Harris 		memcpy(&aer->cpl, cpl, sizeof(*cpl));
8040d7e13ecSJim Harris 		nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
8052868353aSJim Harris 		    NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer,
8062868353aSJim Harris 		    aer->log_page_size, nvme_ctrlr_async_event_log_page_cb,
8072868353aSJim Harris 		    aer);
8082868353aSJim Harris 		/* Wait to notify consumers until after log page is fetched. */
8092868353aSJim Harris 	} else {
8100d7e13ecSJim Harris 		nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,
8110d7e13ecSJim Harris 		    NULL, 0);
812038a5ee4SJim Harris 
8130a0b08ccSJim Harris 		/*
8142868353aSJim Harris 		 * Repost another asynchronous event request to replace the one
8152868353aSJim Harris 		 *  that just completed.
8160a0b08ccSJim Harris 		 */
8170a0b08ccSJim Harris 		nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
8180a0b08ccSJim Harris 	}
8192868353aSJim Harris }
8200a0b08ccSJim Harris 
8210a0b08ccSJim Harris static void
8220a0b08ccSJim Harris nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
8230a0b08ccSJim Harris     struct nvme_async_event_request *aer)
8240a0b08ccSJim Harris {
8250a0b08ccSJim Harris 	struct nvme_request *req;
8260a0b08ccSJim Harris 
8270a0b08ccSJim Harris 	aer->ctrlr = ctrlr;
8281e526bc4SJim Harris 	req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
8290a0b08ccSJim Harris 	aer->req = req;
8300a0b08ccSJim Harris 
8310a0b08ccSJim Harris 	/*
83294143332SJim Harris 	 * Disable timeout here, since asynchronous event requests should by
83394143332SJim Harris 	 *  nature never be timed out.
8340a0b08ccSJim Harris 	 */
8357588c6ccSWarner Losh 	req->timeout = false;
8369544e6dcSChuck Tuffli 	req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
8370a0b08ccSJim Harris 	nvme_ctrlr_submit_admin_request(ctrlr, req);
8380a0b08ccSJim Harris }
8390a0b08ccSJim Harris 
8400a0b08ccSJim Harris static void
841bb0ec6b3SJim Harris nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
842bb0ec6b3SJim Harris {
843d5fc9821SJim Harris 	struct nvme_completion_poll_status	status;
8440a0b08ccSJim Harris 	struct nvme_async_event_request		*aer;
8450a0b08ccSJim Harris 	uint32_t				i;
846bb0ec6b3SJim Harris 
847f439e3a4SAlexander Motin 	ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
848f439e3a4SAlexander Motin 	    NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
849f439e3a4SAlexander Motin 	    NVME_CRIT_WARN_ST_READ_ONLY |
850f439e3a4SAlexander Motin 	    NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP;
851f439e3a4SAlexander Motin 	if (ctrlr->cdata.ver >= NVME_REV(1, 2))
852881534f0SWarner Losh 		ctrlr->async_event_config |= NVME_ASYNC_EVENT_NS_ATTRIBUTE |
853881534f0SWarner Losh 		    NVME_ASYNC_EVENT_FW_ACTIVATE;
854d5fc9821SJim Harris 
85529077eb4SWarner Losh 	status.done = 0;
856d5fc9821SJim Harris 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
857d5fc9821SJim Harris 	    0, NULL, 0, nvme_completion_poll_cb, &status);
858ab0681aaSWarner Losh 	nvme_completion_poll(&status);
859d5fc9821SJim Harris 	if (nvme_completion_is_error(&status.cpl) ||
860d5fc9821SJim Harris 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
861d5fc9821SJim Harris 	    (status.cpl.cdw0 & 0xFFFF) == 0x0000) {
862d5fc9821SJim Harris 		nvme_printf(ctrlr, "temperature threshold not supported\n");
863f439e3a4SAlexander Motin 	} else
864f439e3a4SAlexander Motin 		ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;
865d5fc9821SJim Harris 
866bb2f67fdSJim Harris 	nvme_ctrlr_cmd_set_async_event_config(ctrlr,
867bb2f67fdSJim Harris 	    ctrlr->async_event_config, NULL, NULL);
868bb0ec6b3SJim Harris 
869bb0ec6b3SJim Harris 	/* aerl is a zero-based value, so we need to add 1 here. */
8700a0b08ccSJim Harris 	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
871bb0ec6b3SJim Harris 
8720a0b08ccSJim Harris 	for (i = 0; i < ctrlr->num_aers; i++) {
8730a0b08ccSJim Harris 		aer = &ctrlr->aer[i];
8740a0b08ccSJim Harris 		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
8750a0b08ccSJim Harris 	}
876bb0ec6b3SJim Harris }
877bb0ec6b3SJim Harris 
878bb0ec6b3SJim Harris static void
879bb0ec6b3SJim Harris nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
880bb0ec6b3SJim Harris {
881bb0ec6b3SJim Harris 
882bb0ec6b3SJim Harris 	ctrlr->int_coal_time = 0;
883bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
884bb0ec6b3SJim Harris 	    &ctrlr->int_coal_time);
885bb0ec6b3SJim Harris 
886bb0ec6b3SJim Harris 	ctrlr->int_coal_threshold = 0;
887bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
888bb0ec6b3SJim Harris 	    &ctrlr->int_coal_threshold);
889bb0ec6b3SJim Harris 
890bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
891bb0ec6b3SJim Harris 	    ctrlr->int_coal_threshold, NULL, NULL);
892bb0ec6b3SJim Harris }
893bb0ec6b3SJim Harris 
894be34f216SJim Harris static void
89567abaee9SAlexander Motin nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)
89667abaee9SAlexander Motin {
89767abaee9SAlexander Motin 	struct nvme_hmb_chunk *hmbc;
89867abaee9SAlexander Motin 	int i;
89967abaee9SAlexander Motin 
90067abaee9SAlexander Motin 	if (ctrlr->hmb_desc_paddr) {
90167abaee9SAlexander Motin 		bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);
90267abaee9SAlexander Motin 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
90367abaee9SAlexander Motin 		    ctrlr->hmb_desc_map);
90467abaee9SAlexander Motin 		ctrlr->hmb_desc_paddr = 0;
90567abaee9SAlexander Motin 	}
90667abaee9SAlexander Motin 	if (ctrlr->hmb_desc_tag) {
90767abaee9SAlexander Motin 		bus_dma_tag_destroy(ctrlr->hmb_desc_tag);
908b2cdfb72SAlexander Motin 		ctrlr->hmb_desc_tag = NULL;
90967abaee9SAlexander Motin 	}
91067abaee9SAlexander Motin 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {
91167abaee9SAlexander Motin 		hmbc = &ctrlr->hmb_chunks[i];
91267abaee9SAlexander Motin 		bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);
91367abaee9SAlexander Motin 		bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
91467abaee9SAlexander Motin 		    hmbc->hmbc_map);
91567abaee9SAlexander Motin 	}
91667abaee9SAlexander Motin 	ctrlr->hmb_nchunks = 0;
91767abaee9SAlexander Motin 	if (ctrlr->hmb_tag) {
91867abaee9SAlexander Motin 		bus_dma_tag_destroy(ctrlr->hmb_tag);
91967abaee9SAlexander Motin 		ctrlr->hmb_tag = NULL;
92067abaee9SAlexander Motin 	}
92167abaee9SAlexander Motin 	if (ctrlr->hmb_chunks) {
92267abaee9SAlexander Motin 		free(ctrlr->hmb_chunks, M_NVME);
92367abaee9SAlexander Motin 		ctrlr->hmb_chunks = NULL;
92467abaee9SAlexander Motin 	}
92567abaee9SAlexander Motin }
92667abaee9SAlexander Motin 
92767abaee9SAlexander Motin static void
92867abaee9SAlexander Motin nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)
92967abaee9SAlexander Motin {
93067abaee9SAlexander Motin 	struct nvme_hmb_chunk *hmbc;
93167abaee9SAlexander Motin 	size_t pref, min, minc, size;
93267abaee9SAlexander Motin 	int err, i;
93367abaee9SAlexander Motin 	uint64_t max;
93467abaee9SAlexander Motin 
9351c7dd40eSAlexander Motin 	/* Limit HMB to 5% of RAM size per device by default. */
9361c7dd40eSAlexander Motin 	max = (uint64_t)physmem * PAGE_SIZE / 20;
93767abaee9SAlexander Motin 	TUNABLE_UINT64_FETCH("hw.nvme.hmb_max", &max);
93867abaee9SAlexander Motin 
93967abaee9SAlexander Motin 	min = (long long unsigned)ctrlr->cdata.hmmin * 4096;
9406de4e458SAlexander Motin 	if (max == 0 || max < min)
94167abaee9SAlexander Motin 		return;
94267abaee9SAlexander Motin 	pref = MIN((long long unsigned)ctrlr->cdata.hmpre * 4096, max);
94367abaee9SAlexander Motin 	minc = MAX(ctrlr->cdata.hmminds * 4096, PAGE_SIZE);
94467abaee9SAlexander Motin 	if (min > 0 && ctrlr->cdata.hmmaxd > 0)
94567abaee9SAlexander Motin 		minc = MAX(minc, min / ctrlr->cdata.hmmaxd);
94667abaee9SAlexander Motin 	ctrlr->hmb_chunk = pref;
94767abaee9SAlexander Motin 
94867abaee9SAlexander Motin again:
94967abaee9SAlexander Motin 	ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, PAGE_SIZE);
95067abaee9SAlexander Motin 	ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);
95167abaee9SAlexander Motin 	if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)
95267abaee9SAlexander Motin 		ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;
95367abaee9SAlexander Motin 	ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *
95467abaee9SAlexander Motin 	    ctrlr->hmb_nchunks, M_NVME, M_WAITOK);
95567abaee9SAlexander Motin 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
95667abaee9SAlexander Motin 	    PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
95767abaee9SAlexander Motin 	    ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);
95867abaee9SAlexander Motin 	if (err != 0) {
95967abaee9SAlexander Motin 		nvme_printf(ctrlr, "HMB tag create failed %d\n", err);
96067abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
96167abaee9SAlexander Motin 		return;
96267abaee9SAlexander Motin 	}
96367abaee9SAlexander Motin 
96467abaee9SAlexander Motin 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {
96567abaee9SAlexander Motin 		hmbc = &ctrlr->hmb_chunks[i];
96667abaee9SAlexander Motin 		if (bus_dmamem_alloc(ctrlr->hmb_tag,
96767abaee9SAlexander Motin 		    (void **)&hmbc->hmbc_vaddr, BUS_DMA_NOWAIT,
96867abaee9SAlexander Motin 		    &hmbc->hmbc_map)) {
96967abaee9SAlexander Motin 			nvme_printf(ctrlr, "failed to alloc HMB\n");
97067abaee9SAlexander Motin 			break;
97167abaee9SAlexander Motin 		}
97267abaee9SAlexander Motin 		if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,
97367abaee9SAlexander Motin 		    hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,
97467abaee9SAlexander Motin 		    &hmbc->hmbc_paddr, BUS_DMA_NOWAIT) != 0) {
97567abaee9SAlexander Motin 			bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
97667abaee9SAlexander Motin 			    hmbc->hmbc_map);
97767abaee9SAlexander Motin 			nvme_printf(ctrlr, "failed to load HMB\n");
97867abaee9SAlexander Motin 			break;
97967abaee9SAlexander Motin 		}
98067abaee9SAlexander Motin 		bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,
98167abaee9SAlexander Motin 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
98267abaee9SAlexander Motin 	}
98367abaee9SAlexander Motin 
98467abaee9SAlexander Motin 	if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&
98567abaee9SAlexander Motin 	    ctrlr->hmb_chunk / 2 >= minc) {
98667abaee9SAlexander Motin 		ctrlr->hmb_nchunks = i;
98767abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
98867abaee9SAlexander Motin 		ctrlr->hmb_chunk /= 2;
98967abaee9SAlexander Motin 		goto again;
99067abaee9SAlexander Motin 	}
99167abaee9SAlexander Motin 	ctrlr->hmb_nchunks = i;
99267abaee9SAlexander Motin 	if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {
99367abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
99467abaee9SAlexander Motin 		return;
99567abaee9SAlexander Motin 	}
99667abaee9SAlexander Motin 
99767abaee9SAlexander Motin 	size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;
99867abaee9SAlexander Motin 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
99967abaee9SAlexander Motin 	    16, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
100067abaee9SAlexander Motin 	    size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);
100167abaee9SAlexander Motin 	if (err != 0) {
100267abaee9SAlexander Motin 		nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);
100367abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
100467abaee9SAlexander Motin 		return;
100567abaee9SAlexander Motin 	}
100667abaee9SAlexander Motin 	if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,
100767abaee9SAlexander Motin 	    (void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,
100867abaee9SAlexander Motin 	    &ctrlr->hmb_desc_map)) {
100967abaee9SAlexander Motin 		nvme_printf(ctrlr, "failed to alloc HMB desc\n");
101067abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
101167abaee9SAlexander Motin 		return;
101267abaee9SAlexander Motin 	}
101367abaee9SAlexander Motin 	if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
101467abaee9SAlexander Motin 	    ctrlr->hmb_desc_vaddr, size, nvme_single_map,
101567abaee9SAlexander Motin 	    &ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {
101667abaee9SAlexander Motin 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
101767abaee9SAlexander Motin 		    ctrlr->hmb_desc_map);
101867abaee9SAlexander Motin 		nvme_printf(ctrlr, "failed to load HMB desc\n");
101967abaee9SAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
102067abaee9SAlexander Motin 		return;
102167abaee9SAlexander Motin 	}
102267abaee9SAlexander Motin 
102367abaee9SAlexander Motin 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {
102467abaee9SAlexander Motin 		ctrlr->hmb_desc_vaddr[i].addr =
102567abaee9SAlexander Motin 		    htole64(ctrlr->hmb_chunks[i].hmbc_paddr);
102667abaee9SAlexander Motin 		ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / 4096);
102767abaee9SAlexander Motin 	}
102867abaee9SAlexander Motin 	bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
102967abaee9SAlexander Motin 	    BUS_DMASYNC_PREWRITE);
103067abaee9SAlexander Motin 
103167abaee9SAlexander Motin 	nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",
103267abaee9SAlexander Motin 	    (long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk
103367abaee9SAlexander Motin 	    / 1024 / 1024);
103467abaee9SAlexander Motin }
103567abaee9SAlexander Motin 
103667abaee9SAlexander Motin static void
103767abaee9SAlexander Motin nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)
103867abaee9SAlexander Motin {
103967abaee9SAlexander Motin 	struct nvme_completion_poll_status	status;
104067abaee9SAlexander Motin 	uint32_t cdw11;
104167abaee9SAlexander Motin 
104267abaee9SAlexander Motin 	cdw11 = 0;
104367abaee9SAlexander Motin 	if (enable)
104467abaee9SAlexander Motin 		cdw11 |= 1;
104567abaee9SAlexander Motin 	if (memret)
104667abaee9SAlexander Motin 		cdw11 |= 2;
104767abaee9SAlexander Motin 	status.done = 0;
104867abaee9SAlexander Motin 	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,
104967abaee9SAlexander Motin 	    ctrlr->hmb_nchunks * ctrlr->hmb_chunk / 4096, ctrlr->hmb_desc_paddr,
105067abaee9SAlexander Motin 	    ctrlr->hmb_desc_paddr >> 32, ctrlr->hmb_nchunks, NULL, 0,
105167abaee9SAlexander Motin 	    nvme_completion_poll_cb, &status);
105267abaee9SAlexander Motin 	nvme_completion_poll(&status);
105367abaee9SAlexander Motin 	if (nvme_completion_is_error(&status.cpl))
105467abaee9SAlexander Motin 		nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");
105567abaee9SAlexander Motin }
105667abaee9SAlexander Motin 
105767abaee9SAlexander Motin static void
10584d547561SWarner Losh nvme_ctrlr_start(void *ctrlr_arg, bool resetting)
1059bb0ec6b3SJim Harris {
1060bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = ctrlr_arg;
10612b647da7SJim Harris 	uint32_t old_num_io_queues;
1062b846efd7SJim Harris 	int i;
1063b846efd7SJim Harris 
1064bad42df9SColin Percival 	TSENTER();
1065bad42df9SColin Percival 
10662b647da7SJim Harris 	/*
10672b647da7SJim Harris 	 * Only reset adminq here when we are restarting the
10682b647da7SJim Harris 	 *  controller after a reset.  During initialization,
10692b647da7SJim Harris 	 *  we have already submitted admin commands to get
10702b647da7SJim Harris 	 *  the number of I/O queues supported, so cannot reset
10712b647da7SJim Harris 	 *  the adminq again here.
10722b647da7SJim Harris 	 */
1073ac90f70dSAlexander Motin 	if (resetting) {
1074cb5b7c13SJim Harris 		nvme_qpair_reset(&ctrlr->adminq);
1075ac90f70dSAlexander Motin 		nvme_admin_qpair_enable(&ctrlr->adminq);
1076ac90f70dSAlexander Motin 	}
10772b647da7SJim Harris 
1078701267adSAlexander Motin 	if (ctrlr->ioq != NULL) {
1079cb5b7c13SJim Harris 		for (i = 0; i < ctrlr->num_io_queues; i++)
1080cb5b7c13SJim Harris 			nvme_qpair_reset(&ctrlr->ioq[i]);
1081701267adSAlexander Motin 	}
1082cb5b7c13SJim Harris 
1083701267adSAlexander Motin 	/*
1084701267adSAlexander Motin 	 * If it was a reset on initialization command timeout, just
1085701267adSAlexander Motin 	 * return here, letting initialization code fail gracefully.
1086701267adSAlexander Motin 	 */
1087701267adSAlexander Motin 	if (resetting && !ctrlr->is_initialized)
1088701267adSAlexander Motin 		return;
1089701267adSAlexander Motin 
1090ac90f70dSAlexander Motin 	if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {
1091232e2edbSJim Harris 		nvme_ctrlr_fail(ctrlr);
1092be34f216SJim Harris 		return;
1093232e2edbSJim Harris 	}
1094bb0ec6b3SJim Harris 
10952b647da7SJim Harris 	/*
10962b647da7SJim Harris 	 * The number of qpairs are determined during controller initialization,
10972b647da7SJim Harris 	 *  including using NVMe SET_FEATURES/NUMBER_OF_QUEUES to determine the
10982b647da7SJim Harris 	 *  HW limit.  We call SET_FEATURES again here so that it gets called
10992b647da7SJim Harris 	 *  after any reset for controllers that depend on the driver to
11002b647da7SJim Harris 	 *  explicit specify how many queues it will use.  This value should
11012b647da7SJim Harris 	 *  never change between resets, so panic if somehow that does happen.
11022b647da7SJim Harris 	 */
11034d547561SWarner Losh 	if (resetting) {
11042b647da7SJim Harris 		old_num_io_queues = ctrlr->num_io_queues;
1105232e2edbSJim Harris 		if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
1106232e2edbSJim Harris 			nvme_ctrlr_fail(ctrlr);
1107be34f216SJim Harris 			return;
1108232e2edbSJim Harris 		}
1109bb0ec6b3SJim Harris 
11102b647da7SJim Harris 		if (old_num_io_queues != ctrlr->num_io_queues) {
11117b036d77SJim Harris 			panic("num_io_queues changed from %u to %u",
11127b036d77SJim Harris 			      old_num_io_queues, ctrlr->num_io_queues);
11137b036d77SJim Harris 		}
11142b647da7SJim Harris 	}
11152b647da7SJim Harris 
111667abaee9SAlexander Motin 	if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {
111767abaee9SAlexander Motin 		nvme_ctrlr_hmb_alloc(ctrlr);
111867abaee9SAlexander Motin 		if (ctrlr->hmb_nchunks > 0)
111967abaee9SAlexander Motin 			nvme_ctrlr_hmb_enable(ctrlr, true, false);
112067abaee9SAlexander Motin 	} else if (ctrlr->hmb_nchunks > 0)
112167abaee9SAlexander Motin 		nvme_ctrlr_hmb_enable(ctrlr, true, true);
112267abaee9SAlexander Motin 
1123232e2edbSJim Harris 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
1124232e2edbSJim Harris 		nvme_ctrlr_fail(ctrlr);
1125be34f216SJim Harris 		return;
1126232e2edbSJim Harris 	}
1127bb0ec6b3SJim Harris 
1128232e2edbSJim Harris 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
1129232e2edbSJim Harris 		nvme_ctrlr_fail(ctrlr);
1130be34f216SJim Harris 		return;
1131232e2edbSJim Harris 	}
1132bb0ec6b3SJim Harris 
1133bb0ec6b3SJim Harris 	nvme_ctrlr_configure_aer(ctrlr);
1134bb0ec6b3SJim Harris 	nvme_ctrlr_configure_int_coalescing(ctrlr);
1135bb0ec6b3SJim Harris 
1136b846efd7SJim Harris 	for (i = 0; i < ctrlr->num_io_queues; i++)
1137b846efd7SJim Harris 		nvme_io_qpair_enable(&ctrlr->ioq[i]);
1138bad42df9SColin Percival 	TSEXIT();
1139bb0ec6b3SJim Harris }
1140bb0ec6b3SJim Harris 
1141be34f216SJim Harris void
1142be34f216SJim Harris nvme_ctrlr_start_config_hook(void *arg)
1143be34f216SJim Harris {
1144be34f216SJim Harris 	struct nvme_controller *ctrlr = arg;
114566e59850SWarner Losh 
1146bad42df9SColin Percival 	TSENTER();
1147bad42df9SColin Percival 
1148701267adSAlexander Motin 	if (nvme_ctrlr_hw_reset(ctrlr) != 0) {
1149701267adSAlexander Motin fail:
115066e59850SWarner Losh 		nvme_ctrlr_fail(ctrlr);
115192390644SAlexander Motin 		config_intrhook_disestablish(&ctrlr->config_hook);
115266e59850SWarner Losh 		return;
115366e59850SWarner Losh 	}
115466e59850SWarner Losh 
11554b3da659SWarner Losh #ifdef NVME_2X_RESET
11564b3da659SWarner Losh 	/*
11574b3da659SWarner Losh 	 * Reset controller twice to ensure we do a transition from cc.en==1 to
11584b3da659SWarner Losh 	 * cc.en==0.  This is because we don't really know what status the
11594b3da659SWarner Losh 	 * controller was left in when boot handed off to OS.  Linux doesn't do
11604b3da659SWarner Losh 	 * this, however, and when the controller is in state cc.en == 0, no
11614b3da659SWarner Losh 	 * I/O can happen.
11624b3da659SWarner Losh 	 */
1163701267adSAlexander Motin 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)
1164701267adSAlexander Motin 		goto fail;
11654b3da659SWarner Losh #endif
1166be34f216SJim Harris 
11672b647da7SJim Harris 	nvme_qpair_reset(&ctrlr->adminq);
11682b647da7SJim Harris 	nvme_admin_qpair_enable(&ctrlr->adminq);
11692b647da7SJim Harris 
1170ac90f70dSAlexander Motin 	if (nvme_ctrlr_identify(ctrlr) == 0 &&
1171ac90f70dSAlexander Motin 	    nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&
11722b647da7SJim Harris 	    nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)
11734d547561SWarner Losh 		nvme_ctrlr_start(ctrlr, false);
11742b647da7SJim Harris 	else
1175701267adSAlexander Motin 		goto fail;
11762b647da7SJim Harris 
11772b647da7SJim Harris 	nvme_sysctl_initialize_ctrlr(ctrlr);
1178be34f216SJim Harris 	config_intrhook_disestablish(&ctrlr->config_hook);
1179496a2752SJim Harris 
1180496a2752SJim Harris 	ctrlr->is_initialized = 1;
1181496a2752SJim Harris 	nvme_notify_new_controller(ctrlr);
1182bad42df9SColin Percival 	TSEXIT();
1183b846efd7SJim Harris }
1184b846efd7SJim Harris 
1185bb0ec6b3SJim Harris static void
118648ce3178SJim Harris nvme_ctrlr_reset_task(void *arg, int pending)
118712d191ecSJim Harris {
118812d191ecSJim Harris 	struct nvme_controller	*ctrlr = arg;
118948ce3178SJim Harris 	int			status;
119012d191ecSJim Harris 
1191244b8053SWarner Losh 	nvme_ctrlr_devctl_log(ctrlr, "RESET", "resetting controller");
119248ce3178SJim Harris 	status = nvme_ctrlr_hw_reset(ctrlr);
119348ce3178SJim Harris 	/*
119448ce3178SJim Harris 	 * Use pause instead of DELAY, so that we yield to any nvme interrupt
119548ce3178SJim Harris 	 *  handlers on this CPU that were blocked on a qpair lock. We want
119648ce3178SJim Harris 	 *  all nvme interrupts completed before proceeding with restarting the
119748ce3178SJim Harris 	 *  controller.
119848ce3178SJim Harris 	 *
119948ce3178SJim Harris 	 * XXX - any way to guarantee the interrupt handlers have quiesced?
120048ce3178SJim Harris 	 */
120148ce3178SJim Harris 	pause("nvmereset", hz / 10);
120248ce3178SJim Harris 	if (status == 0)
12034d547561SWarner Losh 		nvme_ctrlr_start(ctrlr, true);
1204232e2edbSJim Harris 	else
1205232e2edbSJim Harris 		nvme_ctrlr_fail(ctrlr);
1206f37c22a3SJim Harris 
1207f37c22a3SJim Harris 	atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
120812d191ecSJim Harris }
120912d191ecSJim Harris 
1210bb1c7be4SWarner Losh /*
1211bb1c7be4SWarner Losh  * Poll all the queues enabled on the device for completion.
1212bb1c7be4SWarner Losh  */
1213bb1c7be4SWarner Losh void
1214bb1c7be4SWarner Losh nvme_ctrlr_poll(struct nvme_controller *ctrlr)
1215bb1c7be4SWarner Losh {
1216bb1c7be4SWarner Losh 	int i;
1217bb1c7be4SWarner Losh 
1218bb1c7be4SWarner Losh 	nvme_qpair_process_completions(&ctrlr->adminq);
1219bb1c7be4SWarner Losh 
1220bb1c7be4SWarner Losh 	for (i = 0; i < ctrlr->num_io_queues; i++)
1221bb1c7be4SWarner Losh 		if (ctrlr->ioq && ctrlr->ioq[i].cpl)
1222bb1c7be4SWarner Losh 			nvme_qpair_process_completions(&ctrlr->ioq[i]);
1223bb1c7be4SWarner Losh }
1224bb1c7be4SWarner Losh 
1225bb1c7be4SWarner Losh /*
12264d547561SWarner Losh  * Poll the single-vector interrupt case: num_io_queues will be 1 and
1227bb1c7be4SWarner Losh  * there's only a single vector. While we're polling, we mask further
1228bb1c7be4SWarner Losh  * interrupts in the controller.
1229bb1c7be4SWarner Losh  */
1230f24c011bSWarner Losh void
1231e3bdf3daSAlexander Motin nvme_ctrlr_shared_handler(void *arg)
1232bb0ec6b3SJim Harris {
1233bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = arg;
1234bb0ec6b3SJim Harris 
12354d6abcb1SJim Harris 	nvme_mmio_write_4(ctrlr, intms, 1);
1236bb1c7be4SWarner Losh 	nvme_ctrlr_poll(ctrlr);
1237bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, intmc, 1);
1238bb0ec6b3SJim Harris }
1239bb0ec6b3SJim Harris 
12407c3f19d7SJim Harris static void
12417c3f19d7SJim Harris nvme_pt_done(void *arg, const struct nvme_completion *cpl)
12427c3f19d7SJim Harris {
12437c3f19d7SJim Harris 	struct nvme_pt_command *pt = arg;
1244c252f637SAlexander Motin 	struct mtx *mtx = pt->driver_lock;
12450d787e9bSWojciech Macek 	uint16_t status;
12467c3f19d7SJim Harris 
12477c3f19d7SJim Harris 	bzero(&pt->cpl, sizeof(pt->cpl));
12487c3f19d7SJim Harris 	pt->cpl.cdw0 = cpl->cdw0;
12490d787e9bSWojciech Macek 
12500d787e9bSWojciech Macek 	status = cpl->status;
12510d787e9bSWojciech Macek 	status &= ~NVME_STATUS_P_MASK;
12520d787e9bSWojciech Macek 	pt->cpl.status = status;
12537c3f19d7SJim Harris 
1254c252f637SAlexander Motin 	mtx_lock(mtx);
1255c252f637SAlexander Motin 	pt->driver_lock = NULL;
12567c3f19d7SJim Harris 	wakeup(pt);
1257c252f637SAlexander Motin 	mtx_unlock(mtx);
12587c3f19d7SJim Harris }
12597c3f19d7SJim Harris 
12607c3f19d7SJim Harris int
12617c3f19d7SJim Harris nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
12627c3f19d7SJim Harris     struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
12637c3f19d7SJim Harris     int is_admin_cmd)
12647c3f19d7SJim Harris {
12657c3f19d7SJim Harris 	struct nvme_request	*req;
12667c3f19d7SJim Harris 	struct mtx		*mtx;
12677c3f19d7SJim Harris 	struct buf		*buf = NULL;
12687c3f19d7SJim Harris 	int			ret = 0;
12697c3f19d7SJim Harris 
12707b68ae1eSJim Harris 	if (pt->len > 0) {
12717b68ae1eSJim Harris 		if (pt->len > ctrlr->max_xfer_size) {
12727b68ae1eSJim Harris 			nvme_printf(ctrlr, "pt->len (%d) "
12737b68ae1eSJim Harris 			    "exceeds max_xfer_size (%d)\n", pt->len,
12747b68ae1eSJim Harris 			    ctrlr->max_xfer_size);
12757b68ae1eSJim Harris 			return EIO;
12767b68ae1eSJim Harris 		}
12777c3f19d7SJim Harris 		if (is_user_buffer) {
12787c3f19d7SJim Harris 			/*
12797c3f19d7SJim Harris 			 * Ensure the user buffer is wired for the duration of
12804d547561SWarner Losh 			 *  this pass-through command.
12817c3f19d7SJim Harris 			 */
12827c3f19d7SJim Harris 			PHOLD(curproc);
1283756a5412SGleb Smirnoff 			buf = uma_zalloc(pbuf_zone, M_WAITOK);
12847c3f19d7SJim Harris 			buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
128544ca4575SBrooks Davis 			if (vmapbuf(buf, pt->buf, pt->len, 1) < 0) {
12867c3f19d7SJim Harris 				ret = EFAULT;
12877c3f19d7SJim Harris 				goto err;
12887c3f19d7SJim Harris 			}
12897c3f19d7SJim Harris 			req = nvme_allocate_request_vaddr(buf->b_data, pt->len,
12907c3f19d7SJim Harris 			    nvme_pt_done, pt);
12917c3f19d7SJim Harris 		} else
12927c3f19d7SJim Harris 			req = nvme_allocate_request_vaddr(pt->buf, pt->len,
12937c3f19d7SJim Harris 			    nvme_pt_done, pt);
12947b68ae1eSJim Harris 	} else
12957c3f19d7SJim Harris 		req = nvme_allocate_request_null(nvme_pt_done, pt);
12967c3f19d7SJim Harris 
12970d787e9bSWojciech Macek 	/* Assume user space already converted to little-endian */
12989544e6dcSChuck Tuffli 	req->cmd.opc = pt->cmd.opc;
12999544e6dcSChuck Tuffli 	req->cmd.fuse = pt->cmd.fuse;
130091182bcfSWarner Losh 	req->cmd.rsvd2 = pt->cmd.rsvd2;
130191182bcfSWarner Losh 	req->cmd.rsvd3 = pt->cmd.rsvd3;
13027c3f19d7SJim Harris 	req->cmd.cdw10 = pt->cmd.cdw10;
13037c3f19d7SJim Harris 	req->cmd.cdw11 = pt->cmd.cdw11;
13047c3f19d7SJim Harris 	req->cmd.cdw12 = pt->cmd.cdw12;
13057c3f19d7SJim Harris 	req->cmd.cdw13 = pt->cmd.cdw13;
13067c3f19d7SJim Harris 	req->cmd.cdw14 = pt->cmd.cdw14;
13077c3f19d7SJim Harris 	req->cmd.cdw15 = pt->cmd.cdw15;
13087c3f19d7SJim Harris 
13090d787e9bSWojciech Macek 	req->cmd.nsid = htole32(nsid);
13107c3f19d7SJim Harris 
1311c252f637SAlexander Motin 	mtx = mtx_pool_find(mtxpool_sleep, pt);
13127c3f19d7SJim Harris 	pt->driver_lock = mtx;
13137c3f19d7SJim Harris 
13147c3f19d7SJim Harris 	if (is_admin_cmd)
13157c3f19d7SJim Harris 		nvme_ctrlr_submit_admin_request(ctrlr, req);
13167c3f19d7SJim Harris 	else
13177c3f19d7SJim Harris 		nvme_ctrlr_submit_io_request(ctrlr, req);
13187c3f19d7SJim Harris 
1319c252f637SAlexander Motin 	mtx_lock(mtx);
1320c252f637SAlexander Motin 	while (pt->driver_lock != NULL)
13217c3f19d7SJim Harris 		mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
13227c3f19d7SJim Harris 	mtx_unlock(mtx);
13237c3f19d7SJim Harris 
13247c3f19d7SJim Harris err:
13257c3f19d7SJim Harris 	if (buf != NULL) {
1326756a5412SGleb Smirnoff 		uma_zfree(pbuf_zone, buf);
13277c3f19d7SJim Harris 		PRELE(curproc);
13287c3f19d7SJim Harris 	}
13297c3f19d7SJim Harris 
13307c3f19d7SJim Harris 	return (ret);
13317c3f19d7SJim Harris }
13327c3f19d7SJim Harris 
1333bb0ec6b3SJim Harris static int
1334bb0ec6b3SJim Harris nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
1335bb0ec6b3SJim Harris     struct thread *td)
1336bb0ec6b3SJim Harris {
1337bb0ec6b3SJim Harris 	struct nvme_controller			*ctrlr;
13387c3f19d7SJim Harris 	struct nvme_pt_command			*pt;
1339bb0ec6b3SJim Harris 
1340bb0ec6b3SJim Harris 	ctrlr = cdev->si_drv1;
1341bb0ec6b3SJim Harris 
1342bb0ec6b3SJim Harris 	switch (cmd) {
1343b846efd7SJim Harris 	case NVME_RESET_CONTROLLER:
1344b846efd7SJim Harris 		nvme_ctrlr_reset(ctrlr);
1345b846efd7SJim Harris 		break;
13467c3f19d7SJim Harris 	case NVME_PASSTHROUGH_CMD:
13477c3f19d7SJim Harris 		pt = (struct nvme_pt_command *)arg;
13480d787e9bSWojciech Macek 		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),
13497c3f19d7SJim Harris 		    1 /* is_user_buffer */, 1 /* is_admin_cmd */));
1350a7bf63beSAlexander Motin 	case NVME_GET_NSID:
1351a7bf63beSAlexander Motin 	{
1352a7bf63beSAlexander Motin 		struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg;
1353a7bf63beSAlexander Motin 		strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),
1354a7bf63beSAlexander Motin 		    sizeof(gnsid->cdev));
13554053f8acSDavid Bright 		gnsid->cdev[sizeof(gnsid->cdev) - 1] = '\0';
1356a7bf63beSAlexander Motin 		gnsid->nsid = 0;
1357a7bf63beSAlexander Motin 		break;
1358a7bf63beSAlexander Motin 	}
1359e32d47f3SDavid Bright 	case NVME_GET_MAX_XFER_SIZE:
1360e32d47f3SDavid Bright 		*(uint64_t *)arg = ctrlr->max_xfer_size;
1361e32d47f3SDavid Bright 		break;
1362bb0ec6b3SJim Harris 	default:
1363bb0ec6b3SJim Harris 		return (ENOTTY);
1364bb0ec6b3SJim Harris 	}
1365bb0ec6b3SJim Harris 
1366bb0ec6b3SJim Harris 	return (0);
1367bb0ec6b3SJim Harris }
1368bb0ec6b3SJim Harris 
1369bb0ec6b3SJim Harris static struct cdevsw nvme_ctrlr_cdevsw = {
1370bb0ec6b3SJim Harris 	.d_version =	D_VERSION,
1371bb0ec6b3SJim Harris 	.d_flags =	0,
1372bb0ec6b3SJim Harris 	.d_ioctl =	nvme_ctrlr_ioctl
1373bb0ec6b3SJim Harris };
1374bb0ec6b3SJim Harris 
1375bb0ec6b3SJim Harris int
1376bb0ec6b3SJim Harris nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
1377bb0ec6b3SJim Harris {
1378e134ecdcSAlexander Motin 	struct make_dev_args	md_args;
13790d787e9bSWojciech Macek 	uint32_t	cap_lo;
13800d787e9bSWojciech Macek 	uint32_t	cap_hi;
13810bed3eabSAlexander Motin 	uint32_t	to, vs, pmrcap;
13820d787e9bSWojciech Macek 	uint8_t		mpsmin;
1383f42ca756SJim Harris 	int		status, timeout_period;
1384bb0ec6b3SJim Harris 
1385bb0ec6b3SJim Harris 	ctrlr->dev = dev;
1386bb0ec6b3SJim Harris 
1387a90b8104SJim Harris 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
13881eab19cbSAlexander Motin 	if (bus_get_domain(dev, &ctrlr->domain) != 0)
13891eab19cbSAlexander Motin 		ctrlr->domain = 0;
1390a90b8104SJim Harris 
1391*6af6a52eSWarner Losh 	ctrlr->cap_lo = cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
1392c44441f8SAlexander Motin 	if (bootverbose) {
1393c44441f8SAlexander Motin 		device_printf(dev, "CapLo: 0x%08x: MQES %u%s%s%s%s, TO %u\n",
1394c44441f8SAlexander Motin 		    cap_lo, NVME_CAP_LO_MQES(cap_lo),
1395c44441f8SAlexander Motin 		    NVME_CAP_LO_CQR(cap_lo) ? ", CQR" : "",
1396c44441f8SAlexander Motin 		    NVME_CAP_LO_AMS(cap_lo) ? ", AMS" : "",
1397c44441f8SAlexander Motin 		    (NVME_CAP_LO_AMS(cap_lo) & 0x1) ? " WRRwUPC" : "",
1398c44441f8SAlexander Motin 		    (NVME_CAP_LO_AMS(cap_lo) & 0x2) ? " VS" : "",
1399c44441f8SAlexander Motin 		    NVME_CAP_LO_TO(cap_lo));
1400c44441f8SAlexander Motin 	}
1401*6af6a52eSWarner Losh 	ctrlr->cap_hi = cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
1402c44441f8SAlexander Motin 	if (bootverbose) {
1403c44441f8SAlexander Motin 		device_printf(dev, "CapHi: 0x%08x: DSTRD %u%s, CSS %x%s, "
1404c44441f8SAlexander Motin 		    "MPSMIN %u, MPSMAX %u%s%s\n", cap_hi,
1405c44441f8SAlexander Motin 		    NVME_CAP_HI_DSTRD(cap_hi),
14060bed3eabSAlexander Motin 		    NVME_CAP_HI_NSSRS(cap_hi) ? ", NSSRS" : "",
1407c44441f8SAlexander Motin 		    NVME_CAP_HI_CSS(cap_hi),
14080bed3eabSAlexander Motin 		    NVME_CAP_HI_BPS(cap_hi) ? ", BPS" : "",
1409c44441f8SAlexander Motin 		    NVME_CAP_HI_MPSMIN(cap_hi),
1410c44441f8SAlexander Motin 		    NVME_CAP_HI_MPSMAX(cap_hi),
14110bed3eabSAlexander Motin 		    NVME_CAP_HI_PMRS(cap_hi) ? ", PMRS" : "",
14120bed3eabSAlexander Motin 		    NVME_CAP_HI_CMBS(cap_hi) ? ", CMBS" : "");
1413c44441f8SAlexander Motin 	}
1414c44441f8SAlexander Motin 	if (bootverbose) {
1415c44441f8SAlexander Motin 		vs = nvme_mmio_read_4(ctrlr, vs);
1416c44441f8SAlexander Motin 		device_printf(dev, "Version: 0x%08x: %d.%d\n", vs,
1417c44441f8SAlexander Motin 		    NVME_MAJOR(vs), NVME_MINOR(vs));
1418c44441f8SAlexander Motin 	}
14190bed3eabSAlexander Motin 	if (bootverbose && NVME_CAP_HI_PMRS(cap_hi)) {
14200bed3eabSAlexander Motin 		pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);
14210bed3eabSAlexander Motin 		device_printf(dev, "PMRCap: 0x%08x: BIR %u%s%s, PMRTU %u, "
14220bed3eabSAlexander Motin 		    "PMRWBM %x, PMRTO %u%s\n", pmrcap,
14230bed3eabSAlexander Motin 		    NVME_PMRCAP_BIR(pmrcap),
14240bed3eabSAlexander Motin 		    NVME_PMRCAP_RDS(pmrcap) ? ", RDS" : "",
14250bed3eabSAlexander Motin 		    NVME_PMRCAP_WDS(pmrcap) ? ", WDS" : "",
14260bed3eabSAlexander Motin 		    NVME_PMRCAP_PMRTU(pmrcap),
14270bed3eabSAlexander Motin 		    NVME_PMRCAP_PMRWBM(pmrcap),
14280bed3eabSAlexander Motin 		    NVME_PMRCAP_PMRTO(pmrcap),
14290bed3eabSAlexander Motin 		    NVME_PMRCAP_CMSS(pmrcap) ? ", CMSS" : "");
14300bed3eabSAlexander Motin 	}
1431c44441f8SAlexander Motin 
1432f93b7f95SWarner Losh 	ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
1433bb0ec6b3SJim Harris 
143462d2cf18SWarner Losh 	mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
14350d787e9bSWojciech Macek 	ctrlr->min_page_size = 1 << (12 + mpsmin);
143602e33484SJim Harris 
1437bb0ec6b3SJim Harris 	/* Get ready timeout value from controller, in units of 500ms. */
143862d2cf18SWarner Losh 	to = NVME_CAP_LO_TO(cap_lo) + 1;
14390d787e9bSWojciech Macek 	ctrlr->ready_timeout_in_ms = to * 500;
1440bb0ec6b3SJim Harris 
144194143332SJim Harris 	timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
144294143332SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
144394143332SJim Harris 	timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
144494143332SJim Harris 	timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
144594143332SJim Harris 	ctrlr->timeout_period = timeout_period;
144694143332SJim Harris 
1447cb5b7c13SJim Harris 	nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
1448cb5b7c13SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
1449cb5b7c13SJim Harris 
145048ce3178SJim Harris 	ctrlr->enable_aborts = 0;
145148ce3178SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
145248ce3178SJim Harris 
14538d09e3c4SJim Harris 	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
1454a965389bSScott Long 	if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)
1455a965389bSScott Long 		return (ENXIO);
1456bb0ec6b3SJim Harris 
1457f0f47121SWarner Losh 	/*
1458f0f47121SWarner Losh 	 * Create 2 threads for the taskqueue. The reset thread will block when
1459f0f47121SWarner Losh 	 * it detects that the controller has failed until all I/O has been
1460f0f47121SWarner Losh 	 * failed up the stack. The fail_req task needs to be able to run in
1461f0f47121SWarner Losh 	 * this case to finish the request failure for some cases.
1462f0f47121SWarner Losh 	 *
1463f0f47121SWarner Losh 	 * We could partially solve this race by draining the failed requeust
1464f0f47121SWarner Losh 	 * queue before proceding to free the sim, though nothing would stop
1465f0f47121SWarner Losh 	 * new I/O from coming in after we do that drain, but before we reach
1466f0f47121SWarner Losh 	 * cam_sim_free, so this big hammer is used instead.
1467f0f47121SWarner Losh 	 */
146812d191ecSJim Harris 	ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
146912d191ecSJim Harris 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);
1470f0f47121SWarner Losh 	taskqueue_start_threads(&ctrlr->taskqueue, 2, PI_DISK, "nvme taskq");
147112d191ecSJim Harris 
1472f37c22a3SJim Harris 	ctrlr->is_resetting = 0;
1473496a2752SJim Harris 	ctrlr->is_initialized = 0;
1474496a2752SJim Harris 	ctrlr->notification_sent = 0;
1475232e2edbSJim Harris 	TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1476232e2edbSJim Harris 	TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
1477232e2edbSJim Harris 	STAILQ_INIT(&ctrlr->fail_req);
14787588c6ccSWarner Losh 	ctrlr->is_failed = false;
1479f37c22a3SJim Harris 
1480e134ecdcSAlexander Motin 	make_dev_args_init(&md_args);
1481e134ecdcSAlexander Motin 	md_args.mda_devsw = &nvme_ctrlr_cdevsw;
1482e134ecdcSAlexander Motin 	md_args.mda_uid = UID_ROOT;
1483e134ecdcSAlexander Motin 	md_args.mda_gid = GID_WHEEL;
1484e134ecdcSAlexander Motin 	md_args.mda_mode = 0600;
1485e134ecdcSAlexander Motin 	md_args.mda_unit = device_get_unit(dev);
1486e134ecdcSAlexander Motin 	md_args.mda_si_drv1 = (void *)ctrlr;
1487e134ecdcSAlexander Motin 	status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",
1488e134ecdcSAlexander Motin 	    device_get_unit(dev));
1489e134ecdcSAlexander Motin 	if (status != 0)
1490e134ecdcSAlexander Motin 		return (ENXIO);
1491e134ecdcSAlexander Motin 
1492bb0ec6b3SJim Harris 	return (0);
1493bb0ec6b3SJim Harris }
1494d281e8fbSJim Harris 
1495d281e8fbSJim Harris void
1496990e741cSJim Harris nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1497990e741cSJim Harris {
149871a28181SAlexander Motin 	int	gone, i;
1499990e741cSJim Harris 
1500502dc84aSWarner Losh 	ctrlr->is_dying = true;
1501502dc84aSWarner Losh 
1502e134ecdcSAlexander Motin 	if (ctrlr->resource == NULL)
1503e134ecdcSAlexander Motin 		goto nores;
150431111372SAlexander Motin 	if (!mtx_initialized(&ctrlr->adminq.lock))
150531111372SAlexander Motin 		goto noadminq;
150612d191ecSJim Harris 
150771a28181SAlexander Motin 	/*
150871a28181SAlexander Motin 	 * Check whether it is a hot unplug or a clean driver detach.
150971a28181SAlexander Motin 	 * If device is not there any more, skip any shutdown commands.
151071a28181SAlexander Motin 	 */
15119600aa31SWarner Losh 	gone = (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);
151271a28181SAlexander Motin 	if (gone)
151371a28181SAlexander Motin 		nvme_ctrlr_fail(ctrlr);
151471a28181SAlexander Motin 	else
1515f439e3a4SAlexander Motin 		nvme_notify_fail_consumers(ctrlr);
1516f439e3a4SAlexander Motin 
1517b846efd7SJim Harris 	for (i = 0; i < NVME_MAX_NAMESPACES; i++)
1518b846efd7SJim Harris 		nvme_ns_destruct(&ctrlr->ns[i]);
1519990e741cSJim Harris 
1520990e741cSJim Harris 	if (ctrlr->cdev)
1521990e741cSJim Harris 		destroy_dev(ctrlr->cdev);
1522990e741cSJim Harris 
15238e61280bSWarner Losh 	if (ctrlr->is_initialized) {
152467abaee9SAlexander Motin 		if (!gone) {
152567abaee9SAlexander Motin 			if (ctrlr->hmb_nchunks > 0)
152667abaee9SAlexander Motin 				nvme_ctrlr_hmb_enable(ctrlr, false, false);
15274d547561SWarner Losh 			nvme_ctrlr_delete_qpairs(ctrlr);
152867abaee9SAlexander Motin 		}
1529701267adSAlexander Motin 		nvme_ctrlr_hmb_free(ctrlr);
1530701267adSAlexander Motin 	}
1531701267adSAlexander Motin 	if (ctrlr->ioq != NULL) {
153271a28181SAlexander Motin 		for (i = 0; i < ctrlr->num_io_queues; i++)
1533990e741cSJim Harris 			nvme_io_qpair_destroy(&ctrlr->ioq[i]);
1534990e741cSJim Harris 		free(ctrlr->ioq, M_NVME);
15358e61280bSWarner Losh 	}
1536550d5d64SAlexander Motin 	nvme_admin_qpair_destroy(&ctrlr->adminq);
1537990e741cSJim Harris 
1538e134ecdcSAlexander Motin 	/*
1539e134ecdcSAlexander Motin 	 *  Notify the controller of a shutdown, even though this is due to
1540e134ecdcSAlexander Motin 	 *   a driver unload, not a system shutdown (this path is not invoked
1541e134ecdcSAlexander Motin 	 *   during shutdown).  This ensures the controller receives a
1542e134ecdcSAlexander Motin 	 *   shutdown notification in case the system is shutdown before
1543e134ecdcSAlexander Motin 	 *   reloading the driver.
1544e134ecdcSAlexander Motin 	 */
154571a28181SAlexander Motin 	if (!gone)
1546e134ecdcSAlexander Motin 		nvme_ctrlr_shutdown(ctrlr);
1547990e741cSJim Harris 
154871a28181SAlexander Motin 	if (!gone)
1549e134ecdcSAlexander Motin 		nvme_ctrlr_disable(ctrlr);
1550e134ecdcSAlexander Motin 
155131111372SAlexander Motin noadminq:
1552e134ecdcSAlexander Motin 	if (ctrlr->taskqueue)
1553e134ecdcSAlexander Motin 		taskqueue_free(ctrlr->taskqueue);
1554990e741cSJim Harris 
1555990e741cSJim Harris 	if (ctrlr->tag)
1556990e741cSJim Harris 		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
1557990e741cSJim Harris 
1558990e741cSJim Harris 	if (ctrlr->res)
1559990e741cSJim Harris 		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
1560990e741cSJim Harris 		    rman_get_rid(ctrlr->res), ctrlr->res);
1561990e741cSJim Harris 
1562e134ecdcSAlexander Motin 	if (ctrlr->bar4_resource != NULL) {
1563e134ecdcSAlexander Motin 		bus_release_resource(dev, SYS_RES_MEMORY,
1564e134ecdcSAlexander Motin 		    ctrlr->bar4_resource_id, ctrlr->bar4_resource);
1565e134ecdcSAlexander Motin 	}
1566e134ecdcSAlexander Motin 
1567e134ecdcSAlexander Motin 	bus_release_resource(dev, SYS_RES_MEMORY,
1568e134ecdcSAlexander Motin 	    ctrlr->resource_id, ctrlr->resource);
1569e134ecdcSAlexander Motin 
1570e134ecdcSAlexander Motin nores:
1571e134ecdcSAlexander Motin 	mtx_destroy(&ctrlr->lock);
1572990e741cSJim Harris }
1573990e741cSJim Harris 
1574990e741cSJim Harris void
157556183abcSJim Harris nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
157656183abcSJim Harris {
15770d787e9bSWojciech Macek 	uint32_t	cc;
15780d787e9bSWojciech Macek 	uint32_t	csts;
15794fbbe523SAlexander Motin 	int		timeout;
158056183abcSJim Harris 
15810d787e9bSWojciech Macek 	cc = nvme_mmio_read_4(ctrlr, cc);
15820d787e9bSWojciech Macek 	cc &= ~(NVME_CC_REG_SHN_MASK << NVME_CC_REG_SHN_SHIFT);
15830d787e9bSWojciech Macek 	cc |= NVME_SHN_NORMAL << NVME_CC_REG_SHN_SHIFT;
15840d787e9bSWojciech Macek 	nvme_mmio_write_4(ctrlr, cc, cc);
15850d787e9bSWojciech Macek 
15864fbbe523SAlexander Motin 	timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :
15874fbbe523SAlexander Motin 	    ((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);
158871a28181SAlexander Motin 	while (1) {
15890d787e9bSWojciech Macek 		csts = nvme_mmio_read_4(ctrlr, csts);
15909600aa31SWarner Losh 		if (csts == NVME_GONE)		/* Hot unplug. */
159171a28181SAlexander Motin 			break;
159271a28181SAlexander Motin 		if (NVME_CSTS_GET_SHST(csts) == NVME_SHST_COMPLETE)
159371a28181SAlexander Motin 			break;
15944fbbe523SAlexander Motin 		if (timeout - ticks < 0) {
15954fbbe523SAlexander Motin 			nvme_printf(ctrlr, "shutdown timeout\n");
159671a28181SAlexander Motin 			break;
159756183abcSJim Harris 		}
15984fbbe523SAlexander Motin 		pause("nvmeshut", 1);
159971a28181SAlexander Motin 	}
160056183abcSJim Harris }
160156183abcSJim Harris 
160256183abcSJim Harris void
1603d281e8fbSJim Harris nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
1604d281e8fbSJim Harris     struct nvme_request *req)
1605d281e8fbSJim Harris {
1606d281e8fbSJim Harris 
16075ae9ed68SJim Harris 	nvme_qpair_submit_request(&ctrlr->adminq, req);
1608d281e8fbSJim Harris }
1609d281e8fbSJim Harris 
1610d281e8fbSJim Harris void
1611d281e8fbSJim Harris nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
1612d281e8fbSJim Harris     struct nvme_request *req)
1613d281e8fbSJim Harris {
1614d281e8fbSJim Harris 	struct nvme_qpair       *qpair;
1615d281e8fbSJim Harris 
16161eab19cbSAlexander Motin 	qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];
16175ae9ed68SJim Harris 	nvme_qpair_submit_request(qpair, req);
1618d281e8fbSJim Harris }
1619038a5ee4SJim Harris 
1620038a5ee4SJim Harris device_t
1621038a5ee4SJim Harris nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
1622038a5ee4SJim Harris {
1623038a5ee4SJim Harris 
1624038a5ee4SJim Harris 	return (ctrlr->dev);
1625038a5ee4SJim Harris }
1626dbba7442SJim Harris 
1627dbba7442SJim Harris const struct nvme_controller_data *
1628dbba7442SJim Harris nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
1629dbba7442SJim Harris {
1630dbba7442SJim Harris 
1631dbba7442SJim Harris 	return (&ctrlr->cdata);
1632dbba7442SJim Harris }
16334d547561SWarner Losh 
16344d547561SWarner Losh int
16354d547561SWarner Losh nvme_ctrlr_suspend(struct nvme_controller *ctrlr)
16364d547561SWarner Losh {
16374d547561SWarner Losh 	int to = hz;
16384d547561SWarner Losh 
16394d547561SWarner Losh 	/*
16404d547561SWarner Losh 	 * Can't touch failed controllers, so it's already suspended.
16414d547561SWarner Losh 	 */
16424d547561SWarner Losh 	if (ctrlr->is_failed)
16434d547561SWarner Losh 		return (0);
16444d547561SWarner Losh 
16454d547561SWarner Losh 	/*
16464d547561SWarner Losh 	 * We don't want the reset taskqueue running, since it does similar
16474d547561SWarner Losh 	 * things, so prevent it from running after we start. Wait for any reset
16484d547561SWarner Losh 	 * that may have been started to complete. The reset process we follow
16494d547561SWarner Losh 	 * will ensure that any new I/O will queue and be given to the hardware
16504d547561SWarner Losh 	 * after we resume (though there should be none).
16514d547561SWarner Losh 	 */
16524d547561SWarner Losh 	while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)
16534d547561SWarner Losh 		pause("nvmesusp", 1);
16544d547561SWarner Losh 	if (to <= 0) {
16554d547561SWarner Losh 		nvme_printf(ctrlr,
16564d547561SWarner Losh 		    "Competing reset task didn't finish. Try again later.\n");
16574d547561SWarner Losh 		return (EWOULDBLOCK);
16584d547561SWarner Losh 	}
16594d547561SWarner Losh 
166067abaee9SAlexander Motin 	if (ctrlr->hmb_nchunks > 0)
166167abaee9SAlexander Motin 		nvme_ctrlr_hmb_enable(ctrlr, false, false);
166267abaee9SAlexander Motin 
16634d547561SWarner Losh 	/*
16644d547561SWarner Losh 	 * Per Section 7.6.2 of NVMe spec 1.4, to properly suspend, we need to
16654d547561SWarner Losh 	 * delete the hardware I/O queues, and then shutdown. This properly
16664d547561SWarner Losh 	 * flushes any metadata the drive may have stored so it can survive
16674d547561SWarner Losh 	 * having its power removed and prevents the unsafe shutdown count from
16684d547561SWarner Losh 	 * incriminating. Once we delete the qpairs, we have to disable them
1669e5e26e4aSWarner Losh 	 * before shutting down.
16704d547561SWarner Losh 	 */
16714d547561SWarner Losh 	nvme_ctrlr_delete_qpairs(ctrlr);
16724d547561SWarner Losh 	nvme_ctrlr_disable_qpairs(ctrlr);
16734d547561SWarner Losh 	nvme_ctrlr_shutdown(ctrlr);
16744d547561SWarner Losh 
16754d547561SWarner Losh 	return (0);
16764d547561SWarner Losh }
16774d547561SWarner Losh 
16784d547561SWarner Losh int
16794d547561SWarner Losh nvme_ctrlr_resume(struct nvme_controller *ctrlr)
16804d547561SWarner Losh {
16814d547561SWarner Losh 
16824d547561SWarner Losh 	/*
16834d547561SWarner Losh 	 * Can't touch failed controllers, so nothing to do to resume.
16844d547561SWarner Losh 	 */
16854d547561SWarner Losh 	if (ctrlr->is_failed)
16864d547561SWarner Losh 		return (0);
16874d547561SWarner Losh 
16884b3da659SWarner Losh 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)
16894b3da659SWarner Losh 		goto fail;
16904b3da659SWarner Losh #ifdef NVME_2X_RESET
16914d547561SWarner Losh 	/*
16924b3da659SWarner Losh 	 * Prior to FreeBSD 13.1, FreeBSD's nvme driver reset the hardware twice
16934b3da659SWarner Losh 	 * to get it into a known good state. However, the hardware's state is
16944b3da659SWarner Losh 	 * good and we don't need to do this for proper functioning.
16954d547561SWarner Losh 	 */
16964d547561SWarner Losh 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)
16974d547561SWarner Losh 		goto fail;
16984b3da659SWarner Losh #endif
16994d547561SWarner Losh 
17004d547561SWarner Losh 	/*
17014053f8acSDavid Bright 	 * Now that we've reset the hardware, we can restart the controller. Any
17024d547561SWarner Losh 	 * I/O that was pending is requeued. Any admin commands are aborted with
17034d547561SWarner Losh 	 * an error. Once we've restarted, take the controller out of reset.
17044d547561SWarner Losh 	 */
17054d547561SWarner Losh 	nvme_ctrlr_start(ctrlr, true);
17064053f8acSDavid Bright 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
17074d547561SWarner Losh 
17084d547561SWarner Losh 	return (0);
17094d547561SWarner Losh fail:
17104d547561SWarner Losh 	/*
17114d547561SWarner Losh 	 * Since we can't bring the controller out of reset, announce and fail
17124d547561SWarner Losh 	 * the controller. However, we have to return success for the resume
17134d547561SWarner Losh 	 * itself, due to questionable APIs.
17144d547561SWarner Losh 	 */
17154d547561SWarner Losh 	nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");
17164d547561SWarner Losh 	nvme_ctrlr_fail(ctrlr);
17174053f8acSDavid Bright 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
17184d547561SWarner Losh 	return (0);
17194d547561SWarner Losh }
1720