xref: /freebsd/sys/dev/nvme/nvme_ctrlr.c (revision d3d381b2b194b4d24853e92eecef55f262688d1a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2012-2016 Intel Corporation
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include "opt_cam.h"
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/buf.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/ioccom.h>
40 #include <sys/proc.h>
41 #include <sys/smp.h>
42 #include <sys/uio.h>
43 #include <sys/endian.h>
44 
45 #include <dev/pci/pcireg.h>
46 #include <dev/pci/pcivar.h>
47 
48 #include "nvme_private.h"
49 
50 #define B4_CHK_RDY_DELAY_MS	2300		/* work around controller bug */
51 
52 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
53 						struct nvme_async_event_request *aer);
54 static void nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr);
55 
56 static int
57 nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr)
58 {
59 
60 	ctrlr->resource_id = PCIR_BAR(0);
61 
62 	ctrlr->resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY,
63 	    &ctrlr->resource_id, RF_ACTIVE);
64 
65 	if(ctrlr->resource == NULL) {
66 		nvme_printf(ctrlr, "unable to allocate pci resource\n");
67 		return (ENOMEM);
68 	}
69 
70 	ctrlr->bus_tag = rman_get_bustag(ctrlr->resource);
71 	ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource);
72 	ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle;
73 
74 	/*
75 	 * The NVMe spec allows for the MSI-X table to be placed behind
76 	 *  BAR 4/5, separate from the control/doorbell registers.  Always
77 	 *  try to map this bar, because it must be mapped prior to calling
78 	 *  pci_alloc_msix().  If the table isn't behind BAR 4/5,
79 	 *  bus_alloc_resource() will just return NULL which is OK.
80 	 */
81 	ctrlr->bar4_resource_id = PCIR_BAR(4);
82 	ctrlr->bar4_resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY,
83 	    &ctrlr->bar4_resource_id, RF_ACTIVE);
84 
85 	return (0);
86 }
87 
88 static int
89 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
90 {
91 	struct nvme_qpair	*qpair;
92 	uint32_t		num_entries;
93 	int			error;
94 
95 	qpair = &ctrlr->adminq;
96 
97 	num_entries = NVME_ADMIN_ENTRIES;
98 	TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
99 	/*
100 	 * If admin_entries was overridden to an invalid value, revert it
101 	 *  back to our default value.
102 	 */
103 	if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
104 	    num_entries > NVME_MAX_ADMIN_ENTRIES) {
105 		nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
106 		    "specified\n", num_entries);
107 		num_entries = NVME_ADMIN_ENTRIES;
108 	}
109 
110 	/*
111 	 * The admin queue's max xfer size is treated differently than the
112 	 *  max I/O xfer size.  16KB is sufficient here - maybe even less?
113 	 */
114 	error = nvme_qpair_construct(qpair,
115 				     0, /* qpair ID */
116 				     0, /* vector */
117 				     num_entries,
118 				     NVME_ADMIN_TRACKERS,
119 				     ctrlr);
120 	return (error);
121 }
122 
123 static int
124 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
125 {
126 	struct nvme_qpair	*qpair;
127 	uint32_t		cap_lo;
128 	uint16_t		mqes;
129 	int			i, error, num_entries, num_trackers;
130 
131 	num_entries = NVME_IO_ENTRIES;
132 	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
133 
134 	/*
135 	 * NVMe spec sets a hard limit of 64K max entries, but
136 	 *  devices may specify a smaller limit, so we need to check
137 	 *  the MQES field in the capabilities register.
138 	 */
139 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
140 	mqes = (cap_lo >> NVME_CAP_LO_REG_MQES_SHIFT) & NVME_CAP_LO_REG_MQES_MASK;
141 	num_entries = min(num_entries, mqes + 1);
142 
143 	num_trackers = NVME_IO_TRACKERS;
144 	TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
145 
146 	num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
147 	num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
148 	/*
149 	 * No need to have more trackers than entries in the submit queue.
150 	 *  Note also that for a queue size of N, we can only have (N-1)
151 	 *  commands outstanding, hence the "-1" here.
152 	 */
153 	num_trackers = min(num_trackers, (num_entries-1));
154 
155 	/*
156 	 * Our best estimate for the maximum number of I/Os that we should
157 	 * noramlly have in flight at one time. This should be viewed as a hint,
158 	 * not a hard limit and will need to be revisitted when the upper layers
159 	 * of the storage system grows multi-queue support.
160 	 */
161 	ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;
162 
163 	/*
164 	 * This was calculated previously when setting up interrupts, but
165 	 *  a controller could theoretically support fewer I/O queues than
166 	 *  MSI-X vectors.  So calculate again here just to be safe.
167 	 */
168 	ctrlr->num_cpus_per_ioq = howmany(mp_ncpus, ctrlr->num_io_queues);
169 
170 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
171 	    M_NVME, M_ZERO | M_WAITOK);
172 
173 	for (i = 0; i < ctrlr->num_io_queues; i++) {
174 		qpair = &ctrlr->ioq[i];
175 
176 		/*
177 		 * Admin queue has ID=0. IO queues start at ID=1 -
178 		 *  hence the 'i+1' here.
179 		 *
180 		 * For I/O queues, use the controller-wide max_xfer_size
181 		 *  calculated in nvme_attach().
182 		 */
183 		error = nvme_qpair_construct(qpair,
184 				     i+1, /* qpair ID */
185 				     ctrlr->msix_enabled ? i+1 : 0, /* vector */
186 				     num_entries,
187 				     num_trackers,
188 				     ctrlr);
189 		if (error)
190 			return (error);
191 
192 		/*
193 		 * Do not bother binding interrupts if we only have one I/O
194 		 *  interrupt thread for this controller.
195 		 */
196 		if (ctrlr->num_io_queues > 1)
197 			bus_bind_intr(ctrlr->dev, qpair->res,
198 			    i * ctrlr->num_cpus_per_ioq);
199 	}
200 
201 	return (0);
202 }
203 
204 static void
205 nvme_ctrlr_fail(struct nvme_controller *ctrlr)
206 {
207 	int i;
208 
209 	ctrlr->is_failed = TRUE;
210 	nvme_qpair_fail(&ctrlr->adminq);
211 	if (ctrlr->ioq != NULL) {
212 		for (i = 0; i < ctrlr->num_io_queues; i++)
213 			nvme_qpair_fail(&ctrlr->ioq[i]);
214 	}
215 	nvme_notify_fail_consumers(ctrlr);
216 }
217 
218 void
219 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
220     struct nvme_request *req)
221 {
222 
223 	mtx_lock(&ctrlr->lock);
224 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
225 	mtx_unlock(&ctrlr->lock);
226 	taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
227 }
228 
229 static void
230 nvme_ctrlr_fail_req_task(void *arg, int pending)
231 {
232 	struct nvme_controller	*ctrlr = arg;
233 	struct nvme_request	*req;
234 
235 	mtx_lock(&ctrlr->lock);
236 	while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {
237 		STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
238 		mtx_unlock(&ctrlr->lock);
239 		nvme_qpair_manual_complete_request(req->qpair, req,
240 		    NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE);
241 		mtx_lock(&ctrlr->lock);
242 	}
243 	mtx_unlock(&ctrlr->lock);
244 }
245 
246 static int
247 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
248 {
249 	int ms_waited;
250 	uint32_t csts;
251 
252 	csts = nvme_mmio_read_4(ctrlr, csts);
253 
254 	ms_waited = 0;
255 	while (((csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK) != desired_val) {
256 		if (ms_waited++ > ctrlr->ready_timeout_in_ms) {
257 			nvme_printf(ctrlr, "controller ready did not become %d "
258 			    "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
259 			return (ENXIO);
260 		}
261 		DELAY(1000);
262 		csts = nvme_mmio_read_4(ctrlr, csts);
263 	}
264 
265 	return (0);
266 }
267 
268 static int
269 nvme_ctrlr_disable(struct nvme_controller *ctrlr)
270 {
271 	uint32_t cc;
272 	uint32_t csts;
273 	uint8_t  en, rdy;
274 	int err;
275 
276 	cc = nvme_mmio_read_4(ctrlr, cc);
277 	csts = nvme_mmio_read_4(ctrlr, csts);
278 
279 	en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
280 	rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
281 
282 	/*
283 	 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
284 	 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
285 	 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
286 	 * isn't the desired value. Short circuit if we're already disabled.
287 	 */
288 	if (en == 1) {
289 		if (rdy == 0) {
290 			/* EN == 1, wait for  RDY == 1 or fail */
291 			err = nvme_ctrlr_wait_for_ready(ctrlr, 1);
292 			if (err != 0)
293 				return (err);
294 		}
295 	} else {
296 		/* EN == 0 already wait for RDY == 0 */
297 		if (rdy == 0)
298 			return (0);
299 		else
300 			return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
301 	}
302 
303 	cc &= ~NVME_CC_REG_EN_MASK;
304 	nvme_mmio_write_4(ctrlr, cc, cc);
305 	/*
306 	 * Some drives have issues with accessing the mmio after we
307 	 * disable, so delay for a bit after we write the bit to
308 	 * cope with these issues.
309 	 */
310 	if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)
311 		pause("nvmeR", B4_CHK_RDY_DELAY_MS * hz / 1000);
312 	return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
313 }
314 
315 static int
316 nvme_ctrlr_enable(struct nvme_controller *ctrlr)
317 {
318 	uint32_t	cc;
319 	uint32_t	csts;
320 	uint32_t	aqa;
321 	uint32_t	qsize;
322 	uint8_t		en, rdy;
323 	int		err;
324 
325 	cc = nvme_mmio_read_4(ctrlr, cc);
326 	csts = nvme_mmio_read_4(ctrlr, csts);
327 
328 	en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
329 	rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
330 
331 	/*
332 	 * See note in nvme_ctrlr_disable. Short circuit if we're already enabled.
333 	 */
334 	if (en == 1) {
335 		if (rdy == 1)
336 			return (0);
337 		else
338 			return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
339 	} else {
340 		/* EN == 0 already wait for RDY == 0 or fail */
341 		err = nvme_ctrlr_wait_for_ready(ctrlr, 0);
342 		if (err != 0)
343 			return (err);
344 	}
345 
346 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
347 	DELAY(5000);
348 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
349 	DELAY(5000);
350 
351 	/* acqs and asqs are 0-based. */
352 	qsize = ctrlr->adminq.num_entries - 1;
353 
354 	aqa = 0;
355 	aqa = (qsize & NVME_AQA_REG_ACQS_MASK) << NVME_AQA_REG_ACQS_SHIFT;
356 	aqa |= (qsize & NVME_AQA_REG_ASQS_MASK) << NVME_AQA_REG_ASQS_SHIFT;
357 	nvme_mmio_write_4(ctrlr, aqa, aqa);
358 	DELAY(5000);
359 
360 	/* Initialization values for CC */
361 	cc = 0;
362 	cc |= 1 << NVME_CC_REG_EN_SHIFT;
363 	cc |= 0 << NVME_CC_REG_CSS_SHIFT;
364 	cc |= 0 << NVME_CC_REG_AMS_SHIFT;
365 	cc |= 0 << NVME_CC_REG_SHN_SHIFT;
366 	cc |= 6 << NVME_CC_REG_IOSQES_SHIFT; /* SQ entry size == 64 == 2^6 */
367 	cc |= 4 << NVME_CC_REG_IOCQES_SHIFT; /* CQ entry size == 16 == 2^4 */
368 
369 	/* This evaluates to 0, which is according to spec. */
370 	cc |= (PAGE_SIZE >> 13) << NVME_CC_REG_MPS_SHIFT;
371 
372 	nvme_mmio_write_4(ctrlr, cc, cc);
373 
374 	return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
375 }
376 
377 int
378 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
379 {
380 	int i, err;
381 
382 	nvme_admin_qpair_disable(&ctrlr->adminq);
383 	/*
384 	 * I/O queues are not allocated before the initial HW
385 	 *  reset, so do not try to disable them.  Use is_initialized
386 	 *  to determine if this is the initial HW reset.
387 	 */
388 	if (ctrlr->is_initialized) {
389 		for (i = 0; i < ctrlr->num_io_queues; i++)
390 			nvme_io_qpair_disable(&ctrlr->ioq[i]);
391 	}
392 
393 	DELAY(100*1000);
394 
395 	err = nvme_ctrlr_disable(ctrlr);
396 	if (err != 0)
397 		return err;
398 	return (nvme_ctrlr_enable(ctrlr));
399 }
400 
401 void
402 nvme_ctrlr_reset(struct nvme_controller *ctrlr)
403 {
404 	int cmpset;
405 
406 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
407 
408 	if (cmpset == 0 || ctrlr->is_failed)
409 		/*
410 		 * Controller is already resetting or has failed.  Return
411 		 *  immediately since there is no need to kick off another
412 		 *  reset in these cases.
413 		 */
414 		return;
415 
416 	taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
417 }
418 
419 static int
420 nvme_ctrlr_identify(struct nvme_controller *ctrlr)
421 {
422 	struct nvme_completion_poll_status	status;
423 
424 	status.done = 0;
425 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
426 	    nvme_completion_poll_cb, &status);
427 	while (!atomic_load_acq_int(&status.done))
428 		pause("nvme", 1);
429 	if (nvme_completion_is_error(&status.cpl)) {
430 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
431 		return (ENXIO);
432 	}
433 
434 	/* Convert data to host endian */
435 	nvme_controller_data_swapbytes(&ctrlr->cdata);
436 
437 	/*
438 	 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
439 	 *  controller supports.
440 	 */
441 	if (ctrlr->cdata.mdts > 0)
442 		ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
443 		    ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
444 
445 	return (0);
446 }
447 
448 static int
449 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
450 {
451 	struct nvme_completion_poll_status	status;
452 	int					cq_allocated, sq_allocated;
453 
454 	status.done = 0;
455 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
456 	    nvme_completion_poll_cb, &status);
457 	while (!atomic_load_acq_int(&status.done))
458 		pause("nvme", 1);
459 	if (nvme_completion_is_error(&status.cpl)) {
460 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
461 		return (ENXIO);
462 	}
463 
464 	/*
465 	 * Data in cdw0 is 0-based.
466 	 * Lower 16-bits indicate number of submission queues allocated.
467 	 * Upper 16-bits indicate number of completion queues allocated.
468 	 */
469 	sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
470 	cq_allocated = (status.cpl.cdw0 >> 16) + 1;
471 
472 	/*
473 	 * Controller may allocate more queues than we requested,
474 	 *  so use the minimum of the number requested and what was
475 	 *  actually allocated.
476 	 */
477 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);
478 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);
479 
480 	return (0);
481 }
482 
483 static int
484 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
485 {
486 	struct nvme_completion_poll_status	status;
487 	struct nvme_qpair			*qpair;
488 	int					i;
489 
490 	for (i = 0; i < ctrlr->num_io_queues; i++) {
491 		qpair = &ctrlr->ioq[i];
492 
493 		status.done = 0;
494 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
495 		    nvme_completion_poll_cb, &status);
496 		while (!atomic_load_acq_int(&status.done))
497 			pause("nvme", 1);
498 		if (nvme_completion_is_error(&status.cpl)) {
499 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
500 			return (ENXIO);
501 		}
502 
503 		status.done = 0;
504 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
505 		    nvme_completion_poll_cb, &status);
506 		while (!atomic_load_acq_int(&status.done))
507 			pause("nvme", 1);
508 		if (nvme_completion_is_error(&status.cpl)) {
509 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
510 			return (ENXIO);
511 		}
512 	}
513 
514 	return (0);
515 }
516 
517 static int
518 nvme_ctrlr_destroy_qpair(struct nvme_controller *ctrlr, struct nvme_qpair *qpair)
519 {
520 	struct nvme_completion_poll_status	status;
521 
522 	status.done = 0;
523 	nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
524 	    nvme_completion_poll_cb, &status);
525 	while (!atomic_load_acq_int(&status.done))
526 		pause("nvme", 1);
527 	if (nvme_completion_is_error(&status.cpl)) {
528 		nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
529 		return (ENXIO);
530 	}
531 
532 	status.done = 0;
533 	nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
534 	    nvme_completion_poll_cb, &status);
535 	while (!atomic_load_acq_int(&status.done))
536 		pause("nvme", 1);
537 	if (nvme_completion_is_error(&status.cpl)) {
538 		nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
539 		return (ENXIO);
540 	}
541 
542 	return (0);
543 }
544 
545 static int
546 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
547 {
548 	struct nvme_namespace	*ns;
549 	uint32_t 		i;
550 
551 	for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
552 		ns = &ctrlr->ns[i];
553 		nvme_ns_construct(ns, i+1, ctrlr);
554 	}
555 
556 	return (0);
557 }
558 
559 static boolean_t
560 is_log_page_id_valid(uint8_t page_id)
561 {
562 
563 	switch (page_id) {
564 	case NVME_LOG_ERROR:
565 	case NVME_LOG_HEALTH_INFORMATION:
566 	case NVME_LOG_FIRMWARE_SLOT:
567 	case NVME_LOG_CHANGED_NAMESPACE:
568 		return (TRUE);
569 	}
570 
571 	return (FALSE);
572 }
573 
574 static uint32_t
575 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
576 {
577 	uint32_t	log_page_size;
578 
579 	switch (page_id) {
580 	case NVME_LOG_ERROR:
581 		log_page_size = min(
582 		    sizeof(struct nvme_error_information_entry) *
583 		    (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);
584 		break;
585 	case NVME_LOG_HEALTH_INFORMATION:
586 		log_page_size = sizeof(struct nvme_health_information_page);
587 		break;
588 	case NVME_LOG_FIRMWARE_SLOT:
589 		log_page_size = sizeof(struct nvme_firmware_page);
590 		break;
591 	case NVME_LOG_CHANGED_NAMESPACE:
592 		log_page_size = sizeof(struct nvme_ns_list);
593 		break;
594 	default:
595 		log_page_size = 0;
596 		break;
597 	}
598 
599 	return (log_page_size);
600 }
601 
602 static void
603 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
604     uint8_t state)
605 {
606 
607 	if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE)
608 		nvme_printf(ctrlr, "available spare space below threshold\n");
609 
610 	if (state & NVME_CRIT_WARN_ST_TEMPERATURE)
611 		nvme_printf(ctrlr, "temperature above threshold\n");
612 
613 	if (state & NVME_CRIT_WARN_ST_DEVICE_RELIABILITY)
614 		nvme_printf(ctrlr, "device reliability degraded\n");
615 
616 	if (state & NVME_CRIT_WARN_ST_READ_ONLY)
617 		nvme_printf(ctrlr, "media placed in read only mode\n");
618 
619 	if (state & NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP)
620 		nvme_printf(ctrlr, "volatile memory backup device failed\n");
621 
622 	if (state & NVME_CRIT_WARN_ST_RESERVED_MASK)
623 		nvme_printf(ctrlr,
624 		    "unknown critical warning(s): state = 0x%02x\n", state);
625 }
626 
627 static void
628 nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl)
629 {
630 	struct nvme_async_event_request		*aer = arg;
631 	struct nvme_health_information_page	*health_info;
632 	struct nvme_ns_list			*nsl;
633 	struct nvme_error_information_entry	*err;
634 	int i;
635 
636 	/*
637 	 * If the log page fetch for some reason completed with an error,
638 	 *  don't pass log page data to the consumers.  In practice, this case
639 	 *  should never happen.
640 	 */
641 	if (nvme_completion_is_error(cpl))
642 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
643 		    aer->log_page_id, NULL, 0);
644 	else {
645 		/* Convert data to host endian */
646 		switch (aer->log_page_id) {
647 		case NVME_LOG_ERROR:
648 			err = (struct nvme_error_information_entry *)aer->log_page_buffer;
649 			for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)
650 				nvme_error_information_entry_swapbytes(err++);
651 			break;
652 		case NVME_LOG_HEALTH_INFORMATION:
653 			nvme_health_information_page_swapbytes(
654 			    (struct nvme_health_information_page *)aer->log_page_buffer);
655 			break;
656 		case NVME_LOG_FIRMWARE_SLOT:
657 			nvme_firmware_page_swapbytes(
658 			    (struct nvme_firmware_page *)aer->log_page_buffer);
659 			break;
660 		case NVME_LOG_CHANGED_NAMESPACE:
661 			nvme_ns_list_swapbytes(
662 			    (struct nvme_ns_list *)aer->log_page_buffer);
663 			break;
664 		case INTEL_LOG_TEMP_STATS:
665 			intel_log_temp_stats_swapbytes(
666 			    (struct intel_log_temp_stats *)aer->log_page_buffer);
667 			break;
668 		default:
669 			break;
670 		}
671 
672 		if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
673 			health_info = (struct nvme_health_information_page *)
674 			    aer->log_page_buffer;
675 			nvme_ctrlr_log_critical_warnings(aer->ctrlr,
676 			    health_info->critical_warning);
677 			/*
678 			 * Critical warnings reported through the
679 			 *  SMART/health log page are persistent, so
680 			 *  clear the associated bits in the async event
681 			 *  config so that we do not receive repeated
682 			 *  notifications for the same event.
683 			 */
684 			aer->ctrlr->async_event_config &=
685 			    ~health_info->critical_warning;
686 			nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
687 			    aer->ctrlr->async_event_config, NULL, NULL);
688 		} else if (aer->log_page_id == NVME_LOG_CHANGED_NAMESPACE &&
689 		    !nvme_use_nvd) {
690 			nsl = (struct nvme_ns_list *)aer->log_page_buffer;
691 			for (i = 0; i < nitems(nsl->ns) && nsl->ns[i] != 0; i++) {
692 				if (nsl->ns[i] > NVME_MAX_NAMESPACES)
693 					break;
694 				nvme_notify_ns(aer->ctrlr, nsl->ns[i]);
695 			}
696 		}
697 
698 
699 		/*
700 		 * Pass the cpl data from the original async event completion,
701 		 *  not the log page fetch.
702 		 */
703 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
704 		    aer->log_page_id, aer->log_page_buffer, aer->log_page_size);
705 	}
706 
707 	/*
708 	 * Repost another asynchronous event request to replace the one
709 	 *  that just completed.
710 	 */
711 	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
712 }
713 
714 static void
715 nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
716 {
717 	struct nvme_async_event_request	*aer = arg;
718 
719 	if (nvme_completion_is_error(cpl)) {
720 		/*
721 		 *  Do not retry failed async event requests.  This avoids
722 		 *  infinite loops where a new async event request is submitted
723 		 *  to replace the one just failed, only to fail again and
724 		 *  perpetuate the loop.
725 		 */
726 		return;
727 	}
728 
729 	/* Associated log page is in bits 23:16 of completion entry dw0. */
730 	aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
731 
732 	nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"
733 	    " page 0x%02x)\n", (cpl->cdw0 & 0x03), (cpl->cdw0 & 0xFF00) >> 8,
734 	    aer->log_page_id);
735 
736 	if (is_log_page_id_valid(aer->log_page_id)) {
737 		aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,
738 		    aer->log_page_id);
739 		memcpy(&aer->cpl, cpl, sizeof(*cpl));
740 		nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
741 		    NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer,
742 		    aer->log_page_size, nvme_ctrlr_async_event_log_page_cb,
743 		    aer);
744 		/* Wait to notify consumers until after log page is fetched. */
745 	} else {
746 		nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,
747 		    NULL, 0);
748 
749 		/*
750 		 * Repost another asynchronous event request to replace the one
751 		 *  that just completed.
752 		 */
753 		nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
754 	}
755 }
756 
757 static void
758 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
759     struct nvme_async_event_request *aer)
760 {
761 	struct nvme_request *req;
762 
763 	aer->ctrlr = ctrlr;
764 	req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
765 	aer->req = req;
766 
767 	/*
768 	 * Disable timeout here, since asynchronous event requests should by
769 	 *  nature never be timed out.
770 	 */
771 	req->timeout = FALSE;
772 	req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
773 	nvme_ctrlr_submit_admin_request(ctrlr, req);
774 }
775 
776 static void
777 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
778 {
779 	struct nvme_completion_poll_status	status;
780 	struct nvme_async_event_request		*aer;
781 	uint32_t				i;
782 
783 	ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
784 	    NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
785 	    NVME_CRIT_WARN_ST_READ_ONLY |
786 	    NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP;
787 	if (ctrlr->cdata.ver >= NVME_REV(1, 2))
788 		ctrlr->async_event_config |= 0x300;
789 
790 	status.done = 0;
791 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
792 	    0, NULL, 0, nvme_completion_poll_cb, &status);
793 	while (!atomic_load_acq_int(&status.done))
794 		pause("nvme", 1);
795 	if (nvme_completion_is_error(&status.cpl) ||
796 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
797 	    (status.cpl.cdw0 & 0xFFFF) == 0x0000) {
798 		nvme_printf(ctrlr, "temperature threshold not supported\n");
799 	} else
800 		ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;
801 
802 	nvme_ctrlr_cmd_set_async_event_config(ctrlr,
803 	    ctrlr->async_event_config, NULL, NULL);
804 
805 	/* aerl is a zero-based value, so we need to add 1 here. */
806 	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
807 
808 	for (i = 0; i < ctrlr->num_aers; i++) {
809 		aer = &ctrlr->aer[i];
810 		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
811 	}
812 }
813 
814 static void
815 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
816 {
817 
818 	ctrlr->int_coal_time = 0;
819 	TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
820 	    &ctrlr->int_coal_time);
821 
822 	ctrlr->int_coal_threshold = 0;
823 	TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
824 	    &ctrlr->int_coal_threshold);
825 
826 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
827 	    ctrlr->int_coal_threshold, NULL, NULL);
828 }
829 
830 static void
831 nvme_ctrlr_start(void *ctrlr_arg)
832 {
833 	struct nvme_controller *ctrlr = ctrlr_arg;
834 	uint32_t old_num_io_queues;
835 	int i;
836 
837 	/*
838 	 * Only reset adminq here when we are restarting the
839 	 *  controller after a reset.  During initialization,
840 	 *  we have already submitted admin commands to get
841 	 *  the number of I/O queues supported, so cannot reset
842 	 *  the adminq again here.
843 	 */
844 	if (ctrlr->is_resetting) {
845 		nvme_qpair_reset(&ctrlr->adminq);
846 	}
847 
848 	for (i = 0; i < ctrlr->num_io_queues; i++)
849 		nvme_qpair_reset(&ctrlr->ioq[i]);
850 
851 	nvme_admin_qpair_enable(&ctrlr->adminq);
852 
853 	if (nvme_ctrlr_identify(ctrlr) != 0) {
854 		nvme_ctrlr_fail(ctrlr);
855 		return;
856 	}
857 
858 	/*
859 	 * The number of qpairs are determined during controller initialization,
860 	 *  including using NVMe SET_FEATURES/NUMBER_OF_QUEUES to determine the
861 	 *  HW limit.  We call SET_FEATURES again here so that it gets called
862 	 *  after any reset for controllers that depend on the driver to
863 	 *  explicit specify how many queues it will use.  This value should
864 	 *  never change between resets, so panic if somehow that does happen.
865 	 */
866 	if (ctrlr->is_resetting) {
867 		old_num_io_queues = ctrlr->num_io_queues;
868 		if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
869 			nvme_ctrlr_fail(ctrlr);
870 			return;
871 		}
872 
873 		if (old_num_io_queues != ctrlr->num_io_queues) {
874 			panic("num_io_queues changed from %u to %u",
875 			      old_num_io_queues, ctrlr->num_io_queues);
876 		}
877 	}
878 
879 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
880 		nvme_ctrlr_fail(ctrlr);
881 		return;
882 	}
883 
884 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
885 		nvme_ctrlr_fail(ctrlr);
886 		return;
887 	}
888 
889 	nvme_ctrlr_configure_aer(ctrlr);
890 	nvme_ctrlr_configure_int_coalescing(ctrlr);
891 
892 	for (i = 0; i < ctrlr->num_io_queues; i++)
893 		nvme_io_qpair_enable(&ctrlr->ioq[i]);
894 }
895 
896 void
897 nvme_ctrlr_start_config_hook(void *arg)
898 {
899 	struct nvme_controller *ctrlr = arg;
900 
901 	nvme_qpair_reset(&ctrlr->adminq);
902 	nvme_admin_qpair_enable(&ctrlr->adminq);
903 
904 	if (nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&
905 	    nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)
906 		nvme_ctrlr_start(ctrlr);
907 	else
908 		nvme_ctrlr_fail(ctrlr);
909 
910 	nvme_sysctl_initialize_ctrlr(ctrlr);
911 	config_intrhook_disestablish(&ctrlr->config_hook);
912 
913 	ctrlr->is_initialized = 1;
914 	nvme_notify_new_controller(ctrlr);
915 }
916 
917 static void
918 nvme_ctrlr_reset_task(void *arg, int pending)
919 {
920 	struct nvme_controller	*ctrlr = arg;
921 	int			status;
922 
923 	nvme_printf(ctrlr, "resetting controller\n");
924 	status = nvme_ctrlr_hw_reset(ctrlr);
925 	/*
926 	 * Use pause instead of DELAY, so that we yield to any nvme interrupt
927 	 *  handlers on this CPU that were blocked on a qpair lock. We want
928 	 *  all nvme interrupts completed before proceeding with restarting the
929 	 *  controller.
930 	 *
931 	 * XXX - any way to guarantee the interrupt handlers have quiesced?
932 	 */
933 	pause("nvmereset", hz / 10);
934 	if (status == 0)
935 		nvme_ctrlr_start(ctrlr);
936 	else
937 		nvme_ctrlr_fail(ctrlr);
938 
939 	atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
940 }
941 
942 /*
943  * Poll all the queues enabled on the device for completion.
944  */
945 void
946 nvme_ctrlr_poll(struct nvme_controller *ctrlr)
947 {
948 	int i;
949 
950 	nvme_qpair_process_completions(&ctrlr->adminq);
951 
952 	for (i = 0; i < ctrlr->num_io_queues; i++)
953 		if (ctrlr->ioq && ctrlr->ioq[i].cpl)
954 			nvme_qpair_process_completions(&ctrlr->ioq[i]);
955 }
956 
957 /*
958  * Poll the single-vector intertrupt case: num_io_queues will be 1 and
959  * there's only a single vector. While we're polling, we mask further
960  * interrupts in the controller.
961  */
962 void
963 nvme_ctrlr_intx_handler(void *arg)
964 {
965 	struct nvme_controller *ctrlr = arg;
966 
967 	nvme_mmio_write_4(ctrlr, intms, 1);
968 	nvme_ctrlr_poll(ctrlr);
969 	nvme_mmio_write_4(ctrlr, intmc, 1);
970 }
971 
972 static int
973 nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr)
974 {
975 
976 	ctrlr->msix_enabled = 0;
977 	ctrlr->num_io_queues = 1;
978 	ctrlr->num_cpus_per_ioq = mp_ncpus;
979 	ctrlr->rid = 0;
980 	ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
981 	    &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE);
982 
983 	if (ctrlr->res == NULL) {
984 		nvme_printf(ctrlr, "unable to allocate shared IRQ\n");
985 		return (ENOMEM);
986 	}
987 
988 	bus_setup_intr(ctrlr->dev, ctrlr->res,
989 	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler,
990 	    ctrlr, &ctrlr->tag);
991 
992 	if (ctrlr->tag == NULL) {
993 		nvme_printf(ctrlr, "unable to setup intx handler\n");
994 		return (ENOMEM);
995 	}
996 
997 	return (0);
998 }
999 
1000 static void
1001 nvme_pt_done(void *arg, const struct nvme_completion *cpl)
1002 {
1003 	struct nvme_pt_command *pt = arg;
1004 	struct mtx *mtx = pt->driver_lock;
1005 	uint16_t status;
1006 
1007 	bzero(&pt->cpl, sizeof(pt->cpl));
1008 	pt->cpl.cdw0 = cpl->cdw0;
1009 
1010 	status = cpl->status;
1011 	status &= ~NVME_STATUS_P_MASK;
1012 	pt->cpl.status = status;
1013 
1014 	mtx_lock(mtx);
1015 	pt->driver_lock = NULL;
1016 	wakeup(pt);
1017 	mtx_unlock(mtx);
1018 }
1019 
1020 int
1021 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
1022     struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
1023     int is_admin_cmd)
1024 {
1025 	struct nvme_request	*req;
1026 	struct mtx		*mtx;
1027 	struct buf		*buf = NULL;
1028 	int			ret = 0;
1029 	vm_offset_t		addr, end;
1030 
1031 	if (pt->len > 0) {
1032 		/*
1033 		 * vmapbuf calls vm_fault_quick_hold_pages which only maps full
1034 		 * pages. Ensure this request has fewer than MAXPHYS bytes when
1035 		 * extended to full pages.
1036 		 */
1037 		addr = (vm_offset_t)pt->buf;
1038 		end = round_page(addr + pt->len);
1039 		addr = trunc_page(addr);
1040 		if (end - addr > MAXPHYS)
1041 			return EIO;
1042 
1043 		if (pt->len > ctrlr->max_xfer_size) {
1044 			nvme_printf(ctrlr, "pt->len (%d) "
1045 			    "exceeds max_xfer_size (%d)\n", pt->len,
1046 			    ctrlr->max_xfer_size);
1047 			return EIO;
1048 		}
1049 		if (is_user_buffer) {
1050 			/*
1051 			 * Ensure the user buffer is wired for the duration of
1052 			 *  this passthrough command.
1053 			 */
1054 			PHOLD(curproc);
1055 			buf = getpbuf(NULL);
1056 			buf->b_data = pt->buf;
1057 			buf->b_bufsize = pt->len;
1058 			buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
1059 #ifdef NVME_UNMAPPED_BIO_SUPPORT
1060 			if (vmapbuf(buf, 1) < 0) {
1061 #else
1062 			if (vmapbuf(buf) < 0) {
1063 #endif
1064 				ret = EFAULT;
1065 				goto err;
1066 			}
1067 			req = nvme_allocate_request_vaddr(buf->b_data, pt->len,
1068 			    nvme_pt_done, pt);
1069 		} else
1070 			req = nvme_allocate_request_vaddr(pt->buf, pt->len,
1071 			    nvme_pt_done, pt);
1072 	} else
1073 		req = nvme_allocate_request_null(nvme_pt_done, pt);
1074 
1075 	/* Assume userspace already converted to little-endian */
1076 	req->cmd.opc = pt->cmd.opc;
1077 	req->cmd.fuse = pt->cmd.fuse;
1078 	req->cmd.cdw10 = pt->cmd.cdw10;
1079 	req->cmd.cdw11 = pt->cmd.cdw11;
1080 	req->cmd.cdw12 = pt->cmd.cdw12;
1081 	req->cmd.cdw13 = pt->cmd.cdw13;
1082 	req->cmd.cdw14 = pt->cmd.cdw14;
1083 	req->cmd.cdw15 = pt->cmd.cdw15;
1084 
1085 	req->cmd.nsid = htole32(nsid);
1086 
1087 	mtx = mtx_pool_find(mtxpool_sleep, pt);
1088 	pt->driver_lock = mtx;
1089 
1090 	if (is_admin_cmd)
1091 		nvme_ctrlr_submit_admin_request(ctrlr, req);
1092 	else
1093 		nvme_ctrlr_submit_io_request(ctrlr, req);
1094 
1095 	mtx_lock(mtx);
1096 	while (pt->driver_lock != NULL)
1097 		mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
1098 	mtx_unlock(mtx);
1099 
1100 err:
1101 	if (buf != NULL) {
1102 		relpbuf(buf, NULL);
1103 		PRELE(curproc);
1104 	}
1105 
1106 	return (ret);
1107 }
1108 
1109 static int
1110 nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
1111     struct thread *td)
1112 {
1113 	struct nvme_controller			*ctrlr;
1114 	struct nvme_pt_command			*pt;
1115 
1116 	ctrlr = cdev->si_drv1;
1117 
1118 	switch (cmd) {
1119 	case NVME_RESET_CONTROLLER:
1120 		nvme_ctrlr_reset(ctrlr);
1121 		break;
1122 	case NVME_PASSTHROUGH_CMD:
1123 		pt = (struct nvme_pt_command *)arg;
1124 		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),
1125 		    1 /* is_user_buffer */, 1 /* is_admin_cmd */));
1126 	default:
1127 		return (ENOTTY);
1128 	}
1129 
1130 	return (0);
1131 }
1132 
1133 static struct cdevsw nvme_ctrlr_cdevsw = {
1134 	.d_version =	D_VERSION,
1135 	.d_flags =	0,
1136 	.d_ioctl =	nvme_ctrlr_ioctl
1137 };
1138 
1139 static void
1140 nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr)
1141 {
1142 	device_t	dev;
1143 	int		per_cpu_io_queues;
1144 	int		min_cpus_per_ioq;
1145 	int		num_vectors_requested, num_vectors_allocated;
1146 	int		num_vectors_available;
1147 
1148 	dev = ctrlr->dev;
1149 	min_cpus_per_ioq = 1;
1150 	TUNABLE_INT_FETCH("hw.nvme.min_cpus_per_ioq", &min_cpus_per_ioq);
1151 
1152 	if (min_cpus_per_ioq < 1) {
1153 		min_cpus_per_ioq = 1;
1154 	} else if (min_cpus_per_ioq > mp_ncpus) {
1155 		min_cpus_per_ioq = mp_ncpus;
1156 	}
1157 
1158 	per_cpu_io_queues = 1;
1159 	TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues);
1160 
1161 	if (per_cpu_io_queues == 0) {
1162 		min_cpus_per_ioq = mp_ncpus;
1163 	}
1164 
1165 	ctrlr->force_intx = 0;
1166 	TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx);
1167 
1168 	/*
1169 	 * FreeBSD currently cannot allocate more than about 190 vectors at
1170 	 *  boot, meaning that systems with high core count and many devices
1171 	 *  requesting per-CPU interrupt vectors will not get their full
1172 	 *  allotment.  So first, try to allocate as many as we may need to
1173 	 *  understand what is available, then immediately release them.
1174 	 *  Then figure out how many of those we will actually use, based on
1175 	 *  assigning an equal number of cores to each I/O queue.
1176 	 */
1177 
1178 	/* One vector for per core I/O queue, plus one vector for admin queue. */
1179 	num_vectors_available = min(pci_msix_count(dev), mp_ncpus + 1);
1180 	if (pci_alloc_msix(dev, &num_vectors_available) != 0) {
1181 		num_vectors_available = 0;
1182 	}
1183 	pci_release_msi(dev);
1184 
1185 	if (ctrlr->force_intx || num_vectors_available < 2) {
1186 		nvme_ctrlr_configure_intx(ctrlr);
1187 		return;
1188 	}
1189 
1190 	/*
1191 	 * Do not use all vectors for I/O queues - one must be saved for the
1192 	 *  admin queue.
1193 	 */
1194 	ctrlr->num_cpus_per_ioq = max(min_cpus_per_ioq,
1195 	    howmany(mp_ncpus, num_vectors_available - 1));
1196 
1197 	ctrlr->num_io_queues = howmany(mp_ncpus, ctrlr->num_cpus_per_ioq);
1198 	num_vectors_requested = ctrlr->num_io_queues + 1;
1199 	num_vectors_allocated = num_vectors_requested;
1200 
1201 	/*
1202 	 * Now just allocate the number of vectors we need.  This should
1203 	 *  succeed, since we previously called pci_alloc_msix()
1204 	 *  successfully returning at least this many vectors, but just to
1205 	 *  be safe, if something goes wrong just revert to INTx.
1206 	 */
1207 	if (pci_alloc_msix(dev, &num_vectors_allocated) != 0) {
1208 		nvme_ctrlr_configure_intx(ctrlr);
1209 		return;
1210 	}
1211 
1212 	if (num_vectors_allocated < num_vectors_requested) {
1213 		pci_release_msi(dev);
1214 		nvme_ctrlr_configure_intx(ctrlr);
1215 		return;
1216 	}
1217 
1218 	ctrlr->msix_enabled = 1;
1219 }
1220 
1221 int
1222 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
1223 {
1224 	struct make_dev_args	md_args;
1225 	uint32_t	cap_lo;
1226 	uint32_t	cap_hi;
1227 	uint8_t		to;
1228 	uint8_t		dstrd;
1229 	uint8_t		mpsmin;
1230 	int		status, timeout_period;
1231 
1232 	ctrlr->dev = dev;
1233 
1234 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
1235 
1236 	status = nvme_ctrlr_allocate_bar(ctrlr);
1237 
1238 	if (status != 0)
1239 		return (status);
1240 
1241 	/*
1242 	 * Software emulators may set the doorbell stride to something
1243 	 *  other than zero, but this driver is not set up to handle that.
1244 	 */
1245 	cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
1246 	dstrd = (cap_hi >> NVME_CAP_HI_REG_DSTRD_SHIFT) & NVME_CAP_HI_REG_DSTRD_MASK;
1247 	if (dstrd != 0)
1248 		return (ENXIO);
1249 
1250 	mpsmin = (cap_hi >> NVME_CAP_HI_REG_MPSMIN_SHIFT) & NVME_CAP_HI_REG_MPSMIN_MASK;
1251 	ctrlr->min_page_size = 1 << (12 + mpsmin);
1252 
1253 	/* Get ready timeout value from controller, in units of 500ms. */
1254 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
1255 	to = (cap_lo >> NVME_CAP_LO_REG_TO_SHIFT) & NVME_CAP_LO_REG_TO_MASK;
1256 	ctrlr->ready_timeout_in_ms = to * 500;
1257 
1258 	timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
1259 	TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
1260 	timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
1261 	timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
1262 	ctrlr->timeout_period = timeout_period;
1263 
1264 	nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
1265 	TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
1266 
1267 	ctrlr->enable_aborts = 0;
1268 	TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
1269 
1270 	nvme_ctrlr_setup_interrupts(ctrlr);
1271 
1272 	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
1273 	if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)
1274 		return (ENXIO);
1275 
1276 	ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1277 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);
1278 	taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq");
1279 
1280 	ctrlr->is_resetting = 0;
1281 	ctrlr->is_initialized = 0;
1282 	ctrlr->notification_sent = 0;
1283 	TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1284 	TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
1285 	STAILQ_INIT(&ctrlr->fail_req);
1286 	ctrlr->is_failed = FALSE;
1287 
1288 	make_dev_args_init(&md_args);
1289 	md_args.mda_devsw = &nvme_ctrlr_cdevsw;
1290 	md_args.mda_uid = UID_ROOT;
1291 	md_args.mda_gid = GID_WHEEL;
1292 	md_args.mda_mode = 0600;
1293 	md_args.mda_unit = device_get_unit(dev);
1294 	md_args.mda_si_drv1 = (void *)ctrlr;
1295 	status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",
1296 	    device_get_unit(dev));
1297 	if (status != 0)
1298 		return (ENXIO);
1299 
1300 	return (0);
1301 }
1302 
1303 void
1304 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1305 {
1306 	int				i;
1307 
1308 	if (ctrlr->resource == NULL)
1309 		goto nores;
1310 
1311 	nvme_notify_fail_consumers(ctrlr);
1312 
1313 	for (i = 0; i < NVME_MAX_NAMESPACES; i++)
1314 		nvme_ns_destruct(&ctrlr->ns[i]);
1315 
1316 	if (ctrlr->cdev)
1317 		destroy_dev(ctrlr->cdev);
1318 
1319 	for (i = 0; i < ctrlr->num_io_queues; i++) {
1320 		nvme_ctrlr_destroy_qpair(ctrlr, &ctrlr->ioq[i]);
1321 		nvme_io_qpair_destroy(&ctrlr->ioq[i]);
1322 	}
1323 	free(ctrlr->ioq, M_NVME);
1324 
1325 	nvme_admin_qpair_destroy(&ctrlr->adminq);
1326 
1327 	/*
1328 	 *  Notify the controller of a shutdown, even though this is due to
1329 	 *   a driver unload, not a system shutdown (this path is not invoked
1330 	 *   during shutdown).  This ensures the controller receives a
1331 	 *   shutdown notification in case the system is shutdown before
1332 	 *   reloading the driver.
1333 	 */
1334 	nvme_ctrlr_shutdown(ctrlr);
1335 
1336 	nvme_ctrlr_disable(ctrlr);
1337 
1338 	if (ctrlr->taskqueue)
1339 		taskqueue_free(ctrlr->taskqueue);
1340 
1341 	if (ctrlr->tag)
1342 		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
1343 
1344 	if (ctrlr->res)
1345 		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
1346 		    rman_get_rid(ctrlr->res), ctrlr->res);
1347 
1348 	if (ctrlr->msix_enabled)
1349 		pci_release_msi(dev);
1350 
1351 	if (ctrlr->bar4_resource != NULL) {
1352 		bus_release_resource(dev, SYS_RES_MEMORY,
1353 		    ctrlr->bar4_resource_id, ctrlr->bar4_resource);
1354 	}
1355 
1356 	bus_release_resource(dev, SYS_RES_MEMORY,
1357 	    ctrlr->resource_id, ctrlr->resource);
1358 
1359 nores:
1360 	mtx_destroy(&ctrlr->lock);
1361 }
1362 
1363 void
1364 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
1365 {
1366 	uint32_t	cc;
1367 	uint32_t	csts;
1368 	int		ticks = 0;
1369 
1370 	cc = nvme_mmio_read_4(ctrlr, cc);
1371 	cc &= ~(NVME_CC_REG_SHN_MASK << NVME_CC_REG_SHN_SHIFT);
1372 	cc |= NVME_SHN_NORMAL << NVME_CC_REG_SHN_SHIFT;
1373 	nvme_mmio_write_4(ctrlr, cc, cc);
1374 
1375 	csts = nvme_mmio_read_4(ctrlr, csts);
1376 	while ((NVME_CSTS_GET_SHST(csts) != NVME_SHST_COMPLETE) && (ticks++ < 5*hz)) {
1377 		pause("nvme shn", 1);
1378 		csts = nvme_mmio_read_4(ctrlr, csts);
1379 	}
1380 	if (NVME_CSTS_GET_SHST(csts) != NVME_SHST_COMPLETE)
1381 		nvme_printf(ctrlr, "did not complete shutdown within 5 seconds "
1382 		    "of notification\n");
1383 }
1384 
1385 void
1386 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
1387     struct nvme_request *req)
1388 {
1389 
1390 	nvme_qpair_submit_request(&ctrlr->adminq, req);
1391 }
1392 
1393 void
1394 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
1395     struct nvme_request *req)
1396 {
1397 	struct nvme_qpair       *qpair;
1398 
1399 	qpair = &ctrlr->ioq[curcpu / ctrlr->num_cpus_per_ioq];
1400 	nvme_qpair_submit_request(qpair, req);
1401 }
1402 
1403 device_t
1404 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
1405 {
1406 
1407 	return (ctrlr->dev);
1408 }
1409 
1410 const struct nvme_controller_data *
1411 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
1412 {
1413 
1414 	return (&ctrlr->cdata);
1415 }
1416