xref: /freebsd/sys/dev/nvme/nvme_ctrlr.c (revision 4ec234c813eed05c166859bba82c882e40826eb9)
1 /*-
2  * Copyright (C) 2012-2013 Intel Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/buf.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/ioccom.h>
36 #include <sys/proc.h>
37 #include <sys/smp.h>
38 #include <sys/uio.h>
39 
40 #include <dev/pci/pcireg.h>
41 #include <dev/pci/pcivar.h>
42 
43 #include "nvme_private.h"
44 
45 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
46 						struct nvme_async_event_request *aer);
47 
48 static int
49 nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr)
50 {
51 
52 	/* Chatham puts the NVMe MMRs behind BAR 2/3, not BAR 0/1. */
53 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
54 		ctrlr->resource_id = PCIR_BAR(2);
55 	else
56 		ctrlr->resource_id = PCIR_BAR(0);
57 
58 	ctrlr->resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY,
59 	    &ctrlr->resource_id, 0, ~0, 1, RF_ACTIVE);
60 
61 	if(ctrlr->resource == NULL) {
62 		nvme_printf(ctrlr, "unable to allocate pci resource\n");
63 		return (ENOMEM);
64 	}
65 
66 	ctrlr->bus_tag = rman_get_bustag(ctrlr->resource);
67 	ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource);
68 	ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle;
69 
70 	/*
71 	 * The NVMe spec allows for the MSI-X table to be placed behind
72 	 *  BAR 4/5, separate from the control/doorbell registers.  Always
73 	 *  try to map this bar, because it must be mapped prior to calling
74 	 *  pci_alloc_msix().  If the table isn't behind BAR 4/5,
75 	 *  bus_alloc_resource() will just return NULL which is OK.
76 	 */
77 	ctrlr->bar4_resource_id = PCIR_BAR(4);
78 	ctrlr->bar4_resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY,
79 	    &ctrlr->bar4_resource_id, 0, ~0, 1, RF_ACTIVE);
80 
81 	return (0);
82 }
83 
84 #ifdef CHATHAM2
85 static int
86 nvme_ctrlr_allocate_chatham_bar(struct nvme_controller *ctrlr)
87 {
88 
89 	ctrlr->chatham_resource_id = PCIR_BAR(CHATHAM_CONTROL_BAR);
90 	ctrlr->chatham_resource = bus_alloc_resource(ctrlr->dev,
91 	    SYS_RES_MEMORY, &ctrlr->chatham_resource_id, 0, ~0, 1,
92 	    RF_ACTIVE);
93 
94 	if(ctrlr->chatham_resource == NULL) {
95 		nvme_printf(ctrlr, "unable to alloc pci resource\n");
96 		return (ENOMEM);
97 	}
98 
99 	ctrlr->chatham_bus_tag = rman_get_bustag(ctrlr->chatham_resource);
100 	ctrlr->chatham_bus_handle =
101 	    rman_get_bushandle(ctrlr->chatham_resource);
102 
103 	return (0);
104 }
105 
106 static void
107 nvme_ctrlr_setup_chatham(struct nvme_controller *ctrlr)
108 {
109 	uint64_t reg1, reg2, reg3;
110 	uint64_t temp1, temp2;
111 	uint32_t temp3;
112 	uint32_t use_flash_timings = 0;
113 
114 	DELAY(10000);
115 
116 	temp3 = chatham_read_4(ctrlr, 0x8080);
117 
118 	device_printf(ctrlr->dev, "Chatham version: 0x%x\n", temp3);
119 
120 	ctrlr->chatham_lbas = chatham_read_4(ctrlr, 0x8068) - 0x110;
121 	ctrlr->chatham_size = ctrlr->chatham_lbas * 512;
122 
123 	device_printf(ctrlr->dev, "Chatham size: %jd\n",
124 	    (intmax_t)ctrlr->chatham_size);
125 
126 	reg1 = reg2 = reg3 = ctrlr->chatham_size - 1;
127 
128 	TUNABLE_INT_FETCH("hw.nvme.use_flash_timings", &use_flash_timings);
129 	if (use_flash_timings) {
130 		device_printf(ctrlr->dev, "Chatham: using flash timings\n");
131 		temp1 = 0x00001b58000007d0LL;
132 		temp2 = 0x000000cb00000131LL;
133 	} else {
134 		device_printf(ctrlr->dev, "Chatham: using DDR timings\n");
135 		temp1 = temp2 = 0x0LL;
136 	}
137 
138 	chatham_write_8(ctrlr, 0x8000, reg1);
139 	chatham_write_8(ctrlr, 0x8008, reg2);
140 	chatham_write_8(ctrlr, 0x8010, reg3);
141 
142 	chatham_write_8(ctrlr, 0x8020, temp1);
143 	temp3 = chatham_read_4(ctrlr, 0x8020);
144 
145 	chatham_write_8(ctrlr, 0x8028, temp2);
146 	temp3 = chatham_read_4(ctrlr, 0x8028);
147 
148 	chatham_write_8(ctrlr, 0x8030, temp1);
149 	chatham_write_8(ctrlr, 0x8038, temp2);
150 	chatham_write_8(ctrlr, 0x8040, temp1);
151 	chatham_write_8(ctrlr, 0x8048, temp2);
152 	chatham_write_8(ctrlr, 0x8050, temp1);
153 	chatham_write_8(ctrlr, 0x8058, temp2);
154 
155 	DELAY(10000);
156 }
157 
158 static void
159 nvme_chatham_populate_cdata(struct nvme_controller *ctrlr)
160 {
161 	struct nvme_controller_data *cdata;
162 
163 	cdata = &ctrlr->cdata;
164 
165 	cdata->vid = 0x8086;
166 	cdata->ssvid = 0x2011;
167 
168 	/*
169 	 * Chatham2 puts garbage data in these fields when we
170 	 *  invoke IDENTIFY_CONTROLLER, so we need to re-zero
171 	 *  the fields before calling bcopy().
172 	 */
173 	memset(cdata->sn, 0, sizeof(cdata->sn));
174 	memcpy(cdata->sn, "2012", strlen("2012"));
175 	memset(cdata->mn, 0, sizeof(cdata->mn));
176 	memcpy(cdata->mn, "CHATHAM2", strlen("CHATHAM2"));
177 	memset(cdata->fr, 0, sizeof(cdata->fr));
178 	memcpy(cdata->fr, "0", strlen("0"));
179 	cdata->rab = 8;
180 	cdata->aerl = 3;
181 	cdata->lpa.ns_smart = 1;
182 	cdata->sqes.min = 6;
183 	cdata->sqes.max = 6;
184 	cdata->cqes.min = 4;
185 	cdata->cqes.max = 4;
186 	cdata->nn = 1;
187 
188 	/* Chatham2 doesn't support DSM command */
189 	cdata->oncs.dsm = 0;
190 
191 	cdata->vwc.present = 1;
192 }
193 #endif /* CHATHAM2 */
194 
195 static void
196 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
197 {
198 	struct nvme_qpair	*qpair;
199 	uint32_t		num_entries;
200 
201 	qpair = &ctrlr->adminq;
202 
203 	num_entries = NVME_ADMIN_ENTRIES;
204 	TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
205 	/*
206 	 * If admin_entries was overridden to an invalid value, revert it
207 	 *  back to our default value.
208 	 */
209 	if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
210 	    num_entries > NVME_MAX_ADMIN_ENTRIES) {
211 		nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
212 		    "specified\n", num_entries);
213 		num_entries = NVME_ADMIN_ENTRIES;
214 	}
215 
216 	/*
217 	 * The admin queue's max xfer size is treated differently than the
218 	 *  max I/O xfer size.  16KB is sufficient here - maybe even less?
219 	 */
220 	nvme_qpair_construct(qpair,
221 			     0, /* qpair ID */
222 			     0, /* vector */
223 			     num_entries,
224 			     NVME_ADMIN_TRACKERS,
225 			     ctrlr);
226 }
227 
228 static int
229 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
230 {
231 	struct nvme_qpair	*qpair;
232 	union cap_lo_register	cap_lo;
233 	int			i, num_entries, num_trackers;
234 
235 	num_entries = NVME_IO_ENTRIES;
236 	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
237 
238 	/*
239 	 * NVMe spec sets a hard limit of 64K max entries, but
240 	 *  devices may specify a smaller limit, so we need to check
241 	 *  the MQES field in the capabilities register.
242 	 */
243 	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo);
244 	num_entries = min(num_entries, cap_lo.bits.mqes+1);
245 
246 	num_trackers = NVME_IO_TRACKERS;
247 	TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
248 
249 	num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
250 	num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
251 	/*
252 	 * No need to have more trackers than entries in the submit queue.
253 	 *  Note also that for a queue size of N, we can only have (N-1)
254 	 *  commands outstanding, hence the "-1" here.
255 	 */
256 	num_trackers = min(num_trackers, (num_entries-1));
257 
258 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
259 	    M_NVME, M_ZERO | M_WAITOK);
260 
261 	for (i = 0; i < ctrlr->num_io_queues; i++) {
262 		qpair = &ctrlr->ioq[i];
263 
264 		/*
265 		 * Admin queue has ID=0. IO queues start at ID=1 -
266 		 *  hence the 'i+1' here.
267 		 *
268 		 * For I/O queues, use the controller-wide max_xfer_size
269 		 *  calculated in nvme_attach().
270 		 */
271 		nvme_qpair_construct(qpair,
272 				     i+1, /* qpair ID */
273 				     ctrlr->msix_enabled ? i+1 : 0, /* vector */
274 				     num_entries,
275 				     num_trackers,
276 				     ctrlr);
277 
278 		if (ctrlr->per_cpu_io_queues)
279 			bus_bind_intr(ctrlr->dev, qpair->res, i);
280 	}
281 
282 	return (0);
283 }
284 
285 static void
286 nvme_ctrlr_fail(struct nvme_controller *ctrlr)
287 {
288 	int i;
289 
290 	ctrlr->is_failed = TRUE;
291 	nvme_qpair_fail(&ctrlr->adminq);
292 	for (i = 0; i < ctrlr->num_io_queues; i++)
293 		nvme_qpair_fail(&ctrlr->ioq[i]);
294 	nvme_notify_fail_consumers(ctrlr);
295 }
296 
297 void
298 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
299     struct nvme_request *req)
300 {
301 
302 	mtx_lock(&ctrlr->lock);
303 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
304 	mtx_unlock(&ctrlr->lock);
305 	taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
306 }
307 
308 static void
309 nvme_ctrlr_fail_req_task(void *arg, int pending)
310 {
311 	struct nvme_controller	*ctrlr = arg;
312 	struct nvme_request	*req;
313 
314 	mtx_lock(&ctrlr->lock);
315 	while (!STAILQ_EMPTY(&ctrlr->fail_req)) {
316 		req = STAILQ_FIRST(&ctrlr->fail_req);
317 		STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
318 		nvme_qpair_manual_complete_request(req->qpair, req,
319 		    NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE);
320 	}
321 	mtx_unlock(&ctrlr->lock);
322 }
323 
324 static int
325 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr)
326 {
327 	int ms_waited;
328 	union cc_register cc;
329 	union csts_register csts;
330 
331 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
332 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
333 
334 	if (!cc.bits.en) {
335 		nvme_printf(ctrlr, "%s called with cc.en = 0\n", __func__);
336 		return (ENXIO);
337 	}
338 
339 	ms_waited = 0;
340 
341 	while (!csts.bits.rdy) {
342 		DELAY(1000);
343 		if (ms_waited++ > ctrlr->ready_timeout_in_ms) {
344 			nvme_printf(ctrlr, "controller did not become ready "
345 			    "within %d ms\n", ctrlr->ready_timeout_in_ms);
346 			return (ENXIO);
347 		}
348 		csts.raw = nvme_mmio_read_4(ctrlr, csts);
349 	}
350 
351 	return (0);
352 }
353 
354 static void
355 nvme_ctrlr_disable(struct nvme_controller *ctrlr)
356 {
357 	union cc_register cc;
358 	union csts_register csts;
359 
360 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
361 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
362 
363 	if (cc.bits.en == 1 && csts.bits.rdy == 0)
364 		nvme_ctrlr_wait_for_ready(ctrlr);
365 
366 	cc.bits.en = 0;
367 	nvme_mmio_write_4(ctrlr, cc, cc.raw);
368 	DELAY(5000);
369 }
370 
371 static int
372 nvme_ctrlr_enable(struct nvme_controller *ctrlr)
373 {
374 	union cc_register	cc;
375 	union csts_register	csts;
376 	union aqa_register	aqa;
377 
378 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
379 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
380 
381 	if (cc.bits.en == 1) {
382 		if (csts.bits.rdy == 1)
383 			return (0);
384 		else
385 			return (nvme_ctrlr_wait_for_ready(ctrlr));
386 	}
387 
388 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
389 	DELAY(5000);
390 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
391 	DELAY(5000);
392 
393 	aqa.raw = 0;
394 	/* acqs and asqs are 0-based. */
395 	aqa.bits.acqs = ctrlr->adminq.num_entries-1;
396 	aqa.bits.asqs = ctrlr->adminq.num_entries-1;
397 	nvme_mmio_write_4(ctrlr, aqa, aqa.raw);
398 	DELAY(5000);
399 
400 	cc.bits.en = 1;
401 	cc.bits.css = 0;
402 	cc.bits.ams = 0;
403 	cc.bits.shn = 0;
404 	cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
405 	cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
406 
407 	/* This evaluates to 0, which is according to spec. */
408 	cc.bits.mps = (PAGE_SIZE >> 13);
409 
410 	nvme_mmio_write_4(ctrlr, cc, cc.raw);
411 	DELAY(5000);
412 
413 	return (nvme_ctrlr_wait_for_ready(ctrlr));
414 }
415 
416 int
417 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
418 {
419 	int i;
420 
421 	nvme_admin_qpair_disable(&ctrlr->adminq);
422 	for (i = 0; i < ctrlr->num_io_queues; i++)
423 		nvme_io_qpair_disable(&ctrlr->ioq[i]);
424 
425 	DELAY(100*1000);
426 
427 	nvme_ctrlr_disable(ctrlr);
428 	return (nvme_ctrlr_enable(ctrlr));
429 }
430 
431 void
432 nvme_ctrlr_reset(struct nvme_controller *ctrlr)
433 {
434 	int cmpset;
435 
436 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
437 
438 	if (cmpset == 0 || ctrlr->is_failed)
439 		/*
440 		 * Controller is already resetting or has failed.  Return
441 		 *  immediately since there is no need to kick off another
442 		 *  reset in these cases.
443 		 */
444 		return;
445 
446 	taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
447 }
448 
449 static int
450 nvme_ctrlr_identify(struct nvme_controller *ctrlr)
451 {
452 	struct nvme_completion_poll_status	status;
453 
454 	status.done = FALSE;
455 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
456 	    nvme_completion_poll_cb, &status);
457 	while (status.done == FALSE)
458 		pause("nvme", 1);
459 	if (nvme_completion_is_error(&status.cpl)) {
460 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
461 		return (ENXIO);
462 	}
463 
464 #ifdef CHATHAM2
465 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
466 		nvme_chatham_populate_cdata(ctrlr);
467 #endif
468 
469 	/*
470 	 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
471 	 *  controller supports.
472 	 */
473 	if (ctrlr->cdata.mdts > 0)
474 		ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
475 		    ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
476 
477 	return (0);
478 }
479 
480 static int
481 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
482 {
483 	struct nvme_completion_poll_status	status;
484 	int					cq_allocated, i, sq_allocated;
485 
486 	status.done = FALSE;
487 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
488 	    nvme_completion_poll_cb, &status);
489 	while (status.done == FALSE)
490 		pause("nvme", 1);
491 	if (nvme_completion_is_error(&status.cpl)) {
492 		nvme_printf(ctrlr, "nvme_set_num_queues failed!\n");
493 		return (ENXIO);
494 	}
495 
496 	/*
497 	 * Data in cdw0 is 0-based.
498 	 * Lower 16-bits indicate number of submission queues allocated.
499 	 * Upper 16-bits indicate number of completion queues allocated.
500 	 */
501 	sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
502 	cq_allocated = (status.cpl.cdw0 >> 16) + 1;
503 
504 	/*
505 	 * Check that the controller was able to allocate the number of
506 	 *  queues we requested.  If not, revert to one IO queue pair.
507 	 */
508 	if (sq_allocated < ctrlr->num_io_queues ||
509 	    cq_allocated < ctrlr->num_io_queues) {
510 
511 		/*
512 		 * Destroy extra IO queue pairs that were created at
513 		 *  controller construction time but are no longer
514 		 *  needed.  This will only happen when a controller
515 		 *  supports fewer queues than MSI-X vectors.  This
516 		 *  is not the normal case, but does occur with the
517 		 *  Chatham prototype board.
518 		 */
519 		for (i = 1; i < ctrlr->num_io_queues; i++)
520 			nvme_io_qpair_destroy(&ctrlr->ioq[i]);
521 
522 		ctrlr->num_io_queues = 1;
523 		ctrlr->per_cpu_io_queues = 0;
524 	}
525 
526 	return (0);
527 }
528 
529 static int
530 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
531 {
532 	struct nvme_completion_poll_status	status;
533 	struct nvme_qpair			*qpair;
534 	int					i;
535 
536 	for (i = 0; i < ctrlr->num_io_queues; i++) {
537 		qpair = &ctrlr->ioq[i];
538 
539 		status.done = FALSE;
540 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
541 		    nvme_completion_poll_cb, &status);
542 		while (status.done == FALSE)
543 			pause("nvme", 1);
544 		if (nvme_completion_is_error(&status.cpl)) {
545 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
546 			return (ENXIO);
547 		}
548 
549 		status.done = FALSE;
550 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
551 		    nvme_completion_poll_cb, &status);
552 		while (status.done == FALSE)
553 			pause("nvme", 1);
554 		if (nvme_completion_is_error(&status.cpl)) {
555 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
556 			return (ENXIO);
557 		}
558 	}
559 
560 	return (0);
561 }
562 
563 static int
564 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
565 {
566 	struct nvme_namespace	*ns;
567 	int			i, status;
568 
569 	for (i = 0; i < ctrlr->cdata.nn; i++) {
570 		ns = &ctrlr->ns[i];
571 		status = nvme_ns_construct(ns, i+1, ctrlr);
572 		if (status != 0)
573 			return (status);
574 	}
575 
576 	return (0);
577 }
578 
579 static boolean_t
580 is_log_page_id_valid(uint8_t page_id)
581 {
582 
583 	switch (page_id) {
584 	case NVME_LOG_ERROR:
585 	case NVME_LOG_HEALTH_INFORMATION:
586 	case NVME_LOG_FIRMWARE_SLOT:
587 		return (TRUE);
588 	}
589 
590 	return (FALSE);
591 }
592 
593 static uint32_t
594 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
595 {
596 	uint32_t	log_page_size;
597 
598 	switch (page_id) {
599 	case NVME_LOG_ERROR:
600 		log_page_size = min(
601 		    sizeof(struct nvme_error_information_entry) *
602 		    ctrlr->cdata.elpe,
603 		    NVME_MAX_AER_LOG_SIZE);
604 		break;
605 	case NVME_LOG_HEALTH_INFORMATION:
606 		log_page_size = sizeof(struct nvme_health_information_page);
607 		break;
608 	case NVME_LOG_FIRMWARE_SLOT:
609 		log_page_size = sizeof(struct nvme_firmware_page);
610 		break;
611 	default:
612 		log_page_size = 0;
613 		break;
614 	}
615 
616 	return (log_page_size);
617 }
618 
619 static void
620 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
621     union nvme_critical_warning_state state)
622 {
623 
624 	if (state.bits.available_spare == 1)
625 		nvme_printf(ctrlr, "available spare space below threshold\n");
626 
627 	if (state.bits.temperature == 1)
628 		nvme_printf(ctrlr, "temperature above threshold\n");
629 
630 	if (state.bits.device_reliability == 1)
631 		nvme_printf(ctrlr, "device reliability degraded\n");
632 
633 	if (state.bits.read_only == 1)
634 		nvme_printf(ctrlr, "media placed in read only mode\n");
635 
636 	if (state.bits.volatile_memory_backup == 1)
637 		nvme_printf(ctrlr, "volatile memory backup device failed\n");
638 
639 	if (state.bits.reserved != 0)
640 		nvme_printf(ctrlr,
641 		    "unknown critical warning(s): state = 0x%02x\n", state.raw);
642 }
643 
644 static void
645 nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl)
646 {
647 	struct nvme_async_event_request		*aer = arg;
648 	struct nvme_health_information_page	*health_info;
649 
650 	/*
651 	 * If the log page fetch for some reason completed with an error,
652 	 *  don't pass log page data to the consumers.  In practice, this case
653 	 *  should never happen.
654 	 */
655 	if (nvme_completion_is_error(cpl))
656 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
657 		    aer->log_page_id, NULL, 0);
658 	else {
659 		if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
660 			health_info = (struct nvme_health_information_page *)
661 			    aer->log_page_buffer;
662 			nvme_ctrlr_log_critical_warnings(aer->ctrlr,
663 			    health_info->critical_warning);
664 			/*
665 			 * Critical warnings reported through the
666 			 *  SMART/health log page are persistent, so
667 			 *  clear the associated bits in the async event
668 			 *  config so that we do not receive repeated
669 			 *  notifications for the same event.
670 			 */
671 			aer->ctrlr->async_event_config.raw &=
672 			    ~health_info->critical_warning.raw;
673 			nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
674 			    aer->ctrlr->async_event_config, NULL, NULL);
675 		}
676 
677 
678 		/*
679 		 * Pass the cpl data from the original async event completion,
680 		 *  not the log page fetch.
681 		 */
682 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
683 		    aer->log_page_id, aer->log_page_buffer, aer->log_page_size);
684 	}
685 
686 	/*
687 	 * Repost another asynchronous event request to replace the one
688 	 *  that just completed.
689 	 */
690 	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
691 }
692 
693 static void
694 nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
695 {
696 	struct nvme_async_event_request	*aer = arg;
697 
698 	if (nvme_completion_is_error(cpl)) {
699 		/*
700 		 *  Do not retry failed async event requests.  This avoids
701 		 *  infinite loops where a new async event request is submitted
702 		 *  to replace the one just failed, only to fail again and
703 		 *  perpetuate the loop.
704 		 */
705 		return;
706 	}
707 
708 	/* Associated log page is in bits 23:16 of completion entry dw0. */
709 	aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
710 
711 	nvme_printf(aer->ctrlr, "async event occurred (log page id=0x%x)\n",
712 	    aer->log_page_id);
713 
714 	if (is_log_page_id_valid(aer->log_page_id)) {
715 		aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,
716 		    aer->log_page_id);
717 		memcpy(&aer->cpl, cpl, sizeof(*cpl));
718 		nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
719 		    NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer,
720 		    aer->log_page_size, nvme_ctrlr_async_event_log_page_cb,
721 		    aer);
722 		/* Wait to notify consumers until after log page is fetched. */
723 	} else {
724 		nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,
725 		    NULL, 0);
726 
727 		/*
728 		 * Repost another asynchronous event request to replace the one
729 		 *  that just completed.
730 		 */
731 		nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
732 	}
733 }
734 
735 static void
736 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
737     struct nvme_async_event_request *aer)
738 {
739 	struct nvme_request *req;
740 
741 	aer->ctrlr = ctrlr;
742 	req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
743 	aer->req = req;
744 
745 	/*
746 	 * Disable timeout here, since asynchronous event requests should by
747 	 *  nature never be timed out.
748 	 */
749 	req->timeout = FALSE;
750 	req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
751 	nvme_ctrlr_submit_admin_request(ctrlr, req);
752 }
753 
754 static void
755 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
756 {
757 	struct nvme_completion_poll_status	status;
758 	struct nvme_async_event_request		*aer;
759 	uint32_t				i;
760 
761 	ctrlr->async_event_config.raw = 0xFF;
762 	ctrlr->async_event_config.bits.reserved = 0;
763 
764 	status.done = FALSE;
765 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
766 	    0, NULL, 0, nvme_completion_poll_cb, &status);
767 	while (status.done == FALSE)
768 		pause("nvme", 1);
769 	if (nvme_completion_is_error(&status.cpl) ||
770 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
771 	    (status.cpl.cdw0 & 0xFFFF) == 0x0000) {
772 		nvme_printf(ctrlr, "temperature threshold not supported\n");
773 		ctrlr->async_event_config.bits.temperature = 0;
774 	}
775 
776 	nvme_ctrlr_cmd_set_async_event_config(ctrlr,
777 	    ctrlr->async_event_config, NULL, NULL);
778 
779 	/* aerl is a zero-based value, so we need to add 1 here. */
780 	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
781 
782 	/* Chatham doesn't support AERs. */
783 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
784 		ctrlr->num_aers = 0;
785 
786 	for (i = 0; i < ctrlr->num_aers; i++) {
787 		aer = &ctrlr->aer[i];
788 		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
789 	}
790 }
791 
792 static void
793 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
794 {
795 
796 	ctrlr->int_coal_time = 0;
797 	TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
798 	    &ctrlr->int_coal_time);
799 
800 	ctrlr->int_coal_threshold = 0;
801 	TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
802 	    &ctrlr->int_coal_threshold);
803 
804 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
805 	    ctrlr->int_coal_threshold, NULL, NULL);
806 }
807 
808 static void
809 nvme_ctrlr_start(void *ctrlr_arg)
810 {
811 	struct nvme_controller *ctrlr = ctrlr_arg;
812 	int i;
813 
814 	nvme_qpair_reset(&ctrlr->adminq);
815 	for (i = 0; i < ctrlr->num_io_queues; i++)
816 		nvme_qpair_reset(&ctrlr->ioq[i]);
817 
818 	nvme_admin_qpair_enable(&ctrlr->adminq);
819 
820 	if (nvme_ctrlr_identify(ctrlr) != 0) {
821 		nvme_ctrlr_fail(ctrlr);
822 		return;
823 	}
824 
825 	if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
826 		nvme_ctrlr_fail(ctrlr);
827 		return;
828 	}
829 
830 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
831 		nvme_ctrlr_fail(ctrlr);
832 		return;
833 	}
834 
835 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
836 		nvme_ctrlr_fail(ctrlr);
837 		return;
838 	}
839 
840 	nvme_ctrlr_configure_aer(ctrlr);
841 	nvme_ctrlr_configure_int_coalescing(ctrlr);
842 
843 	for (i = 0; i < ctrlr->num_io_queues; i++)
844 		nvme_io_qpair_enable(&ctrlr->ioq[i]);
845 
846 	/*
847 	 * Clear software progress marker to 0, to indicate to pre-boot
848 	 *  software that OS driver load was successful.
849 	 *
850 	 * Chatham does not support this feature.
851 	 */
852 	if (pci_get_devid(ctrlr->dev) != CHATHAM_PCI_ID)
853 		nvme_ctrlr_cmd_set_feature(ctrlr,
854 		    NVME_FEAT_SOFTWARE_PROGRESS_MARKER, 0, NULL, 0, NULL, NULL);
855 }
856 
857 void
858 nvme_ctrlr_start_config_hook(void *arg)
859 {
860 	struct nvme_controller *ctrlr = arg;
861 
862 	nvme_ctrlr_start(ctrlr);
863 	config_intrhook_disestablish(&ctrlr->config_hook);
864 }
865 
866 static void
867 nvme_ctrlr_reset_task(void *arg, int pending)
868 {
869 	struct nvme_controller	*ctrlr = arg;
870 	int			status;
871 
872 	nvme_printf(ctrlr, "resetting controller\n");
873 	status = nvme_ctrlr_hw_reset(ctrlr);
874 	/*
875 	 * Use pause instead of DELAY, so that we yield to any nvme interrupt
876 	 *  handlers on this CPU that were blocked on a qpair lock. We want
877 	 *  all nvme interrupts completed before proceeding with restarting the
878 	 *  controller.
879 	 *
880 	 * XXX - any way to guarantee the interrupt handlers have quiesced?
881 	 */
882 	pause("nvmereset", hz / 10);
883 	if (status == 0)
884 		nvme_ctrlr_start(ctrlr);
885 	else
886 		nvme_ctrlr_fail(ctrlr);
887 
888 	atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
889 }
890 
891 static void
892 nvme_ctrlr_intx_handler(void *arg)
893 {
894 	struct nvme_controller *ctrlr = arg;
895 
896 	nvme_mmio_write_4(ctrlr, intms, 1);
897 
898 	nvme_qpair_process_completions(&ctrlr->adminq);
899 
900 	if (ctrlr->ioq[0].cpl)
901 		nvme_qpair_process_completions(&ctrlr->ioq[0]);
902 
903 	nvme_mmio_write_4(ctrlr, intmc, 1);
904 }
905 
906 static int
907 nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr)
908 {
909 
910 	ctrlr->num_io_queues = 1;
911 	ctrlr->per_cpu_io_queues = 0;
912 	ctrlr->rid = 0;
913 	ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
914 	    &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE);
915 
916 	if (ctrlr->res == NULL) {
917 		nvme_printf(ctrlr, "unable to allocate shared IRQ\n");
918 		return (ENOMEM);
919 	}
920 
921 	bus_setup_intr(ctrlr->dev, ctrlr->res,
922 	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler,
923 	    ctrlr, &ctrlr->tag);
924 
925 	if (ctrlr->tag == NULL) {
926 		nvme_printf(ctrlr, "unable to setup intx handler\n");
927 		return (ENOMEM);
928 	}
929 
930 	return (0);
931 }
932 
933 static void
934 nvme_pt_done(void *arg, const struct nvme_completion *cpl)
935 {
936 	struct nvme_pt_command *pt = arg;
937 
938 	bzero(&pt->cpl, sizeof(pt->cpl));
939 	pt->cpl.cdw0 = cpl->cdw0;
940 	pt->cpl.status = cpl->status;
941 	pt->cpl.status.p = 0;
942 
943 	mtx_lock(pt->driver_lock);
944 	wakeup(pt);
945 	mtx_unlock(pt->driver_lock);
946 }
947 
948 int
949 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
950     struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
951     int is_admin_cmd)
952 {
953 	struct nvme_request	*req;
954 	struct mtx		*mtx;
955 	struct buf		*buf = NULL;
956 	int			ret = 0;
957 
958 	if (pt->len > 0) {
959 		if (pt->len > ctrlr->max_xfer_size) {
960 			nvme_printf(ctrlr, "pt->len (%d) "
961 			    "exceeds max_xfer_size (%d)\n", pt->len,
962 			    ctrlr->max_xfer_size);
963 			return EIO;
964 		}
965 		if (is_user_buffer) {
966 			/*
967 			 * Ensure the user buffer is wired for the duration of
968 			 *  this passthrough command.
969 			 */
970 			PHOLD(curproc);
971 			buf = getpbuf(NULL);
972 			buf->b_saveaddr = buf->b_data;
973 			buf->b_data = pt->buf;
974 			buf->b_bufsize = pt->len;
975 			buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
976 #ifdef NVME_UNMAPPED_BIO_SUPPORT
977 			if (vmapbuf(buf, 1) < 0) {
978 #else
979 			if (vmapbuf(buf) < 0) {
980 #endif
981 				ret = EFAULT;
982 				goto err;
983 			}
984 			req = nvme_allocate_request_vaddr(buf->b_data, pt->len,
985 			    nvme_pt_done, pt);
986 		} else
987 			req = nvme_allocate_request_vaddr(pt->buf, pt->len,
988 			    nvme_pt_done, pt);
989 	} else
990 		req = nvme_allocate_request_null(nvme_pt_done, pt);
991 
992 	req->cmd.opc	= pt->cmd.opc;
993 	req->cmd.cdw10	= pt->cmd.cdw10;
994 	req->cmd.cdw11	= pt->cmd.cdw11;
995 	req->cmd.cdw12	= pt->cmd.cdw12;
996 	req->cmd.cdw13	= pt->cmd.cdw13;
997 	req->cmd.cdw14	= pt->cmd.cdw14;
998 	req->cmd.cdw15	= pt->cmd.cdw15;
999 
1000 	req->cmd.nsid = nsid;
1001 
1002 	if (is_admin_cmd)
1003 		mtx = &ctrlr->lock;
1004 	else
1005 		mtx = &ctrlr->ns[nsid-1].lock;
1006 
1007 	mtx_lock(mtx);
1008 	pt->driver_lock = mtx;
1009 
1010 	if (is_admin_cmd)
1011 		nvme_ctrlr_submit_admin_request(ctrlr, req);
1012 	else
1013 		nvme_ctrlr_submit_io_request(ctrlr, req);
1014 
1015 	mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
1016 	mtx_unlock(mtx);
1017 
1018 	pt->driver_lock = NULL;
1019 
1020 err:
1021 	if (buf != NULL) {
1022 		relpbuf(buf, NULL);
1023 		PRELE(curproc);
1024 	}
1025 
1026 	return (ret);
1027 }
1028 
1029 static int
1030 nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
1031     struct thread *td)
1032 {
1033 	struct nvme_controller			*ctrlr;
1034 	struct nvme_pt_command			*pt;
1035 
1036 	ctrlr = cdev->si_drv1;
1037 
1038 	switch (cmd) {
1039 	case NVME_RESET_CONTROLLER:
1040 		nvme_ctrlr_reset(ctrlr);
1041 		break;
1042 	case NVME_PASSTHROUGH_CMD:
1043 		pt = (struct nvme_pt_command *)arg;
1044 #ifdef CHATHAM2
1045 		/*
1046 		 * Chatham IDENTIFY data is spoofed, so copy the spoofed data
1047 		 *  rather than issuing the command to the Chatham controller.
1048 		 */
1049 		if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID &&
1050                     pt->cmd.opc == NVME_OPC_IDENTIFY) {
1051 			if (pt->cmd.cdw10 == 1) {
1052                         	if (pt->len != sizeof(ctrlr->cdata))
1053                                 	return (EINVAL);
1054                         	return (copyout(&ctrlr->cdata, pt->buf,
1055 				    pt->len));
1056 			} else {
1057 				if (pt->len != sizeof(ctrlr->ns[0].data) ||
1058 				    pt->cmd.nsid != 1)
1059 					return (EINVAL);
1060 				return (copyout(&ctrlr->ns[0].data, pt->buf,
1061 				    pt->len));
1062 			}
1063 		}
1064 #endif
1065 		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, pt->cmd.nsid,
1066 		    1 /* is_user_buffer */, 1 /* is_admin_cmd */));
1067 	default:
1068 		return (ENOTTY);
1069 	}
1070 
1071 	return (0);
1072 }
1073 
1074 static struct cdevsw nvme_ctrlr_cdevsw = {
1075 	.d_version =	D_VERSION,
1076 	.d_flags =	0,
1077 	.d_ioctl =	nvme_ctrlr_ioctl
1078 };
1079 
1080 int
1081 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
1082 {
1083 	union cap_lo_register	cap_lo;
1084 	union cap_hi_register	cap_hi;
1085 	int			num_vectors, per_cpu_io_queues, status = 0;
1086 	int			timeout_period;
1087 
1088 	ctrlr->dev = dev;
1089 
1090 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
1091 
1092 	status = nvme_ctrlr_allocate_bar(ctrlr);
1093 
1094 	if (status != 0)
1095 		return (status);
1096 
1097 #ifdef CHATHAM2
1098 	if (pci_get_devid(dev) == CHATHAM_PCI_ID) {
1099 		status = nvme_ctrlr_allocate_chatham_bar(ctrlr);
1100 		if (status != 0)
1101 			return (status);
1102 		nvme_ctrlr_setup_chatham(ctrlr);
1103 	}
1104 #endif
1105 
1106 	/*
1107 	 * Software emulators may set the doorbell stride to something
1108 	 *  other than zero, but this driver is not set up to handle that.
1109 	 */
1110 	cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi);
1111 	if (cap_hi.bits.dstrd != 0)
1112 		return (ENXIO);
1113 
1114 	ctrlr->min_page_size = 1 << (12 + cap_hi.bits.mpsmin);
1115 
1116 	/* Get ready timeout value from controller, in units of 500ms. */
1117 	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo);
1118 	ctrlr->ready_timeout_in_ms = cap_lo.bits.to * 500;
1119 
1120 	timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
1121 	TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
1122 	timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
1123 	timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
1124 	ctrlr->timeout_period = timeout_period;
1125 
1126 	nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
1127 	TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
1128 
1129 	per_cpu_io_queues = 1;
1130 	TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues);
1131 	ctrlr->per_cpu_io_queues = per_cpu_io_queues ? TRUE : FALSE;
1132 
1133 	if (ctrlr->per_cpu_io_queues)
1134 		ctrlr->num_io_queues = mp_ncpus;
1135 	else
1136 		ctrlr->num_io_queues = 1;
1137 
1138 	ctrlr->force_intx = 0;
1139 	TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx);
1140 
1141 	ctrlr->enable_aborts = 0;
1142 	TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
1143 
1144 	ctrlr->msix_enabled = 1;
1145 
1146 	if (ctrlr->force_intx) {
1147 		ctrlr->msix_enabled = 0;
1148 		goto intx;
1149 	}
1150 
1151 	/* One vector per IO queue, plus one vector for admin queue. */
1152 	num_vectors = ctrlr->num_io_queues + 1;
1153 
1154 	if (pci_msix_count(dev) < num_vectors) {
1155 		ctrlr->msix_enabled = 0;
1156 		goto intx;
1157 	}
1158 
1159 	if (pci_alloc_msix(dev, &num_vectors) != 0)
1160 		ctrlr->msix_enabled = 0;
1161 
1162 intx:
1163 
1164 	if (!ctrlr->msix_enabled)
1165 		nvme_ctrlr_configure_intx(ctrlr);
1166 
1167 	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
1168 	nvme_ctrlr_construct_admin_qpair(ctrlr);
1169 	status = nvme_ctrlr_construct_io_qpairs(ctrlr);
1170 
1171 	if (status != 0)
1172 		return (status);
1173 
1174 	ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, device_get_unit(dev),
1175 	    UID_ROOT, GID_WHEEL, 0600, "nvme%d", device_get_unit(dev));
1176 
1177 	if (ctrlr->cdev == NULL)
1178 		return (ENXIO);
1179 
1180 	ctrlr->cdev->si_drv1 = (void *)ctrlr;
1181 
1182 	ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1183 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);
1184 	taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq");
1185 
1186 	ctrlr->is_resetting = 0;
1187 	TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1188 
1189 	TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
1190 	STAILQ_INIT(&ctrlr->fail_req);
1191 	ctrlr->is_failed = FALSE;
1192 
1193 	return (0);
1194 }
1195 
1196 void
1197 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1198 {
1199 	int				i;
1200 
1201 	/*
1202 	 *  Notify the controller of a shutdown, even though this is due to
1203 	 *   a driver unload, not a system shutdown (this path is not invoked
1204 	 *   during shutdown).  This ensures the controller receives a
1205 	 *   shutdown notification in case the system is shutdown before
1206 	 *   reloading the driver.
1207 	 *
1208 	 *  Chatham does not let you re-enable the controller after shutdown
1209 	 *   notification has been received, so do not send it in this case.
1210 	 *   This is OK because Chatham does not depend on the shutdown
1211 	 *   notification anyways.
1212 	 */
1213 	if (pci_get_devid(ctrlr->dev) != CHATHAM_PCI_ID)
1214 		nvme_ctrlr_shutdown(ctrlr);
1215 
1216 	nvme_ctrlr_disable(ctrlr);
1217 	taskqueue_free(ctrlr->taskqueue);
1218 
1219 	for (i = 0; i < NVME_MAX_NAMESPACES; i++)
1220 		nvme_ns_destruct(&ctrlr->ns[i]);
1221 
1222 	if (ctrlr->cdev)
1223 		destroy_dev(ctrlr->cdev);
1224 
1225 	for (i = 0; i < ctrlr->num_io_queues; i++) {
1226 		nvme_io_qpair_destroy(&ctrlr->ioq[i]);
1227 	}
1228 
1229 	free(ctrlr->ioq, M_NVME);
1230 
1231 	nvme_admin_qpair_destroy(&ctrlr->adminq);
1232 
1233 	if (ctrlr->resource != NULL) {
1234 		bus_release_resource(dev, SYS_RES_MEMORY,
1235 		    ctrlr->resource_id, ctrlr->resource);
1236 	}
1237 
1238 	if (ctrlr->bar4_resource != NULL) {
1239 		bus_release_resource(dev, SYS_RES_MEMORY,
1240 		    ctrlr->bar4_resource_id, ctrlr->bar4_resource);
1241 	}
1242 
1243 #ifdef CHATHAM2
1244 	if (ctrlr->chatham_resource != NULL) {
1245 		bus_release_resource(dev, SYS_RES_MEMORY,
1246 		    ctrlr->chatham_resource_id, ctrlr->chatham_resource);
1247 	}
1248 #endif
1249 
1250 	if (ctrlr->tag)
1251 		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
1252 
1253 	if (ctrlr->res)
1254 		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
1255 		    rman_get_rid(ctrlr->res), ctrlr->res);
1256 
1257 	if (ctrlr->msix_enabled)
1258 		pci_release_msi(dev);
1259 }
1260 
1261 void
1262 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
1263 {
1264 	union cc_register	cc;
1265 	union csts_register	csts;
1266 	int			ticks = 0;
1267 
1268 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
1269 	cc.bits.shn = NVME_SHN_NORMAL;
1270 	nvme_mmio_write_4(ctrlr, cc, cc.raw);
1271 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
1272 	while ((csts.bits.shst != NVME_SHST_COMPLETE) && (ticks++ < 5*hz)) {
1273 		pause("nvme shn", 1);
1274 		csts.raw = nvme_mmio_read_4(ctrlr, csts);
1275 	}
1276 	if (csts.bits.shst != NVME_SHST_COMPLETE)
1277 		nvme_printf(ctrlr, "did not complete shutdown within 5 seconds "
1278 		    "of notification\n");
1279 }
1280 
1281 void
1282 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
1283     struct nvme_request *req)
1284 {
1285 
1286 	nvme_qpair_submit_request(&ctrlr->adminq, req);
1287 }
1288 
1289 void
1290 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
1291     struct nvme_request *req)
1292 {
1293 	struct nvme_qpair       *qpair;
1294 
1295 	if (ctrlr->per_cpu_io_queues)
1296 		qpair = &ctrlr->ioq[curcpu];
1297 	else
1298 		qpair = &ctrlr->ioq[0];
1299 
1300 	nvme_qpair_submit_request(qpair, req);
1301 }
1302 
1303 device_t
1304 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
1305 {
1306 
1307 	return (ctrlr->dev);
1308 }
1309 
1310 const struct nvme_controller_data *
1311 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
1312 {
1313 
1314 	return (&ctrlr->cdata);
1315 }
1316