xref: /freebsd/sys/dev/nvme/nvme_ctrlr.c (revision d281e8fbbd9e7aa505e7afa74237238092f9e631)
1bb0ec6b3SJim Harris /*-
2bb0ec6b3SJim Harris  * Copyright (C) 2012 Intel Corporation
3bb0ec6b3SJim Harris  * All rights reserved.
4bb0ec6b3SJim Harris  *
5bb0ec6b3SJim Harris  * Redistribution and use in source and binary forms, with or without
6bb0ec6b3SJim Harris  * modification, are permitted provided that the following conditions
7bb0ec6b3SJim Harris  * are met:
8bb0ec6b3SJim Harris  * 1. Redistributions of source code must retain the above copyright
9bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer.
10bb0ec6b3SJim Harris  * 2. Redistributions in binary form must reproduce the above copyright
11bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer in the
12bb0ec6b3SJim Harris  *    documentation and/or other materials provided with the distribution.
13bb0ec6b3SJim Harris  *
14bb0ec6b3SJim Harris  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15bb0ec6b3SJim Harris  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16bb0ec6b3SJim Harris  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17bb0ec6b3SJim Harris  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18bb0ec6b3SJim Harris  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19bb0ec6b3SJim Harris  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20bb0ec6b3SJim Harris  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21bb0ec6b3SJim Harris  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22bb0ec6b3SJim Harris  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23bb0ec6b3SJim Harris  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24bb0ec6b3SJim Harris  * SUCH DAMAGE.
25bb0ec6b3SJim Harris  */
26bb0ec6b3SJim Harris 
27bb0ec6b3SJim Harris #include <sys/cdefs.h>
28bb0ec6b3SJim Harris __FBSDID("$FreeBSD$");
29bb0ec6b3SJim Harris 
30bb0ec6b3SJim Harris #include <sys/param.h>
31bb0ec6b3SJim Harris #include <sys/bus.h>
32bb0ec6b3SJim Harris #include <sys/conf.h>
33bb0ec6b3SJim Harris #include <sys/ioccom.h>
34bb0ec6b3SJim Harris #include <sys/smp.h>
35bb0ec6b3SJim Harris 
36bb0ec6b3SJim Harris #include <dev/pci/pcireg.h>
37bb0ec6b3SJim Harris #include <dev/pci/pcivar.h>
38bb0ec6b3SJim Harris 
39bb0ec6b3SJim Harris #include "nvme_private.h"
40bb0ec6b3SJim Harris 
41bb0ec6b3SJim Harris static void
42bb0ec6b3SJim Harris nvme_ctrlr_cb(void *arg, const struct nvme_completion *status)
43bb0ec6b3SJim Harris {
44bb0ec6b3SJim Harris 	struct nvme_completion	*cpl = arg;
45bb0ec6b3SJim Harris 	struct mtx		*mtx;
46bb0ec6b3SJim Harris 
47bb0ec6b3SJim Harris 	/*
48bb0ec6b3SJim Harris 	 * Copy status into the argument passed by the caller, so that
49bb0ec6b3SJim Harris 	 *  the caller can check the status to determine if the
50bb0ec6b3SJim Harris 	 *  the request passed or failed.
51bb0ec6b3SJim Harris 	 */
52bb0ec6b3SJim Harris 	memcpy(cpl, status, sizeof(*cpl));
53bb0ec6b3SJim Harris 	mtx = mtx_pool_find(mtxpool_sleep, cpl);
54bb0ec6b3SJim Harris 	mtx_lock(mtx);
55bb0ec6b3SJim Harris 	wakeup(cpl);
56bb0ec6b3SJim Harris 	mtx_unlock(mtx);
57bb0ec6b3SJim Harris }
58bb0ec6b3SJim Harris 
59bb0ec6b3SJim Harris static int
60bb0ec6b3SJim Harris nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr)
61bb0ec6b3SJim Harris {
62bb0ec6b3SJim Harris 
63bb0ec6b3SJim Harris 	/* Chatham puts the NVMe MMRs behind BAR 2/3, not BAR 0/1. */
64bb0ec6b3SJim Harris 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
65bb0ec6b3SJim Harris 		ctrlr->resource_id = PCIR_BAR(2);
66bb0ec6b3SJim Harris 	else
67bb0ec6b3SJim Harris 		ctrlr->resource_id = PCIR_BAR(0);
68bb0ec6b3SJim Harris 
69bb0ec6b3SJim Harris 	ctrlr->resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY,
70bb0ec6b3SJim Harris 	    &ctrlr->resource_id, 0, ~0, 1, RF_ACTIVE);
71bb0ec6b3SJim Harris 
72bb0ec6b3SJim Harris 	if(ctrlr->resource == NULL) {
73bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "unable to allocate pci resource\n");
74bb0ec6b3SJim Harris 		return (ENOMEM);
75bb0ec6b3SJim Harris 	}
76bb0ec6b3SJim Harris 
77bb0ec6b3SJim Harris 	ctrlr->bus_tag = rman_get_bustag(ctrlr->resource);
78bb0ec6b3SJim Harris 	ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource);
79bb0ec6b3SJim Harris 	ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle;
80bb0ec6b3SJim Harris 
81bb0ec6b3SJim Harris 	return (0);
82bb0ec6b3SJim Harris }
83bb0ec6b3SJim Harris 
84bb0ec6b3SJim Harris #ifdef CHATHAM2
85bb0ec6b3SJim Harris static int
86bb0ec6b3SJim Harris nvme_ctrlr_allocate_chatham_bar(struct nvme_controller *ctrlr)
87bb0ec6b3SJim Harris {
88bb0ec6b3SJim Harris 
89bb0ec6b3SJim Harris 	ctrlr->chatham_resource_id = PCIR_BAR(CHATHAM_CONTROL_BAR);
90bb0ec6b3SJim Harris 	ctrlr->chatham_resource = bus_alloc_resource(ctrlr->dev,
91bb0ec6b3SJim Harris 	    SYS_RES_MEMORY, &ctrlr->chatham_resource_id, 0, ~0, 1,
92bb0ec6b3SJim Harris 	    RF_ACTIVE);
93bb0ec6b3SJim Harris 
94bb0ec6b3SJim Harris 	if(ctrlr->chatham_resource == NULL) {
95bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "unable to alloc pci resource\n");
96bb0ec6b3SJim Harris 		return (ENOMEM);
97bb0ec6b3SJim Harris 	}
98bb0ec6b3SJim Harris 
99bb0ec6b3SJim Harris 	ctrlr->chatham_bus_tag = rman_get_bustag(ctrlr->chatham_resource);
100bb0ec6b3SJim Harris 	ctrlr->chatham_bus_handle =
101bb0ec6b3SJim Harris 	    rman_get_bushandle(ctrlr->chatham_resource);
102bb0ec6b3SJim Harris 
103bb0ec6b3SJim Harris 	return (0);
104bb0ec6b3SJim Harris }
105bb0ec6b3SJim Harris 
106bb0ec6b3SJim Harris static void
107bb0ec6b3SJim Harris nvme_ctrlr_setup_chatham(struct nvme_controller *ctrlr)
108bb0ec6b3SJim Harris {
109bb0ec6b3SJim Harris 	uint64_t reg1, reg2, reg3;
110bb0ec6b3SJim Harris 	uint64_t temp1, temp2;
111bb0ec6b3SJim Harris 	uint32_t temp3;
112bb0ec6b3SJim Harris 	uint32_t use_flash_timings = 0;
113bb0ec6b3SJim Harris 
114bb0ec6b3SJim Harris 	DELAY(10000);
115bb0ec6b3SJim Harris 
116bb0ec6b3SJim Harris 	temp3 = chatham_read_4(ctrlr, 0x8080);
117bb0ec6b3SJim Harris 
118bb0ec6b3SJim Harris 	device_printf(ctrlr->dev, "Chatham version: 0x%x\n", temp3);
119bb0ec6b3SJim Harris 
120bb0ec6b3SJim Harris 	ctrlr->chatham_lbas = chatham_read_4(ctrlr, 0x8068) - 0x110;
121bb0ec6b3SJim Harris 	ctrlr->chatham_size = ctrlr->chatham_lbas * 512;
122bb0ec6b3SJim Harris 
123bb0ec6b3SJim Harris 	device_printf(ctrlr->dev, "Chatham size: %lld\n",
124bb0ec6b3SJim Harris 	    (long long)ctrlr->chatham_size);
125bb0ec6b3SJim Harris 
126bb0ec6b3SJim Harris 	reg1 = reg2 = reg3 = ctrlr->chatham_size - 1;
127bb0ec6b3SJim Harris 
128bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.use_flash_timings", &use_flash_timings);
129bb0ec6b3SJim Harris 	if (use_flash_timings) {
130bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "Chatham: using flash timings\n");
131bb0ec6b3SJim Harris 		temp1 = 0x00001b58000007d0LL;
132bb0ec6b3SJim Harris 		temp2 = 0x000000cb00000131LL;
133bb0ec6b3SJim Harris 	} else {
134bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "Chatham: using DDR timings\n");
135bb0ec6b3SJim Harris 		temp1 = temp2 = 0x0LL;
136bb0ec6b3SJim Harris 	}
137bb0ec6b3SJim Harris 
138bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8000, reg1);
139bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8008, reg2);
140bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8010, reg3);
141bb0ec6b3SJim Harris 
142bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8020, temp1);
143bb0ec6b3SJim Harris 	temp3 = chatham_read_4(ctrlr, 0x8020);
144bb0ec6b3SJim Harris 
145bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8028, temp2);
146bb0ec6b3SJim Harris 	temp3 = chatham_read_4(ctrlr, 0x8028);
147bb0ec6b3SJim Harris 
148bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8030, temp1);
149bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8038, temp2);
150bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8040, temp1);
151bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8048, temp2);
152bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8050, temp1);
153bb0ec6b3SJim Harris 	chatham_write_8(ctrlr, 0x8058, temp2);
154bb0ec6b3SJim Harris 
155bb0ec6b3SJim Harris 	DELAY(10000);
156bb0ec6b3SJim Harris }
157bb0ec6b3SJim Harris 
158bb0ec6b3SJim Harris static void
159bb0ec6b3SJim Harris nvme_chatham_populate_cdata(struct nvme_controller *ctrlr)
160bb0ec6b3SJim Harris {
161bb0ec6b3SJim Harris 	struct nvme_controller_data *cdata;
162bb0ec6b3SJim Harris 
163bb0ec6b3SJim Harris 	cdata = &ctrlr->cdata;
164bb0ec6b3SJim Harris 
165bb0ec6b3SJim Harris 	cdata->vid = 0x8086;
166bb0ec6b3SJim Harris 	cdata->ssvid = 0x2011;
167bb0ec6b3SJim Harris 
168bb0ec6b3SJim Harris 	/*
169bb0ec6b3SJim Harris 	 * Chatham2 puts garbage data in these fields when we
170bb0ec6b3SJim Harris 	 *  invoke IDENTIFY_CONTROLLER, so we need to re-zero
171bb0ec6b3SJim Harris 	 *  the fields before calling bcopy().
172bb0ec6b3SJim Harris 	 */
173bb0ec6b3SJim Harris 	memset(cdata->sn, 0, sizeof(cdata->sn));
174bb0ec6b3SJim Harris 	memcpy(cdata->sn, "2012", strlen("2012"));
175bb0ec6b3SJim Harris 	memset(cdata->mn, 0, sizeof(cdata->mn));
176bb0ec6b3SJim Harris 	memcpy(cdata->mn, "CHATHAM2", strlen("CHATHAM2"));
177bb0ec6b3SJim Harris 	memset(cdata->fr, 0, sizeof(cdata->fr));
178bb0ec6b3SJim Harris 	memcpy(cdata->fr, "0", strlen("0"));
179bb0ec6b3SJim Harris 	cdata->rab = 8;
180bb0ec6b3SJim Harris 	cdata->aerl = 3;
181bb0ec6b3SJim Harris 	cdata->lpa.ns_smart = 1;
182bb0ec6b3SJim Harris 	cdata->sqes.min = 6;
183bb0ec6b3SJim Harris 	cdata->sqes.max = 6;
184bb0ec6b3SJim Harris 	cdata->sqes.min = 4;
185bb0ec6b3SJim Harris 	cdata->sqes.max = 4;
186bb0ec6b3SJim Harris 	cdata->nn = 1;
187bb0ec6b3SJim Harris 
188bb0ec6b3SJim Harris 	/* Chatham2 doesn't support DSM command */
189bb0ec6b3SJim Harris 	cdata->oncs.dsm = 0;
190bb0ec6b3SJim Harris 
191bb0ec6b3SJim Harris 	cdata->vwc.present = 1;
192bb0ec6b3SJim Harris }
193bb0ec6b3SJim Harris #endif /* CHATHAM2 */
194bb0ec6b3SJim Harris 
195bb0ec6b3SJim Harris static void
196bb0ec6b3SJim Harris nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
197bb0ec6b3SJim Harris {
198bb0ec6b3SJim Harris 	struct nvme_qpair	*qpair;
199bb0ec6b3SJim Harris 	uint32_t		num_entries;
200bb0ec6b3SJim Harris 
201bb0ec6b3SJim Harris 	qpair = &ctrlr->adminq;
202bb0ec6b3SJim Harris 
203bb0ec6b3SJim Harris 	num_entries = NVME_ADMIN_ENTRIES;
204bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
205bb0ec6b3SJim Harris 	/*
206bb0ec6b3SJim Harris 	 * If admin_entries was overridden to an invalid value, revert it
207bb0ec6b3SJim Harris 	 *  back to our default value.
208bb0ec6b3SJim Harris 	 */
209bb0ec6b3SJim Harris 	if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
210bb0ec6b3SJim Harris 	    num_entries > NVME_MAX_ADMIN_ENTRIES) {
211bb0ec6b3SJim Harris 		printf("nvme: invalid hw.nvme.admin_entries=%d specified\n",
212bb0ec6b3SJim Harris 		    num_entries);
213bb0ec6b3SJim Harris 		num_entries = NVME_ADMIN_ENTRIES;
214bb0ec6b3SJim Harris 	}
215bb0ec6b3SJim Harris 
216bb0ec6b3SJim Harris 	/*
217bb0ec6b3SJim Harris 	 * The admin queue's max xfer size is treated differently than the
218bb0ec6b3SJim Harris 	 *  max I/O xfer size.  16KB is sufficient here - maybe even less?
219bb0ec6b3SJim Harris 	 */
220bb0ec6b3SJim Harris 	nvme_qpair_construct(qpair, 0, 0, num_entries, 16*1024, ctrlr);
221bb0ec6b3SJim Harris }
222bb0ec6b3SJim Harris 
223bb0ec6b3SJim Harris static int
224bb0ec6b3SJim Harris nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
225bb0ec6b3SJim Harris {
226bb0ec6b3SJim Harris 	struct nvme_qpair	*qpair;
227bb0ec6b3SJim Harris 	union cap_lo_register	cap_lo;
228bb0ec6b3SJim Harris 	int			i, num_entries;
229bb0ec6b3SJim Harris 
230bb0ec6b3SJim Harris 	num_entries = NVME_IO_ENTRIES;
231bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
232bb0ec6b3SJim Harris 
233bb0ec6b3SJim Harris 	num_entries = max(num_entries, NVME_MIN_IO_ENTRIES);
234bb0ec6b3SJim Harris 
235bb0ec6b3SJim Harris 	/*
236bb0ec6b3SJim Harris 	 * NVMe spec sets a hard limit of 64K max entries, but
237bb0ec6b3SJim Harris 	 *  devices may specify a smaller limit, so we need to check
238bb0ec6b3SJim Harris 	 *  the MQES field in the capabilities register.
239bb0ec6b3SJim Harris 	 */
240bb0ec6b3SJim Harris 	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo);
241bb0ec6b3SJim Harris 	num_entries = min(num_entries, cap_lo.bits.mqes+1);
242bb0ec6b3SJim Harris 
243bb0ec6b3SJim Harris 	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
244bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.max_xfer_size", &ctrlr->max_xfer_size);
245bb0ec6b3SJim Harris 	/*
246bb0ec6b3SJim Harris 	 * Check that tunable doesn't specify a size greater than what our
247bb0ec6b3SJim Harris 	 *  driver supports, and is an even PAGE_SIZE multiple.
248bb0ec6b3SJim Harris 	 */
249bb0ec6b3SJim Harris 	if (ctrlr->max_xfer_size > NVME_MAX_XFER_SIZE ||
250bb0ec6b3SJim Harris 	    ctrlr->max_xfer_size % PAGE_SIZE)
251bb0ec6b3SJim Harris 		ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
252bb0ec6b3SJim Harris 
253bb0ec6b3SJim Harris 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
254bb0ec6b3SJim Harris 	    M_NVME, M_ZERO | M_NOWAIT);
255bb0ec6b3SJim Harris 
256bb0ec6b3SJim Harris 	if (ctrlr->ioq == NULL)
257bb0ec6b3SJim Harris 		return (ENOMEM);
258bb0ec6b3SJim Harris 
259bb0ec6b3SJim Harris 	for (i = 0; i < ctrlr->num_io_queues; i++) {
260bb0ec6b3SJim Harris 		qpair = &ctrlr->ioq[i];
261bb0ec6b3SJim Harris 
262bb0ec6b3SJim Harris 		/*
263bb0ec6b3SJim Harris 		 * Admin queue has ID=0. IO queues start at ID=1 -
264bb0ec6b3SJim Harris 		 *  hence the 'i+1' here.
265bb0ec6b3SJim Harris 		 *
266bb0ec6b3SJim Harris 		 * For I/O queues, use the controller-wide max_xfer_size
267bb0ec6b3SJim Harris 		 *  calculated in nvme_attach().
268bb0ec6b3SJim Harris 		 */
269bb0ec6b3SJim Harris 		nvme_qpair_construct(qpair,
270bb0ec6b3SJim Harris 				     i+1, /* qpair ID */
271bb0ec6b3SJim Harris 				     ctrlr->msix_enabled ? i+1 : 0, /* vector */
272bb0ec6b3SJim Harris 				     num_entries,
273bb0ec6b3SJim Harris 				     ctrlr->max_xfer_size,
274bb0ec6b3SJim Harris 				     ctrlr);
275bb0ec6b3SJim Harris 
276bb0ec6b3SJim Harris 		if (ctrlr->per_cpu_io_queues)
277bb0ec6b3SJim Harris 			bus_bind_intr(ctrlr->dev, qpair->res, i);
278bb0ec6b3SJim Harris 	}
279bb0ec6b3SJim Harris 
280bb0ec6b3SJim Harris 	return (0);
281bb0ec6b3SJim Harris }
282bb0ec6b3SJim Harris 
283bb0ec6b3SJim Harris static int
284bb0ec6b3SJim Harris nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr)
285bb0ec6b3SJim Harris {
286bb0ec6b3SJim Harris 	int ms_waited;
287bb0ec6b3SJim Harris 	union cc_register cc;
288bb0ec6b3SJim Harris 	union csts_register csts;
289bb0ec6b3SJim Harris 
290bb0ec6b3SJim Harris 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
291bb0ec6b3SJim Harris 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
292bb0ec6b3SJim Harris 
293bb0ec6b3SJim Harris 	if (!cc.bits.en) {
294bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "%s called with cc.en = 0\n",
295bb0ec6b3SJim Harris 		    __func__);
296bb0ec6b3SJim Harris 		return (ENXIO);
297bb0ec6b3SJim Harris 	}
298bb0ec6b3SJim Harris 
299bb0ec6b3SJim Harris 	ms_waited = 0;
300bb0ec6b3SJim Harris 
301bb0ec6b3SJim Harris 	while (!csts.bits.rdy) {
302bb0ec6b3SJim Harris 		DELAY(1000);
303bb0ec6b3SJim Harris 		if (ms_waited++ > ctrlr->ready_timeout_in_ms) {
304bb0ec6b3SJim Harris 			device_printf(ctrlr->dev, "controller did not become "
305bb0ec6b3SJim Harris 			    "ready within %d ms\n", ctrlr->ready_timeout_in_ms);
306bb0ec6b3SJim Harris 			return (ENXIO);
307bb0ec6b3SJim Harris 		}
308bb0ec6b3SJim Harris 		csts.raw = nvme_mmio_read_4(ctrlr, csts);
309bb0ec6b3SJim Harris 	}
310bb0ec6b3SJim Harris 
311bb0ec6b3SJim Harris 	return (0);
312bb0ec6b3SJim Harris }
313bb0ec6b3SJim Harris 
314bb0ec6b3SJim Harris static void
315bb0ec6b3SJim Harris nvme_ctrlr_disable(struct nvme_controller *ctrlr)
316bb0ec6b3SJim Harris {
317bb0ec6b3SJim Harris 	union cc_register cc;
318bb0ec6b3SJim Harris 	union csts_register csts;
319bb0ec6b3SJim Harris 
320bb0ec6b3SJim Harris 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
321bb0ec6b3SJim Harris 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
322bb0ec6b3SJim Harris 
323bb0ec6b3SJim Harris 	if (cc.bits.en == 1 && csts.bits.rdy == 0)
324bb0ec6b3SJim Harris 		nvme_ctrlr_wait_for_ready(ctrlr);
325bb0ec6b3SJim Harris 
326bb0ec6b3SJim Harris 	cc.bits.en = 0;
327bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, cc, cc.raw);
328bb0ec6b3SJim Harris 	DELAY(5000);
329bb0ec6b3SJim Harris }
330bb0ec6b3SJim Harris 
331bb0ec6b3SJim Harris static int
332bb0ec6b3SJim Harris nvme_ctrlr_enable(struct nvme_controller *ctrlr)
333bb0ec6b3SJim Harris {
334bb0ec6b3SJim Harris 	union cc_register	cc;
335bb0ec6b3SJim Harris 	union csts_register	csts;
336bb0ec6b3SJim Harris 	union aqa_register	aqa;
337bb0ec6b3SJim Harris 
338bb0ec6b3SJim Harris 	cc.raw = nvme_mmio_read_4(ctrlr, cc);
339bb0ec6b3SJim Harris 	csts.raw = nvme_mmio_read_4(ctrlr, csts);
340bb0ec6b3SJim Harris 
341bb0ec6b3SJim Harris 	if (cc.bits.en == 1) {
342bb0ec6b3SJim Harris 		if (csts.bits.rdy == 1)
343bb0ec6b3SJim Harris 			return (0);
344bb0ec6b3SJim Harris 		else
345bb0ec6b3SJim Harris 			return (nvme_ctrlr_wait_for_ready(ctrlr));
346bb0ec6b3SJim Harris 	}
347bb0ec6b3SJim Harris 
348bb0ec6b3SJim Harris 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
349bb0ec6b3SJim Harris 	DELAY(5000);
350bb0ec6b3SJim Harris 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
351bb0ec6b3SJim Harris 	DELAY(5000);
352bb0ec6b3SJim Harris 
353bb0ec6b3SJim Harris 	aqa.raw = 0;
354bb0ec6b3SJim Harris 	/* acqs and asqs are 0-based. */
355bb0ec6b3SJim Harris 	aqa.bits.acqs = ctrlr->adminq.num_entries-1;
356bb0ec6b3SJim Harris 	aqa.bits.asqs = ctrlr->adminq.num_entries-1;
357bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, aqa, aqa.raw);
358bb0ec6b3SJim Harris 	DELAY(5000);
359bb0ec6b3SJim Harris 
360bb0ec6b3SJim Harris 	cc.bits.en = 1;
361bb0ec6b3SJim Harris 	cc.bits.css = 0;
362bb0ec6b3SJim Harris 	cc.bits.ams = 0;
363bb0ec6b3SJim Harris 	cc.bits.shn = 0;
364bb0ec6b3SJim Harris 	cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
365bb0ec6b3SJim Harris 	cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
366bb0ec6b3SJim Harris 
367bb0ec6b3SJim Harris 	/* This evaluates to 0, which is according to spec. */
368bb0ec6b3SJim Harris 	cc.bits.mps = (PAGE_SIZE >> 13);
369bb0ec6b3SJim Harris 
370bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, cc, cc.raw);
371bb0ec6b3SJim Harris 	DELAY(5000);
372bb0ec6b3SJim Harris 
373bb0ec6b3SJim Harris 	return (nvme_ctrlr_wait_for_ready(ctrlr));
374bb0ec6b3SJim Harris }
375bb0ec6b3SJim Harris 
376bb0ec6b3SJim Harris int
377bb0ec6b3SJim Harris nvme_ctrlr_reset(struct nvme_controller *ctrlr)
378bb0ec6b3SJim Harris {
379bb0ec6b3SJim Harris 
380bb0ec6b3SJim Harris 	nvme_ctrlr_disable(ctrlr);
381bb0ec6b3SJim Harris 	return (nvme_ctrlr_enable(ctrlr));
382bb0ec6b3SJim Harris }
383bb0ec6b3SJim Harris 
3848a382371SJim Harris /*
3858a382371SJim Harris  * Disable this code for now, since Chatham doesn't support
3868a382371SJim Harris  *  AERs so I have no good way to test them.
3878a382371SJim Harris  */
3888a382371SJim Harris #if 0
389bb0ec6b3SJim Harris static void
390bb0ec6b3SJim Harris nvme_async_event_cb(void *arg, const struct nvme_completion *status)
391bb0ec6b3SJim Harris {
392bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = arg;
393bb0ec6b3SJim Harris 
394bb0ec6b3SJim Harris 	printf("Asynchronous event occurred.\n");
395bb0ec6b3SJim Harris 
396bb0ec6b3SJim Harris 	/* TODO: decode async event type based on status */
397bb0ec6b3SJim Harris 	/* TODO: check status for any error bits */
398bb0ec6b3SJim Harris 
399bb0ec6b3SJim Harris 	/*
400bb0ec6b3SJim Harris 	 * Repost an asynchronous event request so that it can be
401bb0ec6b3SJim Harris 	 *  used again by the controller.
402bb0ec6b3SJim Harris 	 */
403bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_asynchronous_event_request(ctrlr, nvme_async_event_cb,
404bb0ec6b3SJim Harris 	    ctrlr);
405bb0ec6b3SJim Harris }
4068a382371SJim Harris #endif
407bb0ec6b3SJim Harris 
408bb0ec6b3SJim Harris static int
409bb0ec6b3SJim Harris nvme_ctrlr_identify(struct nvme_controller *ctrlr)
410bb0ec6b3SJim Harris {
411bb0ec6b3SJim Harris 	struct mtx		*mtx;
412bb0ec6b3SJim Harris 	struct nvme_completion	cpl;
413bb0ec6b3SJim Harris 	int			status;
414bb0ec6b3SJim Harris 
415bb0ec6b3SJim Harris 	mtx = mtx_pool_find(mtxpool_sleep, &cpl);
416bb0ec6b3SJim Harris 
417bb0ec6b3SJim Harris 	mtx_lock(mtx);
418bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
419bb0ec6b3SJim Harris 	    nvme_ctrlr_cb, &cpl);
420bb0ec6b3SJim Harris 	status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5);
421bb0ec6b3SJim Harris 	mtx_unlock(mtx);
422bb0ec6b3SJim Harris 	if ((status != 0) || cpl.sf_sc || cpl.sf_sct) {
423bb0ec6b3SJim Harris 		printf("nvme_identify_controller failed!\n");
424bb0ec6b3SJim Harris 		return (ENXIO);
425bb0ec6b3SJim Harris 	}
426bb0ec6b3SJim Harris 
427bb0ec6b3SJim Harris #ifdef CHATHAM2
428bb0ec6b3SJim Harris 	if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
429bb0ec6b3SJim Harris 		nvme_chatham_populate_cdata(ctrlr);
430bb0ec6b3SJim Harris #endif
431bb0ec6b3SJim Harris 
432bb0ec6b3SJim Harris 	return (0);
433bb0ec6b3SJim Harris }
434bb0ec6b3SJim Harris 
435bb0ec6b3SJim Harris static int
436bb0ec6b3SJim Harris nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
437bb0ec6b3SJim Harris {
438bb0ec6b3SJim Harris 	struct mtx		*mtx;
439bb0ec6b3SJim Harris 	struct nvme_completion	cpl;
440bb0ec6b3SJim Harris 	int			cq_allocated, sq_allocated, status;
441bb0ec6b3SJim Harris 
442bb0ec6b3SJim Harris 	mtx = mtx_pool_find(mtxpool_sleep, &cpl);
443bb0ec6b3SJim Harris 
444bb0ec6b3SJim Harris 	mtx_lock(mtx);
445bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
446bb0ec6b3SJim Harris 	    nvme_ctrlr_cb, &cpl);
447bb0ec6b3SJim Harris 	status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5);
448bb0ec6b3SJim Harris 	mtx_unlock(mtx);
449bb0ec6b3SJim Harris 	if ((status != 0) || cpl.sf_sc || cpl.sf_sct) {
450bb0ec6b3SJim Harris 		printf("nvme_set_num_queues failed!\n");
451bb0ec6b3SJim Harris 		return (ENXIO);
452bb0ec6b3SJim Harris 	}
453bb0ec6b3SJim Harris 
454bb0ec6b3SJim Harris 	/*
455bb0ec6b3SJim Harris 	 * Data in cdw0 is 0-based.
456bb0ec6b3SJim Harris 	 * Lower 16-bits indicate number of submission queues allocated.
457bb0ec6b3SJim Harris 	 * Upper 16-bits indicate number of completion queues allocated.
458bb0ec6b3SJim Harris 	 */
459bb0ec6b3SJim Harris 	sq_allocated = (cpl.cdw0 & 0xFFFF) + 1;
460bb0ec6b3SJim Harris 	cq_allocated = (cpl.cdw0 >> 16) + 1;
461bb0ec6b3SJim Harris 
462bb0ec6b3SJim Harris 	/*
463bb0ec6b3SJim Harris 	 * Check that the controller was able to allocate the number of
464bb0ec6b3SJim Harris 	 *  queues we requested.  If not, revert to one IO queue.
465bb0ec6b3SJim Harris 	 */
466bb0ec6b3SJim Harris 	if (sq_allocated < ctrlr->num_io_queues ||
467bb0ec6b3SJim Harris 	    cq_allocated < ctrlr->num_io_queues) {
468bb0ec6b3SJim Harris 		ctrlr->num_io_queues = 1;
469bb0ec6b3SJim Harris 		ctrlr->per_cpu_io_queues = 0;
470bb0ec6b3SJim Harris 
471bb0ec6b3SJim Harris 		/* TODO: destroy extra queues that were created
472bb0ec6b3SJim Harris 		 *  previously but now found to be not needed.
473bb0ec6b3SJim Harris 		 */
474bb0ec6b3SJim Harris 	}
475bb0ec6b3SJim Harris 
476bb0ec6b3SJim Harris 	return (0);
477bb0ec6b3SJim Harris }
478bb0ec6b3SJim Harris 
479bb0ec6b3SJim Harris static int
480bb0ec6b3SJim Harris nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
481bb0ec6b3SJim Harris {
482bb0ec6b3SJim Harris 	struct mtx		*mtx;
483bb0ec6b3SJim Harris 	struct nvme_qpair	*qpair;
484bb0ec6b3SJim Harris 	struct nvme_completion	cpl;
485bb0ec6b3SJim Harris 	int			i, status;
486bb0ec6b3SJim Harris 
487bb0ec6b3SJim Harris 	mtx = mtx_pool_find(mtxpool_sleep, &cpl);
488bb0ec6b3SJim Harris 
489bb0ec6b3SJim Harris 	for (i = 0; i < ctrlr->num_io_queues; i++) {
490bb0ec6b3SJim Harris 		qpair = &ctrlr->ioq[i];
491bb0ec6b3SJim Harris 
492bb0ec6b3SJim Harris 		mtx_lock(mtx);
493bb0ec6b3SJim Harris 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
494bb0ec6b3SJim Harris 		    nvme_ctrlr_cb, &cpl);
495bb0ec6b3SJim Harris 		status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5);
496bb0ec6b3SJim Harris 		mtx_unlock(mtx);
497bb0ec6b3SJim Harris 		if ((status != 0) || cpl.sf_sc || cpl.sf_sct) {
498bb0ec6b3SJim Harris 			printf("nvme_create_io_cq failed!\n");
499bb0ec6b3SJim Harris 			return (ENXIO);
500bb0ec6b3SJim Harris 		}
501bb0ec6b3SJim Harris 
502bb0ec6b3SJim Harris 		mtx_lock(mtx);
503bb0ec6b3SJim Harris 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
504bb0ec6b3SJim Harris 		    nvme_ctrlr_cb, &cpl);
505bb0ec6b3SJim Harris 		status = msleep(&cpl, mtx, PRIBIO, "nvme_start", hz*5);
506bb0ec6b3SJim Harris 		mtx_unlock(mtx);
507bb0ec6b3SJim Harris 		if ((status != 0) || cpl.sf_sc || cpl.sf_sct) {
508bb0ec6b3SJim Harris 			printf("nvme_create_io_sq failed!\n");
509bb0ec6b3SJim Harris 			return (ENXIO);
510bb0ec6b3SJim Harris 		}
511bb0ec6b3SJim Harris 	}
512bb0ec6b3SJim Harris 
513bb0ec6b3SJim Harris 	return (0);
514bb0ec6b3SJim Harris }
515bb0ec6b3SJim Harris 
516bb0ec6b3SJim Harris static int
517bb0ec6b3SJim Harris nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
518bb0ec6b3SJim Harris {
519bb0ec6b3SJim Harris 	struct nvme_namespace	*ns;
520bb0ec6b3SJim Harris 	int			i, status;
521bb0ec6b3SJim Harris 
522bb0ec6b3SJim Harris 	for (i = 0; i < ctrlr->cdata.nn; i++) {
523bb0ec6b3SJim Harris 		ns = &ctrlr->ns[i];
524bb0ec6b3SJim Harris 		status = nvme_ns_construct(ns, i+1, ctrlr);
525bb0ec6b3SJim Harris 		if (status != 0)
526bb0ec6b3SJim Harris 			return (status);
527bb0ec6b3SJim Harris 	}
528bb0ec6b3SJim Harris 
529bb0ec6b3SJim Harris 	return (0);
530bb0ec6b3SJim Harris }
531bb0ec6b3SJim Harris 
532bb0ec6b3SJim Harris static void
533bb0ec6b3SJim Harris nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
534bb0ec6b3SJim Harris {
535bb0ec6b3SJim Harris 	union nvme_critical_warning_state	state;
536bb0ec6b3SJim Harris 	uint8_t					num_async_events;
537bb0ec6b3SJim Harris 
538bb0ec6b3SJim Harris 	state.raw = 0xFF;
539bb0ec6b3SJim Harris 	state.bits.reserved = 0;
540bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_set_asynchronous_event_config(ctrlr, state, NULL, NULL);
541bb0ec6b3SJim Harris 
542bb0ec6b3SJim Harris 	/* aerl is a zero-based value, so we need to add 1 here. */
543bb0ec6b3SJim Harris 	num_async_events = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
544bb0ec6b3SJim Harris 
545bb0ec6b3SJim Harris 	/*
546bb0ec6b3SJim Harris 	 * Disable this code for now, since Chatham doesn't support
547bb0ec6b3SJim Harris 	 *  AERs so I have no good way to test them.
548bb0ec6b3SJim Harris 	 */
549bb0ec6b3SJim Harris #if 0
550bb0ec6b3SJim Harris 	for (int i = 0; i < num_async_events; i++)
551bb0ec6b3SJim Harris 		nvme_ctrlr_cmd_asynchronous_event_request(ctrlr,
552bb0ec6b3SJim Harris 		    nvme_async_event_cb, ctrlr);
553bb0ec6b3SJim Harris #endif
554bb0ec6b3SJim Harris }
555bb0ec6b3SJim Harris 
556bb0ec6b3SJim Harris static void
557bb0ec6b3SJim Harris nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
558bb0ec6b3SJim Harris {
559bb0ec6b3SJim Harris 
560bb0ec6b3SJim Harris 	ctrlr->int_coal_time = 0;
561bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
562bb0ec6b3SJim Harris 	    &ctrlr->int_coal_time);
563bb0ec6b3SJim Harris 
564bb0ec6b3SJim Harris 	ctrlr->int_coal_threshold = 0;
565bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
566bb0ec6b3SJim Harris 	    &ctrlr->int_coal_threshold);
567bb0ec6b3SJim Harris 
568bb0ec6b3SJim Harris 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
569bb0ec6b3SJim Harris 	    ctrlr->int_coal_threshold, NULL, NULL);
570bb0ec6b3SJim Harris }
571bb0ec6b3SJim Harris 
572bb0ec6b3SJim Harris void
573bb0ec6b3SJim Harris nvme_ctrlr_start(void *ctrlr_arg)
574bb0ec6b3SJim Harris {
575bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = ctrlr_arg;
576bb0ec6b3SJim Harris 
577bb0ec6b3SJim Harris 	if (nvme_ctrlr_identify(ctrlr) != 0)
578bb0ec6b3SJim Harris 		goto err;
579bb0ec6b3SJim Harris 
580bb0ec6b3SJim Harris 	if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0)
581bb0ec6b3SJim Harris 		goto err;
582bb0ec6b3SJim Harris 
583bb0ec6b3SJim Harris 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0)
584bb0ec6b3SJim Harris 		goto err;
585bb0ec6b3SJim Harris 
586bb0ec6b3SJim Harris 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0)
587bb0ec6b3SJim Harris 		goto err;
588bb0ec6b3SJim Harris 
589bb0ec6b3SJim Harris 	nvme_ctrlr_configure_aer(ctrlr);
590bb0ec6b3SJim Harris 	nvme_ctrlr_configure_int_coalescing(ctrlr);
591bb0ec6b3SJim Harris 
592bb0ec6b3SJim Harris 	ctrlr->is_started = TRUE;
593bb0ec6b3SJim Harris 
594bb0ec6b3SJim Harris err:
595bb0ec6b3SJim Harris 
596bb0ec6b3SJim Harris 	/*
597bb0ec6b3SJim Harris 	 * Initialize sysctls, even if controller failed to start, to
598bb0ec6b3SJim Harris 	 *  assist with debugging admin queue pair.
599bb0ec6b3SJim Harris 	 */
600bb0ec6b3SJim Harris 	nvme_sysctl_initialize_ctrlr(ctrlr);
601bb0ec6b3SJim Harris 	config_intrhook_disestablish(&ctrlr->config_hook);
602bb0ec6b3SJim Harris }
603bb0ec6b3SJim Harris 
604bb0ec6b3SJim Harris static void
605bb0ec6b3SJim Harris nvme_ctrlr_intx_task(void *arg, int pending)
606bb0ec6b3SJim Harris {
607bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = arg;
608bb0ec6b3SJim Harris 
609bb0ec6b3SJim Harris 	nvme_qpair_process_completions(&ctrlr->adminq);
610bb0ec6b3SJim Harris 
611bb0ec6b3SJim Harris 	if (ctrlr->ioq[0].cpl)
612bb0ec6b3SJim Harris 		nvme_qpair_process_completions(&ctrlr->ioq[0]);
613bb0ec6b3SJim Harris 
614bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, intmc, 1);
615bb0ec6b3SJim Harris }
616bb0ec6b3SJim Harris 
617bb0ec6b3SJim Harris static void
618bb0ec6b3SJim Harris nvme_ctrlr_intx_handler(void *arg)
619bb0ec6b3SJim Harris {
620bb0ec6b3SJim Harris 	struct nvme_controller *ctrlr = arg;
621bb0ec6b3SJim Harris 
622bb0ec6b3SJim Harris 	nvme_mmio_write_4(ctrlr, intms, 1);
623bb0ec6b3SJim Harris 	taskqueue_enqueue_fast(ctrlr->taskqueue, &ctrlr->task);
624bb0ec6b3SJim Harris }
625bb0ec6b3SJim Harris 
626bb0ec6b3SJim Harris static int
627bb0ec6b3SJim Harris nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr)
628bb0ec6b3SJim Harris {
629bb0ec6b3SJim Harris 
630bb0ec6b3SJim Harris 	ctrlr->num_io_queues = 1;
631bb0ec6b3SJim Harris 	ctrlr->per_cpu_io_queues = 0;
632bb0ec6b3SJim Harris 	ctrlr->rid = 0;
633bb0ec6b3SJim Harris 	ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
634bb0ec6b3SJim Harris 	    &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE);
635bb0ec6b3SJim Harris 
636bb0ec6b3SJim Harris 	if (ctrlr->res == NULL) {
637bb0ec6b3SJim Harris 		device_printf(ctrlr->dev, "unable to allocate shared IRQ\n");
638bb0ec6b3SJim Harris 		return (ENOMEM);
639bb0ec6b3SJim Harris 	}
640bb0ec6b3SJim Harris 
641bb0ec6b3SJim Harris 	bus_setup_intr(ctrlr->dev, ctrlr->res,
642bb0ec6b3SJim Harris 	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler,
643bb0ec6b3SJim Harris 	    ctrlr, &ctrlr->tag);
644bb0ec6b3SJim Harris 
645bb0ec6b3SJim Harris 	if (ctrlr->tag == NULL) {
646bb0ec6b3SJim Harris 		device_printf(ctrlr->dev,
647bb0ec6b3SJim Harris 		    "unable to setup legacy interrupt handler\n");
648bb0ec6b3SJim Harris 		return (ENOMEM);
649bb0ec6b3SJim Harris 	}
650bb0ec6b3SJim Harris 
651bb0ec6b3SJim Harris 	TASK_INIT(&ctrlr->task, 0, nvme_ctrlr_intx_task, ctrlr);
652bb0ec6b3SJim Harris 	ctrlr->taskqueue = taskqueue_create_fast("nvme_taskq", M_NOWAIT,
653bb0ec6b3SJim Harris 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);
654bb0ec6b3SJim Harris 	taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_NET,
655bb0ec6b3SJim Harris 	    "%s intx taskq", device_get_nameunit(ctrlr->dev));
656bb0ec6b3SJim Harris 
657bb0ec6b3SJim Harris 	return (0);
658bb0ec6b3SJim Harris }
659bb0ec6b3SJim Harris 
660bb0ec6b3SJim Harris static int
661bb0ec6b3SJim Harris nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
662bb0ec6b3SJim Harris     struct thread *td)
663bb0ec6b3SJim Harris {
664bb0ec6b3SJim Harris 	struct nvme_controller	*ctrlr;
665bb0ec6b3SJim Harris 	struct nvme_completion	cpl;
666bb0ec6b3SJim Harris 	struct mtx		*mtx;
667bb0ec6b3SJim Harris 
668bb0ec6b3SJim Harris 	ctrlr = cdev->si_drv1;
669bb0ec6b3SJim Harris 
670bb0ec6b3SJim Harris 	switch (cmd) {
671bb0ec6b3SJim Harris 	case NVME_IDENTIFY_CONTROLLER:
672bb0ec6b3SJim Harris #ifdef CHATHAM2
673bb0ec6b3SJim Harris 		/*
674bb0ec6b3SJim Harris 		 * Don't refresh data on Chatham, since Chatham returns
675bb0ec6b3SJim Harris 		 *  garbage on IDENTIFY anyways.
676bb0ec6b3SJim Harris 		 */
677bb0ec6b3SJim Harris 		if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) {
678bb0ec6b3SJim Harris 			memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata));
679bb0ec6b3SJim Harris 			break;
680bb0ec6b3SJim Harris 		}
681bb0ec6b3SJim Harris #endif
682bb0ec6b3SJim Harris 		/* Refresh data before returning to user. */
683bb0ec6b3SJim Harris 		mtx = mtx_pool_find(mtxpool_sleep, &cpl);
684bb0ec6b3SJim Harris 		mtx_lock(mtx);
685bb0ec6b3SJim Harris 		nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
686bb0ec6b3SJim Harris 		    nvme_ctrlr_cb, &cpl);
687bb0ec6b3SJim Harris 		msleep(&cpl, mtx, PRIBIO, "nvme_ioctl", 0);
688bb0ec6b3SJim Harris 		mtx_unlock(mtx);
689bb0ec6b3SJim Harris 		if (cpl.sf_sc || cpl.sf_sct)
690bb0ec6b3SJim Harris 			return (ENXIO);
691bb0ec6b3SJim Harris 		memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata));
692bb0ec6b3SJim Harris 		break;
693bb0ec6b3SJim Harris 	default:
694bb0ec6b3SJim Harris 		return (ENOTTY);
695bb0ec6b3SJim Harris 	}
696bb0ec6b3SJim Harris 
697bb0ec6b3SJim Harris 	return (0);
698bb0ec6b3SJim Harris }
699bb0ec6b3SJim Harris 
700bb0ec6b3SJim Harris static struct cdevsw nvme_ctrlr_cdevsw = {
701bb0ec6b3SJim Harris 	.d_version =	D_VERSION,
702bb0ec6b3SJim Harris 	.d_flags =	0,
703bb0ec6b3SJim Harris 	.d_ioctl =	nvme_ctrlr_ioctl
704bb0ec6b3SJim Harris };
705bb0ec6b3SJim Harris 
706bb0ec6b3SJim Harris int
707bb0ec6b3SJim Harris nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
708bb0ec6b3SJim Harris {
709bb0ec6b3SJim Harris 	union cap_lo_register	cap_lo;
710bb0ec6b3SJim Harris 	union cap_hi_register	cap_hi;
711bb0ec6b3SJim Harris 	int			num_vectors, per_cpu_io_queues, status = 0;
712bb0ec6b3SJim Harris 
713bb0ec6b3SJim Harris 	ctrlr->dev = dev;
714bb0ec6b3SJim Harris 	ctrlr->is_started = FALSE;
715bb0ec6b3SJim Harris 
716bb0ec6b3SJim Harris 	status = nvme_ctrlr_allocate_bar(ctrlr);
717bb0ec6b3SJim Harris 
718bb0ec6b3SJim Harris 	if (status != 0)
719bb0ec6b3SJim Harris 		return (status);
720bb0ec6b3SJim Harris 
721bb0ec6b3SJim Harris #ifdef CHATHAM2
722bb0ec6b3SJim Harris 	if (pci_get_devid(dev) == CHATHAM_PCI_ID) {
723bb0ec6b3SJim Harris 		status = nvme_ctrlr_allocate_chatham_bar(ctrlr);
724bb0ec6b3SJim Harris 		if (status != 0)
725bb0ec6b3SJim Harris 			return (status);
726bb0ec6b3SJim Harris 		nvme_ctrlr_setup_chatham(ctrlr);
727bb0ec6b3SJim Harris 	}
728bb0ec6b3SJim Harris #endif
729bb0ec6b3SJim Harris 
730bb0ec6b3SJim Harris 	/*
731bb0ec6b3SJim Harris 	 * Software emulators may set the doorbell stride to something
732bb0ec6b3SJim Harris 	 *  other than zero, but this driver is not set up to handle that.
733bb0ec6b3SJim Harris 	 */
734bb0ec6b3SJim Harris 	cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi);
735bb0ec6b3SJim Harris 	if (cap_hi.bits.dstrd != 0)
736bb0ec6b3SJim Harris 		return (ENXIO);
737bb0ec6b3SJim Harris 
738bb0ec6b3SJim Harris 	/* Get ready timeout value from controller, in units of 500ms. */
739bb0ec6b3SJim Harris 	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo);
740bb0ec6b3SJim Harris 	ctrlr->ready_timeout_in_ms = cap_lo.bits.to * 500;
741bb0ec6b3SJim Harris 
742bb0ec6b3SJim Harris 	per_cpu_io_queues = 1;
743bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues);
744bb0ec6b3SJim Harris 	ctrlr->per_cpu_io_queues = per_cpu_io_queues ? TRUE : FALSE;
745bb0ec6b3SJim Harris 
746bb0ec6b3SJim Harris 	if (ctrlr->per_cpu_io_queues)
747bb0ec6b3SJim Harris 		ctrlr->num_io_queues = mp_ncpus;
748bb0ec6b3SJim Harris 	else
749bb0ec6b3SJim Harris 		ctrlr->num_io_queues = 1;
750bb0ec6b3SJim Harris 
751bb0ec6b3SJim Harris 	ctrlr->force_intx = 0;
752bb0ec6b3SJim Harris 	TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx);
753bb0ec6b3SJim Harris 
754bb0ec6b3SJim Harris 	ctrlr->msix_enabled = 1;
755bb0ec6b3SJim Harris 
756bb0ec6b3SJim Harris 	if (ctrlr->force_intx) {
757bb0ec6b3SJim Harris 		ctrlr->msix_enabled = 0;
758bb0ec6b3SJim Harris 		goto intx;
759bb0ec6b3SJim Harris 	}
760bb0ec6b3SJim Harris 
761bb0ec6b3SJim Harris 	/* One vector per IO queue, plus one vector for admin queue. */
762bb0ec6b3SJim Harris 	num_vectors = ctrlr->num_io_queues + 1;
763bb0ec6b3SJim Harris 
764bb0ec6b3SJim Harris 	if (pci_msix_count(dev) < num_vectors) {
765bb0ec6b3SJim Harris 		ctrlr->msix_enabled = 0;
766bb0ec6b3SJim Harris 		goto intx;
767bb0ec6b3SJim Harris 	}
768bb0ec6b3SJim Harris 
769bb0ec6b3SJim Harris 	if (pci_alloc_msix(dev, &num_vectors) != 0)
770bb0ec6b3SJim Harris 		ctrlr->msix_enabled = 0;
771bb0ec6b3SJim Harris 
772bb0ec6b3SJim Harris intx:
773bb0ec6b3SJim Harris 
774bb0ec6b3SJim Harris 	if (!ctrlr->msix_enabled)
775bb0ec6b3SJim Harris 		nvme_ctrlr_configure_intx(ctrlr);
776bb0ec6b3SJim Harris 
777bb0ec6b3SJim Harris 	nvme_ctrlr_construct_admin_qpair(ctrlr);
778bb0ec6b3SJim Harris 
779bb0ec6b3SJim Harris 	status = nvme_ctrlr_construct_io_qpairs(ctrlr);
780bb0ec6b3SJim Harris 
781bb0ec6b3SJim Harris 	if (status != 0)
782bb0ec6b3SJim Harris 		return (status);
783bb0ec6b3SJim Harris 
784bb0ec6b3SJim Harris 	ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
785bb0ec6b3SJim Harris 	    "nvme%d", device_get_unit(dev));
786bb0ec6b3SJim Harris 
787bb0ec6b3SJim Harris 	if (ctrlr->cdev == NULL)
788bb0ec6b3SJim Harris 		return (ENXIO);
789bb0ec6b3SJim Harris 
790bb0ec6b3SJim Harris 	ctrlr->cdev->si_drv1 = (void *)ctrlr;
791bb0ec6b3SJim Harris 
792bb0ec6b3SJim Harris 	return (0);
793bb0ec6b3SJim Harris }
794*d281e8fbSJim Harris 
795*d281e8fbSJim Harris void
796*d281e8fbSJim Harris nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
797*d281e8fbSJim Harris     struct nvme_request *req)
798*d281e8fbSJim Harris {
799*d281e8fbSJim Harris 	struct nvme_qpair       *qpair;
800*d281e8fbSJim Harris 	struct nvme_tracker     *tr;
801*d281e8fbSJim Harris 	int                     err;
802*d281e8fbSJim Harris 
803*d281e8fbSJim Harris 	qpair = &ctrlr->adminq;
804*d281e8fbSJim Harris 
805*d281e8fbSJim Harris 	tr = nvme_qpair_allocate_tracker(qpair);
806*d281e8fbSJim Harris 
807*d281e8fbSJim Harris 	tr->req = req;
808*d281e8fbSJim Harris 
809*d281e8fbSJim Harris 	if (req->payload_size > 0) {
810*d281e8fbSJim Harris 		err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map,
811*d281e8fbSJim Harris 				      req->payload, req->payload_size,
812*d281e8fbSJim Harris 				      nvme_payload_map, tr, 0);
813*d281e8fbSJim Harris 		if (err != 0)
814*d281e8fbSJim Harris 			panic("bus_dmamap_load returned non-zero!\n");
815*d281e8fbSJim Harris 	} else
816*d281e8fbSJim Harris 		nvme_qpair_submit_cmd(tr->qpair, tr);
817*d281e8fbSJim Harris }
818*d281e8fbSJim Harris 
819*d281e8fbSJim Harris void
820*d281e8fbSJim Harris nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
821*d281e8fbSJim Harris     struct nvme_request *req)
822*d281e8fbSJim Harris {
823*d281e8fbSJim Harris 	struct nvme_qpair       *qpair;
824*d281e8fbSJim Harris 	struct nvme_tracker     *tr;
825*d281e8fbSJim Harris 	int                     err;
826*d281e8fbSJim Harris 
827*d281e8fbSJim Harris 	if (ctrlr->per_cpu_io_queues)
828*d281e8fbSJim Harris 		qpair = &ctrlr->ioq[curcpu];
829*d281e8fbSJim Harris 	else
830*d281e8fbSJim Harris 		qpair = &ctrlr->ioq[0];
831*d281e8fbSJim Harris 
832*d281e8fbSJim Harris 	tr = nvme_qpair_allocate_tracker(qpair);
833*d281e8fbSJim Harris 
834*d281e8fbSJim Harris 	tr->req = req;
835*d281e8fbSJim Harris 
836*d281e8fbSJim Harris 	if (req->payload_size > 0) {
837*d281e8fbSJim Harris 		err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map,
838*d281e8fbSJim Harris 				      req->payload, req->payload_size,
839*d281e8fbSJim Harris 				      nvme_payload_map, tr, 0);
840*d281e8fbSJim Harris 		if (err != 0)
841*d281e8fbSJim Harris 			panic("bus_dmamap_load returned non-zero!\n");
842*d281e8fbSJim Harris 	} else
843*d281e8fbSJim Harris 		nvme_qpair_submit_cmd(tr->qpair, tr);
844*d281e8fbSJim Harris }
845