1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2012-2016 Intel Corporation
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_nvme.h"
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf.h>
34 #include <sys/bus.h>
35 #include <sys/conf.h>
36 #include <sys/ioccom.h>
37 #include <sys/proc.h>
38 #include <sys/smp.h>
39 #include <sys/uio.h>
40 #include <sys/sbuf.h>
41 #include <sys/endian.h>
42 #include <sys/stdarg.h>
43 #include <vm/vm.h>
44 #include <vm/vm_page.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_map.h>
47
48 #include "nvme_private.h"
49 #include "nvme_linux.h"
50
51 #include "nvme_if.h"
52
53 #define B4_CHK_RDY_DELAY_MS 2300 /* work around controller bug */
54
55 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
56 struct nvme_async_event_request *aer);
57
58 static void
nvme_ctrlr_barrier(struct nvme_controller * ctrlr,int flags)59 nvme_ctrlr_barrier(struct nvme_controller *ctrlr, int flags)
60 {
61 bus_barrier(ctrlr->resource, 0, rman_get_size(ctrlr->resource), flags);
62 }
63
64 static void
nvme_ctrlr_devctl_va(struct nvme_controller * ctrlr,const char * type,const char * msg,va_list ap)65 nvme_ctrlr_devctl_va(struct nvme_controller *ctrlr, const char *type,
66 const char *msg, va_list ap)
67 {
68 struct sbuf sb;
69 int error;
70
71 if (sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND | SBUF_NOWAIT) == NULL)
72 return;
73 sbuf_printf(&sb, "name=\"%s\" ", device_get_nameunit(ctrlr->dev));
74 sbuf_vprintf(&sb, msg, ap);
75 error = sbuf_finish(&sb);
76 if (error == 0)
77 devctl_notify("nvme", "controller", type, sbuf_data(&sb));
78 sbuf_delete(&sb);
79 }
80
81 static void
nvme_ctrlr_devctl(struct nvme_controller * ctrlr,const char * type,const char * msg,...)82 nvme_ctrlr_devctl(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)
83 {
84 va_list ap;
85
86 va_start(ap, msg);
87 nvme_ctrlr_devctl_va(ctrlr, type, msg, ap);
88 va_end(ap);
89 }
90
91 static void
nvme_ctrlr_devctl_log(struct nvme_controller * ctrlr,const char * type,const char * msg,...)92 nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)
93 {
94 struct sbuf sb;
95 va_list ap;
96 int error;
97
98 if (sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND | SBUF_NOWAIT) == NULL)
99 return;
100 sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));
101 va_start(ap, msg);
102 sbuf_vprintf(&sb, msg, ap);
103 va_end(ap);
104 error = sbuf_finish(&sb);
105 if (error == 0)
106 printf("%s\n", sbuf_data(&sb));
107 sbuf_delete(&sb);
108 va_start(ap, msg);
109 nvme_ctrlr_devctl_va(ctrlr, type, msg, ap);
110 va_end(ap);
111 }
112
113 static int
nvme_ctrlr_construct_admin_qpair(struct nvme_controller * ctrlr)114 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
115 {
116 struct nvme_qpair *qpair;
117 uint32_t num_entries;
118 int error;
119
120 qpair = &ctrlr->adminq;
121 qpair->id = 0;
122 qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
123 qpair->domain = ctrlr->domain;
124
125 num_entries = NVME_ADMIN_ENTRIES;
126 TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
127 /*
128 * If admin_entries was overridden to an invalid value, revert it
129 * back to our default value.
130 */
131 if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
132 num_entries > NVME_MAX_ADMIN_ENTRIES) {
133 nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
134 "specified\n", num_entries);
135 num_entries = NVME_ADMIN_ENTRIES;
136 }
137
138 /*
139 * The admin queue's max xfer size is treated differently than the
140 * max I/O xfer size. 16KB is sufficient here - maybe even less?
141 */
142 error = nvme_qpair_construct(qpair, num_entries, NVME_ADMIN_TRACKERS,
143 ctrlr);
144 return (error);
145 }
146
147 #define QP(ctrlr, c) ((c) * (ctrlr)->num_io_queues / mp_ncpus)
148
149 static int
nvme_ctrlr_construct_io_qpairs(struct nvme_controller * ctrlr)150 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
151 {
152 struct nvme_qpair *qpair;
153 uint32_t cap_lo;
154 uint16_t mqes;
155 int c, error, i, n;
156 int num_entries, num_trackers, max_entries;
157
158 /*
159 * NVMe spec sets a hard limit of 64K max entries, but devices may
160 * specify a smaller limit, so we need to check the MQES field in the
161 * capabilities register. We have to cap the number of entries to the
162 * current stride allows for in BAR 0/1, otherwise the remainder entries
163 * are inaccessible. MQES should reflect this, and this is just a
164 * fail-safe.
165 */
166 max_entries =
167 (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
168 (1 << (ctrlr->dstrd + 1));
169 num_entries = NVME_IO_ENTRIES;
170 TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
171 cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
172 mqes = NVME_CAP_LO_MQES(cap_lo);
173 num_entries = min(num_entries, mqes + 1);
174 num_entries = min(num_entries, max_entries);
175
176 num_trackers = NVME_IO_TRACKERS;
177 TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
178
179 num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
180 num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
181 /*
182 * No need to have more trackers than entries in the submit queue. Note
183 * also that for a queue size of N, we can only have (N-1) commands
184 * outstanding, hence the "-1" here.
185 */
186 num_trackers = min(num_trackers, (num_entries-1));
187
188 /*
189 * Our best estimate for the maximum number of I/Os that we should
190 * normally have in flight at one time. This should be viewed as a hint,
191 * not a hard limit and will need to be revisited when the upper layers
192 * of the storage system grows multi-queue support.
193 */
194 ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;
195
196 ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
197 M_NVME, M_ZERO | M_WAITOK);
198
199 for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {
200 qpair = &ctrlr->ioq[i];
201
202 /*
203 * Admin queue has ID=0. IO queues start at ID=1 -
204 * hence the 'i+1' here.
205 */
206 qpair->id = i + 1;
207 if (ctrlr->num_io_queues > 1) {
208 /* Find number of CPUs served by this queue. */
209 for (n = 1; QP(ctrlr, c + n) == i; n++)
210 ;
211 /* Shuffle multiple NVMe devices between CPUs. */
212 qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;
213 qpair->domain = pcpu_find(qpair->cpu)->pc_domain;
214 } else {
215 qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
216 qpair->domain = ctrlr->domain;
217 }
218
219 /*
220 * For I/O queues, use the controller-wide max_xfer_size
221 * calculated in nvme_attach().
222 */
223 error = nvme_qpair_construct(qpair, num_entries, num_trackers,
224 ctrlr);
225 if (error)
226 return (error);
227
228 /*
229 * Do not bother binding interrupts if we only have one I/O
230 * interrupt thread for this controller.
231 */
232 if (ctrlr->num_io_queues > 1)
233 bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);
234 }
235
236 return (0);
237 }
238
239 static void
nvme_ctrlr_fail(struct nvme_controller * ctrlr,bool admin_also)240 nvme_ctrlr_fail(struct nvme_controller *ctrlr, bool admin_also)
241 {
242 int i;
243
244 /*
245 * No need to disable queues before failing them. Failing is a superet
246 * of disabling (though pedantically we'd abort the AERs silently with
247 * a different error, though when we fail, that hardly matters).
248 */
249 ctrlr->is_failed = true;
250 if (admin_also) {
251 ctrlr->is_failed_admin = true;
252 nvme_qpair_fail(&ctrlr->adminq);
253 }
254 if (ctrlr->ioq != NULL) {
255 for (i = 0; i < ctrlr->num_io_queues; i++) {
256 nvme_qpair_fail(&ctrlr->ioq[i]);
257 }
258 }
259 nvme_notify_fail(ctrlr);
260 }
261
262 /*
263 * Wait for RDY to change.
264 *
265 * Starts sleeping for 1us and geometrically increases it the longer we wait,
266 * capped at 1ms.
267 */
268 static int
nvme_ctrlr_wait_for_ready(struct nvme_controller * ctrlr,int desired_val)269 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
270 {
271 int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);
272 sbintime_t delta_t = SBT_1US;
273 uint32_t csts;
274
275 while (1) {
276 csts = nvme_mmio_read_4(ctrlr, csts);
277 if (csts == NVME_GONE) /* Hot unplug. */
278 return (ENXIO);
279 if (NVMEV(NVME_CSTS_REG_RDY, csts) == desired_val)
280 break;
281 if (timeout - ticks < 0) {
282 nvme_printf(ctrlr, "controller ready did not become %d "
283 "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
284 return (ENXIO);
285 }
286
287 pause_sbt("nvmerdy", delta_t, 0, C_PREL(1));
288 delta_t = min(SBT_1MS, delta_t * 3 / 2);
289 }
290
291 return (0);
292 }
293
294 static int
nvme_ctrlr_disable(struct nvme_controller * ctrlr)295 nvme_ctrlr_disable(struct nvme_controller *ctrlr)
296 {
297 uint32_t cc;
298 uint32_t csts;
299 uint8_t en, rdy;
300 int err;
301
302 cc = nvme_mmio_read_4(ctrlr, cc);
303 csts = nvme_mmio_read_4(ctrlr, csts);
304
305 en = NVMEV(NVME_CC_REG_EN, cc);
306 rdy = NVMEV(NVME_CSTS_REG_RDY, csts);
307
308 /*
309 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
310 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
311 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
312 * isn't the desired value. Short circuit if we're already disabled.
313 */
314 if (en == 0) {
315 /* Wait for RDY == 0 or timeout & fail */
316 if (rdy == 0)
317 return (0);
318 return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
319 }
320 if (rdy == 0) {
321 /* EN == 1, wait for RDY == 1 or timeout & fail */
322 err = nvme_ctrlr_wait_for_ready(ctrlr, 1);
323 if (err != 0)
324 return (err);
325 }
326
327 cc &= ~NVMEM(NVME_CC_REG_EN);
328 nvme_mmio_write_4(ctrlr, cc, cc);
329
330 /*
331 * A few drives have firmware bugs that freeze the drive if we access
332 * the mmio too soon after we disable.
333 */
334 if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)
335 pause("nvmeR", MSEC_2_TICKS(B4_CHK_RDY_DELAY_MS));
336 return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
337 }
338
339 static int
nvme_ctrlr_enable(struct nvme_controller * ctrlr)340 nvme_ctrlr_enable(struct nvme_controller *ctrlr)
341 {
342 uint32_t cc;
343 uint32_t csts;
344 uint32_t aqa;
345 uint32_t qsize;
346 uint8_t en, rdy;
347 int err;
348
349 cc = nvme_mmio_read_4(ctrlr, cc);
350 csts = nvme_mmio_read_4(ctrlr, csts);
351
352 en = NVMEV(NVME_CC_REG_EN, cc);
353 rdy = NVMEV(NVME_CSTS_REG_RDY, csts);
354
355 /*
356 * See note in nvme_ctrlr_disable. Short circuit if we're already enabled.
357 */
358 if (en == 1) {
359 if (rdy == 1)
360 return (0);
361 return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
362 }
363
364 /* EN == 0 already wait for RDY == 0 or timeout & fail */
365 err = nvme_ctrlr_wait_for_ready(ctrlr, 0);
366 if (err != 0)
367 return (err);
368
369 nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
370 nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
371
372 /* acqs and asqs are 0-based. */
373 qsize = ctrlr->adminq.num_entries - 1;
374
375 aqa = 0;
376 aqa |= NVMEF(NVME_AQA_REG_ACQS, qsize);
377 aqa |= NVMEF(NVME_AQA_REG_ASQS, qsize);
378 nvme_mmio_write_4(ctrlr, aqa, aqa);
379
380 /* Initialization values for CC */
381 cc = 0;
382 cc |= NVMEF(NVME_CC_REG_EN, 1);
383 cc |= NVMEF(NVME_CC_REG_CSS, 0);
384 cc |= NVMEF(NVME_CC_REG_AMS, 0);
385 cc |= NVMEF(NVME_CC_REG_SHN, 0);
386 cc |= NVMEF(NVME_CC_REG_IOSQES, 6); /* SQ entry size == 64 == 2^6 */
387 cc |= NVMEF(NVME_CC_REG_IOCQES, 4); /* CQ entry size == 16 == 2^4 */
388
389 /*
390 * Use the Memory Page Size selected during device initialization. Note
391 * that value stored in mps is suitable to use here without adjusting by
392 * NVME_MPS_SHIFT.
393 */
394 cc |= NVMEF(NVME_CC_REG_MPS, ctrlr->mps);
395
396 nvme_ctrlr_barrier(ctrlr, BUS_SPACE_BARRIER_WRITE);
397 nvme_mmio_write_4(ctrlr, cc, cc);
398
399 return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
400 }
401
402 static void
nvme_ctrlr_disable_qpairs(struct nvme_controller * ctrlr)403 nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)
404 {
405 int i;
406
407 nvme_admin_qpair_disable(&ctrlr->adminq);
408 /*
409 * I/O queues are not allocated before the initial HW
410 * reset, so do not try to disable them. Use is_initialized
411 * to determine if this is the initial HW reset.
412 */
413 if (ctrlr->is_initialized) {
414 for (i = 0; i < ctrlr->num_io_queues; i++)
415 nvme_io_qpair_disable(&ctrlr->ioq[i]);
416 }
417 }
418
419 static int
nvme_ctrlr_hw_reset(struct nvme_controller * ctrlr)420 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
421 {
422 int err;
423
424 TSENTER();
425
426 ctrlr->is_failed_admin = true;
427 nvme_ctrlr_disable_qpairs(ctrlr);
428
429 err = nvme_ctrlr_disable(ctrlr);
430 if (err != 0)
431 goto out;
432
433 err = nvme_ctrlr_enable(ctrlr);
434 out:
435 if (err == 0)
436 ctrlr->is_failed_admin = false;
437
438 TSEXIT();
439 return (err);
440 }
441
442 void
nvme_ctrlr_reset(struct nvme_controller * ctrlr)443 nvme_ctrlr_reset(struct nvme_controller *ctrlr)
444 {
445 int cmpset;
446
447 cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
448
449 if (cmpset == 0)
450 /*
451 * Controller is already resetting. Return immediately since
452 * there is no need to kick off another reset.
453 */
454 return;
455
456 if (!ctrlr->is_dying)
457 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
458 }
459
460 static int
nvme_ctrlr_identify(struct nvme_controller * ctrlr)461 nvme_ctrlr_identify(struct nvme_controller *ctrlr)
462 {
463 struct nvme_completion_poll_status status;
464
465 status.done = 0;
466 nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
467 nvme_completion_poll_cb, &status);
468 nvme_completion_poll(&status);
469 if (nvme_completion_is_error(&status.cpl)) {
470 nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
471 return (ENXIO);
472 }
473
474 /* Convert data to host endian */
475 nvme_controller_data_swapbytes(&ctrlr->cdata);
476
477 /*
478 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
479 * controller supports.
480 */
481 if (ctrlr->cdata.mdts > 0)
482 ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
483 1 << (ctrlr->cdata.mdts + NVME_MPS_SHIFT +
484 NVME_CAP_HI_MPSMIN(ctrlr->cap_hi)));
485
486 return (0);
487 }
488
489 static int
nvme_ctrlr_set_num_qpairs(struct nvme_controller * ctrlr)490 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
491 {
492 struct nvme_completion_poll_status status;
493 int cq_allocated, sq_allocated;
494
495 status.done = 0;
496 nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
497 nvme_completion_poll_cb, &status);
498 nvme_completion_poll(&status);
499 if (nvme_completion_is_error(&status.cpl)) {
500 nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
501 return (ENXIO);
502 }
503
504 /*
505 * Data in cdw0 is 0-based.
506 * Lower 16-bits indicate number of submission queues allocated.
507 * Upper 16-bits indicate number of completion queues allocated.
508 */
509 sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
510 cq_allocated = (status.cpl.cdw0 >> 16) + 1;
511
512 /*
513 * Controller may allocate more queues than we requested,
514 * so use the minimum of the number requested and what was
515 * actually allocated.
516 */
517 ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);
518 ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);
519 if (ctrlr->num_io_queues > vm_ndomains)
520 ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;
521
522 return (0);
523 }
524
525 static int
nvme_ctrlr_create_qpairs(struct nvme_controller * ctrlr)526 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
527 {
528 struct nvme_completion_poll_status status;
529 struct nvme_qpair *qpair;
530 int i;
531
532 for (i = 0; i < ctrlr->num_io_queues; i++) {
533 qpair = &ctrlr->ioq[i];
534
535 status.done = 0;
536 nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,
537 nvme_completion_poll_cb, &status);
538 nvme_completion_poll(&status);
539 if (nvme_completion_is_error(&status.cpl)) {
540 nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
541 return (ENXIO);
542 }
543
544 status.done = 0;
545 nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,
546 nvme_completion_poll_cb, &status);
547 nvme_completion_poll(&status);
548 if (nvme_completion_is_error(&status.cpl)) {
549 nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
550 return (ENXIO);
551 }
552 }
553
554 return (0);
555 }
556
557 static int
nvme_ctrlr_delete_qpairs(struct nvme_controller * ctrlr)558 nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)
559 {
560 struct nvme_completion_poll_status status;
561 struct nvme_qpair *qpair;
562
563 for (int i = 0; i < ctrlr->num_io_queues; i++) {
564 qpair = &ctrlr->ioq[i];
565
566 status.done = 0;
567 nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
568 nvme_completion_poll_cb, &status);
569 nvme_completion_poll(&status);
570 if (nvme_completion_is_error(&status.cpl)) {
571 nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
572 return (ENXIO);
573 }
574
575 status.done = 0;
576 nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
577 nvme_completion_poll_cb, &status);
578 nvme_completion_poll(&status);
579 if (nvme_completion_is_error(&status.cpl)) {
580 nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
581 return (ENXIO);
582 }
583 }
584
585 return (0);
586 }
587
588 static int
nvme_ctrlr_construct_namespaces(struct nvme_controller * ctrlr)589 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
590 {
591 struct nvme_namespace *ns;
592 uint32_t i;
593
594 for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
595 ns = &ctrlr->ns[i];
596 nvme_ns_construct(ns, i+1, ctrlr);
597 }
598
599 return (0);
600 }
601
602 static bool
is_log_page_id_valid(uint8_t page_id)603 is_log_page_id_valid(uint8_t page_id)
604 {
605 switch (page_id) {
606 case NVME_LOG_ERROR:
607 case NVME_LOG_HEALTH_INFORMATION:
608 case NVME_LOG_FIRMWARE_SLOT:
609 case NVME_LOG_CHANGED_NAMESPACE:
610 case NVME_LOG_COMMAND_EFFECT:
611 case NVME_LOG_RES_NOTIFICATION:
612 case NVME_LOG_SANITIZE_STATUS:
613 return (true);
614 }
615
616 return (false);
617 }
618
619 static uint32_t
nvme_ctrlr_get_log_page_size(struct nvme_controller * ctrlr,uint8_t page_id)620 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
621 {
622 uint32_t log_page_size;
623
624 switch (page_id) {
625 case NVME_LOG_ERROR:
626 log_page_size = min(
627 sizeof(struct nvme_error_information_entry) *
628 (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);
629 break;
630 case NVME_LOG_HEALTH_INFORMATION:
631 log_page_size = sizeof(struct nvme_health_information_page);
632 break;
633 case NVME_LOG_FIRMWARE_SLOT:
634 log_page_size = sizeof(struct nvme_firmware_page);
635 break;
636 case NVME_LOG_CHANGED_NAMESPACE:
637 log_page_size = sizeof(struct nvme_ns_list);
638 break;
639 case NVME_LOG_COMMAND_EFFECT:
640 log_page_size = sizeof(struct nvme_command_effects_page);
641 break;
642 case NVME_LOG_RES_NOTIFICATION:
643 log_page_size = sizeof(struct nvme_res_notification_page);
644 break;
645 case NVME_LOG_SANITIZE_STATUS:
646 log_page_size = sizeof(struct nvme_sanitize_status_page);
647 break;
648 default:
649 log_page_size = 0;
650 break;
651 }
652
653 return (log_page_size);
654 }
655
656 static void
nvme_ctrlr_log_critical_warnings(struct nvme_controller * ctrlr,uint8_t state)657 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
658 uint8_t state)
659 {
660 if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE)
661 nvme_printf(ctrlr, "SMART WARNING: available spare space below threshold\n");
662
663 if (state & NVME_CRIT_WARN_ST_TEMPERATURE)
664 nvme_printf(ctrlr, "SMART WARNING: temperature above threshold\n");
665
666 if (state & NVME_CRIT_WARN_ST_DEVICE_RELIABILITY)
667 nvme_printf(ctrlr, "SMART WARNING: device reliability degraded\n");
668
669 if (state & NVME_CRIT_WARN_ST_READ_ONLY)
670 nvme_printf(ctrlr, "SMART WARNING: media placed in read only mode\n");
671
672 if (state & NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP)
673 nvme_printf(ctrlr, "SMART WARNING: volatile memory backup device failed\n");
674
675 if (state & NVME_CRIT_WARN_ST_PERSISTENT_MEMORY_REGION)
676 nvme_printf(ctrlr, "SMART WARNING: persistent memory read only or unreliable\n");
677
678 if (state & NVME_CRIT_WARN_ST_RESERVED_MASK)
679 nvme_printf(ctrlr, "SMART WARNING: unknown critical warning(s): state = 0x%02x\n",
680 state & NVME_CRIT_WARN_ST_RESERVED_MASK);
681
682 nvme_ctrlr_devctl(ctrlr, "SMART_ERROR", "state=0x%02x", state);
683 }
684
685 static void
nvme_ctrlr_async_event_cb(void * arg,const struct nvme_completion * cpl)686 nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
687 {
688 struct nvme_async_event_request *aer = arg;
689
690 if (nvme_completion_is_error(cpl)) {
691 /*
692 * Do not retry failed async event requests. This avoids
693 * infinite loops where a new async event request is submitted
694 * to replace the one just failed, only to fail again and
695 * perpetuate the loop.
696 */
697 return;
698 }
699
700 /*
701 * Save the completion status and associated log page is in bits 23:16
702 * of completion entry dw0. Print a message and queue it for further
703 * processing.
704 */
705 memcpy(&aer->cpl, cpl, sizeof(*cpl));
706 aer->log_page_id = NVMEV(NVME_ASYNC_EVENT_LOG_PAGE_ID, cpl->cdw0);
707 nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"
708 " page 0x%02x)\n", NVMEV(NVME_ASYNC_EVENT_TYPE, cpl->cdw0),
709 NVMEV(NVME_ASYNC_EVENT_INFO, cpl->cdw0),
710 aer->log_page_id);
711 taskqueue_enqueue(aer->ctrlr->taskqueue, &aer->task);
712 }
713
714 static void
nvme_ctrlr_construct_and_submit_aer(struct nvme_controller * ctrlr,struct nvme_async_event_request * aer)715 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
716 struct nvme_async_event_request *aer)
717 {
718 struct nvme_request *req;
719
720 /*
721 * We're racing the reset thread, so let that process submit this again.
722 * XXX does this really solve that race? And is that race even possible
723 * since we only reset when we've no theard from the card in a long
724 * time. Why would we get an AER in the middle of that just before we
725 * kick off the reset?
726 */
727 if (ctrlr->is_resetting)
728 return;
729
730 aer->ctrlr = ctrlr;
731 req = nvme_allocate_request_null(M_WAITOK, nvme_ctrlr_async_event_cb,
732 aer);
733 aer->req = req;
734 aer->log_page_id = 0; /* Not a valid page */
735
736 /*
737 * Disable timeout here, since asynchronous event requests should by
738 * nature never be timed out.
739 */
740 req->timeout = false;
741 req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
742 nvme_ctrlr_submit_admin_request(ctrlr, req);
743 }
744
745 static void
nvme_ctrlr_configure_aer(struct nvme_controller * ctrlr)746 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
747 {
748 struct nvme_completion_poll_status status;
749 struct nvme_async_event_request *aer;
750 uint32_t i;
751
752 ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
753 NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
754 NVME_CRIT_WARN_ST_READ_ONLY |
755 NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP;
756 if (ctrlr->cdata.ver >= NVME_REV(1, 2))
757 ctrlr->async_event_config |=
758 ctrlr->cdata.oaes & (NVME_ASYNC_EVENT_NS_ATTRIBUTE |
759 NVME_ASYNC_EVENT_FW_ACTIVATE);
760
761 status.done = 0;
762 nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
763 0, NULL, 0, nvme_completion_poll_cb, &status);
764 nvme_completion_poll(&status);
765 if (nvme_completion_is_error(&status.cpl) ||
766 (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
767 (status.cpl.cdw0 & 0xFFFF) == 0x0000) {
768 nvme_printf(ctrlr, "temperature threshold not supported\n");
769 } else
770 ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;
771
772 nvme_ctrlr_cmd_set_async_event_config(ctrlr,
773 ctrlr->async_event_config, NULL, NULL);
774
775 /* aerl is a zero-based value, so we need to add 1 here. */
776 ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
777
778 for (i = 0; i < ctrlr->num_aers; i++) {
779 aer = &ctrlr->aer[i];
780 nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
781 }
782 }
783
784 static void
nvme_ctrlr_configure_apst(struct nvme_controller * ctrlr)785 nvme_ctrlr_configure_apst(struct nvme_controller *ctrlr)
786 {
787 struct nvme_completion_poll_status status;
788 uint64_t *data;
789 int data_size, i, read_size;
790 bool enable, error = true;
791
792 if (TUNABLE_BOOL_FETCH("hw.nvme.apst_enable", &enable) == 0 ||
793 ctrlr->cdata.apsta == 0)
794 return;
795
796 data_size = 32 * sizeof(*data);
797 data = malloc(data_size, M_NVME, M_WAITOK | M_ZERO);
798
799 if (getenv_array("hw.nvme.apst_data", data, data_size,
800 &read_size, sizeof(*data), GETENV_UNSIGNED) != 0) {
801 for (i = 0; i < read_size / sizeof(*data); ++i)
802 data[i] = htole64(data[i]);
803 } else {
804 status.done = 0;
805 nvme_ctrlr_cmd_get_feature(ctrlr,
806 NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION, 0,
807 data, data_size, nvme_completion_poll_cb, &status);
808 nvme_completion_poll(&status);
809 if (nvme_completion_is_error(&status.cpl))
810 goto out;
811 }
812
813 status.done = 0;
814 nvme_ctrlr_cmd_set_feature(ctrlr,
815 NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION, enable, 0, 0,
816 0, 0, data, data_size, nvme_completion_poll_cb, &status);
817 nvme_completion_poll(&status);
818 error = nvme_completion_is_error(&status.cpl);
819 out:
820 if (error && bootverbose)
821 nvme_printf(ctrlr, "failed to configure APST\n");
822 free(data, M_NVME);
823 }
824
825 static void
nvme_ctrlr_configure_int_coalescing(struct nvme_controller * ctrlr)826 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
827 {
828 ctrlr->int_coal_time = 0;
829 TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
830 &ctrlr->int_coal_time);
831
832 ctrlr->int_coal_threshold = 0;
833 TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
834 &ctrlr->int_coal_threshold);
835
836 nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
837 ctrlr->int_coal_threshold, NULL, NULL);
838 }
839
840 static void
nvme_ctrlr_hmb_free(struct nvme_controller * ctrlr)841 nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)
842 {
843 struct nvme_hmb_chunk *hmbc;
844 int i;
845
846 if (ctrlr->hmb_desc_paddr) {
847 bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);
848 bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
849 ctrlr->hmb_desc_map);
850 ctrlr->hmb_desc_paddr = 0;
851 }
852 if (ctrlr->hmb_desc_tag) {
853 bus_dma_tag_destroy(ctrlr->hmb_desc_tag);
854 ctrlr->hmb_desc_tag = NULL;
855 }
856 for (i = 0; i < ctrlr->hmb_nchunks; i++) {
857 hmbc = &ctrlr->hmb_chunks[i];
858 bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);
859 bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
860 hmbc->hmbc_map);
861 }
862 ctrlr->hmb_nchunks = 0;
863 if (ctrlr->hmb_tag) {
864 bus_dma_tag_destroy(ctrlr->hmb_tag);
865 ctrlr->hmb_tag = NULL;
866 }
867 if (ctrlr->hmb_chunks) {
868 free(ctrlr->hmb_chunks, M_NVME);
869 ctrlr->hmb_chunks = NULL;
870 }
871 }
872
873 static void
nvme_ctrlr_hmb_alloc(struct nvme_controller * ctrlr)874 nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)
875 {
876 struct nvme_hmb_chunk *hmbc;
877 size_t pref, min, minc, size;
878 int err, i;
879 uint64_t max;
880
881 /* Limit HMB to 5% of RAM size per device by default. */
882 max = (uint64_t)physmem * PAGE_SIZE / 20;
883 TUNABLE_UINT64_FETCH("hw.nvme.hmb_max", &max);
884
885 /*
886 * Units of Host Memory Buffer in the Identify info are always in terms
887 * of 4k units.
888 */
889 min = (long long unsigned)ctrlr->cdata.hmmin * NVME_HMB_UNITS;
890 if (max == 0 || max < min)
891 return;
892 pref = MIN((long long unsigned)ctrlr->cdata.hmpre * NVME_HMB_UNITS, max);
893 minc = MAX(ctrlr->cdata.hmminds * NVME_HMB_UNITS, ctrlr->page_size);
894 if (min > 0 && ctrlr->cdata.hmmaxd > 0)
895 minc = MAX(minc, min / ctrlr->cdata.hmmaxd);
896 ctrlr->hmb_chunk = pref;
897
898 again:
899 /*
900 * However, the chunk sizes, number of chunks, and alignment of chunks
901 * are all based on the current MPS (ctrlr->page_size).
902 */
903 ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, ctrlr->page_size);
904 ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);
905 if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)
906 ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;
907 ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *
908 ctrlr->hmb_nchunks, M_NVME, M_WAITOK);
909 err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
910 ctrlr->page_size, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
911 ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);
912 if (err != 0) {
913 nvme_printf(ctrlr, "HMB tag create failed %d\n", err);
914 nvme_ctrlr_hmb_free(ctrlr);
915 return;
916 }
917
918 for (i = 0; i < ctrlr->hmb_nchunks; i++) {
919 hmbc = &ctrlr->hmb_chunks[i];
920 if (bus_dmamem_alloc(ctrlr->hmb_tag,
921 (void **)&hmbc->hmbc_vaddr, BUS_DMA_NOWAIT,
922 &hmbc->hmbc_map)) {
923 nvme_printf(ctrlr, "failed to alloc HMB\n");
924 break;
925 }
926 if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,
927 hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,
928 &hmbc->hmbc_paddr, BUS_DMA_NOWAIT) != 0) {
929 bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
930 hmbc->hmbc_map);
931 nvme_printf(ctrlr, "failed to load HMB\n");
932 break;
933 }
934 bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,
935 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
936 }
937
938 if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&
939 ctrlr->hmb_chunk / 2 >= minc) {
940 ctrlr->hmb_nchunks = i;
941 nvme_ctrlr_hmb_free(ctrlr);
942 ctrlr->hmb_chunk /= 2;
943 goto again;
944 }
945 ctrlr->hmb_nchunks = i;
946 if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {
947 nvme_ctrlr_hmb_free(ctrlr);
948 return;
949 }
950
951 size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;
952 err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
953 PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
954 size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);
955 if (err != 0) {
956 nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);
957 nvme_ctrlr_hmb_free(ctrlr);
958 return;
959 }
960 if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,
961 (void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,
962 &ctrlr->hmb_desc_map)) {
963 nvme_printf(ctrlr, "failed to alloc HMB desc\n");
964 nvme_ctrlr_hmb_free(ctrlr);
965 return;
966 }
967 if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
968 ctrlr->hmb_desc_vaddr, size, nvme_single_map,
969 &ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {
970 bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
971 ctrlr->hmb_desc_map);
972 nvme_printf(ctrlr, "failed to load HMB desc\n");
973 nvme_ctrlr_hmb_free(ctrlr);
974 return;
975 }
976
977 for (i = 0; i < ctrlr->hmb_nchunks; i++) {
978 memset(&ctrlr->hmb_desc_vaddr[i], 0,
979 sizeof(struct nvme_hmb_desc));
980 ctrlr->hmb_desc_vaddr[i].addr =
981 htole64(ctrlr->hmb_chunks[i].hmbc_paddr);
982 ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / ctrlr->page_size);
983 }
984 bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
985 BUS_DMASYNC_PREWRITE);
986
987 nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",
988 (long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk
989 / 1024 / 1024);
990 }
991
992 static void
nvme_ctrlr_hmb_enable(struct nvme_controller * ctrlr,bool enable,bool memret)993 nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)
994 {
995 struct nvme_completion_poll_status status;
996 uint32_t cdw11;
997
998 cdw11 = 0;
999 if (enable)
1000 cdw11 |= 1;
1001 if (memret)
1002 cdw11 |= 2;
1003 status.done = 0;
1004 nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,
1005 ctrlr->hmb_nchunks * ctrlr->hmb_chunk / ctrlr->page_size,
1006 ctrlr->hmb_desc_paddr, ctrlr->hmb_desc_paddr >> 32,
1007 ctrlr->hmb_nchunks, NULL, 0,
1008 nvme_completion_poll_cb, &status);
1009 nvme_completion_poll(&status);
1010 if (nvme_completion_is_error(&status.cpl))
1011 nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");
1012 }
1013
1014 static void
nvme_ctrlr_start(void * ctrlr_arg,bool resetting)1015 nvme_ctrlr_start(void *ctrlr_arg, bool resetting)
1016 {
1017 struct nvme_controller *ctrlr = ctrlr_arg;
1018 uint32_t old_num_io_queues;
1019 int i;
1020
1021 TSENTER();
1022
1023 /*
1024 * Only reset adminq here when we are restarting the
1025 * controller after a reset. During initialization,
1026 * we have already submitted admin commands to get
1027 * the number of I/O queues supported, so cannot reset
1028 * the adminq again here.
1029 */
1030 if (resetting) {
1031 nvme_qpair_reset(&ctrlr->adminq);
1032 nvme_admin_qpair_enable(&ctrlr->adminq);
1033 }
1034
1035 if (ctrlr->ioq != NULL) {
1036 for (i = 0; i < ctrlr->num_io_queues; i++)
1037 nvme_qpair_reset(&ctrlr->ioq[i]);
1038 }
1039
1040 /*
1041 * If it was a reset on initialization command timeout, just
1042 * return here, letting initialization code fail gracefully.
1043 */
1044 if (resetting && !ctrlr->is_initialized)
1045 return;
1046
1047 if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {
1048 nvme_ctrlr_fail(ctrlr, false);
1049 return;
1050 }
1051
1052 /*
1053 * The number of qpairs are determined during controller initialization,
1054 * including using NVMe SET_FEATURES/NUMBER_OF_QUEUES to determine the
1055 * HW limit. We call SET_FEATURES again here so that it gets called
1056 * after any reset for controllers that depend on the driver to
1057 * explicit specify how many queues it will use. This value should
1058 * never change between resets, so panic if somehow that does happen.
1059 */
1060 if (resetting) {
1061 old_num_io_queues = ctrlr->num_io_queues;
1062 if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
1063 nvme_ctrlr_fail(ctrlr, false);
1064 return;
1065 }
1066
1067 if (old_num_io_queues != ctrlr->num_io_queues) {
1068 panic("num_io_queues changed from %u to %u",
1069 old_num_io_queues, ctrlr->num_io_queues);
1070 }
1071 }
1072
1073 if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {
1074 nvme_ctrlr_hmb_alloc(ctrlr);
1075 if (ctrlr->hmb_nchunks > 0)
1076 nvme_ctrlr_hmb_enable(ctrlr, true, false);
1077 } else if (ctrlr->hmb_nchunks > 0)
1078 nvme_ctrlr_hmb_enable(ctrlr, true, true);
1079
1080 if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
1081 nvme_ctrlr_fail(ctrlr, false);
1082 return;
1083 }
1084
1085 if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
1086 nvme_ctrlr_fail(ctrlr, false);
1087 return;
1088 }
1089
1090 nvme_ctrlr_configure_aer(ctrlr);
1091 nvme_ctrlr_configure_apst(ctrlr);
1092 nvme_ctrlr_configure_int_coalescing(ctrlr);
1093
1094 for (i = 0; i < ctrlr->num_io_queues; i++)
1095 nvme_io_qpair_enable(&ctrlr->ioq[i]);
1096 TSEXIT();
1097 }
1098
1099 void
nvme_ctrlr_start_config_hook(void * arg)1100 nvme_ctrlr_start_config_hook(void *arg)
1101 {
1102 struct nvme_controller *ctrlr = arg;
1103
1104 TSENTER();
1105
1106 if (nvme_ctrlr_hw_reset(ctrlr) != 0 || ctrlr->fail_on_reset != 0) {
1107 nvme_ctrlr_fail(ctrlr, true);
1108 config_intrhook_disestablish(&ctrlr->config_hook);
1109 return;
1110 }
1111
1112 nvme_qpair_reset(&ctrlr->adminq);
1113 nvme_admin_qpair_enable(&ctrlr->adminq);
1114
1115 if (nvme_ctrlr_identify(ctrlr) == 0 &&
1116 nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&
1117 nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)
1118 nvme_ctrlr_start(ctrlr, false);
1119 else
1120 nvme_ctrlr_fail(ctrlr, false);
1121
1122 nvme_sysctl_initialize_ctrlr(ctrlr);
1123 config_intrhook_disestablish(&ctrlr->config_hook);
1124
1125 if (!ctrlr->is_failed) {
1126 device_t child;
1127
1128 ctrlr->is_initialized = true;
1129 child = device_add_child(ctrlr->dev, NULL, DEVICE_UNIT_ANY);
1130 device_set_ivars(child, ctrlr);
1131 bus_attach_children(ctrlr->dev);
1132
1133 /*
1134 * Now notify the child of all the known namepsaces
1135 */
1136 for (int i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
1137 struct nvme_namespace *ns = &ctrlr->ns[i];
1138
1139 if (ns->data.nsze == 0)
1140 continue;
1141 NVME_NS_ADDED(child, ns);
1142 }
1143 }
1144 TSEXIT();
1145 }
1146
1147 static void
nvme_ctrlr_reset_task(void * arg,int pending)1148 nvme_ctrlr_reset_task(void *arg, int pending)
1149 {
1150 struct nvme_controller *ctrlr = arg;
1151 int status;
1152
1153 nvme_ctrlr_devctl_log(ctrlr, "RESET", "event=\"start\"");
1154 status = nvme_ctrlr_hw_reset(ctrlr);
1155 if (status == 0) {
1156 nvme_ctrlr_devctl_log(ctrlr, "RESET", "event=\"success\"");
1157 nvme_ctrlr_start(ctrlr, true);
1158 } else {
1159 nvme_ctrlr_devctl_log(ctrlr, "RESET", "event=\"timed_out\"");
1160 nvme_ctrlr_fail(ctrlr, true);
1161 }
1162
1163 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1164 }
1165
1166 static void
nvme_ctrlr_aer_done(void * arg,const struct nvme_completion * cpl)1167 nvme_ctrlr_aer_done(void *arg, const struct nvme_completion *cpl)
1168 {
1169 struct nvme_async_event_request *aer = arg;
1170
1171 mtx_lock(&aer->mtx);
1172 if (nvme_completion_is_error(cpl))
1173 aer->log_page_size = (uint32_t)-1;
1174 else
1175 aer->log_page_size = nvme_ctrlr_get_log_page_size(
1176 aer->ctrlr, aer->log_page_id);
1177 wakeup(aer);
1178 mtx_unlock(&aer->mtx);
1179 }
1180
1181 static void
nvme_ctrlr_aer_task(void * arg,int pending)1182 nvme_ctrlr_aer_task(void *arg, int pending)
1183 {
1184 struct nvme_async_event_request *aer = arg;
1185 struct nvme_controller *ctrlr = aer->ctrlr;
1186 uint32_t len;
1187
1188 /*
1189 * We're resetting, so just punt.
1190 */
1191 if (ctrlr->is_resetting)
1192 return;
1193
1194 if (!is_log_page_id_valid(aer->log_page_id)) {
1195 /*
1196 * Repost another asynchronous event request to replace the one
1197 * that just completed.
1198 */
1199 nvme_notify_async(ctrlr, &aer->cpl, aer->log_page_id, NULL, 0);
1200 nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
1201 goto out;
1202 }
1203
1204 nvme_ctrlr_devctl(ctrlr, "aen", "type=0x%x info=0x%x page=0x%x",
1205 NVMEV(NVME_ASYNC_EVENT_TYPE, aer->cpl.cdw0),
1206 NVMEV(NVME_ASYNC_EVENT_INFO, aer->cpl.cdw0), aer->log_page_id);
1207
1208 aer->log_page_size = 0;
1209 len = nvme_ctrlr_get_log_page_size(aer->ctrlr, aer->log_page_id);
1210 nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
1211 NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer, len,
1212 nvme_ctrlr_aer_done, aer);
1213 mtx_lock(&aer->mtx);
1214 while (aer->log_page_size == 0)
1215 mtx_sleep(aer, &aer->mtx, PRIBIO, "nvme_pt", 0);
1216 mtx_unlock(&aer->mtx);
1217
1218 if (aer->log_page_size == (uint32_t)-1) {
1219 /*
1220 * If the log page fetch for some reason completed with an
1221 * error, don't pass log page data to the consumers. In
1222 * practice, this case should never happen.
1223 */
1224 nvme_notify_async(aer->ctrlr, &aer->cpl, aer->log_page_id,
1225 NULL, 0);
1226 goto out;
1227 }
1228
1229 /* Convert data to host endian */
1230 switch (aer->log_page_id) {
1231 case NVME_LOG_ERROR: {
1232 struct nvme_error_information_entry *err =
1233 (struct nvme_error_information_entry *)aer->log_page_buffer;
1234 for (int i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)
1235 nvme_error_information_entry_swapbytes(err++);
1236 break;
1237 }
1238 case NVME_LOG_HEALTH_INFORMATION:
1239 nvme_health_information_page_swapbytes(
1240 (struct nvme_health_information_page *)aer->log_page_buffer);
1241 break;
1242 case NVME_LOG_CHANGED_NAMESPACE:
1243 nvme_ns_list_swapbytes(
1244 (struct nvme_ns_list *)aer->log_page_buffer);
1245 break;
1246 case NVME_LOG_COMMAND_EFFECT:
1247 nvme_command_effects_page_swapbytes(
1248 (struct nvme_command_effects_page *)aer->log_page_buffer);
1249 break;
1250 case NVME_LOG_RES_NOTIFICATION:
1251 nvme_res_notification_page_swapbytes(
1252 (struct nvme_res_notification_page *)aer->log_page_buffer);
1253 break;
1254 case NVME_LOG_SANITIZE_STATUS:
1255 nvme_sanitize_status_page_swapbytes(
1256 (struct nvme_sanitize_status_page *)aer->log_page_buffer);
1257 break;
1258 default:
1259 break;
1260 }
1261
1262 if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
1263 struct nvme_health_information_page *health_info =
1264 (struct nvme_health_information_page *)aer->log_page_buffer;
1265
1266 /*
1267 * Critical warnings reported through the SMART/health log page
1268 * are persistent, so clear the associated bits in the async
1269 * event config so that we do not receive repeated notifications
1270 * for the same event.
1271 */
1272 nvme_ctrlr_log_critical_warnings(aer->ctrlr,
1273 health_info->critical_warning);
1274 aer->ctrlr->async_event_config &=
1275 ~health_info->critical_warning;
1276 nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
1277 aer->ctrlr->async_event_config, NULL, NULL);
1278 } else if (aer->log_page_id == NVME_LOG_CHANGED_NAMESPACE) {
1279 device_t *children;
1280 int n_children;
1281 struct nvme_ns_list *nsl;
1282
1283 if (device_get_children(aer->ctrlr->dev, &children, &n_children) != 0) {
1284 children = NULL;
1285 n_children = 0;
1286 }
1287 nsl = (struct nvme_ns_list *)aer->log_page_buffer;
1288 for (int i = 0; i < nitems(nsl->ns) && nsl->ns[i] != 0; i++) {
1289 /*
1290 * I think we need to query the name space here and see
1291 * if it went away, arrived, or changed in size and call
1292 * the nuanced routine (after constructing or before
1293 * destructing the namespace). XXX needs more work XXX.
1294 */
1295 for (int j = 0; j < n_children; j++)
1296 NVME_NS_CHANGED(children[j], nsl->ns[i]);
1297 }
1298 free(children, M_TEMP);
1299 }
1300
1301 /*
1302 * Pass the cpl data from the original async event completion, not the
1303 * log page fetch.
1304 */
1305 nvme_notify_async(aer->ctrlr, &aer->cpl, aer->log_page_id,
1306 aer->log_page_buffer, aer->log_page_size);
1307
1308 /*
1309 * Repost another asynchronous event request to replace the one
1310 * that just completed.
1311 */
1312 out:
1313 nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
1314 }
1315
1316 /*
1317 * Poll all the queues enabled on the device for completion.
1318 */
1319 void
nvme_ctrlr_poll(struct nvme_controller * ctrlr)1320 nvme_ctrlr_poll(struct nvme_controller *ctrlr)
1321 {
1322 int i;
1323
1324 nvme_qpair_process_completions(&ctrlr->adminq);
1325
1326 for (i = 0; i < ctrlr->num_io_queues; i++)
1327 if (ctrlr->ioq && ctrlr->ioq[i].cpl)
1328 nvme_qpair_process_completions(&ctrlr->ioq[i]);
1329 }
1330
1331 /*
1332 * Poll the single-vector interrupt case: num_io_queues will be 1 and
1333 * there's only a single vector. While we're polling, we mask further
1334 * interrupts in the controller.
1335 */
1336 void
nvme_ctrlr_shared_handler(void * arg)1337 nvme_ctrlr_shared_handler(void *arg)
1338 {
1339 struct nvme_controller *ctrlr = arg;
1340
1341 nvme_mmio_write_4(ctrlr, intms, 1);
1342 nvme_ctrlr_poll(ctrlr);
1343 nvme_mmio_write_4(ctrlr, intmc, 1);
1344 }
1345
1346 #define NVME_MAX_PAGES (int)(1024 / sizeof(vm_page_t))
1347
1348 static int
nvme_user_ioctl_req(vm_offset_t addr,size_t len,bool is_read,vm_page_t * upages,int max_pages,int * npagesp,struct nvme_request ** req,nvme_cb_fn_t cb_fn,void * cb_arg)1349 nvme_user_ioctl_req(vm_offset_t addr, size_t len, bool is_read,
1350 vm_page_t *upages, int max_pages, int *npagesp, struct nvme_request **req,
1351 nvme_cb_fn_t cb_fn, void *cb_arg)
1352 {
1353 vm_prot_t prot = VM_PROT_READ;
1354 int err;
1355
1356 if (is_read)
1357 prot |= VM_PROT_WRITE; /* Device will write to host memory */
1358 err = vm_fault_hold_pages(&curproc->p_vmspace->vm_map,
1359 addr, len, prot, upages, max_pages, npagesp);
1360 if (err != 0)
1361 return (err);
1362 *req = nvme_allocate_request_null(M_WAITOK, cb_fn, cb_arg);
1363 (*req)->payload = memdesc_vmpages(upages, len, addr & PAGE_MASK);
1364 (*req)->payload_valid = true;
1365 return (0);
1366 }
1367
1368 static void
nvme_user_ioctl_free(vm_page_t * pages,int npage)1369 nvme_user_ioctl_free(vm_page_t *pages, int npage)
1370 {
1371 vm_page_unhold_pages(pages, npage);
1372 }
1373
1374 static void
nvme_pt_done(void * arg,const struct nvme_completion * cpl)1375 nvme_pt_done(void *arg, const struct nvme_completion *cpl)
1376 {
1377 struct nvme_pt_command *pt = arg;
1378 struct mtx *mtx = pt->driver_lock;
1379 uint16_t status;
1380
1381 bzero(&pt->cpl, sizeof(pt->cpl));
1382 pt->cpl.cdw0 = cpl->cdw0;
1383
1384 status = cpl->status;
1385 status &= ~NVMEM(NVME_STATUS_P);
1386 pt->cpl.status = status;
1387
1388 mtx_lock(mtx);
1389 pt->driver_lock = NULL;
1390 wakeup(pt);
1391 mtx_unlock(mtx);
1392 }
1393
1394 int
nvme_ctrlr_passthrough_cmd(struct nvme_controller * ctrlr,struct nvme_pt_command * pt,uint32_t nsid,int is_user,int is_admin_cmd)1395 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
1396 struct nvme_pt_command *pt, uint32_t nsid, int is_user,
1397 int is_admin_cmd)
1398 {
1399 struct nvme_request *req;
1400 struct mtx *mtx;
1401 int ret = 0;
1402 int npages = 0;
1403 vm_page_t upages[NVME_MAX_PAGES];
1404
1405 if (pt->len > 0) {
1406 if (pt->len > ctrlr->max_xfer_size) {
1407 nvme_printf(ctrlr,
1408 "len (%d) exceeds max_xfer_size (%d)\n",
1409 pt->len, ctrlr->max_xfer_size);
1410 return (EIO);
1411 }
1412 if (is_user) {
1413 ret = nvme_user_ioctl_req((vm_offset_t)pt->buf, pt->len,
1414 pt->is_read, upages, nitems(upages), &npages, &req,
1415 nvme_pt_done, pt);
1416 if (ret != 0)
1417 return (ret);
1418 } else
1419 req = nvme_allocate_request_vaddr(pt->buf, pt->len,
1420 M_WAITOK, nvme_pt_done, pt);
1421 } else
1422 req = nvme_allocate_request_null(M_WAITOK, nvme_pt_done, pt);
1423
1424 /* Assume user space already converted to little-endian */
1425 req->cmd.opc = pt->cmd.opc;
1426 req->cmd.fuse = pt->cmd.fuse;
1427 req->cmd.rsvd2 = pt->cmd.rsvd2;
1428 req->cmd.rsvd3 = pt->cmd.rsvd3;
1429 req->cmd.cdw10 = pt->cmd.cdw10;
1430 req->cmd.cdw11 = pt->cmd.cdw11;
1431 req->cmd.cdw12 = pt->cmd.cdw12;
1432 req->cmd.cdw13 = pt->cmd.cdw13;
1433 req->cmd.cdw14 = pt->cmd.cdw14;
1434 req->cmd.cdw15 = pt->cmd.cdw15;
1435
1436 req->cmd.nsid = htole32(nsid);
1437
1438 mtx = mtx_pool_find(mtxpool_sleep, pt);
1439 pt->driver_lock = mtx;
1440
1441 if (is_admin_cmd)
1442 nvme_ctrlr_submit_admin_request(ctrlr, req);
1443 else
1444 nvme_ctrlr_submit_io_request(ctrlr, req);
1445
1446 mtx_lock(mtx);
1447 while (pt->driver_lock != NULL)
1448 mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
1449 mtx_unlock(mtx);
1450
1451 if (npages > 0)
1452 nvme_user_ioctl_free(upages, npages);
1453
1454 return (ret);
1455 }
1456
1457 static void
nvme_npc_done(void * arg,const struct nvme_completion * cpl)1458 nvme_npc_done(void *arg, const struct nvme_completion *cpl)
1459 {
1460 struct nvme_passthru_cmd *npc = arg;
1461 struct mtx *mtx = (void *)(uintptr_t)npc->metadata;
1462
1463 npc->result = cpl->cdw0; /* cpl in host order by now */
1464 mtx_lock(mtx);
1465 npc->metadata = 0;
1466 wakeup(npc);
1467 mtx_unlock(mtx);
1468 }
1469
1470 /* XXX refactor? */
1471
1472 int
nvme_ctrlr_linux_passthru_cmd(struct nvme_controller * ctrlr,struct nvme_passthru_cmd * npc,uint32_t nsid,bool is_user,bool is_admin)1473 nvme_ctrlr_linux_passthru_cmd(struct nvme_controller *ctrlr,
1474 struct nvme_passthru_cmd *npc, uint32_t nsid, bool is_user, bool is_admin)
1475 {
1476 struct nvme_request *req;
1477 struct mtx *mtx;
1478 int ret = 0;
1479 int npages = 0;
1480 vm_page_t upages[NVME_MAX_PAGES];
1481
1482 /*
1483 * We don't support metadata.
1484 */
1485 if (npc->metadata != 0 || npc->metadata_len != 0)
1486 return (EIO);
1487
1488 if (npc->data_len > 0 && npc->addr != 0) {
1489 if (npc->data_len > ctrlr->max_xfer_size) {
1490 nvme_printf(ctrlr,
1491 "data_len (%d) exceeds max_xfer_size (%d)\n",
1492 npc->data_len, ctrlr->max_xfer_size);
1493 return (EIO);
1494 }
1495 if (is_user) {
1496 ret = nvme_user_ioctl_req(npc->addr, npc->data_len,
1497 npc->opcode & 0x1, upages, nitems(upages), &npages,
1498 &req, nvme_npc_done, npc);
1499 if (ret != 0)
1500 return (ret);
1501 } else
1502 req = nvme_allocate_request_vaddr(
1503 (void *)(uintptr_t)npc->addr, npc->data_len,
1504 M_WAITOK, nvme_npc_done, npc);
1505 } else
1506 req = nvme_allocate_request_null(M_WAITOK, nvme_npc_done, npc);
1507
1508 req->cmd.opc = npc->opcode;
1509 req->cmd.fuse = npc->flags;
1510 req->cmd.rsvd2 = htole32(npc->cdw2);
1511 req->cmd.rsvd3 = htole32(npc->cdw3);
1512 req->cmd.cdw10 = htole32(npc->cdw10);
1513 req->cmd.cdw11 = htole32(npc->cdw11);
1514 req->cmd.cdw12 = htole32(npc->cdw12);
1515 req->cmd.cdw13 = htole32(npc->cdw13);
1516 req->cmd.cdw14 = htole32(npc->cdw14);
1517 req->cmd.cdw15 = htole32(npc->cdw15);
1518
1519 req->cmd.nsid = htole32(nsid);
1520
1521 mtx = mtx_pool_find(mtxpool_sleep, npc);
1522 npc->metadata = (uintptr_t) mtx;
1523
1524 /* XXX no timeout passed down */
1525 if (is_admin)
1526 nvme_ctrlr_submit_admin_request(ctrlr, req);
1527 else
1528 nvme_ctrlr_submit_io_request(ctrlr, req);
1529
1530 mtx_lock(mtx);
1531 while (npc->metadata != 0)
1532 mtx_sleep(npc, mtx, PRIBIO, "nvme_npc", 0);
1533 mtx_unlock(mtx);
1534
1535 if (npages > 0)
1536 nvme_user_ioctl_free(upages, npages);
1537
1538 return (ret);
1539 }
1540
1541 static int
nvme_ctrlr_ioctl(struct cdev * cdev,u_long cmd,caddr_t arg,int flag,struct thread * td)1542 nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
1543 struct thread *td)
1544 {
1545 struct nvme_controller *ctrlr;
1546 struct nvme_pt_command *pt;
1547
1548 ctrlr = cdev->si_drv1;
1549
1550 switch (cmd) {
1551 case NVME_IOCTL_RESET: /* Linux compat */
1552 case NVME_RESET_CONTROLLER:
1553 nvme_ctrlr_reset(ctrlr);
1554 break;
1555 case NVME_PASSTHROUGH_CMD:
1556 pt = (struct nvme_pt_command *)arg;
1557 return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),
1558 1 /* is_user_buffer */, 1 /* is_admin_cmd */));
1559 case NVME_GET_NSID:
1560 {
1561 struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg;
1562 strlcpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),
1563 sizeof(gnsid->cdev));
1564 gnsid->nsid = 0;
1565 break;
1566 }
1567 case NVME_GET_MAX_XFER_SIZE:
1568 *(uint64_t *)arg = ctrlr->max_xfer_size;
1569 break;
1570 case NVME_GET_CONTROLLER_DATA:
1571 memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata));
1572 break;
1573 case DIOCGIDENT: {
1574 uint8_t *sn = arg;
1575 nvme_cdata_get_disk_ident(&ctrlr->cdata, sn);
1576 break;
1577 }
1578 /* Linux Compatible (see nvme_linux.h) */
1579 case NVME_IOCTL_ID:
1580 td->td_retval[0] = 0xfffffffful;
1581 return (0);
1582
1583 case NVME_IOCTL_ADMIN_CMD:
1584 case NVME_IOCTL_IO_CMD: {
1585 struct nvme_passthru_cmd *npc = (struct nvme_passthru_cmd *)arg;
1586
1587 return (nvme_ctrlr_linux_passthru_cmd(ctrlr, npc, npc->nsid, true,
1588 cmd == NVME_IOCTL_ADMIN_CMD));
1589 }
1590
1591 default:
1592 return (ENOTTY);
1593 }
1594
1595 return (0);
1596 }
1597
1598 static struct cdevsw nvme_ctrlr_cdevsw = {
1599 .d_version = D_VERSION,
1600 .d_flags = 0,
1601 .d_ioctl = nvme_ctrlr_ioctl
1602 };
1603
1604 int
nvme_ctrlr_construct(struct nvme_controller * ctrlr,device_t dev)1605 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
1606 {
1607 struct make_dev_args md_args;
1608 uint32_t cap_lo;
1609 uint32_t cap_hi;
1610 uint32_t to, vs, pmrcap;
1611 int status, timeout_period;
1612
1613 ctrlr->dev = dev;
1614
1615 mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
1616 if (bus_get_domain(dev, &ctrlr->domain) != 0)
1617 ctrlr->domain = 0;
1618
1619 ctrlr->cap_lo = cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
1620 if (bootverbose) {
1621 device_printf(dev, "CapLo: 0x%08x: MQES %u%s%s%s%s, TO %u\n",
1622 cap_lo, NVME_CAP_LO_MQES(cap_lo),
1623 NVME_CAP_LO_CQR(cap_lo) ? ", CQR" : "",
1624 NVME_CAP_LO_AMS(cap_lo) ? ", AMS" : "",
1625 (NVME_CAP_LO_AMS(cap_lo) & 0x1) ? " WRRwUPC" : "",
1626 (NVME_CAP_LO_AMS(cap_lo) & 0x2) ? " VS" : "",
1627 NVME_CAP_LO_TO(cap_lo));
1628 }
1629 ctrlr->cap_hi = cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
1630 if (bootverbose) {
1631 device_printf(dev, "CapHi: 0x%08x: DSTRD %u%s, CSS %x%s, "
1632 "CPS %x, MPSMIN %u, MPSMAX %u%s%s%s%s%s\n", cap_hi,
1633 NVME_CAP_HI_DSTRD(cap_hi),
1634 NVME_CAP_HI_NSSRS(cap_hi) ? ", NSSRS" : "",
1635 NVME_CAP_HI_CSS(cap_hi),
1636 NVME_CAP_HI_BPS(cap_hi) ? ", BPS" : "",
1637 NVME_CAP_HI_CPS(cap_hi),
1638 NVME_CAP_HI_MPSMIN(cap_hi),
1639 NVME_CAP_HI_MPSMAX(cap_hi),
1640 NVME_CAP_HI_PMRS(cap_hi) ? ", PMRS" : "",
1641 NVME_CAP_HI_CMBS(cap_hi) ? ", CMBS" : "",
1642 NVME_CAP_HI_NSSS(cap_hi) ? ", NSSS" : "",
1643 NVME_CAP_HI_CRWMS(cap_hi) ? ", CRWMS" : "",
1644 NVME_CAP_HI_CRIMS(cap_hi) ? ", CRIMS" : "");
1645 }
1646 if (bootverbose) {
1647 vs = nvme_mmio_read_4(ctrlr, vs);
1648 device_printf(dev, "Version: 0x%08x: %d.%d\n", vs,
1649 NVME_MAJOR(vs), NVME_MINOR(vs));
1650 }
1651 if (bootverbose && NVME_CAP_HI_PMRS(cap_hi)) {
1652 pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);
1653 device_printf(dev, "PMRCap: 0x%08x: BIR %u%s%s, PMRTU %u, "
1654 "PMRWBM %x, PMRTO %u%s\n", pmrcap,
1655 NVME_PMRCAP_BIR(pmrcap),
1656 NVME_PMRCAP_RDS(pmrcap) ? ", RDS" : "",
1657 NVME_PMRCAP_WDS(pmrcap) ? ", WDS" : "",
1658 NVME_PMRCAP_PMRTU(pmrcap),
1659 NVME_PMRCAP_PMRWBM(pmrcap),
1660 NVME_PMRCAP_PMRTO(pmrcap),
1661 NVME_PMRCAP_CMSS(pmrcap) ? ", CMSS" : "");
1662 }
1663
1664 ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
1665
1666 ctrlr->mps = NVME_CAP_HI_MPSMIN(cap_hi);
1667 ctrlr->page_size = 1 << (NVME_MPS_SHIFT + ctrlr->mps);
1668
1669 /* Get ready timeout value from controller, in units of 500ms. */
1670 to = NVME_CAP_LO_TO(cap_lo) + 1;
1671 ctrlr->ready_timeout_in_ms = to * 500;
1672
1673 timeout_period = NVME_ADMIN_TIMEOUT_PERIOD;
1674 TUNABLE_INT_FETCH("hw.nvme.admin_timeout_period", &timeout_period);
1675 timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
1676 timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
1677 ctrlr->admin_timeout_period = timeout_period;
1678
1679 timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
1680 TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
1681 timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
1682 timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
1683 ctrlr->timeout_period = timeout_period;
1684
1685 nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
1686 TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
1687
1688 ctrlr->enable_aborts = 0;
1689 TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
1690
1691 ctrlr->alignment_splits = counter_u64_alloc(M_WAITOK);
1692
1693 /* Cap transfers by the maximum addressable by page-sized PRP (4KB pages -> 2MB). */
1694 ctrlr->max_xfer_size = MIN(maxphys, (ctrlr->page_size / 8 * ctrlr->page_size));
1695 if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)
1696 return (ENXIO);
1697
1698 /*
1699 * Create 2 threads for the taskqueue. The reset thread will block when
1700 * it detects that the controller has failed until all I/O has been
1701 * failed up the stack. The second thread is used for AER events, which
1702 * can block, but only briefly for memory and log page fetching.
1703 */
1704 ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1705 taskqueue_thread_enqueue, &ctrlr->taskqueue);
1706 taskqueue_start_threads(&ctrlr->taskqueue, 2, PI_DISK, "nvme taskq");
1707
1708 ctrlr->is_resetting = 0;
1709 ctrlr->is_initialized = false;
1710 TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1711 for (int i = 0; i < NVME_MAX_ASYNC_EVENTS; i++) {
1712 struct nvme_async_event_request *aer = &ctrlr->aer[i];
1713
1714 TASK_INIT(&aer->task, 0, nvme_ctrlr_aer_task, aer);
1715 mtx_init(&aer->mtx, "AER mutex", NULL, MTX_DEF);
1716 }
1717 ctrlr->is_failed = false;
1718
1719 make_dev_args_init(&md_args);
1720 md_args.mda_devsw = &nvme_ctrlr_cdevsw;
1721 md_args.mda_uid = UID_ROOT;
1722 md_args.mda_gid = GID_WHEEL;
1723 md_args.mda_mode = 0600;
1724 md_args.mda_unit = device_get_unit(dev);
1725 md_args.mda_si_drv1 = (void *)ctrlr;
1726 status = make_dev_s(&md_args, &ctrlr->cdev, "%s",
1727 device_get_nameunit(dev));
1728 if (status != 0)
1729 return (ENXIO);
1730
1731 return (0);
1732 }
1733
1734 /*
1735 * Called on detach, or on error on attach. The nvme_controller won't be used
1736 * again once we return, so we have to tear everything down (so nothing
1737 * references this, no callbacks, etc), but don't need to reset all the state
1738 * since nvme_controller will be freed soon.
1739 */
1740 void
nvme_ctrlr_destruct(struct nvme_controller * ctrlr,device_t dev)1741 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1742 {
1743 int i;
1744 bool gone;
1745
1746 ctrlr->is_dying = true;
1747
1748 if (ctrlr->resource == NULL)
1749 goto nores;
1750 if (!mtx_initialized(&ctrlr->adminq.lock))
1751 goto noadminq;
1752
1753 /*
1754 * Check whether it is a hot unplug or a clean driver detach. If device
1755 * is not there any more, skip any shutdown commands. Some hotplug
1756 * bridges will return zeros instead of ff's when the device is
1757 * departing, so ask the bridge if the device is gone. Some systems can
1758 * remove the drive w/o the bridge knowing its gone (they don't really
1759 * do hotplug), so failsafe with detecting all ff's (impossible with
1760 * this hardware) as the device being gone.
1761 */
1762 gone = bus_child_present(dev) == 0 ||
1763 (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);
1764 if (gone)
1765 nvme_ctrlr_fail(ctrlr, true);
1766 else
1767 nvme_notify_fail(ctrlr);
1768
1769 for (i = 0; i < NVME_MAX_NAMESPACES; i++)
1770 nvme_ns_destruct(&ctrlr->ns[i]);
1771
1772 if (ctrlr->cdev)
1773 destroy_dev(ctrlr->cdev);
1774
1775 if (ctrlr->is_initialized) {
1776 if (!gone) {
1777 if (ctrlr->hmb_nchunks > 0)
1778 nvme_ctrlr_hmb_enable(ctrlr, false, false);
1779 nvme_ctrlr_delete_qpairs(ctrlr);
1780 }
1781 nvme_ctrlr_hmb_free(ctrlr);
1782 }
1783 if (ctrlr->ioq != NULL) {
1784 for (i = 0; i < ctrlr->num_io_queues; i++)
1785 nvme_io_qpair_destroy(&ctrlr->ioq[i]);
1786 free(ctrlr->ioq, M_NVME);
1787 }
1788 nvme_admin_qpair_destroy(&ctrlr->adminq);
1789
1790 /*
1791 * Notify the controller of a shutdown, even though this is due to a
1792 * driver unload, not a system shutdown (this path is not invoked uring
1793 * shutdown). This ensures the controller receives a shutdown
1794 * notification in case the system is shutdown before reloading the
1795 * driver. Some NVMe drives need this to flush their cache to stable
1796 * media and consider it a safe shutdown in SMART stats.
1797 */
1798 if (!gone) {
1799 nvme_ctrlr_shutdown(ctrlr);
1800 nvme_ctrlr_disable(ctrlr);
1801 }
1802
1803 noadminq:
1804 if (ctrlr->taskqueue) {
1805 taskqueue_free(ctrlr->taskqueue);
1806 for (int i = 0; i < NVME_MAX_ASYNC_EVENTS; i++) {
1807 struct nvme_async_event_request *aer = &ctrlr->aer[i];
1808
1809 mtx_destroy(&aer->mtx);
1810 }
1811 }
1812
1813 if (ctrlr->tag)
1814 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
1815
1816 if (ctrlr->res)
1817 bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
1818 rman_get_rid(ctrlr->res), ctrlr->res);
1819
1820 if (ctrlr->msix_table_resource != NULL) {
1821 bus_release_resource(dev, SYS_RES_MEMORY,
1822 ctrlr->msix_table_resource_id, ctrlr->msix_table_resource);
1823 }
1824
1825 if (ctrlr->msix_pba_resource != NULL) {
1826 bus_release_resource(dev, SYS_RES_MEMORY,
1827 ctrlr->msix_pba_resource_id, ctrlr->msix_pba_resource);
1828 }
1829
1830 bus_release_resource(dev, SYS_RES_MEMORY,
1831 ctrlr->resource_id, ctrlr->resource);
1832
1833 nores:
1834 if (ctrlr->alignment_splits)
1835 counter_u64_free(ctrlr->alignment_splits);
1836
1837 mtx_destroy(&ctrlr->lock);
1838 }
1839
1840 void
nvme_ctrlr_shutdown(struct nvme_controller * ctrlr)1841 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
1842 {
1843 uint32_t cc;
1844 uint32_t csts;
1845 int timeout;
1846
1847 cc = nvme_mmio_read_4(ctrlr, cc);
1848 cc &= ~NVMEM(NVME_CC_REG_SHN);
1849 cc |= NVMEF(NVME_CC_REG_SHN, NVME_SHN_NORMAL);
1850 nvme_mmio_write_4(ctrlr, cc, cc);
1851
1852 timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :
1853 ((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);
1854 while (1) {
1855 csts = nvme_mmio_read_4(ctrlr, csts);
1856 if (csts == NVME_GONE) /* Hot unplug. */
1857 break;
1858 if (NVME_CSTS_GET_SHST(csts) == NVME_SHST_COMPLETE)
1859 break;
1860 if (timeout - ticks < 0) {
1861 nvme_printf(ctrlr, "shutdown timeout\n");
1862 break;
1863 }
1864 pause("nvmeshut", 1);
1865 }
1866 }
1867
1868 void
nvme_ctrlr_submit_admin_request(struct nvme_controller * ctrlr,struct nvme_request * req)1869 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
1870 struct nvme_request *req)
1871 {
1872 nvme_qpair_submit_request(&ctrlr->adminq, req);
1873 }
1874
1875 void
nvme_ctrlr_submit_io_request(struct nvme_controller * ctrlr,struct nvme_request * req)1876 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
1877 struct nvme_request *req)
1878 {
1879 struct nvme_qpair *qpair;
1880 int32_t ioq;
1881
1882 ioq = req->ioq == NVME_IOQ_DEFAULT ? QP(ctrlr, curcpu) : req->ioq;
1883 qpair = &ctrlr->ioq[ioq];
1884 nvme_qpair_submit_request(qpair, req);
1885 }
1886
1887 device_t
nvme_ctrlr_get_device(struct nvme_controller * ctrlr)1888 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
1889 {
1890 return (ctrlr->dev);
1891 }
1892
1893 const struct nvme_controller_data *
nvme_ctrlr_get_data(struct nvme_controller * ctrlr)1894 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
1895 {
1896 return (&ctrlr->cdata);
1897 }
1898
1899 int
nvme_ctrlr_suspend(struct nvme_controller * ctrlr)1900 nvme_ctrlr_suspend(struct nvme_controller *ctrlr)
1901 {
1902 int to = hz;
1903
1904 /*
1905 * Can't touch failed controllers, so it's already suspended. User will
1906 * need to do an explicit reset to bring it back, if that's even
1907 * possible.
1908 */
1909 if (ctrlr->is_failed)
1910 return (0);
1911
1912 /*
1913 * We don't want the reset taskqueue running, since it does similar
1914 * things, so prevent it from running after we start. Wait for any reset
1915 * that may have been started to complete. The reset process we follow
1916 * will ensure that any new I/O will queue and be given to the hardware
1917 * after we resume (though there should be none).
1918 */
1919 while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)
1920 pause("nvmesusp", 1);
1921 if (to <= 0) {
1922 nvme_printf(ctrlr,
1923 "Competing reset task didn't finish. Try again later.\n");
1924 return (EWOULDBLOCK);
1925 }
1926
1927 if (ctrlr->hmb_nchunks > 0)
1928 nvme_ctrlr_hmb_enable(ctrlr, false, false);
1929
1930 /*
1931 * Per Section 7.6.2 of NVMe spec 1.4, to properly suspend, we need to
1932 * delete the hardware I/O queues, and then shutdown. This properly
1933 * flushes any metadata the drive may have stored so it can survive
1934 * having its power removed and prevents the unsafe shutdown count from
1935 * incriminating. Once we delete the qpairs, we have to disable them
1936 * before shutting down.
1937 */
1938 nvme_ctrlr_delete_qpairs(ctrlr);
1939 nvme_ctrlr_disable_qpairs(ctrlr);
1940 nvme_ctrlr_shutdown(ctrlr);
1941
1942 return (0);
1943 }
1944
1945 int
nvme_ctrlr_resume(struct nvme_controller * ctrlr)1946 nvme_ctrlr_resume(struct nvme_controller *ctrlr)
1947 {
1948 /*
1949 * Can't touch failed controllers, so nothing to do to resume.
1950 */
1951 if (ctrlr->is_failed)
1952 return (0);
1953
1954 if (nvme_ctrlr_hw_reset(ctrlr) != 0)
1955 goto fail;
1956
1957 /*
1958 * Now that we've reset the hardware, we can restart the controller. Any
1959 * I/O that was pending is requeued. Any admin commands are aborted with
1960 * an error. Once we've restarted, stop flagging the controller as being
1961 * in the reset phase.
1962 */
1963 nvme_ctrlr_start(ctrlr, true);
1964 (void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1965
1966 return (0);
1967 fail:
1968 /*
1969 * Since we can't bring the controller out of reset, announce and fail
1970 * the controller. However, we have to return success for the resume
1971 * itself, due to questionable APIs.
1972 */
1973 nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");
1974 nvme_ctrlr_fail(ctrlr, true);
1975 (void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1976 return (0);
1977 }
1978