xref: /freebsd/sys/dev/nvme/nvme.c (revision f182f928dba2089ceb897191e546f32362035ddc)
1bb0ec6b3SJim Harris /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
4496a2752SJim Harris  * Copyright (C) 2012-2014 Intel Corporation
5bb0ec6b3SJim Harris  * All rights reserved.
6bb0ec6b3SJim Harris  *
7bb0ec6b3SJim Harris  * Redistribution and use in source and binary forms, with or without
8bb0ec6b3SJim Harris  * modification, are permitted provided that the following conditions
9bb0ec6b3SJim Harris  * are met:
10bb0ec6b3SJim Harris  * 1. Redistributions of source code must retain the above copyright
11bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer.
12bb0ec6b3SJim Harris  * 2. Redistributions in binary form must reproduce the above copyright
13bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer in the
14bb0ec6b3SJim Harris  *    documentation and/or other materials provided with the distribution.
15bb0ec6b3SJim Harris  *
16bb0ec6b3SJim Harris  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17bb0ec6b3SJim Harris  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18bb0ec6b3SJim Harris  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19bb0ec6b3SJim Harris  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20bb0ec6b3SJim Harris  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21bb0ec6b3SJim Harris  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22bb0ec6b3SJim Harris  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23bb0ec6b3SJim Harris  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24bb0ec6b3SJim Harris  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25bb0ec6b3SJim Harris  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26bb0ec6b3SJim Harris  * SUCH DAMAGE.
27bb0ec6b3SJim Harris  */
28bb0ec6b3SJim Harris 
29bb0ec6b3SJim Harris #include <sys/cdefs.h>
30bb0ec6b3SJim Harris __FBSDID("$FreeBSD$");
31bb0ec6b3SJim Harris 
32bb0ec6b3SJim Harris #include <sys/param.h>
33bb0ec6b3SJim Harris #include <sys/bus.h>
34bb0ec6b3SJim Harris #include <sys/conf.h>
35bb0ec6b3SJim Harris #include <sys/module.h>
36bb0ec6b3SJim Harris 
37ad697276SJim Harris #include <vm/uma.h>
38ad697276SJim Harris 
39bb0ec6b3SJim Harris #include "nvme_private.h"
40bb0ec6b3SJim Harris 
41bb0ec6b3SJim Harris struct nvme_consumer {
42038a5ee4SJim Harris 	uint32_t		id;
43038a5ee4SJim Harris 	nvme_cons_ns_fn_t	ns_fn;
44038a5ee4SJim Harris 	nvme_cons_ctrlr_fn_t	ctrlr_fn;
45038a5ee4SJim Harris 	nvme_cons_async_fn_t	async_fn;
46232e2edbSJim Harris 	nvme_cons_fail_fn_t	fail_fn;
47bb0ec6b3SJim Harris };
48bb0ec6b3SJim Harris 
49bb0ec6b3SJim Harris struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
50038a5ee4SJim Harris #define	INVALID_CONSUMER_ID	0xFFFF
51bb0ec6b3SJim Harris 
52ad697276SJim Harris uma_zone_t	nvme_request_zone;
53cb5b7c13SJim Harris int32_t		nvme_retry_count;
54c75bdc04SWarner Losh 
55ad697276SJim Harris 
56bb0ec6b3SJim Harris MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
57bb0ec6b3SJim Harris 
58*f182f928SWarner Losh devclass_t nvme_devclass;
59bb0ec6b3SJim Harris 
60bb0ec6b3SJim Harris static void
61ad697276SJim Harris nvme_init(void)
62ad697276SJim Harris {
63038a5ee4SJim Harris 	uint32_t	i;
64038a5ee4SJim Harris 
65ad697276SJim Harris 	nvme_request_zone = uma_zcreate("nvme_request",
66ad697276SJim Harris 	    sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0);
67038a5ee4SJim Harris 
68038a5ee4SJim Harris 	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
69038a5ee4SJim Harris 		nvme_consumer[i].id = INVALID_CONSUMER_ID;
70ad697276SJim Harris }
71ad697276SJim Harris 
72ad697276SJim Harris SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL);
73ad697276SJim Harris 
74ad697276SJim Harris static void
75ad697276SJim Harris nvme_uninit(void)
76ad697276SJim Harris {
77ad697276SJim Harris 	uma_zdestroy(nvme_request_zone);
78ad697276SJim Harris }
79ad697276SJim Harris 
80ad697276SJim Harris SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL);
81ad697276SJim Harris 
82*f182f928SWarner Losh int
83c670f31fSNathan Whitehorn nvme_shutdown(device_t dev)
84bb0ec6b3SJim Harris {
85bb0ec6b3SJim Harris 	struct nvme_controller	*ctrlr;
86bb0ec6b3SJim Harris 
87c670f31fSNathan Whitehorn 	ctrlr = DEVICE2SOFTC(dev);
8856183abcSJim Harris 	nvme_ctrlr_shutdown(ctrlr);
89bb0ec6b3SJim Harris 
90c670f31fSNathan Whitehorn 	return (0);
91bb0ec6b3SJim Harris }
92bb0ec6b3SJim Harris 
93bb0ec6b3SJim Harris void
94bb0ec6b3SJim Harris nvme_dump_command(struct nvme_command *cmd)
95bb0ec6b3SJim Harris {
960d787e9bSWojciech Macek 
974b52061eSDavid E. O'Brien 	printf(
980d787e9bSWojciech Macek "opc:%x f:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
999544e6dcSChuck Tuffli 	    cmd->opc, cmd->fuse, cmd->cid, le32toh(cmd->nsid),
100bb0ec6b3SJim Harris 	    cmd->rsvd2, cmd->rsvd3,
1010d787e9bSWojciech Macek 	    (uintmax_t)le64toh(cmd->mptr), (uintmax_t)le64toh(cmd->prp1), (uintmax_t)le64toh(cmd->prp2),
1020d787e9bSWojciech Macek 	    le32toh(cmd->cdw10), le32toh(cmd->cdw11), le32toh(cmd->cdw12),
1030d787e9bSWojciech Macek 	    le32toh(cmd->cdw13), le32toh(cmd->cdw14), le32toh(cmd->cdw15));
104bb0ec6b3SJim Harris }
105bb0ec6b3SJim Harris 
106bb0ec6b3SJim Harris void
107bb0ec6b3SJim Harris nvme_dump_completion(struct nvme_completion *cpl)
108bb0ec6b3SJim Harris {
1090d787e9bSWojciech Macek 	uint8_t p, sc, sct, m, dnr;
1100d787e9bSWojciech Macek 	uint16_t status;
1110d787e9bSWojciech Macek 
1120d787e9bSWojciech Macek 	status = le16toh(cpl->status);
1130d787e9bSWojciech Macek 
1140d787e9bSWojciech Macek 	p = NVME_STATUS_GET_P(status);
1150d787e9bSWojciech Macek 	sc = NVME_STATUS_GET_SC(status);
1160d787e9bSWojciech Macek 	sct = NVME_STATUS_GET_SCT(status);
1170d787e9bSWojciech Macek 	m = NVME_STATUS_GET_M(status);
1180d787e9bSWojciech Macek 	dnr = NVME_STATUS_GET_DNR(status);
1190d787e9bSWojciech Macek 
120bb0ec6b3SJim Harris 	printf("cdw0:%08x sqhd:%04x sqid:%04x "
121bb0ec6b3SJim Harris 	    "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
1220d787e9bSWojciech Macek 	    le32toh(cpl->cdw0), le16toh(cpl->sqhd), le16toh(cpl->sqid),
1230d787e9bSWojciech Macek 	    cpl->cid, p, sc, sct, m, dnr);
124bb0ec6b3SJim Harris }
125bb0ec6b3SJim Harris 
126*f182f928SWarner Losh int
127bb0ec6b3SJim Harris nvme_attach(device_t dev)
128bb0ec6b3SJim Harris {
129bb0ec6b3SJim Harris 	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
130bb0ec6b3SJim Harris 	int			status;
131bb0ec6b3SJim Harris 
132bb0ec6b3SJim Harris 	status = nvme_ctrlr_construct(ctrlr, dev);
133bb0ec6b3SJim Harris 
1347aa27dbaSJim Harris 	if (status != 0) {
1357aa27dbaSJim Harris 		nvme_ctrlr_destruct(ctrlr, dev);
136bb0ec6b3SJim Harris 		return (status);
1377aa27dbaSJim Harris 	}
138bb0ec6b3SJim Harris 
139bb0ec6b3SJim Harris 	/*
140*f182f928SWarner Losh 	 * Reset controller twice to ensure we do a transition from cc.en==1
141bb0ec6b3SJim Harris 	 * Reset controller twice to ensure we do a transition from cc.en==1
142bb0ec6b3SJim Harris 	 *  to cc.en==0.  This is because we don't really know what status
143bb0ec6b3SJim Harris 	 *  the controller was left in when boot handed off to OS.
144bb0ec6b3SJim Harris 	 */
145b846efd7SJim Harris 	status = nvme_ctrlr_hw_reset(ctrlr);
1467aa27dbaSJim Harris 	if (status != 0) {
1477aa27dbaSJim Harris 		nvme_ctrlr_destruct(ctrlr, dev);
148bb0ec6b3SJim Harris 		return (status);
1497aa27dbaSJim Harris 	}
150bb0ec6b3SJim Harris 
151b846efd7SJim Harris 	status = nvme_ctrlr_hw_reset(ctrlr);
1527aa27dbaSJim Harris 	if (status != 0) {
1537aa27dbaSJim Harris 		nvme_ctrlr_destruct(ctrlr, dev);
154bb0ec6b3SJim Harris 		return (status);
1557aa27dbaSJim Harris 	}
156bb0ec6b3SJim Harris 
157be34f216SJim Harris 	ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook;
158bb0ec6b3SJim Harris 	ctrlr->config_hook.ich_arg = ctrlr;
159bb0ec6b3SJim Harris 
160bb0ec6b3SJim Harris 	config_intrhook_establish(&ctrlr->config_hook);
161bb0ec6b3SJim Harris 
162bb0ec6b3SJim Harris 	return (0);
163bb0ec6b3SJim Harris }
164bb0ec6b3SJim Harris 
165*f182f928SWarner Losh int
166bb0ec6b3SJim Harris nvme_detach (device_t dev)
167bb0ec6b3SJim Harris {
168bb0ec6b3SJim Harris 	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
169bb0ec6b3SJim Harris 
170990e741cSJim Harris 	nvme_ctrlr_destruct(ctrlr, dev);
171bb0ec6b3SJim Harris 	return (0);
172bb0ec6b3SJim Harris }
173bb0ec6b3SJim Harris 
174bb0ec6b3SJim Harris static void
175496a2752SJim Harris nvme_notify(struct nvme_consumer *cons,
176496a2752SJim Harris 	    struct nvme_controller *ctrlr)
177bb0ec6b3SJim Harris {
178038a5ee4SJim Harris 	struct nvme_namespace	*ns;
179038a5ee4SJim Harris 	void			*ctrlr_cookie;
180496a2752SJim Harris 	int			cmpset, ns_idx;
181bb0ec6b3SJim Harris 
182496a2752SJim Harris 	/*
183496a2752SJim Harris 	 * The consumer may register itself after the nvme devices
184496a2752SJim Harris 	 *  have registered with the kernel, but before the
185496a2752SJim Harris 	 *  driver has completed initialization.  In that case,
186496a2752SJim Harris 	 *  return here, and when initialization completes, the
187496a2752SJim Harris 	 *  controller will make sure the consumer gets notified.
188496a2752SJim Harris 	 */
189496a2752SJim Harris 	if (!ctrlr->is_initialized)
190bb0ec6b3SJim Harris 		return;
191bb0ec6b3SJim Harris 
192496a2752SJim Harris 	cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1);
193496a2752SJim Harris 	if (cmpset == 0)
194496a2752SJim Harris 		return;
195496a2752SJim Harris 
196038a5ee4SJim Harris 	if (cons->ctrlr_fn != NULL)
197038a5ee4SJim Harris 		ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr);
198038a5ee4SJim Harris 	else
19951b92c1aSAlexander Motin 		ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de;
200038a5ee4SJim Harris 	ctrlr->cons_cookie[cons->id] = ctrlr_cookie;
20151b92c1aSAlexander Motin 
20251b92c1aSAlexander Motin 	/* ctrlr_fn has failed.  Nothing to notify here any more. */
20351b92c1aSAlexander Motin 	if (ctrlr_cookie == NULL)
20451b92c1aSAlexander Motin 		return;
20551b92c1aSAlexander Motin 
206086d23cfSJim Harris 	if (ctrlr->is_failed) {
20751b92c1aSAlexander Motin 		ctrlr->cons_cookie[cons->id] = NULL;
208086d23cfSJim Harris 		if (cons->fail_fn != NULL)
209086d23cfSJim Harris 			(*cons->fail_fn)(ctrlr_cookie);
210086d23cfSJim Harris 		/*
211086d23cfSJim Harris 		 * Do not notify consumers about the namespaces of a
212086d23cfSJim Harris 		 *  failed controller.
213086d23cfSJim Harris 		 */
214496a2752SJim Harris 		return;
215086d23cfSJim Harris 	}
216a8a18dd5SWarner Losh 	for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) {
217038a5ee4SJim Harris 		ns = &ctrlr->ns[ns_idx];
218a8a18dd5SWarner Losh 		if (ns->data.nsze == 0)
219a8a18dd5SWarner Losh 			continue;
220038a5ee4SJim Harris 		if (cons->ns_fn != NULL)
221038a5ee4SJim Harris 			ns->cons_cookie[cons->id] =
222038a5ee4SJim Harris 			    (*cons->ns_fn)(ns, ctrlr_cookie);
223038a5ee4SJim Harris 	}
224bb0ec6b3SJim Harris }
225bb0ec6b3SJim Harris 
226496a2752SJim Harris void
227496a2752SJim Harris nvme_notify_new_controller(struct nvme_controller *ctrlr)
228496a2752SJim Harris {
229496a2752SJim Harris 	int i;
230496a2752SJim Harris 
231496a2752SJim Harris 	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
232496a2752SJim Harris 		if (nvme_consumer[i].id != INVALID_CONSUMER_ID) {
233496a2752SJim Harris 			nvme_notify(&nvme_consumer[i], ctrlr);
234496a2752SJim Harris 		}
235496a2752SJim Harris 	}
236496a2752SJim Harris }
237496a2752SJim Harris 
238496a2752SJim Harris static void
239496a2752SJim Harris nvme_notify_new_consumer(struct nvme_consumer *cons)
240496a2752SJim Harris {
241496a2752SJim Harris 	device_t		*devlist;
242496a2752SJim Harris 	struct nvme_controller	*ctrlr;
243496a2752SJim Harris 	int			dev_idx, devcount;
244496a2752SJim Harris 
245496a2752SJim Harris 	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
246496a2752SJim Harris 		return;
247496a2752SJim Harris 
248496a2752SJim Harris 	for (dev_idx = 0; dev_idx < devcount; dev_idx++) {
249496a2752SJim Harris 		ctrlr = DEVICE2SOFTC(devlist[dev_idx]);
250496a2752SJim Harris 		nvme_notify(cons, ctrlr);
251496a2752SJim Harris 	}
252496a2752SJim Harris 
253bb0ec6b3SJim Harris 	free(devlist, M_TEMP);
254bb0ec6b3SJim Harris }
255bb0ec6b3SJim Harris 
256038a5ee4SJim Harris void
257038a5ee4SJim Harris nvme_notify_async_consumers(struct nvme_controller *ctrlr,
2580d7e13ecSJim Harris 			    const struct nvme_completion *async_cpl,
2590d7e13ecSJim Harris 			    uint32_t log_page_id, void *log_page_buffer,
2600d7e13ecSJim Harris 			    uint32_t log_page_size)
261038a5ee4SJim Harris {
262038a5ee4SJim Harris 	struct nvme_consumer	*cons;
26351b92c1aSAlexander Motin 	void			*ctrlr_cookie;
264038a5ee4SJim Harris 	uint32_t		i;
265038a5ee4SJim Harris 
266038a5ee4SJim Harris 	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
267038a5ee4SJim Harris 		cons = &nvme_consumer[i];
26851b92c1aSAlexander Motin 		if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL &&
26951b92c1aSAlexander Motin 		    (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
27051b92c1aSAlexander Motin 			(*cons->async_fn)(ctrlr_cookie, async_cpl,
2710d7e13ecSJim Harris 			    log_page_id, log_page_buffer, log_page_size);
272038a5ee4SJim Harris 		}
273038a5ee4SJim Harris 	}
27451b92c1aSAlexander Motin }
275038a5ee4SJim Harris 
276232e2edbSJim Harris void
277232e2edbSJim Harris nvme_notify_fail_consumers(struct nvme_controller *ctrlr)
278232e2edbSJim Harris {
279232e2edbSJim Harris 	struct nvme_consumer	*cons;
28051b92c1aSAlexander Motin 	void			*ctrlr_cookie;
281232e2edbSJim Harris 	uint32_t		i;
282232e2edbSJim Harris 
2830e1fd2ddSJim Harris 	/*
2840e1fd2ddSJim Harris 	 * This controller failed during initialization (i.e. IDENTIFY
2850e1fd2ddSJim Harris 	 *  command failed or timed out).  Do not notify any nvme
2860e1fd2ddSJim Harris 	 *  consumers of the failure here, since the consumer does not
2870e1fd2ddSJim Harris 	 *  even know about the controller yet.
2880e1fd2ddSJim Harris 	 */
2890e1fd2ddSJim Harris 	if (!ctrlr->is_initialized)
2900e1fd2ddSJim Harris 		return;
2910e1fd2ddSJim Harris 
292232e2edbSJim Harris 	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
293232e2edbSJim Harris 		cons = &nvme_consumer[i];
29451b92c1aSAlexander Motin 		if (cons->id != INVALID_CONSUMER_ID &&
29551b92c1aSAlexander Motin 		    (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
29651b92c1aSAlexander Motin 			ctrlr->cons_cookie[i] = NULL;
29751b92c1aSAlexander Motin 			if (cons->fail_fn != NULL)
29851b92c1aSAlexander Motin 				cons->fail_fn(ctrlr_cookie);
29951b92c1aSAlexander Motin 		}
300232e2edbSJim Harris 	}
301232e2edbSJim Harris }
302232e2edbSJim Harris 
303f439e3a4SAlexander Motin void
304f439e3a4SAlexander Motin nvme_notify_ns(struct nvme_controller *ctrlr, int nsid)
305f439e3a4SAlexander Motin {
306f439e3a4SAlexander Motin 	struct nvme_consumer	*cons;
307f439e3a4SAlexander Motin 	struct nvme_namespace	*ns = &ctrlr->ns[nsid - 1];
30851b92c1aSAlexander Motin 	void			*ctrlr_cookie;
309f439e3a4SAlexander Motin 	uint32_t		i;
310f439e3a4SAlexander Motin 
311f439e3a4SAlexander Motin 	if (!ctrlr->is_initialized)
312f439e3a4SAlexander Motin 		return;
313f439e3a4SAlexander Motin 
314f439e3a4SAlexander Motin 	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
315f439e3a4SAlexander Motin 		cons = &nvme_consumer[i];
31651b92c1aSAlexander Motin 		if (cons->id != INVALID_CONSUMER_ID && cons->ns_fn != NULL &&
31751b92c1aSAlexander Motin 		    (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL)
31851b92c1aSAlexander Motin 			ns->cons_cookie[i] = (*cons->ns_fn)(ns, ctrlr_cookie);
319f439e3a4SAlexander Motin 	}
320f439e3a4SAlexander Motin }
321f439e3a4SAlexander Motin 
322bb0ec6b3SJim Harris struct nvme_consumer *
323038a5ee4SJim Harris nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn,
324232e2edbSJim Harris 		       nvme_cons_async_fn_t async_fn,
325232e2edbSJim Harris 		       nvme_cons_fail_fn_t fail_fn)
326bb0ec6b3SJim Harris {
327bb0ec6b3SJim Harris 	int i;
328bb0ec6b3SJim Harris 
329bb0ec6b3SJim Harris 	/*
330204498d7SWarner Losh 	 * TODO: add locking around consumer registration.
331bb0ec6b3SJim Harris 	 */
332bb0ec6b3SJim Harris 	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
333038a5ee4SJim Harris 		if (nvme_consumer[i].id == INVALID_CONSUMER_ID) {
334038a5ee4SJim Harris 			nvme_consumer[i].id = i;
335038a5ee4SJim Harris 			nvme_consumer[i].ns_fn = ns_fn;
336038a5ee4SJim Harris 			nvme_consumer[i].ctrlr_fn = ctrlr_fn;
337038a5ee4SJim Harris 			nvme_consumer[i].async_fn = async_fn;
338232e2edbSJim Harris 			nvme_consumer[i].fail_fn = fail_fn;
339bb0ec6b3SJim Harris 
340496a2752SJim Harris 			nvme_notify_new_consumer(&nvme_consumer[i]);
341bb0ec6b3SJim Harris 			return (&nvme_consumer[i]);
342bb0ec6b3SJim Harris 		}
343bb0ec6b3SJim Harris 
344bb0ec6b3SJim Harris 	printf("nvme(4): consumer not registered - no slots available\n");
345bb0ec6b3SJim Harris 	return (NULL);
346bb0ec6b3SJim Harris }
347bb0ec6b3SJim Harris 
348bb0ec6b3SJim Harris void
349bb0ec6b3SJim Harris nvme_unregister_consumer(struct nvme_consumer *consumer)
350bb0ec6b3SJim Harris {
351bb0ec6b3SJim Harris 
352038a5ee4SJim Harris 	consumer->id = INVALID_CONSUMER_ID;
353bb0ec6b3SJim Harris }
354bb0ec6b3SJim Harris 
355955910a9SJim Harris void
356955910a9SJim Harris nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
357955910a9SJim Harris {
358955910a9SJim Harris 	struct nvme_completion_poll_status	*status = arg;
359955910a9SJim Harris 
360955910a9SJim Harris 	/*
361955910a9SJim Harris 	 * Copy status into the argument passed by the caller, so that
362955910a9SJim Harris 	 *  the caller can check the status to determine if the
363955910a9SJim Harris 	 *  the request passed or failed.
364955910a9SJim Harris 	 */
365955910a9SJim Harris 	memcpy(&status->cpl, cpl, sizeof(*cpl));
36629077eb4SWarner Losh 	atomic_store_rel_int(&status->done, 1);
367955910a9SJim Harris }
368