xref: /freebsd/sys/dev/nvme/nvme_private.h (revision 1323ec571215a77ddd21294f0871979d5ad6b992)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2012-2014 Intel Corporation
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #ifndef __NVME_PRIVATE_H__
32 #define __NVME_PRIVATE_H__
33 
34 #include <sys/param.h>
35 #include <sys/bio.h>
36 #include <sys/bus.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/rman.h>
43 #include <sys/systm.h>
44 #include <sys/taskqueue.h>
45 
46 #include <vm/uma.h>
47 
48 #include <machine/bus.h>
49 
50 #include "nvme.h"
51 
52 #define DEVICE2SOFTC(dev) ((struct nvme_controller *) device_get_softc(dev))
53 
54 MALLOC_DECLARE(M_NVME);
55 
56 #define IDT32_PCI_ID		0x80d0111d /* 32 channel board */
57 #define IDT8_PCI_ID		0x80d2111d /* 8 channel board */
58 
59 #define NVME_ADMIN_TRACKERS	(16)
60 #define NVME_ADMIN_ENTRIES	(128)
61 /* min and max are defined in admin queue attributes section of spec */
62 #define NVME_MIN_ADMIN_ENTRIES	(2)
63 #define NVME_MAX_ADMIN_ENTRIES	(4096)
64 
65 /*
66  * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
67  *  queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
68  *  will allow outstanding on an I/O qpair at any time.  The only advantage in
69  *  having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
70  *  the contents of the submission and completion queues, it will show a longer
71  *  history of data.
72  */
73 #define NVME_IO_ENTRIES		(256)
74 #define NVME_IO_TRACKERS	(128)
75 #define NVME_MIN_IO_TRACKERS	(4)
76 #define NVME_MAX_IO_TRACKERS	(1024)
77 
78 /*
79  * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
80  *  for each controller.
81  */
82 
83 #define NVME_INT_COAL_TIME	(0)	/* disabled */
84 #define NVME_INT_COAL_THRESHOLD (0)	/* 0-based */
85 
86 #define NVME_MAX_NAMESPACES	(16)
87 #define NVME_MAX_CONSUMERS	(2)
88 #define NVME_MAX_ASYNC_EVENTS	(8)
89 
90 #define NVME_DEFAULT_TIMEOUT_PERIOD	(30)    /* in seconds */
91 #define NVME_MIN_TIMEOUT_PERIOD		(5)
92 #define NVME_MAX_TIMEOUT_PERIOD		(120)
93 
94 #define NVME_DEFAULT_RETRY_COUNT	(4)
95 
96 /* Maximum log page size to fetch for AERs. */
97 #define NVME_MAX_AER_LOG_SIZE		(4096)
98 
99 /*
100  * Page size parameters
101  */
102 #define NVME_BASE_SHIFT		12	/* Several parameters (MSP) are 2^(12+x) */
103 
104 /*
105  * Define CACHE_LINE_SIZE here for older FreeBSD versions that do not define
106  *  it.
107  */
108 #ifndef CACHE_LINE_SIZE
109 #define CACHE_LINE_SIZE		(64)
110 #endif
111 
112 #define NVME_GONE		0xfffffffful
113 
114 extern int32_t		nvme_retry_count;
115 extern bool		nvme_verbose_cmd_dump;
116 
117 struct nvme_completion_poll_status {
118 	struct nvme_completion	cpl;
119 	int			done;
120 };
121 
122 extern devclass_t nvme_devclass;
123 
124 #define NVME_REQUEST_VADDR	1
125 #define NVME_REQUEST_NULL	2 /* For requests with no payload. */
126 #define NVME_REQUEST_UIO	3
127 #define NVME_REQUEST_BIO	4
128 #define NVME_REQUEST_CCB        5
129 
130 struct nvme_request {
131 	struct nvme_command		cmd;
132 	struct nvme_qpair		*qpair;
133 	union {
134 		void			*payload;
135 		struct bio		*bio;
136 	} u;
137 	uint32_t			type;
138 	uint32_t			payload_size;
139 	bool				timeout;
140 	nvme_cb_fn_t			cb_fn;
141 	void				*cb_arg;
142 	int32_t				retries;
143 	STAILQ_ENTRY(nvme_request)	stailq;
144 };
145 
146 struct nvme_async_event_request {
147 	struct nvme_controller		*ctrlr;
148 	struct nvme_request		*req;
149 	struct nvme_completion		cpl;
150 	uint32_t			log_page_id;
151 	uint32_t			log_page_size;
152 	uint8_t				log_page_buffer[NVME_MAX_AER_LOG_SIZE];
153 };
154 
155 struct nvme_tracker {
156 	TAILQ_ENTRY(nvme_tracker)	tailq;
157 	struct nvme_request		*req;
158 	struct nvme_qpair		*qpair;
159 	sbintime_t			deadline;
160 	bus_dmamap_t			payload_dma_map;
161 	uint16_t			cid;
162 
163 	uint64_t			*prp;
164 	bus_addr_t			prp_bus_addr;
165 };
166 
167 enum nvme_recovery {
168 	RECOVERY_NONE = 0,		/* Normal operations */
169 	RECOVERY_START,			/* Deadline has passed, start recovering */
170 	RECOVERY_RESET,			/* This pass, initiate reset of controller */
171 	RECOVERY_WAITING,		/* waiting for the reset to complete */
172 };
173 struct nvme_qpair {
174 	struct nvme_controller	*ctrlr;
175 	uint32_t		id;
176 	int			domain;
177 	int			cpu;
178 
179 	uint16_t		vector;
180 	int			rid;
181 	struct resource		*res;
182 	void 			*tag;
183 
184 	struct callout		timer;
185 	sbintime_t		deadline;
186 	bool			timer_armed;
187 	enum nvme_recovery	recovery_state;
188 
189 	uint32_t		num_entries;
190 	uint32_t		num_trackers;
191 	uint32_t		sq_tdbl_off;
192 	uint32_t		cq_hdbl_off;
193 
194 	uint32_t		phase;
195 	uint32_t		sq_head;
196 	uint32_t		sq_tail;
197 	uint32_t		cq_head;
198 
199 	int64_t			num_cmds;
200 	int64_t			num_intr_handler_calls;
201 	int64_t			num_retries;
202 	int64_t			num_failures;
203 	int64_t			num_ignored;
204 
205 	struct nvme_command	*cmd;
206 	struct nvme_completion	*cpl;
207 
208 	bus_dma_tag_t		dma_tag;
209 	bus_dma_tag_t		dma_tag_payload;
210 
211 	bus_dmamap_t		queuemem_map;
212 	uint64_t		cmd_bus_addr;
213 	uint64_t		cpl_bus_addr;
214 
215 	TAILQ_HEAD(, nvme_tracker)	free_tr;
216 	TAILQ_HEAD(, nvme_tracker)	outstanding_tr;
217 	STAILQ_HEAD(, nvme_request)	queued_req;
218 
219 	struct nvme_tracker	**act_tr;
220 
221 	struct mtx		lock __aligned(CACHE_LINE_SIZE);
222 
223 } __aligned(CACHE_LINE_SIZE);
224 
225 struct nvme_namespace {
226 	struct nvme_controller		*ctrlr;
227 	struct nvme_namespace_data	data;
228 	uint32_t			id;
229 	uint32_t			flags;
230 	struct cdev			*cdev;
231 	void				*cons_cookie[NVME_MAX_CONSUMERS];
232 	uint32_t			boundary;
233 	struct mtx			lock;
234 };
235 
236 /*
237  * One of these per allocated PCI device.
238  */
239 struct nvme_controller {
240 	device_t		dev;
241 
242 	struct mtx		lock;
243 	int			domain;
244 	uint32_t		ready_timeout_in_ms;
245 	uint32_t		quirks;
246 #define	QUIRK_DELAY_B4_CHK_RDY	1		/* Can't touch MMIO on disable */
247 #define	QUIRK_DISABLE_TIMEOUT	2		/* Disable broken completion timeout feature */
248 #define	QUIRK_INTEL_ALIGNMENT	4		/* Pre NVMe 1.3 performance alignment */
249 #define QUIRK_AHCI		8		/* Attached via AHCI redirect */
250 
251 	bus_space_tag_t		bus_tag;
252 	bus_space_handle_t	bus_handle;
253 	int			resource_id;
254 	struct resource		*resource;
255 
256 	/*
257 	 * The NVMe spec allows for the MSI-X table to be placed in BAR 4/5,
258 	 *  separate from the control registers which are in BAR 0/1.  These
259 	 *  members track the mapping of BAR 4/5 for that reason.
260 	 */
261 	int			bar4_resource_id;
262 	struct resource		*bar4_resource;
263 
264 	int			msi_count;
265 	uint32_t		enable_aborts;
266 
267 	uint32_t		num_io_queues;
268 	uint32_t		max_hw_pend_io;
269 
270 	/* Fields for tracking progress during controller initialization. */
271 	struct intr_config_hook	config_hook;
272 	uint32_t		ns_identified;
273 	uint32_t		queues_created;
274 
275 	struct task		reset_task;
276 	struct task		fail_req_task;
277 	struct taskqueue	*taskqueue;
278 
279 	/* For shared legacy interrupt. */
280 	int			rid;
281 	struct resource		*res;
282 	void			*tag;
283 
284 	/** maximum i/o size in bytes */
285 	uint32_t		max_xfer_size;
286 
287 	/** LO and HI capacity mask */
288 	uint32_t		cap_lo;
289 	uint32_t		cap_hi;
290 
291 	/** minimum page size supported by this controller in bytes */
292 	uint32_t		min_page_size;
293 
294 	/** interrupt coalescing time period (in microseconds) */
295 	uint32_t		int_coal_time;
296 
297 	/** interrupt coalescing threshold */
298 	uint32_t		int_coal_threshold;
299 
300 	/** timeout period in seconds */
301 	uint32_t		timeout_period;
302 
303 	/** doorbell stride */
304 	uint32_t		dstrd;
305 
306 	struct nvme_qpair	adminq;
307 	struct nvme_qpair	*ioq;
308 
309 	struct nvme_registers		*regs;
310 
311 	struct nvme_controller_data	cdata;
312 	struct nvme_namespace		ns[NVME_MAX_NAMESPACES];
313 
314 	struct cdev			*cdev;
315 
316 	/** bit mask of event types currently enabled for async events */
317 	uint32_t			async_event_config;
318 
319 	uint32_t			num_aers;
320 	struct nvme_async_event_request	aer[NVME_MAX_ASYNC_EVENTS];
321 
322 	void				*cons_cookie[NVME_MAX_CONSUMERS];
323 
324 	uint32_t			is_resetting;
325 	uint32_t			is_initialized;
326 	uint32_t			notification_sent;
327 
328 	bool				is_failed;
329 	bool				is_dying;
330 	STAILQ_HEAD(, nvme_request)	fail_req;
331 
332 	/* Host Memory Buffer */
333 	int				hmb_nchunks;
334 	size_t				hmb_chunk;
335 	bus_dma_tag_t			hmb_tag;
336 	struct nvme_hmb_chunk {
337 		bus_dmamap_t		hmbc_map;
338 		void			*hmbc_vaddr;
339 		uint64_t		hmbc_paddr;
340 	} *hmb_chunks;
341 	bus_dma_tag_t			hmb_desc_tag;
342 	bus_dmamap_t			hmb_desc_map;
343 	struct nvme_hmb_desc		*hmb_desc_vaddr;
344 	uint64_t			hmb_desc_paddr;
345 };
346 
347 #define nvme_mmio_offsetof(reg)						       \
348 	offsetof(struct nvme_registers, reg)
349 
350 #define nvme_mmio_read_4(sc, reg)					       \
351 	bus_space_read_4((sc)->bus_tag, (sc)->bus_handle,		       \
352 	    nvme_mmio_offsetof(reg))
353 
354 #define nvme_mmio_write_4(sc, reg, val)					       \
355 	bus_space_write_4((sc)->bus_tag, (sc)->bus_handle,		       \
356 	    nvme_mmio_offsetof(reg), val)
357 
358 #define nvme_mmio_write_8(sc, reg, val)					       \
359 	do {								       \
360 		bus_space_write_4((sc)->bus_tag, (sc)->bus_handle,	       \
361 		    nvme_mmio_offsetof(reg), val & 0xFFFFFFFF); 	       \
362 		bus_space_write_4((sc)->bus_tag, (sc)->bus_handle,	       \
363 		    nvme_mmio_offsetof(reg)+4,				       \
364 		    (val & 0xFFFFFFFF00000000ULL) >> 32);		       \
365 	} while (0);
366 
367 #define nvme_printf(ctrlr, fmt, args...)	\
368     device_printf(ctrlr->dev, fmt, ##args)
369 
370 void	nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg);
371 
372 void	nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
373 					   void *payload,
374 					   nvme_cb_fn_t cb_fn, void *cb_arg);
375 void	nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
376 					  uint32_t nsid, void *payload,
377 					  nvme_cb_fn_t cb_fn, void *cb_arg);
378 void	nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
379 						uint32_t microseconds,
380 						uint32_t threshold,
381 						nvme_cb_fn_t cb_fn,
382 						void *cb_arg);
383 void	nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
384 				      struct nvme_error_information_entry *payload,
385 				      uint32_t num_entries, /* 0 = max */
386 				      nvme_cb_fn_t cb_fn,
387 				      void *cb_arg);
388 void	nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
389 						   uint32_t nsid,
390 						   struct nvme_health_information_page *payload,
391 						   nvme_cb_fn_t cb_fn,
392 						   void *cb_arg);
393 void	nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
394 					 struct nvme_firmware_page *payload,
395 					 nvme_cb_fn_t cb_fn,
396 					 void *cb_arg);
397 void	nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
398 				    struct nvme_qpair *io_que,
399 				    nvme_cb_fn_t cb_fn, void *cb_arg);
400 void	nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
401 				    struct nvme_qpair *io_que,
402 				    nvme_cb_fn_t cb_fn, void *cb_arg);
403 void	nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
404 				    struct nvme_qpair *io_que,
405 				    nvme_cb_fn_t cb_fn, void *cb_arg);
406 void	nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
407 				    struct nvme_qpair *io_que,
408 				    nvme_cb_fn_t cb_fn, void *cb_arg);
409 void	nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
410 				      uint32_t num_queues, nvme_cb_fn_t cb_fn,
411 				      void *cb_arg);
412 void	nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
413 					      uint32_t state,
414 					      nvme_cb_fn_t cb_fn, void *cb_arg);
415 void	nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
416 			     uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
417 
418 void	nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
419 
420 int	nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev);
421 void	nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev);
422 void	nvme_ctrlr_shutdown(struct nvme_controller *ctrlr);
423 void	nvme_ctrlr_reset(struct nvme_controller *ctrlr);
424 /* ctrlr defined as void * to allow use with config_intrhook. */
425 void	nvme_ctrlr_start_config_hook(void *ctrlr_arg);
426 void	nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
427 					struct nvme_request *req);
428 void	nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
429 				     struct nvme_request *req);
430 void	nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
431 				       struct nvme_request *req);
432 
433 int	nvme_qpair_construct(struct nvme_qpair *qpair,
434 			     uint32_t num_entries, uint32_t num_trackers,
435 			     struct nvme_controller *ctrlr);
436 void	nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
437 				  struct nvme_tracker *tr);
438 bool	nvme_qpair_process_completions(struct nvme_qpair *qpair);
439 void	nvme_qpair_submit_request(struct nvme_qpair *qpair,
440 				  struct nvme_request *req);
441 void	nvme_qpair_reset(struct nvme_qpair *qpair);
442 void	nvme_qpair_fail(struct nvme_qpair *qpair);
443 void	nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
444 					   struct nvme_request *req,
445                                            uint32_t sct, uint32_t sc);
446 
447 void	nvme_admin_qpair_enable(struct nvme_qpair *qpair);
448 void	nvme_admin_qpair_disable(struct nvme_qpair *qpair);
449 void	nvme_admin_qpair_destroy(struct nvme_qpair *qpair);
450 
451 void	nvme_io_qpair_enable(struct nvme_qpair *qpair);
452 void	nvme_io_qpair_disable(struct nvme_qpair *qpair);
453 void	nvme_io_qpair_destroy(struct nvme_qpair *qpair);
454 
455 int	nvme_ns_construct(struct nvme_namespace *ns, uint32_t id,
456 			  struct nvme_controller *ctrlr);
457 void	nvme_ns_destruct(struct nvme_namespace *ns);
458 
459 void	nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr);
460 
461 void	nvme_dump_command(struct nvme_command *cmd);
462 void	nvme_dump_completion(struct nvme_completion *cpl);
463 
464 int	nvme_attach(device_t dev);
465 int	nvme_shutdown(device_t dev);
466 int	nvme_detach(device_t dev);
467 
468 /*
469  * Wait for a command to complete using the nvme_completion_poll_cb.  Used in
470  * limited contexts where the caller knows it's OK to block briefly while the
471  * command runs. The ISR will run the callback which will set status->done to
472  * true, usually within microseconds. If not, then after one second timeout
473  * handler should reset the controller and abort all outstanding requests
474  * including this polled one. If still not after ten seconds, then something is
475  * wrong with the driver, and panic is the only way to recover.
476  *
477  * Most commands using this interface aren't actual I/O to the drive's media so
478  * complete within a few microseconds. Adaptively spin for one tick to catch the
479  * vast majority of these without waiting for a tick plus scheduling delays. Since
480  * these are on startup, this drastically reduces startup time.
481  */
482 static __inline
483 void
484 nvme_completion_poll(struct nvme_completion_poll_status *status)
485 {
486 	int timeout = ticks + 10 * hz;
487 	sbintime_t delta_t = SBT_1US;
488 
489 	while (!atomic_load_acq_int(&status->done)) {
490 		if (timeout - ticks < 0)
491 			panic("NVME polled command failed to complete within 10s.");
492 		pause_sbt("nvme", delta_t, 0, C_PREL(1));
493 		delta_t = min(SBT_1MS, delta_t * 3 / 2);
494 	}
495 }
496 
497 static __inline void
498 nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
499 {
500 	uint64_t *bus_addr = (uint64_t *)arg;
501 
502 	KASSERT(nseg == 1, ("number of segments (%d) is not 1", nseg));
503 	if (error != 0)
504 		printf("nvme_single_map err %d\n", error);
505 	*bus_addr = seg[0].ds_addr;
506 }
507 
508 static __inline struct nvme_request *
509 _nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
510 {
511 	struct nvme_request *req;
512 
513 	req = malloc(sizeof(*req), M_NVME, M_NOWAIT | M_ZERO);
514 	if (req != NULL) {
515 		req->cb_fn = cb_fn;
516 		req->cb_arg = cb_arg;
517 		req->timeout = true;
518 	}
519 	return (req);
520 }
521 
522 static __inline struct nvme_request *
523 nvme_allocate_request_vaddr(void *payload, uint32_t payload_size,
524     nvme_cb_fn_t cb_fn, void *cb_arg)
525 {
526 	struct nvme_request *req;
527 
528 	req = _nvme_allocate_request(cb_fn, cb_arg);
529 	if (req != NULL) {
530 		req->type = NVME_REQUEST_VADDR;
531 		req->u.payload = payload;
532 		req->payload_size = payload_size;
533 	}
534 	return (req);
535 }
536 
537 static __inline struct nvme_request *
538 nvme_allocate_request_null(nvme_cb_fn_t cb_fn, void *cb_arg)
539 {
540 	struct nvme_request *req;
541 
542 	req = _nvme_allocate_request(cb_fn, cb_arg);
543 	if (req != NULL)
544 		req->type = NVME_REQUEST_NULL;
545 	return (req);
546 }
547 
548 static __inline struct nvme_request *
549 nvme_allocate_request_bio(struct bio *bio, nvme_cb_fn_t cb_fn, void *cb_arg)
550 {
551 	struct nvme_request *req;
552 
553 	req = _nvme_allocate_request(cb_fn, cb_arg);
554 	if (req != NULL) {
555 		req->type = NVME_REQUEST_BIO;
556 		req->u.bio = bio;
557 	}
558 	return (req);
559 }
560 
561 static __inline struct nvme_request *
562 nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg)
563 {
564 	struct nvme_request *req;
565 
566 	req = _nvme_allocate_request(cb_fn, cb_arg);
567 	if (req != NULL) {
568 		req->type = NVME_REQUEST_CCB;
569 		req->u.payload = ccb;
570 	}
571 
572 	return (req);
573 }
574 
575 #define nvme_free_request(req)	free(req, M_NVME)
576 
577 void	nvme_notify_async_consumers(struct nvme_controller *ctrlr,
578 				    const struct nvme_completion *async_cpl,
579 				    uint32_t log_page_id, void *log_page_buffer,
580 				    uint32_t log_page_size);
581 void	nvme_notify_fail_consumers(struct nvme_controller *ctrlr);
582 void	nvme_notify_new_controller(struct nvme_controller *ctrlr);
583 void	nvme_notify_ns(struct nvme_controller *ctrlr, int nsid);
584 
585 void	nvme_ctrlr_shared_handler(void *arg);
586 void	nvme_ctrlr_poll(struct nvme_controller *ctrlr);
587 
588 int	nvme_ctrlr_suspend(struct nvme_controller *ctrlr);
589 int	nvme_ctrlr_resume(struct nvme_controller *ctrlr);
590 
591 #endif /* __NVME_PRIVATE_H__ */
592