xref: /freebsd/sys/cam/nvme/nvme_xpt.c (revision 32100375a661c1e16588ddfa7b90ca8d26cb9786)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2015 Netflix, Inc.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer,
11  *    without modification, immediately at the beginning of the file.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  *
27  * derived from ata_xpt.c: Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/endian.h>
36 #include <sys/systm.h>
37 #include <sys/types.h>
38 #include <sys/malloc.h>
39 #include <sys/kernel.h>
40 #include <sys/time.h>
41 #include <sys/conf.h>
42 #include <sys/fcntl.h>
43 #include <sys/sbuf.h>
44 
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/sysctl.h>
48 
49 #include <cam/cam.h>
50 #include <cam/cam_ccb.h>
51 #include <cam/cam_queue.h>
52 #include <cam/cam_periph.h>
53 #include <cam/cam_sim.h>
54 #include <cam/cam_xpt.h>
55 #include <cam/cam_xpt_sim.h>
56 #include <cam/cam_xpt_periph.h>
57 #include <cam/cam_xpt_internal.h>
58 #include <cam/cam_debug.h>
59 
60 #include <cam/scsi/scsi_all.h>
61 #include <cam/scsi/scsi_message.h>
62 #include <cam/nvme/nvme_all.h>
63 #include <machine/stdarg.h>	/* for xpt_print below */
64 #include "opt_cam.h"
65 
66 struct nvme_quirk_entry {
67 	u_int quirks;
68 #define CAM_QUIRK_MAXTAGS 1
69 	u_int mintags;
70 	u_int maxtags;
71 };
72 
73 /* Not even sure why we need this */
74 static periph_init_t nvme_probe_periph_init;
75 
76 static struct periph_driver nvme_probe_driver =
77 {
78 	nvme_probe_periph_init, "nvme_probe",
79 	TAILQ_HEAD_INITIALIZER(nvme_probe_driver.units), /* generation */ 0,
80 	CAM_PERIPH_DRV_EARLY
81 };
82 
83 PERIPHDRIVER_DECLARE(nvme_probe, nvme_probe_driver);
84 
85 typedef enum {
86 	NVME_PROBE_IDENTIFY_CD,
87 	NVME_PROBE_IDENTIFY_NS,
88 	NVME_PROBE_DONE,
89 	NVME_PROBE_INVALID
90 } nvme_probe_action;
91 
92 static char *nvme_probe_action_text[] = {
93 	"NVME_PROBE_IDENTIFY_CD",
94 	"NVME_PROBE_IDENTIFY_NS",
95 	"NVME_PROBE_DONE",
96 	"NVME_PROBE_INVALID"
97 };
98 
99 #define NVME_PROBE_SET_ACTION(softc, newaction)	\
100 do {									\
101 	char **text;							\
102 	text = nvme_probe_action_text;					\
103 	CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE,		\
104 	    ("Probe %s to %s\n", text[(softc)->action],			\
105 	    text[(newaction)]));					\
106 	(softc)->action = (newaction);					\
107 } while(0)
108 
109 typedef enum {
110 	NVME_PROBE_NO_ANNOUNCE	= 0x04
111 } nvme_probe_flags;
112 
113 typedef struct {
114 	TAILQ_HEAD(, ccb_hdr) request_ccbs;
115 	union {
116 		struct nvme_controller_data	cd;
117 		struct nvme_namespace_data	ns;
118 	};
119 	nvme_probe_action	action;
120 	nvme_probe_flags	flags;
121 	int		restart;
122 	struct cam_periph *periph;
123 } nvme_probe_softc;
124 
125 static struct nvme_quirk_entry nvme_quirk_table[] =
126 {
127 	{
128 //		{
129 //		  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
130 //		  /*vendor*/"*", /*product*/"*", /*revision*/"*"
131 //		},
132 		.quirks = 0, .mintags = 0, .maxtags = 0
133 	},
134 };
135 
136 static const int nvme_quirk_table_size =
137 	sizeof(nvme_quirk_table) / sizeof(*nvme_quirk_table);
138 
139 static cam_status	nvme_probe_register(struct cam_periph *periph,
140 				      void *arg);
141 static void	 nvme_probe_schedule(struct cam_periph *nvme_probe_periph);
142 static void	 nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb);
143 static void	 nvme_probe_done(struct cam_periph *periph, union ccb *done_ccb);
144 static void	 nvme_probe_cleanup(struct cam_periph *periph);
145 //static void	 nvme_find_quirk(struct cam_ed *device);
146 static void	 nvme_scan_lun(struct cam_periph *periph,
147 			       struct cam_path *path, cam_flags flags,
148 			       union ccb *ccb);
149 static struct cam_ed *
150 		 nvme_alloc_device(struct cam_eb *bus, struct cam_et *target,
151 				   lun_id_t lun_id);
152 static void	 nvme_device_transport(struct cam_path *path);
153 static void	 nvme_dev_async(u_int32_t async_code,
154 				struct cam_eb *bus,
155 				struct cam_et *target,
156 				struct cam_ed *device,
157 				void *async_arg);
158 static void	 nvme_action(union ccb *start_ccb);
159 static void	 nvme_announce_periph(struct cam_periph *periph);
160 static void	 nvme_proto_announce(struct cam_ed *device);
161 static void	 nvme_proto_denounce(struct cam_ed *device);
162 static void	 nvme_proto_debug_out(union ccb *ccb);
163 
164 static struct xpt_xport_ops nvme_xport_ops = {
165 	.alloc_device = nvme_alloc_device,
166 	.action = nvme_action,
167 	.async = nvme_dev_async,
168 	.announce = nvme_announce_periph,
169 };
170 #define NVME_XPT_XPORT(x, X)			\
171 static struct xpt_xport nvme_xport_ ## x = {	\
172 	.xport = XPORT_ ## X,			\
173 	.name = #x,				\
174 	.ops = &nvme_xport_ops,			\
175 };						\
176 CAM_XPT_XPORT(nvme_xport_ ## x);
177 
178 NVME_XPT_XPORT(nvme, NVME);
179 
180 #undef NVME_XPT_XPORT
181 
182 static struct xpt_proto_ops nvme_proto_ops = {
183 	.announce = nvme_proto_announce,
184 	.denounce = nvme_proto_denounce,
185 	.debug_out = nvme_proto_debug_out,
186 };
187 static struct xpt_proto nvme_proto = {
188 	.proto = PROTO_NVME,
189 	.name = "nvme",
190 	.ops = &nvme_proto_ops,
191 };
192 CAM_XPT_PROTO(nvme_proto);
193 
194 static void
195 nvme_probe_periph_init(void)
196 {
197 }
198 
199 static cam_status
200 nvme_probe_register(struct cam_periph *periph, void *arg)
201 {
202 	union ccb *request_ccb;	/* CCB representing the probe request */
203 	nvme_probe_softc *softc;
204 
205 	request_ccb = (union ccb *)arg;
206 	if (request_ccb == NULL) {
207 		printf("nvme_probe_register: no probe CCB, "
208 		       "can't register device\n");
209 		return(CAM_REQ_CMP_ERR);
210 	}
211 
212 	softc = (nvme_probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT);
213 
214 	if (softc == NULL) {
215 		printf("nvme_probe_register: Unable to probe new device. "
216 		       "Unable to allocate softc\n");
217 		return(CAM_REQ_CMP_ERR);
218 	}
219 	TAILQ_INIT(&softc->request_ccbs);
220 	TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
221 			  periph_links.tqe);
222 	softc->flags = 0;
223 	periph->softc = softc;
224 	softc->periph = periph;
225 	softc->action = NVME_PROBE_INVALID;
226 	if (cam_periph_acquire(periph) != 0)
227 		return (CAM_REQ_CMP_ERR);
228 
229 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
230 
231 //	nvme_device_transport(periph->path);
232 	nvme_probe_schedule(periph);
233 
234 	return(CAM_REQ_CMP);
235 }
236 
237 static void
238 nvme_probe_schedule(struct cam_periph *periph)
239 {
240 	union ccb *ccb;
241 	nvme_probe_softc *softc;
242 
243 	softc = (nvme_probe_softc *)periph->softc;
244 	ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
245 
246 	NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_CD);
247 
248 	if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
249 		softc->flags |= NVME_PROBE_NO_ANNOUNCE;
250 	else
251 		softc->flags &= ~NVME_PROBE_NO_ANNOUNCE;
252 
253 	xpt_schedule(periph, CAM_PRIORITY_XPT);
254 }
255 
256 static void
257 nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb)
258 {
259 	struct ccb_nvmeio *nvmeio;
260 	nvme_probe_softc *softc;
261 	struct cam_path *path;
262 	lun_id_t lun;
263 
264 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_probe_start\n"));
265 
266 	softc = (nvme_probe_softc *)periph->softc;
267 	path = start_ccb->ccb_h.path;
268 	nvmeio = &start_ccb->nvmeio;
269 	lun = xpt_path_lun_id(periph->path);
270 
271 	if (softc->restart) {
272 		softc->restart = 0;
273 		NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_CD);
274 	}
275 
276 	switch (softc->action) {
277 	case NVME_PROBE_IDENTIFY_CD:
278 		cam_fill_nvmeadmin(nvmeio,
279 		    0,			/* retries */
280 		    nvme_probe_done,	/* cbfcnp */
281 		    CAM_DIR_IN,		/* flags */
282 		    (uint8_t *)&softc->cd,	/* data_ptr */
283 		    sizeof(softc->cd),		/* dxfer_len */
284 		    30 * 1000); /* timeout 30s */
285 		nvme_ns_cmd(nvmeio, NVME_OPC_IDENTIFY, 0,
286 		    1, 0, 0, 0, 0, 0);
287 		break;
288 	case NVME_PROBE_IDENTIFY_NS:
289 		cam_fill_nvmeadmin(nvmeio,
290 		    0,			/* retries */
291 		    nvme_probe_done,	/* cbfcnp */
292 		    CAM_DIR_IN,		/* flags */
293 		    (uint8_t *)&softc->ns,	/* data_ptr */
294 		    sizeof(softc->ns),		/* dxfer_len */
295 		    30 * 1000); /* timeout 30s */
296 		nvme_ns_cmd(nvmeio, NVME_OPC_IDENTIFY, lun,
297 		    0, 0, 0, 0, 0, 0);
298 		break;
299 	default:
300 		panic("nvme_probe_start: invalid action state 0x%x\n", softc->action);
301 	}
302 	start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
303 	xpt_action(start_ccb);
304 }
305 
306 static void
307 nvme_probe_done(struct cam_periph *periph, union ccb *done_ccb)
308 {
309 	struct nvme_namespace_data *nvme_data;
310 	struct nvme_controller_data *nvme_cdata;
311 	nvme_probe_softc *softc;
312 	struct cam_path *path;
313 	cam_status status;
314 	u_int32_t  priority;
315 	int found = 1;
316 
317 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_probe_done\n"));
318 
319 	softc = (nvme_probe_softc *)periph->softc;
320 	path = done_ccb->ccb_h.path;
321 	priority = done_ccb->ccb_h.pinfo.priority;
322 
323 	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
324 		if (cam_periph_error(done_ccb,
325 			0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0
326 		    ) == ERESTART) {
327 out:
328 			/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
329 			cam_release_devq(path, 0, 0, 0, FALSE);
330 			return;
331 		}
332 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
333 			/* Don't wedge the queue */
334 			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
335 		}
336 		status = done_ccb->ccb_h.status & CAM_STATUS_MASK;
337 
338 		/*
339 		 * If we get to this point, we got an error status back
340 		 * from the inquiry and the error status doesn't require
341 		 * automatically retrying the command.  Therefore, the
342 		 * inquiry failed.  If we had inquiry information before
343 		 * for this device, but this latest inquiry command failed,
344 		 * the device has probably gone away.  If this device isn't
345 		 * already marked unconfigured, notify the peripheral
346 		 * drivers that this device is no more.
347 		 */
348 device_fail:	if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
349 			xpt_async(AC_LOST_DEVICE, path, NULL);
350 		NVME_PROBE_SET_ACTION(softc, NVME_PROBE_INVALID);
351 		found = 0;
352 		goto done;
353 	}
354 	if (softc->restart)
355 		goto done;
356 	switch (softc->action) {
357 	case NVME_PROBE_IDENTIFY_CD:
358 		nvme_controller_data_swapbytes(&softc->cd);
359 
360 		nvme_cdata = path->device->nvme_cdata;
361 		if (nvme_cdata == NULL) {
362 			nvme_cdata = malloc(sizeof(*nvme_cdata), M_CAMXPT,
363 			    M_NOWAIT);
364 			if (nvme_cdata == NULL) {
365 				xpt_print(path, "Can't allocate memory");
366 				goto device_fail;
367 			}
368 		}
369 		bcopy(&softc->cd, nvme_cdata, sizeof(*nvme_cdata));
370 		path->device->nvme_cdata = nvme_cdata;
371 
372 //		nvme_find_quirk(path->device);
373 		nvme_device_transport(path);
374 		NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_NS);
375 		xpt_release_ccb(done_ccb);
376 		xpt_schedule(periph, priority);
377 		goto out;
378 	case NVME_PROBE_IDENTIFY_NS:
379 		nvme_namespace_data_swapbytes(&softc->ns);
380 
381 		/* Check that the namespace exists. */
382 		if (softc->ns.nsze == 0)
383 			goto device_fail;
384 
385 		nvme_data = path->device->nvme_data;
386 		if (nvme_data == NULL) {
387 			nvme_data = malloc(sizeof(*nvme_data), M_CAMXPT,
388 			    M_NOWAIT);
389 			if (nvme_data == NULL) {
390 				xpt_print(path, "Can't allocate memory");
391 				goto device_fail;
392 			}
393 		}
394 		bcopy(&softc->ns, nvme_data, sizeof(*nvme_data));
395 		path->device->nvme_data = nvme_data;
396 
397 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
398 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
399 			xpt_acquire_device(path->device);
400 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
401 			xpt_action(done_ccb);
402 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
403 		}
404 		NVME_PROBE_SET_ACTION(softc, NVME_PROBE_DONE);
405 		break;
406 	default:
407 		panic("nvme_probe_done: invalid action state 0x%x\n", softc->action);
408 	}
409 done:
410 	if (softc->restart) {
411 		softc->restart = 0;
412 		xpt_release_ccb(done_ccb);
413 		nvme_probe_schedule(periph);
414 		goto out;
415 	}
416 	xpt_release_ccb(done_ccb);
417 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n"));
418 	while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) {
419 		TAILQ_REMOVE(&softc->request_ccbs,
420 		    &done_ccb->ccb_h, periph_links.tqe);
421 		done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR;
422 		xpt_done(done_ccb);
423 	}
424 	/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
425 	cam_release_devq(path, 0, 0, 0, FALSE);
426 	cam_periph_invalidate(periph);
427 	cam_periph_release_locked(periph);
428 }
429 
430 static void
431 nvme_probe_cleanup(struct cam_periph *periph)
432 {
433 
434 	free(periph->softc, M_CAMXPT);
435 }
436 
437 #if 0
438 /* XXX should be used, don't delete */
439 static void
440 nvme_find_quirk(struct cam_ed *device)
441 {
442 	struct nvme_quirk_entry *quirk;
443 	caddr_t	match;
444 
445 	match = cam_quirkmatch((caddr_t)&device->nvme_data,
446 			       (caddr_t)nvme_quirk_table,
447 			       nvme_quirk_table_size,
448 			       sizeof(*nvme_quirk_table), nvme_identify_match);
449 
450 	if (match == NULL)
451 		panic("xpt_find_quirk: device didn't match wildcard entry!!");
452 
453 	quirk = (struct nvme_quirk_entry *)match;
454 	device->quirk = quirk;
455 	if (quirk->quirks & CAM_QUIRK_MAXTAGS) {
456 		device->mintags = quirk->mintags;
457 		device->maxtags = quirk->maxtags;
458 	}
459 }
460 #endif
461 
462 static void
463 nvme_scan_lun(struct cam_periph *periph, struct cam_path *path,
464 	     cam_flags flags, union ccb *request_ccb)
465 {
466 	struct ccb_pathinq cpi;
467 	cam_status status;
468 	struct cam_periph *old_periph;
469 	int lock;
470 
471 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun\n"));
472 
473 	xpt_path_inq(&cpi, path);
474 
475 	if (cpi.ccb_h.status != CAM_REQ_CMP) {
476 		if (request_ccb != NULL) {
477 			request_ccb->ccb_h.status = cpi.ccb_h.status;
478 			xpt_done(request_ccb);
479 		}
480 		return;
481 	}
482 
483 	if (xpt_path_lun_id(path) == CAM_LUN_WILDCARD) {
484 		CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun ignoring bus\n"));
485 		request_ccb->ccb_h.status = CAM_REQ_CMP;	/* XXX signal error ? */
486 		xpt_done(request_ccb);
487 		return;
488 	}
489 
490 	lock = (xpt_path_owned(path) == 0);
491 	if (lock)
492 		xpt_path_lock(path);
493 	if ((old_periph = cam_periph_find(path, "nvme_probe")) != NULL) {
494 		if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
495 			nvme_probe_softc *softc;
496 
497 			softc = (nvme_probe_softc *)old_periph->softc;
498 			TAILQ_INSERT_TAIL(&softc->request_ccbs,
499 				&request_ccb->ccb_h, periph_links.tqe);
500 			softc->restart = 1;
501 			CAM_DEBUG(path, CAM_DEBUG_TRACE,
502 			    ("restarting nvme_probe device\n"));
503 		} else {
504 			request_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
505 			CAM_DEBUG(path, CAM_DEBUG_TRACE,
506 			    ("Failing to restart nvme_probe device\n"));
507 			xpt_done(request_ccb);
508 		}
509 	} else {
510 		CAM_DEBUG(path, CAM_DEBUG_TRACE,
511 		    ("Adding nvme_probe device\n"));
512 		status = cam_periph_alloc(nvme_probe_register, NULL, nvme_probe_cleanup,
513 					  nvme_probe_start, "nvme_probe",
514 					  CAM_PERIPH_BIO,
515 					  request_ccb->ccb_h.path, NULL, 0,
516 					  request_ccb);
517 
518 		if (status != CAM_REQ_CMP) {
519 			xpt_print(path, "xpt_scan_lun: cam_alloc_periph "
520 			    "returned an error, can't continue probe\n");
521 			request_ccb->ccb_h.status = status;
522 			xpt_done(request_ccb);
523 		}
524 	}
525 	if (lock)
526 		xpt_path_unlock(path);
527 }
528 
529 static struct cam_ed *
530 nvme_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
531 {
532 	struct nvme_quirk_entry *quirk;
533 	struct cam_ed *device;
534 
535 	device = xpt_alloc_device(bus, target, lun_id);
536 	if (device == NULL)
537 		return (NULL);
538 
539 	/*
540 	 * Take the default quirk entry until we have inquiry
541 	 * data from nvme and can determine a better quirk to use.
542 	 */
543 	quirk = &nvme_quirk_table[nvme_quirk_table_size - 1];
544 	device->quirk = (void *)quirk;
545 	device->mintags = 0;
546 	device->maxtags = 0;
547 	device->inq_flags = 0;
548 	device->queue_flags = 0;
549 	device->device_id = NULL;	/* XXX Need to set this somewhere */
550 	device->device_id_len = 0;
551 	device->serial_num = NULL;	/* XXX Need to set this somewhere */
552 	device->serial_num_len = 0;
553 	return (device);
554 }
555 
556 static void
557 nvme_device_transport(struct cam_path *path)
558 {
559 	struct ccb_pathinq cpi;
560 	struct ccb_trans_settings cts;
561 	/* XXX get data from nvme namespace and other info ??? */
562 
563 	/* Get transport information from the SIM */
564 	xpt_path_inq(&cpi, path);
565 
566 	path->device->transport = cpi.transport;
567 	path->device->transport_version = cpi.transport_version;
568 
569 	path->device->protocol = cpi.protocol;
570 	path->device->protocol_version = cpi.protocol_version;
571 
572 	/* Tell the controller what we think */
573 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
574 	cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
575 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
576 	cts.transport = path->device->transport;
577 	cts.transport_version = path->device->transport_version;
578 	cts.protocol = path->device->protocol;
579 	cts.protocol_version = path->device->protocol_version;
580 	cts.proto_specific.valid = 0;
581 	cts.xport_specific.valid = 0;
582 	xpt_action((union ccb *)&cts);
583 }
584 
585 static void
586 nvme_dev_advinfo(union ccb *start_ccb)
587 {
588 	struct cam_ed *device;
589 	struct ccb_dev_advinfo *cdai;
590 	off_t amt;
591 
592 	xpt_path_assert(start_ccb->ccb_h.path, MA_OWNED);
593 	start_ccb->ccb_h.status = CAM_REQ_INVALID;
594 	device = start_ccb->ccb_h.path->device;
595 	cdai = &start_ccb->cdai;
596 	switch(cdai->buftype) {
597 	case CDAI_TYPE_SCSI_DEVID:
598 		if (cdai->flags & CDAI_FLAG_STORE)
599 			return;
600 		cdai->provsiz = device->device_id_len;
601 		if (device->device_id_len == 0)
602 			break;
603 		amt = device->device_id_len;
604 		if (cdai->provsiz > cdai->bufsiz)
605 			amt = cdai->bufsiz;
606 		memcpy(cdai->buf, device->device_id, amt);
607 		break;
608 	case CDAI_TYPE_SERIAL_NUM:
609 		if (cdai->flags & CDAI_FLAG_STORE)
610 			return;
611 		cdai->provsiz = device->serial_num_len;
612 		if (device->serial_num_len == 0)
613 			break;
614 		amt = device->serial_num_len;
615 		if (cdai->provsiz > cdai->bufsiz)
616 			amt = cdai->bufsiz;
617 		memcpy(cdai->buf, device->serial_num, amt);
618 		break;
619 	case CDAI_TYPE_PHYS_PATH:
620 		if (cdai->flags & CDAI_FLAG_STORE) {
621 			if (device->physpath != NULL)
622 				free(device->physpath, M_CAMXPT);
623 			device->physpath_len = cdai->bufsiz;
624 			/* Clear existing buffer if zero length */
625 			if (cdai->bufsiz == 0)
626 				break;
627 			device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT);
628 			if (device->physpath == NULL) {
629 				start_ccb->ccb_h.status = CAM_REQ_ABORTED;
630 				return;
631 			}
632 			memcpy(device->physpath, cdai->buf, cdai->bufsiz);
633 		} else {
634 			cdai->provsiz = device->physpath_len;
635 			if (device->physpath_len == 0)
636 				break;
637 			amt = device->physpath_len;
638 			if (cdai->provsiz > cdai->bufsiz)
639 				amt = cdai->bufsiz;
640 			memcpy(cdai->buf, device->physpath, amt);
641 		}
642 		break;
643 	case CDAI_TYPE_NVME_CNTRL:
644 		if (cdai->flags & CDAI_FLAG_STORE)
645 			return;
646 		amt = sizeof(struct nvme_controller_data);
647 		cdai->provsiz = amt;
648 		if (amt > cdai->bufsiz)
649 			amt = cdai->bufsiz;
650 		memcpy(cdai->buf, device->nvme_cdata, amt);
651 		break;
652 	case CDAI_TYPE_NVME_NS:
653 		if (cdai->flags & CDAI_FLAG_STORE)
654 			return;
655 		amt = sizeof(struct nvme_namespace_data);
656 		cdai->provsiz = amt;
657 		if (amt > cdai->bufsiz)
658 			amt = cdai->bufsiz;
659 		memcpy(cdai->buf, device->nvme_data, amt);
660 		break;
661 	default:
662 		return;
663 	}
664 	start_ccb->ccb_h.status = CAM_REQ_CMP;
665 
666 	if (cdai->flags & CDAI_FLAG_STORE) {
667 		xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
668 			  (void *)(uintptr_t)cdai->buftype);
669 	}
670 }
671 
672 static void
673 nvme_action(union ccb *start_ccb)
674 {
675 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
676 	    ("nvme_action: func= %#x\n", start_ccb->ccb_h.func_code));
677 
678 	switch (start_ccb->ccb_h.func_code) {
679 	case XPT_SCAN_BUS:
680 	case XPT_SCAN_TGT:
681 	case XPT_SCAN_LUN:
682 		nvme_scan_lun(start_ccb->ccb_h.path->periph,
683 			      start_ccb->ccb_h.path, start_ccb->crcn.flags,
684 			      start_ccb);
685 		break;
686 	case XPT_DEV_ADVINFO:
687 		nvme_dev_advinfo(start_ccb);
688 		break;
689 
690 	default:
691 		xpt_action_default(start_ccb);
692 		break;
693 	}
694 }
695 
696 /*
697  * Handle any per-device event notifications that require action by the XPT.
698  */
699 static void
700 nvme_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target,
701 	      struct cam_ed *device, void *async_arg)
702 {
703 
704 	/*
705 	 * We only need to handle events for real devices.
706 	 */
707 	if (target->target_id == CAM_TARGET_WILDCARD
708 	 || device->lun_id == CAM_LUN_WILDCARD)
709 		return;
710 
711 	if (async_code == AC_LOST_DEVICE &&
712 	    (device->flags & CAM_DEV_UNCONFIGURED) == 0) {
713 		device->flags |= CAM_DEV_UNCONFIGURED;
714 		xpt_release_device(device);
715 	}
716 }
717 
718 static void
719 nvme_announce_periph(struct cam_periph *periph)
720 {
721 	struct	ccb_pathinq cpi;
722 	struct	ccb_trans_settings cts;
723 	struct	cam_path *path = periph->path;
724 	struct ccb_trans_settings_nvme	*nvmex;
725 	struct sbuf	sb;
726 	char		buffer[120];
727 
728 	cam_periph_assert(periph, MA_OWNED);
729 
730 	/* Ask the SIM for connection details */
731 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
732 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
733 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
734 	xpt_action((union ccb*)&cts);
735 	if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
736 		return;
737 	nvmex = &cts.xport_specific.nvme;
738 
739 	/* Ask the SIM for its base transfer speed */
740 	xpt_path_inq(&cpi, periph->path);
741 	sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN);
742 	sbuf_printf(&sb, "%s%d: nvme version %d.%d",
743 	    periph->periph_name, periph->unit_number,
744 	    NVME_MAJOR(nvmex->spec),
745 	    NVME_MINOR(nvmex->spec));
746 	if (nvmex->valid & CTS_NVME_VALID_LINK)
747 		sbuf_printf(&sb, " x%d (max x%d) lanes PCIe Gen%d (max Gen%d) link",
748 		    nvmex->lanes, nvmex->max_lanes,
749 		    nvmex->speed, nvmex->max_speed);
750 	sbuf_printf(&sb, "\n");
751 	sbuf_finish(&sb);
752 	sbuf_putbuf(&sb);
753 }
754 
755 static void
756 nvme_proto_announce(struct cam_ed *device)
757 {
758 	struct sbuf	sb;
759 	char		buffer[120];
760 
761 	sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN);
762 	nvme_print_ident(device->nvme_cdata, device->nvme_data, &sb);
763 	sbuf_finish(&sb);
764 	sbuf_putbuf(&sb);
765 }
766 
767 static void
768 nvme_proto_denounce(struct cam_ed *device)
769 {
770 
771 	nvme_proto_announce(device);
772 }
773 
774 static void
775 nvme_proto_debug_out(union ccb *ccb)
776 {
777 	char cdb_str[(sizeof(struct nvme_command) * 3) + 1];
778 
779 	if (ccb->ccb_h.func_code != XPT_NVME_IO &&
780 	    ccb->ccb_h.func_code != XPT_NVME_ADMIN)
781 		return;
782 
783 	CAM_DEBUG(ccb->ccb_h.path,
784 	    CAM_DEBUG_CDB,("%s. NCB: %s\n", nvme_op_string(&ccb->nvmeio.cmd,
785 		ccb->ccb_h.func_code == XPT_NVME_ADMIN),
786 		nvme_cmd_string(&ccb->nvmeio.cmd, cdb_str, sizeof(cdb_str))));
787 }
788 
789