xref: /titanic_41/usr/src/uts/common/io/fibre-channel/ulp/fcsm.c (revision 96d9f183facd90dbbc2268c9a51689be0b6a0b46)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * fcsm - ULP Module for Fibre Channel SAN Management
28  */
29 
30 #include <sys/types.h>
31 #include <sys/file.h>
32 #include <sys/kmem.h>
33 #include <sys/scsi/scsi.h>
34 #include <sys/var.h>
35 #include <sys/byteorder.h>
36 #include <sys/fibre-channel/fc.h>
37 #include <sys/fibre-channel/impl/fc_ulpif.h>
38 #include <sys/fibre-channel/ulp/fcsm.h>
39 
40 /* Definitions */
41 #define	FCSM_VERSION		"20090729-1.28"
42 #define	FCSM_NAME_VERSION	"SunFC FCSM v" FCSM_VERSION
43 
44 /* Global Variables */
45 static char		fcsm_name[] = "FCSM";
46 static void		*fcsm_state = NULL;
47 static kmutex_t		fcsm_global_mutex;
48 static uint32_t		fcsm_flag = FCSM_IDLE;
49 static dev_info_t	*fcsm_dip = NULL;
50 static fcsm_t		*fcsm_port_head = NULL;
51 static kmem_cache_t	*fcsm_job_cache = NULL;
52 static int		fcsm_num_attaching = 0;
53 static int		fcsm_num_detaching = 0;
54 static int		fcsm_detached = 0;
55 
56 static int		fcsm_max_cmd_retries = FCSM_MAX_CMD_RETRIES;
57 static int		fcsm_retry_interval = FCSM_RETRY_INTERVAL;
58 static int		fcsm_retry_ticker = FCSM_RETRY_TICKER;
59 static int		fcsm_offline_ticker = FCSM_OFFLINE_TICKER;
60 static int		fcsm_max_job_retries = FCSM_MAX_JOB_RETRIES;
61 static clock_t		fcsm_retry_ticks;
62 static clock_t		fcsm_offline_ticks;
63 
64 
65 
66 #ifdef DEBUG
67 uint32_t		fcsm_debug = (SMDL_TRACE | SMDL_IO |
68     SMDL_ERR | SMDL_INFO);
69 #endif
70 
71 
72 /* Character/Block entry points */
73 struct cb_ops	fcsm_cb_ops = {
74 	fcsm_open,	/* open */
75 	fcsm_close,	/* close */
76 	nodev,		/* strategy */
77 	nodev,		/* print */
78 	nodev,		/* dump */
79 	nodev,		/* read */
80 	nodev,		/* write */
81 	fcsm_ioctl,	/* ioctl */
82 	nodev,		/* devmap */
83 	nodev,		/* mmap */
84 	nodev,		/* segmap */
85 	nochpoll,	/* poll */
86 	ddi_prop_op,
87 	NULL,		/* streams info */
88 	D_NEW | D_MP,
89 	CB_REV,
90 	nodev,		/* aread */
91 	nodev		/* awrite */
92 };
93 
94 struct dev_ops fcsm_ops = {
95 	DEVO_REV,
96 	0,		/* refcnt */
97 	fcsm_getinfo,	/* get info */
98 	nulldev,	/* identify (obsolete) */
99 	nulldev,	/* probe (not required for self-identifying devices) */
100 	fcsm_attach,	/* attach */
101 	fcsm_detach,	/* detach */
102 	nodev,		/* reset */
103 	&fcsm_cb_ops,	/* char/block entry points structure for leaf drivers */
104 	NULL,		/* bus operations for nexus driver */
105 	NULL		/* power management */
106 };
107 
108 
109 struct modldrv modldrv = {
110 	&mod_driverops,
111 	FCSM_NAME_VERSION,
112 	&fcsm_ops
113 };
114 
115 struct modlinkage modlinkage = {
116 	MODREV_1,
117 	&modldrv,
118 	NULL
119 };
120 
121 static fc_ulp_modinfo_t fcsm_modinfo = {
122 	&fcsm_modinfo,		/* ulp_handle */
123 	FCTL_ULP_MODREV_4,	/* ulp_rev */
124 	FC_TYPE_FC_SERVICES,	/* ulp_type */
125 	fcsm_name,		/* ulp_name */
126 	0,			/* ulp_statec_mask: get all statec callbacks */
127 	fcsm_port_attach,	/* ulp_port_attach */
128 	fcsm_port_detach,	/* ulp_port_detach */
129 	fcsm_port_ioctl,	/* ulp_port_ioctl */
130 	fcsm_els_cb,		/* ulp_els_callback */
131 	fcsm_data_cb,		/* ulp_data_callback */
132 	fcsm_statec_cb		/* ulp_statec_callback */
133 };
134 
135 struct fcsm_xlat_pkt_state {
136 	uchar_t	xlat_state;
137 	int	xlat_rval;
138 } fcsm_xlat_pkt_state [] = {
139 	{ FC_PKT_SUCCESS,		FC_SUCCESS },
140 	{ FC_PKT_REMOTE_STOP,		FC_FAILURE },
141 	{ FC_PKT_LOCAL_RJT,		FC_TRANSPORT_ERROR },
142 	{ FC_PKT_NPORT_RJT,		FC_PREJECT },
143 	{ FC_PKT_FABRIC_RJT,		FC_FREJECT },
144 	{ FC_PKT_LOCAL_BSY,		FC_TRAN_BUSY },
145 	{ FC_PKT_TRAN_BSY,		FC_TRAN_BUSY },
146 	{ FC_PKT_NPORT_BSY,		FC_PBUSY },
147 	{ FC_PKT_FABRIC_BSY,		FC_FBUSY },
148 	{ FC_PKT_LS_RJT,		FC_PREJECT },
149 	{ FC_PKT_BA_RJT,		FC_PREJECT },
150 	{ FC_PKT_TIMEOUT,		FC_FAILURE },
151 	{ FC_PKT_FS_RJT,		FC_FAILURE },
152 	{ FC_PKT_TRAN_ERROR,		FC_TRANSPORT_ERROR },
153 	{ FC_PKT_FAILURE,		FC_FAILURE },
154 	{ FC_PKT_PORT_OFFLINE,		FC_OFFLINE },
155 	{ FC_PKT_ELS_IN_PROGRESS,	FC_FAILURE }
156 };
157 
158 struct fcsm_xlat_port_state {
159 	uint32_t	xlat_pstate;
160 	caddr_t		xlat_state_str;
161 } fcsm_xlat_port_state [] = {
162 	{ FC_STATE_OFFLINE,		"OFFLINE" },
163 	{ FC_STATE_ONLINE,		"ONLINE" },
164 	{ FC_STATE_LOOP,		"LOOP" },
165 	{ FC_STATE_NAMESERVICE,		"NAMESERVICE" },
166 	{ FC_STATE_RESET,		"RESET" },
167 	{ FC_STATE_RESET_REQUESTED,	"RESET_REQUESTED" },
168 	{ FC_STATE_LIP,			"LIP" },
169 	{ FC_STATE_LIP_LBIT_SET,	"LIP_LBIT_SET" },
170 	{ FC_STATE_DEVICE_CHANGE,	"DEVICE_CHANGE" },
171 	{ FC_STATE_TARGET_PORT_RESET,	"TARGET_PORT_RESET" }
172 };
173 
174 struct fcsm_xlat_topology {
175 	uint32_t	xlat_top;
176 	caddr_t		xlat_top_str;
177 } fcsm_xlat_topology [] = {
178 	{ FC_TOP_UNKNOWN,	"UNKNOWN" },
179 	{ FC_TOP_PRIVATE_LOOP,	"Private Loop" },
180 	{ FC_TOP_PUBLIC_LOOP,	"Public Loop" },
181 	{ FC_TOP_FABRIC,	"Fabric" },
182 	{ FC_TOP_PT_PT,		"Point-to-Point" },
183 	{ FC_TOP_NO_NS,		"NO_NS" }
184 };
185 
186 struct fcsm_xlat_dev_type {
187 	uint32_t	xlat_type;
188 	caddr_t		xlat_str;
189 } fcsm_xlat_dev_type [] = {
190 	{ PORT_DEVICE_NOCHANGE,		"No Change" },
191 	{ PORT_DEVICE_NEW,		"New" },
192 	{ PORT_DEVICE_OLD,		"Old" },
193 	{ PORT_DEVICE_CHANGED,		"Changed" },
194 	{ PORT_DEVICE_DELETE,		"Delete" },
195 	{ PORT_DEVICE_USER_LOGIN,	"User Login" },
196 	{ PORT_DEVICE_USER_LOGOUT,	"User Logout" },
197 	{ PORT_DEVICE_USER_CREATE,	"User Create" },
198 	{ PORT_DEVICE_USER_DELETE,	"User Delete" }
199 };
200 
201 int
202 _init(void)
203 {
204 	int		rval;
205 
206 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "_init"));
207 
208 	fcsm_retry_ticks = drv_usectohz(fcsm_retry_ticker * 1000 * 1000);
209 	fcsm_offline_ticks = drv_usectohz(fcsm_offline_ticker * 1000 * 1000);
210 
211 	if (rval = ddi_soft_state_init(&fcsm_state, sizeof (fcsm_t),
212 	    FCSM_INIT_INSTANCES)) {
213 		fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
214 		    "_init: ddi_soft_state_init failed");
215 		return (ENOMEM);
216 	}
217 
218 	mutex_init(&fcsm_global_mutex, NULL, MUTEX_DRIVER, NULL);
219 
220 	fcsm_job_cache = kmem_cache_create("fcsm_job_cache",
221 	    sizeof (fcsm_job_t), 8, fcsm_job_cache_constructor,
222 	    fcsm_job_cache_destructor, NULL, NULL, NULL, 0);
223 
224 	if (fcsm_job_cache == NULL) {
225 		mutex_destroy(&fcsm_global_mutex);
226 		ddi_soft_state_fini(&fcsm_state);
227 		return (ENOMEM);
228 	}
229 
230 	/*
231 	 * Now call fc_ulp_add to add this ULP in the transport layer
232 	 * database. This will cause 'ulp_port_attach' callback function
233 	 * to be called.
234 	 */
235 	rval = fc_ulp_add(&fcsm_modinfo);
236 	if (rval != 0) {
237 		switch (rval) {
238 		case FC_ULP_SAMEMODULE:
239 			fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
240 			    "_init: FC SAN Management module is already "
241 			    "registered with transport layer");
242 			rval = EEXIST;
243 			break;
244 
245 		case FC_ULP_SAMETYPE:
246 			fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
247 			    "_init: Another module with same type 0x%x is "
248 			    "already registered with transport layer",
249 			    fcsm_modinfo.ulp_type);
250 			rval = EEXIST;
251 			break;
252 
253 		case FC_BADULP:
254 			fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
255 			    "_init: Please upgrade this module. Current "
256 			    "version 0x%x is not the most recent version",
257 			    fcsm_modinfo.ulp_rev);
258 			rval = EIO;
259 			break;
260 		default:
261 			fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
262 			    "_init: fc_ulp_add failed with status 0x%x", rval);
263 			rval = EIO;
264 			break;
265 		}
266 		kmem_cache_destroy(fcsm_job_cache);
267 		mutex_destroy(&fcsm_global_mutex);
268 		ddi_soft_state_fini(&fcsm_state);
269 		return (rval);
270 	}
271 
272 	if ((rval = mod_install(&modlinkage)) != 0) {
273 		FCSM_DEBUG(SMDL_ERR, (CE_WARN, SM_LOG, NULL, NULL,
274 		    "_init: mod_install failed with status 0x%x", rval));
275 		(void) fc_ulp_remove(&fcsm_modinfo);
276 		kmem_cache_destroy(fcsm_job_cache);
277 		mutex_destroy(&fcsm_global_mutex);
278 		ddi_soft_state_fini(&fcsm_state);
279 		return (rval);
280 	}
281 
282 	return (rval);
283 }
284 
285 int
286 _fini(void)
287 {
288 	int	rval;
289 #ifdef	DEBUG
290 	int	status;
291 #endif /* DEBUG */
292 
293 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "_fini"));
294 
295 	/*
296 	 * don't start cleaning up until we know that the module remove
297 	 * has worked  -- if this works, then we know that each instance
298 	 * has successfully been DDI_DETACHed
299 	 */
300 	if ((rval = mod_remove(&modlinkage)) != 0) {
301 		return (rval);
302 	}
303 
304 #ifdef DEBUG
305 	status = fc_ulp_remove(&fcsm_modinfo);
306 	if (status != 0) {
307 		FCSM_DEBUG(SMDL_ERR, (CE_WARN, SM_LOG, NULL, NULL,
308 		    "_fini: fc_ulp_remove failed with status 0x%x", status));
309 	}
310 #else
311 	(void) fc_ulp_remove(&fcsm_modinfo);
312 #endif /* DEBUG */
313 
314 	fcsm_detached = 0;
315 
316 	/*
317 	 * It is possible to modunload fcsm manually, which will cause
318 	 * a bypass of all the port_detach functionality.  We may need
319 	 * to force that code path to be executed to properly clean up
320 	 * in that case.
321 	 */
322 	fcsm_force_port_detach_all();
323 
324 	kmem_cache_destroy(fcsm_job_cache);
325 	mutex_destroy(&fcsm_global_mutex);
326 	ddi_soft_state_fini(&fcsm_state);
327 
328 	return (rval);
329 }
330 
331 
332 int
333 _info(struct modinfo *modinfop)
334 {
335 	return (mod_info(&modlinkage, modinfop));
336 }
337 
338 /* ARGSUSED */
339 static int
340 fcsm_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
341 {
342 	int rval = DDI_FAILURE;
343 
344 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
345 	    "attach: cmd 0x%x", cmd));
346 
347 	switch (cmd) {
348 	case DDI_ATTACH:
349 		mutex_enter(&fcsm_global_mutex);
350 		if (fcsm_dip != NULL) {
351 			mutex_exit(&fcsm_global_mutex);
352 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
353 			    "attach: duplicate attach of fcsm!!"));
354 			break;
355 		}
356 
357 		fcsm_dip = dip;
358 
359 		/*
360 		 * The detach routine cleans up all the port instances
361 		 * i.e. it detaches all ports.
362 		 * If _fini never got called after detach, then
363 		 * perform an fc_ulp_remove() followed by fc_ulp_add()
364 		 * to ensure that port_attach callbacks are called
365 		 * again.
366 		 */
367 		if (fcsm_detached) {
368 			int status;
369 
370 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
371 			    "attach: rebinding to transport driver"));
372 
373 			mutex_exit(&fcsm_global_mutex);
374 
375 			(void) fc_ulp_remove(&fcsm_modinfo);
376 
377 			/*
378 			 * Reset the detached flag, so that ports can attach
379 			 */
380 			mutex_enter(&fcsm_global_mutex);
381 			fcsm_detached = 0;
382 			mutex_exit(&fcsm_global_mutex);
383 
384 			status = fc_ulp_add(&fcsm_modinfo);
385 
386 			if (status != 0) {
387 				/*
388 				 * ULP add failed. So set the
389 				 * detached flag again
390 				 */
391 				mutex_enter(&fcsm_global_mutex);
392 				fcsm_detached = 1;
393 				mutex_exit(&fcsm_global_mutex);
394 
395 				switch (status) {
396 				case FC_ULP_SAMEMODULE:
397 					fcsm_display(CE_WARN, SM_LOG, NULL,
398 					    NULL, "attach: FC SAN Management "
399 					    "module is already "
400 					    "registered with transport layer");
401 					break;
402 
403 				case FC_ULP_SAMETYPE:
404 					fcsm_display(CE_WARN, SM_LOG, NULL,
405 					    NULL, "attach: Another module with "
406 					    "same type 0x%x is already "
407 					    "registered with transport layer",
408 					    fcsm_modinfo.ulp_type);
409 					break;
410 
411 				case FC_BADULP:
412 					fcsm_display(CE_WARN, SM_LOG, NULL,
413 					    NULL, "attach: Please upgrade this "
414 					    "module. Current version 0x%x is "
415 					    "not the most recent version",
416 					    fcsm_modinfo.ulp_rev);
417 					break;
418 				default:
419 					fcsm_display(CE_WARN, SM_LOG, NULL,
420 					    NULL, "attach: fc_ulp_add failed "
421 					    "with status 0x%x", status);
422 					break;
423 				}
424 
425 				/* Return failure */
426 				break;
427 			}
428 
429 			mutex_enter(&fcsm_global_mutex);
430 		}
431 
432 		/* Create a minor node */
433 		if (ddi_create_minor_node(fcsm_dip, "fcsm", S_IFCHR,
434 		    NULL, DDI_PSEUDO, 0) == DDI_SUCCESS) {
435 			/* Announce presence of the device */
436 			mutex_exit(&fcsm_global_mutex);
437 			ddi_report_dev(dip);
438 			rval = DDI_SUCCESS;
439 		} else {
440 			fcsm_dip = NULL;
441 			mutex_exit(&fcsm_global_mutex);
442 			fcsm_display(CE_WARN, SM_LOG_AND_CONSOLE,
443 			    NULL, NULL, "attach: create minor node failed");
444 		}
445 		break;
446 
447 	case DDI_RESUME:
448 		rval = DDI_SUCCESS;
449 		break;
450 
451 	default:
452 		FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, NULL, NULL,
453 		    "attach: unknown cmd 0x%x dip 0x%p", cmd, dip));
454 		break;
455 	}
456 
457 	return (rval);
458 }
459 
460 /* ARGSUSED */
461 static int
462 fcsm_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
463 {
464 	int	instance;
465 	int	rval = DDI_SUCCESS;
466 
467 	instance = getminor((dev_t)arg);
468 
469 	switch (cmd) {
470 	case DDI_INFO_DEVT2INSTANCE:
471 		*result = (void *)(long)instance; /* minor number is instance */
472 		break;
473 
474 	case DDI_INFO_DEVT2DEVINFO:
475 		mutex_enter(&fcsm_global_mutex);
476 		*result = (void *)fcsm_dip;
477 		mutex_exit(&fcsm_global_mutex);
478 		break;
479 
480 	default:
481 		rval = DDI_FAILURE;
482 		break;
483 	}
484 
485 	return (rval);
486 }
487 
488 
489 /* ARGSUSED */
490 static int
491 fcsm_port_attach(opaque_t ulph, fc_ulp_port_info_t *pinfo,
492     fc_attach_cmd_t cmd, uint32_t s_id)
493 {
494 	int	instance;
495 	int	rval = FC_FAILURE;
496 
497 	instance = ddi_get_instance(pinfo->port_dip);
498 
499 	/*
500 	 * Set the attaching flag, so that fcsm_detach will fail, if
501 	 * port attach is in progress.
502 	 */
503 	mutex_enter(&fcsm_global_mutex);
504 	if (fcsm_detached) {
505 		mutex_exit(&fcsm_global_mutex);
506 
507 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
508 		    "port_attach: end. detach in progress. failing attach "
509 		    "instance 0x%x", instance));
510 		return (((cmd == FC_CMD_POWER_UP) || (cmd == FC_CMD_RESUME)) ?
511 		    FC_FAILURE_SILENT : FC_FAILURE);
512 	}
513 
514 	fcsm_num_attaching++;
515 	mutex_exit(&fcsm_global_mutex);
516 
517 	switch (cmd) {
518 	case FC_CMD_ATTACH:
519 		if (fcsm_handle_port_attach(pinfo, s_id, instance)
520 		    != DDI_SUCCESS) {
521 			ASSERT(ddi_get_soft_state(fcsm_state,
522 			    instance) == NULL);
523 			break;
524 		}
525 		rval = FC_SUCCESS;
526 		break;
527 
528 	case FC_CMD_RESUME:
529 	case FC_CMD_POWER_UP: {
530 		fcsm_t	*fcsm;
531 		char fcsm_pathname[MAXPATHLEN];
532 
533 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
534 		    "port_attach: cmd 0x%x instance 0x%x", cmd, instance));
535 
536 		/* Get the soft state structure */
537 		if ((fcsm = ddi_get_soft_state(fcsm_state, instance)) == NULL) {
538 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
539 			    "port_attach: instance 0x%x, cmd 0x%x "
540 			    "get softstate failed", instance, cmd));
541 			break;
542 		}
543 
544 		ASSERT(fcsm->sm_instance == instance);
545 
546 		/* If this instance is not attached, then return failure */
547 		mutex_enter(&fcsm->sm_mutex);
548 		if ((fcsm->sm_flags & FCSM_ATTACHED) == 0) {
549 			mutex_exit(&fcsm->sm_mutex);
550 			fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
551 			    "port_detach: port is not attached");
552 			break;
553 		}
554 		mutex_exit(&fcsm->sm_mutex);
555 
556 		if (fcsm_handle_port_resume(ulph, pinfo, cmd, s_id, fcsm) !=
557 		    DDI_SUCCESS) {
558 			break;
559 		}
560 
561 		(void) ddi_pathname(fcsm->sm_port_info.port_dip, fcsm_pathname);
562 		fcsm_display(CE_NOTE, SM_LOG, fcsm, NULL,
563 		    "attached to path %s", fcsm_pathname);
564 		rval = FC_SUCCESS;
565 		break;
566 	}
567 
568 	default:
569 		FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, NULL, NULL,
570 		    "port_attach: unknown cmd 0x%x for port 0x%x",
571 		    cmd, instance));
572 		break;
573 	}
574 
575 	mutex_enter(&fcsm_global_mutex);
576 	fcsm_num_attaching--;
577 	mutex_exit(&fcsm_global_mutex);
578 	return (rval);
579 }
580 
581 
582 static int
583 fcsm_handle_port_attach(fc_ulp_port_info_t *pinfo, uint32_t s_id, int instance)
584 {
585 	fcsm_t		*fcsm;
586 	kthread_t	*thread;
587 	char		name[32];
588 	char fcsm_pathname[MAXPATHLEN];
589 
590 	/* Allocate a soft state structure for the port */
591 	if (ddi_soft_state_zalloc(fcsm_state, instance) != DDI_SUCCESS) {
592 		fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
593 		    "port_attach: instance 0x%x, soft state alloc failed",
594 		    instance);
595 		return (DDI_FAILURE);
596 	}
597 
598 	if ((fcsm = ddi_get_soft_state(fcsm_state, instance)) == NULL) {
599 		fcsm_display(CE_WARN, SM_LOG, NULL, NULL,
600 		    "port_attach: instance 0x%x, get soft state failed",
601 		    instance);
602 		ddi_soft_state_free(fcsm_state, instance);
603 		return (DDI_FAILURE);
604 	}
605 
606 
607 	/* Initialize the mutex */
608 	mutex_init(&fcsm->sm_mutex, NULL, MUTEX_DRIVER, NULL);
609 	cv_init(&fcsm->sm_job_cv, NULL, CV_DRIVER, NULL);
610 
611 	mutex_enter(&fcsm->sm_mutex);
612 	fcsm->sm_flags		|= FCSM_ATTACHING;
613 	fcsm->sm_sid		= s_id;
614 	fcsm->sm_instance	= instance;
615 	fcsm->sm_port_state	= pinfo->port_state;
616 
617 	/*
618 	 * Make a copy of the port_information structure, since fctl
619 	 * uses a temporary structure.
620 	 */
621 	fcsm->sm_port_info	= *pinfo;	/* Structure copy !!! */
622 	mutex_exit(&fcsm->sm_mutex);
623 
624 
625 	(void) sprintf(name, "fcsm%d_cmd_cache", fcsm->sm_instance);
626 	fcsm->sm_cmd_cache = kmem_cache_create(name,
627 	    sizeof (fcsm_cmd_t) + pinfo->port_fca_pkt_size, 8,
628 	    fcsm_cmd_cache_constructor, fcsm_cmd_cache_destructor,
629 	    NULL, (void *)fcsm, NULL, 0);
630 	if (fcsm->sm_cmd_cache == NULL) {
631 		fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
632 		    "port_attach: pkt cache create failed");
633 		cv_destroy(&fcsm->sm_job_cv);
634 		mutex_destroy(&fcsm->sm_mutex);
635 		ddi_soft_state_free(fcsm_state, instance);
636 		return (DDI_FAILURE);
637 	}
638 
639 	thread = thread_create((caddr_t)NULL, 0, fcsm_job_thread,
640 	    (caddr_t)fcsm, 0, &p0, TS_RUN, v.v_maxsyspri-2);
641 	if (thread == NULL) {
642 		fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
643 		    "port_attach: job thread create failed");
644 		kmem_cache_destroy(fcsm->sm_cmd_cache);
645 		cv_destroy(&fcsm->sm_job_cv);
646 		mutex_destroy(&fcsm->sm_mutex);
647 		ddi_soft_state_free(fcsm_state, instance);
648 		return (DDI_FAILURE);
649 	}
650 
651 	fcsm->sm_thread = thread;
652 
653 	/* Add this structure to fcsm global linked list */
654 	mutex_enter(&fcsm_global_mutex);
655 	if (fcsm_port_head == NULL) {
656 		fcsm_port_head = fcsm;
657 	} else {
658 		fcsm->sm_next = fcsm_port_head;
659 		fcsm_port_head = fcsm;
660 	}
661 	mutex_exit(&fcsm_global_mutex);
662 
663 	mutex_enter(&fcsm->sm_mutex);
664 	fcsm->sm_flags &= ~FCSM_ATTACHING;
665 	fcsm->sm_flags |= FCSM_ATTACHED;
666 	fcsm->sm_port_top = pinfo->port_flags;
667 	fcsm->sm_port_state = pinfo->port_state;
668 	if (pinfo->port_acc_attr == NULL) {
669 		/*
670 		 * The corresponding FCA doesn't support DMA at all
671 		 */
672 		fcsm->sm_flags |= FCSM_USING_NODMA_FCA;
673 	}
674 	mutex_exit(&fcsm->sm_mutex);
675 
676 	(void) ddi_pathname(fcsm->sm_port_info.port_dip, fcsm_pathname);
677 	fcsm_display(CE_NOTE, SM_LOG, fcsm, NULL,
678 	    "attached to path %s", fcsm_pathname);
679 
680 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
681 	    "port_attach: state <%s>(0x%x) topology <%s>(0x%x)",
682 	    fcsm_port_state_to_str(FC_PORT_STATE_MASK(pinfo->port_state)),
683 	    pinfo->port_state,
684 	    fcsm_topology_to_str(pinfo->port_flags), pinfo->port_flags));
685 
686 	return (DDI_SUCCESS);
687 }
688 
689 static int
690 fcsm_handle_port_resume(opaque_t ulph, fc_ulp_port_info_t *pinfo,
691     fc_attach_cmd_t cmd, uint32_t s_id, fcsm_t *fcsm)
692 {
693 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
694 	    "port_resume: cmd 0x%x", cmd));
695 
696 	mutex_enter(&fcsm->sm_mutex);
697 
698 	switch (cmd) {
699 	case FC_CMD_RESUME:
700 		ASSERT(!(fcsm->sm_flags & FCSM_POWER_DOWN));
701 		fcsm->sm_flags &= ~FCSM_SUSPENDED;
702 		break;
703 
704 	case FC_CMD_POWER_UP:
705 		/* If port is suspended, then no need to resume */
706 		fcsm->sm_flags &= ~FCSM_POWER_DOWN;
707 		if (fcsm->sm_flags & FCSM_SUSPENDED) {
708 			mutex_exit(&fcsm->sm_mutex);
709 			return (DDI_SUCCESS);
710 		}
711 		break;
712 	default:
713 		mutex_exit(&fcsm->sm_mutex);
714 		return (DDI_FAILURE);
715 	}
716 
717 	fcsm->sm_sid = s_id;
718 
719 	/*
720 	 * Make a copy of the new port_information structure
721 	 */
722 	fcsm->sm_port_info	= *pinfo;	/* Structure copy !!! */
723 	mutex_exit(&fcsm->sm_mutex);
724 
725 	fcsm_resume_port(fcsm);
726 
727 	/*
728 	 * Invoke state change processing.
729 	 * This will ensure that
730 	 *    - offline timer is started if new port state changed to offline.
731 	 *    - MGMT_SERVER_LOGIN flag is reset.
732 	 *    - Port topology is updated.
733 	 */
734 	fcsm_statec_cb(ulph, (opaque_t)pinfo->port_handle, pinfo->port_state,
735 	    pinfo->port_flags, NULL, 0, s_id);
736 
737 	return (DDI_SUCCESS);
738 }
739 
740 
741 /* ARGSUSED */
742 static int
743 fcsm_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
744 {
745 	int	rval = DDI_SUCCESS;
746 
747 	switch (cmd) {
748 	case DDI_DETACH: {
749 		fcsm_t	*fcsm;
750 
751 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
752 		    "detach: start. cmd <DETACH>", cmd));
753 
754 		mutex_enter(&fcsm_global_mutex);
755 
756 		/*
757 		 * If port attach/detach in progress, then wait for 5 seconds
758 		 * for them to complete.
759 		 */
760 		if (fcsm_num_attaching || fcsm_num_detaching) {
761 			int count;
762 
763 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
764 			    "detach: wait for port attach/detach to complete"));
765 
766 			count = 0;
767 			while ((count++ <= 30) &&
768 			    (fcsm_num_attaching || fcsm_num_detaching)) {
769 				mutex_exit(&fcsm_global_mutex);
770 				delay(drv_usectohz(1000000));
771 				mutex_enter(&fcsm_global_mutex);
772 			}
773 
774 			/* Port attach/detach still in prog, so fail detach */
775 			if (fcsm_num_attaching || fcsm_num_detaching) {
776 				mutex_exit(&fcsm_global_mutex);
777 				FCSM_DEBUG(SMDL_ERR, (CE_WARN, SM_LOG, NULL,
778 				    NULL, "detach: Failing detach. port "
779 				    "attach/detach in progress"));
780 				rval = DDI_FAILURE;
781 				break;
782 			}
783 		}
784 
785 		if (fcsm_port_head == NULL) {
786 			/* Not much do, Succeed to detach. */
787 			ddi_remove_minor_node(fcsm_dip, NULL);
788 			fcsm_dip = NULL;
789 			fcsm_detached = 0;
790 			mutex_exit(&fcsm_global_mutex);
791 			break;
792 		}
793 
794 		/*
795 		 * Check to see, if any ports are active.
796 		 * If not, then set the DETACHING flag to indicate
797 		 * that they are being detached.
798 		 */
799 		fcsm = fcsm_port_head;
800 		while (fcsm != NULL) {
801 
802 			mutex_enter(&fcsm->sm_mutex);
803 			if (!(fcsm->sm_flags & FCSM_ATTACHED) ||
804 			    fcsm->sm_ncmds || fcsm->sm_cb_count) {
805 				/* port is busy. We can't detach */
806 				mutex_exit(&fcsm->sm_mutex);
807 				break;
808 			}
809 
810 			fcsm->sm_flags |= FCSM_DETACHING;
811 			mutex_exit(&fcsm->sm_mutex);
812 
813 			fcsm = fcsm->sm_next;
814 		}
815 
816 		/*
817 		 * If all ports could not be marked for detaching,
818 		 * then clear the flags and fail the detach.
819 		 * Also if a port attach is currently in progress
820 		 * then fail the detach.
821 		 */
822 		if (fcsm != NULL || fcsm_num_attaching || fcsm_num_detaching) {
823 			/*
824 			 * Some ports were busy, so can't detach.
825 			 * Clear the DETACHING flag and return failure
826 			 */
827 			fcsm = fcsm_port_head;
828 			while (fcsm != NULL) {
829 				mutex_enter(&fcsm->sm_mutex);
830 				if (fcsm->sm_flags & FCSM_DETACHING) {
831 					fcsm->sm_flags &= ~FCSM_DETACHING;
832 				}
833 				mutex_exit(&fcsm->sm_mutex);
834 
835 				fcsm = fcsm->sm_next;
836 			}
837 			mutex_exit(&fcsm_global_mutex);
838 			return (DDI_FAILURE);
839 		} else {
840 			fcsm_detached = 1;
841 			/*
842 			 * Mark all the detaching ports as detached, as we
843 			 * will be detaching them
844 			 */
845 			fcsm = fcsm_port_head;
846 			while (fcsm != NULL) {
847 				mutex_enter(&fcsm->sm_mutex);
848 				fcsm->sm_flags &= ~FCSM_DETACHING;
849 				fcsm->sm_flags |= FCSM_DETACHED;
850 				mutex_exit(&fcsm->sm_mutex);
851 
852 				fcsm = fcsm->sm_next;
853 			}
854 		}
855 		mutex_exit(&fcsm_global_mutex);
856 
857 
858 		/*
859 		 * Go ahead and detach the ports
860 		 */
861 		mutex_enter(&fcsm_global_mutex);
862 		while (fcsm_port_head != NULL) {
863 			fcsm = fcsm_port_head;
864 			mutex_exit(&fcsm_global_mutex);
865 
866 			/*
867 			 * Call fcsm_cleanup_port(). This cleansup and
868 			 * removes the fcsm structure from global linked list
869 			 */
870 			fcsm_cleanup_port(fcsm);
871 
872 			/*
873 			 * Soft state cleanup done.
874 			 * Remember that fcsm struct doesn't exist anymore.
875 			 */
876 
877 			mutex_enter(&fcsm_global_mutex);
878 		}
879 
880 		ddi_remove_minor_node(fcsm_dip, NULL);
881 		fcsm_dip = NULL;
882 		mutex_exit(&fcsm_global_mutex);
883 		break;
884 	}
885 
886 	case DDI_SUSPEND:
887 		rval = DDI_SUCCESS;
888 		break;
889 
890 	default:
891 		FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, NULL, NULL,
892 		    "detach: unknown cmd 0x%x", cmd));
893 		rval = DDI_FAILURE;
894 		break;
895 	}
896 
897 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
898 	    "detach: end. cmd 0x%x, rval 0x%x", cmd, rval));
899 
900 	return (rval);
901 }
902 
903 
904 /* ARGSUSED */
905 static void
906 fcsm_force_port_detach_all(void)
907 {
908 	fcsm_t	*fcsm;
909 
910 	fcsm = fcsm_port_head;
911 
912 	while (fcsm) {
913 		fcsm_cleanup_port(fcsm);
914 		/*
915 		 * fcsm_cleanup_port will remove the current fcsm structure
916 		 * from the list, which will cause fcsm_port_head to point
917 		 * to what would have been the next structure on the list.
918 		 */
919 		fcsm = fcsm_port_head;
920 	}
921 }
922 
923 
924 /* ARGSUSED */
925 static int
926 fcsm_port_detach(opaque_t ulph, fc_ulp_port_info_t *pinfo, fc_detach_cmd_t cmd)
927 {
928 	int	instance;
929 	int	rval = FC_FAILURE;
930 	fcsm_t	*fcsm;
931 
932 	instance = ddi_get_instance(pinfo->port_dip);
933 
934 	mutex_enter(&fcsm_global_mutex);
935 	if (fcsm_detached) {
936 		mutex_exit(&fcsm_global_mutex);
937 
938 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
939 		    "port_detach: end. instance 0x%x, fcsm is detached",
940 		    instance));
941 		return (FC_SUCCESS);
942 	}
943 	fcsm_num_detaching++;	/* Set the flag */
944 	mutex_exit(&fcsm_global_mutex);
945 
946 	/* Get the soft state structure */
947 	if ((fcsm = ddi_get_soft_state(fcsm_state, instance)) == NULL) {
948 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
949 		    "port_detach: instance 0x%x, cmd 0x%x get softstate failed",
950 		    instance, cmd));
951 		mutex_enter(&fcsm_global_mutex);
952 		fcsm_num_detaching--;
953 		mutex_exit(&fcsm_global_mutex);
954 		return (rval);
955 	}
956 
957 	ASSERT(fcsm->sm_instance == instance);
958 
959 	/* If this instance is not attached, then fail the detach */
960 	mutex_enter(&fcsm->sm_mutex);
961 	if ((fcsm->sm_flags & FCSM_ATTACHED) == 0) {
962 		mutex_exit(&fcsm->sm_mutex);
963 		fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
964 		    "port_detach: port is not attached");
965 		mutex_enter(&fcsm_global_mutex);
966 		fcsm_num_detaching--;
967 		mutex_exit(&fcsm_global_mutex);
968 		return (rval);
969 	}
970 	mutex_exit(&fcsm->sm_mutex);
971 
972 	/*
973 	 * If fcsm has been detached, then all instance has already been
974 	 * detached or are being detached. So succeed this detach.
975 	 */
976 
977 	switch (cmd) {
978 	case FC_CMD_DETACH:
979 	case FC_CMD_SUSPEND:
980 	case FC_CMD_POWER_DOWN:
981 		break;
982 
983 	default:
984 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
985 		    "port_detach: port unknown cmd 0x%x", cmd));
986 		mutex_enter(&fcsm_global_mutex);
987 		fcsm_num_detaching--;
988 		mutex_exit(&fcsm_global_mutex);
989 		return (rval);
990 	};
991 
992 	if (fcsm_handle_port_detach(pinfo, fcsm, cmd) == DDI_SUCCESS) {
993 		rval = FC_SUCCESS;
994 	}
995 
996 	mutex_enter(&fcsm_global_mutex);
997 	fcsm_num_detaching--;
998 	mutex_exit(&fcsm_global_mutex);
999 
1000 	/* If it was a detach, then fcsm state structure no longer exists */
1001 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1002 	    "port_detach: end. cmd 0x%x rval 0x%x", cmd, rval));
1003 	return (rval);
1004 }
1005 
1006 
1007 static int
1008 fcsm_handle_port_detach(fc_ulp_port_info_t *pinfo, fcsm_t *fcsm,
1009     fc_detach_cmd_t cmd)
1010 {
1011 	uint32_t	flag;
1012 	int		count;
1013 #ifdef DEBUG
1014 	char		pathname[MAXPATHLEN];
1015 #endif /* DEBUG */
1016 
1017 	/*
1018 	 * If port is already powered down OR suspended and there is nothing
1019 	 * else to do then just return.
1020 	 * Otherwise, set the flag, so that no more new activity will be
1021 	 * initiated on this port.
1022 	 */
1023 	mutex_enter(&fcsm->sm_mutex);
1024 
1025 	switch (cmd) {
1026 	case FC_CMD_DETACH:
1027 		flag = FCSM_DETACHING;
1028 		break;
1029 
1030 	case FC_CMD_SUSPEND:
1031 	case FC_CMD_POWER_DOWN:
1032 		((cmd == FC_CMD_SUSPEND) ? (flag = FCSM_SUSPENDED) :
1033 		    (flag = FCSM_POWER_DOWN));
1034 		if (fcsm->sm_flags &
1035 		    (FCSM_POWER_DOWN | FCSM_SUSPENDED)) {
1036 			fcsm->sm_flags |= flag;
1037 			mutex_exit(&fcsm->sm_mutex);
1038 			return (DDI_SUCCESS);
1039 		}
1040 		break;
1041 
1042 	default:
1043 		mutex_exit(&fcsm->sm_mutex);
1044 		return (DDI_FAILURE);
1045 	};
1046 
1047 	fcsm->sm_flags |= flag;
1048 
1049 	/*
1050 	 * If some commands are pending OR callback in progress, then
1051 	 * wait for some finite amount of time for their completion.
1052 	 * TODO: add more checks here to check for cmd timeout, offline
1053 	 * timeout and other (??) threads.
1054 	 */
1055 	count = 0;
1056 	while ((count++ <= 30) && (fcsm->sm_ncmds || fcsm->sm_cb_count)) {
1057 		mutex_exit(&fcsm->sm_mutex);
1058 		delay(drv_usectohz(1000000));
1059 		mutex_enter(&fcsm->sm_mutex);
1060 	}
1061 	if (fcsm->sm_ncmds || fcsm->sm_cb_count) {
1062 		fcsm->sm_flags &= ~flag;
1063 		mutex_exit(&fcsm->sm_mutex);
1064 		fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
1065 		    "port_detach: Failing suspend, port is busy");
1066 		return (DDI_FAILURE);
1067 	}
1068 	if (flag == FCSM_DETACHING) {
1069 		fcsm->sm_flags &= ~FCSM_DETACHING;
1070 		fcsm->sm_flags |= FCSM_DETACHED;
1071 	}
1072 
1073 	mutex_exit(&fcsm->sm_mutex);
1074 
1075 	FCSM_DEBUG(SMDL_INFO, (CE_CONT, SM_LOG, fcsm, NULL,
1076 	    "port_detach: cmd 0x%x pathname <%s>",
1077 	    cmd, ddi_pathname(pinfo->port_dip, pathname)));
1078 
1079 	if (cmd == FC_CMD_DETACH) {
1080 		fcsm_cleanup_port(fcsm);
1081 		/*
1082 		 * Soft state cleanup done.
1083 		 * Always remember that fcsm struct doesn't exist anymore.
1084 		 */
1085 	} else {
1086 		fcsm_suspend_port(fcsm);
1087 	}
1088 
1089 	return (DDI_SUCCESS);
1090 }
1091 
1092 static void
1093 fcsm_suspend_port(fcsm_t *fcsm)
1094 {
1095 	mutex_enter(&fcsm->sm_mutex);
1096 
1097 	if (fcsm->sm_offline_tid != NULL) {
1098 		timeout_id_t	tid;
1099 
1100 		tid = fcsm->sm_offline_tid;
1101 		fcsm->sm_offline_tid = (timeout_id_t)NULL;
1102 		mutex_exit(&fcsm->sm_mutex);
1103 		(void) untimeout(tid);
1104 		mutex_enter(&fcsm->sm_mutex);
1105 		fcsm->sm_flags |= FCSM_RESTORE_OFFLINE_TIMEOUT;
1106 	}
1107 
1108 	if (fcsm->sm_retry_tid != NULL) {
1109 		timeout_id_t	tid;
1110 
1111 		tid = fcsm->sm_retry_tid;
1112 		fcsm->sm_retry_tid = (timeout_id_t)NULL;
1113 		mutex_exit(&fcsm->sm_mutex);
1114 		(void) untimeout(tid);
1115 		mutex_enter(&fcsm->sm_mutex);
1116 		fcsm->sm_flags |= FCSM_RESTORE_RETRY_TIMEOUT;
1117 	}
1118 
1119 	mutex_exit(&fcsm->sm_mutex);
1120 }
1121 
1122 static void
1123 fcsm_resume_port(fcsm_t *fcsm)
1124 {
1125 	mutex_enter(&fcsm->sm_mutex);
1126 
1127 	if (fcsm->sm_flags & FCSM_RESTORE_OFFLINE_TIMEOUT) {
1128 		fcsm->sm_flags &= ~FCSM_RESTORE_OFFLINE_TIMEOUT;
1129 
1130 		/*
1131 		 * If port if offline, link is not marked down and offline
1132 		 * timer is not already running, then restart offline timer.
1133 		 */
1134 		if (!(fcsm->sm_flags & FCSM_LINK_DOWN) &&
1135 		    fcsm->sm_offline_tid == NULL &&
1136 		    (fcsm->sm_flags & FCSM_PORT_OFFLINE)) {
1137 			fcsm->sm_offline_tid = timeout(fcsm_offline_timeout,
1138 			    (caddr_t)fcsm, fcsm_offline_ticks);
1139 		}
1140 	}
1141 
1142 	if (fcsm->sm_flags & FCSM_RESTORE_RETRY_TIMEOUT) {
1143 		fcsm->sm_flags &= ~FCSM_RESTORE_RETRY_TIMEOUT;
1144 
1145 		/*
1146 		 * If retry queue is not suspended and some cmds are waiting
1147 		 * to be retried, then restart the retry timer
1148 		 */
1149 		if (fcsm->sm_retry_head && fcsm->sm_retry_tid == NULL) {
1150 			fcsm->sm_retry_tid = timeout(fcsm_retry_timeout,
1151 			    (caddr_t)fcsm, fcsm_retry_ticks);
1152 		}
1153 	}
1154 	mutex_exit(&fcsm->sm_mutex);
1155 }
1156 
1157 static void
1158 fcsm_cleanup_port(fcsm_t *fcsm)
1159 {
1160 	fcsm_t		*curr, *prev;
1161 	int		status;
1162 	fcsm_job_t	*job;
1163 
1164 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1165 	    "fcsm_cleanup_port: entered"));
1166 
1167 	/*
1168 	 * Kill the job thread
1169 	 */
1170 	job = fcsm_alloc_job(KM_SLEEP);
1171 	ASSERT(job != NULL);
1172 	fcsm_init_job(job, fcsm->sm_instance, FCSM_JOB_THREAD_SHUTDOWN,
1173 	    FCSM_JOBFLAG_SYNC, NULL, NULL, NULL, NULL);
1174 
1175 	status = fcsm_process_job(job, 0);
1176 	ASSERT(status == FC_SUCCESS);
1177 
1178 	ASSERT(job->job_result == FC_SUCCESS);
1179 	fcsm_dealloc_job(job);
1180 
1181 	/*
1182 	 * We got here after ensuring the no commands are pending or active.
1183 	 * Therefore retry timeout thread should NOT be running.
1184 	 * Kill the offline timeout thread if currently running.
1185 	 */
1186 	mutex_enter(&fcsm->sm_mutex);
1187 
1188 	ASSERT(fcsm->sm_retry_tid == NULL);
1189 
1190 	if (fcsm->sm_offline_tid != NULL) {
1191 		timeout_id_t	tid;
1192 
1193 		tid = fcsm->sm_offline_tid;
1194 		fcsm->sm_offline_tid = (timeout_id_t)NULL;
1195 		mutex_exit(&fcsm->sm_mutex);
1196 		(void) untimeout(tid);
1197 	} else {
1198 		mutex_exit(&fcsm->sm_mutex);
1199 	}
1200 
1201 	/* Remove from the fcsm state structure from global linked list */
1202 	mutex_enter(&fcsm_global_mutex);
1203 	curr = fcsm_port_head;
1204 	prev = NULL;
1205 	while (curr != fcsm && curr != NULL) {
1206 		prev = curr;
1207 		curr = curr->sm_next;
1208 	}
1209 	ASSERT(curr != NULL);
1210 
1211 	if (prev == NULL) {
1212 		fcsm_port_head = curr->sm_next;
1213 	} else {
1214 		prev->sm_next = curr->sm_next;
1215 	}
1216 	mutex_exit(&fcsm_global_mutex);
1217 
1218 	if (fcsm->sm_cmd_cache != NULL) {
1219 		kmem_cache_destroy(fcsm->sm_cmd_cache);
1220 	}
1221 	cv_destroy(&fcsm->sm_job_cv);
1222 	mutex_destroy(&fcsm->sm_mutex);
1223 
1224 	/* Free the fcsm state structure */
1225 	ddi_soft_state_free(fcsm_state, fcsm->sm_instance);
1226 }
1227 
1228 
1229 /* ARGSUSED */
1230 static void
1231 fcsm_statec_cb(opaque_t ulph, opaque_t port_handle, uint32_t port_state,
1232     uint32_t port_top, fc_portmap_t *devlist, uint32_t dev_cnt,
1233     uint32_t port_sid)
1234 {
1235 	fcsm_t		*fcsm;
1236 	timeout_id_t	offline_tid, retry_tid;
1237 
1238 	mutex_enter(&fcsm_global_mutex);
1239 	if (fcsm_detached) {
1240 		mutex_exit(&fcsm_global_mutex);
1241 		return;
1242 	}
1243 
1244 	fcsm = ddi_get_soft_state(fcsm_state,
1245 	    fc_ulp_get_port_instance(port_handle));
1246 	if (fcsm == NULL) {
1247 		mutex_exit(&fcsm_global_mutex);
1248 		FCSM_DEBUG(SMDL_TRACE, (CE_NOTE, SM_LOG, NULL, NULL,
1249 		    "statec_cb: instance 0x%x not found",
1250 		    fc_ulp_get_port_instance(port_handle)));
1251 		return;
1252 	}
1253 	mutex_enter(&fcsm->sm_mutex);
1254 	ASSERT(fcsm->sm_instance == fc_ulp_get_port_instance(port_handle));
1255 	if ((fcsm->sm_flags & FCSM_ATTACHED) == 0) {
1256 		mutex_exit(&fcsm->sm_mutex);
1257 		mutex_exit(&fcsm_global_mutex);
1258 		FCSM_DEBUG(SMDL_TRACE, (CE_NOTE, SM_LOG, fcsm, NULL,
1259 		    "statec_cb: port not attached"));
1260 		return;
1261 	}
1262 
1263 	ASSERT(fcsm->sm_cb_count >= 0);
1264 
1265 	fcsm->sm_cb_count++;
1266 	mutex_exit(&fcsm->sm_mutex);
1267 	mutex_exit(&fcsm_global_mutex);
1268 
1269 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1270 	    "statec_cb: state <%s>(0x%x) topology <%s>(0x%x) dev_cnt %d",
1271 	    fcsm_port_state_to_str(FC_PORT_STATE_MASK(port_state)), port_state,
1272 	    fcsm_topology_to_str(port_top), port_top, dev_cnt));
1273 
1274 	fcsm_disp_devlist(fcsm, devlist, dev_cnt);
1275 
1276 	mutex_enter(&fcsm->sm_mutex);
1277 
1278 	/*
1279 	 * Reset the Mgmt server Login flag, so that login is performed again.
1280 	 */
1281 	fcsm->sm_flags &= ~FCSM_MGMT_SERVER_LOGGED_IN;
1282 
1283 	fcsm->sm_sid = port_sid;
1284 	fcsm->sm_port_top = port_top;
1285 	fcsm->sm_port_state = port_state;
1286 
1287 	switch (port_state) {
1288 	case FC_STATE_OFFLINE:
1289 	case FC_STATE_RESET:
1290 	case FC_STATE_RESET_REQUESTED:
1291 		fcsm->sm_flags |= FCSM_PORT_OFFLINE;
1292 		break;
1293 
1294 	case FC_STATE_ONLINE:
1295 	case FC_STATE_LOOP:
1296 	case FC_STATE_LIP:
1297 	case FC_STATE_LIP_LBIT_SET:
1298 		fcsm->sm_flags &= ~FCSM_PORT_OFFLINE;
1299 		fcsm->sm_flags &= ~FCSM_LINK_DOWN;
1300 		break;
1301 
1302 	case FC_STATE_NAMESERVICE:
1303 	case FC_STATE_DEVICE_CHANGE:
1304 	case FC_STATE_TARGET_PORT_RESET:
1305 	default:
1306 		/* Do nothing */
1307 		break;
1308 	}
1309 
1310 	offline_tid = retry_tid = NULL;
1311 	if (fcsm->sm_flags & FCSM_PORT_OFFLINE) {
1312 		/*
1313 		 * Port is offline.
1314 		 * Suspend cmd processing and start offline timeout thread.
1315 		 */
1316 		if (fcsm->sm_offline_tid == NULL) {
1317 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1318 			    "statec_cb: schedule offline timeout thread"));
1319 			fcsm->sm_flags |= FCSM_CMD_RETRY_Q_SUSPENDED;
1320 			/* Stop the cmd retry thread */
1321 			retry_tid = fcsm->sm_retry_tid;
1322 			fcsm->sm_retry_tid = (timeout_id_t)NULL;
1323 
1324 			fcsm->sm_offline_tid = timeout(fcsm_offline_timeout,
1325 			    (caddr_t)fcsm, fcsm_offline_ticks);
1326 		}
1327 
1328 	} else {
1329 		/*
1330 		 * Port is online.
1331 		 * Cancel offline timeout thread and resume command processing.
1332 		 */
1333 		if (fcsm->sm_offline_tid) {
1334 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1335 			    "statec_cb: cancel offline timeout thread"));
1336 			offline_tid = fcsm->sm_offline_tid;
1337 			fcsm->sm_offline_tid = (timeout_id_t)NULL;
1338 		}
1339 
1340 		fcsm->sm_flags &= ~FCSM_CMD_RETRY_Q_SUSPENDED;
1341 		/* Start retry thread if needed */
1342 		if (fcsm->sm_retry_head && fcsm->sm_retry_tid == NULL) {
1343 			fcsm->sm_retry_tid = timeout(fcsm_retry_timeout,
1344 			    (caddr_t)fcsm, fcsm_retry_ticks);
1345 		}
1346 	}
1347 
1348 	mutex_exit(&fcsm->sm_mutex);
1349 
1350 	if (offline_tid != NULL) {
1351 		(void) untimeout(offline_tid);
1352 	}
1353 
1354 	if (retry_tid != NULL) {
1355 		(void) untimeout(retry_tid);
1356 	}
1357 
1358 	mutex_enter(&fcsm->sm_mutex);
1359 	fcsm->sm_cb_count--;
1360 	ASSERT(fcsm->sm_cb_count >= 0);
1361 	mutex_exit(&fcsm->sm_mutex);
1362 }
1363 
1364 
1365 static void
1366 fcsm_offline_timeout(void *handle)
1367 {
1368 	fcsm_t	*fcsm = (fcsm_t *)handle;
1369 
1370 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1371 	    "offline_timeout"));
1372 
1373 	mutex_enter(&fcsm->sm_mutex);
1374 	if (fcsm->sm_flags & FCSM_PORT_OFFLINE) {
1375 		fcsm->sm_flags |= FCSM_LINK_DOWN;
1376 	}
1377 	fcsm->sm_offline_tid = (timeout_id_t)NULL;
1378 	fcsm->sm_flags &= ~FCSM_CMD_RETRY_Q_SUSPENDED;
1379 
1380 	/* Start the retry thread if needed */
1381 	if (fcsm->sm_retry_head && fcsm->sm_retry_tid == NULL) {
1382 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1383 		    "offline_timeout: reschedule cmd retry thread"));
1384 		ASSERT(fcsm->sm_retry_tid == NULL);
1385 		fcsm->sm_retry_tid = timeout(fcsm_retry_timeout,
1386 		    (caddr_t)fcsm, fcsm_retry_ticks);
1387 	}
1388 	mutex_exit(&fcsm->sm_mutex);
1389 }
1390 
1391 /* ARGSUSED */
1392 static int
1393 fcsm_els_cb(opaque_t ulph, opaque_t port_handle, fc_unsol_buf_t *buf,
1394     uint32_t claimed)
1395 {
1396 	return (FC_UNCLAIMED);
1397 }
1398 
1399 
1400 /* ARGSUSED */
1401 static int
1402 fcsm_data_cb(opaque_t ulph, opaque_t port_handle, fc_unsol_buf_t *buf,
1403     uint32_t claimed)
1404 {
1405 	return (FC_UNCLAIMED);
1406 }
1407 
1408 
1409 /* ARGSUSED */
1410 static int
1411 fcsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
1412     int *rval_p)
1413 {
1414 	int retval = 0;
1415 
1416 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "ioctl: start"));
1417 
1418 	mutex_enter(&fcsm_global_mutex);
1419 	if (!(fcsm_flag & FCSM_OPEN)) {
1420 		mutex_exit(&fcsm_global_mutex);
1421 		return (ENXIO);
1422 	}
1423 	mutex_exit(&fcsm_global_mutex);
1424 
1425 	/* Allow only root to talk */
1426 	if (drv_priv(credp)) {
1427 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1428 		    "ioctl: end (disallowing underprivileged user)"));
1429 		return (EPERM);
1430 	}
1431 
1432 	switch (cmd) {
1433 
1434 	case FCSMIO_CMD: {
1435 		fcio_t	fcio;
1436 		int	status;
1437 #ifdef	_MULTI_DATAMODEL
1438 		switch (ddi_model_convert_from(mode & FMODELS)) {
1439 		case DDI_MODEL_ILP32: {
1440 			struct fcio32 fcio32;
1441 
1442 			if (status = ddi_copyin((void *)arg, (void *)&fcio32,
1443 			    sizeof (struct fcio32), mode)) {
1444 				retval = EFAULT;
1445 				break;
1446 			}
1447 			fcio.fcio_xfer = fcio32.fcio_xfer;
1448 			fcio.fcio_cmd = fcio32.fcio_cmd;
1449 			fcio.fcio_flags = fcio32.fcio_flags;
1450 			fcio.fcio_cmd_flags = fcio32.fcio_cmd_flags;
1451 			fcio.fcio_ilen = (size_t)fcio32.fcio_ilen;
1452 			fcio.fcio_ibuf = (caddr_t)(long)fcio32.fcio_ibuf;
1453 			fcio.fcio_olen = (size_t)fcio32.fcio_olen;
1454 			fcio.fcio_obuf = (caddr_t)(long)fcio32.fcio_obuf;
1455 			fcio.fcio_alen = (size_t)fcio32.fcio_alen;
1456 			fcio.fcio_abuf = (caddr_t)(long)fcio32.fcio_abuf;
1457 			fcio.fcio_errno = fcio32.fcio_errno;
1458 			break;
1459 		}
1460 
1461 		case DDI_MODEL_NONE:
1462 			if (status = ddi_copyin((void *)arg, (void *)&fcio,
1463 			    sizeof (fcio_t), mode)) {
1464 				retval = EFAULT;
1465 			}
1466 			break;
1467 		}
1468 #else	/* _MULTI_DATAMODEL */
1469 		if (status = ddi_copyin((void *)arg, (void *)&fcio,
1470 		    sizeof (fcio_t), mode)) {
1471 			retval = EFAULT;
1472 			break;
1473 		}
1474 #endif	/* _MULTI_DATAMODEL */
1475 		if (!status) {
1476 			retval = fcsm_fciocmd(arg, mode, credp, &fcio);
1477 		}
1478 		break;
1479 	}
1480 
1481 	default:
1482 		retval = ENOTTY;
1483 		break;
1484 	}
1485 
1486 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "ioctl: end"));
1487 	return (retval);
1488 }
1489 
1490 /* ARGSUSED */
1491 static int
1492 fcsm_port_ioctl(opaque_t ulph, opaque_t port_handle, dev_t dev, int cmd,
1493     intptr_t arg, int mode, cred_t *credp, int *rval, uint32_t claimed)
1494 {
1495 	return (FC_UNCLAIMED);
1496 }
1497 
1498 
1499 /* ARGSUSED */
1500 static int
1501 fcsm_fciocmd(intptr_t arg, int mode, cred_t *credp, fcio_t *fcio)
1502 {
1503 	int  retval = 0;
1504 
1505 	switch (fcio->fcio_cmd) {
1506 	case  FCSMIO_CT_CMD: {
1507 		fcsm_t		*fcsm;
1508 		caddr_t		user_ibuf, user_obuf;
1509 		caddr_t		req_iu, rsp_iu, abuf;
1510 		int		status, instance, count;
1511 
1512 		if ((fcio->fcio_xfer != FCIO_XFER_RW) ||
1513 		    (fcio->fcio_ilen == 0) || (fcio->fcio_ibuf == 0) ||
1514 		    (fcio->fcio_olen == 0) || (fcio->fcio_obuf == 0) ||
1515 		    (fcio->fcio_alen == 0) || (fcio->fcio_abuf == 0) ||
1516 		    (fcio->fcio_flags != 0) || (fcio->fcio_cmd_flags != 0) ||
1517 		    (fcio->fcio_ilen > FCSM_MAX_CT_SIZE) ||
1518 		    (fcio->fcio_olen > FCSM_MAX_CT_SIZE) ||
1519 		    (fcio->fcio_alen > MAXPATHLEN)) {
1520 			retval = EINVAL;
1521 			break;
1522 		}
1523 
1524 		/*
1525 		 * Get the destination port for which this ioctl
1526 		 * is targeted. The abuf will have the fp_minor
1527 		 * number.
1528 		 */
1529 		abuf = kmem_zalloc(fcio->fcio_alen, KM_SLEEP);
1530 		ASSERT(abuf != NULL);
1531 		if (ddi_copyin(fcio->fcio_abuf, abuf, fcio->fcio_alen, mode)) {
1532 			retval = EFAULT;
1533 			kmem_free(abuf, fcio->fcio_alen);
1534 			break;
1535 		}
1536 
1537 		instance = *((int *)abuf);
1538 		kmem_free(abuf, fcio->fcio_alen);
1539 
1540 		if (instance < 0) {
1541 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
1542 			    "fciocmd: instance 0x%x, invalid instance",
1543 			    instance));
1544 			retval = ENXIO;
1545 			break;
1546 		}
1547 
1548 		/*
1549 		 * We confirmed that path corresponds to our port driver
1550 		 * and a valid instance.
1551 		 * If this port instance is not yet attached, then wait
1552 		 * for a finite time for attach to complete
1553 		 */
1554 		fcsm = ddi_get_soft_state(fcsm_state, instance);
1555 		count = 0;
1556 		while (count++ <= 30) {
1557 			if (fcsm != NULL) {
1558 				mutex_enter(&fcsm->sm_mutex);
1559 				if (fcsm->sm_flags & FCSM_ATTACHED) {
1560 					mutex_exit(&fcsm->sm_mutex);
1561 					break;
1562 				}
1563 				mutex_exit(&fcsm->sm_mutex);
1564 			}
1565 			if (count == 1) {
1566 				FCSM_DEBUG(SMDL_TRACE,
1567 				    (CE_WARN, SM_LOG, NULL, NULL,
1568 				    "fciocmd: instance 0x%x, "
1569 				    "wait for port attach", instance));
1570 			}
1571 			delay(drv_usectohz(1000000));
1572 			fcsm = ddi_get_soft_state(fcsm_state, instance);
1573 		}
1574 		if (count > 30) {
1575 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, NULL, NULL,
1576 			    "fciocmd: instance 0x%x, port not attached",
1577 			    instance));
1578 			retval = ENXIO;
1579 			break;
1580 		}
1581 
1582 		req_iu = kmem_zalloc(fcio->fcio_ilen, KM_SLEEP);
1583 		rsp_iu = kmem_zalloc(fcio->fcio_olen, KM_SLEEP);
1584 		ASSERT((req_iu != NULL) && (rsp_iu != NULL));
1585 
1586 		if (ddi_copyin(fcio->fcio_ibuf, req_iu,
1587 		    fcio->fcio_ilen, mode)) {
1588 			retval = EFAULT;
1589 			kmem_free(req_iu, fcio->fcio_ilen);
1590 			kmem_free(rsp_iu, fcio->fcio_olen);
1591 			break;
1592 		}
1593 
1594 		user_ibuf = fcio->fcio_ibuf;
1595 		user_obuf = fcio->fcio_obuf;
1596 		fcio->fcio_ibuf = req_iu;
1597 		fcio->fcio_obuf = rsp_iu;
1598 
1599 		status = fcsm_ct_passthru(fcsm->sm_instance, fcio, KM_SLEEP,
1600 		    FCSM_JOBFLAG_SYNC, NULL);
1601 		if (status != FC_SUCCESS) {
1602 			retval = EIO;
1603 		}
1604 
1605 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1606 		    "fciocmd: cmd 0x%x completion status 0x%x",
1607 		    fcio->fcio_cmd, status));
1608 		fcio->fcio_errno = status;
1609 		fcio->fcio_ibuf = user_ibuf;
1610 		fcio->fcio_obuf = user_obuf;
1611 
1612 		if (ddi_copyout(rsp_iu, fcio->fcio_obuf,
1613 		    fcio->fcio_olen, mode)) {
1614 			retval = EFAULT;
1615 			kmem_free(req_iu, fcio->fcio_ilen);
1616 			kmem_free(rsp_iu, fcio->fcio_olen);
1617 			break;
1618 		}
1619 
1620 		kmem_free(req_iu, fcio->fcio_ilen);
1621 		kmem_free(rsp_iu, fcio->fcio_olen);
1622 
1623 		if (fcsm_fcio_copyout(fcio, arg, mode)) {
1624 			retval = EFAULT;
1625 		}
1626 		break;
1627 	}
1628 
1629 	case  FCSMIO_ADAPTER_LIST: {
1630 		fc_hba_list_t	*list;
1631 		int			count;
1632 
1633 		if ((fcio->fcio_xfer != FCIO_XFER_RW) ||
1634 		    (fcio->fcio_olen == 0) || (fcio->fcio_obuf == 0)) {
1635 			retval = EINVAL;
1636 			break;
1637 		}
1638 
1639 		list = kmem_zalloc(fcio->fcio_olen, KM_SLEEP);
1640 
1641 		if (ddi_copyin(fcio->fcio_obuf, list, fcio->fcio_olen, mode)) {
1642 			retval = EFAULT;
1643 			break;
1644 		}
1645 		list->version = FC_HBA_LIST_VERSION;
1646 
1647 		if (fcio->fcio_olen < MAXPATHLEN * list->numAdapters) {
1648 			retval = EFAULT;
1649 			break;
1650 		}
1651 
1652 		count = fc_ulp_get_adapter_paths((char *)list->hbaPaths,
1653 		    list->numAdapters);
1654 		if (count < 0) {
1655 			/* Did something go wrong? */
1656 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1657 			    "Error fetching adapter list."));
1658 			retval = ENXIO;
1659 			kmem_free(list, fcio->fcio_olen);
1660 			break;
1661 		}
1662 		/* Sucess (or short buffer) */
1663 		list->numAdapters = count;
1664 		if (ddi_copyout(list, fcio->fcio_obuf,
1665 		    fcio->fcio_olen, mode)) {
1666 			retval = EFAULT;
1667 		}
1668 		kmem_free(list, fcio->fcio_olen);
1669 		break;
1670 	}
1671 
1672 	default:
1673 		FCSM_DEBUG(SMDL_TRACE, (CE_NOTE, SM_LOG, NULL, NULL,
1674 		    "fciocmd: unknown cmd <0x%x>", fcio->fcio_cmd));
1675 		retval = ENOTTY;
1676 		break;
1677 	}
1678 
1679 	return (retval);
1680 }
1681 
1682 static int
1683 fcsm_fcio_copyout(fcio_t *fcio, intptr_t arg, int mode)
1684 {
1685 	int status;
1686 
1687 #ifdef	_MULTI_DATAMODEL
1688 	switch (ddi_model_convert_from(mode & FMODELS)) {
1689 	case DDI_MODEL_ILP32: {
1690 		struct fcio32 fcio32;
1691 
1692 		fcio32.fcio_xfer = fcio->fcio_xfer;
1693 		fcio32.fcio_cmd = fcio->fcio_cmd;
1694 		fcio32.fcio_flags = fcio->fcio_flags;
1695 		fcio32.fcio_cmd_flags = fcio->fcio_cmd_flags;
1696 		fcio32.fcio_ilen = fcio->fcio_ilen;
1697 		fcio32.fcio_ibuf = (caddr32_t)(long)fcio->fcio_ibuf;
1698 		fcio32.fcio_olen = fcio->fcio_olen;
1699 		fcio32.fcio_obuf = (caddr32_t)(long)fcio->fcio_obuf;
1700 		fcio32.fcio_alen = fcio->fcio_alen;
1701 		fcio32.fcio_abuf = (caddr32_t)(long)fcio->fcio_abuf;
1702 		fcio32.fcio_errno = fcio->fcio_errno;
1703 
1704 		status = ddi_copyout((void *)&fcio32, (void *)arg,
1705 		    sizeof (struct fcio32), mode);
1706 		break;
1707 	}
1708 	case DDI_MODEL_NONE:
1709 		status = ddi_copyout((void *)fcio, (void *)arg,
1710 		    sizeof (fcio_t), mode);
1711 		break;
1712 	}
1713 #else	/* _MULTI_DATAMODEL */
1714 	status = ddi_copyout((void *)fcio, (void *)arg, sizeof (fcio_t), mode);
1715 #endif	/* _MULTI_DATAMODEL */
1716 
1717 	return (status);
1718 }
1719 
1720 
1721 /* ARGSUSED */
1722 static int
1723 fcsm_open(dev_t *devp, int flags, int otyp, cred_t *credp)
1724 {
1725 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "open"));
1726 
1727 	if (otyp != OTYP_CHR) {
1728 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1729 		    "fcsm_open: failed. open type 0x%x for minor 0x%x is not "
1730 		    "OTYP_CHR", otyp, getminor(*devp)));
1731 		return (EINVAL);
1732 	}
1733 
1734 	/*
1735 	 * Allow anybody to open (both root and non-root users).
1736 	 * Previlege level checks are made on the per ioctl basis.
1737 	 */
1738 	mutex_enter(&fcsm_global_mutex);
1739 	if (flags & FEXCL) {
1740 		if (fcsm_flag & FCSM_OPEN) {
1741 			mutex_exit(&fcsm_global_mutex);
1742 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1743 			    "fcsm_open: exclusive open of 0x%x failed",
1744 			    getminor(*devp)));
1745 			return (EBUSY);
1746 		} else {
1747 			ASSERT(fcsm_flag == FCSM_IDLE);
1748 			fcsm_flag |= FCSM_EXCL;
1749 		}
1750 	} else {
1751 		if (fcsm_flag & FCSM_EXCL) {
1752 			mutex_exit(&fcsm_global_mutex);
1753 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1754 			    "fcsm_open: failed. Device minor 0x%x is in "
1755 			    "exclusive open mode", getminor(*devp)));
1756 			return (EBUSY);
1757 		}
1758 
1759 	}
1760 	fcsm_flag |= FCSM_OPEN;
1761 	mutex_exit(&fcsm_global_mutex);
1762 	return (0);
1763 }
1764 
1765 
1766 /* ARGSUSED */
1767 static int
1768 fcsm_close(dev_t dev, int flag, int otyp, cred_t *credp)
1769 {
1770 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL, "close"));
1771 
1772 	if (otyp != OTYP_CHR) {
1773 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1774 		    "fcsm_close: failed. close type 0x%x for minor 0x%x is not "
1775 		    "OTYP_CHR", otyp, getminor(dev)));
1776 		return (EINVAL);
1777 	}
1778 
1779 	mutex_enter(&fcsm_global_mutex);
1780 	if ((fcsm_flag & FCSM_OPEN) == 0) {
1781 		mutex_exit(&fcsm_global_mutex);
1782 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
1783 		    "fcsm_close: failed. minor 0x%x is already closed",
1784 		    getminor(dev)));
1785 		return (ENODEV);
1786 	}
1787 	fcsm_flag = FCSM_IDLE;
1788 	mutex_exit(&fcsm_global_mutex);
1789 	return (0);
1790 }
1791 
1792 
1793 /* ARGSUSED */
1794 static void
1795 fcsm_disp_devlist(fcsm_t *fcsm, fc_portmap_t *devlist, uint32_t dev_cnt)
1796 {
1797 	fc_portmap_t	*map;
1798 	uint32_t	i;
1799 
1800 	if (dev_cnt == 0) {
1801 		return;
1802 	}
1803 
1804 	ASSERT(devlist != NULL);
1805 	for (i = 0; i < dev_cnt; i++) {
1806 		map = &devlist[i];
1807 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
1808 		    "list[%d]: ID 0x%x WWN %x:%x:%x:%x:%x:%x:%x:%x "
1809 		    "state (0x%x) "
1810 		    "type <%s>(0x%x) "
1811 		    "flags (0x%x)",
1812 		    i, map->map_did.port_id,
1813 		    map->map_pwwn.raw_wwn[0], map->map_pwwn.raw_wwn[1],
1814 		    map->map_pwwn.raw_wwn[2], map->map_pwwn.raw_wwn[3],
1815 		    map->map_pwwn.raw_wwn[4], map->map_pwwn.raw_wwn[5],
1816 		    map->map_pwwn.raw_wwn[6], map->map_pwwn.raw_wwn[7],
1817 		    map->map_state,
1818 		    fcsm_dev_type_to_str(map->map_type), map->map_type,
1819 		    map->map_flags));
1820 	}
1821 }
1822 
1823 /* ARGSUSED */
1824 static void
1825 fcsm_display(int level, int flags, fcsm_t *fcsm, fc_packet_t *pkt,
1826     const char *fmt, ...)
1827 {
1828 	caddr_t	buf;
1829 	va_list	ap;
1830 
1831 	buf = kmem_zalloc(256, KM_NOSLEEP);
1832 	if (buf == NULL) {
1833 		return;
1834 	}
1835 
1836 	if (fcsm) {
1837 		(void) sprintf(buf + strlen(buf), "fcsm(%d): ",
1838 		    ddi_get_instance(fcsm->sm_port_info.port_dip));
1839 	} else {
1840 		(void) sprintf(buf, "fcsm: ");
1841 	}
1842 
1843 	va_start(ap, fmt);
1844 	(void) vsprintf(buf + strlen(buf), fmt, ap);
1845 	va_end(ap);
1846 
1847 	if (pkt) {
1848 		caddr_t state, reason, action, expln;
1849 
1850 		(void) fc_ulp_pkt_error(pkt, &state, &reason, &action, &expln);
1851 
1852 		(void) sprintf(buf + strlen(buf),
1853 		    " state: %s(0x%x); reason: %s(0x%x)",
1854 		    state, pkt->pkt_state, reason, pkt->pkt_reason);
1855 	}
1856 
1857 	switch (flags) {
1858 	case SM_LOG:
1859 		cmn_err(level, "!%s", buf);
1860 		break;
1861 
1862 	case SM_CONSOLE:
1863 		cmn_err(level, "^%s", buf);
1864 		break;
1865 
1866 	default:
1867 		cmn_err(level, "%s", buf);
1868 		break;
1869 	}
1870 
1871 	kmem_free(buf, 256);
1872 }
1873 
1874 
1875 /*
1876  * Convert FC packet state to FC errno
1877  */
1878 int
1879 fcsm_pkt_state_to_rval(uchar_t state, uint32_t reason)
1880 {
1881 	int count;
1882 
1883 	if (state == FC_PKT_LOCAL_RJT && (reason == FC_REASON_NO_CONNECTION ||
1884 	    reason == FC_REASON_LOGIN_REQUIRED)) {
1885 		return (FC_LOGINREQ);
1886 	} else if (state == FC_PKT_PORT_OFFLINE &&
1887 	    reason == FC_REASON_LOGIN_REQUIRED) {
1888 		return (FC_LOGINREQ);
1889 	}
1890 
1891 	for (count = 0; count < sizeof (fcsm_xlat_pkt_state) /
1892 	    sizeof (fcsm_xlat_pkt_state[0]); count++) {
1893 		if (fcsm_xlat_pkt_state[count].xlat_state == state) {
1894 			return (fcsm_xlat_pkt_state[count].xlat_rval);
1895 		}
1896 	}
1897 
1898 	return (FC_FAILURE);
1899 }
1900 
1901 
1902 /*
1903  * Convert port state state to descriptive string
1904  */
1905 caddr_t
1906 fcsm_port_state_to_str(uint32_t port_state)
1907 {
1908 	int count;
1909 
1910 	for (count = 0; count < sizeof (fcsm_xlat_port_state) /
1911 	    sizeof (fcsm_xlat_port_state[0]); count++) {
1912 		if (fcsm_xlat_port_state[count].xlat_pstate == port_state) {
1913 			return (fcsm_xlat_port_state[count].xlat_state_str);
1914 		}
1915 	}
1916 
1917 	return (NULL);
1918 }
1919 
1920 
1921 /*
1922  * Convert port topology state to descriptive string
1923  */
1924 caddr_t
1925 fcsm_topology_to_str(uint32_t topology)
1926 {
1927 	int count;
1928 
1929 	for (count = 0; count < sizeof (fcsm_xlat_topology) /
1930 	    sizeof (fcsm_xlat_topology[0]); count++) {
1931 		if (fcsm_xlat_topology[count].xlat_top == topology) {
1932 			return (fcsm_xlat_topology[count].xlat_top_str);
1933 		}
1934 	}
1935 
1936 	return (NULL);
1937 }
1938 
1939 
1940 /*
1941  * Convert port topology state to descriptive string
1942  */
1943 static caddr_t
1944 fcsm_dev_type_to_str(uint32_t type)
1945 {
1946 	int count;
1947 
1948 	for (count = 0; count < sizeof (fcsm_xlat_dev_type) /
1949 	    sizeof (fcsm_xlat_dev_type[0]); count++) {
1950 		if (fcsm_xlat_dev_type[count].xlat_type == type) {
1951 			return (fcsm_xlat_dev_type[count].xlat_str);
1952 		}
1953 	}
1954 
1955 	return (NULL);
1956 }
1957 
1958 static int
1959 fcsm_cmd_cache_constructor(void *buf, void *cdarg, int kmflags)
1960 {
1961 	fcsm_cmd_t		*cmd = (fcsm_cmd_t *)buf;
1962 	fcsm_t			*fcsm = (fcsm_t *)cdarg;
1963 	int			(*callback)(caddr_t);
1964 	fc_packet_t		*pkt;
1965 	fc_ulp_port_info_t	*pinfo;
1966 
1967 	ASSERT(fcsm != NULL && buf != NULL);
1968 	callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP: DDI_DMA_DONTWAIT;
1969 
1970 	cmd->cmd_fp_pkt		= &cmd->cmd_fc_packet;
1971 	cmd->cmd_job		= NULL;
1972 	cmd->cmd_fcsm		= fcsm;
1973 	cmd->cmd_dma_flags	= 0;
1974 
1975 	pkt = &cmd->cmd_fc_packet;
1976 
1977 	pkt->pkt_ulp_rscn_infop = NULL;
1978 	pkt->pkt_fca_private = (opaque_t)((caddr_t)cmd + sizeof (fcsm_cmd_t));
1979 	pkt->pkt_ulp_private = (opaque_t)cmd;
1980 
1981 	if (!(fcsm->sm_flags & FCSM_USING_NODMA_FCA)) {
1982 		pinfo = &fcsm->sm_port_info;
1983 		if (ddi_dma_alloc_handle(pinfo->port_dip,
1984 		    pinfo->port_cmd_dma_attr,
1985 		    callback, NULL, &pkt->pkt_cmd_dma) != DDI_SUCCESS) {
1986 			return (1);
1987 		}
1988 
1989 		if (ddi_dma_alloc_handle(pinfo->port_dip,
1990 		    pinfo->port_resp_dma_attr,
1991 		    callback, NULL, &pkt->pkt_resp_dma) != DDI_SUCCESS) {
1992 			ddi_dma_free_handle(&pkt->pkt_cmd_dma);
1993 			return (1);
1994 		}
1995 	} else {
1996 		pkt->pkt_cmd_dma  = NULL;
1997 		pkt->pkt_cmd	  = NULL;
1998 		pkt->pkt_resp_dma = NULL;
1999 		pkt->pkt_resp	  = NULL;
2000 	}
2001 
2002 	pkt->pkt_cmd_acc = pkt->pkt_resp_acc = NULL;
2003 	pkt->pkt_cmd_cookie_cnt = pkt->pkt_resp_cookie_cnt =
2004 	    pkt->pkt_data_cookie_cnt = 0;
2005 	pkt->pkt_cmd_cookie = pkt->pkt_resp_cookie =
2006 	    pkt->pkt_data_cookie = NULL;
2007 
2008 	return (0);
2009 }
2010 
2011 
2012 /* ARGSUSED */
2013 static void
2014 fcsm_cmd_cache_destructor(void *buf, void *cdarg)
2015 {
2016 	fcsm_cmd_t	*cmd = (fcsm_cmd_t *)buf;
2017 	fcsm_t		*fcsm = (fcsm_t *)cdarg;
2018 	fc_packet_t	*pkt;
2019 
2020 	ASSERT(fcsm == cmd->cmd_fcsm);
2021 
2022 	pkt = cmd->cmd_fp_pkt;
2023 
2024 	if (pkt->pkt_cmd_dma != NULL) {
2025 		ddi_dma_free_handle(&pkt->pkt_cmd_dma);
2026 	}
2027 
2028 	if (pkt->pkt_resp_dma != NULL) {
2029 		ddi_dma_free_handle(&pkt->pkt_resp_dma);
2030 	}
2031 }
2032 
2033 
2034 static fcsm_cmd_t *
2035 fcsm_alloc_cmd(fcsm_t *fcsm, uint32_t cmd_len, uint32_t resp_len, int sleep)
2036 {
2037 	fcsm_cmd_t	*cmd;
2038 	fc_packet_t	*pkt;
2039 	int		rval;
2040 	ulong_t		real_len;
2041 	int		(*callback)(caddr_t);
2042 	ddi_dma_cookie_t	pkt_cookie;
2043 	ddi_dma_cookie_t	*cp;
2044 	uint32_t		cnt;
2045 	fc_ulp_port_info_t	*pinfo;
2046 
2047 	ASSERT(fcsm != NULL);
2048 	pinfo = &fcsm->sm_port_info;
2049 
2050 	callback = (sleep == KM_SLEEP) ? DDI_DMA_SLEEP: DDI_DMA_DONTWAIT;
2051 
2052 	cmd = (fcsm_cmd_t *)kmem_cache_alloc(fcsm->sm_cmd_cache, sleep);
2053 	if (cmd == NULL) {
2054 		FCSM_DEBUG(SMDL_ERR, (CE_WARN, SM_LOG, fcsm, NULL,
2055 		    "alloc_cmd: kmem_cache_alloc failed"));
2056 		return (NULL);
2057 	}
2058 
2059 	cmd->cmd_retry_count	= 0;
2060 	cmd->cmd_max_retries	= 0;
2061 	cmd->cmd_retry_interval	= 0;
2062 	cmd->cmd_transport	= NULL;
2063 
2064 	ASSERT(cmd->cmd_dma_flags == 0);
2065 	ASSERT(cmd->cmd_fp_pkt == &cmd->cmd_fc_packet);
2066 	pkt = cmd->cmd_fp_pkt;
2067 
2068 	/* Zero out the important fc_packet fields */
2069 	pkt->pkt_pd		= NULL;
2070 	pkt->pkt_datalen	= 0;
2071 	pkt->pkt_data		= NULL;
2072 	pkt->pkt_state		= 0;
2073 	pkt->pkt_action		= 0;
2074 	pkt->pkt_reason		= 0;
2075 	pkt->pkt_expln		= 0;
2076 
2077 	/*
2078 	 * Now that pkt_pd is initialized, we can call fc_ulp_init_packet
2079 	 */
2080 
2081 	if (fc_ulp_init_packet((opaque_t)pinfo->port_handle, pkt, sleep)
2082 	    != FC_SUCCESS) {
2083 		kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2084 		return (NULL);
2085 	}
2086 
2087 	if ((cmd_len) && !(fcsm->sm_flags & FCSM_USING_NODMA_FCA)) {
2088 		ASSERT(pkt->pkt_cmd_dma != NULL);
2089 
2090 		rval = ddi_dma_mem_alloc(pkt->pkt_cmd_dma, cmd_len,
2091 		    fcsm->sm_port_info.port_acc_attr, DDI_DMA_CONSISTENT,
2092 		    callback, NULL, (caddr_t *)&pkt->pkt_cmd, &real_len,
2093 		    &pkt->pkt_cmd_acc);
2094 
2095 		if (rval != DDI_SUCCESS) {
2096 			(void) fc_ulp_uninit_packet(
2097 			    (opaque_t)pinfo->port_handle, pkt);
2098 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2099 			fcsm_free_cmd_dma(cmd);
2100 			return (NULL);
2101 		}
2102 
2103 		cmd->cmd_dma_flags |= FCSM_CF_CMD_VALID_DMA_MEM;
2104 
2105 		if (real_len < cmd_len) {
2106 			(void) fc_ulp_uninit_packet(
2107 			    (opaque_t)pinfo->port_handle, pkt);
2108 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2109 			fcsm_free_cmd_dma(cmd);
2110 			return (NULL);
2111 		}
2112 
2113 		rval = ddi_dma_addr_bind_handle(pkt->pkt_cmd_dma, NULL,
2114 		    pkt->pkt_cmd, real_len, DDI_DMA_WRITE | DDI_DMA_CONSISTENT,
2115 		    callback, NULL, &pkt_cookie, &pkt->pkt_cmd_cookie_cnt);
2116 
2117 		if (rval != DDI_DMA_MAPPED) {
2118 			(void) fc_ulp_uninit_packet(
2119 			    (opaque_t)pinfo->port_handle, pkt);
2120 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2121 			fcsm_free_cmd_dma(cmd);
2122 			return (NULL);
2123 		}
2124 
2125 		cmd->cmd_dma_flags |= FCSM_CF_CMD_VALID_DMA_BIND;
2126 
2127 		if (pkt->pkt_cmd_cookie_cnt >
2128 		    pinfo->port_cmd_dma_attr->dma_attr_sgllen) {
2129 			(void) fc_ulp_uninit_packet(
2130 			    (opaque_t)pinfo->port_handle, pkt);
2131 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2132 			fcsm_free_cmd_dma(cmd);
2133 			return (NULL);
2134 		}
2135 
2136 		ASSERT(pkt->pkt_cmd_cookie_cnt != 0);
2137 
2138 		cp = pkt->pkt_cmd_cookie = (ddi_dma_cookie_t *)kmem_alloc(
2139 		    pkt->pkt_cmd_cookie_cnt * sizeof (pkt_cookie),
2140 		    KM_NOSLEEP);
2141 
2142 		if (cp == NULL) {
2143 			(void) fc_ulp_uninit_packet(
2144 			    (opaque_t)pinfo->port_handle, pkt);
2145 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2146 			fcsm_free_cmd_dma(cmd);
2147 			return (NULL);
2148 		}
2149 
2150 		*cp = pkt_cookie;
2151 		cp++;
2152 		for (cnt = 1; cnt < pkt->pkt_cmd_cookie_cnt; cnt++, cp++) {
2153 			ddi_dma_nextcookie(pkt->pkt_cmd_dma, &pkt_cookie);
2154 			*cp = pkt_cookie;
2155 		}
2156 	} else if (cmd_len != 0) {
2157 		pkt->pkt_cmd = kmem_zalloc(cmd_len, KM_SLEEP);
2158 	}
2159 
2160 	if ((resp_len) && !(fcsm->sm_flags & FCSM_USING_NODMA_FCA)) {
2161 		ASSERT(pkt->pkt_resp_dma != NULL);
2162 
2163 		rval = ddi_dma_mem_alloc(pkt->pkt_resp_dma, resp_len,
2164 		    fcsm->sm_port_info.port_acc_attr, DDI_DMA_CONSISTENT,
2165 		    callback, NULL, (caddr_t *)&pkt->pkt_resp, &real_len,
2166 		    &pkt->pkt_resp_acc);
2167 
2168 		if (rval != DDI_SUCCESS) {
2169 			(void) fc_ulp_uninit_packet(
2170 			    (opaque_t)pinfo->port_handle, pkt);
2171 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2172 			fcsm_free_cmd_dma(cmd);
2173 			return (NULL);
2174 		}
2175 
2176 		cmd->cmd_dma_flags |= FCSM_CF_RESP_VALID_DMA_MEM;
2177 
2178 		if (real_len < resp_len) {
2179 			(void) fc_ulp_uninit_packet(
2180 			    (opaque_t)pinfo->port_handle, pkt);
2181 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2182 			fcsm_free_cmd_dma(cmd);
2183 			return (NULL);
2184 		}
2185 
2186 		rval = ddi_dma_addr_bind_handle(pkt->pkt_resp_dma, NULL,
2187 		    pkt->pkt_resp, real_len, DDI_DMA_READ | DDI_DMA_CONSISTENT,
2188 		    callback, NULL, &pkt_cookie, &pkt->pkt_resp_cookie_cnt);
2189 
2190 		if (rval != DDI_DMA_MAPPED) {
2191 			(void) fc_ulp_uninit_packet(
2192 			    (opaque_t)pinfo->port_handle, pkt);
2193 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2194 			fcsm_free_cmd_dma(cmd);
2195 			return (NULL);
2196 		}
2197 
2198 		cmd->cmd_dma_flags |= FCSM_CF_RESP_VALID_DMA_BIND;
2199 
2200 		if (pkt->pkt_resp_cookie_cnt >
2201 		    pinfo->port_resp_dma_attr->dma_attr_sgllen) {
2202 			(void) fc_ulp_uninit_packet(
2203 			    (opaque_t)pinfo->port_handle, pkt);
2204 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2205 			fcsm_free_cmd_dma(cmd);
2206 			return (NULL);
2207 		}
2208 
2209 		ASSERT(pkt->pkt_resp_cookie_cnt != 0);
2210 
2211 		cp = pkt->pkt_resp_cookie = (ddi_dma_cookie_t *)kmem_alloc(
2212 		    pkt->pkt_resp_cookie_cnt * sizeof (pkt_cookie),
2213 		    KM_NOSLEEP);
2214 
2215 		if (cp == NULL) {
2216 			(void) fc_ulp_uninit_packet(
2217 			    (opaque_t)pinfo->port_handle, pkt);
2218 			kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2219 			fcsm_free_cmd_dma(cmd);
2220 			return (NULL);
2221 		}
2222 
2223 		*cp = pkt_cookie;
2224 		cp++;
2225 		for (cnt = 1; cnt < pkt->pkt_resp_cookie_cnt; cnt++, cp++) {
2226 			ddi_dma_nextcookie(pkt->pkt_resp_dma, &pkt_cookie);
2227 			*cp = pkt_cookie;
2228 		}
2229 	} else if (resp_len != 0) {
2230 		pkt->pkt_resp = kmem_zalloc(resp_len, KM_SLEEP);
2231 	}
2232 
2233 	pkt->pkt_cmdlen = cmd_len;
2234 	pkt->pkt_rsplen = resp_len;
2235 
2236 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2237 	    "alloc_cmd: cmd 0x%p", (void *)cmd));
2238 	return (cmd);
2239 }
2240 
2241 static void
2242 fcsm_free_cmd(fcsm_cmd_t *cmd)
2243 {
2244 	fcsm_t		*fcsm;
2245 
2246 	fcsm = cmd->cmd_fcsm;
2247 	ASSERT(fcsm != NULL);
2248 
2249 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2250 	    "free_cmd: cmd 0x%p", (void *)cmd));
2251 
2252 	fcsm_free_cmd_dma(cmd);
2253 
2254 	(void) fc_ulp_uninit_packet((opaque_t)fcsm->sm_port_info.port_handle,
2255 	    cmd->cmd_fp_pkt);
2256 	kmem_cache_free(fcsm->sm_cmd_cache, (void *)cmd);
2257 }
2258 
2259 static void
2260 fcsm_free_cmd_dma(fcsm_cmd_t *cmd)
2261 {
2262 	fc_packet_t	*pkt;
2263 
2264 	pkt = cmd->cmd_fp_pkt;
2265 	ASSERT(pkt != NULL);
2266 
2267 	if (cmd->cmd_fcsm->sm_flags & FCSM_USING_NODMA_FCA) {
2268 		if (pkt->pkt_cmd) {
2269 			kmem_free(pkt->pkt_cmd, pkt->pkt_cmdlen);
2270 			pkt->pkt_cmd = NULL;
2271 		}
2272 
2273 		if (pkt->pkt_resp) {
2274 			kmem_free(pkt->pkt_resp, pkt->pkt_rsplen);
2275 			pkt->pkt_resp = NULL;
2276 		}
2277 	}
2278 
2279 	pkt->pkt_cmdlen = 0;
2280 	pkt->pkt_rsplen = 0;
2281 	pkt->pkt_tran_type = 0;
2282 	pkt->pkt_tran_flags = 0;
2283 
2284 	if (pkt->pkt_cmd_cookie != NULL) {
2285 		kmem_free(pkt->pkt_cmd_cookie, pkt->pkt_cmd_cookie_cnt *
2286 		    sizeof (ddi_dma_cookie_t));
2287 		pkt->pkt_cmd_cookie = NULL;
2288 	}
2289 
2290 	if (pkt->pkt_resp_cookie != NULL) {
2291 		kmem_free(pkt->pkt_resp_cookie, pkt->pkt_resp_cookie_cnt *
2292 		    sizeof (ddi_dma_cookie_t));
2293 		pkt->pkt_resp_cookie = NULL;
2294 	}
2295 
2296 	if (cmd->cmd_dma_flags & FCSM_CF_CMD_VALID_DMA_BIND) {
2297 		(void) ddi_dma_unbind_handle(pkt->pkt_cmd_dma);
2298 	}
2299 
2300 	if (cmd->cmd_dma_flags & FCSM_CF_CMD_VALID_DMA_MEM) {
2301 		if (pkt->pkt_cmd_acc) {
2302 			ddi_dma_mem_free(&pkt->pkt_cmd_acc);
2303 		}
2304 	}
2305 
2306 	if (cmd->cmd_dma_flags & FCSM_CF_RESP_VALID_DMA_BIND) {
2307 		(void) ddi_dma_unbind_handle(pkt->pkt_resp_dma);
2308 	}
2309 
2310 	if (cmd->cmd_dma_flags & FCSM_CF_RESP_VALID_DMA_MEM) {
2311 		if (pkt->pkt_resp_acc) {
2312 			ddi_dma_mem_free(&pkt->pkt_resp_acc);
2313 		}
2314 	}
2315 
2316 	cmd->cmd_dma_flags = 0;
2317 }
2318 
2319 /* ARGSUSED */
2320 static int
2321 fcsm_job_cache_constructor(void *buf, void *cdarg, int kmflag)
2322 {
2323 	fcsm_job_t *job = (fcsm_job_t *)buf;
2324 
2325 	mutex_init(&job->job_mutex, NULL, MUTEX_DRIVER, NULL);
2326 	sema_init(&job->job_sema, 0, NULL, SEMA_DEFAULT, NULL);
2327 
2328 	return (0);
2329 }
2330 
2331 /* ARGSUSED */
2332 static void
2333 fcsm_job_cache_destructor(void *buf, void *cdarg)
2334 {
2335 	fcsm_job_t *job = (fcsm_job_t *)buf;
2336 
2337 	sema_destroy(&job->job_sema);
2338 	mutex_destroy(&job->job_mutex);
2339 }
2340 
2341 
2342 static fcsm_job_t *
2343 fcsm_alloc_job(int sleep)
2344 {
2345 	fcsm_job_t	*job;
2346 
2347 	job = (fcsm_job_t *)kmem_cache_alloc(fcsm_job_cache, sleep);
2348 	if (job != NULL) {
2349 		job->job_code		= FCSM_JOB_NONE;
2350 		job->job_flags		= 0;
2351 		job->job_port_instance	= -1;
2352 		job->job_result		= -1;
2353 		job->job_arg		= (opaque_t)0;
2354 		job->job_caller_priv	= (opaque_t)0;
2355 		job->job_comp		= NULL;
2356 		job->job_comp_arg	= (opaque_t)0;
2357 		job->job_priv		= (void *)0;
2358 		job->job_priv_flags	= 0;
2359 		job->job_next		= 0;
2360 	}
2361 
2362 	return (job);
2363 }
2364 
2365 static void
2366 fcsm_dealloc_job(fcsm_job_t *job)
2367 {
2368 	kmem_cache_free(fcsm_job_cache, (void *)job);
2369 }
2370 
2371 
2372 static void
2373 fcsm_init_job(fcsm_job_t *job, int instance, uint32_t command, uint32_t flags,
2374     opaque_t arg, opaque_t caller_priv,
2375     void (*comp)(opaque_t, fcsm_job_t *, int), opaque_t comp_arg)
2376 {
2377 	ASSERT(job != NULL);
2378 	job->job_port_instance	= instance;
2379 	job->job_code		= command;
2380 	job->job_flags		= flags;
2381 	job->job_arg		= arg;
2382 	job->job_caller_priv	= caller_priv;
2383 	job->job_comp		= comp;
2384 	job->job_comp_arg	= comp_arg;
2385 	job->job_retry_count	= 0;
2386 }
2387 
2388 static int
2389 fcsm_process_job(fcsm_job_t *job, int priority_flag)
2390 {
2391 	fcsm_t	*fcsm;
2392 	int	sync;
2393 
2394 	ASSERT(job != NULL);
2395 	ASSERT(!MUTEX_HELD(&job->job_mutex));
2396 
2397 	fcsm = ddi_get_soft_state(fcsm_state, job->job_port_instance);
2398 
2399 	if (fcsm == NULL) {
2400 		FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, NULL, NULL,
2401 		    "process_job: port instance 0x%x not found",
2402 		    job->job_port_instance));
2403 		return (FC_BADDEV);
2404 	}
2405 
2406 	mutex_enter(&job->job_mutex);
2407 	/* Both SYNC and ASYNC flags should not be set */
2408 	ASSERT(((job->job_flags & (FCSM_JOBFLAG_SYNC | FCSM_JOBFLAG_ASYNC)) ==
2409 	    FCSM_JOBFLAG_SYNC) || ((job->job_flags &
2410 	    (FCSM_JOBFLAG_SYNC | FCSM_JOBFLAG_ASYNC)) == FCSM_JOBFLAG_ASYNC));
2411 	/*
2412 	 * Check if job is a synchronous job. We might not be able to
2413 	 * check it reliably after enque_job(), if job is an ASYNC job.
2414 	 */
2415 	sync = job->job_flags & FCSM_JOBFLAG_SYNC;
2416 	mutex_exit(&job->job_mutex);
2417 
2418 	/* Queue the job for processing by job thread */
2419 	fcsm_enque_job(fcsm, job, priority_flag);
2420 
2421 	/* Wait for job completion, if it is a synchronous job */
2422 	if (sync) {
2423 		/*
2424 		 * This is a Synchronous Job. So job structure is available.
2425 		 * Caller is responsible for freeing it.
2426 		 */
2427 		FCSM_DEBUG(SMDL_ERR, (CE_CONT, SM_LOG, fcsm, NULL,
2428 		    "process_job: Waiting for sync job <%p> completion",
2429 		    (void *)job));
2430 		sema_p(&job->job_sema);
2431 	}
2432 
2433 	return (FC_SUCCESS);
2434 }
2435 
2436 static void
2437 fcsm_enque_job(fcsm_t *fcsm, fcsm_job_t *job, int priority_flag)
2438 {
2439 	ASSERT(!MUTEX_HELD(&fcsm->sm_mutex));
2440 
2441 	mutex_enter(&fcsm->sm_mutex);
2442 	/* Queue the job at the head or tail depending on the job priority */
2443 	if (priority_flag) {
2444 		FCSM_DEBUG(SMDL_INFO, (CE_CONT, SM_LOG, fcsm, NULL,
2445 		    "enque_job: job 0x%p is high priority", job));
2446 		/* Queue at the head */
2447 		if (fcsm->sm_job_tail == NULL) {
2448 			ASSERT(fcsm->sm_job_head == NULL);
2449 			fcsm->sm_job_head = fcsm->sm_job_tail = job;
2450 		} else {
2451 			ASSERT(fcsm->sm_job_head != NULL);
2452 			job->job_next = fcsm->sm_job_head;
2453 			fcsm->sm_job_head = job;
2454 		}
2455 	} else {
2456 		FCSM_DEBUG(SMDL_INFO, (CE_CONT, SM_LOG, fcsm, NULL,
2457 		    "enque_job: job 0x%p is normal", job));
2458 		/* Queue at the tail */
2459 		if (fcsm->sm_job_tail == NULL) {
2460 			ASSERT(fcsm->sm_job_head == NULL);
2461 			fcsm->sm_job_head = fcsm->sm_job_tail = job;
2462 		} else {
2463 			ASSERT(fcsm->sm_job_head != NULL);
2464 			fcsm->sm_job_tail->job_next = job;
2465 			fcsm->sm_job_tail = job;
2466 		}
2467 		job->job_next = NULL;
2468 	}
2469 
2470 	/* Signal the job thread to process the job */
2471 	cv_signal(&fcsm->sm_job_cv);
2472 	mutex_exit(&fcsm->sm_mutex);
2473 }
2474 
2475 static int
2476 fcsm_retry_job(fcsm_t *fcsm, fcsm_job_t *job)
2477 {
2478 	/*
2479 	 * If it is a CT passthru job and status is login required, then
2480 	 * retry the job so that login can be performed again.
2481 	 * Ensure that this retry is performed a finite number of times,
2482 	 * so that a faulty fabric does not cause us to retry forever.
2483 	 */
2484 
2485 	switch (job->job_code) {
2486 	case FCSM_JOB_CT_PASSTHRU: {
2487 		uint32_t	jobflag;
2488 		fc_ct_header_t	*ct_header;
2489 
2490 		if (job->job_result != FC_LOGINREQ) {
2491 			break;
2492 		}
2493 
2494 		/*
2495 		 * If it is a management server command
2496 		 * then Reset the Management server login flag, so that login
2497 		 * gets re-established.
2498 		 * If it is a Name server command,
2499 		 * then it is 'fp' responsibility to perform the login.
2500 		 */
2501 		ASSERT(job->job_arg != NULL);
2502 		ct_header =
2503 		    (fc_ct_header_t *)((fcio_t *)job->job_arg)->fcio_ibuf;
2504 		if (ct_header->ct_fcstype == FCSTYPE_MGMTSERVICE) {
2505 			mutex_enter(&fcsm->sm_mutex);
2506 			fcsm->sm_flags &= ~FCSM_MGMT_SERVER_LOGGED_IN;
2507 			mutex_exit(&fcsm->sm_mutex);
2508 		}
2509 
2510 		if (job->job_retry_count >= fcsm_max_job_retries) {
2511 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2512 			    "retry_job: job 0x%p max retries (%d) reached",
2513 			    (void *)job, job->job_retry_count));
2514 			break;
2515 		}
2516 
2517 		/*
2518 		 * Login is required again. Retry the command, so that
2519 		 * login will get performed again.
2520 		 */
2521 		mutex_enter(&job->job_mutex);
2522 		job->job_retry_count++;
2523 		jobflag = job->job_flags;
2524 		mutex_exit(&job->job_mutex);
2525 
2526 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2527 		    "retry_job: retry(%d) job 0x%p",
2528 		    job->job_retry_count, (void *)job));
2529 		/*
2530 		 * This job should get picked up before the
2531 		 * other jobs sitting in the queue.
2532 		 * Requeue the command at the head and then
2533 		 * reset the SERIALIZE flag.
2534 		 */
2535 		fcsm_enque_job(fcsm, job, 1);
2536 		if (jobflag & FCSM_JOBFLAG_SERIALIZE) {
2537 			mutex_enter(&fcsm->sm_mutex);
2538 			ASSERT(fcsm->sm_flags & FCSM_SERIALIZE_JOBTHREAD);
2539 			fcsm->sm_flags &= ~FCSM_SERIALIZE_JOBTHREAD;
2540 
2541 			/* Signal the job thread to process the job */
2542 			cv_signal(&fcsm->sm_job_cv);
2543 			mutex_exit(&fcsm->sm_mutex);
2544 		}
2545 
2546 		/* Command is queued for retrying */
2547 		return (0);
2548 	}
2549 
2550 	default:
2551 		break;
2552 	}
2553 	return (1);
2554 }
2555 
2556 static void
2557 fcsm_jobdone(fcsm_job_t *job)
2558 {
2559 	fcsm_t	*fcsm;
2560 
2561 	fcsm = ddi_get_soft_state(fcsm_state, job->job_port_instance);
2562 	ASSERT(fcsm != NULL);
2563 
2564 	if (job->job_result != FC_SUCCESS) {
2565 		if (fcsm_retry_job(fcsm, job) == 0) {
2566 			/* Job retried. so just return from here */
2567 			return;
2568 		}
2569 	}
2570 
2571 	if (job->job_comp) {
2572 		job->job_comp(job->job_comp_arg, job, job->job_result);
2573 	}
2574 
2575 	mutex_enter(&job->job_mutex);
2576 	if (job->job_flags & FCSM_JOBFLAG_SERIALIZE) {
2577 		mutex_exit(&job->job_mutex);
2578 		mutex_enter(&fcsm->sm_mutex);
2579 		ASSERT(fcsm->sm_flags & FCSM_SERIALIZE_JOBTHREAD);
2580 		fcsm->sm_flags &= ~FCSM_SERIALIZE_JOBTHREAD;
2581 
2582 		/* Signal the job thread to process the job */
2583 		cv_signal(&fcsm->sm_job_cv);
2584 		mutex_exit(&fcsm->sm_mutex);
2585 		mutex_enter(&job->job_mutex);
2586 	}
2587 
2588 	if (job->job_flags & FCSM_JOBFLAG_SYNC) {
2589 		mutex_exit(&job->job_mutex);
2590 		sema_v(&job->job_sema);
2591 	} else {
2592 		mutex_exit(&job->job_mutex);
2593 		/* Async job, free the job structure */
2594 		fcsm_dealloc_job(job);
2595 	}
2596 }
2597 
2598 fcsm_job_t *
2599 fcsm_deque_job(fcsm_t *fcsm)
2600 {
2601 	fcsm_job_t	*job;
2602 
2603 	ASSERT(MUTEX_HELD(&fcsm->sm_mutex));
2604 
2605 	if (fcsm->sm_job_head == NULL) {
2606 		ASSERT(fcsm->sm_job_tail == NULL);
2607 		job = NULL;
2608 	} else {
2609 		ASSERT(fcsm->sm_job_tail != NULL);
2610 		job = fcsm->sm_job_head;
2611 		if (job->job_next == NULL) {
2612 			ASSERT(fcsm->sm_job_tail == job);
2613 			fcsm->sm_job_tail = NULL;
2614 		}
2615 		fcsm->sm_job_head = job->job_next;
2616 		job->job_next = NULL;
2617 	}
2618 
2619 	return (job);
2620 }
2621 
2622 
2623 /* Dedicated per port thread to process various commands */
2624 static void
2625 fcsm_job_thread(fcsm_t *fcsm)
2626 {
2627 	fcsm_job_t	*job;
2628 
2629 	ASSERT(fcsm != NULL);
2630 #ifndef __lock_lint
2631 	CALLB_CPR_INIT(&fcsm->sm_cpr_info, &fcsm->sm_mutex,
2632 	    callb_generic_cpr, "fcsm_job_thread");
2633 #endif /* __lock_lint */
2634 
2635 	for (;;) {
2636 		mutex_enter(&fcsm->sm_mutex);
2637 
2638 		while (fcsm->sm_job_head == NULL ||
2639 		    fcsm->sm_flags & FCSM_SERIALIZE_JOBTHREAD) {
2640 			CALLB_CPR_SAFE_BEGIN(&fcsm->sm_cpr_info);
2641 			cv_wait(&fcsm->sm_job_cv, &fcsm->sm_mutex);
2642 			CALLB_CPR_SAFE_END(&fcsm->sm_cpr_info, &fcsm->sm_mutex);
2643 		}
2644 
2645 		job = fcsm_deque_job(fcsm);
2646 
2647 		mutex_exit(&fcsm->sm_mutex);
2648 
2649 		mutex_enter(&job->job_mutex);
2650 		if (job->job_flags & FCSM_JOBFLAG_SERIALIZE) {
2651 			mutex_exit(&job->job_mutex);
2652 
2653 			mutex_enter(&fcsm->sm_mutex);
2654 			ASSERT(!(fcsm->sm_flags & FCSM_SERIALIZE_JOBTHREAD));
2655 			fcsm->sm_flags |= FCSM_SERIALIZE_JOBTHREAD;
2656 			mutex_exit(&fcsm->sm_mutex);
2657 		} else {
2658 			mutex_exit(&job->job_mutex);
2659 		}
2660 
2661 		ASSERT(fcsm->sm_instance == job->job_port_instance);
2662 
2663 		switch (job->job_code) {
2664 		case FCSM_JOB_NONE:
2665 			fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
2666 			    "job_thread: uninitialized job code");
2667 			job->job_result = FC_FAILURE;
2668 			fcsm_jobdone(job);
2669 			break;
2670 
2671 		case FCSM_JOB_THREAD_SHUTDOWN:
2672 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2673 			    "job_thread: job code <JOB PORT SHUTDOWN>"));
2674 
2675 			/*
2676 			 * There should not be any pending jobs, when this
2677 			 * is being called.
2678 			 */
2679 			mutex_enter(&fcsm->sm_mutex);
2680 			ASSERT(fcsm->sm_job_head == NULL);
2681 			ASSERT(fcsm->sm_job_tail == NULL);
2682 			ASSERT(fcsm->sm_retry_head == NULL);
2683 			ASSERT(fcsm->sm_retry_tail == NULL);
2684 			job->job_result = FC_SUCCESS;
2685 #ifndef __lock_lint
2686 			CALLB_CPR_EXIT(&fcsm->sm_cpr_info);
2687 #endif
2688 			/* CPR_EXIT has also dropped the fcsm->sm_mutex */
2689 
2690 			fcsm_jobdone(job);
2691 			thread_exit();
2692 			/* NOTREACHED */
2693 			break;
2694 
2695 		case FCSM_JOB_LOGIN_NAME_SERVER:
2696 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2697 			    "job_thread: job code <LOGIN_NAME_SERVER>"));
2698 			job->job_result = FC_SUCCESS;
2699 			fcsm_jobdone(job);
2700 			break;
2701 
2702 		case FCSM_JOB_LOGIN_MGMT_SERVER:
2703 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2704 			    "job_thread: job code <LOGIN_MGMT_SERVER>"));
2705 			fcsm_job_login_mgmt_server(job);
2706 			break;
2707 
2708 		case FCSM_JOB_CT_PASSTHRU:
2709 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2710 			    "job_thread: job code <CT_PASSTHRU>"));
2711 			fcsm_job_ct_passthru(job);
2712 			break;
2713 
2714 		default:
2715 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2716 			    "job_thread: job code <UNKNOWN>"));
2717 			job->job_result = FC_FAILURE;
2718 			fcsm_jobdone(job);
2719 			break;
2720 		}
2721 	}
2722 
2723 	/* NOTREACHED */
2724 }
2725 
2726 
2727 static void
2728 fcsm_ct_init(fcsm_t *fcsm, fcsm_cmd_t *cmd, fc_ct_aiu_t *req_iu, size_t req_len,
2729     void (*comp_func)())
2730 {
2731 	fc_packet_t	*pkt;
2732 
2733 	pkt = cmd->cmd_fp_pkt;
2734 	ASSERT(pkt != NULL);
2735 
2736 	ASSERT(req_iu->aiu_header.ct_fcstype == FCSTYPE_MGMTSERVICE ||
2737 	    (req_iu->aiu_header.ct_fcstype == FCSTYPE_DIRECTORY &&
2738 	    req_iu->aiu_header.ct_fcssubtype == FCSSUB_DS_NAME_SERVER));
2739 
2740 
2741 	/* Set the pkt d_id properly */
2742 	if (req_iu->aiu_header.ct_fcstype == FCSTYPE_MGMTSERVICE) {
2743 		pkt->pkt_cmd_fhdr.d_id	= FS_MANAGEMENT_SERVER;
2744 	} else {
2745 		pkt->pkt_cmd_fhdr.d_id	= FS_NAME_SERVER;
2746 	}
2747 
2748 	pkt->pkt_cmd_fhdr.r_ctl	= R_CTL_UNSOL_CONTROL;
2749 	pkt->pkt_cmd_fhdr.rsvd	= 0;
2750 	pkt->pkt_cmd_fhdr.s_id	= fcsm->sm_sid;
2751 	pkt->pkt_cmd_fhdr.type	= FC_TYPE_FC_SERVICES;
2752 	pkt->pkt_cmd_fhdr.f_ctl	= F_CTL_SEQ_INITIATIVE |
2753 	    F_CTL_FIRST_SEQ | F_CTL_END_SEQ;
2754 	pkt->pkt_cmd_fhdr.seq_id = 0;
2755 	pkt->pkt_cmd_fhdr.df_ctl = 0;
2756 	pkt->pkt_cmd_fhdr.seq_cnt = 0;
2757 	pkt->pkt_cmd_fhdr.ox_id = 0xffff;
2758 	pkt->pkt_cmd_fhdr.rx_id = 0xffff;
2759 	pkt->pkt_cmd_fhdr.ro	= 0;
2760 
2761 	pkt->pkt_timeout	= FCSM_MS_TIMEOUT;
2762 	pkt->pkt_comp		= comp_func;
2763 
2764 	FCSM_REP_WR(pkt->pkt_cmd_acc, req_iu, pkt->pkt_cmd, req_len);
2765 
2766 	cmd->cmd_transport = fc_ulp_transport;
2767 }
2768 
2769 static void
2770 fcsm_ct_intr(fcsm_cmd_t *cmd)
2771 {
2772 	fc_packet_t	*pkt;
2773 	fcsm_job_t	*job;
2774 	fcio_t		*fcio;
2775 	fcsm_t		*fcsm;
2776 
2777 	pkt = cmd->cmd_fp_pkt;
2778 	job = cmd->cmd_job;
2779 	ASSERT(job != NULL);
2780 
2781 	fcio = job->job_arg;
2782 	ASSERT(fcio != NULL);
2783 
2784 	if (pkt->pkt_state != FC_PKT_SUCCESS) {
2785 		FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, cmd->cmd_fcsm, pkt,
2786 		    "ct_intr: CT command <0x%x> to did 0x%x failed",
2787 		    ((fc_ct_aiu_t *)fcio->fcio_ibuf)->aiu_header.ct_cmdrsp,
2788 		    pkt->pkt_cmd_fhdr.d_id));
2789 	} else {
2790 		/* Get the CT response payload */
2791 		fcsm = cmd->cmd_fcsm;
2792 		FCSM_REP_RD(pkt->pkt_resp_acc, fcio->fcio_obuf,
2793 		    pkt->pkt_resp, fcio->fcio_olen);
2794 	}
2795 
2796 	job->job_result =
2797 	    fcsm_pkt_state_to_rval(pkt->pkt_state, pkt->pkt_reason);
2798 
2799 	fcsm_free_cmd(cmd);
2800 
2801 	fcsm_jobdone(job);
2802 }
2803 
2804 
2805 static void
2806 fcsm_job_ct_passthru(fcsm_job_t *job)
2807 {
2808 	fcsm_t		*fcsm;
2809 	fcio_t		*fcio;
2810 	fcsm_cmd_t	*cmd;
2811 	int		status;
2812 	fc_ct_header_t	*ct_header;
2813 
2814 	ASSERT(job != NULL);
2815 	ASSERT(job->job_port_instance != -1);
2816 
2817 	job->job_result = FC_FAILURE;
2818 	fcsm = ddi_get_soft_state(fcsm_state, job->job_port_instance);
2819 	if (fcsm == NULL) {
2820 		fcsm_jobdone(job);
2821 		return;
2822 	}
2823 
2824 	/*
2825 	 * Process the CT Passthru job only if port is attached
2826 	 * to a FABRIC.
2827 	 */
2828 	if (!FC_TOP_EXTERNAL(fcsm->sm_port_top)) {
2829 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2830 		    "job_ct_passthru: end (non-fabric port)"));
2831 		job->job_result = FC_BADDEV;
2832 		fcsm_jobdone(job);
2833 		return;
2834 	}
2835 
2836 	fcio = job->job_arg;
2837 	ASSERT(fcio != NULL);
2838 
2839 	/*
2840 	 * If it is NOT a Management Seriver (MS) or Name Server (NS) command
2841 	 * then complete the command with failure.
2842 	 */
2843 	ct_header = (fc_ct_header_t *)fcio->fcio_ibuf;
2844 
2845 	/*
2846 	 * According to libHBAAPI spec, CT header from libHBAAPI would always
2847 	 * be big endian, so we must swap CT header before continue in little
2848 	 * endian platforms.
2849 	 */
2850 	mutex_enter(&job->job_mutex);
2851 	if (!(job->job_flags & FCSM_JOBFLAG_CTHEADER_BE)) {
2852 		job->job_flags |= FCSM_JOBFLAG_CTHEADER_BE;
2853 		*((uint32_t *)((uint32_t *)ct_header + 0)) =
2854 		    BE_32(*((uint32_t *)((uint32_t *)ct_header + 0)));
2855 		*((uint32_t *)((uint32_t *)ct_header + 1)) =
2856 		    BE_32(*((uint32_t *)((uint32_t *)ct_header + 1)));
2857 		*((uint32_t *)((uint32_t *)ct_header + 2)) =
2858 		    BE_32(*((uint32_t *)((uint32_t *)ct_header + 2)));
2859 		*((uint32_t *)((uint32_t *)ct_header + 3)) =
2860 		    BE_32(*((uint32_t *)((uint32_t *)ct_header + 3)));
2861 	}
2862 	mutex_exit(&job->job_mutex);
2863 
2864 	if (ct_header->ct_fcstype == FCSTYPE_MGMTSERVICE) {
2865 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2866 		    "job_ct_passthru: Management Server Cmd"));
2867 	} else if (ct_header->ct_fcstype == FCSTYPE_DIRECTORY) {
2868 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2869 		    "job_ct_passthru: Name Server Cmd"));
2870 	} else {
2871 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2872 		    "job_ct_passthru: Unsupported Destination "
2873 		    "gs_type <0x%x> gs_subtype <0x%x>",
2874 		    ct_header->ct_fcstype, ct_header->ct_fcssubtype));
2875 	}
2876 
2877 	if (ct_header->ct_fcstype != FCSTYPE_MGMTSERVICE &&
2878 	    (ct_header->ct_fcstype != FCSTYPE_DIRECTORY ||
2879 	    ct_header->ct_fcssubtype != FCSSUB_DS_NAME_SERVER)) {
2880 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2881 		    "job_ct_passthru: end (Not a Name Server OR "
2882 		    "Mgmt Server Cmd)"));
2883 		job->job_result = FC_BADCMD;
2884 		fcsm_jobdone(job);
2885 		return;
2886 	}
2887 
2888 	/*
2889 	 * If it is an MS command and we are not logged in to the management
2890 	 * server, then start the login and requeue the command.
2891 	 * If login to management server is in progress, then reque the
2892 	 * command to wait for login to complete.
2893 	 */
2894 	mutex_enter(&fcsm->sm_mutex);
2895 	if ((ct_header->ct_fcstype == FCSTYPE_MGMTSERVICE) &&
2896 	    !(fcsm->sm_flags & FCSM_MGMT_SERVER_LOGGED_IN)) {
2897 		mutex_exit(&fcsm->sm_mutex);
2898 		if (fcsm_login_and_process_job(fcsm, job) != FC_SUCCESS) {
2899 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2900 			    "job_ct_passthru: perform login failed"));
2901 			job->job_result = FC_FAILURE;
2902 			fcsm_jobdone(job);
2903 		}
2904 		return;
2905 	}
2906 	mutex_exit(&fcsm->sm_mutex);
2907 
2908 	/*
2909 	 * We are already logged in to the management server.
2910 	 * Issue the CT Passthru command
2911 	 */
2912 	cmd = fcsm_alloc_cmd(fcsm, fcio->fcio_ilen, fcio->fcio_olen, KM_SLEEP);
2913 	if (cmd == NULL) {
2914 		job->job_result = FC_NOMEM;
2915 		fcsm_jobdone(job);
2916 		return;
2917 	}
2918 
2919 	FCSM_INIT_CMD(cmd, job, FC_TRAN_INTR | FC_TRAN_CLASS3, FC_PKT_EXCHANGE,
2920 	    fcsm_max_cmd_retries, fcsm_ct_intr);
2921 
2922 	fcsm_ct_init(fcsm, cmd, (fc_ct_aiu_t *)fcio->fcio_ibuf, fcio->fcio_ilen,
2923 	    fcsm_pkt_common_intr);
2924 
2925 	if ((status = fcsm_issue_cmd(cmd)) != FC_SUCCESS) {
2926 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, cmd->cmd_fcsm, NULL,
2927 		    "job_ct_passthru: issue CT Passthru failed, status 0x%x",
2928 		    status));
2929 		job->job_result = status;
2930 		fcsm_free_cmd(cmd);
2931 		fcsm_jobdone(job);
2932 		return;
2933 	}
2934 }
2935 
2936 static int
2937 fcsm_login_and_process_job(fcsm_t *fcsm, fcsm_job_t *orig_job)
2938 {
2939 	fcsm_job_t	*login_job;
2940 #ifdef DEBUG
2941 	int		status;
2942 #endif /* DEBUG */
2943 
2944 	if (orig_job->job_code != FCSM_JOB_CT_PASSTHRU) {
2945 		return (FC_FAILURE);
2946 	}
2947 
2948 	FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
2949 	    "login_and_process_job: start login."));
2950 
2951 	mutex_enter(&fcsm->sm_mutex);
2952 	if (fcsm->sm_flags & FCSM_MGMT_SERVER_LOGGED_IN) {
2953 		/*
2954 		 * Directory server login completed just now, while the
2955 		 * mutex was dropped. Just queue the command again for
2956 		 * processing.
2957 		 */
2958 		mutex_exit(&fcsm->sm_mutex);
2959 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2960 		    "login_and_process_job: got job 0x%p. login just "
2961 		    "completed", (void *)orig_job));
2962 		fcsm_enque_job(fcsm, orig_job, 0);
2963 		return (FC_SUCCESS);
2964 	}
2965 
2966 	if (fcsm->sm_flags & FCSM_MGMT_SERVER_LOGIN_IN_PROG) {
2967 		/*
2968 		 * Ideally we shouldn't have come here, since login
2969 		 * job has the serialize flag set.
2970 		 * Anyway, put the command back on the queue.
2971 		 */
2972 		mutex_exit(&fcsm->sm_mutex);
2973 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
2974 		    "login_and_process_job: got job 0x%p while login to "
2975 		    "management server in progress", (void *)orig_job));
2976 		fcsm_enque_job(fcsm, orig_job, 0);
2977 		return (FC_SUCCESS);
2978 	}
2979 
2980 	fcsm->sm_flags |= FCSM_MGMT_SERVER_LOGIN_IN_PROG;
2981 	mutex_exit(&fcsm->sm_mutex);
2982 
2983 	login_job = fcsm_alloc_job(KM_SLEEP);
2984 	ASSERT(login_job != NULL);
2985 
2986 	/*
2987 	 * Mark the login job as SERIALIZE, so that all other jobs will
2988 	 * be processed after completing the login.
2989 	 * Save the original job (CT Passthru job) in the caller private
2990 	 * field in the job structure, so that CT command can be issued
2991 	 * after login has completed.
2992 	 */
2993 	fcsm_init_job(login_job, fcsm->sm_instance, FCSM_JOB_LOGIN_MGMT_SERVER,
2994 	    FCSM_JOBFLAG_ASYNC | FCSM_JOBFLAG_SERIALIZE,
2995 	    (opaque_t)NULL, (opaque_t)orig_job, fcsm_login_ms_comp, NULL);
2996 	orig_job->job_priv = (void *)login_job;
2997 
2998 #ifdef DEBUG
2999 	status = fcsm_process_job(login_job, 1);
3000 	ASSERT(status == FC_SUCCESS);
3001 #else /* DEBUG */
3002 	(void) fcsm_process_job(login_job, 1);
3003 #endif /* DEBUG */
3004 	return (FC_SUCCESS);
3005 }
3006 
3007 
3008 /* ARGSUSED */
3009 static void
3010 fcsm_login_ms_comp(opaque_t comp_arg, fcsm_job_t *login_job, int result)
3011 {
3012 	fcsm_t		*fcsm;
3013 	fcsm_job_t	*orig_job;
3014 
3015 	ASSERT(login_job != NULL);
3016 
3017 	orig_job = (fcsm_job_t *)login_job->job_caller_priv;
3018 
3019 	ASSERT(orig_job != NULL);
3020 	ASSERT(orig_job->job_priv == (void *)login_job);
3021 	orig_job->job_priv = NULL;
3022 
3023 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
3024 	    "login_ms_comp: result 0x%x", login_job->job_result));
3025 
3026 	/* Set the login flag in the per port fcsm structure */
3027 	ASSERT(login_job->job_port_instance == orig_job->job_port_instance);
3028 	fcsm = ddi_get_soft_state(fcsm_state, login_job->job_port_instance);
3029 	ASSERT(fcsm != NULL);
3030 
3031 	mutex_enter(&fcsm->sm_mutex);
3032 	ASSERT((fcsm->sm_flags & FCSM_MGMT_SERVER_LOGGED_IN) == 0);
3033 	ASSERT(fcsm->sm_flags & FCSM_MGMT_SERVER_LOGIN_IN_PROG);
3034 	fcsm->sm_flags &= ~FCSM_MGMT_SERVER_LOGIN_IN_PROG;
3035 	if (login_job->job_result != FC_SUCCESS) {
3036 		caddr_t	msg;
3037 
3038 		/*
3039 		 * Login failed. Complete the original job with FC_LOGINREQ
3040 		 * status. Retry of that job will cause login to be
3041 		 * retried.
3042 		 */
3043 		mutex_exit(&fcsm->sm_mutex);
3044 		orig_job->job_result = FC_LOGINREQ;
3045 		fcsm_jobdone(orig_job);
3046 
3047 		(void) fc_ulp_error(login_job->job_result, &msg);
3048 		fcsm_display(CE_WARN, SM_LOG, fcsm, NULL,
3049 		    "login_ms_comp: Management server login failed: <%s>", msg);
3050 		return;
3051 	}
3052 	fcsm->sm_flags |= FCSM_MGMT_SERVER_LOGGED_IN;
3053 	mutex_exit(&fcsm->sm_mutex);
3054 
3055 	/*
3056 	 * Queue the original job at the head of the queue for processing.
3057 	 */
3058 	fcsm_enque_job(fcsm, orig_job, 1);
3059 }
3060 
3061 
3062 static void
3063 fcsm_els_init(fcsm_cmd_t *cmd, uint32_t d_id)
3064 {
3065 	fc_packet_t	*pkt;
3066 	fcsm_t		*fcsm;
3067 
3068 	fcsm = cmd->cmd_fcsm;
3069 	pkt = cmd->cmd_fp_pkt;
3070 	ASSERT(fcsm != NULL && pkt != NULL);
3071 
3072 	pkt->pkt_cmd_fhdr.r_ctl	= R_CTL_ELS_REQ;
3073 	pkt->pkt_cmd_fhdr.d_id	= d_id;
3074 	pkt->pkt_cmd_fhdr.rsvd	= 0;
3075 	pkt->pkt_cmd_fhdr.s_id	= fcsm->sm_sid;
3076 	pkt->pkt_cmd_fhdr.type	= FC_TYPE_EXTENDED_LS;
3077 	pkt->pkt_cmd_fhdr.f_ctl	= F_CTL_SEQ_INITIATIVE | F_CTL_FIRST_SEQ;
3078 	pkt->pkt_cmd_fhdr.seq_id = 0;
3079 	pkt->pkt_cmd_fhdr.df_ctl = 0;
3080 	pkt->pkt_cmd_fhdr.seq_cnt = 0;
3081 	pkt->pkt_cmd_fhdr.ox_id = 0xffff;
3082 	pkt->pkt_cmd_fhdr.rx_id = 0xffff;
3083 	pkt->pkt_cmd_fhdr.ro	= 0;
3084 
3085 	pkt->pkt_timeout	= FCSM_ELS_TIMEOUT;
3086 }
3087 
3088 
3089 static int
3090 fcsm_xlogi_init(fcsm_t *fcsm, fcsm_cmd_t *cmd, uint32_t d_id,
3091     void (*comp_func)(), uchar_t ls_code)
3092 {
3093 	ls_code_t	payload;
3094 	fc_packet_t	*pkt;
3095 	la_els_logi_t	*login_params;
3096 	int		status;
3097 
3098 	login_params = (la_els_logi_t *)
3099 	    kmem_zalloc(sizeof (la_els_logi_t), KM_SLEEP);
3100 	if (login_params == NULL) {
3101 		return (FC_NOMEM);
3102 	}
3103 
3104 	status = fc_ulp_get_port_login_params(fcsm->sm_port_info.port_handle,
3105 	    login_params);
3106 	if (status != FC_SUCCESS) {
3107 		kmem_free(login_params, sizeof (la_els_logi_t));
3108 		return (status);
3109 	}
3110 
3111 	pkt = cmd->cmd_fp_pkt;
3112 
3113 	fcsm_els_init(cmd, d_id);
3114 	pkt->pkt_comp = comp_func;
3115 
3116 	payload.ls_code = ls_code;
3117 	payload.mbz = 0;
3118 
3119 	FCSM_REP_WR(pkt->pkt_cmd_acc, login_params,
3120 	    pkt->pkt_cmd, sizeof (la_els_logi_t));
3121 	FCSM_REP_WR(pkt->pkt_cmd_acc, &payload,
3122 	    pkt->pkt_cmd, sizeof (payload));
3123 
3124 	cmd->cmd_transport = fc_ulp_issue_els;
3125 
3126 	kmem_free(login_params, sizeof (la_els_logi_t));
3127 
3128 	return (FC_SUCCESS);
3129 }
3130 
3131 static void
3132 fcsm_xlogi_intr(fcsm_cmd_t *cmd)
3133 {
3134 	fc_packet_t	*pkt;
3135 	fcsm_job_t	*job;
3136 	fcsm_t		*fcsm;
3137 
3138 	pkt = cmd->cmd_fp_pkt;
3139 	job = cmd->cmd_job;
3140 	ASSERT(job != NULL);
3141 
3142 	fcsm = cmd->cmd_fcsm;
3143 	ASSERT(fcsm != NULL);
3144 
3145 	if (pkt->pkt_state != FC_PKT_SUCCESS) {
3146 		fcsm_display(CE_WARN, SM_LOG, fcsm, pkt,
3147 		    "xlogi_intr: login to DID 0x%x failed",
3148 		    pkt->pkt_cmd_fhdr.d_id);
3149 	} else {
3150 		/* Get the Login parameters of the Management Server */
3151 		FCSM_REP_RD(pkt->pkt_resp_acc, &fcsm->sm_ms_service_params,
3152 		    pkt->pkt_resp, sizeof (la_els_logi_t));
3153 	}
3154 
3155 	job->job_result =
3156 	    fcsm_pkt_state_to_rval(pkt->pkt_state, pkt->pkt_reason);
3157 
3158 	fcsm_free_cmd(cmd);
3159 
3160 	fcsm_jobdone(job);
3161 }
3162 
3163 static void
3164 fcsm_job_login_mgmt_server(fcsm_job_t *job)
3165 {
3166 	fcsm_t		*fcsm;
3167 	fcsm_cmd_t	*cmd;
3168 	int		status;
3169 
3170 	ASSERT(job != NULL);
3171 	ASSERT(job->job_port_instance != -1);
3172 
3173 	fcsm = ddi_get_soft_state(fcsm_state, job->job_port_instance);
3174 	if (fcsm == NULL) {
3175 		job->job_result = FC_NOMEM;
3176 		fcsm_jobdone(job);
3177 		return;
3178 	}
3179 
3180 	/*
3181 	 * Issue the  Login command to the management server.
3182 	 */
3183 	cmd = fcsm_alloc_cmd(fcsm, sizeof (la_els_logi_t),
3184 	    sizeof (la_els_logi_t), KM_SLEEP);
3185 	if (cmd == NULL) {
3186 		job->job_result = FC_NOMEM;
3187 		fcsm_jobdone(job);
3188 		return;
3189 	}
3190 
3191 	FCSM_INIT_CMD(cmd, job, FC_TRAN_INTR | FC_TRAN_CLASS3, FC_PKT_EXCHANGE,
3192 	    fcsm_max_cmd_retries, fcsm_xlogi_intr);
3193 
3194 	status = fcsm_xlogi_init(fcsm, cmd, FS_MANAGEMENT_SERVER,
3195 	    fcsm_pkt_common_intr, LA_ELS_PLOGI);
3196 
3197 	if (status != FC_SUCCESS) {
3198 		FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
3199 		    "job_login_mgmt_server: plogi init failed. status 0x%x",
3200 		    status));
3201 		job->job_result = status;
3202 		fcsm_free_cmd(cmd);
3203 		fcsm_jobdone(job);
3204 		return;
3205 	}
3206 
3207 	if ((status = fcsm_issue_cmd(cmd)) != FC_SUCCESS) {
3208 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, cmd->cmd_fcsm, NULL,
3209 		    "job_ct_passthru: issue login cmd failed, status 0x%x",
3210 		    status));
3211 		job->job_result = status;
3212 		fcsm_free_cmd(cmd);
3213 		fcsm_jobdone(job);
3214 		return;
3215 	}
3216 }
3217 
3218 
3219 int
3220 fcsm_ct_passthru(int instance, fcio_t *fcio, int sleep, int job_flags,
3221     void (*func)(fcio_t *))
3222 {
3223 	fcsm_job_t	*job;
3224 	int		status;
3225 
3226 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
3227 	    "ct_passthru: instance 0x%x fcio 0x%p", instance, fcio));
3228 	job = fcsm_alloc_job(sleep);
3229 	ASSERT(sleep == KM_NOSLEEP || job != NULL);
3230 
3231 	fcsm_init_job(job, instance, FCSM_JOB_CT_PASSTHRU, job_flags,
3232 	    (opaque_t)fcio, (opaque_t)func, fcsm_ct_passthru_comp, NULL);
3233 	status = fcsm_process_job(job, 0);
3234 	if (status != FC_SUCCESS) {
3235 		/* Job could not be issued. So free the job and return */
3236 		fcsm_dealloc_job(job);
3237 		return (status);
3238 	}
3239 
3240 	if (job_flags & FCSM_JOBFLAG_SYNC) {
3241 		status = job->job_result;
3242 		fcsm_dealloc_job(job);
3243 	}
3244 
3245 	return (status);
3246 }
3247 
3248 
3249 /* ARGSUSED */
3250 static void
3251 fcsm_ct_passthru_comp(opaque_t comp_arg, fcsm_job_t *job, int result)
3252 {
3253 	ASSERT(job != NULL);
3254 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
3255 	    "ct_passthru_comp: result 0x%x port 0x%x",
3256 	    job->job_result, job->job_port_instance));
3257 }
3258 
3259 
3260 static void
3261 fcsm_pkt_common_intr(fc_packet_t *pkt)
3262 {
3263 	fcsm_cmd_t	*cmd;
3264 	int		jobstatus;
3265 	fcsm_t		*fcsm;
3266 
3267 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, NULL, NULL,
3268 	    "pkt_common_intr"));
3269 
3270 	cmd = (fcsm_cmd_t *)pkt->pkt_ulp_private;
3271 	ASSERT(cmd != NULL);
3272 
3273 	if (pkt->pkt_state == FC_PKT_SUCCESS) {
3274 		/* Command completed successfully. Just complete the command */
3275 		cmd->cmd_comp(cmd);
3276 		return;
3277 	}
3278 
3279 	fcsm = cmd->cmd_fcsm;
3280 	ASSERT(fcsm != NULL);
3281 
3282 	FCSM_DEBUG(SMDL_ERR, (CE_NOTE, SM_LOG, cmd->cmd_fcsm, pkt,
3283 	    "fc packet to DID 0x%x failed for pkt 0x%p",
3284 	    pkt->pkt_cmd_fhdr.d_id, pkt));
3285 
3286 	mutex_enter(&fcsm->sm_mutex);
3287 	if (fcsm->sm_flags & FCSM_LINK_DOWN) {
3288 		/*
3289 		 * No need to retry the command. The link previously
3290 		 * suffered an offline	timeout.
3291 		 */
3292 		mutex_exit(&fcsm->sm_mutex);
3293 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, cmd->cmd_fcsm, NULL,
3294 		    "pkt_common_intr: end. Link is down"));
3295 		cmd->cmd_comp(cmd);
3296 		return;
3297 	}
3298 	mutex_exit(&fcsm->sm_mutex);
3299 
3300 	jobstatus = fcsm_pkt_state_to_rval(pkt->pkt_state, pkt->pkt_reason);
3301 	if (jobstatus == FC_LOGINREQ) {
3302 		/*
3303 		 * Login to the destination is required. No need to
3304 		 * retry this cmd again.
3305 		 */
3306 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, cmd->cmd_fcsm, NULL,
3307 		    "pkt_common_intr: end. LOGIN required"));
3308 		cmd->cmd_comp(cmd);
3309 		return;
3310 	}
3311 
3312 	switch (pkt->pkt_state) {
3313 	case FC_PKT_PORT_OFFLINE:
3314 	case FC_PKT_LOCAL_RJT:
3315 	case FC_PKT_TIMEOUT: {
3316 		uchar_t		pkt_state;
3317 
3318 		pkt_state = pkt->pkt_state;
3319 		cmd->cmd_retry_interval = fcsm_retry_interval;
3320 		if (fcsm_retry_cmd(cmd) != 0) {
3321 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG,
3322 			    cmd->cmd_fcsm, NULL,
3323 			    "common_intr: max retries(%d) reached, status 0x%x",
3324 			    cmd->cmd_retry_count));
3325 
3326 			/*
3327 			 * Restore the pkt_state to the actual failure status
3328 			 * received at the time of pkt completion.
3329 			 */
3330 			pkt->pkt_state = pkt_state;
3331 			pkt->pkt_reason = 0;
3332 			cmd->cmd_comp(cmd);
3333 		} else {
3334 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG,
3335 			    cmd->cmd_fcsm, NULL,
3336 			    "pkt_common_intr: retry(%d) on pkt state (0x%x)",
3337 			    cmd->cmd_retry_count, pkt_state));
3338 		}
3339 		break;
3340 	}
3341 	default:
3342 		cmd->cmd_comp(cmd);
3343 		break;
3344 	}
3345 }
3346 
3347 static int
3348 fcsm_issue_cmd(fcsm_cmd_t *cmd)
3349 {
3350 	fc_packet_t	*pkt;
3351 	fcsm_t		*fcsm;
3352 	int		status;
3353 
3354 	pkt = cmd->cmd_fp_pkt;
3355 	fcsm = cmd->cmd_fcsm;
3356 
3357 	/* Explicitly invalidate this field till fcsm decides to use it */
3358 	pkt->pkt_ulp_rscn_infop = NULL;
3359 
3360 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3361 	    "issue_cmd: entry"));
3362 
3363 	ASSERT(!MUTEX_HELD(&fcsm->sm_mutex));
3364 	mutex_enter(&fcsm->sm_mutex);
3365 	if (fcsm->sm_flags & FCSM_LINK_DOWN) {
3366 		/*
3367 		 * Update the pkt_state/pkt_reason appropriately.
3368 		 * Caller of this function can decide whether to call
3369 		 * 'pkt->pkt_comp' or use the 'status' returned by this func.
3370 		 */
3371 		mutex_exit(&fcsm->sm_mutex);
3372 		pkt->pkt_state = FC_PKT_PORT_OFFLINE;
3373 		pkt->pkt_reason = FC_REASON_OFFLINE;
3374 		return (FC_OFFLINE);
3375 	}
3376 	mutex_exit(&fcsm->sm_mutex);
3377 
3378 	ASSERT(cmd->cmd_transport != NULL);
3379 	status = cmd->cmd_transport(fcsm->sm_port_info.port_handle, pkt);
3380 	if (status != FC_SUCCESS) {
3381 		switch (status) {
3382 		case FC_LOGINREQ:
3383 			/*
3384 			 * No need to retry. Return the cause of failure.
3385 			 * Also update the pkt_state/pkt_reason. Caller of
3386 			 * this function can decide, whether to call
3387 			 * 'pkt->pkt_comp' or use the 'status' code returned
3388 			 * by this function.
3389 			 */
3390 			pkt->pkt_state = FC_PKT_LOCAL_RJT;
3391 			pkt->pkt_reason = FC_REASON_LOGIN_REQUIRED;
3392 			break;
3393 
3394 		case FC_DEVICE_BUSY_NEW_RSCN:
3395 			/*
3396 			 * There was a newer RSCN than what fcsm knows about.
3397 			 * So, just retry again
3398 			 */
3399 			cmd->cmd_retry_count = 0;
3400 			/*FALLTHROUGH*/
3401 		case FC_OFFLINE:
3402 		case FC_STATEC_BUSY:
3403 			/*
3404 			 * TODO: set flag, so that command is retried after
3405 			 * port is back online.
3406 			 * FALL Through for now.
3407 			 */
3408 
3409 		case FC_TRAN_BUSY:
3410 		case FC_NOMEM:
3411 		case FC_DEVICE_BUSY:
3412 			cmd->cmd_retry_interval = fcsm_retry_interval;
3413 			if (fcsm_retry_cmd(cmd) != 0) {
3414 				FCSM_DEBUG(SMDL_TRACE,
3415 				    (CE_WARN, SM_LOG, fcsm, NULL,
3416 				    "issue_cmd: max retries (%d) reached",
3417 				    cmd->cmd_retry_count));
3418 
3419 				/*
3420 				 * status variable is not changed here.
3421 				 * Return the cause of the original
3422 				 * cmd_transport failure.
3423 				 * Update the pkt_state/pkt_reason. Caller
3424 				 * of this function can decide whether to
3425 				 * call 'pkt->pkt_comp' or use the 'status'
3426 				 * code returned by this function.
3427 				 */
3428 				pkt->pkt_state = FC_PKT_TRAN_BSY;
3429 				pkt->pkt_reason = 0;
3430 			} else {
3431 				FCSM_DEBUG(SMDL_TRACE,
3432 				    (CE_WARN, SM_LOG, fcsm, NULL,
3433 				    "issue_cmd: retry (%d) on fc status (0x%x)",
3434 				    cmd->cmd_retry_count, status));
3435 
3436 				status = FC_SUCCESS;
3437 			}
3438 			break;
3439 
3440 		default:
3441 			FCSM_DEBUG(SMDL_TRACE, (CE_WARN, SM_LOG, fcsm, NULL,
3442 			    "issue_cmd: failure status 0x%x", status));
3443 
3444 			pkt->pkt_state = FC_PKT_TRAN_ERROR;
3445 			pkt->pkt_reason = 0;
3446 			break;
3447 
3448 
3449 		}
3450 	}
3451 
3452 	return (status);
3453 }
3454 
3455 
3456 static int
3457 fcsm_retry_cmd(fcsm_cmd_t *cmd)
3458 {
3459 	if (cmd->cmd_retry_count < cmd->cmd_max_retries) {
3460 		cmd->cmd_retry_count++;
3461 		fcsm_enque_cmd(cmd->cmd_fcsm, cmd);
3462 		return (0);
3463 	}
3464 
3465 	return (1);
3466 }
3467 
3468 static void
3469 fcsm_enque_cmd(fcsm_t *fcsm, fcsm_cmd_t *cmd)
3470 {
3471 	ASSERT(!MUTEX_HELD(&fcsm->sm_mutex));
3472 
3473 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL, "enque_cmd"));
3474 
3475 	cmd->cmd_next = NULL;
3476 	mutex_enter(&fcsm->sm_mutex);
3477 	if (fcsm->sm_retry_tail) {
3478 		ASSERT(fcsm->sm_retry_head != NULL);
3479 		fcsm->sm_retry_tail->cmd_next = cmd;
3480 		fcsm->sm_retry_tail = cmd;
3481 	} else {
3482 		ASSERT(fcsm->sm_retry_tail == NULL);
3483 		fcsm->sm_retry_head = fcsm->sm_retry_tail = cmd;
3484 
3485 		/* Schedule retry thread, if not already running */
3486 		if (fcsm->sm_retry_tid == NULL) {
3487 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3488 			    "enque_cmd: schedule retry thread"));
3489 			fcsm->sm_retry_tid = timeout(fcsm_retry_timeout,
3490 			    (caddr_t)fcsm, fcsm_retry_ticks);
3491 		}
3492 	}
3493 	mutex_exit(&fcsm->sm_mutex);
3494 }
3495 
3496 
3497 static fcsm_cmd_t *
3498 fcsm_deque_cmd(fcsm_t *fcsm)
3499 {
3500 	fcsm_cmd_t	*cmd;
3501 
3502 	ASSERT(!MUTEX_HELD(&fcsm->sm_mutex));
3503 
3504 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL, "deque_cmd"));
3505 
3506 	mutex_enter(&fcsm->sm_mutex);
3507 	if (fcsm->sm_retry_head == NULL) {
3508 		ASSERT(fcsm->sm_retry_tail == NULL);
3509 		cmd = NULL;
3510 	} else {
3511 		cmd = fcsm->sm_retry_head;
3512 		fcsm->sm_retry_head = cmd->cmd_next;
3513 		if (fcsm->sm_retry_head == NULL) {
3514 			fcsm->sm_retry_tail = NULL;
3515 		}
3516 		cmd->cmd_next = NULL;
3517 	}
3518 	mutex_exit(&fcsm->sm_mutex);
3519 
3520 	return (cmd);
3521 }
3522 
3523 static void
3524 fcsm_retry_timeout(void *handle)
3525 {
3526 	fcsm_t		*fcsm;
3527 	fcsm_cmd_t	*curr_tail;
3528 	fcsm_cmd_t	*cmd;
3529 	int		done = 0;
3530 	int		linkdown;
3531 
3532 	fcsm = (fcsm_t *)handle;
3533 
3534 	FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL, "retry_timeout"));
3535 
3536 	/*
3537 	 * If retry cmd queue is suspended, then go away.
3538 	 * This retry thread will be restarted, when cmd queue resumes.
3539 	 */
3540 	mutex_enter(&fcsm->sm_mutex);
3541 	if (fcsm->sm_flags & FCSM_CMD_RETRY_Q_SUSPENDED) {
3542 		/*
3543 		 * Clear the retry_tid, to indicate that this routine is not
3544 		 * currently being rescheduled.
3545 		 */
3546 		fcsm->sm_retry_tid = (timeout_id_t)NULL;
3547 		mutex_exit(&fcsm->sm_mutex);
3548 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3549 		    "retry_timeout: end. No processing. "
3550 		    "Queue is currently suspended for this instance"));
3551 		return;
3552 	}
3553 
3554 	linkdown = (fcsm->sm_flags & FCSM_LINK_DOWN) ? 1 : 0;
3555 
3556 	/*
3557 	 * Save the curr_tail, so that we only process the commands
3558 	 * which are in the queue at this time.
3559 	 */
3560 	curr_tail = fcsm->sm_retry_tail;
3561 	mutex_exit(&fcsm->sm_mutex);
3562 
3563 	/*
3564 	 * Check for done flag before dequeing the command.
3565 	 * Dequeing before checking the done flag will cause a command
3566 	 * to be lost.
3567 	 */
3568 	while ((!done) && ((cmd = fcsm_deque_cmd(fcsm)) != NULL)) {
3569 
3570 		if (cmd == curr_tail) {
3571 			done = 1;
3572 		}
3573 
3574 		cmd->cmd_retry_interval -= fcsm_retry_ticker;
3575 
3576 		if (linkdown) {
3577 			fc_packet_t *pkt;
3578 
3579 			/*
3580 			 * No need to retry the command. The link has
3581 			 * suffered an offline	timeout.
3582 			 */
3583 			pkt = cmd->cmd_fp_pkt;
3584 			pkt->pkt_state = FC_PKT_PORT_OFFLINE;
3585 			pkt->pkt_reason = FC_REASON_OFFLINE;
3586 			pkt->pkt_comp(pkt);
3587 			continue;
3588 		}
3589 
3590 		if (cmd->cmd_retry_interval <= 0) {
3591 			/* Retry the command */
3592 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3593 			    "retry_timeout: issue cmd 0x%p", (void *)cmd));
3594 			if (fcsm_issue_cmd(cmd) != FC_SUCCESS) {
3595 				cmd->cmd_fp_pkt->pkt_comp(cmd->cmd_fp_pkt);
3596 			}
3597 		} else {
3598 			/*
3599 			 * Put the command back on the queue. Retry time
3600 			 * has not yet reached.
3601 			 */
3602 			FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3603 			    "retry_timeout: queue cmd 0x%p", (void *)cmd));
3604 			fcsm_enque_cmd(fcsm, cmd);
3605 		}
3606 	}
3607 
3608 	mutex_enter(&fcsm->sm_mutex);
3609 	if (fcsm->sm_retry_head) {
3610 		/* Activate timer */
3611 		fcsm->sm_retry_tid = timeout(fcsm_retry_timeout,
3612 		    (caddr_t)fcsm, fcsm_retry_ticks);
3613 		FCSM_DEBUG(SMDL_TRACE, (CE_CONT, SM_LOG, fcsm, NULL,
3614 		    "retry_timeout: retry thread rescheduled"));
3615 	} else {
3616 		/*
3617 		 * Reset the tid variable. The first thread which queues the
3618 		 * command, will restart the timer.
3619 		 */
3620 		fcsm->sm_retry_tid = (timeout_id_t)NULL;
3621 	}
3622 	mutex_exit(&fcsm->sm_mutex);
3623 }
3624