xref: /freebsd/sys/dev/isci/isci_controller.c (revision f0cfa1b168014f56c02b83e5f28412cc5f78d117)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * BSD LICENSE
5  *
6  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  *   * Redistributions of source code must retain the above copyright
14  *     notice, this list of conditions and the following disclaimer.
15  *   * Redistributions in binary form must reproduce the above copyright
16  *     notice, this list of conditions and the following disclaimer in
17  *     the documentation and/or other materials provided with the
18  *     distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 
36 #include <dev/isci/isci.h>
37 
38 #include <sys/conf.h>
39 #include <sys/malloc.h>
40 
41 #include <cam/cam_periph.h>
42 #include <cam/cam_xpt_periph.h>
43 
44 #include <dev/isci/scil/sci_memory_descriptor_list.h>
45 #include <dev/isci/scil/sci_memory_descriptor_list_decorator.h>
46 
47 #include <dev/isci/scil/scif_controller.h>
48 #include <dev/isci/scil/scif_library.h>
49 #include <dev/isci/scil/scif_io_request.h>
50 #include <dev/isci/scil/scif_task_request.h>
51 #include <dev/isci/scil/scif_remote_device.h>
52 #include <dev/isci/scil/scif_domain.h>
53 #include <dev/isci/scil/scif_user_callback.h>
54 #include <dev/isci/scil/scic_sgpio.h>
55 
56 #include <dev/led/led.h>
57 
58 void isci_action(struct cam_sim *sim, union ccb *ccb);
59 void isci_poll(struct cam_sim *sim);
60 
61 #define ccb_sim_ptr sim_priv.entries[0].ptr
62 
63 /**
64  * @brief This user callback will inform the user that the controller has
65  *        had a serious unexpected error.  The user should not the error,
66  *        disable interrupts, and wait for current ongoing processing to
67  *        complete.  Subsequently, the user should reset the controller.
68  *
69  * @param[in]  controller This parameter specifies the controller that had
70  *                        an error.
71  *
72  * @return none
73  */
74 void scif_cb_controller_error(SCI_CONTROLLER_HANDLE_T controller,
75     SCI_CONTROLLER_ERROR error)
76 {
77 
78 	isci_log_message(0, "ISCI", "scif_cb_controller_error: 0x%x\n",
79 	    error);
80 }
81 
82 /**
83  * @brief This user callback will inform the user that the controller has
84  *        finished the start process.
85  *
86  * @param[in]  controller This parameter specifies the controller that was
87  *             started.
88  * @param[in]  completion_status This parameter specifies the results of
89  *             the start operation.  SCI_SUCCESS indicates successful
90  *             completion.
91  *
92  * @return none
93  */
94 void scif_cb_controller_start_complete(SCI_CONTROLLER_HANDLE_T controller,
95     SCI_STATUS completion_status)
96 {
97 	uint32_t index;
98 	struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
99 	    sci_object_get_association(controller);
100 
101 	isci_controller->is_started = TRUE;
102 
103 	/* Set bits for all domains.  We will clear them one-by-one once
104 	 *  the domains complete discovery, or return error when calling
105 	 *  scif_domain_discover.  Once all bits are clear, we will register
106 	 *  the controller with CAM.
107 	 */
108 	isci_controller->initial_discovery_mask = (1 << SCI_MAX_DOMAINS) - 1;
109 
110 	for(index = 0; index < SCI_MAX_DOMAINS; index++) {
111 		SCI_STATUS status;
112 		SCI_DOMAIN_HANDLE_T domain =
113 		    isci_controller->domain[index].sci_object;
114 
115 		status = scif_domain_discover(
116 			domain,
117 			scif_domain_get_suggested_discover_timeout(domain),
118 			DEVICE_TIMEOUT
119 		);
120 
121 		if (status != SCI_SUCCESS)
122 		{
123 			isci_controller_domain_discovery_complete(
124 			    isci_controller, &isci_controller->domain[index]);
125 		}
126 	}
127 }
128 
129 /**
130  * @brief This user callback will inform the user that the controller has
131  *        finished the stop process. Note, after user calls
132  *        scif_controller_stop(), before user receives this controller stop
133  *        complete callback, user should not expect any callback from
134  *        framework, such like scif_cb_domain_change_notification().
135  *
136  * @param[in]  controller This parameter specifies the controller that was
137  *             stopped.
138  * @param[in]  completion_status This parameter specifies the results of
139  *             the stop operation.  SCI_SUCCESS indicates successful
140  *             completion.
141  *
142  * @return none
143  */
144 void scif_cb_controller_stop_complete(SCI_CONTROLLER_HANDLE_T controller,
145     SCI_STATUS completion_status)
146 {
147 	struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
148 	    sci_object_get_association(controller);
149 
150 	isci_controller->is_started = FALSE;
151 }
152 
153 static void
154 isci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
155 {
156 	SCI_PHYSICAL_ADDRESS *phys_addr = arg;
157 
158 	*phys_addr = seg[0].ds_addr;
159 }
160 
161 /**
162  * @brief This method will be invoked to allocate memory dynamically.
163  *
164  * @param[in]  controller This parameter represents the controller
165  *             object for which to allocate memory.
166  * @param[out] mde This parameter represents the memory descriptor to
167  *             be filled in by the user that will reference the newly
168  *             allocated memory.
169  *
170  * @return none
171  */
172 void scif_cb_controller_allocate_memory(SCI_CONTROLLER_HANDLE_T controller,
173     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T *mde)
174 {
175 	struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
176 	    sci_object_get_association(controller);
177 
178 	/*
179 	 * Note this routine is only used for buffers needed to translate
180 	 * SCSI UNMAP commands to ATA DSM commands for SATA disks.
181 	 *
182 	 * We first try to pull a buffer from the controller's pool, and only
183 	 * call contigmalloc if one isn't there.
184 	 */
185 	if (!sci_pool_empty(isci_controller->unmap_buffer_pool)) {
186 		sci_pool_get(isci_controller->unmap_buffer_pool,
187 		    mde->virtual_address);
188 	} else
189 		mde->virtual_address = contigmalloc(PAGE_SIZE,
190 		    M_ISCI, M_NOWAIT, 0, BUS_SPACE_MAXADDR,
191 		    mde->constant_memory_alignment, 0);
192 
193 	if (mde->virtual_address != NULL)
194 		bus_dmamap_load(isci_controller->buffer_dma_tag,
195 		    NULL, mde->virtual_address, PAGE_SIZE,
196 		    isci_single_map, &mde->physical_address,
197 		    BUS_DMA_NOWAIT);
198 }
199 
200 /**
201  * @brief This method will be invoked to allocate memory dynamically.
202  *
203  * @param[in]  controller This parameter represents the controller
204  *             object for which to allocate memory.
205  * @param[out] mde This parameter represents the memory descriptor to
206  *             be filled in by the user that will reference the newly
207  *             allocated memory.
208  *
209  * @return none
210  */
211 void scif_cb_controller_free_memory(SCI_CONTROLLER_HANDLE_T controller,
212     SCI_PHYSICAL_MEMORY_DESCRIPTOR_T * mde)
213 {
214 	struct ISCI_CONTROLLER *isci_controller = (struct ISCI_CONTROLLER *)
215 	    sci_object_get_association(controller);
216 
217 	/*
218 	 * Put the buffer back into the controller's buffer pool, rather
219 	 * than invoking configfree.  This helps reduce chance we won't
220 	 * have buffers available when system is under memory pressure.
221 	 */
222 	sci_pool_put(isci_controller->unmap_buffer_pool,
223 	    mde->virtual_address);
224 }
225 
226 void isci_controller_construct(struct ISCI_CONTROLLER *controller,
227     struct isci_softc *isci)
228 {
229 	SCI_CONTROLLER_HANDLE_T scif_controller_handle;
230 
231 	scif_library_allocate_controller(isci->sci_library_handle,
232 	    &scif_controller_handle);
233 
234 	scif_controller_construct(isci->sci_library_handle,
235 	    scif_controller_handle, NULL);
236 
237 	controller->isci = isci;
238 	controller->scif_controller_handle = scif_controller_handle;
239 
240 	/* This allows us to later use
241 	 *  sci_object_get_association(scif_controller_handle)
242 	 * inside of a callback routine to get our struct ISCI_CONTROLLER object
243 	 */
244 	sci_object_set_association(scif_controller_handle, (void *)controller);
245 
246 	controller->is_started = FALSE;
247 	controller->is_frozen = FALSE;
248 	controller->release_queued_ccbs = FALSE;
249 	controller->sim = NULL;
250 	controller->initial_discovery_mask = 0;
251 
252 	sci_fast_list_init(&controller->pending_device_reset_list);
253 
254 	mtx_init(&controller->lock, "isci", NULL, MTX_DEF);
255 
256 	uint32_t domain_index;
257 
258 	for(domain_index = 0; domain_index < SCI_MAX_DOMAINS; domain_index++) {
259 		isci_domain_construct( &controller->domain[domain_index],
260 		    domain_index, controller);
261 	}
262 
263 	controller->timer_memory = malloc(
264 	    sizeof(struct ISCI_TIMER) * SCI_MAX_TIMERS, M_ISCI,
265 	    M_NOWAIT | M_ZERO);
266 
267 	sci_pool_initialize(controller->timer_pool);
268 
269 	struct ISCI_TIMER *timer = (struct ISCI_TIMER *)
270 	    controller->timer_memory;
271 
272 	for ( int i = 0; i < SCI_MAX_TIMERS; i++ ) {
273 		sci_pool_put(controller->timer_pool, timer++);
274 	}
275 
276 	sci_pool_initialize(controller->unmap_buffer_pool);
277 }
278 
279 static void isci_led_fault_func(void *priv, int onoff)
280 {
281 	struct ISCI_PHY *phy = priv;
282 
283 	/* map onoff to the fault LED */
284 	phy->led_fault = onoff;
285 	scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
286 		phy->led_fault, phy->led_locate, 0);
287 }
288 
289 static void isci_led_locate_func(void *priv, int onoff)
290 {
291 	struct ISCI_PHY *phy = priv;
292 
293 	/* map onoff to the locate LED */
294 	phy->led_locate = onoff;
295 	scic_sgpio_update_led_state(phy->handle, 1 << phy->index,
296 		phy->led_fault, phy->led_locate, 0);
297 }
298 
299 SCI_STATUS isci_controller_initialize(struct ISCI_CONTROLLER *controller)
300 {
301 	SCIC_USER_PARAMETERS_T scic_user_parameters;
302 	SCI_CONTROLLER_HANDLE_T scic_controller_handle;
303 	char led_name[64];
304 	unsigned long tunable;
305 	uint32_t io_shortage;
306 	uint32_t fail_on_timeout;
307 	int i;
308 
309 	scic_controller_handle =
310 	    scif_controller_get_scic_handle(controller->scif_controller_handle);
311 
312 	if (controller->isci->oem_parameters_found == TRUE)
313 	{
314 		scic_oem_parameters_set(
315 		    scic_controller_handle,
316 		    &controller->oem_parameters,
317 		    (uint8_t)(controller->oem_parameters_version));
318 	}
319 
320 	scic_user_parameters_get(scic_controller_handle, &scic_user_parameters);
321 
322 	if (TUNABLE_ULONG_FETCH("hw.isci.no_outbound_task_timeout", &tunable))
323 		scic_user_parameters.sds1.no_outbound_task_timeout =
324 		    (uint8_t)tunable;
325 
326 	if (TUNABLE_ULONG_FETCH("hw.isci.ssp_max_occupancy_timeout", &tunable))
327 		scic_user_parameters.sds1.ssp_max_occupancy_timeout =
328 		    (uint16_t)tunable;
329 
330 	if (TUNABLE_ULONG_FETCH("hw.isci.stp_max_occupancy_timeout", &tunable))
331 		scic_user_parameters.sds1.stp_max_occupancy_timeout =
332 		    (uint16_t)tunable;
333 
334 	if (TUNABLE_ULONG_FETCH("hw.isci.ssp_inactivity_timeout", &tunable))
335 		scic_user_parameters.sds1.ssp_inactivity_timeout =
336 		    (uint16_t)tunable;
337 
338 	if (TUNABLE_ULONG_FETCH("hw.isci.stp_inactivity_timeout", &tunable))
339 		scic_user_parameters.sds1.stp_inactivity_timeout =
340 		    (uint16_t)tunable;
341 
342 	if (TUNABLE_ULONG_FETCH("hw.isci.max_speed_generation", &tunable))
343 		for (i = 0; i < SCI_MAX_PHYS; i++)
344 			scic_user_parameters.sds1.phys[i].max_speed_generation =
345 			    (uint8_t)tunable;
346 
347 	scic_user_parameters_set(scic_controller_handle, &scic_user_parameters);
348 
349 	/* Scheduler bug in SCU requires SCIL to reserve some task contexts as a
350 	 *  a workaround - one per domain.
351 	 */
352 	controller->queue_depth = SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS;
353 
354 	if (TUNABLE_INT_FETCH("hw.isci.controller_queue_depth",
355 	    &controller->queue_depth)) {
356 		controller->queue_depth = max(1, min(controller->queue_depth,
357 		    SCI_MAX_IO_REQUESTS - SCI_MAX_DOMAINS));
358 	}
359 
360 	/* Reserve one request so that we can ensure we have one available TC
361 	 *  to do internal device resets.
362 	 */
363 	controller->sim_queue_depth = controller->queue_depth - 1;
364 
365 	/* Although we save one TC to do internal device resets, it is possible
366 	 *  we could end up using several TCs for simultaneous device resets
367 	 *  while at the same time having CAM fill our controller queue.  To
368 	 *  simulate this condition, and how our driver handles it, we can set
369 	 *  this io_shortage parameter, which will tell CAM that we have a
370 	 *  large queue depth than we really do.
371 	 */
372 	io_shortage = 0;
373 	TUNABLE_INT_FETCH("hw.isci.io_shortage", &io_shortage);
374 	controller->sim_queue_depth += io_shortage;
375 
376 	fail_on_timeout = 1;
377 	TUNABLE_INT_FETCH("hw.isci.fail_on_task_timeout", &fail_on_timeout);
378 	controller->fail_on_task_timeout = fail_on_timeout;
379 
380 	/* Attach to CAM using xpt_bus_register now, then immediately freeze
381 	 *  the simq.  It will get released later when initial domain discovery
382 	 *  is complete.
383 	 */
384 	controller->has_been_scanned = FALSE;
385 	mtx_lock(&controller->lock);
386 	isci_controller_attach_to_cam(controller);
387 	xpt_freeze_simq(controller->sim, 1);
388 	mtx_unlock(&controller->lock);
389 
390 	for (i = 0; i < SCI_MAX_PHYS; i++) {
391 		controller->phys[i].handle = scic_controller_handle;
392 		controller->phys[i].index = i;
393 
394 		/* fault */
395 		controller->phys[i].led_fault = 0;
396 		sprintf(led_name, "isci.bus%d.port%d.fault", controller->index, i);
397 		controller->phys[i].cdev_fault = led_create(isci_led_fault_func,
398 		    &controller->phys[i], led_name);
399 
400 		/* locate */
401 		controller->phys[i].led_locate = 0;
402 		sprintf(led_name, "isci.bus%d.port%d.locate", controller->index, i);
403 		controller->phys[i].cdev_locate = led_create(isci_led_locate_func,
404 		    &controller->phys[i], led_name);
405 	}
406 
407 	return (scif_controller_initialize(controller->scif_controller_handle));
408 }
409 
410 int isci_controller_allocate_memory(struct ISCI_CONTROLLER *controller)
411 {
412 	int error;
413 	device_t device =  controller->isci->device;
414 	uint32_t max_segment_size = isci_io_request_get_max_io_size();
415 	uint32_t status = 0;
416 	struct ISCI_MEMORY *uncached_controller_memory =
417 	    &controller->uncached_controller_memory;
418 	struct ISCI_MEMORY *cached_controller_memory =
419 	    &controller->cached_controller_memory;
420 	struct ISCI_MEMORY *request_memory =
421 	    &controller->request_memory;
422 	POINTER_UINT virtual_address;
423 	bus_addr_t physical_address;
424 
425 	controller->mdl = sci_controller_get_memory_descriptor_list_handle(
426 	    controller->scif_controller_handle);
427 
428 	uncached_controller_memory->size = sci_mdl_decorator_get_memory_size(
429 	    controller->mdl, SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS);
430 
431 	error = isci_allocate_dma_buffer(device, uncached_controller_memory);
432 
433 	if (error != 0)
434 	    return (error);
435 
436 	sci_mdl_decorator_assign_memory( controller->mdl,
437 	    SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
438 	    uncached_controller_memory->virtual_address,
439 	    uncached_controller_memory->physical_address);
440 
441 	cached_controller_memory->size = sci_mdl_decorator_get_memory_size(
442 	    controller->mdl,
443 	    SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS
444 	);
445 
446 	error = isci_allocate_dma_buffer(device, cached_controller_memory);
447 
448 	if (error != 0)
449 	    return (error);
450 
451 	sci_mdl_decorator_assign_memory(controller->mdl,
452 	    SCI_MDE_ATTRIBUTE_CACHEABLE | SCI_MDE_ATTRIBUTE_PHYSICALLY_CONTIGUOUS,
453 	    cached_controller_memory->virtual_address,
454 	    cached_controller_memory->physical_address);
455 
456 	request_memory->size =
457 	    controller->queue_depth * isci_io_request_get_object_size();
458 
459 	error = isci_allocate_dma_buffer(device, request_memory);
460 
461 	if (error != 0)
462 	    return (error);
463 
464 	/* For STP PIO testing, we want to ensure we can force multiple SGLs
465 	 *  since this has been a problem area in SCIL.  This tunable parameter
466 	 *  will allow us to force DMA segments to a smaller size, ensuring
467 	 *  that even if a physically contiguous buffer is attached to this
468 	 *  I/O, the DMA subsystem will pass us multiple segments in our DMA
469 	 *  load callback.
470 	 */
471 	TUNABLE_INT_FETCH("hw.isci.max_segment_size", &max_segment_size);
472 
473 	/* Create DMA tag for our I/O requests.  Then we can create DMA maps based off
474 	 *  of this tag and store them in each of our ISCI_IO_REQUEST objects.  This
475 	 *  will enable better performance than creating the DMA maps every time we get
476 	 *  an I/O.
477 	 */
478 	status = bus_dma_tag_create(bus_get_dma_tag(device), 0x1, 0x0,
479 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
480 	    isci_io_request_get_max_io_size(),
481 	    SCI_MAX_SCATTER_GATHER_ELEMENTS, max_segment_size, 0, NULL, NULL,
482 	    &controller->buffer_dma_tag);
483 
484 	sci_pool_initialize(controller->request_pool);
485 
486 	virtual_address = request_memory->virtual_address;
487 	physical_address = request_memory->physical_address;
488 
489 	for (int i = 0; i < controller->queue_depth; i++) {
490 		struct ISCI_REQUEST *request =
491 		    (struct ISCI_REQUEST *)virtual_address;
492 
493 		isci_request_construct(request,
494 		    controller->scif_controller_handle,
495 		    controller->buffer_dma_tag, physical_address);
496 
497 		sci_pool_put(controller->request_pool, request);
498 
499 		virtual_address += isci_request_get_object_size();
500 		physical_address += isci_request_get_object_size();
501 	}
502 
503 	uint32_t remote_device_size = sizeof(struct ISCI_REMOTE_DEVICE) +
504 	    scif_remote_device_get_object_size();
505 
506 	controller->remote_device_memory = (uint8_t *) malloc(
507 	    remote_device_size * SCI_MAX_REMOTE_DEVICES, M_ISCI,
508 	    M_NOWAIT | M_ZERO);
509 
510 	sci_pool_initialize(controller->remote_device_pool);
511 
512 	uint8_t *remote_device_memory_ptr = controller->remote_device_memory;
513 
514 	for (int i = 0; i < SCI_MAX_REMOTE_DEVICES; i++) {
515 		struct ISCI_REMOTE_DEVICE *remote_device =
516 		    (struct ISCI_REMOTE_DEVICE *)remote_device_memory_ptr;
517 
518 		controller->remote_device[i] = NULL;
519 		remote_device->index = i;
520 		remote_device->is_resetting = FALSE;
521 		remote_device->frozen_lun_mask = 0;
522 		sci_fast_list_element_init(remote_device,
523 		    &remote_device->pending_device_reset_element);
524 		TAILQ_INIT(&remote_device->queued_ccbs);
525 		remote_device->release_queued_ccb = FALSE;
526 		remote_device->queued_ccb_in_progress = NULL;
527 
528 		/*
529 		 * For the first SCI_MAX_DOMAINS device objects, do not put
530 		 *  them in the pool, rather assign them to each domain.  This
531 		 *  ensures that any device attached directly to port "i" will
532 		 *  always get CAM target id "i".
533 		 */
534 		if (i < SCI_MAX_DOMAINS)
535 			controller->domain[i].da_remote_device = remote_device;
536 		else
537 			sci_pool_put(controller->remote_device_pool,
538 			    remote_device);
539 		remote_device_memory_ptr += remote_device_size;
540 	}
541 
542 	return (0);
543 }
544 
545 void isci_controller_start(void *controller_handle)
546 {
547 	struct ISCI_CONTROLLER *controller =
548 	    (struct ISCI_CONTROLLER *)controller_handle;
549 	SCI_CONTROLLER_HANDLE_T scif_controller_handle =
550 	    controller->scif_controller_handle;
551 
552 	scif_controller_start(scif_controller_handle,
553 	    scif_controller_get_suggested_start_timeout(scif_controller_handle));
554 
555 	scic_controller_enable_interrupts(
556 	    scif_controller_get_scic_handle(controller->scif_controller_handle));
557 }
558 
559 void isci_controller_domain_discovery_complete(
560     struct ISCI_CONTROLLER *isci_controller, struct ISCI_DOMAIN *isci_domain)
561 {
562 	if (!isci_controller->has_been_scanned)
563 	{
564 		/* Controller has not been scanned yet.  We'll clear
565 		 *  the discovery bit for this domain, then check if all bits
566 		 *  are now clear.  That would indicate that all domains are
567 		 *  done with discovery and we can then proceed with initial
568 		 *  scan.
569 		 */
570 
571 		isci_controller->initial_discovery_mask &=
572 		    ~(1 << isci_domain->index);
573 
574 		if (isci_controller->initial_discovery_mask == 0) {
575 			struct isci_softc *driver = isci_controller->isci;
576 			uint8_t next_index = isci_controller->index + 1;
577 
578 			isci_controller->has_been_scanned = TRUE;
579 
580 			/* Unfreeze simq to allow initial scan to proceed. */
581 			xpt_release_simq(isci_controller->sim, TRUE);
582 
583 #if __FreeBSD_version < 800000
584 			/* When driver is loaded after boot, we need to
585 			 *  explicitly rescan here for versions <8.0, because
586 			 *  CAM only automatically scans new buses at boot
587 			 *  time.
588 			 */
589 			union ccb *ccb = xpt_alloc_ccb_nowait();
590 
591 			xpt_create_path(&ccb->ccb_h.path, NULL,
592 			    cam_sim_path(isci_controller->sim),
593 			    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
594 
595 			xpt_rescan(ccb);
596 #endif
597 
598 			if (next_index < driver->controller_count) {
599 				/*  There are more controllers that need to
600 				 *   start.  So start the next one.
601 				 */
602 				isci_controller_start(
603 				    &driver->controllers[next_index]);
604 			}
605 			else
606 			{
607 				/* All controllers have been started and completed discovery.
608 				 *  Disestablish the config hook while will signal to the
609 				 *  kernel during boot that it is safe to try to find and
610 				 *  mount the root partition.
611 				 */
612 				config_intrhook_disestablish(
613 				    &driver->config_hook);
614 			}
615 		}
616 	}
617 }
618 
619 int isci_controller_attach_to_cam(struct ISCI_CONTROLLER *controller)
620 {
621 	struct isci_softc *isci = controller->isci;
622 	device_t parent = device_get_parent(isci->device);
623 	int unit = device_get_unit(isci->device);
624 	struct cam_devq *isci_devq = cam_simq_alloc(controller->sim_queue_depth);
625 
626 	if(isci_devq == NULL) {
627 		isci_log_message(0, "ISCI", "isci_devq is NULL \n");
628 		return (-1);
629 	}
630 
631 	controller->sim = cam_sim_alloc(isci_action, isci_poll, "isci",
632 	    controller, unit, &controller->lock, controller->sim_queue_depth,
633 	    controller->sim_queue_depth, isci_devq);
634 
635 	if(controller->sim == NULL) {
636 		isci_log_message(0, "ISCI", "cam_sim_alloc... fails\n");
637 		cam_simq_free(isci_devq);
638 		return (-1);
639 	}
640 
641 	if(xpt_bus_register(controller->sim, parent, controller->index)
642 	    != CAM_SUCCESS) {
643 		isci_log_message(0, "ISCI", "xpt_bus_register...fails \n");
644 		cam_sim_free(controller->sim, TRUE);
645 		mtx_unlock(&controller->lock);
646 		return (-1);
647 	}
648 
649 	if(xpt_create_path(&controller->path, NULL,
650 	    cam_sim_path(controller->sim), CAM_TARGET_WILDCARD,
651 	    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
652 		isci_log_message(0, "ISCI", "xpt_create_path....fails\n");
653 		xpt_bus_deregister(cam_sim_path(controller->sim));
654 		cam_sim_free(controller->sim, TRUE);
655 		mtx_unlock(&controller->lock);
656 		return (-1);
657 	}
658 
659 	return (0);
660 }
661 
662 void isci_poll(struct cam_sim *sim)
663 {
664 	struct ISCI_CONTROLLER *controller =
665 	    (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
666 
667 	isci_interrupt_poll_handler(controller);
668 }
669 
670 void isci_action(struct cam_sim *sim, union ccb *ccb)
671 {
672 	struct ISCI_CONTROLLER *controller =
673 	    (struct ISCI_CONTROLLER *)cam_sim_softc(sim);
674 
675 	switch ( ccb->ccb_h.func_code ) {
676 	case XPT_PATH_INQ:
677 		{
678 			struct ccb_pathinq *cpi = &ccb->cpi;
679 			int bus = cam_sim_bus(sim);
680 			ccb->ccb_h.ccb_sim_ptr = sim;
681 			cpi->version_num = 1;
682 			cpi->hba_inquiry = PI_TAG_ABLE;
683 			cpi->target_sprt = 0;
684 			cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN |
685 			    PIM_UNMAPPED;
686 			cpi->hba_eng_cnt = 0;
687 			cpi->max_target = SCI_MAX_REMOTE_DEVICES - 1;
688 			cpi->max_lun = ISCI_MAX_LUN;
689 #if __FreeBSD_version >= 800102
690 			cpi->maxio = isci_io_request_get_max_io_size();
691 #endif
692 			cpi->unit_number = cam_sim_unit(sim);
693 			cpi->bus_id = bus;
694 			cpi->initiator_id = SCI_MAX_REMOTE_DEVICES;
695 			cpi->base_transfer_speed = 300000;
696 			strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
697 			strlcpy(cpi->hba_vid, "Intel Corp.", HBA_IDLEN);
698 			strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
699 			cpi->transport = XPORT_SAS;
700 			cpi->transport_version = 0;
701 			cpi->protocol = PROTO_SCSI;
702 			cpi->protocol_version = SCSI_REV_SPC2;
703 			cpi->ccb_h.status = CAM_REQ_CMP;
704 			xpt_done(ccb);
705 		}
706 		break;
707 	case XPT_GET_TRAN_SETTINGS:
708 		{
709 			struct ccb_trans_settings *general_settings = &ccb->cts;
710 			struct ccb_trans_settings_sas *sas_settings =
711 			    &general_settings->xport_specific.sas;
712 			struct ccb_trans_settings_scsi *scsi_settings =
713 			    &general_settings->proto_specific.scsi;
714 			struct ISCI_REMOTE_DEVICE *remote_device;
715 
716 			remote_device = controller->remote_device[ccb->ccb_h.target_id];
717 
718 			if (remote_device == NULL) {
719 				ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
720 				ccb->ccb_h.status &= ~CAM_STATUS_MASK;
721 				ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
722 				xpt_done(ccb);
723 				break;
724 			}
725 
726 			general_settings->protocol = PROTO_SCSI;
727 			general_settings->transport = XPORT_SAS;
728 			general_settings->protocol_version = SCSI_REV_SPC2;
729 			general_settings->transport_version = 0;
730 			scsi_settings->valid = CTS_SCSI_VALID_TQ;
731 			scsi_settings->flags = CTS_SCSI_FLAGS_TAG_ENB;
732 			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
733 			ccb->ccb_h.status |= CAM_REQ_CMP;
734 
735 			sas_settings->bitrate =
736 			    isci_remote_device_get_bitrate(remote_device);
737 
738 			if (sas_settings->bitrate != 0)
739 				sas_settings->valid = CTS_SAS_VALID_SPEED;
740 
741 			xpt_done(ccb);
742 		}
743 		break;
744 	case XPT_SCSI_IO:
745 		if (ccb->ccb_h.flags & CAM_CDB_PHYS) {
746 			ccb->ccb_h.status = CAM_REQ_INVALID;
747 			xpt_done(ccb);
748 			break;
749 		}
750 		isci_io_request_execute_scsi_io(ccb, controller);
751 		break;
752 #if __FreeBSD_version >= 900026
753 	case XPT_SMP_IO:
754 		isci_io_request_execute_smp_io(ccb, controller);
755 		break;
756 #endif
757 	case XPT_SET_TRAN_SETTINGS:
758 		ccb->ccb_h.status &= ~CAM_STATUS_MASK;
759 		ccb->ccb_h.status |= CAM_REQ_CMP;
760 		xpt_done(ccb);
761 		break;
762 	case XPT_CALC_GEOMETRY:
763 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
764 		xpt_done(ccb);
765 		break;
766 	case XPT_RESET_DEV:
767 		{
768 			struct ISCI_REMOTE_DEVICE *remote_device =
769 			    controller->remote_device[ccb->ccb_h.target_id];
770 
771 			if (remote_device != NULL)
772 				isci_remote_device_reset(remote_device, ccb);
773 			else {
774 				ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
775 				ccb->ccb_h.status &= ~CAM_STATUS_MASK;
776 				ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
777 				xpt_done(ccb);
778 			}
779 		}
780 		break;
781 	case XPT_RESET_BUS:
782 		ccb->ccb_h.status = CAM_REQ_CMP;
783 		xpt_done(ccb);
784 		break;
785 	default:
786 		isci_log_message(0, "ISCI", "Unhandled func_code 0x%x\n",
787 		    ccb->ccb_h.func_code);
788 		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
789 		ccb->ccb_h.status &= ~CAM_STATUS_MASK;
790 		ccb->ccb_h.status |= CAM_REQ_INVALID;
791 		xpt_done(ccb);
792 		break;
793 	}
794 }
795 
796 /*
797  * Unfortunately, SCIL doesn't cleanly handle retry conditions.
798  *  CAM_REQUEUE_REQ works only when no one is using the pass(4) interface.  So
799  *  when SCIL denotes an I/O needs to be retried (typically because of mixing
800  *  tagged/non-tagged ATA commands, or running out of NCQ slots), we queue
801  *  these I/O internally.  Once SCIL completes an I/O to this device, or we get
802  *  a ready notification, we will retry the first I/O on the queue.
803  *  Unfortunately, SCIL also doesn't cleanly handle starting the new I/O within
804  *  the context of the completion handler, so we need to retry these I/O after
805  *  the completion handler is done executing.
806  */
807 void
808 isci_controller_release_queued_ccbs(struct ISCI_CONTROLLER *controller)
809 {
810 	struct ISCI_REMOTE_DEVICE *dev;
811 	struct ccb_hdr *ccb_h;
812 	uint8_t *ptr;
813 	int dev_idx;
814 
815 	KASSERT(mtx_owned(&controller->lock), ("controller lock not owned"));
816 
817 	controller->release_queued_ccbs = FALSE;
818 	for (dev_idx = 0;
819 	     dev_idx < SCI_MAX_REMOTE_DEVICES;
820 	     dev_idx++) {
821 
822 		dev = controller->remote_device[dev_idx];
823 		if (dev != NULL &&
824 		    dev->release_queued_ccb == TRUE &&
825 		    dev->queued_ccb_in_progress == NULL) {
826 			dev->release_queued_ccb = FALSE;
827 			ccb_h = TAILQ_FIRST(&dev->queued_ccbs);
828 
829 			if (ccb_h == NULL)
830 				continue;
831 
832 			ptr = scsiio_cdb_ptr(&((union ccb *)ccb_h)->csio);
833 			isci_log_message(1, "ISCI", "release %p %x\n", ccb_h, *ptr);
834 
835 			dev->queued_ccb_in_progress = (union ccb *)ccb_h;
836 			isci_io_request_execute_scsi_io(
837 			    (union ccb *)ccb_h, controller);
838 		}
839 	}
840 }
841