1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 */
15
16 /*
17 * This file contains code necessary to send SCSI commands to HBA.
18 */
19 #include <smartpqi.h>
20
21 /*
22 * []------------------------------------------------------------------[]
23 * | Forward declarations for support/utility functions |
24 * []------------------------------------------------------------------[]
25 */
26 static void aio_io_complete(pqi_io_request_t *io, void *context);
27 static void raid_io_complete(pqi_io_request_t *io, void *context);
28 static void build_aio_sg_list(pqi_state_t s,
29 pqi_aio_path_request_t *rqst, pqi_cmd_t cmd, pqi_io_request_t *);
30 static void build_raid_sg_list(pqi_state_t s,
31 pqi_raid_path_request_t *rqst, pqi_cmd_t cmd, pqi_io_request_t *);
32 static pqi_io_request_t *setup_aio_request(pqi_state_t s, pqi_cmd_t cmd);
33 static pqi_io_request_t *setup_raid_request(pqi_state_t s, pqi_cmd_t cmd);
34 static uint32_t read_heartbeat_counter(pqi_state_t s);
35 static void take_ctlr_offline(pqi_state_t s);
36 static uint32_t free_elem_count(pqi_index_t pi, pqi_index_t ci,
37 uint32_t per_iq);
38 static void ack_event(pqi_state_t s, pqi_event_t e);
39 static boolean_t is_aio_enabled(pqi_device_t d);
40 static void lun_reset_worker(void *v);
41 static void lun_reset_complete(pqi_io_request_t *io, void *ctx);
42
43 #define DIV_UP(n, d) ((n + (d - 1)) / d)
44
45 /*
46 * []------------------------------------------------------------------[]
47 * | Main entry points in file. |
48 * []------------------------------------------------------------------[]
49 */
50
51 /*
52 * pqi_watchdog -- interrupt count and/or heartbeat must increase over time.
53 */
54 void
pqi_watchdog(void * v)55 pqi_watchdog(void *v)
56 {
57 pqi_state_t s = v;
58 uint32_t hb;
59
60 if (pqi_is_offline(s))
61 return;
62
63 hb = read_heartbeat_counter(s);
64 if ((s->s_last_intr_count == s->s_intr_count) &&
65 (s->s_last_heartbeat_count == hb)) {
66 dev_err(s->s_dip, CE_NOTE, "No heartbeat");
67 pqi_show_dev_state(s);
68 take_ctlr_offline(s);
69 } else {
70 s->s_last_intr_count = s->s_intr_count;
71 s->s_last_heartbeat_count = hb;
72 s->s_watchdog = timeout(pqi_watchdog, s,
73 drv_usectohz(WATCHDOG));
74 }
75 }
76
77 /*
78 * pqi_start_io -- queues command to HBA.
79 *
80 * This method can be called either from the upper layer with a non-zero
81 * io argument or called during an interrupt to load the outgoing queue
82 * with more commands.
83 */
84 void
pqi_start_io(pqi_state_t s,pqi_queue_group_t * qg,pqi_path_t path,pqi_io_request_t * io)85 pqi_start_io(pqi_state_t s, pqi_queue_group_t *qg, pqi_path_t path,
86 pqi_io_request_t *io)
87 {
88 pqi_iu_header_t *rqst;
89 size_t iu_len;
90 size_t copy_to_end;
91 pqi_index_t iq_pi;
92 pqi_index_t iq_ci;
93 uint32_t elem_needed;
94 uint32_t elem_to_end;
95 caddr_t next_elem;
96 int sending = 0;
97
98 mutex_enter(&qg->submit_lock[path]);
99 if (io != NULL)
100 list_insert_tail(&qg->request_list[path], io);
101
102
103 iq_pi = qg->iq_pi_copy[path];
104 while ((io = list_head(&qg->request_list[path])) != NULL) {
105
106 /* ---- Primary cause for !active is controller failure ---- */
107 if (qg->qg_active == B_FALSE && io->io_cmd) {
108 list_remove(&qg->request_list[path], io);
109 mutex_enter(&io->io_cmd->pc_device->pd_mutex);
110 pqi_fail_cmd(io->io_cmd, CMD_DEV_GONE, STAT_TERMINATED);
111 mutex_exit(&io->io_cmd->pc_device->pd_mutex);
112 continue;
113 }
114
115 rqst = io->io_iu;
116 iu_len = rqst->iu_length + PQI_REQUEST_HEADER_LENGTH;
117 elem_needed = DIV_UP(iu_len, PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
118 (void) ddi_dma_sync(s->s_queue_dma->handle,
119 (uintptr_t)qg->iq_ci[path] -
120 (uintptr_t)s->s_queue_dma->alloc_memory, sizeof (iq_ci),
121 DDI_DMA_SYNC_FORCPU);
122 iq_ci = *qg->iq_ci[path];
123
124 if (elem_needed > free_elem_count(iq_pi, iq_ci,
125 s->s_num_elements_per_iq))
126 break;
127
128 io->io_pi = iq_pi;
129 rqst->iu_id = qg->oq_id;
130 next_elem = qg->iq_element_array[path] +
131 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
132 elem_to_end = s->s_num_elements_per_iq - iq_pi;
133 if (elem_needed <= elem_to_end) {
134 (void) memcpy(next_elem, rqst, iu_len);
135 (void) ddi_dma_sync(s->s_queue_dma->handle,
136 (uintptr_t)next_elem -
137 (uintptr_t)s->s_queue_dma->alloc_memory, iu_len,
138 DDI_DMA_SYNC_FORDEV);
139 } else {
140 copy_to_end = elem_to_end *
141 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
142 (void) memcpy(next_elem, rqst, copy_to_end);
143 (void) ddi_dma_sync(s->s_queue_dma->handle,
144 (uintptr_t)next_elem -
145 (uintptr_t)s->s_queue_dma->alloc_memory,
146 copy_to_end, DDI_DMA_SYNC_FORDEV);
147 (void) memcpy(qg->iq_element_array[path],
148 (caddr_t)rqst + copy_to_end,
149 iu_len - copy_to_end);
150 (void) ddi_dma_sync(s->s_queue_dma->handle,
151 0, iu_len - copy_to_end, DDI_DMA_SYNC_FORDEV);
152 }
153 sending += elem_needed;
154 if (io->io_cmd != NULL)
155 pqi_cmd_sm(io->io_cmd, PQI_CMD_STARTED, B_TRUE);
156 else if ((rqst->iu_type == PQI_REQUEST_IU_RAID_PATH_IO) &&
157 (s->s_debug_level & (DBG_LVL_CDB | DBG_LVL_RQST)))
158 pqi_dump_io(io);
159
160 iq_pi = (iq_pi + elem_needed) % s->s_num_elements_per_iq;
161 list_remove(&qg->request_list[path], io);
162 }
163
164 qg->submit_count += sending;
165 if (iq_pi != qg->iq_pi_copy[path]) {
166 qg->iq_pi_copy[path] = iq_pi;
167 ddi_put32(s->s_datap, qg->iq_pi[path], iq_pi);
168 } else {
169 ASSERT0(sending);
170 }
171 mutex_exit(&qg->submit_lock[path]);
172 }
173
174 int
pqi_transport_command(pqi_state_t s,pqi_cmd_t cmd)175 pqi_transport_command(pqi_state_t s, pqi_cmd_t cmd)
176 {
177 pqi_device_t devp = cmd->pc_device;
178 int path;
179 pqi_io_request_t *io;
180
181 if (is_aio_enabled(devp) == B_TRUE) {
182 path = AIO_PATH;
183 io = setup_aio_request(s, cmd);
184 } else {
185 path = RAID_PATH;
186 io = setup_raid_request(s, cmd);
187 }
188
189 if (io == NULL)
190 return (TRAN_BUSY);
191
192 cmd->pc_io_rqst = io;
193
194 pqi_start_io(s, &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP],
195 path, io);
196
197 return (TRAN_ACCEPT);
198 }
199
200 void
pqi_do_rescan(void * v)201 pqi_do_rescan(void *v)
202 {
203 pqi_state_t s = v;
204 int circ = 0;
205 int circ1 = 0;
206
207 ndi_devi_enter(scsi_vhci_dip, &circ1);
208 ndi_devi_enter(s->s_dip, &circ);
209 pqi_rescan_devices(s);
210 (void) pqi_config_all(s->s_dip, s);
211 ndi_devi_exit(s->s_dip, circ);
212 ndi_devi_exit(scsi_vhci_dip, circ1);
213 }
214
215 void
pqi_event_worker(void * v)216 pqi_event_worker(void *v)
217 {
218 pqi_state_t s = v;
219 int i;
220 pqi_event_t e;
221 boolean_t non_heartbeat = B_FALSE;
222
223 if (pqi_is_offline(s))
224 return;
225
226 e = s->s_events;
227 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
228 if (e->ev_pending == B_TRUE) {
229 e->ev_pending = B_FALSE;
230 ack_event(s, e);
231 if (pqi_map_event(PQI_EVENT_TYPE_HEARTBEAT) != i)
232 non_heartbeat = B_TRUE;
233 }
234 e++;
235 }
236
237 if (non_heartbeat == B_TRUE)
238 pqi_do_rescan(s);
239 }
240
241 /*
242 * pqi_fail_cmd -- given a reason and stats the command is failed.
243 *
244 * NOTE: pqi_device->pd_mutex must be held. Also note that during the
245 * call to pqi_cmd_sm() the lock will be dropped and reacquired.
246 */
247 void
pqi_fail_cmd(pqi_cmd_t cmd,uchar_t reason,uint_t stats)248 pqi_fail_cmd(pqi_cmd_t cmd, uchar_t reason, uint_t stats)
249 {
250 struct scsi_pkt *pkt = CMD2PKT(cmd);
251
252 ASSERT(MUTEX_HELD(&cmd->pc_device->pd_mutex));
253
254 pkt->pkt_reason = reason;
255 pkt->pkt_statistics = stats;
256
257 pqi_cmd_sm(cmd, PQI_CMD_FATAL, B_FALSE);
258 }
259
260 void
pqi_fail_drive_cmds(pqi_device_t devp)261 pqi_fail_drive_cmds(pqi_device_t devp)
262 {
263 pqi_cmd_t cmd;
264
265 restart:
266 mutex_enter(&devp->pd_mutex);
267 while ((cmd = list_head(&devp->pd_cmd_list)) != NULL) {
268
269 if (cmd->pc_flags & PQI_FLAG_FINISHING) {
270 /*
271 * This will be a very short wait since
272 * raid_io_complete is a quick function that will
273 * call pqi_cmd_sm() which removes the command
274 * from pd_cmd_list.
275 */
276 mutex_exit(&devp->pd_mutex);
277 drv_usecwait(100);
278 goto restart;
279 }
280 pqi_fail_cmd(cmd, CMD_DEV_GONE, STAT_TERMINATED);
281 }
282
283 mutex_exit(&devp->pd_mutex);
284 }
285
286 uint32_t
pqi_disable_intr(pqi_state_t s)287 pqi_disable_intr(pqi_state_t s)
288 {
289 uint32_t db;
290 uint32_t rval;
291
292 rval = db = G32(s, sis_host_to_ctrl_doorbell);
293 db &= ~(SIS_ENABLE_MSIX | SIS_ENABLE_INTX);
294 S32(s, sis_host_to_ctrl_doorbell, db);
295 return (rval);
296 }
297
298 void
pqi_enable_intr(pqi_state_t s,uint32_t old_state)299 pqi_enable_intr(pqi_state_t s, uint32_t old_state)
300 {
301 S32(s, sis_host_to_ctrl_doorbell, old_state);
302 }
303
304 typedef struct reset_closure {
305 pqi_state_t rc_s;
306 pqi_device_t rc_d;
307 } *reset_closure_t;
308
309 /*
310 * pqi_lun_reset -- set up callback to reset the device
311 *
312 * Dispatch queue is used here because the call tree can come from the interrupt
313 * routine. (pqi_process_io_intr -> aio_io_complete -> SCSA -> tran_reset ->
314 * pqi_lun_reset). If pqi_lun_reset were to actually do the reset work it would
315 * then wait for an interrupt which would never arrive since the current thread
316 * would be the interrupt thread. So, start a task to reset the device and
317 * wait for completion.
318 */
319 boolean_t
pqi_lun_reset(pqi_state_t s,pqi_device_t d)320 pqi_lun_reset(pqi_state_t s, pqi_device_t d)
321 {
322 reset_closure_t r = kmem_alloc(sizeof (struct reset_closure), KM_SLEEP);
323
324 r->rc_s = s;
325 r->rc_d = d;
326 (void) ddi_taskq_dispatch(s->s_events_taskq, lun_reset_worker, r, 0);
327 return (B_TRUE);
328 }
329
330 /*
331 * []------------------------------------------------------------------[]
332 * | Support/utility functions for main entry points |
333 * []------------------------------------------------------------------[]
334 */
335
336 static void
lun_reset_worker(void * v)337 lun_reset_worker(void *v)
338 {
339 reset_closure_t r = v;
340 pqi_state_t s;
341 pqi_device_t d;
342 pqi_io_request_t *io;
343 ksema_t sema;
344 pqi_task_management_rqst_t *rqst;
345
346 s = r->rc_s;
347 d = r->rc_d;
348 kmem_free(r, sizeof (*r));
349 sema_p(&s->s_sync_rqst);
350 s->s_sync_expire = gethrtime() + (SYNC_CMDS_TIMEOUT_SECS * NANOSEC);
351
352 sema_init(&sema, 0, NULL, SEMA_DRIVER, NULL);
353
354 io = pqi_alloc_io(s);
355 io->io_cb = lun_reset_complete;
356 io->io_context = &sema;
357
358 rqst = io->io_iu;
359 (void) memset(rqst, 0, sizeof (*rqst));
360
361 rqst->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
362 rqst->header.iu_length = sizeof (*rqst) - PQI_REQUEST_HEADER_LENGTH;
363 rqst->request_id = io->io_index;
364 (void) memcpy(rqst->lun_number, d->pd_scsi3addr,
365 sizeof (rqst->lun_number));
366 rqst->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
367
368 s->s_sync_io = io;
369 pqi_start_io(s, &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
370 io);
371
372 sema_p(&sema);
373 pqi_free_io(io);
374 s->s_sync_io = NULL;
375 s->s_sync_expire = 0;
376
377 sema_v(&s->s_sync_rqst);
378 }
379
380 /*ARGSUSED*/
381 static void
lun_reset_complete(pqi_io_request_t * io,void * ctx)382 lun_reset_complete(pqi_io_request_t *io, void *ctx)
383 {
384 sema_v((ksema_t *)ctx);
385 }
386
387 static void
send_event_ack(pqi_state_t s,pqi_event_acknowledge_request_t * rqst)388 send_event_ack(pqi_state_t s, pqi_event_acknowledge_request_t *rqst)
389 {
390 pqi_queue_group_t *qg;
391 caddr_t next_element;
392 pqi_index_t iq_ci;
393 pqi_index_t iq_pi;
394 int ms_timeo = 1000 * 10;
395
396 qg = &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP];
397 rqst->header.iu_id = qg->oq_id;
398
399 for (;;) {
400 mutex_enter(&qg->submit_lock[RAID_PATH]);
401 iq_pi = qg->iq_pi_copy[RAID_PATH];
402 iq_ci = ddi_get32(s->s_queue_dma->acc, qg->iq_ci[RAID_PATH]);
403
404 if (free_elem_count(iq_pi, iq_ci, s->s_num_elements_per_iq))
405 break;
406
407 mutex_exit(&qg->submit_lock[RAID_PATH]);
408 if (pqi_is_offline(s))
409 return;
410 }
411 next_element = qg->iq_element_array[RAID_PATH] +
412 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
413
414 (void) memcpy(next_element, rqst, sizeof (*rqst));
415 (void) ddi_dma_sync(s->s_queue_dma->handle, 0, 0, DDI_DMA_SYNC_FORDEV);
416
417 iq_pi = (iq_pi + 1) % s->s_num_elements_per_iq;
418 qg->iq_pi_copy[RAID_PATH] = iq_pi;
419
420 ddi_put32(s->s_datap, qg->iq_pi[RAID_PATH], iq_pi);
421
422 /*
423 * Special case processing for events required. The driver must
424 * wait until the acknowledgement is processed before proceeding.
425 * Unfortunately, the HBA doesn't provide an interrupt which means
426 * the code must busy wait.
427 * Code will wait up to 10 seconds.
428 */
429 while (ms_timeo--) {
430 drv_usecwait(1000);
431 iq_ci = ddi_get32(s->s_queue_dma->acc, qg->iq_ci[RAID_PATH]);
432 if (iq_pi == iq_ci)
433 break;
434 }
435
436 mutex_exit(&qg->submit_lock[RAID_PATH]);
437 }
438
439 static void
ack_event(pqi_state_t s,pqi_event_t e)440 ack_event(pqi_state_t s, pqi_event_t e)
441 {
442 pqi_event_acknowledge_request_t rqst;
443
444 (void) memset(&rqst, 0, sizeof (rqst));
445 rqst.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
446 rqst.header.iu_length = sizeof (rqst) - PQI_REQUEST_HEADER_LENGTH;
447 rqst.event_type = e->ev_type;
448 rqst.event_id = e->ev_id;
449 rqst.additional_event_id = e->ev_additional;
450
451 send_event_ack(s, &rqst);
452 }
453
454 static pqi_io_request_t *
setup_aio_request(pqi_state_t s,pqi_cmd_t cmd)455 setup_aio_request(pqi_state_t s, pqi_cmd_t cmd)
456 {
457 pqi_io_request_t *io;
458 pqi_aio_path_request_t *rqst;
459 pqi_device_t devp = cmd->pc_device;
460
461 /* ---- Most likely received a signal during a cv_wait ---- */
462 if ((io = pqi_alloc_io(s)) == NULL)
463 return (NULL);
464
465 io->io_cb = aio_io_complete;
466 io->io_cmd = cmd;
467 io->io_raid_bypass = 0;
468
469 rqst = io->io_iu;
470 (void) memset(rqst, 0, sizeof (*rqst));
471
472 rqst->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
473 rqst->nexus_id = devp->pd_aio_handle;
474 rqst->buffer_length = cmd->pc_dma_count;
475 rqst->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
476 rqst->request_id = io->io_index;
477 rqst->error_index = rqst->request_id;
478 rqst->cdb_length = cmd->pc_cmdlen;
479 (void) memcpy(rqst->cdb, cmd->pc_cdb, cmd->pc_cmdlen);
480 (void) memcpy(rqst->lun_number, devp->pd_scsi3addr,
481 sizeof (rqst->lun_number));
482
483 if (cmd->pc_flags & PQI_FLAG_DMA_VALID) {
484 if (cmd->pc_flags & PQI_FLAG_IO_READ)
485 rqst->data_direction = SOP_READ_FLAG;
486 else
487 rqst->data_direction = SOP_WRITE_FLAG;
488 } else {
489 rqst->data_direction = SOP_NO_DIRECTION_FLAG;
490 }
491
492 build_aio_sg_list(s, rqst, cmd, io);
493 return (io);
494 }
495
496 static pqi_io_request_t *
setup_raid_request(pqi_state_t s,pqi_cmd_t cmd)497 setup_raid_request(pqi_state_t s, pqi_cmd_t cmd)
498 {
499 pqi_io_request_t *io;
500 pqi_raid_path_request_t *rqst;
501 pqi_device_t devp = cmd->pc_device;
502
503 /* ---- Most likely received a signal during a cv_wait ---- */
504 if ((io = pqi_alloc_io(s)) == NULL)
505 return (NULL);
506
507 io->io_cb = raid_io_complete;
508 io->io_cmd = cmd;
509 io->io_raid_bypass = 0;
510
511 rqst = io->io_iu;
512 (void) memset(rqst, 0, sizeof (*rqst));
513 rqst->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
514 rqst->rp_data_len = cmd->pc_dma_count;
515 rqst->rp_task_attr = SOP_TASK_ATTRIBUTE_SIMPLE;
516 rqst->rp_id = io->io_index;
517 rqst->rp_error_index = rqst->rp_id;
518 (void) memcpy(rqst->rp_lun, devp->pd_scsi3addr, sizeof (rqst->rp_lun));
519 (void) memcpy(rqst->rp_cdb, cmd->pc_cdb, cmd->pc_cmdlen);
520
521 ASSERT(cmd->pc_cmdlen <= 16);
522 rqst->rp_additional_cdb = SOP_ADDITIONAL_CDB_BYTES_0;
523
524 if (cmd->pc_flags & PQI_FLAG_DMA_VALID) {
525 if (cmd->pc_flags & PQI_FLAG_IO_READ)
526 rqst->rp_data_dir = SOP_READ_FLAG;
527 else
528 rqst->rp_data_dir = SOP_WRITE_FLAG;
529 } else {
530 rqst->rp_data_dir = SOP_NO_DIRECTION_FLAG;
531 }
532
533 build_raid_sg_list(s, rqst, cmd, io);
534 return (io);
535 }
536
537 /*ARGSUSED*/
538 pqi_cmd_t
pqi_process_comp_ring(pqi_state_t s)539 pqi_process_comp_ring(pqi_state_t s)
540 {
541 return (NULL);
542 }
543
544 static void
raid_io_complete(pqi_io_request_t * io,void * context)545 raid_io_complete(pqi_io_request_t *io, void *context)
546 {
547 /*
548 * ---- XXX Not sure if this complete function will be the same
549 * or different in the end. If it's the same this will be removed
550 * and aio_io_complete will have it's named changed to something
551 * more generic.
552 */
553 aio_io_complete(io, context);
554 }
555
556 /*
557 * special_error_check -- See if sense buffer matches "offline" status.
558 *
559 * spc3r23 section 4.5.6 -- Sense key and sense code definitions.
560 * Sense key == 5 (KEY_ILLEGAL_REQUEST) indicates one of several conditions
561 * a) Command addressed to incorrect logical unit.
562 * b) Command had an invalid task attribute.
563 * ...
564 * Table 28 also shows that ASC 0x26 and ASCQ of 0x00 is an INVALID FIELD
565 * IN PARAMETER LIST.
566 * At no other time does this combination of KEY/ASC/ASCQ occur except when
567 * a device or cable is pulled from the system along with a Hotplug event.
568 * Without documentation it's only a guess, but it's the best that's available.
569 * So, if the conditions are true the command packet pkt_reason will be changed
570 * to CMD_DEV_GONE which causes MPxIO to switch to the other path and the
571 * Hotplug event will cause a scan to occur which removes other inactive
572 * devices in case of a cable pull.
573 */
574 boolean_t
special_error_check(pqi_cmd_t cmd)575 special_error_check(pqi_cmd_t cmd)
576 {
577 struct scsi_arq_status *arq;
578
579 /* LINTED E_BAD_PTR_CAST_ALIGN */
580 arq = (struct scsi_arq_status *)cmd->pc_pkt->pkt_scbp;
581
582 if (((*cmd->pc_pkt->pkt_scbp & STATUS_MASK) == STATUS_CHECK) &&
583 (arq->sts_sensedata.es_key == KEY_ILLEGAL_REQUEST) &&
584 (arq->sts_sensedata.es_add_code == 0x26) &&
585 (arq->sts_sensedata.es_qual_code == 0)) {
586 return (B_TRUE);
587 } else {
588 return (B_FALSE);
589 }
590 }
591
592 /*ARGSUSED*/
593 static void
aio_io_complete(pqi_io_request_t * io,void * context)594 aio_io_complete(pqi_io_request_t *io, void *context)
595 {
596 pqi_cmd_t cmd = io->io_cmd;
597 struct scsi_pkt *pkt = CMD2PKT(cmd);
598
599 if (cmd->pc_flags & (PQI_FLAG_IO_READ | PQI_FLAG_IO_IOPB))
600 (void) ddi_dma_sync(cmd->pc_dmahdl, 0, 0, DDI_DMA_SYNC_FORCPU);
601
602 switch (io->io_status) {
603 case PQI_DATA_IN_OUT_UNDERFLOW:
604 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
605 STATE_SENT_CMD | STATE_GOT_STATUS;
606 if (pkt->pkt_resid == cmd->pc_dma_count) {
607 pkt->pkt_reason = CMD_INCOMPLETE;
608 } else {
609 pkt->pkt_state |= STATE_XFERRED_DATA;
610 pkt->pkt_reason = CMD_CMPLT;
611 }
612 break;
613
614 case PQI_DATA_IN_OUT_GOOD:
615 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
616 STATE_SENT_CMD | STATE_GOT_STATUS;
617 if (cmd->pc_flags & PQI_FLAG_DMA_VALID)
618 pkt->pkt_state |= STATE_XFERRED_DATA;
619 pkt->pkt_reason = CMD_CMPLT;
620 pkt->pkt_resid = 0;
621 pkt->pkt_statistics = 0;
622 break;
623
624 case PQI_DATA_IN_OUT_ERROR:
625 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
626 STATE_SENT_CMD;
627 if (pkt->pkt_resid != cmd->pc_dma_count) {
628 pkt->pkt_state |= STATE_XFERRED_DATA;
629 pkt->pkt_reason = CMD_CMPLT;
630 } else {
631 pkt->pkt_reason = CMD_CMPLT;
632 }
633 break;
634
635 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
636 pkt->pkt_reason = CMD_TERMINATED;
637 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET;
638 break;
639
640 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
641 pkt->pkt_reason = CMD_CMPLT;
642 pkt->pkt_state |= STATE_GOT_BUS;
643 break;
644
645 default:
646 pkt->pkt_reason = CMD_INCOMPLETE;
647 break;
648 }
649
650 if (special_error_check(cmd) == B_TRUE) {
651 pkt->pkt_reason = CMD_DEV_GONE;
652 pkt->pkt_statistics = STAT_TERMINATED;
653
654 pqi_cmd_sm(cmd, PQI_CMD_FATAL, B_TRUE);
655 } else {
656 pqi_cmd_sm(cmd, PQI_CMD_CMPLT, B_TRUE);
657 }
658 }
659
660 static void
fail_outstanding_cmds(pqi_state_t s)661 fail_outstanding_cmds(pqi_state_t s)
662 {
663 pqi_device_t devp;
664 int i;
665 pqi_queue_group_t *qg;
666
667 ASSERT(MUTEX_HELD(&s->s_mutex));
668 if (s->s_sync_io != NULL) {
669 s->s_sync_io->io_status = PQI_DATA_IN_OUT_UNSOLICITED_ABORT;
670 (s->s_sync_io->io_cb)(s->s_sync_io,
671 s->s_sync_io->io_context);
672 }
673
674 for (i = 0; i < s->s_num_queue_groups; i++) {
675 qg = &s->s_queue_groups[i];
676 mutex_enter(&qg->submit_lock[RAID_PATH]);
677 mutex_enter(&qg->submit_lock[AIO_PATH]);
678 qg->qg_active = B_FALSE;
679 mutex_exit(&qg->submit_lock[AIO_PATH]);
680 mutex_exit(&qg->submit_lock[RAID_PATH]);
681 }
682
683 for (devp = list_head(&s->s_devnodes); devp != NULL;
684 devp = list_next(&s->s_devnodes, devp)) {
685 pqi_fail_drive_cmds(devp);
686 }
687 }
688
689 static void
set_sg_descriptor(pqi_sg_entry_t * sg,ddi_dma_cookie_t * cookie)690 set_sg_descriptor(pqi_sg_entry_t *sg, ddi_dma_cookie_t *cookie)
691 {
692 sg->sg_addr = cookie->dmac_laddress;
693 sg->sg_len = cookie->dmac_size;
694 sg->sg_flags = 0;
695 }
696
697 static void
build_aio_sg_list(pqi_state_t s,pqi_aio_path_request_t * rqst,pqi_cmd_t cmd,pqi_io_request_t * io)698 build_aio_sg_list(pqi_state_t s, pqi_aio_path_request_t *rqst,
699 pqi_cmd_t cmd, pqi_io_request_t *io)
700 {
701 int i;
702 int max_sg_per_iu;
703 uint16_t iu_length;
704 uint8_t chained;
705 uint8_t num_sg_in_iu = 0;
706 ddi_dma_cookie_t *cookies;
707 pqi_sg_entry_t *sg;
708
709 iu_length = offsetof(struct pqi_aio_path_request, ap_sglist) -
710 PQI_REQUEST_HEADER_LENGTH;
711
712 if (cmd->pc_dmaccount == 0)
713 goto out;
714 sg = rqst->ap_sglist;
715 cookies = cmd->pc_cached_cookies;
716 max_sg_per_iu = s->s_max_sg_per_iu - 1;
717 i = 0;
718 chained = 0;
719
720 for (;;) {
721 set_sg_descriptor(sg, cookies);
722 if (!chained)
723 num_sg_in_iu++;
724 i++;
725 if (i == cmd->pc_dmaccount)
726 break;
727 sg++;
728 cookies++;
729 if (i == max_sg_per_iu) {
730 sg->sg_addr = io->io_sg_chain_dma->dma_addr;
731 sg->sg_len = (cmd->pc_dmaccount - num_sg_in_iu) *
732 sizeof (*sg);
733 sg->sg_flags = CISS_SG_CHAIN;
734 chained = 1;
735 num_sg_in_iu++;
736 sg = (pqi_sg_entry_t *)
737 io->io_sg_chain_dma->alloc_memory;
738 }
739 }
740 sg->sg_flags = CISS_SG_LAST;
741 rqst->partial = chained;
742 if (chained) {
743 (void) ddi_dma_sync(io->io_sg_chain_dma->handle, 0, 0,
744 DDI_DMA_SYNC_FORDEV);
745 }
746 iu_length += num_sg_in_iu * sizeof (*sg);
747
748 out:
749 rqst->header.iu_length = iu_length;
750 rqst->num_sg_descriptors = num_sg_in_iu;
751 }
752
753 static void
build_raid_sg_list(pqi_state_t s,pqi_raid_path_request_t * rqst,pqi_cmd_t cmd,pqi_io_request_t * io)754 build_raid_sg_list(pqi_state_t s, pqi_raid_path_request_t *rqst,
755 pqi_cmd_t cmd, pqi_io_request_t *io)
756 {
757 int i = 0;
758 int max_sg_per_iu;
759 int num_sg_in_iu = 0;
760 uint16_t iu_length;
761 uint8_t chained = 0;
762 ddi_dma_cookie_t *cookies;
763 pqi_sg_entry_t *sg;
764
765 iu_length = offsetof(struct pqi_raid_path_request, rp_sglist) -
766 PQI_REQUEST_HEADER_LENGTH;
767
768 if (cmd->pc_dmaccount == 0)
769 goto out;
770
771 sg = rqst->rp_sglist;
772 cookies = cmd->pc_cached_cookies;
773 max_sg_per_iu = s->s_max_sg_per_iu - 1;
774
775 for (;;) {
776 set_sg_descriptor(sg, cookies);
777 if (!chained)
778 num_sg_in_iu++;
779 i++;
780 if (i == cmd->pc_dmaccount)
781 break;
782 sg++;
783 cookies++;
784 if (i == max_sg_per_iu) {
785 ASSERT(io->io_sg_chain_dma != NULL);
786 sg->sg_addr = io->io_sg_chain_dma->dma_addr;
787 sg->sg_len = (cmd->pc_dmaccount - num_sg_in_iu) *
788 sizeof (*sg);
789 sg->sg_flags = CISS_SG_CHAIN;
790 chained = 1;
791 num_sg_in_iu++;
792 sg = (pqi_sg_entry_t *)
793 io->io_sg_chain_dma->alloc_memory;
794 }
795 }
796 sg->sg_flags = CISS_SG_LAST;
797 rqst->rp_partial = chained;
798 if (chained) {
799 (void) ddi_dma_sync(io->io_sg_chain_dma->handle, 0, 0,
800 DDI_DMA_SYNC_FORDEV);
801 }
802 iu_length += num_sg_in_iu * sizeof (*sg);
803
804 out:
805 rqst->header.iu_length = iu_length;
806 }
807
808 static uint32_t
read_heartbeat_counter(pqi_state_t s)809 read_heartbeat_counter(pqi_state_t s)
810 {
811 return (ddi_get32(s->s_datap, s->s_heartbeat_counter));
812 }
813
814 static void
take_ctlr_offline(pqi_state_t s)815 take_ctlr_offline(pqi_state_t s)
816 {
817 int circ = 0;
818 int circ1 = 0;
819
820 mutex_enter(&s->s_mutex);
821 s->s_offline = 1;
822 s->s_watchdog = 0;
823 fail_outstanding_cmds(s);
824 mutex_exit(&s->s_mutex);
825
826 /*
827 * This will have the effect of releasing the device's dip
828 * structure from the NDI layer do to s_offline == 1.
829 */
830 ndi_devi_enter(scsi_vhci_dip, &circ1);
831 ndi_devi_enter(s->s_dip, &circ);
832 (void) pqi_config_all(s->s_dip, s);
833 ndi_devi_exit(s->s_dip, circ);
834 ndi_devi_exit(scsi_vhci_dip, circ1);
835 }
836
837 static uint32_t
free_elem_count(pqi_index_t pi,pqi_index_t ci,uint32_t per_iq)838 free_elem_count(pqi_index_t pi, pqi_index_t ci, uint32_t per_iq)
839 {
840 pqi_index_t used;
841 if (pi >= ci) {
842 used = pi - ci;
843 } else {
844 used = per_iq - ci + pi;
845 }
846 return (per_iq - used - 1);
847 }
848
849 static boolean_t
is_aio_enabled(pqi_device_t d)850 is_aio_enabled(pqi_device_t d)
851 {
852 return (d->pd_aio_enabled ? B_TRUE : B_FALSE);
853 }
854