Lines Matching +full:ext +full:- +full:push +full:- +full:pull
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
5 * Copyright (c) 2011-2015 LSI Corp.
6 * Copyright (c) 2013-2015 Avago Technologies
30 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
121 /* Added this union to smoothly convert le64toh cm->cm_desc.Words.
135 /* Rate limit chain-fail messages to 1 per minute */
164 if (curthread->td_no_sleeping != 0) in mps_diag_reset()
169 /* Push the magic sequence */ in mps_diag_reset()
176 if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) in mps_diag_reset()
177 msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, in mps_diag_reset()
210 if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) in mps_diag_reset()
211 msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, in mps_diag_reset()
281 sleep_flags = (sc->mps_flags & MPS_FLAGS_ATTACH_DONE) in mps_transition_ready()
389 prireqcr = MAX(1, sc->max_prireqframes); in mps_resize_queues()
390 prireqcr = MIN(prireqcr, sc->facts->HighPriorityCredit); in mps_resize_queues()
392 reqcr = MAX(2, sc->max_reqframes); in mps_resize_queues()
393 reqcr = MIN(reqcr, sc->facts->RequestCredit); in mps_resize_queues()
395 sc->num_reqs = prireqcr + reqcr; in mps_resize_queues()
396 sc->num_prireqs = prireqcr; in mps_resize_queues()
397 sc->num_replies = MIN(sc->max_replyframes + sc->max_evtframes, in mps_resize_queues()
398 sc->facts->MaxReplyDescriptorPostQueueDepth) - 1; in mps_resize_queues()
401 sc->reqframesz = sc->facts->IOCRequestFrameSize * 4; in mps_resize_queues()
405 * ((SGEs per frame - 1 for chain element) * Max Chain Depth) in mps_resize_queues()
412 sges_per_frame = sc->reqframesz / sizeof(MPI2_SGE_SIMPLE64) - 1; in mps_resize_queues()
413 maxio = (sges_per_frame * sc->facts->MaxChainDepth + 1) * PAGE_SIZE; in mps_resize_queues()
419 if (sc->max_io_pages > 0) { in mps_resize_queues()
420 maxio = min(maxio, sc->max_io_pages * PAGE_SIZE); in mps_resize_queues()
421 sc->maxio = maxio; in mps_resize_queues()
423 sc->maxio = maxio; in mps_resize_queues()
427 sc->num_chains = (maxio / PAGE_SIZE + sges_per_frame - 2) / in mps_resize_queues()
429 if (sc->max_chains > 0 && sc->max_chains < sc->num_chains) in mps_resize_queues()
430 sc->num_chains = sc->max_chains; in mps_resize_queues()
433 * Figure out the number of MSIx-based queues. If the firmware or in mps_resize_queues()
435 * the queues to be useful then don't enable multi-queue. in mps_resize_queues()
437 if (sc->facts->MaxMSIxVectors < 2) in mps_resize_queues()
438 sc->msi_msgs = 1; in mps_resize_queues()
440 if (sc->msi_msgs > 1) { in mps_resize_queues()
441 sc->msi_msgs = MIN(sc->msi_msgs, mp_ncpus); in mps_resize_queues()
442 sc->msi_msgs = MIN(sc->msi_msgs, sc->facts->MaxMSIxVectors); in mps_resize_queues()
443 if (sc->num_reqs / sc->msi_msgs < 2) in mps_resize_queues()
444 sc->msi_msgs = 1; in mps_resize_queues()
448 sc->msi_msgs, sc->num_reqs, sc->num_replies); in mps_resize_queues()
452 * This is called during attach and when re-initializing due to a Diag Reset.
454 * If called from attach, de-allocation is not required because the driver has
456 * allocated structures based on IOC Facts will need to be freed and re-
470 bcopy(sc->facts, &saved_facts, sizeof(MPI2_IOC_FACTS_REPLY)); in mps_iocfacts_allocate()
475 * a re-initialization and only return the error if attaching so the OS in mps_iocfacts_allocate()
478 if ((error = mps_get_iocfacts(sc, sc->facts)) != 0) { in mps_iocfacts_allocate()
489 MPS_DPRINT_PAGE(sc, MPS_XINFO, iocfacts, sc->facts); in mps_iocfacts_allocate()
491 snprintf(sc->fw_version, sizeof(sc->fw_version), in mps_iocfacts_allocate()
493 sc->facts->FWVersion.Struct.Major, in mps_iocfacts_allocate()
494 sc->facts->FWVersion.Struct.Minor, in mps_iocfacts_allocate()
495 sc->facts->FWVersion.Struct.Unit, in mps_iocfacts_allocate()
496 sc->facts->FWVersion.Struct.Dev); in mps_iocfacts_allocate()
498 snprintf(sc->msg_version, sizeof(sc->msg_version), "%d.%d", in mps_iocfacts_allocate()
499 (sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MAJOR_MASK) >> in mps_iocfacts_allocate()
501 (sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MINOR_MASK) >> in mps_iocfacts_allocate()
504 mps_dprint(sc, MPS_INFO, "Firmware: %s, Driver: %s\n", sc->fw_version, in mps_iocfacts_allocate()
507 sc->facts->IOCCapabilities, in mps_iocfacts_allocate()
519 if (attaching && ((sc->facts->IOCCapabilities & in mps_iocfacts_allocate()
536 saved_mode = sc->ir_firmware; in mps_iocfacts_allocate()
537 if (sc->facts->IOCCapabilities & in mps_iocfacts_allocate()
539 sc->ir_firmware = 1; in mps_iocfacts_allocate()
541 if (sc->ir_firmware != saved_mode) { in mps_iocfacts_allocate()
549 sc->mps_flags &= ~MPS_FLAGS_REALLOCATED; in mps_iocfacts_allocate()
552 ((saved_facts.MsgVersion != sc->facts->MsgVersion) || in mps_iocfacts_allocate()
553 (saved_facts.HeaderVersion != sc->facts->HeaderVersion) || in mps_iocfacts_allocate()
554 (saved_facts.MaxChainDepth != sc->facts->MaxChainDepth) || in mps_iocfacts_allocate()
555 (saved_facts.RequestCredit != sc->facts->RequestCredit) || in mps_iocfacts_allocate()
556 (saved_facts.ProductID != sc->facts->ProductID) || in mps_iocfacts_allocate()
557 (saved_facts.IOCCapabilities != sc->facts->IOCCapabilities) || in mps_iocfacts_allocate()
559 sc->facts->IOCRequestFrameSize) || in mps_iocfacts_allocate()
560 (saved_facts.MaxTargets != sc->facts->MaxTargets) || in mps_iocfacts_allocate()
561 (saved_facts.MaxSasExpanders != sc->facts->MaxSasExpanders) || in mps_iocfacts_allocate()
562 (saved_facts.MaxEnclosures != sc->facts->MaxEnclosures) || in mps_iocfacts_allocate()
563 (saved_facts.HighPriorityCredit != sc->facts->HighPriorityCredit) || in mps_iocfacts_allocate()
565 sc->facts->MaxReplyDescriptorPostQueueDepth) || in mps_iocfacts_allocate()
566 (saved_facts.ReplyFrameSize != sc->facts->ReplyFrameSize) || in mps_iocfacts_allocate()
567 (saved_facts.MaxVolumes != sc->facts->MaxVolumes) || in mps_iocfacts_allocate()
569 sc->facts->MaxPersistentEntries))) { in mps_iocfacts_allocate()
573 sc->mps_flags |= MPS_FLAGS_REALLOCATED; in mps_iocfacts_allocate()
577 * Some things should be done if attaching or re-allocating after a Diag in mps_iocfacts_allocate()
586 if (sc->facts->IOCCapabilities & in mps_iocfacts_allocate()
588 sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_TRACE]. in mps_iocfacts_allocate()
590 if (sc->facts->IOCCapabilities & in mps_iocfacts_allocate()
592 sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_SNAPSHOT]. in mps_iocfacts_allocate()
594 if (sc->facts->IOCCapabilities & in mps_iocfacts_allocate()
596 sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_EXTENDED]. in mps_iocfacts_allocate()
602 if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) in mps_iocfacts_allocate()
603 sc->eedp_enabled = TRUE; in mps_iocfacts_allocate()
604 if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) in mps_iocfacts_allocate()
605 sc->control_TLR = TRUE; in mps_iocfacts_allocate()
612 TAILQ_INIT(&sc->req_list); in mps_iocfacts_allocate()
613 TAILQ_INIT(&sc->high_priority_req_list); in mps_iocfacts_allocate()
614 TAILQ_INIT(&sc->chain_list); in mps_iocfacts_allocate()
615 TAILQ_INIT(&sc->tm_list); in mps_iocfacts_allocate()
657 bzero(sc->free_queue, sc->fqdepth * 4); in mps_iocfacts_allocate()
683 sc->replypostindex = 0; in mps_iocfacts_allocate()
684 mps_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex); in mps_iocfacts_allocate()
710 * XXX If the number of MSI-X vectors changes during re-init, this in mps_iocfacts_allocate()
724 sc->WD_available = FALSE; in mps_iocfacts_allocate()
725 if (pci_get_device(sc->mps_dev) == MPI2_MFGPAGE_DEVID_SSS6200) in mps_iocfacts_allocate()
726 sc->WD_available = TRUE; in mps_iocfacts_allocate()
743 if (sc->free_busaddr != 0) in mps_iocfacts_free()
744 bus_dmamap_unload(sc->queues_dmat, sc->queues_map); in mps_iocfacts_free()
745 if (sc->free_queue != NULL) in mps_iocfacts_free()
746 bus_dmamem_free(sc->queues_dmat, sc->free_queue, in mps_iocfacts_free()
747 sc->queues_map); in mps_iocfacts_free()
748 if (sc->queues_dmat != NULL) in mps_iocfacts_free()
749 bus_dma_tag_destroy(sc->queues_dmat); in mps_iocfacts_free()
751 if (sc->chain_frames != NULL) { in mps_iocfacts_free()
752 bus_dmamap_unload(sc->chain_dmat, sc->chain_map); in mps_iocfacts_free()
753 bus_dmamem_free(sc->chain_dmat, sc->chain_frames, in mps_iocfacts_free()
754 sc->chain_map); in mps_iocfacts_free()
756 if (sc->chain_dmat != NULL) in mps_iocfacts_free()
757 bus_dma_tag_destroy(sc->chain_dmat); in mps_iocfacts_free()
759 if (sc->sense_busaddr != 0) in mps_iocfacts_free()
760 bus_dmamap_unload(sc->sense_dmat, sc->sense_map); in mps_iocfacts_free()
761 if (sc->sense_frames != NULL) in mps_iocfacts_free()
762 bus_dmamem_free(sc->sense_dmat, sc->sense_frames, in mps_iocfacts_free()
763 sc->sense_map); in mps_iocfacts_free()
764 if (sc->sense_dmat != NULL) in mps_iocfacts_free()
765 bus_dma_tag_destroy(sc->sense_dmat); in mps_iocfacts_free()
767 if (sc->reply_busaddr != 0) in mps_iocfacts_free()
768 bus_dmamap_unload(sc->reply_dmat, sc->reply_map); in mps_iocfacts_free()
769 if (sc->reply_frames != NULL) in mps_iocfacts_free()
770 bus_dmamem_free(sc->reply_dmat, sc->reply_frames, in mps_iocfacts_free()
771 sc->reply_map); in mps_iocfacts_free()
772 if (sc->reply_dmat != NULL) in mps_iocfacts_free()
773 bus_dma_tag_destroy(sc->reply_dmat); in mps_iocfacts_free()
775 if (sc->req_busaddr != 0) in mps_iocfacts_free()
776 bus_dmamap_unload(sc->req_dmat, sc->req_map); in mps_iocfacts_free()
777 if (sc->req_frames != NULL) in mps_iocfacts_free()
778 bus_dmamem_free(sc->req_dmat, sc->req_frames, sc->req_map); in mps_iocfacts_free()
779 if (sc->req_dmat != NULL) in mps_iocfacts_free()
780 bus_dma_tag_destroy(sc->req_dmat); in mps_iocfacts_free()
782 if (sc->chains != NULL) in mps_iocfacts_free()
783 free(sc->chains, M_MPT2); in mps_iocfacts_free()
784 if (sc->commands != NULL) { in mps_iocfacts_free()
785 for (i = 1; i < sc->num_reqs; i++) { in mps_iocfacts_free()
786 cm = &sc->commands[i]; in mps_iocfacts_free()
787 bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap); in mps_iocfacts_free()
789 free(sc->commands, M_MPT2); in mps_iocfacts_free()
791 if (sc->buffer_dmat != NULL) in mps_iocfacts_free()
792 bus_dma_tag_destroy(sc->buffer_dmat); in mps_iocfacts_free()
795 free(sc->queues, M_MPT2); in mps_iocfacts_free()
796 sc->queues = NULL; in mps_iocfacts_free()
813 sassc = sc->sassc; in mps_reinit()
817 mtx_assert(&sc->mps_mtx, MA_OWNED); in mps_reinit()
820 if (sc->mps_flags & MPS_FLAGS_DIAGRESET) { in mps_reinit()
828 sc->mps_flags |= MPS_FLAGS_DIAGRESET; in mps_reinit()
843 /* Restore the PCI state, including the MSI-X registers */ in mps_reinit()
861 * Mapping structures will be re-allocated after getting IOC Page8, so in mps_reinit()
874 sc->mps_flags &= ~MPS_FLAGS_DIAGRESET; in mps_reinit()
878 * Some mapping info is based in IOC Page8 data, so re-initialize the in mps_reinit()
891 sc, sc->replypostindex, sc->replyfreeindex); in mps_reinit()
934 if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) in mps_wait_db_ack()
935 msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, in mps_wait_db_ack()
942 } while (--cntdn); in mps_wait_db_ack()
976 if (curthread->td_no_sleeping != 0) in mps_request_sync()
1008 /* Clock out the message data synchronously in 32-bit dwords*/ in mps_request_sync()
1020 /* Clock in the reply in 16-bit words. The total length of the in mps_request_sync()
1041 ioc_sz = reply->MsgLength; in mps_request_sync()
1051 residual = ioc_sz * 2 - count; in mps_request_sync()
1068 * Pull out residual words that won't fit into the provided buffer. in mps_request_sync()
1072 while (residual--) { in mps_request_sync()
1100 cm->cm_desc.Default.SMID, cm, cm->cm_ccb); in mps_enqueue_request()
1102 if (sc->mps_flags & MPS_FLAGS_ATTACH_DONE && !(sc->mps_flags & MPS_FLAGS_SHUTDOWN)) in mps_enqueue_request()
1103 mtx_assert(&sc->mps_mtx, MA_OWNED); in mps_enqueue_request()
1105 if (++sc->io_cmds_active > sc->io_cmds_highwater) in mps_enqueue_request()
1106 sc->io_cmds_highwater++; in mps_enqueue_request()
1107 rd.u.low = cm->cm_desc.Words.Low; in mps_enqueue_request()
1108 rd.u.high = cm->cm_desc.Words.High; in mps_enqueue_request()
1111 KASSERT(cm->cm_state == MPS_CM_STATE_BUSY, in mps_enqueue_request()
1112 ("command not busy, state = %u\n", cm->cm_state)); in mps_enqueue_request()
1113 cm->cm_state = MPS_CM_STATE_INQUEUE; in mps_enqueue_request()
1115 /* TODO-We may need to make below regwrite atomic */ in mps_enqueue_request()
1171 if ((sc->pqdepth == 0) || (sc->fqdepth == 0) || (sc->reqframesz == 0) in mps_send_iocinit()
1172 || (sc->replyframesz == 0)) { in mps_send_iocinit()
1185 * deliberately in the lower 32bits of memory. This is a micro- in mps_send_iocinit()
1192 init.SystemRequestFrameSize = htole16((uint16_t)(sc->reqframesz / 4)); in mps_send_iocinit()
1193 init.ReplyDescriptorPostQueueDepth = htole16(sc->pqdepth); in mps_send_iocinit()
1194 init.ReplyFreeQueueDepth = htole16(sc->fqdepth); in mps_send_iocinit()
1198 init.SystemRequestFrameBaseAddress.Low = htole32((uint32_t)sc->req_busaddr); in mps_send_iocinit()
1200 init.ReplyDescriptorPostQueueAddress.Low = htole32((uint32_t)sc->post_busaddr); in mps_send_iocinit()
1202 init.ReplyFreeQueueAddress.Low = htole32((uint32_t)sc->free_busaddr); in mps_send_iocinit()
1247 mps_lock(ctx->softc); in mps_memaddr_wait_cb()
1248 ctx->error = error; in mps_memaddr_wait_cb()
1249 ctx->completed = 1; in mps_memaddr_wait_cb()
1250 if ((error == 0) && (ctx->abandoned == 0)) { in mps_memaddr_wait_cb()
1251 *ctx->addr = segs[0].ds_addr; in mps_memaddr_wait_cb()
1255 if (ctx->abandoned != 0) in mps_memaddr_wait_cb()
1261 mps_unlock(ctx->softc); in mps_memaddr_wait_cb()
1264 bus_dmamap_unload(ctx->buffer_dmat, in mps_memaddr_wait_cb()
1265 ctx->buffer_dmamap); in mps_memaddr_wait_cb()
1266 *ctx->addr = 0; in mps_memaddr_wait_cb()
1279 nq = sc->msi_msgs; in mps_alloc_queues()
1282 sc->queues = malloc(sizeof(struct mps_queue) * nq, M_MPT2, in mps_alloc_queues()
1284 if (sc->queues == NULL) in mps_alloc_queues()
1288 q = &sc->queues[i]; in mps_alloc_queues()
1290 q->sc = sc; in mps_alloc_queues()
1291 q->qnum = i; in mps_alloc_queues()
1312 * contains filled-in reply frames sent from the firmware to the host. in mps_alloc_hw_queues()
1316 sc->fqdepth = roundup2(sc->num_replies + 1, 16); in mps_alloc_hw_queues()
1317 sc->pqdepth = roundup2(sc->num_replies + 1, 16); in mps_alloc_hw_queues()
1318 fqsize= sc->fqdepth * 4; in mps_alloc_hw_queues()
1319 pqsize = sc->pqdepth * 8; in mps_alloc_hw_queues()
1322 bus_dma_template_init(&t, sc->mps_parent_dmat); in mps_alloc_hw_queues()
1326 if (bus_dma_template_tag(&t, &sc->queues_dmat)) { in mps_alloc_hw_queues()
1330 if (bus_dmamem_alloc(sc->queues_dmat, (void **)&queues, BUS_DMA_NOWAIT, in mps_alloc_hw_queues()
1331 &sc->queues_map)) { in mps_alloc_hw_queues()
1336 bus_dmamap_load(sc->queues_dmat, sc->queues_map, queues, qsize, in mps_alloc_hw_queues()
1339 sc->free_queue = (uint32_t *)queues; in mps_alloc_hw_queues()
1340 sc->free_busaddr = queues_busaddr; in mps_alloc_hw_queues()
1341 sc->post_queue = (MPI2_REPLY_DESCRIPTORS_UNION *)(queues + fqsize); in mps_alloc_hw_queues()
1342 sc->post_busaddr = queues_busaddr + fqsize; in mps_alloc_hw_queues()
1344 (uintmax_t)sc->free_busaddr, fqsize); in mps_alloc_hw_queues()
1346 (uintmax_t)sc->post_busaddr, pqsize); in mps_alloc_hw_queues()
1358 sc->replyframesz = sc->facts->ReplyFrameSize * 4; in mps_alloc_replies()
1361 * sc->num_replies should be one less than sc->fqdepth. We need to in mps_alloc_replies()
1362 * allocate space for sc->fqdepth replies, but only sc->num_replies in mps_alloc_replies()
1365 num_replies = max(sc->fqdepth, sc->num_replies); in mps_alloc_replies()
1367 rsize = sc->replyframesz * num_replies; in mps_alloc_replies()
1368 bus_dma_template_init(&t, sc->mps_parent_dmat); in mps_alloc_replies()
1372 if (bus_dma_template_tag(&t, &sc->reply_dmat)) { in mps_alloc_replies()
1376 if (bus_dmamem_alloc(sc->reply_dmat, (void **)&sc->reply_frames, in mps_alloc_replies()
1377 BUS_DMA_NOWAIT, &sc->reply_map)) { in mps_alloc_replies()
1381 bzero(sc->reply_frames, rsize); in mps_alloc_replies()
1382 bus_dmamap_load(sc->reply_dmat, sc->reply_map, sc->reply_frames, rsize, in mps_alloc_replies()
1383 mps_memaddr_cb, &sc->reply_busaddr, 0); in mps_alloc_replies()
1386 (uintmax_t)sc->reply_busaddr, rsize); in mps_alloc_replies()
1403 KASSERT(segs[s].ds_addr + segs[s].ds_len - 1 <= BUS_SPACE_MAXADDR_32BIT, in mps_load_chains_cb()
1406 for (bo = 0; bo + sc->reqframesz <= segs[s].ds_len; in mps_load_chains_cb()
1407 bo += sc->reqframesz) { in mps_load_chains_cb()
1408 chain = &sc->chains[i++]; in mps_load_chains_cb()
1409 chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o); in mps_load_chains_cb()
1410 chain->chain_busaddr = segs[s].ds_addr + bo; in mps_load_chains_cb()
1411 o += sc->reqframesz; in mps_load_chains_cb()
1415 o += segs[s].ds_len - bo; in mps_load_chains_cb()
1417 sc->chain_free_lowwater = i; in mps_load_chains_cb()
1427 rsize = sc->reqframesz * sc->num_reqs; in mps_alloc_requests()
1428 bus_dma_template_init(&t, sc->mps_parent_dmat); in mps_alloc_requests()
1432 if (bus_dma_template_tag(&t, &sc->req_dmat)) { in mps_alloc_requests()
1436 if (bus_dmamem_alloc(sc->req_dmat, (void **)&sc->req_frames, in mps_alloc_requests()
1437 BUS_DMA_NOWAIT, &sc->req_map)) { in mps_alloc_requests()
1441 bzero(sc->req_frames, rsize); in mps_alloc_requests()
1442 bus_dmamap_load(sc->req_dmat, sc->req_map, sc->req_frames, rsize, in mps_alloc_requests()
1443 mps_memaddr_cb, &sc->req_busaddr, 0); in mps_alloc_requests()
1445 (uintmax_t)sc->req_busaddr, rsize); in mps_alloc_requests()
1447 sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2, in mps_alloc_requests()
1449 if (!sc->chains) { in mps_alloc_requests()
1453 rsize = sc->reqframesz * sc->num_chains; in mps_alloc_requests()
1454 bus_dma_template_clone(&t, sc->req_dmat); in mps_alloc_requests()
1458 if (bus_dma_template_tag(&t, &sc->chain_dmat)) { in mps_alloc_requests()
1462 if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, in mps_alloc_requests()
1463 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->chain_map)) { in mps_alloc_requests()
1467 if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, in mps_alloc_requests()
1470 bus_dmamem_free(sc->chain_dmat, sc->chain_frames, in mps_alloc_requests()
1471 sc->chain_map); in mps_alloc_requests()
1475 rsize = MPS_SENSE_LEN * sc->num_reqs; in mps_alloc_requests()
1476 bus_dma_template_clone(&t, sc->req_dmat); in mps_alloc_requests()
1479 if (bus_dma_template_tag(&t, &sc->sense_dmat)) { in mps_alloc_requests()
1483 if (bus_dmamem_alloc(sc->sense_dmat, (void **)&sc->sense_frames, in mps_alloc_requests()
1484 BUS_DMA_NOWAIT, &sc->sense_map)) { in mps_alloc_requests()
1488 bzero(sc->sense_frames, rsize); in mps_alloc_requests()
1489 bus_dmamap_load(sc->sense_dmat, sc->sense_map, sc->sense_frames, rsize, in mps_alloc_requests()
1490 mps_memaddr_cb, &sc->sense_busaddr, 0); in mps_alloc_requests()
1492 (uintmax_t)sc->sense_busaddr, rsize); in mps_alloc_requests()
1494 nsegs = (sc->maxio / PAGE_SIZE) + 1; in mps_alloc_requests()
1495 bus_dma_template_init(&t, sc->mps_parent_dmat); in mps_alloc_requests()
1499 BD_LOCKFUNCARG(&sc->mps_mtx), in mps_alloc_requests()
1501 if (bus_dma_template_tag(&t, &sc->buffer_dmat)) { in mps_alloc_requests()
1510 sc->commands = malloc(sizeof(struct mps_command) * sc->num_reqs, in mps_alloc_requests()
1512 for (i = 1; i < sc->num_reqs; i++) { in mps_alloc_requests()
1513 cm = &sc->commands[i]; in mps_alloc_requests()
1514 cm->cm_req = sc->req_frames + i * sc->reqframesz; in mps_alloc_requests()
1515 cm->cm_req_busaddr = sc->req_busaddr + i * sc->reqframesz; in mps_alloc_requests()
1516 cm->cm_sense = &sc->sense_frames[i]; in mps_alloc_requests()
1517 cm->cm_sense_busaddr = sc->sense_busaddr + i * MPS_SENSE_LEN; in mps_alloc_requests()
1518 cm->cm_desc.Default.SMID = i; in mps_alloc_requests()
1519 cm->cm_sc = sc; in mps_alloc_requests()
1520 cm->cm_state = MPS_CM_STATE_BUSY; in mps_alloc_requests()
1521 TAILQ_INIT(&cm->cm_chain_list); in mps_alloc_requests()
1522 callout_init_mtx(&cm->cm_callout, &sc->mps_mtx, 0); in mps_alloc_requests()
1525 if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) == 0) in mps_alloc_requests()
1526 if (i <= sc->num_prireqs) in mps_alloc_requests()
1532 sc->num_reqs = i; in mps_alloc_requests()
1545 memset((uint8_t *)sc->post_queue, 0xff, sc->pqdepth * 8); in mps_init_queues()
1549 * have space for on the queue. So sc->num_replies (the number we in mps_init_queues()
1550 * use) should be less than sc->fqdepth (allocated size). in mps_init_queues()
1552 if (sc->num_replies >= sc->fqdepth) in mps_init_queues()
1558 for (i = 0; i < sc->fqdepth; i++) in mps_init_queues()
1559 sc->free_queue[i] = sc->reply_busaddr + (i * sc->replyframesz); in mps_init_queues()
1560 sc->replyfreeindex = sc->num_replies; in mps_init_queues()
1566 * Next are the global settings, if they exist. Highest are the per-unit
1575 sc->mps_debug = MPS_INFO|MPS_FAULT; in mps_get_tunables()
1576 sc->disable_msix = 0; in mps_get_tunables()
1577 sc->disable_msi = 0; in mps_get_tunables()
1578 sc->max_msix = MPS_MSIX_MAX; in mps_get_tunables()
1579 sc->max_chains = MPS_CHAIN_FRAMES; in mps_get_tunables()
1580 sc->max_io_pages = MPS_MAXIO_PAGES; in mps_get_tunables()
1581 sc->enable_ssu = MPS_SSU_ENABLE_SSD_DISABLE_HDD; in mps_get_tunables()
1582 sc->spinup_wait_time = DEFAULT_SPINUP_WAIT; in mps_get_tunables()
1583 sc->use_phynum = 1; in mps_get_tunables()
1584 sc->max_reqframes = MPS_REQ_FRAMES; in mps_get_tunables()
1585 sc->max_prireqframes = MPS_PRI_REQ_FRAMES; in mps_get_tunables()
1586 sc->max_replyframes = MPS_REPLY_FRAMES; in mps_get_tunables()
1587 sc->max_evtframes = MPS_EVT_REPLY_FRAMES; in mps_get_tunables()
1595 TUNABLE_INT_FETCH("hw.mps.disable_msix", &sc->disable_msix); in mps_get_tunables()
1596 TUNABLE_INT_FETCH("hw.mps.disable_msi", &sc->disable_msi); in mps_get_tunables()
1597 TUNABLE_INT_FETCH("hw.mps.max_msix", &sc->max_msix); in mps_get_tunables()
1598 TUNABLE_INT_FETCH("hw.mps.max_chains", &sc->max_chains); in mps_get_tunables()
1599 TUNABLE_INT_FETCH("hw.mps.max_io_pages", &sc->max_io_pages); in mps_get_tunables()
1600 TUNABLE_INT_FETCH("hw.mps.enable_ssu", &sc->enable_ssu); in mps_get_tunables()
1601 TUNABLE_INT_FETCH("hw.mps.spinup_wait_time", &sc->spinup_wait_time); in mps_get_tunables()
1602 TUNABLE_INT_FETCH("hw.mps.use_phy_num", &sc->use_phynum); in mps_get_tunables()
1603 TUNABLE_INT_FETCH("hw.mps.max_reqframes", &sc->max_reqframes); in mps_get_tunables()
1604 TUNABLE_INT_FETCH("hw.mps.max_prireqframes", &sc->max_prireqframes); in mps_get_tunables()
1605 TUNABLE_INT_FETCH("hw.mps.max_replyframes", &sc->max_replyframes); in mps_get_tunables()
1606 TUNABLE_INT_FETCH("hw.mps.max_evtframes", &sc->max_evtframes); in mps_get_tunables()
1608 /* Grab the unit-instance variables */ in mps_get_tunables()
1610 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1616 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1617 TUNABLE_INT_FETCH(tmpstr, &sc->disable_msix); in mps_get_tunables()
1620 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1621 TUNABLE_INT_FETCH(tmpstr, &sc->disable_msi); in mps_get_tunables()
1624 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1625 TUNABLE_INT_FETCH(tmpstr, &sc->max_msix); in mps_get_tunables()
1628 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1629 TUNABLE_INT_FETCH(tmpstr, &sc->max_chains); in mps_get_tunables()
1632 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1633 TUNABLE_INT_FETCH(tmpstr, &sc->max_io_pages); in mps_get_tunables()
1635 bzero(sc->exclude_ids, sizeof(sc->exclude_ids)); in mps_get_tunables()
1637 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1638 TUNABLE_STR_FETCH(tmpstr, sc->exclude_ids, sizeof(sc->exclude_ids)); in mps_get_tunables()
1641 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1642 TUNABLE_INT_FETCH(tmpstr, &sc->enable_ssu); in mps_get_tunables()
1645 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1646 TUNABLE_INT_FETCH(tmpstr, &sc->spinup_wait_time); in mps_get_tunables()
1649 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1650 TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum); in mps_get_tunables()
1653 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1654 TUNABLE_INT_FETCH(tmpstr, &sc->max_reqframes); in mps_get_tunables()
1657 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1658 TUNABLE_INT_FETCH(tmpstr, &sc->max_prireqframes); in mps_get_tunables()
1661 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1662 TUNABLE_INT_FETCH(tmpstr, &sc->max_replyframes); in mps_get_tunables()
1665 device_get_unit(sc->mps_dev)); in mps_get_tunables()
1666 TUNABLE_INT_FETCH(tmpstr, &sc->max_evtframes); in mps_get_tunables()
1682 device_get_unit(sc->mps_dev)); in mps_setup_sysctl()
1683 snprintf(tmpstr2, sizeof(tmpstr2), "%d", device_get_unit(sc->mps_dev)); in mps_setup_sysctl()
1685 sysctl_ctx = device_get_sysctl_ctx(sc->mps_dev); in mps_setup_sysctl()
1687 sysctl_tree = device_get_sysctl_tree(sc->mps_dev); in mps_setup_sysctl()
1690 sysctl_ctx_init(&sc->sysctl_ctx); in mps_setup_sysctl()
1691 sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, in mps_setup_sysctl()
1694 if (sc->sysctl_tree == NULL) in mps_setup_sysctl()
1696 sysctl_ctx = &sc->sysctl_ctx; in mps_setup_sysctl()
1697 sysctl_tree = sc->sysctl_tree; in mps_setup_sysctl()
1705 OID_AUTO, "disable_msix", CTLFLAG_RD, &sc->disable_msix, 0, in mps_setup_sysctl()
1706 "Disable the use of MSI-X interrupts"); in mps_setup_sysctl()
1709 OID_AUTO, "disable_msi", CTLFLAG_RD, &sc->disable_msi, 0, in mps_setup_sysctl()
1713 OID_AUTO, "max_msix", CTLFLAG_RD, &sc->max_msix, 0, in mps_setup_sysctl()
1714 "User-defined maximum number of MSIX queues"); in mps_setup_sysctl()
1717 OID_AUTO, "msix_msgs", CTLFLAG_RD, &sc->msi_msgs, 0, in mps_setup_sysctl()
1721 OID_AUTO, "max_reqframes", CTLFLAG_RD, &sc->max_reqframes, 0, in mps_setup_sysctl()
1725 OID_AUTO, "max_prireqframes", CTLFLAG_RD, &sc->max_prireqframes, 0, in mps_setup_sysctl()
1729 OID_AUTO, "max_replyframes", CTLFLAG_RD, &sc->max_replyframes, 0, in mps_setup_sysctl()
1733 OID_AUTO, "max_evtframes", CTLFLAG_RD, &sc->max_evtframes, 0, in mps_setup_sysctl()
1737 OID_AUTO, "firmware_version", CTLFLAG_RD, sc->fw_version, in mps_setup_sysctl()
1738 strlen(sc->fw_version), "firmware version"); in mps_setup_sysctl()
1745 OID_AUTO, "msg_version", CTLFLAG_RD, sc->msg_version, in mps_setup_sysctl()
1746 strlen(sc->msg_version), "message interface version (deprecated)"); in mps_setup_sysctl()
1750 &sc->io_cmds_active, 0, "number of currently active commands"); in mps_setup_sysctl()
1754 &sc->io_cmds_highwater, 0, "maximum active commands seen"); in mps_setup_sysctl()
1758 &sc->chain_free, 0, "number of free chain elements"); in mps_setup_sysctl()
1762 &sc->chain_free_lowwater, 0,"lowest number of free chain elements"); in mps_setup_sysctl()
1766 &sc->max_chains, 0,"maximum chain frames that will be allocated"); in mps_setup_sysctl()
1770 &sc->max_io_pages, 0,"maximum pages to allow per I/O (if <1 use " in mps_setup_sysctl()
1774 OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0, in mps_setup_sysctl()
1779 &sc->chain_alloc_fail, "chain allocation failures"); in mps_setup_sysctl()
1783 &sc->spinup_wait_time, DEFAULT_SPINUP_WAIT, "seconds to wait for " in mps_setup_sysctl()
1803 &sc->dump_reqs_alltypes, 0, in mps_setup_sysctl()
1807 OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0, in mps_setup_sysctl()
1851 debug = sc->mps_debug; in mps_debug_sysctl()
1858 if (debug & string->flag) in mps_debug_sysctl()
1859 sbuf_printf(sbuf, ",%s", string->name); in mps_debug_sysctl()
1865 if (error || req->newptr == NULL) in mps_debug_sysctl()
1868 len = req->newlen - req->newidx; in mps_debug_sysctl()
1896 } else if (*list == '-') { in mps_parse_debug()
1915 if (strcasecmp(token, string->name) == 0) { in mps_parse_debug()
1916 flags |= string->flag; in mps_parse_debug()
1924 sc->mps_debug = flags; in mps_parse_debug()
1927 sc->mps_debug |= flags; in mps_parse_debug()
1930 sc->mps_debug &= (~flags); in mps_parse_debug()
1965 numreqs = sc->num_reqs; in mps_dump_reqs()
1967 if (req->newptr != NULL) in mps_dump_reqs()
1970 if (smid == 0 || smid > sc->num_reqs) in mps_dump_reqs()
1972 if (numreqs <= 0 || (numreqs + smid > sc->num_reqs)) in mps_dump_reqs()
1973 numreqs = sc->num_reqs; in mps_dump_reqs()
1978 cm = &sc->commands[i]; in mps_dump_reqs()
1979 if ((sc->dump_reqs_alltypes == 0) && (cm->cm_state != state)) in mps_dump_reqs()
1982 hdr.state = cm->cm_state; in mps_dump_reqs()
1984 hdr.deschi = cm->cm_desc.Words.High; in mps_dump_reqs()
1985 hdr.desclo = cm->cm_desc.Words.Low; in mps_dump_reqs()
1986 TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link, in mps_dump_reqs()
1990 sbuf_bcat(sb, cm->cm_req, 128); in mps_dump_reqs()
1991 TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link, in mps_dump_reqs()
1993 sbuf_bcat(sb, chain->chain, 128); in mps_dump_reqs()
2009 mtx_init(&sc->mps_mtx, "MPT2SAS lock", NULL, MTX_DEF); in mps_attach()
2010 callout_init_mtx(&sc->periodic, &sc->mps_mtx, 0); in mps_attach()
2011 callout_init_mtx(&sc->device_check_callout, &sc->mps_mtx, 0); in mps_attach()
2012 TAILQ_INIT(&sc->event_list); in mps_attach()
2013 timevalclear(&sc->lastfail); in mps_attach()
2021 sc->facts = malloc(sizeof(MPI2_IOC_FACTS_REPLY), M_MPT2, in mps_attach()
2023 if(!sc->facts) { in mps_attach()
2031 * A Diag Reset will also call mps_iocfacts_allocate and re-read the IOC in mps_attach()
2050 sc->mps_ich.ich_func = mps_startup; in mps_attach()
2051 sc->mps_ich.ich_arg = sc; in mps_attach()
2052 if (config_intrhook_establish(&sc->mps_ich) != 0) { in mps_attach()
2061 sc->shutdown_eh = EVENTHANDLER_REGISTER(shutdown_final, in mps_attach()
2064 if (sc->shutdown_eh == NULL) in mps_attach()
2070 sc->mps_flags |= MPS_FLAGS_ATTACH_DONE; in mps_attach()
2076 /* Run through any late-start handlers. */
2095 config_intrhook_disestablish(&sc->mps_ich); in mps_startup()
2096 sc->mps_ich.ich_arg = NULL; in mps_startup()
2109 if (sc->mps_flags & MPS_FLAGS_SHUTDOWN) in mps_periodic()
2118 callout_reset_sbt(&sc->periodic, MPS_PERIODIC_DELAY * SBT_1S, 0, in mps_periodic()
2130 switch (event->Event) { in mps_log_evt_handler()
2133 if (sc->mps_debug & MPS_EVENT) in mps_log_evt_handler()
2134 hexdump(event->EventData, event->EventDataLength, NULL, 0); in mps_log_evt_handler()
2137 entry = (MPI2_EVENT_DATA_LOG_ENTRY_ADDED *)event->EventData; in mps_log_evt_handler()
2139 "0x%x Sequence %d:\n", entry->LogEntryQualifier, in mps_log_evt_handler()
2140 entry->LogSequence); in mps_log_evt_handler()
2158 &sc->mps_log_eh); in mps_attach_log()
2167 if (sc->mps_log_eh != NULL) in mps_detach_log()
2168 mps_deregister_events(sc, sc->mps_log_eh); in mps_detach_log()
2184 sc->mps_flags |= MPS_FLAGS_SHUTDOWN; in mps_free()
2187 callout_drain(&sc->periodic); in mps_free()
2188 callout_drain(&sc->device_check_callout); in mps_free()
2207 if (sc->facts != NULL) in mps_free()
2208 free(sc->facts, M_MPT2); in mps_free()
2216 if (sc->sysctl_tree != NULL) in mps_free()
2217 sysctl_ctx_free(&sc->sysctl_ctx); in mps_free()
2220 if (sc->shutdown_eh != NULL) in mps_free()
2221 EVENTHANDLER_DEREGISTER(shutdown_final, sc->shutdown_eh); in mps_free()
2223 mtx_destroy(&sc->mps_mtx); in mps_free()
2239 KASSERT(cm->cm_state == MPS_CM_STATE_INQUEUE, in mps_complete_command()
2240 ("command not inqueue, state = %u\n", cm->cm_state)); in mps_complete_command()
2241 cm->cm_state = MPS_CM_STATE_BUSY; in mps_complete_command()
2242 if (cm->cm_flags & MPS_CM_FLAGS_POLLED) in mps_complete_command()
2243 cm->cm_flags |= MPS_CM_FLAGS_COMPLETE; in mps_complete_command()
2245 if (cm->cm_complete != NULL) { in mps_complete_command()
2248 __func__, cm, cm->cm_complete, cm->cm_complete_data, in mps_complete_command()
2249 cm->cm_reply); in mps_complete_command()
2250 cm->cm_complete(sc, cm); in mps_complete_command()
2253 if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) { in mps_complete_command()
2258 if (cm->cm_sc->io_cmds_active != 0) { in mps_complete_command()
2259 cm->cm_sc->io_cmds_active--; in mps_complete_command()
2262 "out of sync - resynching to 0\n"); in mps_complete_command()
2319 sc_status = le16toh(mpi_reply->IOCStatus); in mps_display_reply_info()
2321 mps_sas_log_info(sc, le32toh(mpi_reply->IOCLogInfo)); in mps_display_reply_info()
2334 * needed for both INTx interrupts and driver-driven polling in mps_intr()
2380 pq = sc->replypostindex; in mps_intr_locked()
2383 __func__, sc, sc->replypostindex); in mps_intr_locked()
2387 desc = &sc->post_queue[sc->replypostindex]; in mps_intr_locked()
2399 flags = desc->Default.ReplyFlags & in mps_intr_locked()
2402 || (le32toh(desc->Words.High) == 0xffffffff)) in mps_intr_locked()
2412 if (++sc->replypostindex >= sc->pqdepth) in mps_intr_locked()
2413 sc->replypostindex = 0; in mps_intr_locked()
2417 cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)]; in mps_intr_locked()
2418 cm->cm_reply = NULL; in mps_intr_locked()
2426 * Re-compose the reply address from the address in mps_intr_locked()
2432 * (sc->reply_frames). in mps_intr_locked()
2434 baddr = le32toh(desc->AddressReply.ReplyFrameAddress); in mps_intr_locked()
2435 reply = sc->reply_frames + in mps_intr_locked()
2436 (baddr - ((uint32_t)sc->reply_busaddr)); in mps_intr_locked()
2443 if ((reply < sc->reply_frames) in mps_intr_locked()
2444 || (reply > (sc->reply_frames + in mps_intr_locked()
2445 (sc->fqdepth * sc->replyframesz)))) { in mps_intr_locked()
2450 sc->reply_frames, sc->fqdepth, in mps_intr_locked()
2451 sc->replyframesz); in mps_intr_locked()
2453 /* LSI-TODO. See Linux Code for Graceful exit */ in mps_intr_locked()
2456 if (le16toh(desc->AddressReply.SMID) == 0) { in mps_intr_locked()
2457 if (((MPI2_DEFAULT_REPLY *)reply)->Function == in mps_intr_locked()
2468 if ((le16toh(rel_rep->IOCStatus) & in mps_intr_locked()
2473 &sc->fw_diag_buffer_list[ in mps_intr_locked()
2474 rel_rep->BufferType]; in mps_intr_locked()
2475 pBuffer->valid_data = TRUE; in mps_intr_locked()
2476 pBuffer->owned_by_firmware = in mps_intr_locked()
2478 pBuffer->immediate = FALSE; in mps_intr_locked()
2490 cm = &sc->commands[ in mps_intr_locked()
2491 le16toh(desc->AddressReply.SMID)]; in mps_intr_locked()
2492 if (cm->cm_state == MPS_CM_STATE_INQUEUE) { in mps_intr_locked()
2493 cm->cm_reply = reply; in mps_intr_locked()
2494 cm->cm_reply_data = le32toh( in mps_intr_locked()
2495 desc->AddressReply.ReplyFrameAddress); in mps_intr_locked()
2500 cm->cm_state, cm); in mps_intr_locked()
2511 desc->Default.ReplyFlags); in mps_intr_locked()
2519 if (cm->cm_reply) in mps_intr_locked()
2520 mps_display_reply_info(sc,cm->cm_reply); in mps_intr_locked()
2525 if (pq != sc->replypostindex) { in mps_intr_locked()
2527 __func__, sc, sc->replypostindex); in mps_intr_locked()
2529 sc->replypostindex); in mps_intr_locked()
2542 event = le16toh(reply->Event); in mps_dispatch_event()
2543 TAILQ_FOREACH(eh, &sc->event_list, eh_list) { in mps_dispatch_event()
2544 if (isset(eh->mask, event)) { in mps_dispatch_event()
2545 eh->callback(sc, data, reply); in mps_dispatch_event()
2566 if (cm->cm_reply) in mps_reregister_events_complete()
2568 (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply); in mps_reregister_events_complete()
2589 eh->callback = cb; in mps_register_events()
2590 eh->data = data; in mps_register_events()
2591 TAILQ_INSERT_TAIL(&sc->event_list, eh, eh_list); in mps_register_events()
2611 bcopy(mask, &handle->mask[0], sizeof(u32) * in mps_update_events()
2615 sc->event_mask[i] = -1; in mps_update_events()
2618 sc->event_mask[i] &= ~handle->mask[i]; in mps_update_events()
2622 evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req; in mps_update_events()
2623 evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; in mps_update_events()
2624 evtreq->MsgFlags = 0; in mps_update_events()
2625 evtreq->SASBroadcastPrimitiveMasks = 0; in mps_update_events()
2630 bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) * in mps_update_events()
2635 evtreq->EventMasks[i] = in mps_update_events()
2636 htole32(sc->event_mask[i]); in mps_update_events()
2638 cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; in mps_update_events()
2639 cm->cm_data = NULL; in mps_update_events()
2643 reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply; in mps_update_events()
2645 (reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) in mps_update_events()
2671 sc->event_mask[i] = -1; in mps_reregister_events()
2673 TAILQ_FOREACH(eh, &sc->event_list, eh_list) { in mps_reregister_events()
2675 sc->event_mask[i] &= ~eh->mask[i]; in mps_reregister_events()
2680 evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req; in mps_reregister_events()
2681 evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; in mps_reregister_events()
2682 evtreq->MsgFlags = 0; in mps_reregister_events()
2683 evtreq->SASBroadcastPrimitiveMasks = 0; in mps_reregister_events()
2688 bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) * in mps_reregister_events()
2693 evtreq->EventMasks[i] = in mps_reregister_events()
2694 htole32(sc->event_mask[i]); in mps_reregister_events()
2696 cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; in mps_reregister_events()
2697 cm->cm_data = NULL; in mps_reregister_events()
2698 cm->cm_complete = mps_reregister_events_complete; in mps_reregister_events()
2711 TAILQ_REMOVE(&sc->event_list, handle, eh_list); in mps_deregister_events()
2726 if (cm->cm_sglsize < MPS_SGC_SIZE) in mps_add_chain()
2729 chain = mps_alloc_chain(cm->cm_sc); in mps_add_chain()
2733 space = cm->cm_sc->reqframesz; in mps_add_chain()
2736 * Note: a double-linked list is used to make it easier to in mps_add_chain()
2739 TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link); in mps_add_chain()
2741 sgc = (MPI2_SGE_CHAIN64 *)&cm->cm_sge->MpiChain; in mps_add_chain()
2742 sgc->Length = htole16(space); in mps_add_chain()
2743 sgc->NextChainOffset = 0; in mps_add_chain()
2744 /* TODO Looks like bug in Setting sgc->Flags. in mps_add_chain()
2745 * sgc->Flags = ( MPI2_SGE_FLAGS_CHAIN_ELEMENT | MPI2_SGE_FLAGS_64_BIT_ADDRESSING | in mps_add_chain()
2750 sgc->Flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT | MPI2_SGE_FLAGS_64_BIT_ADDRESSING; in mps_add_chain()
2751 sgc->Address.High = htole32(chain->chain_busaddr >> 32); in mps_add_chain()
2752 sgc->Address.Low = htole32(chain->chain_busaddr); in mps_add_chain()
2754 cm->cm_sge = (MPI2_SGE_IO_UNION *)&chain->chain->MpiSimple; in mps_add_chain()
2755 cm->cm_sglsize = space; in mps_add_chain()
2760 * Add one scatter-gather element (chain, simple, transaction context)
2761 * to the scatter-gather list for a command. Maintain cm_sglsize and
2773 type = (tc->Flags & MPI2_SGE_FLAGS_ELEMENT_MASK); in mps_push_sge()
2778 if (len != tc->DetailsLength + 4) in mps_push_sge()
2780 tc->DetailsLength + 4, len); in mps_push_sge()
2784 /* Driver only uses 64-bit chain elements */ in mps_push_sge()
2790 /* Driver only uses 64-bit SGE simple elements */ in mps_push_sge()
2794 if (((le32toh(sge->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT) & in mps_push_sge()
2796 panic("SGE simple %p not marked 64-bit?", sge); in mps_push_sge()
2800 panic("Unexpected SGE %p, flags %02x", tc, tc->Flags); in mps_push_sge()
2816 if (cm->cm_sglsize < MPS_SGC_SIZE) in mps_push_sge()
2819 if (segsleft >= 1 && cm->cm_sglsize < len + MPS_SGC_SIZE) { in mps_push_sge()
2831 cm->cm_sglsize < len + MPS_SGC_SIZE + MPS_SGE64_SIZE) { in mps_push_sge()
2839 sge->FlagsLength |= htole32( in mps_push_sge()
2848 cm->cm_sglsize -= len; in mps_push_sge()
2849 bcopy(sgep, cm->cm_sge, len); in mps_push_sge()
2850 cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len); in mps_push_sge()
2856 if (segsleft == 1 && cm->cm_sglsize < len) in mps_push_sge()
2858 cm->cm_sglsize, len); in mps_push_sge()
2861 if (segsleft == 2 && cm->cm_sglsize < len + MPS_SGE64_SIZE) in mps_push_sge()
2863 cm->cm_sglsize, len); in mps_push_sge()
2868 * If this is a bi-directional request, need to account for that in mps_push_sge()
2869 * here. Save the pre-filled sge values. These will be used in mps_push_sge()
2871 * cm_out_len is non-zero, this is a bi-directional request, so in mps_push_sge()
2874 * 2 SGL's for a bi-directional request, they both use the same in mps_push_sge()
2877 saved_buf_len = le32toh(sge->FlagsLength) & 0x00FFFFFF; in mps_push_sge()
2878 saved_address_low = sge->Address.Low; in mps_push_sge()
2879 saved_address_high = sge->Address.High; in mps_push_sge()
2880 if (cm->cm_out_len) { in mps_push_sge()
2881 sge->FlagsLength = htole32(cm->cm_out_len | in mps_push_sge()
2887 cm->cm_sglsize -= len; in mps_push_sge()
2888 bcopy(sgep, cm->cm_sge, len); in mps_push_sge()
2889 cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge in mps_push_sge()
2899 if (cm->cm_flags & MPS_CM_FLAGS_DATAIN) { in mps_push_sge()
2908 sge->FlagsLength = htole32(saved_buf_len); in mps_push_sge()
2909 sge->Address.Low = saved_address_low; in mps_push_sge()
2910 sge->Address.High = saved_address_high; in mps_push_sge()
2913 cm->cm_sglsize -= len; in mps_push_sge()
2914 bcopy(sgep, cm->cm_sge, len); in mps_push_sge()
2915 cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len); in mps_push_sge()
2920 * Add one dma segment to the scatter-gather list for a command.
2929 * This driver always uses 64-bit address elements for simplicity. in mps_add_dmaseg()
2948 sc = cm->cm_sc; in mps_data_cb()
2954 if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) { in mps_data_cb()
2958 cm->cm_max_segs); in mps_data_cb()
2962 * Set up DMA direction flags. Bi-directional requests are also handled in mps_data_cb()
2966 if (cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) { in mps_data_cb()
2988 } else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT) { in mps_data_cb()
2995 if ((cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) && (i != 0)) { in mps_data_cb()
2999 sflags, nsegs - i); in mps_data_cb()
3002 if (ratecheck(&sc->lastfail, &mps_chainfail_interval)) in mps_data_cb()
3005 cm->cm_flags |= MPS_CM_FLAGS_CHAIN_FAILED; in mps_data_cb()
3012 cm->cm_state = MPS_CM_STATE_INQUEUE; in mps_data_cb()
3018 bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir); in mps_data_cb()
3035 * assumed that if you have a command in-hand, then you have enough credits
3043 if (cm->cm_flags & MPS_CM_FLAGS_USE_UIO) { in mps_map_command()
3044 error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap, in mps_map_command()
3045 &cm->cm_uio, mps_data_cb2, cm, 0); in mps_map_command()
3046 } else if (cm->cm_flags & MPS_CM_FLAGS_USE_CCB) { in mps_map_command()
3047 error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap, in mps_map_command()
3048 cm->cm_data, mps_data_cb, cm, 0); in mps_map_command()
3049 } else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) { in mps_map_command()
3050 error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap, in mps_map_command()
3051 cm->cm_data, cm->cm_length, mps_data_cb, cm, 0); in mps_map_command()
3053 /* Add a zero-length element as needed */ in mps_map_command()
3054 if (cm->cm_sge != NULL) in mps_map_command()
3075 if (sc->mps_flags & MPS_FLAGS_DIAGRESET) in mps_wait_command()
3078 cm->cm_complete = NULL; in mps_wait_command()
3079 cm->cm_flags |= MPS_CM_FLAGS_POLLED; in mps_wait_command()
3089 if (curthread->td_no_sleeping != 0) in mps_wait_command()
3092 if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) { in mps_wait_command()
3093 cm->cm_flags |= MPS_CM_FLAGS_WAKEUP; in mps_wait_command()
3094 error = msleep(cm, &sc->mps_mtx, 0, "mpswait", timeout*hz); in mps_wait_command()
3104 while ((cm->cm_flags & MPS_CM_FLAGS_COMPLETE) == 0) { in mps_wait_command()
3121 if (cm->cm_timeout_handler == NULL) { in mps_wait_command()
3129 cm->cm_timeout_handler(sc, cm); in mps_wait_command()
3130 if (sc->mps_flags & MPS_FLAGS_REALLOCATED) { in mps_wait_command()
3153 if (sc->mps_flags & MPS_FLAGS_BUSY) { in mps_read_config_page()
3162 req = (MPI2_CONFIG_REQUEST *)cm->cm_req; in mps_read_config_page()
3163 req->Function = MPI2_FUNCTION_CONFIG; in mps_read_config_page()
3164 req->Action = params->action; in mps_read_config_page()
3165 req->SGLFlags = 0; in mps_read_config_page()
3166 req->ChainOffset = 0; in mps_read_config_page()
3167 req->PageAddress = params->page_address; in mps_read_config_page()
3168 if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) { in mps_read_config_page()
3171 hdr = ¶ms->hdr.Ext; in mps_read_config_page()
3172 req->ExtPageType = hdr->ExtPageType; in mps_read_config_page()
3173 req->ExtPageLength = hdr->ExtPageLength; in mps_read_config_page()
3174 req->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED; in mps_read_config_page()
3175 req->Header.PageLength = 0; /* Must be set to zero */ in mps_read_config_page()
3176 req->Header.PageNumber = hdr->PageNumber; in mps_read_config_page()
3177 req->Header.PageVersion = hdr->PageVersion; in mps_read_config_page()
3181 hdr = ¶ms->hdr.Struct; in mps_read_config_page()
3182 req->Header.PageType = hdr->PageType; in mps_read_config_page()
3183 req->Header.PageNumber = hdr->PageNumber; in mps_read_config_page()
3184 req->Header.PageLength = hdr->PageLength; in mps_read_config_page()
3185 req->Header.PageVersion = hdr->PageVersion; in mps_read_config_page()
3188 cm->cm_data = params->buffer; in mps_read_config_page()
3189 cm->cm_length = params->length; in mps_read_config_page()
3190 if (cm->cm_data != NULL) { in mps_read_config_page()
3191 cm->cm_sge = &req->PageBufferSGE; in mps_read_config_page()
3192 cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION); in mps_read_config_page()
3193 cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE | MPS_CM_FLAGS_DATAIN; in mps_read_config_page()
3195 cm->cm_sge = NULL; in mps_read_config_page()
3196 cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; in mps_read_config_page()
3198 cm->cm_complete_data = params; in mps_read_config_page()
3199 if (params->callback != NULL) { in mps_read_config_page()
3200 cm->cm_complete = mps_config_complete; in mps_read_config_page()
3230 params = cm->cm_complete_data; in mps_config_complete()
3232 if (cm->cm_data != NULL) { in mps_config_complete()
3233 bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, in mps_config_complete()
3235 bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap); in mps_config_complete()
3242 if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) { in mps_config_complete()
3243 params->status = MPI2_IOCSTATUS_BUSY; in mps_config_complete()
3247 reply = (MPI2_CONFIG_REPLY *)cm->cm_reply; in mps_config_complete()
3249 params->status = MPI2_IOCSTATUS_BUSY; in mps_config_complete()
3252 params->status = reply->IOCStatus; in mps_config_complete()
3253 if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) { in mps_config_complete()
3254 params->hdr.Ext.ExtPageType = reply->ExtPageType; in mps_config_complete()
3255 params->hdr.Ext.ExtPageLength = reply->ExtPageLength; in mps_config_complete()
3256 params->hdr.Ext.PageType = reply->Header.PageType; in mps_config_complete()
3257 params->hdr.Ext.PageNumber = reply->Header.PageNumber; in mps_config_complete()
3258 params->hdr.Ext.PageVersion = reply->Header.PageVersion; in mps_config_complete()
3260 params->hdr.Struct.PageType = reply->Header.PageType; in mps_config_complete()
3261 params->hdr.Struct.PageNumber = reply->Header.PageNumber; in mps_config_complete()
3262 params->hdr.Struct.PageLength = reply->Header.PageLength; in mps_config_complete()
3263 params->hdr.Struct.PageVersion = reply->Header.PageVersion; in mps_config_complete()
3268 if (params->callback != NULL) in mps_config_complete()
3269 params->callback(sc, params); in mps_config_complete()