/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" /* * This file implements the MAD send logic in IBMF. */ #include #define IBMF_SEND_WR_ID_TO_ADDR(id, ptr) \ (ptr) = (void *)(uintptr_t)(id) extern int ibmf_trace_level; static void ibmf_i_do_send_cb(void *taskq_arg); static void ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp, ibmf_send_wqe_t *send_wqep); /* * ibmf_i_issue_pkt(): * Post an IB packet on the specified QP's send queue */ int ibmf_i_issue_pkt(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp, ibmf_qp_handle_t ibmf_qp_handle, ibmf_send_wqe_t *send_wqep) { int ret; ibt_status_t status; ibt_wr_ds_t sgl[1]; ibt_qp_hdl_t ibt_qp_handle; _NOTE(ASSUMING_PROTECTED(*send_wqep)) _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_start, IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() enter, clientp = %p, msg = %p, " "qp_hdl = %p, swqep = %p\n", tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp, tnf_opaque, ibmf_qp_handle, ibmf_qp_handle, tnf_opaque, send_wqep, send_wqep); ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); ASSERT(MUTEX_NOT_HELD(&clientp->ic_mutex)); /* * if the qp handle provided in ibmf_send_pkt() * is not the default qp handle for this client, * then the wqe must be sent on this qp, * else use the default qp handle set up during ibmf_register() */ if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { ibt_qp_handle = clientp->ic_qp->iq_qp_handle; } else { ibt_qp_handle = ((ibmf_alt_qp_t *)ibmf_qp_handle)->isq_qp_handle; } /* initialize the send WQE */ ibmf_i_init_send_wqe(clientp, msgimplp, sgl, send_wqep, msgimplp->im_ud_dest, ibt_qp_handle, ibmf_qp_handle); _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) /* * Issue the wqe to the transport. * NOTE: ibt_post_send() will not block, so, it is ok * to hold the msgimpl mutex across this call. */ status = ibt_post_send(send_wqep->send_qp_handle, &send_wqep->send_wr, 1, NULL); if (status != IBT_SUCCESS) { mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, send_pkt_failed, 1); mutex_exit(&clientp->ic_kstat_mutex); IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_issue_pkt_err, IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(): %s, status = %d\n", tnf_string, msg, "post send failure", tnf_uint, ibt_status, status); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(() exit\n"); return (IBMF_TRANSPORT_FAILURE); } ret = IBMF_SUCCESS; /* bump the number of active sends */ if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&clientp->ic_mutex); clientp->ic_sends_active++; mutex_exit(&clientp->ic_mutex); mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, sends_active, 1); mutex_exit(&clientp->ic_kstat_mutex); } else { ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)ibmf_qp_handle; mutex_enter(&qpp->isq_mutex); qpp->isq_sends_active++; mutex_exit(&qpp->isq_mutex); mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, sends_active, 1); mutex_exit(&clientp->ic_kstat_mutex); } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() exit\n"); return (ret); } /* * ibmf_i_send_pkt() * Send an IB packet after allocating send resources */ int ibmf_i_send_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle, ibmf_msg_impl_t *msgimplp, int block) { ibmf_send_wqe_t *send_wqep; int status; IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_start, IBMF_TNF_TRACE, "", "ibmf_i_send_pkt(): clientp = 0x%p, qp_hdl = 0x%p, " "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp, tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp, tnf_uint, block, block); ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) /* * Reset send_done to indicate we have not received the completion * for this send yet. */ msgimplp->im_trans_state_flags &= ~IBMF_TRANS_STATE_FLAG_SEND_DONE; /* * Allocate resources needed to send a UD packet including the * send WQE context */ status = ibmf_i_alloc_send_resources(clientp->ic_myci, msgimplp, block, &send_wqep); if (status != IBMF_SUCCESS) { IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err, IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n", tnf_string, msg, "unable to allocate send resources", tnf_uint, status, status); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n"); return (status); } /* Set the segment number in the send WQE context */ if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) send_wqep->send_rmpp_segment = msgimplp->im_rmpp_ctx.rmpp_ns; _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) /* * Increment the count of pending send completions. * Only when this count is zero should the client be notified * of completion of the transaction. */ msgimplp->im_pending_send_compls += 1; /* Send the packet */ status = ibmf_i_issue_pkt(clientp, msgimplp, ibmf_qp_handle, send_wqep); if (status != IBMF_SUCCESS) { ibmf_i_free_send_resources(clientp->ic_myci, msgimplp, send_wqep); IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err, IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n", tnf_string, msg, "unable to issue packet", tnf_uint, status, status); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n"); return (status); } IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit, status = %d\n", tnf_uint, status, status); return (IBMF_SUCCESS); } /* * ibmf_i_send_single_pkt(): * Send a single IB packet. Only used to send non-RMPP packets. */ int ibmf_i_send_single_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle, ibmf_msg_impl_t *msgimplp, int block) { int status; IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_start, IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt(): clientp = 0x%p, qp_hdl = 0x%p, " "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp, tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp, tnf_uint, block, block); ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); status = ibmf_i_send_pkt(clientp, ibmf_qp_handle, msgimplp, block); if (status != IBMF_SUCCESS) { IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_single_pkt_err, IBMF_TNF_ERROR, "", "ibmf_i_send_single_pkt(): %s, msgp = 0x%p\n", tnf_string, msg, "unable to send packet", tnf_uint, status, status); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt() exit\n"); return (status); } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_end, IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt() exit\n"); return (IBMF_SUCCESS); } /* * ibmf_i_handle_send_completion(): * Process the WQE from the SQ identified in the work completion entry. */ /* ARGSUSED */ void ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp) { ibmf_client_t *clientp, *cclientp; ibmf_send_wqe_t *send_wqep; ibmf_qp_handle_t ibmf_qp_handle; ibmf_alt_qp_t *qpp; int ret; IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n", tnf_opaque, cip, cip, tnf_opaque, wcp, wcp); _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) ASSERT(wcp->wc_id != NULL); ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id)); /* get the IBMF send WQE context */ IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep); ASSERT(send_wqep != NULL); /* get the client context */ cclientp = clientp = send_wqep->send_client; /* Check if this is a completion for a BUSY MAD sent by IBMF */ if (clientp == NULL) { ibmf_msg_impl_t *msgimplp; IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion(): NULL client\n"); msgimplp = send_wqep->send_msg; /* * Deregister registered memory and free it, and * free up the send WQE context */ (void) ibt_deregister_mr(cip->ci_ci_handle, send_wqep->send_mem_hdl); kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE); kmem_free(send_wqep, sizeof (ibmf_send_wqe_t)); /* Free up the message context */ ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest); ibmf_i_clean_ud_dest_list(cip, B_FALSE); kmem_free(msgimplp, sizeof (ibmf_msg_impl_t)); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); return; } /* get the QP handle */ ibmf_qp_handle = send_wqep->send_ibmf_qp_handle; qpp = (ibmf_alt_qp_t *)ibmf_qp_handle; ASSERT(clientp != NULL); /* decrement the number of active sends */ if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&clientp->ic_mutex); clientp->ic_sends_active--; mutex_exit(&clientp->ic_mutex); } else { mutex_enter(&qpp->isq_mutex); qpp->isq_sends_active--; mutex_exit(&qpp->isq_mutex); } mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, sends_active, 1); mutex_exit(&clientp->ic_kstat_mutex); send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status); /* * issue the callback using taskq. If no taskq or if the * dispatch fails, we do the send processing in the callback context * which is the interrupt context */ if (cclientp->ic_send_taskq == NULL) { /* Do the processing in callback context */ mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); ibmf_i_do_send_cb((void *)send_wqep); mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", "ibmf_i_handle_send_completion(): %s\n", tnf_string, msg, "ci_send_taskq == NULL"); IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); return; } mutex_enter(&clientp->ic_kstat_mutex); IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); /* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */ if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) { ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb, send_wqep, TQ_NOSLEEP); if (ret == 0) { IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", "ibmf_i_handle_send_completion(): %s\n", tnf_string, msg, "send: dispatch failed"); ibmf_i_do_send_cb((void *)send_wqep); } } else { ibmf_i_do_send_cb((void *)send_wqep); } mutex_enter(&clientp->ic_kstat_mutex); IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); mutex_exit(&clientp->ic_kstat_mutex); _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", "ibmf_i_handle_send_completion() exit\n"); } /* * ibmf_i_do_send_cb(): * Do the send completion processing */ static void ibmf_i_do_send_cb(void *taskq_arg) { ibmf_ci_t *cip; ibmf_msg_impl_t *msgimplp; ibmf_client_t *clientp; ibmf_send_wqe_t *send_wqep; boolean_t found; int msg_trans_state_flags, msg_flags; uint_t ref_cnt; ibmf_qp_handle_t ibmf_qp_handle; struct kmem_cache *kmem_cachep; timeout_id_t msg_rp_unset_id, msg_tr_unset_id; timeout_id_t msg_rp_set_id, msg_tr_set_id; ibmf_alt_qp_t *altqp; boolean_t inc_refcnt; send_wqep = taskq_arg; IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_cb_start, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb() enter, send_wqep = %p\n", tnf_opaque, send_wqep, send_wqep); clientp = send_wqep->send_client; cip = clientp->ic_myci; msgimplp = send_wqep->send_msg; /* get the QP handle */ ibmf_qp_handle = send_wqep->send_ibmf_qp_handle; /* Get the WQE kmem cache pointer based on the QP type */ if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) kmem_cachep = cip->ci_send_wqes_cache; else { altqp = (ibmf_alt_qp_t *)ibmf_qp_handle; kmem_cachep = altqp->isq_send_wqes_cache; } /* Look for a message in the client's message list */ inc_refcnt = B_TRUE; found = ibmf_i_find_msg_client(clientp, msgimplp, inc_refcnt); /* * If the message context was not found, then it's likely * been freed up. So, do nothing in this timeout handler */ if (found == B_FALSE) { kmem_cache_free(kmem_cachep, send_wqep); mutex_enter(&cip->ci_mutex); IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); mutex_exit(&cip->ci_mutex); if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&cip->ci_mutex); cip->ci_wqes_alloced--; if (cip->ci_wqes_alloced == 0) cv_signal(&cip->ci_wqes_cv); mutex_exit(&cip->ci_mutex); } else { mutex_enter(&altqp->isq_mutex); altqp->isq_wqes_alloced--; if (altqp->isq_wqes_alloced == 0) cv_signal(&altqp->isq_wqes_cv); mutex_exit(&altqp->isq_mutex); } IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb(): %s\n", tnf_string, msg, "Message not found, return without processing send cb"); return; } /* Grab the message context lock */ mutex_enter(&msgimplp->im_mutex); /* * Decrement the count of pending send completions for * this transaction */ msgimplp->im_pending_send_compls -= 1; /* * If the pending send completions is not zero, then we must * not attempt to notify the client of a transaction completion * in this instance of the send completion handler. Notification * of transaction completion should be provided only by the * last send completion so that all send completions are accounted * for before the client is notified and subsequently attempts to * reuse the message for an other transaction. * If this is not done, the message may be reused while the * send WR from the old transaction is still active in the QP's WQ. * This could result in an attempt to modify the address handle with * information for the new transaction which could be potentially * incompatible, such as an incorrect port number. Such an * incompatible modification of the address handle of the old * transaction could result in a QP error. */ if (msgimplp->im_pending_send_compls != 0) { IBMF_MSG_DECR_REFCNT(msgimplp); mutex_exit(&msgimplp->im_mutex); kmem_cache_free(kmem_cachep, send_wqep); mutex_enter(&cip->ci_mutex); IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); mutex_exit(&cip->ci_mutex); if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&cip->ci_mutex); cip->ci_wqes_alloced--; if (cip->ci_wqes_alloced == 0) cv_signal(&cip->ci_wqes_cv); mutex_exit(&cip->ci_mutex); } else { mutex_enter(&altqp->isq_mutex); altqp->isq_wqes_alloced--; if (altqp->isq_wqes_alloced == 0) cv_signal(&altqp->isq_wqes_cv); mutex_exit(&altqp->isq_mutex); } IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb(): %s\n", tnf_string, msg, "Message found with pending send completions, " "return without processing send cb"); return; } /* * If the message has been marked unitialized or done * release the message mutex and return */ if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) || (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) { IBMF_MSG_DECR_REFCNT(msgimplp); msg_trans_state_flags = msgimplp->im_trans_state_flags; msg_flags = msgimplp->im_flags; ref_cnt = msgimplp->im_ref_count; mutex_exit(&msgimplp->im_mutex); /* * This thread may notify the client only if the * transaction is done, the message has been removed * from the client's message list, and the message * reference count is 0. * If the transaction is done, and the message reference * count = 0, there is still a possibility that a * packet could arrive for the message and its reference * count increased if the message is still on the list. * If the message is still on the list, it will be * removed by a call to ibmf_i_client_rem_msg() at * the completion point of the transaction. * So, the reference count should be checked after the * message has been removed. */ if ((msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) && !(msg_flags & IBMF_MSG_FLAGS_ON_LIST) && (ref_cnt == 0)) { ibmf_i_notify_sequence(clientp, msgimplp, msg_flags); } kmem_cache_free(kmem_cachep, send_wqep); mutex_enter(&cip->ci_mutex); IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); mutex_exit(&cip->ci_mutex); if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&cip->ci_mutex); cip->ci_wqes_alloced--; if (cip->ci_wqes_alloced == 0) cv_signal(&cip->ci_wqes_cv); mutex_exit(&cip->ci_mutex); } else { mutex_enter(&altqp->isq_mutex); altqp->isq_wqes_alloced--; if (altqp->isq_wqes_alloced == 0) cv_signal(&altqp->isq_wqes_cv); mutex_exit(&altqp->isq_mutex); } IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb(): %s, msg = %p\n", tnf_string, msg, "Message marked for removal, return without processing " "send cb", tnf_opaque, msgimplp, msgimplp); return; } /* Perform send completion processing of the message context */ ibmf_i_do_send_compl((ibmf_handle_t)clientp, msgimplp, send_wqep); msg_rp_unset_id = msg_tr_unset_id = msg_rp_set_id = msg_tr_set_id = 0; /* Save the message flags before releasing the mutex */ msg_trans_state_flags = msgimplp->im_trans_state_flags; msg_flags = msgimplp->im_flags; msg_rp_unset_id = msgimplp->im_rp_unset_timeout_id; msg_tr_unset_id = msgimplp->im_tr_unset_timeout_id; msgimplp->im_rp_unset_timeout_id = 0; msgimplp->im_tr_unset_timeout_id = 0; /* * Decrement the message reference count * This count was inceremented when the message was found on the * client's message list */ IBMF_MSG_DECR_REFCNT(msgimplp); if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) { if (msgimplp->im_rp_timeout_id != 0) { msg_rp_set_id = msgimplp->im_rp_timeout_id; msgimplp->im_rp_timeout_id = 0; } if (msgimplp->im_tr_timeout_id != 0) { msg_tr_set_id = msgimplp->im_tr_timeout_id; msgimplp->im_tr_timeout_id = 0; } } mutex_exit(&msgimplp->im_mutex); if (msg_rp_unset_id != 0) { (void) untimeout(msg_rp_unset_id); } if (msg_tr_unset_id != 0) { (void) untimeout(msg_tr_unset_id); } if (msg_rp_set_id != 0) { (void) untimeout(msg_rp_set_id); } if (msg_tr_set_id != 0) { (void) untimeout(msg_tr_set_id); } IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb(): %s, msg = %p\n", tnf_string, msg, "Send callback done. Dec ref count", tnf_opaque, msgimplp, msgimplp); /* * If the transaction is done, signal the block thread if the * transaction is blocking, or call the client's transaction done * notification callback */ if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) { /* Remove the message from the client's message list */ ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt); /* * Notify the client if the message reference count is zero. * At this point, we know that the transaction is done and * the message has been removed from the client's message list. * So, we only need to make sure the reference count is zero * before notifying the client. */ if (ref_cnt == 0) { ibmf_i_notify_sequence(clientp, msgimplp, msg_flags); } } kmem_cache_free(kmem_cachep, send_wqep); mutex_enter(&cip->ci_mutex); IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); mutex_exit(&cip->ci_mutex); if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { mutex_enter(&cip->ci_mutex); cip->ci_wqes_alloced--; if (cip->ci_wqes_alloced == 0) cv_signal(&cip->ci_wqes_cv); mutex_exit(&cip->ci_mutex); } else { mutex_enter(&altqp->isq_mutex); altqp->isq_wqes_alloced--; if (altqp->isq_wqes_alloced == 0) cv_signal(&altqp->isq_wqes_cv); mutex_exit(&altqp->isq_mutex); } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_cb_end, IBMF_TNF_TRACE, "", "ibmf_i_do_send_cb() exit\n"); } /* * ibmf_i_do_send_compl(): * Determine if the transaction is complete */ /* ARGSUSED */ static void ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp, ibmf_send_wqe_t *send_wqep) { IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_start, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): ibmf_hdl = 0x%p " "msgp = %p, send_wqep = 0x%p, msg_flags = 0x%x\n", tnf_opaque, ibmf_hdl, ibmf_handle, tnf_opaque, msgimplp, msgimplp, tnf_opaque, send_wqep, send_wqep, tnf_opaque, msg_flags, msgimplp->im_flags); ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); /* * For RMPP transactions, we only care about the final packet of the * transaction. For others, the code does not need to wait for the send * completion (although bad things can happen if it never occurs). * The final packets of a transaction are sent when the state is either * ABORT or RECEVR_TERMINATE. * Don't mark the transaction as send_done if there are still more * packets to be sent, including doing the second part of a double-sided * transaction. */ if ((msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) || (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)) { IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_compl, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): %s msgp = %p, rmpp_state = 0x%x\n", tnf_string, msg, "Received send callback for RMPP trans", tnf_opaque, msg, msgimplp, tnf_opaque, rmpp_state, msgimplp->im_rmpp_ctx.rmpp_state); /* * For ABORT state, we should not return control to * the client from the send completion handler. * Control should be returned in the error timeout handler. * * The exception is when the IBMF_TRANS_STATE_FLAG_RECV_DONE * flag has already been set. This flag is set when * ibmf_i_terminate_transaction is called from one of the * three timeout handlers. In this case return control from * here. */ if (msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_ABORT) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SEND_DONE; if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_RECV_DONE) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; } } if ((msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_RECEVR_TERMINATE) || (msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_DONE)) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SEND_DONE; if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_RECV_DONE) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; } } /* * If the transaction is a send-only RMPP, then * set the SEND_DONE flag on every send completion * as long as there are no outstanding ones. * This is needed so that the transaction can return * in the receive path, where ibmf_i_terminate_transaction * is called from ibmf_i_rmpp_sender_active_flow, * after checking if the SEND_DONE flag is set. * When a new MAD is sent as part of the RMPP transaction, * the SEND_DONE flag will get reset. * The RECV_DONE indicates that the last ACK was received. */ if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) { if (msgimplp->im_pending_send_compls == 0) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SEND_DONE; if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_RECV_DONE) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; } } } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit\n"); return; } /* * Only non-RMPP send completion gets here. * If the send is a single-packet send that does not use RMPP, and if * the transaction is not a sequenced transaction, call the transaction * callback handler after flagging the transaction as done. If the * message is sequenced, start a timer to bound the wait for the first * data packet of the response. */ if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) { IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_do_send_compl, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): %s msgp = %p\n", tnf_string, msg, "Sequenced transaction, setting response timer", tnf_opaque, msg, msgimplp); /* * Check if the send completion already occured, * which could imply that this is a send completion * for some previous transaction that has come in very late. * In this case exit here. */ if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_SEND_DONE) { IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit, " "Duplicate SEND completion\n"); return; } /* mark as send_compl happened */ msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SEND_DONE; if (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_RECV_DONE) { msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit, RECV_DONE\n"); return; } /* * check if response was received before send * completion */ if (((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) == 0) && ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_RECV_ACTIVE) == 0)) { /* set timer for first packet of response */ ibmf_i_set_timer(ibmf_i_send_timeout, msgimplp, IBMF_RESP_TIMER); } } else { msgimplp->im_msg_status = IBMF_SUCCESS; msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_SEND_DONE; msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; } IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit\n"); }