Lines Matching +full:cmd +full:- +full:db
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
43 #include "mlx4-abi.h"
48 struct ibv_query_device cmd; in mlx4_query_device() local
53 ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd); in mlx4_query_device()
61 snprintf(attr->fw_ver, sizeof attr->fw_ver, in mlx4_query_device()
74 struct mlx4_query_device_ex cmd = {}; in mlx4_query_device_ex() local
83 &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), in mlx4_query_device_ex()
90 mctx->core_clock.offset = resp.hca_core_clock_offset; in mlx4_query_device_ex()
91 mctx->core_clock.offset_valid = 1; in mlx4_query_device_ex()
98 snprintf(attr->orig_attr.fw_ver, sizeof attr->orig_attr.fw_ver, in mlx4_query_device_ex()
111 if (!ctx->hca_core_clock) in mlx4_read_clock()
112 return -EOPNOTSUPP; in mlx4_read_clock()
116 clockhi = be32toh(READL(ctx->hca_core_clock)); in mlx4_read_clock()
117 clocklo = be32toh(READL(ctx->hca_core_clock + 4)); in mlx4_read_clock()
118 clockhi1 = be32toh(READL(ctx->hca_core_clock)); in mlx4_read_clock()
134 if (values->comp_mask & IBV_VALUES_MASK_RAW_CLOCK) { in mlx4_query_rt_values()
139 values->raw_clock.tv_sec = 0; in mlx4_query_rt_values()
140 values->raw_clock.tv_nsec = cycles; in mlx4_query_rt_values()
145 values->comp_mask = comp_mask; in mlx4_query_rt_values()
153 struct ibv_query_port cmd; in mlx4_query_port() local
156 err = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); in mlx4_query_port()
159 if (!mctx->port_query_cache[port - 1].valid) { in mlx4_query_port()
160 mctx->port_query_cache[port - 1].link_layer = in mlx4_query_port()
161 attr->link_layer; in mlx4_query_port()
162 mctx->port_query_cache[port - 1].caps = in mlx4_query_port()
163 attr->port_cap_flags; in mlx4_query_port()
164 mctx->port_query_cache[port - 1].valid = 1; in mlx4_query_port()
177 return -EINVAL; in query_port_cache()
178 if (mctx->port_query_cache[port_num - 1].valid) { in query_port_cache()
179 port_attr->link_layer = in query_port_cache()
180 mctx-> in query_port_cache()
181 port_query_cache[port_num - 1]. in query_port_cache()
183 port_attr->port_cap_flags = in query_port_cache()
184 mctx-> in query_port_cache()
185 port_query_cache[port_num - 1]. in query_port_cache()
196 struct ibv_alloc_pd cmd; in mlx4_alloc_pd() local
204 if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd, in mlx4_alloc_pd()
210 pd->pdn = resp.pdn; in mlx4_alloc_pd()
212 return &pd->ibv_pd; in mlx4_alloc_pd()
230 struct ibv_open_xrcd cmd; in mlx4_open_xrcd() local
240 &cmd, sizeof cmd, &resp, sizeof resp); in mlx4_open_xrcd()
244 return &xrcd->xrcd; in mlx4_open_xrcd()
267 struct ibv_reg_mr cmd; in mlx4_reg_mr() local
276 access, mr, &cmd, sizeof cmd, in mlx4_reg_mr()
291 struct ibv_rereg_mr cmd; in mlx4_rereg_mr() local
300 &cmd, sizeof(cmd), in mlx4_rereg_mr()
319 struct ibv_alloc_mw cmd; in mlx4_alloc_mw() local
327 ret = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), in mlx4_alloc_mw()
341 struct ibv_dealloc_mw cmd; in mlx4_dealloc_mw() local
343 ret = ibv_cmd_dealloc_mw(mw, &cmd, sizeof(cmd)); in mlx4_dealloc_mw()
362 wr.wr_id = mw_bind->wr_id; in mlx4_bind_mw()
363 wr.send_flags = mw_bind->send_flags; in mlx4_bind_mw()
366 wr.bind_mw.rkey = ibv_inc_rkey(mw->rkey); in mlx4_bind_mw()
367 wr.bind_mw.bind_info = mw_bind->bind_info; in mlx4_bind_mw()
375 mw->rkey = wr.bind_mw.rkey; in mlx4_bind_mw()
408 struct mlx4_create_cq cmd = {}; in mlx4_cmd_create_cq() local
412 cmd.buf_addr = (uintptr_t) cq->buf.buf; in mlx4_cmd_create_cq()
413 cmd.db_addr = (uintptr_t) cq->set_ci_db; in mlx4_cmd_create_cq()
415 ret = ibv_cmd_create_cq(context, cq_attr->cqe, cq_attr->channel, in mlx4_cmd_create_cq()
416 cq_attr->comp_vector, in mlx4_cmd_create_cq()
417 ibv_cq_ex_to_cq(&cq->ibv_cq), in mlx4_cmd_create_cq()
418 &cmd.ibv_cmd, sizeof(cmd), in mlx4_cmd_create_cq()
421 cq->cqn = resp.cqn; in mlx4_cmd_create_cq()
431 struct mlx4_create_cq_ex cmd = {}; in mlx4_cmd_create_cq_ex() local
435 cmd.buf_addr = (uintptr_t) cq->buf.buf; in mlx4_cmd_create_cq_ex()
436 cmd.db_addr = (uintptr_t) cq->set_ci_db; in mlx4_cmd_create_cq_ex()
439 &cq->ibv_cq, &cmd.ibv_cmd, in mlx4_cmd_create_cq_ex()
440 sizeof(cmd.ibv_cmd), in mlx4_cmd_create_cq_ex()
441 sizeof(cmd), in mlx4_cmd_create_cq_ex()
446 cq->cqn = resp.cqn; in mlx4_cmd_create_cq_ex()
460 if (cq_attr->cqe > 0x3fffff) { in create_cq()
465 if (cq_attr->comp_mask & ~CREATE_CQ_SUPPORTED_COMP_MASK) { in create_cq()
470 if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && in create_cq()
471 cq_attr->flags & ~CREATE_CQ_SUPPORTED_FLAGS) { in create_cq()
476 if (cq_attr->wc_flags & ~CREATE_CQ_SUPPORTED_WC_FLAGS) in create_cq()
482 if ((cq_attr->wc_flags & (IBV_WC_EX_WITH_SLID | IBV_WC_EX_WITH_SL)) && in create_cq()
483 (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)) { in create_cq()
492 cq->cons_index = 0; in create_cq()
494 if (pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE)) in create_cq()
497 cq_attr->cqe = align_queue_size(cq_attr->cqe + 1); in create_cq()
499 if (mlx4_alloc_cq_buf(to_mdev(context->device), &cq->buf, cq_attr->cqe, mctx->cqe_size)) in create_cq()
502 cq->cqe_size = mctx->cqe_size; in create_cq()
503 cq->set_ci_db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_CQ); in create_cq()
504 if (!cq->set_ci_db) in create_cq()
507 cq->arm_db = cq->set_ci_db + 1; in create_cq()
508 *cq->arm_db = 0; in create_cq()
509 cq->arm_sn = 1; in create_cq()
510 *cq->set_ci_db = 0; in create_cq()
511 cq->flags = cq_alloc_flags; in create_cq()
513 if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && in create_cq()
514 cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) in create_cq()
515 cq->flags |= MLX4_CQ_FLAGS_SINGLE_THREADED; in create_cq()
517 --cq_attr->cqe; in create_cq()
530 return &cq->ibv_cq; in create_cq()
533 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_CQ, cq->set_ci_db); in create_cq()
536 mlx4_free_buf(&cq->buf); in create_cq()
539 pthread_spin_destroy(&cq->lock); in create_cq()
567 struct ibv_cq_init_attr_ex cq_attr_c = {.cqe = cq_attr->cqe, in mlx4_create_cq_ex()
568 .channel = cq_attr->channel, in mlx4_create_cq_ex()
569 .comp_vector = cq_attr->comp_vector, in mlx4_create_cq_ex()
570 .wc_flags = cq_attr->wc_flags, in mlx4_create_cq_ex()
571 .comp_mask = cq_attr->comp_mask, in mlx4_create_cq_ex()
572 .flags = cq_attr->flags}; in mlx4_create_cq_ex()
580 struct mlx4_resize_cq cmd; in mlx4_resize_cq() local
589 pthread_spin_lock(&cq->lock); in mlx4_resize_cq()
592 if (cqe == ibcq->cqe + 1) { in mlx4_resize_cq()
604 ret = mlx4_alloc_cq_buf(to_mdev(ibcq->context->device), &buf, cqe, cq->cqe_size); in mlx4_resize_cq()
608 old_cqe = ibcq->cqe; in mlx4_resize_cq()
609 cmd.buf_addr = (uintptr_t) buf.buf; in mlx4_resize_cq()
611 ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd, in mlx4_resize_cq()
620 mlx4_free_buf(&cq->buf); in mlx4_resize_cq()
621 cq->buf = buf; in mlx4_resize_cq()
625 pthread_spin_unlock(&cq->lock); in mlx4_resize_cq()
638 pthread_spin_destroy(&to_mcq(cq)->lock); in mlx4_destroy_cq()
639 mlx4_free_db(to_mctx(cq->context), MLX4_DB_TYPE_CQ, to_mcq(cq)->set_ci_db); in mlx4_destroy_cq()
640 mlx4_free_buf(&to_mcq(cq)->buf); in mlx4_destroy_cq()
649 struct mlx4_create_srq cmd; in mlx4_create_srq() local
655 if (attr->attr.max_wr > 1 << 16 || attr->attr.max_sge > 64) in mlx4_create_srq()
662 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) in mlx4_create_srq()
665 srq->max = align_queue_size(attr->attr.max_wr + 1); in mlx4_create_srq()
666 srq->max_gs = attr->attr.max_sge; in mlx4_create_srq()
667 srq->counter = 0; in mlx4_create_srq()
668 srq->ext_srq = 0; in mlx4_create_srq()
670 if (mlx4_alloc_srq_buf(pd, &attr->attr, srq)) in mlx4_create_srq()
673 srq->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ); in mlx4_create_srq()
674 if (!srq->db) in mlx4_create_srq()
677 *srq->db = 0; in mlx4_create_srq()
679 cmd.buf_addr = (uintptr_t) srq->buf.buf; in mlx4_create_srq()
680 cmd.db_addr = (uintptr_t) srq->db; in mlx4_create_srq()
682 ret = ibv_cmd_create_srq(pd, &srq->verbs_srq.srq, attr, in mlx4_create_srq()
683 &cmd.ibv_cmd, sizeof cmd, in mlx4_create_srq()
688 return &srq->verbs_srq.srq; in mlx4_create_srq()
691 mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, srq->db); in mlx4_create_srq()
694 free(srq->wrid); in mlx4_create_srq()
695 mlx4_free_buf(&srq->buf); in mlx4_create_srq()
698 pthread_spin_destroy(&srq->lock); in mlx4_create_srq()
709 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) || in mlx4_create_srq_ex()
710 (attr_ex->srq_type == IBV_SRQT_BASIC)) in mlx4_create_srq_ex()
711 return mlx4_create_srq(attr_ex->pd, (struct ibv_srq_init_attr *) attr_ex); in mlx4_create_srq_ex()
712 else if (attr_ex->srq_type == IBV_SRQT_XRC) in mlx4_create_srq_ex()
722 struct ibv_modify_srq cmd; in mlx4_modify_srq() local
724 return ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof cmd); in mlx4_modify_srq()
730 struct ibv_query_srq cmd; in mlx4_query_srq() local
732 return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd); in mlx4_query_srq()
739 if (to_msrq(srq)->ext_srq) in mlx4_destroy_srq()
746 mlx4_free_db(to_mctx(srq->context), MLX4_DB_TYPE_RQ, to_msrq(srq)->db); in mlx4_destroy_srq()
747 mlx4_free_buf(&to_msrq(srq)->buf); in mlx4_destroy_srq()
748 free(to_msrq(srq)->wrid); in mlx4_destroy_srq()
749 pthread_spin_destroy(&to_msrq(srq)->lock); in mlx4_destroy_srq()
757 struct mlx4_create_qp *cmd, in mlx4_cmd_create_qp_ex() argument
765 memcpy(&cmd_ex.ibv_cmd.base, &cmd->ibv_cmd.user_handle, in mlx4_cmd_create_qp_ex()
766 offsetof(typeof(cmd->ibv_cmd), is_srq) + in mlx4_cmd_create_qp_ex()
767 sizeof(cmd->ibv_cmd.is_srq) - in mlx4_cmd_create_qp_ex()
768 offsetof(typeof(cmd->ibv_cmd), user_handle)); in mlx4_cmd_create_qp_ex()
770 memcpy(&cmd_ex.drv_ex, &cmd->buf_addr, in mlx4_cmd_create_qp_ex()
771 offsetof(typeof(*cmd), sq_no_prefetch) + in mlx4_cmd_create_qp_ex()
772 sizeof(cmd->sq_no_prefetch) - sizeof(cmd->ibv_cmd)); in mlx4_cmd_create_qp_ex()
774 ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, in mlx4_cmd_create_qp_ex()
775 sizeof(qp->verbs_qp), attr, in mlx4_cmd_create_qp_ex()
796 struct mlx4_create_qp cmd; in mlx4_create_qp_ex() local
802 if (ctx->max_qp_wr) { /* mlx4_query_device succeeded */ in mlx4_create_qp_ex()
803 if (attr->cap.max_send_wr > ctx->max_qp_wr || in mlx4_create_qp_ex()
804 attr->cap.max_recv_wr > ctx->max_qp_wr || in mlx4_create_qp_ex()
805 attr->cap.max_send_sge > ctx->max_sge || in mlx4_create_qp_ex()
806 attr->cap.max_recv_sge > ctx->max_sge) in mlx4_create_qp_ex()
809 if (attr->cap.max_send_wr > 65536 || in mlx4_create_qp_ex()
810 attr->cap.max_recv_wr > 65536 || in mlx4_create_qp_ex()
811 attr->cap.max_send_sge > 64 || in mlx4_create_qp_ex()
812 attr->cap.max_recv_sge > 64) in mlx4_create_qp_ex()
815 if (attr->cap.max_inline_data > 1024) in mlx4_create_qp_ex()
818 if (attr->comp_mask & ~MLX4_CREATE_QP_SUP_COMP_MASK) in mlx4_create_qp_ex()
825 if (attr->qp_type == IBV_QPT_XRC_RECV) { in mlx4_create_qp_ex()
826 attr->cap.max_send_wr = qp->sq.wqe_cnt = 0; in mlx4_create_qp_ex()
828 mlx4_calc_sq_wqe_size(&attr->cap, attr->qp_type, qp); in mlx4_create_qp_ex()
833 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1; in mlx4_create_qp_ex()
834 qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes); in mlx4_create_qp_ex()
837 if (attr->srq || attr->qp_type == IBV_QPT_XRC_SEND || in mlx4_create_qp_ex()
838 attr->qp_type == IBV_QPT_XRC_RECV) { in mlx4_create_qp_ex()
839 attr->cap.max_recv_wr = qp->rq.wqe_cnt = attr->cap.max_recv_sge = 0; in mlx4_create_qp_ex()
841 qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr); in mlx4_create_qp_ex()
842 if (attr->cap.max_recv_sge < 1) in mlx4_create_qp_ex()
843 attr->cap.max_recv_sge = 1; in mlx4_create_qp_ex()
844 if (attr->cap.max_recv_wr < 1) in mlx4_create_qp_ex()
845 attr->cap.max_recv_wr = 1; in mlx4_create_qp_ex()
848 if (mlx4_alloc_qp_buf(context, &attr->cap, attr->qp_type, qp)) in mlx4_create_qp_ex()
853 if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE)) in mlx4_create_qp_ex()
855 if (pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) in mlx4_create_qp_ex()
858 if (attr->cap.max_recv_sge) { in mlx4_create_qp_ex()
859 qp->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ); in mlx4_create_qp_ex()
860 if (!qp->db) in mlx4_create_qp_ex()
863 *qp->db = 0; in mlx4_create_qp_ex()
864 cmd.db_addr = (uintptr_t) qp->db; in mlx4_create_qp_ex()
866 cmd.db_addr = 0; in mlx4_create_qp_ex()
869 cmd.buf_addr = (uintptr_t) qp->buf.buf; in mlx4_create_qp_ex()
870 cmd.log_sq_stride = qp->sq.wqe_shift; in mlx4_create_qp_ex()
871 for (cmd.log_sq_bb_count = 0; in mlx4_create_qp_ex()
872 qp->sq.wqe_cnt > 1 << cmd.log_sq_bb_count; in mlx4_create_qp_ex()
873 ++cmd.log_sq_bb_count) in mlx4_create_qp_ex()
875 cmd.sq_no_prefetch = 0; /* OK for ABI 2: just a reserved field */ in mlx4_create_qp_ex()
876 memset(cmd.reserved, 0, sizeof cmd.reserved); in mlx4_create_qp_ex()
877 pthread_mutex_lock(&to_mctx(context)->qp_table_mutex); in mlx4_create_qp_ex()
879 if (attr->comp_mask & MLX4_CREATE_QP_EX2_COMP_MASK) in mlx4_create_qp_ex()
880 ret = mlx4_cmd_create_qp_ex(context, attr, &cmd, qp); in mlx4_create_qp_ex()
882 ret = ibv_cmd_create_qp_ex(context, &qp->verbs_qp, in mlx4_create_qp_ex()
883 sizeof(qp->verbs_qp), attr, in mlx4_create_qp_ex()
884 &cmd.ibv_cmd, sizeof(cmd), &resp, in mlx4_create_qp_ex()
889 if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) { in mlx4_create_qp_ex()
890 ret = mlx4_store_qp(to_mctx(context), qp->verbs_qp.qp.qp_num, qp); in mlx4_create_qp_ex()
894 pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex); in mlx4_create_qp_ex()
896 qp->rq.wqe_cnt = qp->rq.max_post = attr->cap.max_recv_wr; in mlx4_create_qp_ex()
897 qp->rq.max_gs = attr->cap.max_recv_sge; in mlx4_create_qp_ex()
898 if (attr->qp_type != IBV_QPT_XRC_RECV) in mlx4_create_qp_ex()
899 mlx4_set_sq_sizes(qp, &attr->cap, attr->qp_type); in mlx4_create_qp_ex()
901 qp->doorbell_qpn = htobe32(qp->verbs_qp.qp.qp_num << 8); in mlx4_create_qp_ex()
902 if (attr->sq_sig_all) in mlx4_create_qp_ex()
903 qp->sq_signal_bits = htobe32(MLX4_WQE_CTRL_CQ_UPDATE); in mlx4_create_qp_ex()
905 qp->sq_signal_bits = 0; in mlx4_create_qp_ex()
907 return &qp->verbs_qp.qp; in mlx4_create_qp_ex()
910 ibv_cmd_destroy_qp(&qp->verbs_qp.qp); in mlx4_create_qp_ex()
913 pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex); in mlx4_create_qp_ex()
914 if (attr->cap.max_recv_sge) in mlx4_create_qp_ex()
915 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, qp->db); in mlx4_create_qp_ex()
917 pthread_spin_destroy(&qp->rq.lock); in mlx4_create_qp_ex()
919 pthread_spin_destroy(&qp->sq.lock); in mlx4_create_qp_ex()
921 free(qp->sq.wrid); in mlx4_create_qp_ex()
922 if (qp->rq.wqe_cnt) in mlx4_create_qp_ex()
923 free(qp->rq.wrid); in mlx4_create_qp_ex()
924 mlx4_free_buf(&qp->buf); in mlx4_create_qp_ex()
940 qp = mlx4_create_qp_ex(pd->context, &attr_ex); in mlx4_create_qp()
948 struct ibv_open_qp cmd; in mlx4_open_qp() local
957 ret = ibv_cmd_open_qp(context, &qp->verbs_qp, sizeof(qp->verbs_qp), attr, in mlx4_open_qp()
958 &cmd, sizeof cmd, &resp, sizeof resp); in mlx4_open_qp()
962 return &qp->verbs_qp.qp; in mlx4_open_qp()
973 struct ibv_query_qp cmd; in mlx4_query_qp() local
977 ret = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof cmd); in mlx4_query_qp()
981 init_attr->cap.max_send_wr = qp->sq.max_post; in mlx4_query_qp()
982 init_attr->cap.max_send_sge = qp->sq.max_gs; in mlx4_query_qp()
983 init_attr->cap.max_inline_data = qp->max_inline_data; in mlx4_query_qp()
985 attr->cap = init_attr->cap; in mlx4_query_qp()
993 struct ibv_modify_qp cmd = {}; in mlx4_modify_qp() local
1001 ret = ibv_query_port(qp->context, attr->port_num, in mlx4_modify_qp()
1005 mqp->link_layer = port_attr.link_layer; in mlx4_modify_qp()
1007 ret = ibv_query_device(qp->context, &device_attr); in mlx4_modify_qp()
1011 switch(qp->qp_type) { in mlx4_modify_qp()
1013 if ((mqp->link_layer == IBV_LINK_LAYER_INFINIBAND) && in mlx4_modify_qp()
1015 mqp->qp_cap_cache |= MLX4_CSUM_SUPPORT_UD_OVER_IB | in mlx4_modify_qp()
1019 if ((mqp->link_layer == IBV_LINK_LAYER_ETHERNET) && in mlx4_modify_qp()
1021 mqp->qp_cap_cache |= MLX4_CSUM_SUPPORT_RAW_OVER_ETH | in mlx4_modify_qp()
1030 if (qp->state == IBV_QPS_RESET && in mlx4_modify_qp()
1032 attr->qp_state == IBV_QPS_INIT) { in mlx4_modify_qp()
1036 ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd); in mlx4_modify_qp()
1040 attr->qp_state == IBV_QPS_RESET) { in mlx4_modify_qp()
1041 if (qp->recv_cq) in mlx4_modify_qp()
1042 mlx4_cq_clean(to_mcq(qp->recv_cq), qp->qp_num, in mlx4_modify_qp()
1043 qp->srq ? to_msrq(qp->srq) : NULL); in mlx4_modify_qp()
1044 if (qp->send_cq && qp->send_cq != qp->recv_cq) in mlx4_modify_qp()
1045 mlx4_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL); in mlx4_modify_qp()
1048 if (to_mqp(qp)->rq.wqe_cnt) in mlx4_modify_qp()
1049 *to_mqp(qp)->db = 0; in mlx4_modify_qp()
1057 struct mlx4_cq *send_cq = to_mcq(qp->send_cq); in mlx4_lock_cqs()
1058 struct mlx4_cq *recv_cq = to_mcq(qp->recv_cq); in mlx4_lock_cqs()
1060 if (!qp->send_cq || !qp->recv_cq) { in mlx4_lock_cqs()
1061 if (qp->send_cq) in mlx4_lock_cqs()
1062 pthread_spin_lock(&send_cq->lock); in mlx4_lock_cqs()
1063 else if (qp->recv_cq) in mlx4_lock_cqs()
1064 pthread_spin_lock(&recv_cq->lock); in mlx4_lock_cqs()
1066 pthread_spin_lock(&send_cq->lock); in mlx4_lock_cqs()
1067 } else if (send_cq->cqn < recv_cq->cqn) { in mlx4_lock_cqs()
1068 pthread_spin_lock(&send_cq->lock); in mlx4_lock_cqs()
1069 pthread_spin_lock(&recv_cq->lock); in mlx4_lock_cqs()
1071 pthread_spin_lock(&recv_cq->lock); in mlx4_lock_cqs()
1072 pthread_spin_lock(&send_cq->lock); in mlx4_lock_cqs()
1078 struct mlx4_cq *send_cq = to_mcq(qp->send_cq); in mlx4_unlock_cqs()
1079 struct mlx4_cq *recv_cq = to_mcq(qp->recv_cq); in mlx4_unlock_cqs()
1082 if (!qp->send_cq || !qp->recv_cq) { in mlx4_unlock_cqs()
1083 if (qp->send_cq) in mlx4_unlock_cqs()
1084 pthread_spin_unlock(&send_cq->lock); in mlx4_unlock_cqs()
1085 else if (qp->recv_cq) in mlx4_unlock_cqs()
1086 pthread_spin_unlock(&recv_cq->lock); in mlx4_unlock_cqs()
1088 pthread_spin_unlock(&send_cq->lock); in mlx4_unlock_cqs()
1089 } else if (send_cq->cqn < recv_cq->cqn) { in mlx4_unlock_cqs()
1090 pthread_spin_unlock(&recv_cq->lock); in mlx4_unlock_cqs()
1091 pthread_spin_unlock(&send_cq->lock); in mlx4_unlock_cqs()
1093 pthread_spin_unlock(&send_cq->lock); in mlx4_unlock_cqs()
1094 pthread_spin_unlock(&recv_cq->lock); in mlx4_unlock_cqs()
1103 pthread_mutex_lock(&to_mctx(ibqp->context)->qp_table_mutex); in mlx4_destroy_qp()
1106 pthread_mutex_unlock(&to_mctx(ibqp->context)->qp_table_mutex); in mlx4_destroy_qp()
1112 if (ibqp->recv_cq) in mlx4_destroy_qp()
1113 __mlx4_cq_clean(to_mcq(ibqp->recv_cq), ibqp->qp_num, in mlx4_destroy_qp()
1114 ibqp->srq ? to_msrq(ibqp->srq) : NULL); in mlx4_destroy_qp()
1115 if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq) in mlx4_destroy_qp()
1116 __mlx4_cq_clean(to_mcq(ibqp->send_cq), ibqp->qp_num, NULL); in mlx4_destroy_qp()
1118 if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) in mlx4_destroy_qp()
1119 mlx4_clear_qp(to_mctx(ibqp->context), ibqp->qp_num); in mlx4_destroy_qp()
1122 pthread_mutex_unlock(&to_mctx(ibqp->context)->qp_table_mutex); in mlx4_destroy_qp()
1124 pthread_spin_destroy(&qp->rq.lock); in mlx4_destroy_qp()
1125 pthread_spin_destroy(&qp->sq.lock); in mlx4_destroy_qp()
1127 if (qp->rq.wqe_cnt) { in mlx4_destroy_qp()
1128 mlx4_free_db(to_mctx(ibqp->context), MLX4_DB_TYPE_RQ, qp->db); in mlx4_destroy_qp()
1129 free(qp->rq.wrid); in mlx4_destroy_qp()
1131 if (qp->sq.wqe_cnt) in mlx4_destroy_qp()
1132 free(qp->sq.wrid); in mlx4_destroy_qp()
1133 mlx4_free_buf(&qp->buf); in mlx4_destroy_qp()
1141 uint32_t *tmp = (uint32_t *)gid->raw; in link_local_gid()
1153 return gid->raw[0] == 0xff; in is_multicast_gid()
1159 vid = gid->raw[11] << 8 | gid->raw[12]; in get_vlan_id()
1170 if (link_local_gid(&attr->grh.dgid)) { in mlx4_resolve_grh_to_l2()
1171 memcpy(ah->mac, &attr->grh.dgid.raw[8], 3); in mlx4_resolve_grh_to_l2()
1172 memcpy(ah->mac + 3, &attr->grh.dgid.raw[13], 3); in mlx4_resolve_grh_to_l2()
1173 ah->mac[0] ^= 2; in mlx4_resolve_grh_to_l2()
1175 vid = get_vlan_id(&attr->grh.dgid); in mlx4_resolve_grh_to_l2()
1176 } else if (is_multicast_gid(&attr->grh.dgid)) { in mlx4_resolve_grh_to_l2()
1177 ah->mac[0] = 0x33; in mlx4_resolve_grh_to_l2()
1178 ah->mac[1] = 0x33; in mlx4_resolve_grh_to_l2()
1180 ah->mac[i] = attr->grh.dgid.raw[i + 10]; in mlx4_resolve_grh_to_l2()
1182 err = ibv_query_gid(pd->context, attr->port_num, in mlx4_resolve_grh_to_l2()
1183 attr->grh.sgid_index, &sgid); in mlx4_resolve_grh_to_l2()
1187 ah->av.dlid = htobe16(0xc000); in mlx4_resolve_grh_to_l2()
1188 ah->av.port_pd |= htobe32(1 << 31); in mlx4_resolve_grh_to_l2()
1195 ah->av.port_pd |= htobe32(1 << 29); in mlx4_resolve_grh_to_l2()
1196 ah->vlan = vid | ((attr->sl & 7) << 13); in mlx4_resolve_grh_to_l2()
1207 if (query_port_cache(pd->context, attr->port_num, &port_attr)) in mlx4_create_ah()
1214 memset(&ah->av, 0, sizeof ah->av); in mlx4_create_ah()
1216 ah->av.port_pd = htobe32(to_mpd(pd)->pdn | (attr->port_num << 24)); in mlx4_create_ah()
1219 ah->av.g_slid = attr->src_path_bits; in mlx4_create_ah()
1220 ah->av.dlid = htobe16(attr->dlid); in mlx4_create_ah()
1221 ah->av.sl_tclass_flowlabel = htobe32(attr->sl << 28); in mlx4_create_ah()
1223 ah->av.sl_tclass_flowlabel = htobe32(attr->sl << 29); in mlx4_create_ah()
1225 if (attr->static_rate) { in mlx4_create_ah()
1226 ah->av.stat_rate = attr->static_rate + MLX4_STAT_RATE_OFFSET; in mlx4_create_ah()
1229 if (attr->is_global) { in mlx4_create_ah()
1230 ah->av.g_slid |= 0x80; in mlx4_create_ah()
1231 ah->av.gid_index = attr->grh.sgid_index; in mlx4_create_ah()
1232 ah->av.hop_limit = attr->grh.hop_limit; in mlx4_create_ah()
1233 ah->av.sl_tclass_flowlabel |= in mlx4_create_ah()
1234 htobe32((attr->grh.traffic_class << 20) | in mlx4_create_ah()
1235 attr->grh.flow_label); in mlx4_create_ah()
1236 memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); in mlx4_create_ah()
1243 if (ibv_resolve_eth_l2_from_gid(pd->context, attr, in mlx4_create_ah()
1244 ah->mac, &vid)) { in mlx4_create_ah()
1250 ah->av.port_pd |= htobe32(1 << 29); in mlx4_create_ah()
1251 ah->vlan = vid | in mlx4_create_ah()
1252 ((attr->sl & 7) << 13); in mlx4_create_ah()
1263 return &ah->ibv_ah; in mlx4_create_ah()