1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (C) 2019 - 2020 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 /*$FreeBSD$*/ 35 36 #ifndef IRDMA_UMAIN_H 37 #define IRDMA_UMAIN_H 38 39 #include <sys/queue.h> 40 #include <infiniband/verbs.h> 41 #include <infiniband/driver.h> 42 43 #include "osdep.h" 44 #include "irdma.h" 45 #include "irdma_defs.h" 46 #include "i40iw_hw.h" 47 #include "irdma_user.h" 48 49 #ifndef likely 50 #define likely(x) __builtin_expect((x), 1) 51 #endif 52 #ifndef unlikely 53 #define unlikely(x) __builtin_expect((x), 0) 54 #endif 55 #define PFX "libirdma-" 56 57 #define IRDMA_BASE_PUSH_PAGE 1 58 #define IRDMA_U_MINCQ_SIZE 4 59 #define IRDMA_DB_SHADOW_AREA_SIZE 64 60 #define IRDMA_DB_CQ_OFFSET 64 61 62 LIST_HEAD(list_head, irdma_cq_buf); 63 LIST_HEAD(list_head_cmpl, irdma_cmpl_gen); 64 65 enum irdma_supported_wc_flags { 66 IRDMA_CQ_SUPPORTED_WC_FLAGS = IBV_WC_EX_WITH_BYTE_LEN 67 | IBV_WC_EX_WITH_IMM 68 | IBV_WC_EX_WITH_QP_NUM 69 | IBV_WC_EX_WITH_SRC_QP 70 | IBV_WC_EX_WITH_SL 71 | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP, 72 }; 73 74 struct irdma_udevice { 75 struct verbs_device ibv_dev; 76 }; 77 78 struct irdma_uah { 79 struct ibv_ah ibv_ah; 80 uint32_t ah_id; 81 struct ibv_global_route grh; 82 }; 83 84 struct irdma_upd { 85 struct ibv_pd ibv_pd; 86 void *arm_cq_page; 87 void *arm_cq; 88 uint32_t pd_id; 89 }; 90 91 struct irdma_uvcontext { 92 struct ibv_context ibv_ctx; 93 struct irdma_upd *iwupd; 94 struct irdma_uk_attrs uk_attrs; 95 void *db; 96 int abi_ver; 97 bool legacy_mode; 98 }; 99 100 struct irdma_uqp; 101 102 struct irdma_cq_buf { 103 LIST_ENTRY(irdma_cq_buf) list; 104 struct irdma_cq_uk cq; 105 struct verbs_mr vmr; 106 }; 107 108 struct verbs_cq { 109 union { 110 struct ibv_cq cq; 111 struct ibv_cq_ex cq_ex; 112 }; 113 }; 114 115 struct irdma_cmpl_gen { 116 LIST_ENTRY(irdma_cmpl_gen) list; 117 struct irdma_cq_poll_info cpi; 118 }; 119 120 struct irdma_ucq { 121 struct verbs_cq verbs_cq; 122 struct verbs_mr vmr; 123 struct verbs_mr vmr_shadow_area; 124 pthread_spinlock_t lock; 125 size_t buf_size; 126 bool is_armed; 127 bool skip_arm; 128 bool arm_sol; 129 bool skip_sol; 130 int comp_vector; 131 uint32_t report_rtt; 132 struct irdma_uqp *uqp; 133 struct irdma_cq_uk cq; 134 struct list_head resize_list; 135 /* for extended CQ completion fields */ 136 struct irdma_cq_poll_info cur_cqe; 137 struct list_head_cmpl cmpl_generated; 138 }; 139 140 struct irdma_uqp { 141 struct ibv_qp ibv_qp; 142 struct ibv_qp_attr attr; 143 struct irdma_ucq *send_cq; 144 struct irdma_ucq *recv_cq; 145 struct verbs_mr vmr; 146 size_t buf_size; 147 uint32_t irdma_drv_opt; 148 pthread_spinlock_t lock; 149 uint16_t sq_sig_all; 150 uint16_t qperr; 151 uint16_t rsvd; 152 uint32_t pending_rcvs; 153 uint32_t wq_size; 154 struct ibv_recv_wr *pend_rx_wr; 155 struct irdma_qp_uk qp; 156 enum ibv_qp_type qp_type; 157 enum ibv_qp_attr_mask attr_mask; 158 struct irdma_sge *recv_sges; 159 pthread_t flush_thread; 160 }; 161 162 struct irdma_umr { 163 struct verbs_mr vmr; 164 uint32_t acc_flags; 165 }; 166 167 /* irdma_uverbs.c */ 168 int irdma_uquery_device_ex(struct ibv_context *context, 169 const struct ibv_query_device_ex_input *input, 170 struct ibv_device_attr_ex *attr, size_t attr_size); 171 int irdma_uquery_port(struct ibv_context *context, uint8_t port, 172 struct ibv_port_attr *attr); 173 struct ibv_pd *irdma_ualloc_pd(struct ibv_context *context); 174 int irdma_ufree_pd(struct ibv_pd *pd); 175 int irdma_uquery_device(struct ibv_context *, struct ibv_device_attr *); 176 struct ibv_mr *irdma_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, 177 int access); 178 int irdma_udereg_mr(struct ibv_mr *mr); 179 struct ibv_mw *irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); 180 int irdma_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw, 181 struct ibv_mw_bind *mw_bind); 182 int irdma_udealloc_mw(struct ibv_mw *mw); 183 struct ibv_cq *irdma_ucreate_cq(struct ibv_context *context, int cqe, 184 struct ibv_comp_channel *channel, 185 int comp_vector); 186 struct ibv_cq_ex *irdma_ucreate_cq_ex(struct ibv_context *context, 187 struct ibv_cq_init_attr_ex *attr_ex); 188 void irdma_ibvcq_ex_fill_priv_funcs(struct irdma_ucq *iwucq, 189 struct ibv_cq_init_attr_ex *attr_ex); 190 int irdma_uresize_cq(struct ibv_cq *cq, int cqe); 191 int irdma_udestroy_cq(struct ibv_cq *cq); 192 int irdma_upoll_cq(struct ibv_cq *cq, int entries, struct ibv_wc *entry); 193 int irdma_uarm_cq(struct ibv_cq *cq, int solicited); 194 void irdma_cq_event(struct ibv_cq *cq); 195 struct ibv_qp *irdma_ucreate_qp(struct ibv_pd *pd, 196 struct ibv_qp_init_attr *attr); 197 int irdma_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, 198 struct ibv_qp_init_attr *init_attr); 199 int irdma_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 200 int attr_mask); 201 int irdma_udestroy_qp(struct ibv_qp *qp); 202 int irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, 203 struct ibv_send_wr **bad_wr); 204 int irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, 205 struct ibv_recv_wr **bad_wr); 206 struct ibv_ah *irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr); 207 int irdma_udestroy_ah(struct ibv_ah *ibah); 208 int irdma_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, 209 uint16_t lid); 210 int irdma_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, 211 uint16_t lid); 212 void irdma_async_event(struct ibv_context *context, 213 struct ibv_async_event *event); 214 void irdma_set_hw_attrs(struct irdma_hw_attrs *attrs); 215 void *irdma_mmap(int fd, off_t offset); 216 void irdma_munmap(void *map); 217 void *irdma_flush_thread(void *arg); 218 #endif /* IRDMA_UMAIN_H */ 219