1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_IB_CLIENTS_IBD_H 27 #define _SYS_IB_CLIENTS_IBD_H 28 29 #ifdef __cplusplus 30 extern "C" { 31 #endif 32 33 /* 34 * IETF defined IPoIB encapsulation header, with 2b of ethertype 35 * followed by 2 reserved bytes. This is at the start of the 36 * datagram sent to and received over the wire by the driver. 37 */ 38 typedef struct ipoib_header { 39 ushort_t ipoib_type; 40 ushort_t ipoib_mbz; 41 } ipoib_hdr_t; 42 43 #define IPOIB_HDRSIZE sizeof (struct ipoib_header) 44 45 /* 46 * IETF defined IPoIB link address; IBA QPN, followed by GID, 47 * which has a prefix and suffix, as reported via ARP. 48 */ 49 typedef struct ipoib_mac { 50 uint32_t ipoib_qpn; 51 uint32_t ipoib_gidpref[2]; 52 uint32_t ipoib_gidsuff[2]; 53 } ipoib_mac_t; 54 55 #define IPOIB_ADDRL sizeof (struct ipoib_mac) 56 57 /* 58 * Pseudo header prepended to datagram in DLIOCRAW transmit path 59 * and when GLD hands the datagram to the gldm_send entry point. 60 */ 61 typedef struct ipoib_ptxhdr { 62 ipoib_mac_t ipoib_dest; 63 ipoib_hdr_t ipoib_rhdr; 64 } ipoib_ptxhdr_t; 65 66 #define IPOIBDLSAP(p, offset) ((ipoib_ptxhdr_t *)((caddr_t)(p)+offset)) 67 68 /* 69 * The pseudo-GRH structure that sits before the data in the 70 * receive buffer, and is overlaid on top of the real GRH. 71 * The driver sets the ipoib_vertcflow to 0 if the pseudo-GRH 72 * does not hold valid information. If it is indicated valid, 73 * the driver must additionally provide the sender's qpn in 74 * network byte order in ipoib_sqpn, and not touch the 75 * remaining parts which were DMA'ed in by the IBA hardware. 76 */ 77 typedef struct ipoib_pgrh { 78 uint32_t ipoib_vertcflow; 79 uint32_t ipoib_sqpn; 80 uint32_t ipoib_sgid_pref[2]; 81 uint32_t ipoib_sgid_suff[2]; 82 uint32_t ipoib_dgid_pref[2]; 83 uint32_t ipoib_dgid_suff[2]; 84 } ipoib_pgrh_t; 85 86 /* 87 * The GRH is also dma'ed into recv buffers, thus space needs 88 * to be allocated for them. 89 */ 90 #define IPOIB_GRH_SIZE sizeof (ipoib_pgrh_t) 91 92 #if defined(_KERNEL) && !defined(_BOOT) 93 94 #include <sys/ib/ibtl/ibti.h> 95 #include <sys/ib/ib_pkt_hdrs.h> 96 #include <sys/list.h> 97 #include <sys/mac_provider.h> 98 #include <sys/mac_ib.h> 99 #include <sys/modhash.h> 100 101 #define IBD_HIWAT (64*1024) /* drv flow control high water */ 102 #define IBD_LOWAT (1024) /* drv flow control low water */ 103 #define IBD_IDNUM 0 /* ibd module ID; zero works */ 104 105 #define IBD_MAX_SQSEG 3 106 #define IBD_MAX_RQSEG 1 107 108 typedef struct ibd_copybuf_s { 109 ibt_mr_hdl_t ic_mr_hdl; 110 ibt_wr_ds_t ic_sgl; 111 ibt_mr_desc_t ic_mr_desc; 112 uint8_t *ic_bufaddr; 113 } ibd_copybuf_t; 114 115 typedef struct ibd_mblkbuf_s { 116 ibt_mr_hdl_t im_mr_hdl; 117 ibt_mr_desc_t im_mr_desc; 118 } ibd_mblkbuf_t; 119 120 /* 121 * Structure to encapsulate various types of async requests. 122 */ 123 typedef struct ibd_acache_rq { 124 struct list_node rq_list; /* list of pending work */ 125 int rq_op; /* what operation */ 126 ipoib_mac_t rq_mac; 127 ib_gid_t rq_gid; 128 void *rq_ptr; 129 } ibd_req_t; 130 131 132 typedef struct ibd_mcache { 133 struct list_node mc_list; /* full/non list */ 134 uint8_t mc_jstate; 135 boolean_t mc_fullreap; 136 ibt_mcg_info_t mc_info; 137 ibd_req_t mc_req; /* to queue LEAVE req */ 138 } ibd_mce_t; 139 140 typedef struct ibd_acache_s { 141 struct list_node ac_list; /* free/active list */ 142 ibt_ud_dest_hdl_t ac_dest; 143 ipoib_mac_t ac_mac; 144 uint32_t ac_ref; 145 ibd_mce_t *ac_mce; /* for MCG AHs */ 146 } ibd_ace_t; 147 148 typedef enum {IBD_WQE_SEND, IBD_WQE_RECV} ibd_wqe_type_t; 149 150 typedef struct ibd_wqe_s { 151 struct ibd_wqe_s *w_next; 152 struct ibd_wqe_s *w_prev; 153 ibd_wqe_type_t w_type; 154 ibd_copybuf_t w_copybuf; 155 mblk_t *im_mblk; 156 } ibd_wqe_t; 157 158 typedef struct ibd_swqe_s { 159 ibd_wqe_t w_ibd_swqe; 160 ibt_send_wr_t w_swr; 161 ibt_wr_ds_t w_smblk_sgl[IBD_MAX_SQSEG]; 162 ibd_mblkbuf_t w_smblkbuf[IBD_MAX_SQSEG]; 163 ibd_ace_t *w_ahandle; 164 } ibd_swqe_t; 165 166 #define swqe_next w_ibd_swqe.w_next 167 #define swqe_prev w_ibd_swqe.w_prev 168 #define swqe_type w_ibd_swqe.w_type 169 #define swqe_copybuf w_ibd_swqe.w_copybuf 170 #define swqe_im_mblk w_ibd_swqe.im_mblk 171 #define SWQE_TO_WQE(swqe) (ibd_wqe_t *)&((swqe)->w_ibd_swqe) 172 #define WQE_TO_SWQE(wqe) (ibd_swqe_t *)wqe 173 174 typedef struct ibd_rwqe_s { 175 ibd_wqe_t w_ibd_rwqe; 176 struct ibd_state_s *w_state; 177 ibt_recv_wr_t w_rwr; 178 boolean_t w_freeing_wqe; 179 frtn_t w_freemsg_cb; 180 } ibd_rwqe_t; 181 182 #define rwqe_next w_ibd_rwqe.w_next 183 #define rwqe_prev w_ibd_rwqe.w_prev 184 #define rwqe_type w_ibd_rwqe.w_type 185 #define rwqe_copybuf w_ibd_rwqe.w_copybuf 186 #define rwqe_im_mblk w_ibd_rwqe.im_mblk 187 #define RWQE_TO_WQE(rwqe) (ibd_wqe_t *)&((rwqe)->w_ibd_rwqe) 188 #define WQE_TO_RWQE(wqe) (ibd_rwqe_t *)wqe 189 190 191 typedef struct ibd_list_s { 192 ibd_wqe_t *dl_head; 193 ibd_wqe_t *dl_tail; 194 union { 195 boolean_t pending_sends; 196 uint32_t bufs_outstanding; 197 } ustat; 198 uint32_t dl_cnt; 199 kmutex_t dl_mutex; 200 } ibd_list_t; 201 202 #define dl_pending_sends ustat.pending_sends 203 #define dl_bufs_outstanding ustat.bufs_outstanding 204 205 /* 206 * This structure maintains information per port per HCA 207 * (per network interface). 208 */ 209 typedef struct ibd_state_s { 210 dev_info_t *id_dip; 211 ibt_clnt_hdl_t id_ibt_hdl; 212 ibt_hca_hdl_t id_hca_hdl; 213 ibt_pd_hdl_t id_pd_hdl; 214 kmem_cache_t *id_req_kmc; 215 216 uint32_t id_max_sqseg; 217 ibd_list_t id_tx_list; 218 ddi_softintr_t id_tx; 219 uint32_t id_tx_sends; 220 kmutex_t id_txcomp_lock; 221 ibt_cq_hdl_t id_scq_hdl; 222 ibt_wc_t *id_txwcs; 223 uint32_t id_txwcs_size; 224 225 uint32_t id_num_rwqe; 226 ibd_list_t id_rx_list; 227 ddi_softintr_t id_rx; 228 ibt_cq_hdl_t id_rcq_hdl; 229 void *id_fifos; 230 int id_nfifos; 231 ibt_wc_t *id_rxwcs; 232 uint32_t id_rxwcs_size; 233 kmutex_t id_rx_mutex; 234 235 ibt_channel_hdl_t id_chnl_hdl; 236 ib_pkey_t id_pkey; 237 uint16_t id_pkix; 238 uint8_t id_port; 239 ibt_mcg_info_t *id_mcinfo; 240 241 mac_handle_t id_mh; 242 ib_gid_t id_sgid; 243 ib_qpn_t id_qpnum; 244 ipoib_mac_t id_macaddr; 245 ib_gid_t id_mgid; 246 ipoib_mac_t id_bcaddr; 247 248 int id_mtu; 249 uchar_t id_scope; 250 251 struct list id_req_list; 252 253 kmutex_t id_acache_req_lock; 254 kcondvar_t id_acache_req_cv; 255 kt_did_t id_async_thrid; 256 257 kmutex_t id_ac_mutex; 258 mod_hash_t *id_ah_active_hash; 259 struct list id_ah_free; 260 struct list id_ah_active; 261 ipoib_mac_t id_ah_addr; 262 ibd_req_t id_ah_req; 263 char id_ah_op; 264 ibd_ace_t *id_ac_list; 265 266 kmutex_t id_mc_mutex; 267 struct list id_mc_full; 268 struct list id_mc_non; 269 270 kmutex_t id_trap_lock; 271 kcondvar_t id_trap_cv; 272 boolean_t id_trap_stop; 273 uint32_t id_trap_inprog; 274 275 char id_prom_op; 276 277 kmutex_t id_sched_lock; 278 boolean_t id_sched_needed; 279 280 kmutex_t id_link_mutex; 281 link_state_t id_link_state; 282 uint64_t id_link_speed; 283 284 uint64_t id_ah_error; 285 uint64_t id_rx_short; 286 uint64_t id_num_intrs; 287 uint64_t id_tx_short; 288 uint32_t id_num_swqe; 289 290 uint64_t id_xmt_bytes; 291 uint64_t id_recv_bytes; 292 uint64_t id_multi_xmt; 293 uint64_t id_brd_xmt; 294 uint64_t id_multi_rcv; 295 uint64_t id_brd_rcv; 296 uint64_t id_xmt_pkt; 297 uint64_t id_rcv_pkt; 298 } ibd_state_t; 299 300 #endif /* _KERNEL && !_BOOT */ 301 302 #ifdef __cplusplus 303 } 304 #endif 305 306 #endif /* _SYS_IB_CLIENTS_IBD_H */ 307