1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_IB_CLIENTS_IBD_H 27 #define _SYS_IB_CLIENTS_IBD_H 28 29 #pragma ident "%Z%%M% %I% %E% SMI" 30 31 #ifdef __cplusplus 32 extern "C" { 33 #endif 34 35 /* 36 * IETF defined IPoIB encapsulation header, with 2b of ethertype 37 * followed by 2 reserved bytes. This is at the start of the 38 * datagram sent to and received over the wire by the driver. 39 */ 40 typedef struct ipoib_header { 41 ushort_t ipoib_type; 42 ushort_t ipoib_mbz; 43 } ipoib_hdr_t; 44 45 #define IPOIB_HDRSIZE sizeof (struct ipoib_header) 46 47 /* 48 * IETF defined IPoIB link address; IBA QPN, followed by GID, 49 * which has a prefix and suffix, as reported via ARP. 50 */ 51 typedef struct ipoib_mac { 52 uint32_t ipoib_qpn; 53 uint32_t ipoib_gidpref[2]; 54 uint32_t ipoib_gidsuff[2]; 55 } ipoib_mac_t; 56 57 #define IPOIB_ADDRL sizeof (struct ipoib_mac) 58 59 /* 60 * Pseudo header prepended to datagram in DLIOCRAW transmit path 61 * and when GLD hands the datagram to the gldm_send entry point. 62 */ 63 typedef struct ipoib_ptxhdr { 64 ipoib_mac_t ipoib_dest; 65 ipoib_hdr_t ipoib_rhdr; 66 } ipoib_ptxhdr_t; 67 68 #define IPOIBDLSAP(p, offset) ((ipoib_ptxhdr_t *)((caddr_t)(p)+offset)) 69 70 /* 71 * The pseudo-GRH structure that sits before the data in the 72 * receive buffer, and is overlaid on top of the real GRH. 73 * The driver sets the ipoib_vertcflow to 0 if the pseudo-GRH 74 * does not hold valid information. If it is indicated valid, 75 * the driver must additionally provide the sender's qpn in 76 * network byte order in ipoib_sqpn, and not touch the 77 * remaining parts which were DMA'ed in by the IBA hardware. 78 */ 79 typedef struct ipoib_pgrh { 80 uint32_t ipoib_vertcflow; 81 uint32_t ipoib_sqpn; 82 uint32_t ipoib_sgid_pref[2]; 83 uint32_t ipoib_sgid_suff[2]; 84 uint32_t ipoib_dgid_pref[2]; 85 uint32_t ipoib_dgid_suff[2]; 86 } ipoib_pgrh_t; 87 88 /* 89 * The GRH is also dma'ed into recv buffers, thus space needs 90 * to be allocated for them. 91 */ 92 #define IPOIB_GRH_SIZE sizeof (ipoib_pgrh_t) 93 94 #if defined(_KERNEL) && !defined(_BOOT) 95 96 #include <sys/ib/ibtl/ibti.h> 97 #include <sys/ib/ib_pkt_hdrs.h> 98 #include <sys/list.h> 99 #include <sys/mac.h> 100 #include <sys/mac_ib.h> 101 #include <sys/modhash.h> 102 103 #define IBD_HIWAT (64*1024) /* drv flow control high water */ 104 #define IBD_LOWAT (1024) /* drv flow control low water */ 105 #define IBD_IDNUM 0 /* ibd module ID; zero works */ 106 107 #define IBD_MAX_SQSEG 3 108 #define IBD_MAX_RQSEG 1 109 110 typedef struct ibd_copybuf_s { 111 ibt_mr_hdl_t ic_mr_hdl; 112 ibt_wr_ds_t ic_sgl; 113 ibt_mr_desc_t ic_mr_desc; 114 uint8_t *ic_bufaddr; 115 } ibd_copybuf_t; 116 117 typedef struct ibd_mblkbuf_s { 118 ibt_mr_hdl_t im_mr_hdl; 119 ibt_mr_desc_t im_mr_desc; 120 } ibd_mblkbuf_t; 121 122 /* 123 * Structure to encapsulate various types of async requests. 124 */ 125 typedef struct ibd_acache_rq { 126 struct list_node rq_list; /* list of pending work */ 127 int rq_op; /* what operation */ 128 ipoib_mac_t rq_mac; 129 ib_gid_t rq_gid; 130 void *rq_ptr; 131 } ibd_req_t; 132 133 134 typedef struct ibd_mcache { 135 struct list_node mc_list; /* full/non list */ 136 uint8_t mc_jstate; 137 boolean_t mc_fullreap; 138 ibt_mcg_info_t mc_info; 139 ibd_req_t mc_req; /* to queue LEAVE req */ 140 } ibd_mce_t; 141 142 typedef struct ibd_acache_s { 143 struct list_node ac_list; /* free/active list */ 144 ibt_ud_dest_hdl_t ac_dest; 145 ipoib_mac_t ac_mac; 146 uint32_t ac_ref; 147 ibd_mce_t *ac_mce; /* for MCG AHs */ 148 } ibd_ace_t; 149 150 typedef enum {IBD_WQE_SEND, IBD_WQE_RECV} ibd_wqe_type_t; 151 152 typedef struct ibd_wqe_s { 153 struct ibd_wqe_s *w_next; 154 struct ibd_wqe_s *w_prev; 155 ibd_wqe_type_t w_type; 156 ibd_copybuf_t w_copybuf; 157 mblk_t *im_mblk; 158 } ibd_wqe_t; 159 160 typedef struct ibd_swqe_s { 161 ibd_wqe_t w_ibd_swqe; 162 ibt_send_wr_t w_swr; 163 ibt_wr_ds_t w_smblk_sgl[IBD_MAX_SQSEG]; 164 ibd_mblkbuf_t w_smblkbuf[IBD_MAX_SQSEG]; 165 ibd_ace_t *w_ahandle; 166 } ibd_swqe_t; 167 168 #define swqe_next w_ibd_swqe.w_next 169 #define swqe_prev w_ibd_swqe.w_prev 170 #define swqe_type w_ibd_swqe.w_type 171 #define swqe_copybuf w_ibd_swqe.w_copybuf 172 #define swqe_im_mblk w_ibd_swqe.im_mblk 173 #define SWQE_TO_WQE(swqe) (ibd_wqe_t *)&((swqe)->w_ibd_swqe) 174 #define WQE_TO_SWQE(wqe) (ibd_swqe_t *)wqe 175 176 typedef struct ibd_rwqe_s { 177 ibd_wqe_t w_ibd_rwqe; 178 struct ibd_state_s *w_state; 179 ibt_recv_wr_t w_rwr; 180 boolean_t w_freeing_wqe; 181 frtn_t w_freemsg_cb; 182 } ibd_rwqe_t; 183 184 #define rwqe_next w_ibd_rwqe.w_next 185 #define rwqe_prev w_ibd_rwqe.w_prev 186 #define rwqe_type w_ibd_rwqe.w_type 187 #define rwqe_copybuf w_ibd_rwqe.w_copybuf 188 #define rwqe_im_mblk w_ibd_rwqe.im_mblk 189 #define RWQE_TO_WQE(rwqe) (ibd_wqe_t *)&((rwqe)->w_ibd_rwqe) 190 #define WQE_TO_RWQE(wqe) (ibd_rwqe_t *)wqe 191 192 193 typedef struct ibd_list_s { 194 ibd_wqe_t *dl_head; 195 ibd_wqe_t *dl_tail; 196 union { 197 boolean_t pending_sends; 198 uint32_t bufs_outstanding; 199 } ustat; 200 uint32_t dl_cnt; 201 kmutex_t dl_mutex; 202 } ibd_list_t; 203 204 #define dl_pending_sends ustat.pending_sends 205 #define dl_bufs_outstanding ustat.bufs_outstanding 206 207 /* 208 * This structure maintains information per port per HCA 209 * (per network interface). 210 */ 211 typedef struct ibd_state_s { 212 dev_info_t *id_dip; 213 ibt_clnt_hdl_t id_ibt_hdl; 214 ibt_hca_hdl_t id_hca_hdl; 215 ibt_pd_hdl_t id_pd_hdl; 216 kmem_cache_t *id_req_kmc; 217 218 uint32_t id_max_sqseg; 219 ibd_list_t id_tx_list; 220 ddi_softintr_t id_tx; 221 uint32_t id_tx_sends; 222 kmutex_t id_txcomp_lock; 223 ibt_cq_hdl_t id_scq_hdl; 224 ibt_wc_t *id_txwcs; 225 uint32_t id_txwcs_size; 226 227 uint32_t id_num_rwqe; 228 ibd_list_t id_rx_list; 229 ddi_softintr_t id_rx; 230 ibt_cq_hdl_t id_rcq_hdl; 231 void *id_fifos; 232 int id_nfifos; 233 ibt_wc_t *id_rxwcs; 234 uint32_t id_rxwcs_size; 235 kmutex_t id_rx_mutex; 236 237 ibt_channel_hdl_t id_chnl_hdl; 238 ib_pkey_t id_pkey; 239 uint16_t id_pkix; 240 uint8_t id_port; 241 ibt_mcg_info_t *id_mcinfo; 242 243 mac_handle_t id_mh; 244 ib_gid_t id_sgid; 245 ib_qpn_t id_qpnum; 246 ipoib_mac_t id_macaddr; 247 ib_gid_t id_mgid; 248 ipoib_mac_t id_bcaddr; 249 250 int id_mtu; 251 uchar_t id_scope; 252 253 struct list id_req_list; 254 255 kmutex_t id_acache_req_lock; 256 kcondvar_t id_acache_req_cv; 257 kt_did_t id_async_thrid; 258 259 kmutex_t id_ac_mutex; 260 mod_hash_t *id_ah_active_hash; 261 struct list id_ah_free; 262 struct list id_ah_active; 263 ipoib_mac_t id_ah_addr; 264 ibd_req_t id_ah_req; 265 char id_ah_op; 266 ibd_ace_t *id_ac_list; 267 268 kmutex_t id_mc_mutex; 269 struct list id_mc_full; 270 struct list id_mc_non; 271 272 kmutex_t id_trap_lock; 273 kcondvar_t id_trap_cv; 274 boolean_t id_trap_stop; 275 uint32_t id_trap_inprog; 276 277 char id_prom_op; 278 279 kmutex_t id_sched_lock; 280 boolean_t id_sched_needed; 281 282 kmutex_t id_link_mutex; 283 link_state_t id_link_state; 284 uint64_t id_link_speed; 285 286 uint64_t id_ah_error; 287 uint64_t id_rx_short; 288 uint64_t id_num_intrs; 289 uint64_t id_tx_short; 290 uint32_t id_num_swqe; 291 292 uint64_t id_xmt_bytes; 293 uint64_t id_recv_bytes; 294 uint64_t id_multi_xmt; 295 uint64_t id_brd_xmt; 296 uint64_t id_multi_rcv; 297 uint64_t id_brd_rcv; 298 uint64_t id_xmt_pkt; 299 uint64_t id_rcv_pkt; 300 } ibd_state_t; 301 302 #endif /* _KERNEL && !_BOOT */ 303 304 #ifdef __cplusplus 305 } 306 #endif 307 308 #endif /* _SYS_IB_CLIENTS_IBD_H */ 309