1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _ISER_IB_H 27 #define _ISER_IB_H 28 29 #ifdef __cplusplus 30 extern "C" { 31 #endif 32 33 #include <sys/types.h> 34 #include <sys/socket.h> 35 #include <netinet/in.h> 36 #include <sys/ib/ibtl/ibti.h> 37 #include <sys/iscsi_protocol.h> 38 39 /* 40 * iser_ib.h 41 * Definitions and macros related to iSER InfiniBand transport, 42 * and the use of the Solaris IBTI (InfiniBand Transport Interface). 43 */ 44 45 struct iser_state_s; 46 47 extern struct iser_state_s *iser_state; 48 extern ddi_taskq_t *iser_taskq; 49 50 /* 51 * iser_hca_s holds all the information about the Infinband HCAs in use. 52 */ 53 typedef struct iser_hca_s { 54 list_node_t hca_node; 55 boolean_t hca_failed; 56 ibt_clnt_hdl_t hca_clnt_hdl; 57 ibt_hca_hdl_t hca_hdl; 58 ibt_hca_attr_t hca_attr; 59 ibt_pd_hdl_t hca_pdhdl; 60 ib_guid_t hca_guid; 61 uint_t hca_num_ports; 62 ibt_hca_portinfo_t *hca_port_info; 63 uint_t hca_port_info_sz; 64 65 /* Per PD (per HCA) message and data buffer caches */ 66 struct iser_vmem_mr_pool_s *hca_msg_pool; /* Use iser_msg_cache */ 67 kmem_cache_t *iser_msg_cache; 68 struct iser_vmem_mr_pool_s *hca_buf_pool; /* Use iser_buf_cache */ 69 kmem_cache_t *iser_buf_cache; 70 } iser_hca_t; 71 72 /* RQ low water mark percentage */ 73 #define ISER_IB_RQ_LWM_PCT 80 74 75 /* Maximum number of WRs to post on the RQ at a time */ 76 #define ISER_IB_RQ_POST_MAX 64 77 78 /* Maximum number of SCQ WCs to poll at a time */ 79 #define ISER_IB_SCQ_POLL_MAX 8 80 81 /* 82 * iser_qp_t encodes data related to a Queue Pair (QP) in use by 83 * iSER. Each QP consists of two Work Queues (WQs), one Send Queue 84 * (SQ) and on Receive Queue (RQ). Most of the data in the QP 85 * handle relates to monitoring the posted depth of the RQ. 86 * 87 * Note that we are explicitly using slightly less than a power-of-2 88 * number for our queue sizes. The HCA driver will round up for us, 89 * and this affords us some headroom. 90 */ 91 #ifdef _LP64 92 #define ISER_IB_RECVQ_SIZE 400 93 #else 94 /* Memory is very limited on 32-bit kernels */ 95 #define ISER_IB_RECVQ_SIZE 100 96 #endif 97 #define ISER_IB_SENDQ_SIZE 2000 98 #define ISER_IB_SGLIST_SIZE 1 99 100 #define ISER_IB_DEFAULT_IRD 2 101 #define ISER_IB_DEFAULT_ORD 4 102 103 typedef struct iser_qp_s { 104 kmutex_t qp_lock; 105 uint_t sq_size; 106 uint_t rq_size; 107 uint32_t rq_depth; 108 uint32_t rq_level; 109 uint32_t rq_min_post_level; 110 uint32_t rq_lwm; 111 boolean_t rq_taskqpending; 112 } iser_qp_t; 113 114 /* 115 * iSER RC channel information 116 */ 117 typedef struct iser_chan_s { 118 kmutex_t ic_chan_lock; 119 120 /* IBT channel handle */ 121 ibt_channel_hdl_t ic_chanhdl; 122 123 /* local and remote IP addresses and port numbers */ 124 ibt_ip_addr_t ic_localip; 125 ibt_ip_addr_t ic_remoteip; 126 in_port_t ic_lport; 127 in_port_t ic_rport; 128 129 /* 130 * The local HCA GUID, the service ID, Destination GID, Source GID 131 * the primary hca port on which the channel is connected is 132 * stored in ic_ibt_path 133 */ 134 ibt_path_info_t ic_ibt_path; 135 136 /* 137 * Information related to the HCA handle and the queues. 138 */ 139 iser_hca_t *ic_hca; 140 ibt_cq_hdl_t ic_sendcq; 141 ibt_cq_hdl_t ic_recvcq; 142 uint_t ic_sendcq_sz; 143 uint_t ic_recvcq_sz; 144 iser_qp_t ic_qp; 145 146 /* Used to track the number of WRs posted on the SQ */ 147 kmutex_t ic_sq_post_lock; 148 uint_t ic_sq_post_count; 149 uint_t ic_sq_max_post_count; 150 151 /* 152 * To help identify the channel end point and some connection 153 * specifics, maintain a pointer to the connection on which 154 * this channel originated 155 */ 156 struct iser_conn_s *ic_conn; 157 } iser_chan_t; 158 159 int iser_ib_init(void); 160 161 int iser_ib_fini(void); 162 163 int iser_ib_register_service(idm_svc_t *idm_svc); 164 165 int iser_ib_bind_service(idm_svc_t *idm_svc); 166 167 void iser_ib_unbind_service(idm_svc_t *idm_svc); 168 169 void iser_ib_deregister_service(idm_svc_t *idm_svc); 170 171 void iser_ib_conv_sockaddr2ibtaddr(idm_sockaddr_t *saddr, 172 ibt_ip_addr_t *ibt_addr); 173 174 void iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage *ss, 175 ibt_ip_addr_t *ibt_addr, in_port_t port); 176 177 int iser_ib_get_paths( 178 ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip, ibt_path_info_t *path, 179 ibt_path_ip_src_t *path_src_ip); 180 181 iser_chan_t *iser_ib_alloc_channel_pathlookup( 182 ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip); 183 184 iser_chan_t *iser_ib_alloc_channel_nopathlookup( 185 ib_guid_t hca_guid, uint8_t hca_port); 186 187 iser_chan_t *iser_ib_alloc_rc_channel(iser_hca_t *hca, uint8_t hca_port); 188 189 int iser_ib_open_rc_channel(iser_chan_t *chan); 190 191 void iser_ib_close_rc_channel(iser_chan_t *chan); 192 193 void iser_ib_free_rc_channel(iser_chan_t *chan); 194 195 int iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl); 196 197 void iser_ib_post_recv(ibt_channel_hdl_t chanhdl); 198 199 void iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg); 200 201 void iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg); 202 203 void iser_ib_async_handler(void *clntp, ibt_hca_hdl_t hdl, 204 ibt_async_code_t code, ibt_async_event_t *event); 205 206 #ifdef __cplusplus 207 } 208 #endif 209 210 #endif /* _ISER_IB_H */ 211