1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H 27 #define _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H 28 29 /* 30 * saa_impl.h 31 */ 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 37 #include <sys/ib/mgt/ibmf/ibmf_saa.h> 38 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 39 40 #define SAA_MAX_CLIENTS_PER_PORT 100 41 #define SAA_MAD_BASE_VERSION 1 42 #define SAA_MAD_CLASS_VERSION 2 43 #define IBMF_SAA_RETRANS_RETRIES 0 44 #define IBMF_SAA_MAX_SUBNET_TIMEOUT 20 45 #define IBMF_SAA_MAX_RESP_TIME 20 46 #define IBMF_SAA_MAX_BUSY_RETRY_COUNT 10 47 #define IBMF_SAA_MAX_WAIT_TIME_IN_SECS 60 48 #define IBMF_SAA_TRANS_WAIT_TIME_IN_SECS 240 49 #define IBMF_SAA_BUSY_RETRY_SLEEP_SECS 1 /* seconds between retry */ 50 51 /* 52 * saa_port_s: 53 * Linked list of ports that saa is using. Each port is registered to ibmf. 54 * Multiple saa clients can associate with an saa port 55 */ 56 typedef enum saa_port_state_s { 57 IBMF_SAA_PORT_STATE_REGISTERING, 58 IBMF_SAA_PORT_STATE_READY, 59 IBMF_SAA_PORT_STATE_INVALID, /* client MUST close */ 60 IBMF_SAA_PORT_STATE_PURGING /* being purged */ 61 } saa_port_state_t; 62 63 typedef struct saa_port_s { 64 65 struct saa_port_s *next; 66 67 kmutex_t saa_pt_mutex; 68 69 /* registration synchronization: only one client registers to ibmf */ 70 kcondvar_t saa_pt_ibmf_reg_cv; 71 72 /* state and client reference counts */ 73 saa_port_state_t saa_pt_state; 74 int saa_pt_reference_count; 75 76 /* port identification and ibmf registration info */ 77 ib_guid_t saa_pt_port_guid; 78 ibmf_register_info_t saa_pt_ibmf_reginfo; 79 80 ibmf_handle_t saa_pt_ibmf_handle; 81 ibmf_impl_caps_t saa_pt_ibmf_impl_features; 82 ibmf_qp_handle_t saa_pt_qp_handle; 83 ib_qpn_t saa_pt_qpn; 84 85 /* transaction parameters */ 86 int saa_pt_timeout; /* from portinfo */ 87 uint16_t saa_pt_sa_cap_mask; /* from classportinfo */ 88 89 ibmf_addr_info_t saa_pt_ibmf_addr_info; 90 ibmf_global_addr_info_t saa_pt_ibmf_global_addr; 91 uint32_t saa_pt_ibmf_msg_flags; 92 boolean_t saa_pt_redirect_active; /* SA has redirected */ 93 94 ibmf_retrans_t saa_pt_ibmf_retrans; 95 uint64_t saa_pt_current_tid; 96 int saa_pt_num_outstanding_trans; 97 98 /* kstats */ 99 kmutex_t saa_pt_kstat_mutex; 100 struct kstat *saa_pt_kstatp; 101 102 /* sync. for receiving informinfo req packets */ 103 kmutex_t saa_pt_event_sub_mutex; 104 uint8_t saa_pt_event_sub_arrive_mask; 105 uint8_t saa_pt_event_sub_success_mask; 106 uint8_t saa_pt_event_sub_last_success_mask; 107 struct saa_client_data_s *saa_pt_event_sub_client_list; 108 109 /* node guid and port num, saved for easy ibt_queries */ 110 ib_guid_t saa_pt_node_guid; 111 uint8_t saa_pt_port_num; 112 113 /* latest hrtime that packet from SA was received */ 114 hrtime_t saa_pt_sa_uptime; 115 } saa_port_t; 116 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_mutex, 117 saa_port_t::saa_pt_reference_count 118 saa_port_t::saa_pt_ibmf_reg_cv 119 saa_port_t::saa_pt_ibmf_retrans 120 saa_port_t::saa_pt_current_tid 121 saa_port_t::saa_pt_num_outstanding_trans 122 saa_port_t::saa_pt_timeout 123 saa_port_t::saa_pt_ibmf_addr_info 124 saa_port_t::saa_pt_ibmf_global_addr 125 saa_port_t::saa_pt_ibmf_msg_flags 126 saa_port_t::saa_pt_redirect_active)) 127 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_kstat_mutex, 128 saa_port_t::saa_pt_kstatp)) 129 130 131 #define IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE \ 132 (IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_CA | \ 133 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SWITCH | \ 134 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_ROUTER | \ 135 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM) 136 137 typedef struct ibmf_saa_kstat_s { 138 kstat_named_t clients_registered; /* # saa registrants */ 139 kstat_named_t clients_reg_failed; /* # failed registrants */ 140 kstat_named_t outstanding_requests; /* # outstanding requests */ 141 kstat_named_t total_requests; /* # requests ever made */ 142 kstat_named_t failed_requests; /* # failed requests */ 143 kstat_named_t requests_timedout; /* # requests that timedout */ 144 } ibmf_saa_kstat_t; 145 146 #define IBMF_SAA_ADD32_KSTATS(subnetp, xx, val) \ 147 if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) { \ 148 ibmf_saa_kstat_t *kp; \ 149 kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\ 150 kp->xx.value.ui32 += val; \ 151 } 152 153 #define IBMF_SAA_SUB32_KSTATS(subnetp, xx, val) \ 154 if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) { \ 155 ibmf_saa_kstat_t *kp; \ 156 kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\ 157 kp->xx.value.ui32 -= val; \ 158 } 159 160 typedef enum _saa_client_state_e { 161 SAA_CLIENT_STATE_ACTIVE, 162 SAA_CLIENT_STATE_WAITING, 163 SAA_CLIENT_STATE_CLOSED 164 } saa_client_state_t; 165 166 typedef struct saa_client_data_s { 167 void *next; 168 169 /* set for valid handles */ 170 void *saa_client_sig; 171 saa_port_t *saa_client_port; 172 kmutex_t saa_client_mutex; 173 int saa_client_num_pending_trans; 174 kcondvar_t saa_client_state_cv; 175 saa_client_state_t saa_client_state; 176 ib_smkey_t saa_client_sm_key; 177 178 int saa_client_event_cb_num_active; 179 kcondvar_t saa_client_event_cb_cv; 180 181 ibmf_saa_subnet_event_cb_t saa_client_event_cb; 182 void *saa_client_event_cb_arg; 183 } saa_client_data_t; 184 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_port)) 185 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_sig)) 186 187 188 typedef struct saa_state_s { 189 190 saa_port_t *saa_port_list; 191 kmutex_t saa_port_list_mutex; 192 taskq_t *saa_event_taskq; 193 } saa_state_t; 194 _NOTE(MUTEX_PROTECTS_DATA(saa_state_t::saa_port_list_mutex, 195 saa_port_t::next 196 saa_state_t::saa_port_list)) 197 _NOTE(READ_ONLY_DATA(saa_state_t::saa_event_taskq)) 198 199 /* 200 * special callback used specifically for handling informinfo responses; 201 * extra parameter is producer_type 202 */ 203 typedef void (*ibmf_saa_sub_cb_t) (void *, size_t, char *, int, uint32_t); 204 205 /* 206 * saa_impl_trans_info_t: 207 * Convenience structure wich allows ibmf_access_sa to group all the fields 208 * into one structure as a parameter to the send request function. 209 * This structure is allocated by ibmf_access_sa() and freed by ibmf_access_sa() 210 * in the sync case and by the ibmf_msg_transport callback in the async case 211 */ 212 typedef struct saa_impl_trans_info_t { 213 214 saa_client_data_t *si_trans_client_data; 215 saa_port_t *si_trans_port; 216 217 /* used to tell send_request about request mad */ 218 size_t si_trans_template_length; /* for unknown attr */ 219 uint16_t si_trans_attr_id; 220 uint64_t si_trans_component_mask; 221 void *si_trans_template; 222 uint8_t si_trans_method; 223 224 /* used for async call to tell send_request which callback to use */ 225 ibmf_saa_cb_t si_trans_callback; 226 void *si_trans_callback_arg; 227 228 /* 229 * used to tell ibmf_access_sa about response if the request was sync. 230 * If the request was async, the ibmf_msg_transport callback function 231 * will fill these values directly into the si_trans_callback 232 */ 233 int si_trans_status; 234 void *si_trans_result; 235 size_t si_trans_length; 236 237 /* fields needed for specific case of handling InformInfo requests */ 238 239 /* 240 * producer_type indicates the notice producer type that the 241 * subscription was for. There is no way to tell which type the 242 * response packet is for. 243 */ 244 uint32_t si_trans_sub_producer_type; 245 246 /* 247 * separate callback typedef to provide the producer type to the 248 * subscription response handler (ibmf_saa_impl_get_informinfo_cb) 249 */ 250 ibmf_saa_sub_cb_t si_trans_sub_callback; 251 252 /* 253 * some unsubscribe requests are sequenced, others are unsequenced 254 * (depending on context that generated unsubscribe request) 255 */ 256 boolean_t si_trans_unseq_unsubscribe; 257 258 /* trans flags saved in case sm lid changes and msg must be resent */ 259 uint8_t si_trans_transport_flags; 260 261 /* counter for retrying requests which return MAD_BUSY status */ 262 uint8_t si_trans_retry_busy_count; 263 264 /* hrtime that we initiated this transaction */ 265 hrtime_t si_trans_send_time; 266 } saa_impl_trans_info_t; 267 _NOTE(SCHEME_PROTECTS_DATA("private callback arg", saa_impl_trans_info_t)) 268 269 typedef struct ibmf_saa_event_taskq_args_s { 270 saa_client_data_t *et_client; 271 ibmf_saa_subnet_event_t et_subnet_event; 272 ibmf_saa_event_details_t *et_event_details; 273 ib_mad_notice_t *et_notice; 274 ibmf_saa_subnet_event_cb_t et_callback; 275 void *et_callback_arg; 276 } ibmf_saa_event_taskq_args_t; 277 _NOTE(READ_ONLY_DATA(ibmf_saa_event_taskq_args_t::et_subnet_event 278 ibmf_saa_event_taskq_args_t::et_event_details 279 ibmf_saa_event_taskq_args_t::et_client 280 ibmf_saa_event_taskq_args_t::et_notice 281 ibmf_saa_event_taskq_args_t::et_callback 282 ibmf_saa_event_taskq_args_t::et_callback_arg)) 283 284 /* 285 * Public Functions 286 */ 287 int ibmf_saa_impl_init(); 288 int ibmf_saa_impl_fini(); 289 boolean_t ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref); 290 void ibmf_saa_impl_purge(); 291 int ibmf_saa_impl_add_client(saa_port_t *saa_portp); 292 int ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp); 293 int ibmf_saa_impl_init_kstats(saa_port_t *saa_portp); 294 void ibmf_saa_impl_register_failed(saa_port_t *saa_portp); 295 int ibmf_saa_impl_register_port(saa_port_t *saa_portp); 296 void ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp); 297 int ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info); 298 299 void ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 300 void *args); 301 302 void 303 ibmf_saa_add_event_subscriber(saa_client_data_t *client, 304 ibmf_saa_subnet_event_args_t *event_args); 305 306 void ibmf_saa_subscribe_events(saa_port_t *saa_portp, boolean_t subscribe, 307 boolean_t seq_unsubscribe); 308 309 void ibmf_saa_subscribe_sm_events(saa_port_t *saa_portp); 310 311 void 312 ibmf_saa_notify_event_clients(saa_port_t *saa_portp, 313 ibmf_saa_event_details_t *event_details, 314 ibmf_saa_subnet_event_t subnet_event, 315 saa_client_data_t *registering_client); 316 317 void 318 ibmf_saa_report_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 319 void *args); 320 #ifdef __cplusplus 321 } 322 #endif 323 324 #endif /* _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H */ 325