1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H 27 #define _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H 28 29 #pragma ident "%Z%%M% %I% %E% SMI" 30 31 /* 32 * saa_impl.h 33 */ 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #include <sys/ib/mgt/ibmf/ibmf_saa.h> 40 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 41 42 #define SAA_MAX_CLIENTS_PER_PORT 100 43 #define SAA_MAD_BASE_VERSION 1 44 #define SAA_MAD_CLASS_VERSION 2 45 #define IBMF_SAA_RETRANS_RETRIES 2 46 #define IBMF_SAA_MAX_SUBNET_TIMEOUT 17 47 #define IBMF_SAA_MAX_BUSY_RETRY_COUNT 10 48 #define IBMF_SAA_MAX_WAIT_TIME_IN_SECS 60 49 #define IBMF_SAA_TRANS_WAIT_TIME_IN_SECS 240 50 #define IBMF_SAA_BUSY_RETRY_SLEEP_SECS 1 /* seconds between retry */ 51 52 /* 53 * saa_port_s: 54 * Linked list of ports that saa is using. Each port is registered to ibmf. 55 * Multiple saa clients can associate with an saa port 56 */ 57 typedef enum saa_port_state_s { 58 IBMF_SAA_PORT_STATE_REGISTERING, 59 IBMF_SAA_PORT_STATE_READY, 60 IBMF_SAA_PORT_STATE_INVALID, /* client MUST close */ 61 IBMF_SAA_PORT_STATE_PURGING /* being purged */ 62 } saa_port_state_t; 63 64 typedef struct saa_port_s { 65 66 struct saa_port_s *next; 67 68 kmutex_t saa_pt_mutex; 69 70 /* registration synchronization: only one client registers to ibmf */ 71 kcondvar_t saa_pt_ibmf_reg_cv; 72 73 /* state and client reference counts */ 74 saa_port_state_t saa_pt_state; 75 int saa_pt_reference_count; 76 77 /* port identification and ibmf registration info */ 78 ib_guid_t saa_pt_port_guid; 79 ibmf_register_info_t saa_pt_ibmf_reginfo; 80 81 ibmf_handle_t saa_pt_ibmf_handle; 82 ibmf_impl_caps_t saa_pt_ibmf_impl_features; 83 ibmf_qp_handle_t saa_pt_qp_handle; 84 ib_qpn_t saa_pt_qpn; 85 86 /* transaction parameters */ 87 int saa_pt_timeout; /* from portinfo */ 88 uint16_t saa_pt_sa_cap_mask; /* from classportinfo */ 89 90 ibmf_addr_info_t saa_pt_ibmf_addr_info; 91 ibmf_global_addr_info_t saa_pt_ibmf_global_addr; 92 uint32_t saa_pt_ibmf_msg_flags; 93 boolean_t saa_pt_redirect_active; /* SA has redirected */ 94 95 ibmf_retrans_t saa_pt_ibmf_retrans; 96 uint64_t saa_pt_current_tid; 97 int saa_pt_num_outstanding_trans; 98 99 /* kstats */ 100 kmutex_t saa_pt_kstat_mutex; 101 struct kstat *saa_pt_kstatp; 102 103 /* sync. for receiving informinfo req packets */ 104 kmutex_t saa_pt_event_sub_mutex; 105 uint8_t saa_pt_event_sub_arrive_mask; 106 uint8_t saa_pt_event_sub_success_mask; 107 uint8_t saa_pt_event_sub_last_success_mask; 108 struct saa_client_data_s *saa_pt_event_sub_client_list; 109 110 /* node guid and port num, saved for easy ibt_queries */ 111 ib_guid_t saa_pt_node_guid; 112 uint8_t saa_pt_port_num; 113 114 /* latest hrtime that packet from SA was received */ 115 hrtime_t saa_pt_sa_uptime; 116 } saa_port_t; 117 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_mutex, 118 saa_port_t::saa_pt_reference_count 119 saa_port_t::saa_pt_ibmf_reg_cv 120 saa_port_t::saa_pt_ibmf_retrans 121 saa_port_t::saa_pt_current_tid 122 saa_port_t::saa_pt_num_outstanding_trans 123 saa_port_t::saa_pt_timeout 124 saa_port_t::saa_pt_ibmf_addr_info 125 saa_port_t::saa_pt_ibmf_global_addr 126 saa_port_t::saa_pt_ibmf_msg_flags 127 saa_port_t::saa_pt_redirect_active)) 128 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_kstat_mutex, 129 saa_port_t::saa_pt_kstatp)) 130 131 132 #define IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE \ 133 (IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_CA | \ 134 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SWITCH | \ 135 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_ROUTER | \ 136 IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM) 137 138 typedef struct ibmf_saa_kstat_s { 139 kstat_named_t clients_registered; /* # saa registrants */ 140 kstat_named_t clients_reg_failed; /* # failed registrants */ 141 kstat_named_t outstanding_requests; /* # outstanding requests */ 142 kstat_named_t total_requests; /* # requests ever made */ 143 kstat_named_t failed_requests; /* # failed requests */ 144 kstat_named_t requests_timedout; /* # requests that timedout */ 145 } ibmf_saa_kstat_t; 146 147 #define IBMF_SAA_ADD32_KSTATS(subnetp, xx, val) \ 148 if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) { \ 149 ibmf_saa_kstat_t *kp; \ 150 kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\ 151 kp->xx.value.ui32 += val; \ 152 } 153 154 #define IBMF_SAA_SUB32_KSTATS(subnetp, xx, val) \ 155 if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) { \ 156 ibmf_saa_kstat_t *kp; \ 157 kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\ 158 kp->xx.value.ui32 -= val; \ 159 } 160 161 typedef enum _saa_client_state_e { 162 SAA_CLIENT_STATE_ACTIVE, 163 SAA_CLIENT_STATE_WAITING, 164 SAA_CLIENT_STATE_CLOSED 165 } saa_client_state_t; 166 167 typedef struct saa_client_data_s { 168 void *next; 169 170 /* set for valid handles */ 171 void *saa_client_sig; 172 saa_port_t *saa_client_port; 173 kmutex_t saa_client_mutex; 174 int saa_client_num_pending_trans; 175 kcondvar_t saa_client_state_cv; 176 saa_client_state_t saa_client_state; 177 ib_smkey_t saa_client_sm_key; 178 179 int saa_client_event_cb_num_active; 180 kcondvar_t saa_client_event_cb_cv; 181 182 ibmf_saa_subnet_event_cb_t saa_client_event_cb; 183 void *saa_client_event_cb_arg; 184 } saa_client_data_t; 185 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_port)) 186 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_sig)) 187 188 189 typedef struct saa_state_s { 190 191 saa_port_t *saa_port_list; 192 kmutex_t saa_port_list_mutex; 193 taskq_t *saa_event_taskq; 194 } saa_state_t; 195 _NOTE(MUTEX_PROTECTS_DATA(saa_state_t::saa_port_list_mutex, 196 saa_port_t::next 197 saa_state_t::saa_port_list)) 198 _NOTE(READ_ONLY_DATA(saa_state_t::saa_event_taskq)) 199 200 /* 201 * special callback used specifically for handling informinfo responses; 202 * extra parameter is producer_type 203 */ 204 typedef void (*ibmf_saa_sub_cb_t) (void *, size_t, char *, int, uint32_t); 205 206 /* 207 * saa_impl_trans_info_t: 208 * Convenience structure wich allows ibmf_access_sa to group all the fields 209 * into one structure as a parameter to the send request function. 210 * This structure is allocated by ibmf_access_sa() and freed by ibmf_access_sa() 211 * in the sync case and by the ibmf_msg_transport callback in the async case 212 */ 213 typedef struct saa_impl_trans_info_t { 214 215 saa_client_data_t *si_trans_client_data; 216 saa_port_t *si_trans_port; 217 218 /* used to tell send_request about request mad */ 219 size_t si_trans_template_length; /* for unknown attr */ 220 uint16_t si_trans_attr_id; 221 uint64_t si_trans_component_mask; 222 void *si_trans_template; 223 uint8_t si_trans_method; 224 225 /* used for async call to tell send_request which callback to use */ 226 ibmf_saa_cb_t si_trans_callback; 227 void *si_trans_callback_arg; 228 229 /* 230 * used to tell ibmf_access_sa about response if the request was sync. 231 * If the request was async, the ibmf_msg_transport callback function 232 * will fill these values directly into the si_trans_callback 233 */ 234 int si_trans_status; 235 void *si_trans_result; 236 size_t si_trans_length; 237 238 /* fields needed for specific case of handling InformInfo requests */ 239 240 /* 241 * producer_type indicates the notice producer type that the 242 * subscription was for. There is no way to tell which type the 243 * response packet is for. 244 */ 245 uint32_t si_trans_sub_producer_type; 246 247 /* 248 * separate callback typedef to provide the producer type to the 249 * subscription response handler (ibmf_saa_impl_get_informinfo_cb) 250 */ 251 ibmf_saa_sub_cb_t si_trans_sub_callback; 252 253 /* 254 * some unsubscribe requests are sequenced, others are unsequenced 255 * (depending on context that generated unsubscribe request) 256 */ 257 boolean_t si_trans_unseq_unsubscribe; 258 259 /* trans flags saved in case sm lid changes and msg must be resent */ 260 uint8_t si_trans_transport_flags; 261 262 /* counter for retrying requests which return MAD_BUSY status */ 263 uint8_t si_trans_retry_busy_count; 264 265 /* hrtime that we initiated this transaction */ 266 hrtime_t si_trans_send_time; 267 } saa_impl_trans_info_t; 268 _NOTE(SCHEME_PROTECTS_DATA("private callback arg", saa_impl_trans_info_t)) 269 270 typedef struct ibmf_saa_event_taskq_args_s { 271 saa_client_data_t *et_client; 272 ibmf_saa_subnet_event_t et_subnet_event; 273 ibmf_saa_event_details_t *et_event_details; 274 ib_mad_notice_t *et_notice; 275 ibmf_saa_subnet_event_cb_t et_callback; 276 void *et_callback_arg; 277 } ibmf_saa_event_taskq_args_t; 278 _NOTE(READ_ONLY_DATA(ibmf_saa_event_taskq_args_t::et_subnet_event 279 ibmf_saa_event_taskq_args_t::et_event_details 280 ibmf_saa_event_taskq_args_t::et_client 281 ibmf_saa_event_taskq_args_t::et_notice 282 ibmf_saa_event_taskq_args_t::et_callback 283 ibmf_saa_event_taskq_args_t::et_callback_arg)) 284 285 /* 286 * Public Functions 287 */ 288 int ibmf_saa_impl_init(); 289 int ibmf_saa_impl_fini(); 290 boolean_t ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref); 291 void ibmf_saa_impl_purge(); 292 int ibmf_saa_impl_add_client(saa_port_t *saa_portp); 293 int ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp); 294 int ibmf_saa_impl_init_kstats(saa_port_t *saa_portp); 295 void ibmf_saa_impl_register_failed(saa_port_t *saa_portp); 296 int ibmf_saa_impl_register_port(saa_port_t *saa_portp); 297 void ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp); 298 int ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info); 299 300 void ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 301 void *args); 302 303 void 304 ibmf_saa_add_event_subscriber(saa_client_data_t *client, 305 ibmf_saa_subnet_event_args_t *event_args); 306 307 void ibmf_saa_subscribe_events(saa_port_t *saa_portp, boolean_t subscribe, 308 boolean_t seq_unsubscribe); 309 310 void 311 ibmf_saa_notify_event_clients(saa_port_t *saa_portp, 312 ibmf_saa_event_details_t *event_details, 313 ibmf_saa_subnet_event_t subnet_event, 314 saa_client_data_t *registering_client); 315 316 void 317 ibmf_saa_report_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, 318 void *args); 319 #ifdef __cplusplus 320 } 321 #endif 322 323 #endif /* _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H */ 324