1 /* 2 * Copyright (c) 2004-2006 Intel Corporation. All rights reserved. 3 * Copyright (c) 2004 Topspin Corporation. All rights reserved. 4 * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * $Id$ 35 */ 36 #if !defined(CM_H) 37 #define CM_H 38 39 #include <infiniband/verbs.h> 40 #include <infiniband/sa.h> 41 #ifdef _KERNEL 42 #include <linux/types.h> 43 #include <sys/endian.h> 44 #else 45 #include <infiniband/types.h> 46 #include <infiniband/endian.h> 47 #endif 48 49 #ifdef __cplusplus 50 extern "C" { 51 #endif 52 53 enum ib_cm_event_type { 54 IB_CM_REQ_ERROR, 55 IB_CM_REQ_RECEIVED, 56 IB_CM_REP_ERROR, 57 IB_CM_REP_RECEIVED, 58 IB_CM_RTU_RECEIVED, 59 IB_CM_USER_ESTABLISHED, 60 IB_CM_DREQ_ERROR, 61 IB_CM_DREQ_RECEIVED, 62 IB_CM_DREP_RECEIVED, 63 IB_CM_TIMEWAIT_EXIT, 64 IB_CM_MRA_RECEIVED, 65 IB_CM_REJ_RECEIVED, 66 IB_CM_LAP_ERROR, 67 IB_CM_LAP_RECEIVED, 68 IB_CM_APR_RECEIVED, 69 IB_CM_SIDR_REQ_ERROR, 70 IB_CM_SIDR_REQ_RECEIVED, 71 IB_CM_SIDR_REP_RECEIVED 72 }; 73 74 enum ib_cm_data_size { 75 IB_CM_REQ_PRIVATE_DATA_SIZE = 92, 76 IB_CM_MRA_PRIVATE_DATA_SIZE = 222, 77 IB_CM_REJ_PRIVATE_DATA_SIZE = 148, 78 IB_CM_REP_PRIVATE_DATA_SIZE = 196, 79 IB_CM_RTU_PRIVATE_DATA_SIZE = 224, 80 IB_CM_DREQ_PRIVATE_DATA_SIZE = 220, 81 IB_CM_DREP_PRIVATE_DATA_SIZE = 224, 82 IB_CM_REJ_ARI_LENGTH = 72, 83 IB_CM_LAP_PRIVATE_DATA_SIZE = 168, 84 IB_CM_APR_PRIVATE_DATA_SIZE = 148, 85 IB_CM_APR_INFO_LENGTH = 72, 86 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE = 216, 87 IB_CM_SIDR_REP_PRIVATE_DATA_SIZE = 136, 88 IB_CM_SIDR_REP_INFO_LENGTH = 72 89 }; 90 91 struct ib_cm_device { 92 struct ibv_context *device_context; 93 int fd; 94 }; 95 96 struct ib_cm_id { 97 void *context; 98 struct ib_cm_device *device; 99 uint32_t handle; 100 }; 101 102 struct ib_cm_req_event_param { 103 struct ib_cm_id *listen_id; 104 uint8_t port; 105 106 struct ibv_sa_path_rec *primary_path; 107 struct ibv_sa_path_rec *alternate_path; 108 109 __be64 remote_ca_guid; 110 uint32_t remote_qkey; 111 uint32_t remote_qpn; 112 enum ibv_qp_type qp_type; 113 114 uint32_t starting_psn; 115 uint8_t responder_resources; 116 uint8_t initiator_depth; 117 unsigned int local_cm_response_timeout:5; 118 unsigned int flow_control:1; 119 unsigned int remote_cm_response_timeout:5; 120 unsigned int retry_count:3; 121 unsigned int rnr_retry_count:3; 122 unsigned int srq:1; 123 }; 124 125 struct ib_cm_rep_event_param { 126 __be64 remote_ca_guid; 127 uint32_t remote_qkey; 128 uint32_t remote_qpn; 129 uint32_t starting_psn; 130 uint8_t responder_resources; 131 uint8_t initiator_depth; 132 unsigned int target_ack_delay:5; 133 unsigned int failover_accepted:2; 134 unsigned int flow_control:1; 135 unsigned int rnr_retry_count:3; 136 unsigned int srq:1; 137 }; 138 139 enum ib_cm_rej_reason { 140 IB_CM_REJ_NO_QP = 1, 141 IB_CM_REJ_NO_EEC = 2, 142 IB_CM_REJ_NO_RESOURCES = 3, 143 IB_CM_REJ_TIMEOUT = 4, 144 IB_CM_REJ_UNSUPPORTED = 5, 145 IB_CM_REJ_INVALID_COMM_ID = 6, 146 IB_CM_REJ_INVALID_COMM_INSTANCE = 7, 147 IB_CM_REJ_INVALID_SERVICE_ID = 8, 148 IB_CM_REJ_INVALID_TRANSPORT_TYPE = 9, 149 IB_CM_REJ_STALE_CONN = 10, 150 IB_CM_REJ_RDC_NOT_EXIST = 11, 151 IB_CM_REJ_INVALID_GID = 12, 152 IB_CM_REJ_INVALID_LID = 13, 153 IB_CM_REJ_INVALID_SL = 14, 154 IB_CM_REJ_INVALID_TRAFFIC_CLASS = 15, 155 IB_CM_REJ_INVALID_HOP_LIMIT = 16, 156 IB_CM_REJ_INVALID_PACKET_RATE = 17, 157 IB_CM_REJ_INVALID_ALT_GID = 18, 158 IB_CM_REJ_INVALID_ALT_LID = 19, 159 IB_CM_REJ_INVALID_ALT_SL = 20, 160 IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS = 21, 161 IB_CM_REJ_INVALID_ALT_HOP_LIMIT = 22, 162 IB_CM_REJ_INVALID_ALT_PACKET_RATE = 23, 163 IB_CM_REJ_PORT_CM_REDIRECT = 24, 164 IB_CM_REJ_PORT_REDIRECT = 25, 165 IB_CM_REJ_INVALID_MTU = 26, 166 IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES = 27, 167 IB_CM_REJ_CONSUMER_DEFINED = 28, 168 IB_CM_REJ_INVALID_RNR_RETRY = 29, 169 IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID = 30, 170 IB_CM_REJ_INVALID_CLASS_VERSION = 31, 171 IB_CM_REJ_INVALID_FLOW_LABEL = 32, 172 IB_CM_REJ_INVALID_ALT_FLOW_LABEL = 33 173 }; 174 175 struct ib_cm_rej_event_param { 176 enum ib_cm_rej_reason reason; 177 void *ari; 178 uint8_t ari_length; 179 }; 180 181 struct ib_cm_mra_event_param { 182 uint8_t service_timeout; 183 }; 184 185 struct ib_cm_lap_event_param { 186 struct ibv_sa_path_rec *alternate_path; 187 }; 188 189 enum ib_cm_apr_status { 190 IB_CM_APR_SUCCESS, 191 IB_CM_APR_INVALID_COMM_ID, 192 IB_CM_APR_UNSUPPORTED, 193 IB_CM_APR_REJECT, 194 IB_CM_APR_REDIRECT, 195 IB_CM_APR_IS_CURRENT, 196 IB_CM_APR_INVALID_QPN_EECN, 197 IB_CM_APR_INVALID_LID, 198 IB_CM_APR_INVALID_GID, 199 IB_CM_APR_INVALID_FLOW_LABEL, 200 IB_CM_APR_INVALID_TCLASS, 201 IB_CM_APR_INVALID_HOP_LIMIT, 202 IB_CM_APR_INVALID_PACKET_RATE, 203 IB_CM_APR_INVALID_SL 204 }; 205 206 struct ib_cm_apr_event_param { 207 enum ib_cm_apr_status ap_status; 208 void *apr_info; 209 uint8_t info_len; 210 }; 211 212 struct ib_cm_sidr_req_event_param { 213 struct ib_cm_id *listen_id; 214 uint8_t port; 215 uint16_t pkey; 216 }; 217 218 enum ib_cm_sidr_status { 219 IB_SIDR_SUCCESS, 220 IB_SIDR_UNSUPPORTED, 221 IB_SIDR_REJECT, 222 IB_SIDR_NO_QP, 223 IB_SIDR_REDIRECT, 224 IB_SIDR_UNSUPPORTED_VERSION 225 }; 226 227 struct ib_cm_sidr_rep_event_param { 228 enum ib_cm_sidr_status status; 229 uint32_t qkey; 230 uint32_t qpn; 231 void *info; 232 uint8_t info_len; 233 }; 234 235 struct ib_cm_event { 236 struct ib_cm_id *cm_id; 237 enum ib_cm_event_type event; 238 union { 239 struct ib_cm_req_event_param req_rcvd; 240 struct ib_cm_rep_event_param rep_rcvd; 241 /* No data for RTU received events. */ 242 struct ib_cm_rej_event_param rej_rcvd; 243 struct ib_cm_mra_event_param mra_rcvd; 244 struct ib_cm_lap_event_param lap_rcvd; 245 struct ib_cm_apr_event_param apr_rcvd; 246 /* No data for DREQ/DREP received events. */ 247 struct ib_cm_sidr_req_event_param sidr_req_rcvd; 248 struct ib_cm_sidr_rep_event_param sidr_rep_rcvd; 249 enum ibv_wc_status send_status; 250 } param; 251 252 void *private_data; 253 }; 254 255 /** 256 * ib_cm_get_event - Retrieves the next pending communications event, 257 * if no event is pending waits for an event. 258 * @device: CM device to retrieve the event. 259 * @event: Allocated information about the next communication event. 260 * Event should be freed using ib_cm_ack_event() 261 * 262 * IB_CM_REQ_RECEIVED and IB_CM_SIDR_REQ_RECEIVED communication events 263 * generated as a result of listen requests result in the allocation of a 264 * new @cm_id. 265 * Clients are responsible for destroying the new @cm_id. For peer-to-peer 266 * IB_CM_REQ_RECEIVED and all other events, the returned @cm_id corresponds 267 * to a user's existing communication identifier. 268 */ 269 int ib_cm_get_event(struct ib_cm_device *device, struct ib_cm_event **event); 270 271 /** 272 * ib_cm_ack_event - Free a communications event. 273 * @event: Event to be released. 274 * 275 * All events which are allocated by ib_cm_get_event() must be released, 276 * there should be a one-to-one correspondence between successful gets 277 * and puts. 278 */ 279 int ib_cm_ack_event(struct ib_cm_event *event); 280 281 /** 282 * ib_cm_open_device - Returns the device the CM uses to submit requests 283 * and retrieve events, corresponding to the specified verbs device. 284 * 285 * The CM device contains the file descriptor that the CM uses to 286 * communicate with the kernel CM component. The primary use of the 287 * file descriptor is to test for CM readiness events. When the CM 288 * becomes ready to READ there is a pending event ready, and a subsequent 289 * call to ib_cm_get_event will not block. 290 * Note: The user should not read or write directly to the CM file 291 * descriptor, it will likely result in an error or unexpected 292 * results. 293 */ 294 struct ib_cm_device* ib_cm_open_device(struct ibv_context *device_context); 295 296 /** 297 * ib_cm_close_device - Close a CM device. 298 * @device: Device to close. 299 */ 300 void ib_cm_close_device(struct ib_cm_device *device); 301 302 /** 303 * ib_cm_create_id - Allocate a communication identifier. 304 * 305 * Communication identifiers are used to track connection states, service 306 * ID resolution requests, and listen requests. 307 */ 308 int ib_cm_create_id(struct ib_cm_device *device, 309 struct ib_cm_id **cm_id, void *context); 310 311 /** 312 * ib_cm_destroy_id - Destroy a connection identifier. 313 * @cm_id: Connection identifier to destroy. 314 */ 315 int ib_cm_destroy_id(struct ib_cm_id *cm_id); 316 317 struct ib_cm_attr_param { 318 __be64 service_id; 319 __be64 service_mask; 320 __be32 local_id; 321 __be32 remote_id; 322 }; 323 324 /** 325 * ib_cm_attr_id - Get connection identifier attributes. 326 * @cm_id: Connection identifier to retrieve attributes. 327 * @param: Destination of retreived parameters. 328 * 329 * Not all parameters are valid during all connection states. 330 */ 331 int ib_cm_attr_id(struct ib_cm_id *cm_id, 332 struct ib_cm_attr_param *param); 333 334 #define IB_CM_ASSIGN_SERVICE_ID_MASK htobe64(0xFF00000000000000ULL) 335 #define IB_CM_ASSIGN_SERVICE_ID htobe64(0x0200000000000000ULL) 336 337 /** 338 * ib_cm_listen - Initiates listening on the specified service ID for 339 * connection and service ID resolution requests. 340 * @cm_id: Connection identifier associated with the listen request. 341 * @service_id: Service identifier matched against incoming connection 342 * and service ID resolution requests. The service ID should be specified 343 * network-byte order. 344 * @service_mask: Mask applied to service ID used to listen across a 345 * range of service IDs. If set to 0, the service ID is matched 346 * exactly. 347 */ 348 int ib_cm_listen(struct ib_cm_id *cm_id, 349 __be64 service_id, 350 __be64 service_mask); 351 352 struct ib_cm_req_param { 353 struct ibv_sa_path_rec *primary_path; 354 struct ibv_sa_path_rec *alternate_path; 355 __be64 service_id; 356 uint32_t qp_num; 357 enum ibv_qp_type qp_type; 358 uint32_t starting_psn; 359 void *private_data; 360 uint8_t private_data_len; 361 uint8_t peer_to_peer; 362 uint8_t responder_resources; 363 uint8_t initiator_depth; 364 uint8_t remote_cm_response_timeout; 365 uint8_t flow_control; 366 uint8_t local_cm_response_timeout; 367 uint8_t retry_count; 368 uint8_t rnr_retry_count; 369 uint8_t max_cm_retries; 370 uint8_t srq; 371 }; 372 373 /** 374 * ib_cm_send_req - Sends a connection request to the remote node. 375 * @cm_id: Connection identifier that will be associated with the 376 * connection request. 377 * @param: Connection request information needed to establish the 378 * connection. 379 */ 380 int ib_cm_send_req(struct ib_cm_id *cm_id, 381 struct ib_cm_req_param *param); 382 383 struct ib_cm_rep_param { 384 uint32_t qp_num; 385 uint32_t starting_psn; 386 void *private_data; 387 uint8_t private_data_len; 388 uint8_t responder_resources; 389 uint8_t initiator_depth; 390 uint8_t target_ack_delay; 391 uint8_t failover_accepted; 392 uint8_t flow_control; 393 uint8_t rnr_retry_count; 394 uint8_t srq; 395 }; 396 397 /** 398 * ib_cm_send_rep - Sends a connection reply in response to a connection 399 * request. 400 * @cm_id: Connection identifier that will be associated with the 401 * connection request. 402 * @param: Connection reply information needed to establish the 403 * connection. 404 */ 405 int ib_cm_send_rep(struct ib_cm_id *cm_id, 406 struct ib_cm_rep_param *param); 407 408 /** 409 * ib_cm_send_rtu - Sends a connection ready to use message in response 410 * to a connection reply message. 411 * @cm_id: Connection identifier associated with the connection request. 412 * @private_data: Optional user-defined private data sent with the 413 * ready to use message. 414 * @private_data_len: Size of the private data buffer, in bytes. 415 */ 416 int ib_cm_send_rtu(struct ib_cm_id *cm_id, 417 void *private_data, 418 uint8_t private_data_len); 419 420 /** 421 * ib_cm_send_dreq - Sends a disconnection request for an existing 422 * connection. 423 * @cm_id: Connection identifier associated with the connection being 424 * released. 425 * @private_data: Optional user-defined private data sent with the 426 * disconnection request message. 427 * @private_data_len: Size of the private data buffer, in bytes. 428 */ 429 int ib_cm_send_dreq(struct ib_cm_id *cm_id, 430 void *private_data, 431 uint8_t private_data_len); 432 433 /** 434 * ib_cm_send_drep - Sends a disconnection reply to a disconnection request. 435 * @cm_id: Connection identifier associated with the connection being 436 * released. 437 * @private_data: Optional user-defined private data sent with the 438 * disconnection reply message. 439 * @private_data_len: Size of the private data buffer, in bytes. 440 */ 441 int ib_cm_send_drep(struct ib_cm_id *cm_id, 442 void *private_data, 443 uint8_t private_data_len); 444 445 /** 446 * ib_cm_notify - Notifies the CM of an event reported to the consumer. 447 * @cm_id: Connection identifier to transition to established. 448 * @event: Type of event. 449 * 450 * This routine should be invoked by users to notify the CM of relevant 451 * communication events. Events that should be reported to the CM and 452 * when to report them are: 453 * 454 * IBV_EVENT_COMM_EST - Used when a message is received on a connected 455 * QP before an RTU has been received. 456 * IBV_EVENT_PATH_MIG - Notifies the CM that the connection has failed over 457 * to the alternate path. 458 */ 459 int ib_cm_notify(struct ib_cm_id *cm_id, enum ibv_event_type event); 460 461 /** 462 * ib_cm_send_rej - Sends a connection rejection message to the 463 * remote node. 464 * @cm_id: Connection identifier associated with the connection being 465 * rejected. 466 * @reason: Reason for the connection request rejection. 467 * @ari: Optional additional rejection information. 468 * @ari_length: Size of the additional rejection information, in bytes. 469 * @private_data: Optional user-defined private data sent with the 470 * rejection message. 471 * @private_data_len: Size of the private data buffer, in bytes. 472 */ 473 int ib_cm_send_rej(struct ib_cm_id *cm_id, 474 enum ib_cm_rej_reason reason, 475 void *ari, 476 uint8_t ari_length, 477 void *private_data, 478 uint8_t private_data_len); 479 480 /** 481 * ib_cm_send_mra - Sends a message receipt acknowledgement to a connection 482 * message. 483 * @cm_id: Connection identifier associated with the connection message. 484 * @service_timeout: The maximum time required for the sender to reply to 485 * to the connection message. 486 * @private_data: Optional user-defined private data sent with the 487 * message receipt acknowledgement. 488 * @private_data_len: Size of the private data buffer, in bytes. 489 */ 490 int ib_cm_send_mra(struct ib_cm_id *cm_id, 491 uint8_t service_timeout, 492 void *private_data, 493 uint8_t private_data_len); 494 495 /** 496 * ib_cm_send_lap - Sends a load alternate path request. 497 * @cm_id: Connection identifier associated with the load alternate path 498 * message. 499 * @alternate_path: A path record that identifies the alternate path to 500 * load. 501 * @private_data: Optional user-defined private data sent with the 502 * load alternate path message. 503 * @private_data_len: Size of the private data buffer, in bytes. 504 */ 505 int ib_cm_send_lap(struct ib_cm_id *cm_id, 506 struct ibv_sa_path_rec *alternate_path, 507 void *private_data, 508 uint8_t private_data_len); 509 510 /** 511 * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning 512 * to a specified QP state. 513 * @cm_id: Communication identifier associated with the QP attributes to 514 * initialize. 515 * @qp_attr: On input, specifies the desired QP state. On output, the 516 * mandatory and desired optional attributes will be set in order to 517 * modify the QP to the specified state. 518 * @qp_attr_mask: The QP attribute mask that may be used to transition the 519 * QP to the specified state. 520 * 521 * Users must set the @qp_attr->qp_state to the desired QP state. This call 522 * will set all required attributes for the given transition, along with 523 * known optional attributes. Users may override the attributes returned from 524 * this call before calling ib_modify_qp. 525 */ 526 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, 527 struct ibv_qp_attr *qp_attr, 528 int *qp_attr_mask); 529 530 /** 531 * ib_cm_send_apr - Sends an alternate path response message in response to 532 * a load alternate path request. 533 * @cm_id: Connection identifier associated with the alternate path response. 534 * @status: Reply status sent with the alternate path response. 535 * @info: Optional additional information sent with the alternate path 536 * response. 537 * @info_length: Size of the additional information, in bytes. 538 * @private_data: Optional user-defined private data sent with the 539 * alternate path response message. 540 * @private_data_len: Size of the private data buffer, in bytes. 541 */ 542 int ib_cm_send_apr(struct ib_cm_id *cm_id, 543 enum ib_cm_apr_status status, 544 void *info, 545 uint8_t info_length, 546 void *private_data, 547 uint8_t private_data_len); 548 549 struct ib_cm_sidr_req_param { 550 struct ibv_sa_path_rec *path; 551 __be64 service_id; 552 int timeout_ms; 553 void *private_data; 554 uint8_t private_data_len; 555 uint8_t max_cm_retries; 556 }; 557 558 /** 559 * ib_cm_send_sidr_req - Sends a service ID resolution request to the 560 * remote node. 561 * @cm_id: Communication identifier that will be associated with the 562 * service ID resolution request. 563 * @param: Service ID resolution request information. 564 */ 565 int ib_cm_send_sidr_req(struct ib_cm_id *cm_id, 566 struct ib_cm_sidr_req_param *param); 567 568 struct ib_cm_sidr_rep_param { 569 uint32_t qp_num; 570 uint32_t qkey; 571 enum ib_cm_sidr_status status; 572 void *info; 573 uint8_t info_length; 574 void *private_data; 575 uint8_t private_data_len; 576 }; 577 578 /** 579 * ib_cm_send_sidr_rep - Sends a service ID resolution reply to the 580 * remote node. 581 * @cm_id: Communication identifier associated with the received service ID 582 * resolution request. 583 * @param: Service ID resolution reply information. 584 */ 585 int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id, 586 struct ib_cm_sidr_rep_param *param); 587 588 #ifdef __cplusplus 589 } 590 #endif 591 592 #endif /* CM_H */ 593