1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright (c) 2008 Oracle. All rights reserved. 27 * 28 * This software is available to you under a choice of one of two 29 * licenses. You may choose to be licensed under the terms of the GNU 30 * General Public License (GPL) Version 2, available from the file 31 * COPYING in the main directory of this source tree, or the 32 * OpenIB.org BSD license below: 33 * 34 * Redistribution and use in source and binary forms, with or 35 * without modification, are permitted provided that the following 36 * conditions are met: 37 * 38 * - Redistributions of source code must retain the above 39 * copyright notice, this list of conditions and the following 40 * disclaimer. 41 * 42 * - Redistributions in binary form must reproduce the above 43 * copyright notice, this list of conditions and the following 44 * disclaimer in the documentation and/or other materials 45 * provided with the distribution. 46 * 47 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 50 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 51 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 52 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 53 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 * SOFTWARE. 55 * 56 */ 57 58 /* 59 * Include this file if the application uses rdsv3 sockets. 60 */ 61 62 /* 63 * This file contains definitions from the ofed rds.h and rds_rdma.h 64 * header file. 65 */ 66 #ifndef _RDSV3_RDS_H 67 #define _RDSV3_RDS_H 68 69 #include <sys/types.h> 70 71 #ifdef __cplusplus 72 extern "C" { 73 #endif 74 75 #define RDS_IB_ABI_VERSION 0x301 76 77 #define AF_RDS AF_INET_OFFLOAD 78 #define PF_RDS AF_INET_OFFLOAD 79 80 #define SOL_RDS 272 81 82 /* 83 * setsockopt/getsockopt for SOL_RDS 84 */ 85 #define RDSV3_CANCEL_SENT_TO 1 86 #define RDSV3_GET_MR 2 87 #define RDSV3_FREE_MR 3 88 /* deprecated: RDS_BARRIER 4 */ 89 #define RDSV3_RECVERR 5 90 #define RDSV3_CONG_MONITOR 6 91 #define RDSV3_GET_MR_FOR_DEST 7 92 93 /* 94 * Control message types for SOL_RDS. 95 * 96 * RDS_CMSG_RDMA_ARGS (sendmsg) 97 * Request a RDMA transfer to/from the specified 98 * memory ranges. 99 * The cmsg_data is a struct rdsv3_rdma_args. 100 * RDS_CMSG_RDMA_DEST (recvmsg, sendmsg) 101 * Kernel informs application about intended 102 * source/destination of a RDMA transfer 103 * RDS_CMSG_RDMA_MAP (sendmsg) 104 * Application asks kernel to map the given 105 * memory range into a IB MR, and send the 106 * R_Key along in an RDS extension header. 107 * The cmsg_data is a struct rdsv3_get_mr_args, 108 * the same as for the GET_MR setsockopt. 109 * RDS_CMSG_RDMA_STATUS (recvmsg) 110 * Returns the status of a completed RDMA operation. 111 */ 112 #define RDSV3_CMSG_RDMA_ARGS 1 113 #define RDSV3_CMSG_RDMA_DEST 2 114 #define RDSV3_CMSG_RDMA_MAP 3 115 #define RDSV3_CMSG_RDMA_STATUS 4 116 #define RDSV3_CMSG_CONG_UPDATE 5 117 118 /* rds-info related */ 119 120 #define RDSV3_INFO_FIRST 10000 121 #define RDSV3_INFO_COUNTERS 10000 122 #define RDSV3_INFO_CONNECTIONS 10001 123 /* 10002 aka RDS_INFO_FLOWS is deprecated */ 124 #define RDSV3_INFO_SEND_MESSAGES 10003 125 #define RDSV3_INFO_RETRANS_MESSAGES 10004 126 #define RDSV3_INFO_RECV_MESSAGES 10005 127 #define RDSV3_INFO_SOCKETS 10006 128 #define RDSV3_INFO_TCP_SOCKETS 10007 129 #define RDSV3_INFO_IB_CONNECTIONS 10008 130 #define RDSV3_INFO_CONNECTION_STATS 10009 131 #define RDSV3_INFO_IWARP_CONNECTIONS 10010 132 #define RDSV3_INFO_LAST 10010 133 134 #ifndef __lock_lint 135 #pragma pack(1) 136 struct rdsv3_info_counter { 137 uint8_t name[32]; 138 uint64_t value; 139 } __attribute__((packed)); 140 #pragma pack() 141 #else 142 struct rdsv3_info_counter { 143 uint8_t name[32]; 144 uint64_t value; 145 }; 146 #endif 147 148 #define RDSV3_INFO_CONNECTION_FLAG_SENDING 0x01 149 #define RDSV3_INFO_CONNECTION_FLAG_CONNECTING 0x02 150 #define RDSV3_INFO_CONNECTION_FLAG_CONNECTED 0x04 151 152 #define TRANSNAMSIZ 16 153 154 #ifndef __lock_lint 155 #pragma pack(1) 156 struct rdsv3_info_connection { 157 uint64_t next_tx_seq; 158 uint64_t next_rx_seq; 159 uint32_t laddr; /* network order */ 160 uint32_t faddr; /* network order */ 161 uint8_t transport[15]; /* null term ascii */ 162 uint8_t flags; 163 } __attribute__((packed)); 164 #pragma pack() 165 #else 166 struct rdsv3_info_connection { 167 uint64_t next_tx_seq; 168 uint64_t next_rx_seq; 169 uint32_t laddr; /* network order */ 170 uint32_t faddr; /* network order */ 171 uint8_t transport[15]; /* null term ascii */ 172 uint8_t flags; 173 }; 174 #endif 175 176 #ifndef __lock_lint 177 #pragma pack(1) 178 struct rdsv3_info_flow { 179 uint32_t laddr; /* network order */ 180 uint32_t faddr; /* network order */ 181 uint32_t bytes; 182 uint16_t lport; /* network order */ 183 uint16_t fport; /* network order */ 184 } __attribute__((packed)); 185 #pragma pack() 186 #else 187 struct rdsv3_info_flow { 188 uint32_t laddr; /* network order */ 189 uint32_t faddr; /* network order */ 190 uint32_t bytes; 191 uint16_t lport; /* network order */ 192 uint16_t fport; /* network order */ 193 }; 194 #endif 195 196 #define RDSV3_INFO_MESSAGE_FLAG_ACK 0x01 197 #define RDSV3_INFO_MESSAGE_FLAG_FAST_ACK 0x02 198 199 #ifndef __lock_lint 200 #pragma pack(1) 201 struct rdsv3_info_message { 202 uint64_t seq; 203 uint32_t len; 204 uint32_t laddr; /* network order */ 205 uint32_t faddr; /* network order */ 206 uint16_t lport; /* network order */ 207 uint16_t fport; /* network order */ 208 uint8_t flags; 209 } __attribute__((packed)); 210 #pragma pack() 211 #else 212 struct rdsv3_info_message { 213 uint64_t seq; 214 uint32_t len; 215 uint32_t laddr; /* network order */ 216 uint32_t faddr; /* network order */ 217 uint16_t lport; /* network order */ 218 uint16_t fport; /* network order */ 219 uint8_t flags; 220 }; 221 #endif 222 223 #ifndef __lock_lint 224 #pragma pack(1) 225 struct rdsv3_info_socket { 226 uint32_t sndbuf; 227 uint32_t bound_addr; /* network order */ 228 uint32_t connected_addr; /* network order */ 229 uint16_t bound_port; /* network order */ 230 uint16_t connected_port; /* network order */ 231 uint32_t rcvbuf; 232 uint64_t inum; 233 } __attribute__((packed)); 234 #pragma pack() 235 #else 236 struct rdsv3_info_socket { 237 uint32_t sndbuf; 238 uint32_t bound_addr; /* network order */ 239 uint32_t connected_addr; /* network order */ 240 uint16_t bound_port; /* network order */ 241 uint16_t connected_port; /* network order */ 242 uint32_t rcvbuf; 243 uint64_t inum; 244 }; 245 #endif 246 247 #ifndef __lock_lint 248 #pragma pack(1) 249 struct rdsv3_info_socket_v1 { 250 uint32_t sndbuf; 251 uint32_t bound_addr; /* network order */ 252 uint32_t connected_addr; /* network order */ 253 uint16_t bound_port; /* network order */ 254 uint16_t connected_port; /* network order */ 255 uint32_t rcvbuf; 256 } __attribute__((packed)); 257 #pragma pack() 258 #else 259 struct rdsv3_info_socket_v1 { 260 uint32_t sndbuf; 261 uint32_t bound_addr; /* network order */ 262 uint32_t connected_addr; /* network order */ 263 uint16_t bound_port; /* network order */ 264 uint16_t connected_port; /* network order */ 265 uint32_t rcvbuf; 266 }; 267 #endif 268 269 #define RDS_IB_GID_LEN 16 270 struct rdsv3_info_rdma_connection { 271 uint32_t src_addr; /* network order */ 272 uint32_t dst_addr; /* network order */ 273 uint8_t src_gid[RDS_IB_GID_LEN]; 274 uint8_t dst_gid[RDS_IB_GID_LEN]; 275 276 uint32_t max_send_wr; 277 uint32_t max_recv_wr; 278 uint32_t max_send_sge; 279 uint32_t rdma_mr_max; 280 uint32_t rdma_mr_size; 281 }; 282 283 #define rdsv3_info_ib_connection rdsv3_info_rdma_connection 284 #define rdma_fmr_max rdma_mr_max 285 #define rdma_fmr_size rdma_mr_size 286 287 /* 288 * Congestion monitoring. 289 * Congestion control in RDS happens at the host connection 290 * level by exchanging a bitmap marking congested ports. 291 * By default, a process sleeping in poll() is always woken 292 * up when the congestion map is updated. 293 * With explicit monitoring, an application can have more 294 * fine-grained control. 295 * The application installs a 64bit mask value in the socket, 296 * where each bit corresponds to a group of ports. 297 * When a congestion update arrives, RDS checks the set of 298 * ports that are now uncongested against the list bit mask 299 * installed in the socket, and if they overlap, we queue a 300 * cong_notification on the socket. 301 * 302 * To install the congestion monitor bitmask, use RDS_CONG_MONITOR 303 * with the 64bit mask. 304 * Congestion updates are received via RDS_CMSG_CONG_UPDATE 305 * control messages. 306 * 307 * The correspondence between bits and ports is 308 * 1 << (portnum % 64) 309 */ 310 #define RDSV3_CONG_MONITOR_SIZE 64 311 #define RDSV3_CONG_MONITOR_BIT(port) \ 312 (((unsigned int) port) % RDSV3_CONG_MONITOR_SIZE) 313 #define RDSV3_CONG_MONITOR_MASK(port) (1ULL << RDSV3_CONG_MONITOR_BIT(port)) 314 315 /* 316 * RDMA related types 317 */ 318 319 /* 320 * This encapsulates a remote memory location. 321 * In the current implementation, it contains the R_Key 322 * of the remote memory region, and the offset into it 323 * (so that the application does not have to worry about 324 * alignment). 325 */ 326 typedef uint64_t rdsv3_rdma_cookie_t; 327 328 struct rdsv3_iovec { 329 uint64_t addr; 330 uint64_t bytes; 331 }; 332 333 struct rdsv3_get_mr_args { 334 struct rdsv3_iovec vec; 335 uint64_t cookie_addr; 336 uint64_t flags; 337 }; 338 339 #if 1 /* 1 at 1.5.1 */ 340 #include <sys/socket_impl.h> 341 struct rdsv3_get_mr_for_dest_args { 342 struct sockaddr_storage dest_addr; 343 struct rdsv3_iovec vec; 344 uint64_t cookie_addr; 345 uint64_t flags; 346 }; 347 #endif 348 349 struct rdsv3_free_mr_args { 350 rdsv3_rdma_cookie_t cookie; 351 uint64_t flags; 352 }; 353 354 struct rdsv3_rdma_args { 355 rdsv3_rdma_cookie_t cookie; 356 struct rdsv3_iovec remote_vec; 357 uint64_t local_vec_addr; 358 uint64_t nr_local; 359 uint64_t flags; 360 uint64_t user_token; 361 }; 362 363 struct rdsv3_rdma_notify { 364 uint64_t user_token; 365 int32_t status; 366 }; 367 368 #define RDSV3_RDMA_SUCCESS 0 369 #define RDSV3_RDMA_REMOTE_ERROR 1 370 #define RDSV3_RDMA_CANCELED 2 371 #define RDSV3_RDMA_DROPPED 3 372 #define RDSV3_RDMA_OTHER_ERROR 4 373 374 /* 375 * Common set of flags for all RDMA related structs 376 */ 377 #define RDSV3_RDMA_READWRITE 0x0001 378 #define RDSV3_RDMA_FENCE 0x0002 /* use FENCE for immediate send */ 379 #define RDSV3_RDMA_INVALIDATE 0x0004 /* invalidate R_Key after freeing MR */ 380 #define RDSV3_RDMA_USE_ONCE 0x0008 /* free MR after use */ 381 #define RDSV3_RDMA_DONTWAIT 0x0010 /* Don't wait in SET_BARRIER */ 382 #define RDSV3_RDMA_NOTIFY_ME 0x0020 /* Notify when operation completes */ 383 384 #ifdef __cplusplus 385 } 386 #endif 387 388 #endif /* _RDSV3_RDS_H */ 389