1 /* 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 3 */ 4 5 /* 6 * This file contains code imported from the OFED rds source file 7 * rdma_transport.c * Oracle elects to have and use the contents of 8 * rdma_transport.c under and governed by the OpenIB.org BSD license 9 * (see below for full license text). However, the following notice 10 * accompanied the original version of this file: 11 */ 12 13 /* 14 * Copyright (c) 2009 Oracle. All rights reserved. 15 * 16 * This software is available to you under a choice of one of two 17 * licenses. You may choose to be licensed under the terms of the GNU 18 * General Public License (GPL) Version 2, available from the file 19 * COPYING in the main directory of this source tree, or the 20 * OpenIB.org BSD license below: 21 * 22 * Redistribution and use in source and binary forms, with or 23 * without modification, are permitted provided that the following 24 * conditions are met: 25 * 26 * - Redistributions of source code must retain the above 27 * copyright notice, this list of conditions and the following 28 * disclaimer. 29 * 30 * - Redistributions in binary form must reproduce the above 31 * copyright notice, this list of conditions and the following 32 * disclaimer in the documentation and/or other materials 33 * provided with the distribution. 34 * 35 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 36 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 37 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 38 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 39 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 40 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 41 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 42 * SOFTWARE. 43 * 44 */ 45 #include <sys/ib/clients/of/rdma/ib_verbs.h> 46 #include <sys/ib/clients/of/rdma/ib_addr.h> 47 #include <sys/ib/clients/of/rdma/rdma_cm.h> 48 49 #include <sys/ib/clients/rdsv3/ib.h> 50 #include <sys/ib/clients/rdsv3/rdma_transport.h> 51 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 52 53 kmutex_t rdsv3_rdma_listen_id_lock; 54 struct rdma_cm_id *rdsv3_rdma_listen_id = NULL; 55 56 int 57 rdsv3_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 58 struct rdma_cm_event *event) 59 { 60 /* this can be null in the listening path */ 61 struct rdsv3_connection *conn = cm_id->context; 62 struct rdsv3_transport *trans; 63 int ret = 0; 64 65 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 66 "conn %p id %p handling event %u", conn, cm_id, event->event); 67 68 trans = &rdsv3_ib_transport; 69 70 /* 71 * Prevent shutdown from tearing down the connection 72 * while we're executing. 73 */ 74 if (conn) { 75 mutex_enter(&conn->c_cm_lock); 76 77 /* 78 * If the connection is being shut down, bail out 79 * right away. We return 0 so cm_id doesn't get 80 * destroyed prematurely 81 */ 82 if (rdsv3_conn_state(conn) == RDSV3_CONN_DISCONNECTING) { 83 /* 84 * Reject incoming connections while we're tearing 85 * down an existing one. 86 */ 87 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 88 ret = 1; 89 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 90 "conn %p id %p incoming event %u when " 91 "disconnecting", conn, cm_id, event->event); 92 goto out; 93 } 94 } 95 96 switch (event->event) { 97 case RDMA_CM_EVENT_CONNECT_REQUEST: 98 ret = trans->cm_handle_connect(cm_id, event); 99 break; 100 101 case RDMA_CM_EVENT_ADDR_RESOLVED: 102 /* XXX do we need to clean up if this fails? */ 103 ret = rdma_resolve_route(cm_id, 104 RDSV3_RDMA_RESOLVE_TIMEOUT_MS); 105 break; 106 107 case RDMA_CM_EVENT_ROUTE_RESOLVED: 108 /* XXX worry about racing with listen acceptance */ 109 ret = trans->cm_initiate_connect(cm_id); 110 break; 111 112 case RDMA_CM_EVENT_ESTABLISHED: 113 trans->cm_connect_complete(conn, event); 114 break; 115 116 case RDMA_CM_EVENT_ADDR_ERROR: 117 case RDMA_CM_EVENT_ROUTE_ERROR: 118 case RDMA_CM_EVENT_CONNECT_ERROR: 119 case RDMA_CM_EVENT_UNREACHABLE: 120 case RDMA_CM_EVENT_REJECTED: 121 case RDMA_CM_EVENT_DEVICE_REMOVAL: 122 case RDMA_CM_EVENT_ADDR_CHANGE: 123 if (conn) 124 rdsv3_conn_drop(conn); 125 break; 126 127 case RDMA_CM_EVENT_DISCONNECTED: 128 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 129 "RDS/RDMA: DISCONNECT event - dropping connection " 130 "cm_id: %p", cm_id); 131 if (conn) { 132 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 133 "RDS/RDMA: DISCONNECT event - dropping connection " 134 "%u.%u.%u.%u->%u.%u.%u.%u", NIPQUAD(conn->c_laddr), 135 NIPQUAD(conn->c_faddr)); 136 rdsv3_conn_drop(conn); 137 } 138 break; 139 140 default: 141 /* things like device disconnect? */ 142 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 143 "unknown event %u!\n", event->event); 144 RDSV3_PANIC(); 145 break; 146 } 147 148 out: 149 if (conn) 150 mutex_exit(&conn->c_cm_lock); 151 152 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 153 "id %p event %u handling ret %d", cm_id, event->event, ret); 154 155 return (ret); 156 } 157 158 static int 159 rdsv3_rdma_listen_init(void) 160 { 161 struct sockaddr_in sin; 162 struct rdma_cm_id *cm_id; 163 int ret; 164 165 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", "Enter"); 166 167 cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, NULL, RDMA_PS_TCP); 168 if (IS_ERR(cm_id)) { 169 ret = PTR_ERR(cm_id); 170 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 171 "RDS/RDMA: failed to setup listener, " 172 "rdma_create_id() returned %d", ret); 173 goto out; 174 } 175 176 sin.sin_family = PF_INET; 177 sin.sin_addr.s_addr = (uint32_t)htonl(INADDR_ANY); 178 sin.sin_port = (uint16_t)htons(RDSV3_PORT); 179 180 /* 181 * XXX I bet this binds the cm_id to a device. If we want to support 182 * fail-over we'll have to take this into consideration. 183 */ 184 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); 185 if (ret) { 186 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 187 "RDS/RDMA: failed to setup listener, " 188 "rdma_bind_addr() returned %d", ret); 189 goto out; 190 } 191 192 ret = rdma_listen(cm_id, 128); 193 if (ret) { 194 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 195 "RDS/RDMA: failed to setup listener, " 196 "rdma_listen() returned %d", ret); 197 goto out; 198 } 199 200 RDSV3_DPRINTF5("rdsv3_rdma_listen_init", 201 "cm %p listening on port %u", cm_id, RDSV3_PORT); 202 203 rdsv3_rdma_listen_id = cm_id; 204 cm_id = NULL; 205 206 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 207 "Return: rdsv3_rdma_listen_id: %p", rdsv3_rdma_listen_id); 208 out: 209 if (cm_id) 210 rdma_destroy_id(cm_id); 211 return (ret); 212 } 213 214 static void rdsv3_rdma_listen_stop(void) 215 { 216 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "cm %p", rdsv3_rdma_listen_id); 217 rdma_destroy_id(rdsv3_rdma_listen_id); 218 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "Return"); 219 } 220 221 /* 222 * This function can be called via two routes. 223 * 1. During attach on a worker thread. 224 * 2. From rdsv3_create() for 1st socket. 225 */ 226 void 227 rdsv3_rdma_init() 228 { 229 int ret; 230 231 RDSV3_DPRINTF2("rdsv3_rdma_init", "Enter"); 232 233 mutex_enter(&rdsv3_rdma_listen_id_lock); 234 if (rdsv3_rdma_listen_id != NULL) { 235 RDSV3_DPRINTF5("rdsv3_rdma_init", 236 "rdsv3_rdma_listen_id is already initialized: %p", 237 rdsv3_rdma_listen_id); 238 mutex_exit(&rdsv3_rdma_listen_id_lock); 239 return; 240 } 241 242 ret = rdsv3_rdma_listen_init(); 243 if (ret) { 244 mutex_exit(&rdsv3_rdma_listen_id_lock); 245 return; 246 } 247 248 ret = rdsv3_ib_init(); 249 if (ret) { 250 rdsv3_rdma_listen_stop(); 251 } 252 mutex_exit(&rdsv3_rdma_listen_id_lock); 253 254 RDSV3_DPRINTF2("rdsv3_rdma_init", "Return"); 255 } 256 257 /*ARGSUSED*/ 258 void 259 rdsv3_rdma_exit(void *arg) 260 { 261 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Enter"); 262 263 /* stop listening first to ensure no new connections are attempted */ 264 if (rdsv3_rdma_listen_id) { 265 rdsv3_rdma_listen_stop(); 266 rdsv3_ib_exit(); 267 rdsv3_rdma_listen_id = NULL; 268 } 269 270 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Return"); 271 } 272