1 /* 2 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2009 HNR Consulting. All rights reserved. 4 * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 #if HAVE_CONFIG_H 37 # include <config.h> 38 #endif /* HAVE_CONFIG_H */ 39 40 #include <stdio.h> 41 #include <stdlib.h> 42 #include <string.h> 43 #include <errno.h> 44 45 #include <infiniband/umad.h> 46 #include <infiniband/mad.h> 47 48 #include "mad_internal.h" 49 50 int ibdebug; 51 52 static struct ibmad_port mad_port; 53 struct ibmad_port *ibmp = &mad_port; 54 55 static int iberrs; 56 57 int madrpc_retries = MAD_DEF_RETRIES; 58 int madrpc_timeout = MAD_DEF_TIMEOUT_MS; 59 60 static void *save_mad; 61 static int save_mad_len = 256; 62 63 #undef DEBUG 64 #define DEBUG if (ibdebug) IBWARN 65 #define ERRS(fmt, ...) do { \ 66 if (iberrs || ibdebug) \ 67 IBWARN(fmt, ## __VA_ARGS__); \ 68 } while (0) 69 70 #define MAD_TID(mad) (*((uint64_t *)((char *)(mad) + 8))) 71 72 void madrpc_show_errors(int set) 73 { 74 iberrs = set; 75 } 76 77 void madrpc_save_mad(void *madbuf, int len) 78 { 79 save_mad = madbuf; 80 save_mad_len = len; 81 } 82 83 int madrpc_set_retries(int retries) 84 { 85 if (retries > 0) 86 madrpc_retries = retries; 87 return madrpc_retries; 88 } 89 90 int madrpc_set_timeout(int timeout) 91 { 92 madrpc_timeout = timeout; 93 return 0; 94 } 95 96 void mad_rpc_set_retries(struct ibmad_port *port, int retries) 97 { 98 port->retries = retries; 99 } 100 101 void mad_rpc_set_timeout(struct ibmad_port *port, int timeout) 102 { 103 port->timeout = timeout; 104 } 105 106 int madrpc_def_timeout(void) 107 { 108 return madrpc_timeout; 109 } 110 111 int madrpc_portid(void) 112 { 113 return ibmp->port_id; 114 } 115 116 int mad_rpc_portid(struct ibmad_port *srcport) 117 { 118 return srcport->port_id; 119 } 120 121 int mad_rpc_class_agent(struct ibmad_port *port, int class) 122 { 123 if (class < 1 || class >= MAX_CLASS) 124 return -1; 125 return port->class_agents[class]; 126 } 127 128 static int 129 _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, 130 int timeout, int max_retries, int *p_error) 131 { 132 uint32_t trid; /* only low 32 bits - see mad_trid() */ 133 int retries; 134 int length, status; 135 136 if (ibdebug > 1) { 137 IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len); 138 xdump(stderr, "send buf\n", sndbuf, umad_size() + len); 139 } 140 141 if (save_mad) { 142 memcpy(save_mad, umad_get_mad(sndbuf), 143 save_mad_len < len ? save_mad_len : len); 144 save_mad = 0; 145 } 146 147 if (max_retries <= 0) { 148 errno = EINVAL; 149 *p_error = EINVAL; 150 ERRS("max_retries %d <= 0", max_retries); 151 return -1; 152 } 153 154 trid = 155 (uint32_t) mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F); 156 157 for (retries = 0; retries < max_retries; retries++) { 158 if (retries) 159 ERRS("retry %d (timeout %d ms)", retries, timeout); 160 161 length = len; 162 if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) { 163 IBWARN("send failed; %s", strerror(errno)); 164 return -1; 165 } 166 167 /* Use same timeout on receive side just in case */ 168 /* send packet is lost somewhere. */ 169 do { 170 length = len; 171 if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) { 172 IBWARN("recv failed: %s", strerror(errno)); 173 return -1; 174 } 175 176 if (ibdebug > 2) 177 umad_addr_dump(umad_get_mad_addr(rcvbuf)); 178 if (ibdebug > 1) { 179 IBWARN("rcv buf:"); 180 xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), 181 IB_MAD_SIZE); 182 } 183 } while ((uint32_t) 184 mad_get_field64(umad_get_mad(rcvbuf), 0, 185 IB_MAD_TRID_F) != trid); 186 187 status = umad_status(rcvbuf); 188 if (!status) 189 return length; /* done */ 190 if (status == ENOMEM) 191 return length; 192 } 193 194 errno = status; 195 *p_error = ETIMEDOUT; 196 ERRS("timeout after %d retries, %d ms", retries, timeout * retries); 197 return -1; 198 } 199 200 static int redirect_port(ib_portid_t * port, uint8_t * mad) 201 { 202 port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); 203 if (!port->lid) { 204 IBWARN("GID-based redirection is not supported"); 205 return -1; 206 } 207 208 port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); 209 port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); 210 port->sl = (uint8_t) mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); 211 212 /* TODO: Reverse map redirection P_Key to P_Key index */ 213 214 if (ibdebug) 215 IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x", 216 port->lid, port->qp, port->qkey, port->sl); 217 218 return 0; 219 } 220 221 void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, 222 ib_portid_t * dport, void *payload, void *rcvdata) 223 { 224 int status, len; 225 uint8_t sndbuf[1024], rcvbuf[1024], *mad; 226 ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc; 227 int error = 0; 228 229 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1) 230 rpcv1->error = 0; 231 do { 232 len = 0; 233 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); 234 235 if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) 236 return NULL; 237 238 if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, 239 port->class_agents[rpc->mgtclass & 0xff], 240 len, mad_get_timeout(port, rpc->timeout), 241 mad_get_retries(port), &error)) < 0) { 242 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == 243 IB_MAD_RPC_VERSION1) 244 rpcv1->error = error; 245 IBWARN("_do_madrpc failed; dport (%s)", 246 portid2str(dport)); 247 return NULL; 248 } 249 250 mad = umad_get_mad(rcvbuf); 251 status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F); 252 253 /* check for exact match instead of only the redirect bit; 254 * that way, weird statuses cause an error, too */ 255 if (status == IB_MAD_STS_REDIRECT) { 256 /* update dport for next request and retry */ 257 /* bail if redirection fails */ 258 if (redirect_port(dport, mad)) 259 break; 260 } else 261 break; 262 } while (1); 263 264 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1) 265 rpcv1->error = error; 266 rpc->rstatus = status; 267 268 if (status != 0) { 269 ERRS("MAD completed with error status 0x%x; dport (%s)", 270 status, portid2str(dport)); 271 errno = EIO; 272 return NULL; 273 } 274 275 if (ibdebug) { 276 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz); 277 xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz); 278 } 279 280 if (rcvdata) 281 memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz); 282 283 return rcvdata; 284 } 285 286 void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc, 287 ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data) 288 { 289 int status, len; 290 uint8_t sndbuf[1024], rcvbuf[1024], *mad; 291 ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc; 292 int error = 0; 293 294 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); 295 296 DEBUG("rmpp %p data %p", rmpp, data); 297 298 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1) 299 rpcv1->error = 0; 300 if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0) 301 return NULL; 302 303 if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, 304 port->class_agents[rpc->mgtclass & 0xff], 305 len, mad_get_timeout(port, rpc->timeout), 306 mad_get_retries(port), &error)) < 0) { 307 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1) 308 rpcv1->error = error; 309 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); 310 return NULL; 311 } 312 313 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1) 314 rpcv1->error = error; 315 316 mad = umad_get_mad(rcvbuf); 317 318 if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) { 319 ERRS("MAD completed with error status 0x%x; dport (%s)", 320 status, portid2str(dport)); 321 errno = EIO; 322 return NULL; 323 } 324 325 if (ibdebug) { 326 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz); 327 xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs, 328 rpc->datasz); 329 } 330 331 if (rmpp) { 332 rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F); 333 if ((rmpp->flags & 0x3) && 334 mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) { 335 IBWARN("bad rmpp version"); 336 return NULL; 337 } 338 rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F); 339 rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F); 340 DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status); 341 rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F); 342 rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F); 343 } 344 345 if (data) 346 memcpy(data, mad + rpc->dataoffs, rpc->datasz); 347 348 rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F); 349 350 return data; 351 } 352 353 void *madrpc(ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata) 354 { 355 return mad_rpc(ibmp, rpc, dport, payload, rcvdata); 356 } 357 358 void *madrpc_rmpp(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, 359 void *data) 360 { 361 return mad_rpc_rmpp(ibmp, rpc, dport, rmpp, data); 362 } 363 364 void 365 madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes) 366 { 367 int fd; 368 369 if (umad_init() < 0) 370 IBPANIC("can't init UMAD library"); 371 372 if ((fd = umad_open_port(dev_name, dev_port)) < 0) 373 IBPANIC("can't open UMAD port (%s:%d)", 374 dev_name ? dev_name : "(nil)", dev_port); 375 376 if (num_classes >= MAX_CLASS) 377 IBPANIC("too many classes %d requested", num_classes); 378 379 ibmp->port_id = fd; 380 memset(ibmp->class_agents, 0xff, sizeof ibmp->class_agents); 381 while (num_classes--) { 382 uint8_t rmpp_version = 0; 383 int mgmt = *mgmt_classes++; 384 385 if (mgmt == IB_SA_CLASS) 386 rmpp_version = 1; 387 if (mad_register_client_via(mgmt, rmpp_version, ibmp) < 0) 388 IBPANIC("client_register for mgmt class %d failed", 389 mgmt); 390 } 391 } 392 393 struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, 394 int *mgmt_classes, int num_classes) 395 { 396 struct ibmad_port *p; 397 int port_id; 398 399 if (num_classes >= MAX_CLASS) { 400 IBWARN("too many classes %d requested", num_classes); 401 errno = EINVAL; 402 return NULL; 403 } 404 405 if (umad_init() < 0) { 406 IBWARN("can't init UMAD library"); 407 errno = ENODEV; 408 return NULL; 409 } 410 411 p = malloc(sizeof(*p)); 412 if (!p) { 413 errno = ENOMEM; 414 return NULL; 415 } 416 memset(p, 0, sizeof(*p)); 417 418 if ((port_id = umad_open_port(dev_name, dev_port)) < 0) { 419 IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port); 420 if (!errno) 421 errno = EIO; 422 free(p); 423 return NULL; 424 } 425 426 p->port_id = port_id; 427 if (dev_name) 428 strncpy(p->ca_name, dev_name, sizeof p->ca_name - 1); 429 p->portnum = dev_port; 430 431 memset(p->class_agents, 0xff, sizeof p->class_agents); 432 while (num_classes--) { 433 uint8_t rmpp_version = 0; 434 int mgmt = *mgmt_classes++; 435 436 if (mgmt == IB_SA_CLASS) 437 rmpp_version = 1; 438 if (mgmt < 0 || mgmt >= MAX_CLASS || 439 mad_register_client_via(mgmt, rmpp_version, p) < 0) { 440 IBWARN("client_register for mgmt %d failed", mgmt); 441 if (!errno) 442 errno = EINVAL; 443 umad_close_port(port_id); 444 free(p); 445 return NULL; 446 } 447 } 448 449 return p; 450 } 451 452 void mad_rpc_close_port(struct ibmad_port *port) 453 { 454 umad_close_port(port->port_id); 455 free(port); 456 } 457