1 /*
2 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2009 HNR Consulting. All rights reserved.
4 * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 #if HAVE_CONFIG_H
37 # include <config.h>
38 #endif /* HAVE_CONFIG_H */
39
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43 #include <errno.h>
44
45 #include <infiniband/umad.h>
46 #include <infiniband/mad.h>
47
48 #include "mad_internal.h"
49
50 int ibdebug;
51
52 static struct ibmad_port mad_port;
53 struct ibmad_port *ibmp = &mad_port;
54
55 static int iberrs;
56
57 int madrpc_retries = MAD_DEF_RETRIES;
58 int madrpc_timeout = MAD_DEF_TIMEOUT_MS;
59
60 static void *save_mad;
61 static int save_mad_len = 256;
62
63 #undef DEBUG
64 #define DEBUG if (ibdebug) IBWARN
65 #define ERRS(fmt, ...) do { \
66 if (iberrs || ibdebug) \
67 IBWARN(fmt, ## __VA_ARGS__); \
68 } while (0)
69
70 #define MAD_TID(mad) (*((uint64_t *)((char *)(mad) + 8)))
71
madrpc_show_errors(int set)72 void madrpc_show_errors(int set)
73 {
74 iberrs = set;
75 }
76
madrpc_save_mad(void * madbuf,int len)77 void madrpc_save_mad(void *madbuf, int len)
78 {
79 save_mad = madbuf;
80 save_mad_len = len;
81 }
82
madrpc_set_retries(int retries)83 int madrpc_set_retries(int retries)
84 {
85 if (retries > 0)
86 madrpc_retries = retries;
87 return madrpc_retries;
88 }
89
madrpc_set_timeout(int timeout)90 int madrpc_set_timeout(int timeout)
91 {
92 madrpc_timeout = timeout;
93 return 0;
94 }
95
mad_rpc_set_retries(struct ibmad_port * port,int retries)96 void mad_rpc_set_retries(struct ibmad_port *port, int retries)
97 {
98 port->retries = retries;
99 }
100
mad_rpc_set_timeout(struct ibmad_port * port,int timeout)101 void mad_rpc_set_timeout(struct ibmad_port *port, int timeout)
102 {
103 port->timeout = timeout;
104 }
105
madrpc_def_timeout(void)106 int madrpc_def_timeout(void)
107 {
108 return madrpc_timeout;
109 }
110
madrpc_portid(void)111 int madrpc_portid(void)
112 {
113 return ibmp->port_id;
114 }
115
mad_rpc_portid(struct ibmad_port * srcport)116 int mad_rpc_portid(struct ibmad_port *srcport)
117 {
118 return srcport->port_id;
119 }
120
mad_rpc_class_agent(struct ibmad_port * port,int class)121 int mad_rpc_class_agent(struct ibmad_port *port, int class)
122 {
123 if (class < 1 || class >= MAX_CLASS)
124 return -1;
125 return port->class_agents[class];
126 }
127
128 static int
_do_madrpc(int port_id,void * sndbuf,void * rcvbuf,int agentid,int len,int timeout,int max_retries,int * p_error)129 _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
130 int timeout, int max_retries, int *p_error)
131 {
132 uint32_t trid; /* only low 32 bits - see mad_trid() */
133 int retries;
134 int length, status;
135
136 if (ibdebug > 1) {
137 IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
138 xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
139 }
140
141 if (save_mad) {
142 memcpy(save_mad, umad_get_mad(sndbuf),
143 save_mad_len < len ? save_mad_len : len);
144 save_mad = 0;
145 }
146
147 if (max_retries <= 0) {
148 errno = EINVAL;
149 *p_error = EINVAL;
150 ERRS("max_retries %d <= 0", max_retries);
151 return -1;
152 }
153
154 trid =
155 (uint32_t) mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
156
157 for (retries = 0; retries < max_retries; retries++) {
158 if (retries)
159 ERRS("retry %d (timeout %d ms)", retries, timeout);
160
161 length = len;
162 if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
163 IBWARN("send failed; %s", strerror(errno));
164 return -1;
165 }
166
167 /* Use same timeout on receive side just in case */
168 /* send packet is lost somewhere. */
169 do {
170 length = len;
171 if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
172 IBWARN("recv failed: %s", strerror(errno));
173 return -1;
174 }
175
176 if (ibdebug > 2)
177 umad_addr_dump(umad_get_mad_addr(rcvbuf));
178 if (ibdebug > 1) {
179 IBWARN("rcv buf:");
180 xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf),
181 IB_MAD_SIZE);
182 }
183 } while ((uint32_t)
184 mad_get_field64(umad_get_mad(rcvbuf), 0,
185 IB_MAD_TRID_F) != trid);
186
187 status = umad_status(rcvbuf);
188 if (!status)
189 return length; /* done */
190 if (status == ENOMEM)
191 return length;
192 }
193
194 errno = status;
195 *p_error = ETIMEDOUT;
196 ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
197 return -1;
198 }
199
redirect_port(ib_portid_t * port,uint8_t * mad)200 static int redirect_port(ib_portid_t * port, uint8_t * mad)
201 {
202 port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F);
203 if (!port->lid) {
204 IBWARN("GID-based redirection is not supported");
205 return -1;
206 }
207
208 port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F);
209 port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F);
210 port->sl = (uint8_t) mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F);
211
212 /* TODO: Reverse map redirection P_Key to P_Key index */
213
214 if (ibdebug)
215 IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x",
216 port->lid, port->qp, port->qkey, port->sl);
217
218 return 0;
219 }
220
mad_rpc(const struct ibmad_port * port,ib_rpc_t * rpc,ib_portid_t * dport,void * payload,void * rcvdata)221 void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc,
222 ib_portid_t * dport, void *payload, void *rcvdata)
223 {
224 int status, len;
225 uint8_t sndbuf[1024], rcvbuf[1024], *mad;
226 ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
227 int error = 0;
228
229 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
230 rpcv1->error = 0;
231 do {
232 len = 0;
233 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
234
235 if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
236 return NULL;
237
238 if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
239 port->class_agents[rpc->mgtclass & 0xff],
240 len, mad_get_timeout(port, rpc->timeout),
241 mad_get_retries(port), &error)) < 0) {
242 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) ==
243 IB_MAD_RPC_VERSION1)
244 rpcv1->error = error;
245 IBWARN("_do_madrpc failed; dport (%s)",
246 portid2str(dport));
247 return NULL;
248 }
249
250 mad = umad_get_mad(rcvbuf);
251 status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F);
252
253 /* check for exact match instead of only the redirect bit;
254 * that way, weird statuses cause an error, too */
255 if (status == IB_MAD_STS_REDIRECT) {
256 /* update dport for next request and retry */
257 /* bail if redirection fails */
258 if (redirect_port(dport, mad))
259 break;
260 } else
261 break;
262 } while (1);
263
264 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
265 rpcv1->error = error;
266 rpc->rstatus = status;
267
268 if (status != 0) {
269 ERRS("MAD completed with error status 0x%x; dport (%s)",
270 status, portid2str(dport));
271 errno = EIO;
272 return NULL;
273 }
274
275 if (ibdebug) {
276 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
277 xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
278 }
279
280 if (rcvdata)
281 memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
282
283 return rcvdata;
284 }
285
mad_rpc_rmpp(const struct ibmad_port * port,ib_rpc_t * rpc,ib_portid_t * dport,ib_rmpp_hdr_t * rmpp,void * data)286 void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc,
287 ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data)
288 {
289 int status, len;
290 uint8_t sndbuf[1024], rcvbuf[1024], *mad;
291 ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
292 int error = 0;
293
294 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
295
296 DEBUG("rmpp %p data %p", rmpp, data);
297
298 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
299 rpcv1->error = 0;
300 if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
301 return NULL;
302
303 if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
304 port->class_agents[rpc->mgtclass & 0xff],
305 len, mad_get_timeout(port, rpc->timeout),
306 mad_get_retries(port), &error)) < 0) {
307 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
308 rpcv1->error = error;
309 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
310 return NULL;
311 }
312
313 if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
314 rpcv1->error = error;
315
316 mad = umad_get_mad(rcvbuf);
317
318 if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
319 ERRS("MAD completed with error status 0x%x; dport (%s)",
320 status, portid2str(dport));
321 errno = EIO;
322 return NULL;
323 }
324
325 if (ibdebug) {
326 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
327 xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
328 rpc->datasz);
329 }
330
331 if (rmpp) {
332 rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
333 if ((rmpp->flags & 0x3) &&
334 mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
335 IBWARN("bad rmpp version");
336 return NULL;
337 }
338 rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
339 rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
340 DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
341 rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
342 rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
343 }
344
345 if (data)
346 memcpy(data, mad + rpc->dataoffs, rpc->datasz);
347
348 rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
349
350 return data;
351 }
352
madrpc(ib_rpc_t * rpc,ib_portid_t * dport,void * payload,void * rcvdata)353 void *madrpc(ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata)
354 {
355 return mad_rpc(ibmp, rpc, dport, payload, rcvdata);
356 }
357
madrpc_rmpp(ib_rpc_t * rpc,ib_portid_t * dport,ib_rmpp_hdr_t * rmpp,void * data)358 void *madrpc_rmpp(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp,
359 void *data)
360 {
361 return mad_rpc_rmpp(ibmp, rpc, dport, rmpp, data);
362 }
363
364 void
madrpc_init(char * dev_name,int dev_port,int * mgmt_classes,int num_classes)365 madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
366 {
367 int fd;
368
369 if (umad_init() < 0)
370 IBPANIC("can't init UMAD library");
371
372 if ((fd = umad_open_port(dev_name, dev_port)) < 0)
373 IBPANIC("can't open UMAD port (%s:%d)",
374 dev_name ? dev_name : "(nil)", dev_port);
375
376 if (num_classes >= MAX_CLASS)
377 IBPANIC("too many classes %d requested", num_classes);
378
379 ibmp->port_id = fd;
380 memset(ibmp->class_agents, 0xff, sizeof ibmp->class_agents);
381 while (num_classes--) {
382 uint8_t rmpp_version = 0;
383 int mgmt = *mgmt_classes++;
384
385 if (mgmt == IB_SA_CLASS)
386 rmpp_version = 1;
387 if (mad_register_client_via(mgmt, rmpp_version, ibmp) < 0)
388 IBPANIC("client_register for mgmt class %d failed",
389 mgmt);
390 }
391 }
392
mad_rpc_open_port(char * dev_name,int dev_port,int * mgmt_classes,int num_classes)393 struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port,
394 int *mgmt_classes, int num_classes)
395 {
396 struct ibmad_port *p;
397 int port_id;
398
399 if (num_classes >= MAX_CLASS) {
400 IBWARN("too many classes %d requested", num_classes);
401 errno = EINVAL;
402 return NULL;
403 }
404
405 if (umad_init() < 0) {
406 IBWARN("can't init UMAD library");
407 errno = ENODEV;
408 return NULL;
409 }
410
411 p = malloc(sizeof(*p));
412 if (!p) {
413 errno = ENOMEM;
414 return NULL;
415 }
416 memset(p, 0, sizeof(*p));
417
418 if ((port_id = umad_open_port(dev_name, dev_port)) < 0) {
419 IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
420 if (!errno)
421 errno = EIO;
422 free(p);
423 return NULL;
424 }
425
426 p->port_id = port_id;
427 if (dev_name)
428 strncpy(p->ca_name, dev_name, sizeof p->ca_name - 1);
429 p->portnum = dev_port;
430
431 memset(p->class_agents, 0xff, sizeof p->class_agents);
432 while (num_classes--) {
433 uint8_t rmpp_version = 0;
434 int mgmt = *mgmt_classes++;
435
436 if (mgmt == IB_SA_CLASS)
437 rmpp_version = 1;
438 if (mgmt < 0 || mgmt >= MAX_CLASS ||
439 mad_register_client_via(mgmt, rmpp_version, p) < 0) {
440 IBWARN("client_register for mgmt %d failed", mgmt);
441 if (!errno)
442 errno = EINVAL;
443 umad_close_port(port_id);
444 free(p);
445 return NULL;
446 }
447 }
448
449 return p;
450 }
451
mad_rpc_close_port(struct ibmad_port * port)452 void mad_rpc_close_port(struct ibmad_port *port)
453 {
454 umad_close_port(port->port_id);
455 free(port);
456 }
457