1 /* 2 * Copyright (c) 2010 Lawrence Livermore National Laboratory 3 * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 */ 34 35 #if HAVE_CONFIG_H 36 # include <config.h> 37 #endif /* HAVE_CONFIG_H */ 38 39 #include <errno.h> 40 #include <infiniband/ibnetdisc.h> 41 #include <infiniband/umad.h> 42 #include "internal.h" 43 44 extern int mlnx_ext_port_info_err(smp_engine_t * engine, ibnd_smp_t * smp, 45 uint8_t * mad, void *cb_data); 46 47 static void queue_smp(smp_engine_t * engine, ibnd_smp_t * smp) 48 { 49 smp->qnext = NULL; 50 if (!engine->smp_queue_head) { 51 engine->smp_queue_head = smp; 52 engine->smp_queue_tail = smp; 53 } else { 54 engine->smp_queue_tail->qnext = smp; 55 engine->smp_queue_tail = smp; 56 } 57 } 58 59 static ibnd_smp_t *get_smp(smp_engine_t * engine) 60 { 61 ibnd_smp_t *head = engine->smp_queue_head; 62 ibnd_smp_t *tail = engine->smp_queue_tail; 63 ibnd_smp_t *rc = head; 64 if (head) { 65 if (tail == head) 66 engine->smp_queue_tail = NULL; 67 engine->smp_queue_head = head->qnext; 68 } 69 return rc; 70 } 71 72 static int send_smp(ibnd_smp_t * smp, smp_engine_t * engine) 73 { 74 int rc = 0; 75 uint8_t umad[1024]; 76 ib_rpc_t *rpc = &smp->rpc; 77 int agent = 0; 78 79 memset(umad, 0, umad_size() + IB_MAD_SIZE); 80 81 if (rpc->mgtclass == IB_SMI_CLASS) { 82 agent = engine->smi_agent; 83 } else if (rpc->mgtclass == IB_SMI_DIRECT_CLASS) { 84 agent = engine->smi_dir_agent; 85 } else { 86 IBND_ERROR("Invalid class for RPC\n"); 87 return (-EIO); 88 } 89 90 if ((rc = mad_build_pkt(umad, &smp->rpc, &smp->path, NULL, NULL)) 91 < 0) { 92 IBND_ERROR("mad_build_pkt failed; %d\n", rc); 93 return rc; 94 } 95 96 if ((rc = umad_send(engine->umad_fd, agent, umad, IB_MAD_SIZE, 97 engine->cfg->timeout_ms, engine->cfg->retries)) < 0) { 98 IBND_ERROR("send failed; %d\n", rc); 99 return rc; 100 } 101 102 return 0; 103 } 104 105 static int process_smp_queue(smp_engine_t * engine) 106 { 107 int rc = 0; 108 ibnd_smp_t *smp; 109 while (cl_qmap_count(&engine->smps_on_wire) 110 < engine->cfg->max_smps) { 111 smp = get_smp(engine); 112 if (!smp) 113 return 0; 114 115 if ((rc = send_smp(smp, engine)) != 0) { 116 free(smp); 117 return rc; 118 } 119 cl_qmap_insert(&engine->smps_on_wire, (uint32_t) smp->rpc.trid, 120 (cl_map_item_t *) smp); 121 engine->total_smps++; 122 } 123 return 0; 124 } 125 126 int issue_smp(smp_engine_t * engine, ib_portid_t * portid, 127 unsigned attrid, unsigned mod, smp_comp_cb_t cb, void *cb_data) 128 { 129 ibnd_smp_t *smp = calloc(1, sizeof *smp); 130 if (!smp) { 131 IBND_ERROR("OOM\n"); 132 return -ENOMEM; 133 } 134 135 smp->cb = cb; 136 smp->cb_data = cb_data; 137 smp->path = *portid; 138 smp->rpc.method = IB_MAD_METHOD_GET; 139 smp->rpc.attr.id = attrid; 140 smp->rpc.attr.mod = mod; 141 smp->rpc.timeout = engine->cfg->timeout_ms; 142 smp->rpc.datasz = IB_SMP_DATA_SIZE; 143 smp->rpc.dataoffs = IB_SMP_DATA_OFFS; 144 smp->rpc.trid = mad_trid(); 145 smp->rpc.mkey = engine->cfg->mkey; 146 147 if (portid->lid <= 0 || portid->drpath.drslid == 0xffff || 148 portid->drpath.drdlid == 0xffff) 149 smp->rpc.mgtclass = IB_SMI_DIRECT_CLASS; /* direct SMI */ 150 else 151 smp->rpc.mgtclass = IB_SMI_CLASS; /* Lid routed SMI */ 152 153 portid->sl = 0; 154 portid->qp = 0; 155 156 queue_smp(engine, smp); 157 return process_smp_queue(engine); 158 } 159 160 static int process_one_recv(smp_engine_t * engine) 161 { 162 int rc = 0; 163 int status = 0; 164 ibnd_smp_t *smp; 165 uint8_t *mad; 166 uint32_t trid; 167 uint8_t umad[sizeof(struct ib_user_mad) + IB_MAD_SIZE]; 168 int length = umad_size() + IB_MAD_SIZE; 169 170 memset(umad, 0, sizeof(umad)); 171 172 /* wait for the next message */ 173 if ((rc = umad_recv(engine->umad_fd, umad, &length, 174 -1)) < 0) { 175 IBND_ERROR("umad_recv failed: %d\n", rc); 176 return -1; 177 } 178 179 mad = umad_get_mad(umad); 180 trid = (uint32_t) mad_get_field64(mad, 0, IB_MAD_TRID_F); 181 182 smp = (ibnd_smp_t *) cl_qmap_remove(&engine->smps_on_wire, trid); 183 if ((cl_map_item_t *) smp == cl_qmap_end(&engine->smps_on_wire)) { 184 IBND_ERROR("Failed to find matching smp for trid (%x)\n", trid); 185 return -1; 186 } 187 188 rc = process_smp_queue(engine); 189 if (rc) 190 goto error; 191 192 if ((status = umad_status(umad))) { 193 IBND_ERROR("umad (%s Attr 0x%x:%u) bad status %d; %s\n", 194 portid2str(&smp->path), smp->rpc.attr.id, 195 smp->rpc.attr.mod, status, strerror(status)); 196 if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO) 197 rc = mlnx_ext_port_info_err(engine, smp, mad, 198 smp->cb_data); 199 } else if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F))) { 200 IBND_ERROR("mad (%s Attr 0x%x:%u) bad status 0x%x\n", 201 portid2str(&smp->path), smp->rpc.attr.id, 202 smp->rpc.attr.mod, status); 203 if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO) 204 rc = mlnx_ext_port_info_err(engine, smp, mad, 205 smp->cb_data); 206 } else 207 rc = smp->cb(engine, smp, mad, smp->cb_data); 208 209 error: 210 free(smp); 211 return rc; 212 } 213 214 int smp_engine_init(smp_engine_t * engine, char * ca_name, int ca_port, 215 void *user_data, ibnd_config_t *cfg) 216 { 217 memset(engine, 0, sizeof(*engine)); 218 219 if (umad_init() < 0) { 220 IBND_ERROR("umad_init failed\n"); 221 return -EIO; 222 } 223 224 engine->umad_fd = umad_open_port(ca_name, ca_port); 225 if (engine->umad_fd < 0) { 226 IBND_ERROR("can't open UMAD port (%s:%d)\n", ca_name, ca_port); 227 return -EIO; 228 } 229 230 if ((engine->smi_agent = umad_register(engine->umad_fd, 231 IB_SMI_CLASS, 1, 0, 0)) < 0) { 232 IBND_ERROR("Failed to register SMI agent on (%s:%d)\n", 233 ca_name, ca_port); 234 goto eio_close; 235 } 236 237 if ((engine->smi_dir_agent = umad_register(engine->umad_fd, 238 IB_SMI_DIRECT_CLASS, 1, 0, 0)) < 0) { 239 IBND_ERROR("Failed to register SMI_DIRECT agent on (%s:%d)\n", 240 ca_name, ca_port); 241 goto eio_close; 242 } 243 244 engine->user_data = user_data; 245 cl_qmap_init(&engine->smps_on_wire); 246 engine->cfg = cfg; 247 return (0); 248 249 eio_close: 250 umad_close_port(engine->umad_fd); 251 return (-EIO); 252 } 253 254 void smp_engine_destroy(smp_engine_t * engine) 255 { 256 cl_map_item_t *item; 257 ibnd_smp_t *smp; 258 259 /* remove queued smps */ 260 smp = get_smp(engine); 261 if (smp) 262 IBND_ERROR("outstanding SMP's\n"); 263 for ( /* */ ; smp; smp = get_smp(engine)) 264 free(smp); 265 266 /* remove smps from the wire queue */ 267 item = cl_qmap_head(&engine->smps_on_wire); 268 if (item != cl_qmap_end(&engine->smps_on_wire)) 269 IBND_ERROR("outstanding SMP's on wire\n"); 270 for ( /* */ ; item != cl_qmap_end(&engine->smps_on_wire); 271 item = cl_qmap_head(&engine->smps_on_wire)) { 272 cl_qmap_remove_item(&engine->smps_on_wire, item); 273 free(item); 274 } 275 276 umad_close_port(engine->umad_fd); 277 } 278 279 int process_mads(smp_engine_t * engine) 280 { 281 int rc; 282 while (!cl_is_qmap_empty(&engine->smps_on_wire)) 283 if ((rc = process_one_recv(engine)) != 0) 284 return rc; 285 return 0; 286 } 287