1 /* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #define _GNU_SOURCE 34 #include <config.h> 35 36 #include <infiniband/endian.h> 37 #include <stdio.h> 38 #include <sys/types.h> 39 #include <sys/stat.h> 40 #include <fcntl.h> 41 #include <unistd.h> 42 #include <stdlib.h> 43 #include <alloca.h> 44 #include <errno.h> 45 46 #include "ibverbs.h" 47 48 /* Hack to avoid GCC's -Wmissing-prototypes and the similar error from sparse 49 with these prototypes. Symbol versionining requires the goofy names, the 50 prototype must match the version in verbs.h. 51 */ 52 struct ibv_device **__ibv_get_device_list(int *num_devices); 53 void __ibv_free_device_list(struct ibv_device **list); 54 const char *__ibv_get_device_name(struct ibv_device *device); 55 __be64 __ibv_get_device_guid(struct ibv_device *device); 56 struct ibv_context *__ibv_open_device(struct ibv_device *device); 57 int __ibv_close_device(struct ibv_context *context); 58 int __ibv_get_async_event(struct ibv_context *context, 59 struct ibv_async_event *event); 60 void __ibv_ack_async_event(struct ibv_async_event *event); 61 62 static pthread_once_t device_list_once = PTHREAD_ONCE_INIT; 63 static int num_devices; 64 static struct ibv_device **device_list; 65 66 static void count_devices(void) 67 { 68 num_devices = ibverbs_init(&device_list); 69 } 70 71 struct ibv_device **__ibv_get_device_list(int *num) 72 { 73 struct ibv_device **l; 74 int i; 75 76 if (num) 77 *num = 0; 78 79 pthread_once(&device_list_once, count_devices); 80 81 if (num_devices < 0) { 82 errno = -num_devices; 83 return NULL; 84 } 85 86 l = calloc(num_devices + 1, sizeof (struct ibv_device *)); 87 if (!l) { 88 errno = ENOMEM; 89 return NULL; 90 } 91 92 for (i = 0; i < num_devices; ++i) 93 l[i] = device_list[i]; 94 if (num) 95 *num = num_devices; 96 97 return l; 98 } 99 default_symver(__ibv_get_device_list, ibv_get_device_list); 100 101 void __ibv_free_device_list(struct ibv_device **list) 102 { 103 free(list); 104 } 105 default_symver(__ibv_free_device_list, ibv_free_device_list); 106 107 const char *__ibv_get_device_name(struct ibv_device *device) 108 { 109 return device->name; 110 } 111 default_symver(__ibv_get_device_name, ibv_get_device_name); 112 113 __be64 __ibv_get_device_guid(struct ibv_device *device) 114 { 115 char attr[24]; 116 uint64_t guid = 0; 117 uint16_t parts[4]; 118 int i; 119 120 if (ibv_read_sysfs_file(device->ibdev_path, "node_guid", 121 attr, sizeof attr) < 0) 122 return 0; 123 124 if (sscanf(attr, "%hx:%hx:%hx:%hx", 125 parts, parts + 1, parts + 2, parts + 3) != 4) 126 return 0; 127 128 for (i = 0; i < 4; ++i) 129 guid = (guid << 16) | parts[i]; 130 131 return htobe64(guid); 132 } 133 default_symver(__ibv_get_device_guid, ibv_get_device_guid); 134 135 int verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, 136 struct ibv_comp_channel *channel, 137 void *cq_context) 138 { 139 int err = 0; 140 141 cq->context = context; 142 cq->channel = channel; 143 144 err = pthread_mutex_init(&cq->mutex, NULL); 145 if (err) 146 return err; 147 err = pthread_cond_init(&cq->cond, NULL); 148 if (err) 149 goto err; 150 151 if (cq->channel) { 152 pthread_mutex_lock(&context->mutex); 153 ++cq->channel->refcnt; 154 pthread_mutex_unlock(&context->mutex); 155 } 156 157 cq->cq_context = cq_context; 158 cq->comp_events_completed = 0; 159 cq->async_events_completed = 0; 160 161 return err; 162 163 err: 164 pthread_mutex_destroy(&cq->mutex); 165 166 return err; 167 } 168 169 void verbs_cleanup_cq(struct ibv_cq *cq) 170 { 171 pthread_cond_destroy(&cq->cond); 172 pthread_mutex_destroy(&cq->mutex); 173 } 174 175 static struct ibv_cq_ex * 176 __lib_ibv_create_cq_ex(struct ibv_context *context, 177 struct ibv_cq_init_attr_ex *cq_attr) 178 { 179 struct verbs_context *vctx = verbs_get_ctx(context); 180 struct ibv_cq_ex *cq; 181 int err = 0; 182 183 if (cq_attr->wc_flags & ~IBV_CREATE_CQ_SUP_WC_FLAGS) { 184 errno = EOPNOTSUPP; 185 return NULL; 186 } 187 188 cq = vctx->priv->create_cq_ex(context, cq_attr); 189 if (!cq) 190 return NULL; 191 192 err = verbs_init_cq(ibv_cq_ex_to_cq(cq), context, 193 cq_attr->channel, cq_attr->cq_context); 194 if (err) 195 goto err; 196 197 return cq; 198 199 err: 200 context->ops.destroy_cq(ibv_cq_ex_to_cq(cq)); 201 202 return NULL; 203 } 204 205 struct ibv_context *__ibv_open_device(struct ibv_device *device) 206 { 207 struct verbs_device *verbs_device = verbs_get_device(device); 208 char *devpath; 209 int cmd_fd, ret; 210 struct ibv_context *context; 211 struct verbs_context *context_ex; 212 213 if (asprintf(&devpath, "/dev/%s", device->dev_name) < 0) 214 return NULL; 215 216 /* 217 * We'll only be doing writes, but we need O_RDWR in case the 218 * provider needs to mmap() the file. 219 */ 220 cmd_fd = open(devpath, O_RDWR | O_CLOEXEC); 221 free(devpath); 222 223 if (cmd_fd < 0) 224 return NULL; 225 226 if (!verbs_device->ops->init_context) { 227 context = verbs_device->ops->alloc_context(device, cmd_fd); 228 if (!context) 229 goto err; 230 231 if (pthread_mutex_init(&context->mutex, NULL)) { 232 verbs_device->ops->free_context(context); 233 goto err; 234 } 235 } else { 236 struct verbs_ex_private *priv; 237 238 /* Library now allocates the context */ 239 context_ex = calloc(1, sizeof(*context_ex) + 240 verbs_device->size_of_context); 241 if (!context_ex) { 242 errno = ENOMEM; 243 goto err; 244 } 245 246 priv = calloc(1, sizeof(*priv)); 247 if (!priv) { 248 errno = ENOMEM; 249 goto err_context; 250 } 251 252 context_ex->priv = priv; 253 context_ex->context.abi_compat = __VERBS_ABI_IS_EXTENDED; 254 context_ex->sz = sizeof(*context_ex); 255 256 context = &context_ex->context; 257 if (pthread_mutex_init(&context->mutex, NULL)) 258 goto verbs_err; 259 260 ret = verbs_device->ops->init_context(verbs_device, context, cmd_fd); 261 if (ret) 262 goto err_mutex; 263 /* 264 * In order to maintain backward/forward binary compatibility 265 * with apps compiled against libibverbs-1.1.8 that use the 266 * flow steering addition, we need to set the two 267 * ABI_placeholder entries to match the driver set flow 268 * entries. This is because apps compiled against 269 * libibverbs-1.1.8 use an inline ibv_create_flow and 270 * ibv_destroy_flow function that looks in the placeholder 271 * spots for the proper entry points. For apps compiled 272 * against libibverbs-1.1.9 and later, the inline functions 273 * will be looking in the right place. 274 */ 275 context_ex->ABI_placeholder1 = (void (*)(void)) context_ex->ibv_create_flow; 276 context_ex->ABI_placeholder2 = (void (*)(void)) context_ex->ibv_destroy_flow; 277 278 if (context_ex->create_cq_ex) { 279 priv->create_cq_ex = context_ex->create_cq_ex; 280 context_ex->create_cq_ex = __lib_ibv_create_cq_ex; 281 } 282 } 283 284 context->device = device; 285 context->cmd_fd = cmd_fd; 286 287 return context; 288 289 err_mutex: 290 pthread_mutex_destroy(&context->mutex); 291 verbs_err: 292 free(context_ex->priv); 293 err_context: 294 free(context_ex); 295 err: 296 close(cmd_fd); 297 return NULL; 298 } 299 default_symver(__ibv_open_device, ibv_open_device); 300 301 int __ibv_close_device(struct ibv_context *context) 302 { 303 int async_fd = context->async_fd; 304 int cmd_fd = context->cmd_fd; 305 struct verbs_context *context_ex; 306 struct verbs_device *verbs_device = verbs_get_device(context->device); 307 308 pthread_mutex_destroy(&context->mutex); 309 context_ex = verbs_get_ctx(context); 310 if (context_ex) { 311 verbs_device->ops->uninit_context(verbs_device, context); 312 free(context_ex->priv); 313 free(context_ex); 314 } else { 315 verbs_device->ops->free_context(context); 316 } 317 318 close(async_fd); 319 close(cmd_fd); 320 321 return 0; 322 } 323 default_symver(__ibv_close_device, ibv_close_device); 324 325 int __ibv_get_async_event(struct ibv_context *context, 326 struct ibv_async_event *event) 327 { 328 struct ibv_kern_async_event ev; 329 330 if (read(context->async_fd, &ev, sizeof ev) != sizeof ev) 331 return -1; 332 333 event->event_type = ev.event_type; 334 335 switch (event->event_type) { 336 case IBV_EVENT_CQ_ERR: 337 event->element.cq = (void *) (uintptr_t) ev.element; 338 break; 339 340 case IBV_EVENT_QP_FATAL: 341 case IBV_EVENT_QP_REQ_ERR: 342 case IBV_EVENT_QP_ACCESS_ERR: 343 case IBV_EVENT_COMM_EST: 344 case IBV_EVENT_SQ_DRAINED: 345 case IBV_EVENT_PATH_MIG: 346 case IBV_EVENT_PATH_MIG_ERR: 347 case IBV_EVENT_QP_LAST_WQE_REACHED: 348 event->element.qp = (void *) (uintptr_t) ev.element; 349 break; 350 351 case IBV_EVENT_SRQ_ERR: 352 case IBV_EVENT_SRQ_LIMIT_REACHED: 353 event->element.srq = (void *) (uintptr_t) ev.element; 354 break; 355 356 case IBV_EVENT_WQ_FATAL: 357 event->element.wq = (void *) (uintptr_t) ev.element; 358 break; 359 default: 360 event->element.port_num = ev.element; 361 break; 362 } 363 364 if (context->ops.async_event) 365 context->ops.async_event(event); 366 367 return 0; 368 } 369 default_symver(__ibv_get_async_event, ibv_get_async_event); 370 371 void __ibv_ack_async_event(struct ibv_async_event *event) 372 { 373 switch (event->event_type) { 374 case IBV_EVENT_CQ_ERR: 375 { 376 struct ibv_cq *cq = event->element.cq; 377 378 pthread_mutex_lock(&cq->mutex); 379 ++cq->async_events_completed; 380 pthread_cond_signal(&cq->cond); 381 pthread_mutex_unlock(&cq->mutex); 382 383 return; 384 } 385 386 case IBV_EVENT_QP_FATAL: 387 case IBV_EVENT_QP_REQ_ERR: 388 case IBV_EVENT_QP_ACCESS_ERR: 389 case IBV_EVENT_COMM_EST: 390 case IBV_EVENT_SQ_DRAINED: 391 case IBV_EVENT_PATH_MIG: 392 case IBV_EVENT_PATH_MIG_ERR: 393 case IBV_EVENT_QP_LAST_WQE_REACHED: 394 { 395 struct ibv_qp *qp = event->element.qp; 396 397 pthread_mutex_lock(&qp->mutex); 398 ++qp->events_completed; 399 pthread_cond_signal(&qp->cond); 400 pthread_mutex_unlock(&qp->mutex); 401 402 return; 403 } 404 405 case IBV_EVENT_SRQ_ERR: 406 case IBV_EVENT_SRQ_LIMIT_REACHED: 407 { 408 struct ibv_srq *srq = event->element.srq; 409 410 pthread_mutex_lock(&srq->mutex); 411 ++srq->events_completed; 412 pthread_cond_signal(&srq->cond); 413 pthread_mutex_unlock(&srq->mutex); 414 415 return; 416 } 417 418 case IBV_EVENT_WQ_FATAL: 419 { 420 struct ibv_wq *wq = event->element.wq; 421 422 pthread_mutex_lock(&wq->mutex); 423 ++wq->events_completed; 424 pthread_cond_signal(&wq->cond); 425 pthread_mutex_unlock(&wq->mutex); 426 427 return; 428 } 429 430 default: 431 return; 432 } 433 } 434 default_symver(__ibv_ack_async_event, ibv_ack_async_event); 435 436 int __ibv_init_wq(struct ibv_wq *wq) 437 { 438 int err = 0; 439 wq->events_completed = 0; 440 err = pthread_mutex_init(&wq->mutex, NULL); 441 if (err) 442 return err; 443 444 err = pthread_cond_init(&wq->cond, NULL); 445 if (err) 446 goto err; 447 448 return err; 449 450 err: 451 pthread_mutex_destroy(&wq->mutex); 452 453 return err; 454 } 455 default_symver(__ibv_init_wq, ibv_init_wq); 456 457 void __ibv_cleanup_wq(struct ibv_wq *wq) 458 { 459 pthread_cond_destroy(&wq->cond); 460 pthread_mutex_destroy(&wq->mutex); 461 } 462 default_symver(__ibv_cleanup_wq, ibv_cleanup_wq); 463