1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/string.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/slab.h> 39 #include <linux/init.h> 40 #include <linux/mutex.h> 41 #include <linux/netdevice.h> 42 #include <linux/security.h> 43 #include <linux/notifier.h> 44 #include <rdma/rdma_netlink.h> 45 #include <rdma/ib_addr.h> 46 #include <rdma/ib_cache.h> 47 48 #include "core_priv.h" 49 50 MODULE_AUTHOR("Roland Dreier"); 51 MODULE_DESCRIPTION("core kernel InfiniBand API"); 52 MODULE_LICENSE("Dual BSD/GPL"); 53 54 struct ib_client_data { 55 struct list_head list; 56 struct ib_client *client; 57 void * data; 58 /* The device or client is going down. Do not call client or device 59 * callbacks other than remove(). */ 60 bool going_down; 61 }; 62 63 struct workqueue_struct *ib_comp_wq; 64 struct workqueue_struct *ib_comp_unbound_wq; 65 struct workqueue_struct *ib_wq; 66 EXPORT_SYMBOL_GPL(ib_wq); 67 68 /* The device_list and client_list contain devices and clients after their 69 * registration has completed, and the devices and clients are removed 70 * during unregistration. */ 71 static LIST_HEAD(device_list); 72 static LIST_HEAD(client_list); 73 74 /* 75 * device_mutex and lists_rwsem protect access to both device_list and 76 * client_list. device_mutex protects writer access by device and client 77 * registration / de-registration. lists_rwsem protects reader access to 78 * these lists. Iterators of these lists must lock it for read, while updates 79 * to the lists must be done with a write lock. A special case is when the 80 * device_mutex is locked. In this case locking the lists for read access is 81 * not necessary as the device_mutex implies it. 82 * 83 * lists_rwsem also protects access to the client data list. 84 */ 85 static DEFINE_MUTEX(device_mutex); 86 static DECLARE_RWSEM(lists_rwsem); 87 88 static int ib_security_change(struct notifier_block *nb, unsigned long event, 89 void *lsm_data); 90 static void ib_policy_change_task(struct work_struct *work); 91 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 92 93 static struct notifier_block ibdev_lsm_nb = { 94 .notifier_call = ib_security_change, 95 }; 96 97 static int ib_device_check_mandatory(struct ib_device *device) 98 { 99 #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x } 100 static const struct { 101 size_t offset; 102 char *name; 103 } mandatory_table[] = { 104 IB_MANDATORY_FUNC(query_device), 105 IB_MANDATORY_FUNC(query_port), 106 IB_MANDATORY_FUNC(query_pkey), 107 IB_MANDATORY_FUNC(alloc_pd), 108 IB_MANDATORY_FUNC(dealloc_pd), 109 IB_MANDATORY_FUNC(create_qp), 110 IB_MANDATORY_FUNC(modify_qp), 111 IB_MANDATORY_FUNC(destroy_qp), 112 IB_MANDATORY_FUNC(post_send), 113 IB_MANDATORY_FUNC(post_recv), 114 IB_MANDATORY_FUNC(create_cq), 115 IB_MANDATORY_FUNC(destroy_cq), 116 IB_MANDATORY_FUNC(poll_cq), 117 IB_MANDATORY_FUNC(req_notify_cq), 118 IB_MANDATORY_FUNC(get_dma_mr), 119 IB_MANDATORY_FUNC(dereg_mr), 120 IB_MANDATORY_FUNC(get_port_immutable) 121 }; 122 int i; 123 124 for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 125 if (!*(void **) ((void *) &device->ops + 126 mandatory_table[i].offset)) { 127 dev_warn(&device->dev, 128 "Device is missing mandatory function %s\n", 129 mandatory_table[i].name); 130 return -EINVAL; 131 } 132 } 133 134 return 0; 135 } 136 137 static struct ib_device *__ib_device_get_by_index(u32 index) 138 { 139 struct ib_device *device; 140 141 list_for_each_entry(device, &device_list, core_list) 142 if (device->index == index) 143 return device; 144 145 return NULL; 146 } 147 148 /* 149 * Caller must perform ib_device_put() to return the device reference count 150 * when ib_device_get_by_index() returns valid device pointer. 151 */ 152 struct ib_device *ib_device_get_by_index(u32 index) 153 { 154 struct ib_device *device; 155 156 down_read(&lists_rwsem); 157 device = __ib_device_get_by_index(index); 158 if (device) { 159 /* Do not return a device if unregistration has started. */ 160 if (!refcount_inc_not_zero(&device->refcount)) 161 device = NULL; 162 } 163 up_read(&lists_rwsem); 164 return device; 165 } 166 167 void ib_device_put(struct ib_device *device) 168 { 169 if (refcount_dec_and_test(&device->refcount)) 170 complete(&device->unreg_completion); 171 } 172 173 static struct ib_device *__ib_device_get_by_name(const char *name) 174 { 175 struct ib_device *device; 176 177 list_for_each_entry(device, &device_list, core_list) 178 if (!strcmp(name, dev_name(&device->dev))) 179 return device; 180 181 return NULL; 182 } 183 184 int ib_device_rename(struct ib_device *ibdev, const char *name) 185 { 186 struct ib_device *device; 187 int ret = 0; 188 189 if (!strcmp(name, dev_name(&ibdev->dev))) 190 return ret; 191 192 mutex_lock(&device_mutex); 193 list_for_each_entry(device, &device_list, core_list) { 194 if (!strcmp(name, dev_name(&device->dev))) { 195 ret = -EEXIST; 196 goto out; 197 } 198 } 199 200 ret = device_rename(&ibdev->dev, name); 201 if (ret) 202 goto out; 203 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 204 out: 205 mutex_unlock(&device_mutex); 206 return ret; 207 } 208 209 static int alloc_name(struct ib_device *ibdev, const char *name) 210 { 211 unsigned long *inuse; 212 struct ib_device *device; 213 int i; 214 215 inuse = (unsigned long *) get_zeroed_page(GFP_KERNEL); 216 if (!inuse) 217 return -ENOMEM; 218 219 list_for_each_entry(device, &device_list, core_list) { 220 char buf[IB_DEVICE_NAME_MAX]; 221 222 if (sscanf(dev_name(&device->dev), name, &i) != 1) 223 continue; 224 if (i < 0 || i >= PAGE_SIZE * 8) 225 continue; 226 snprintf(buf, sizeof buf, name, i); 227 if (!strcmp(buf, dev_name(&device->dev))) 228 set_bit(i, inuse); 229 } 230 231 i = find_first_zero_bit(inuse, PAGE_SIZE * 8); 232 free_page((unsigned long) inuse); 233 234 return dev_set_name(&ibdev->dev, name, i); 235 } 236 237 static void ib_device_release(struct device *device) 238 { 239 struct ib_device *dev = container_of(device, struct ib_device, dev); 240 241 WARN_ON(dev->reg_state == IB_DEV_REGISTERED); 242 if (dev->reg_state == IB_DEV_UNREGISTERED) { 243 /* 244 * In IB_DEV_UNINITIALIZED state, cache or port table 245 * is not even created. Free cache and port table only when 246 * device reaches UNREGISTERED state. 247 */ 248 ib_cache_release_one(dev); 249 kfree(dev->port_immutable); 250 } 251 kfree(dev); 252 } 253 254 static int ib_device_uevent(struct device *device, 255 struct kobj_uevent_env *env) 256 { 257 if (add_uevent_var(env, "NAME=%s", dev_name(device))) 258 return -ENOMEM; 259 260 /* 261 * It would be nice to pass the node GUID with the event... 262 */ 263 264 return 0; 265 } 266 267 static struct class ib_class = { 268 .name = "infiniband", 269 .dev_release = ib_device_release, 270 .dev_uevent = ib_device_uevent, 271 }; 272 273 /** 274 * ib_alloc_device - allocate an IB device struct 275 * @size:size of structure to allocate 276 * 277 * Low-level drivers should use ib_alloc_device() to allocate &struct 278 * ib_device. @size is the size of the structure to be allocated, 279 * including any private data used by the low-level driver. 280 * ib_dealloc_device() must be used to free structures allocated with 281 * ib_alloc_device(). 282 */ 283 struct ib_device *ib_alloc_device(size_t size) 284 { 285 struct ib_device *device; 286 287 if (WARN_ON(size < sizeof(struct ib_device))) 288 return NULL; 289 290 device = kzalloc(size, GFP_KERNEL); 291 if (!device) 292 return NULL; 293 294 rdma_restrack_init(&device->res); 295 296 device->dev.class = &ib_class; 297 device_initialize(&device->dev); 298 299 dev_set_drvdata(&device->dev, device); 300 301 INIT_LIST_HEAD(&device->event_handler_list); 302 spin_lock_init(&device->event_handler_lock); 303 rwlock_init(&device->client_data_lock); 304 INIT_LIST_HEAD(&device->client_data_list); 305 INIT_LIST_HEAD(&device->port_list); 306 refcount_set(&device->refcount, 1); 307 init_completion(&device->unreg_completion); 308 309 return device; 310 } 311 EXPORT_SYMBOL(ib_alloc_device); 312 313 /** 314 * ib_dealloc_device - free an IB device struct 315 * @device:structure to free 316 * 317 * Free a structure allocated with ib_alloc_device(). 318 */ 319 void ib_dealloc_device(struct ib_device *device) 320 { 321 WARN_ON(!list_empty(&device->client_data_list)); 322 WARN_ON(device->reg_state != IB_DEV_UNREGISTERED && 323 device->reg_state != IB_DEV_UNINITIALIZED); 324 rdma_restrack_clean(&device->res); 325 put_device(&device->dev); 326 } 327 EXPORT_SYMBOL(ib_dealloc_device); 328 329 static int add_client_context(struct ib_device *device, struct ib_client *client) 330 { 331 struct ib_client_data *context; 332 333 context = kmalloc(sizeof(*context), GFP_KERNEL); 334 if (!context) 335 return -ENOMEM; 336 337 context->client = client; 338 context->data = NULL; 339 context->going_down = false; 340 341 down_write(&lists_rwsem); 342 write_lock_irq(&device->client_data_lock); 343 list_add(&context->list, &device->client_data_list); 344 write_unlock_irq(&device->client_data_lock); 345 up_write(&lists_rwsem); 346 347 return 0; 348 } 349 350 static int verify_immutable(const struct ib_device *dev, u8 port) 351 { 352 return WARN_ON(!rdma_cap_ib_mad(dev, port) && 353 rdma_max_mad_size(dev, port) != 0); 354 } 355 356 static int read_port_immutable(struct ib_device *device) 357 { 358 int ret; 359 u8 start_port = rdma_start_port(device); 360 u8 end_port = rdma_end_port(device); 361 u8 port; 362 363 /** 364 * device->port_immutable is indexed directly by the port number to make 365 * access to this data as efficient as possible. 366 * 367 * Therefore port_immutable is declared as a 1 based array with 368 * potential empty slots at the beginning. 369 */ 370 device->port_immutable = kcalloc(end_port + 1, 371 sizeof(*device->port_immutable), 372 GFP_KERNEL); 373 if (!device->port_immutable) 374 return -ENOMEM; 375 376 for (port = start_port; port <= end_port; ++port) { 377 ret = device->ops.get_port_immutable( 378 device, port, &device->port_immutable[port]); 379 if (ret) 380 return ret; 381 382 if (verify_immutable(device, port)) 383 return -EINVAL; 384 } 385 return 0; 386 } 387 388 void ib_get_device_fw_str(struct ib_device *dev, char *str) 389 { 390 if (dev->ops.get_dev_fw_str) 391 dev->ops.get_dev_fw_str(dev, str); 392 else 393 str[0] = '\0'; 394 } 395 EXPORT_SYMBOL(ib_get_device_fw_str); 396 397 static int setup_port_pkey_list(struct ib_device *device) 398 { 399 int i; 400 401 /** 402 * device->port_pkey_list is indexed directly by the port number, 403 * Therefore it is declared as a 1 based array with potential empty 404 * slots at the beginning. 405 */ 406 device->port_pkey_list = kcalloc(rdma_end_port(device) + 1, 407 sizeof(*device->port_pkey_list), 408 GFP_KERNEL); 409 410 if (!device->port_pkey_list) 411 return -ENOMEM; 412 413 for (i = 0; i < (rdma_end_port(device) + 1); i++) { 414 spin_lock_init(&device->port_pkey_list[i].list_lock); 415 INIT_LIST_HEAD(&device->port_pkey_list[i].pkey_list); 416 } 417 418 return 0; 419 } 420 421 static void ib_policy_change_task(struct work_struct *work) 422 { 423 struct ib_device *dev; 424 425 down_read(&lists_rwsem); 426 list_for_each_entry(dev, &device_list, core_list) { 427 int i; 428 429 for (i = rdma_start_port(dev); i <= rdma_end_port(dev); i++) { 430 u64 sp; 431 int ret = ib_get_cached_subnet_prefix(dev, 432 i, 433 &sp); 434 435 WARN_ONCE(ret, 436 "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 437 ret); 438 if (!ret) 439 ib_security_cache_change(dev, i, sp); 440 } 441 } 442 up_read(&lists_rwsem); 443 } 444 445 static int ib_security_change(struct notifier_block *nb, unsigned long event, 446 void *lsm_data) 447 { 448 if (event != LSM_POLICY_CHANGE) 449 return NOTIFY_DONE; 450 451 schedule_work(&ib_policy_change_work); 452 453 return NOTIFY_OK; 454 } 455 456 /** 457 * __dev_new_index - allocate an device index 458 * 459 * Returns a suitable unique value for a new device interface 460 * number. It assumes that there are less than 2^32-1 ib devices 461 * will be present in the system. 462 */ 463 static u32 __dev_new_index(void) 464 { 465 /* 466 * The device index to allow stable naming. 467 * Similar to struct net -> ifindex. 468 */ 469 static u32 index; 470 471 for (;;) { 472 if (!(++index)) 473 index = 1; 474 475 if (!__ib_device_get_by_index(index)) 476 return index; 477 } 478 } 479 480 static void setup_dma_device(struct ib_device *device) 481 { 482 struct device *parent = device->dev.parent; 483 484 WARN_ON_ONCE(device->dma_device); 485 if (device->dev.dma_ops) { 486 /* 487 * The caller provided custom DMA operations. Copy the 488 * DMA-related fields that are used by e.g. dma_alloc_coherent() 489 * into device->dev. 490 */ 491 device->dma_device = &device->dev; 492 if (!device->dev.dma_mask) { 493 if (parent) 494 device->dev.dma_mask = parent->dma_mask; 495 else 496 WARN_ON_ONCE(true); 497 } 498 if (!device->dev.coherent_dma_mask) { 499 if (parent) 500 device->dev.coherent_dma_mask = 501 parent->coherent_dma_mask; 502 else 503 WARN_ON_ONCE(true); 504 } 505 } else { 506 /* 507 * The caller did not provide custom DMA operations. Use the 508 * DMA mapping operations of the parent device. 509 */ 510 WARN_ON_ONCE(!parent); 511 device->dma_device = parent; 512 } 513 } 514 515 static void cleanup_device(struct ib_device *device) 516 { 517 ib_cache_cleanup_one(device); 518 ib_cache_release_one(device); 519 kfree(device->port_pkey_list); 520 kfree(device->port_immutable); 521 } 522 523 static int setup_device(struct ib_device *device) 524 { 525 struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 526 int ret; 527 528 ret = ib_device_check_mandatory(device); 529 if (ret) 530 return ret; 531 532 ret = read_port_immutable(device); 533 if (ret) { 534 dev_warn(&device->dev, 535 "Couldn't create per port immutable data\n"); 536 return ret; 537 } 538 539 memset(&device->attrs, 0, sizeof(device->attrs)); 540 ret = device->ops.query_device(device, &device->attrs, &uhw); 541 if (ret) { 542 dev_warn(&device->dev, 543 "Couldn't query the device attributes\n"); 544 goto port_cleanup; 545 } 546 547 ret = setup_port_pkey_list(device); 548 if (ret) { 549 dev_warn(&device->dev, "Couldn't create per port_pkey_list\n"); 550 goto port_cleanup; 551 } 552 553 ret = ib_cache_setup_one(device); 554 if (ret) { 555 dev_warn(&device->dev, 556 "Couldn't set up InfiniBand P_Key/GID cache\n"); 557 goto pkey_cleanup; 558 } 559 return 0; 560 561 pkey_cleanup: 562 kfree(device->port_pkey_list); 563 port_cleanup: 564 kfree(device->port_immutable); 565 return ret; 566 } 567 568 /** 569 * ib_register_device - Register an IB device with IB core 570 * @device:Device to register 571 * 572 * Low-level drivers use ib_register_device() to register their 573 * devices with the IB core. All registered clients will receive a 574 * callback for each device that is added. @device must be allocated 575 * with ib_alloc_device(). 576 */ 577 int ib_register_device(struct ib_device *device, const char *name, 578 int (*port_callback)(struct ib_device *, u8, 579 struct kobject *)) 580 { 581 int ret; 582 struct ib_client *client; 583 584 setup_dma_device(device); 585 586 mutex_lock(&device_mutex); 587 588 if (strchr(name, '%')) { 589 ret = alloc_name(device, name); 590 if (ret) 591 goto out; 592 } else { 593 ret = dev_set_name(&device->dev, name); 594 if (ret) 595 goto out; 596 } 597 if (__ib_device_get_by_name(dev_name(&device->dev))) { 598 ret = -ENFILE; 599 goto out; 600 } 601 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); 602 603 ret = setup_device(device); 604 if (ret) 605 goto out; 606 607 device->index = __dev_new_index(); 608 609 ret = ib_device_register_rdmacg(device); 610 if (ret) { 611 dev_warn(&device->dev, 612 "Couldn't register device with rdma cgroup\n"); 613 goto dev_cleanup; 614 } 615 616 ret = ib_device_register_sysfs(device, port_callback); 617 if (ret) { 618 dev_warn(&device->dev, 619 "Couldn't register device with driver model\n"); 620 goto cg_cleanup; 621 } 622 623 device->reg_state = IB_DEV_REGISTERED; 624 625 list_for_each_entry(client, &client_list, list) 626 if (!add_client_context(device, client) && client->add) 627 client->add(device); 628 629 down_write(&lists_rwsem); 630 list_add_tail(&device->core_list, &device_list); 631 up_write(&lists_rwsem); 632 mutex_unlock(&device_mutex); 633 return 0; 634 635 cg_cleanup: 636 ib_device_unregister_rdmacg(device); 637 dev_cleanup: 638 cleanup_device(device); 639 out: 640 mutex_unlock(&device_mutex); 641 return ret; 642 } 643 EXPORT_SYMBOL(ib_register_device); 644 645 /** 646 * ib_unregister_device - Unregister an IB device 647 * @device:Device to unregister 648 * 649 * Unregister an IB device. All clients will receive a remove callback. 650 */ 651 void ib_unregister_device(struct ib_device *device) 652 { 653 struct ib_client_data *context, *tmp; 654 unsigned long flags; 655 656 /* 657 * Wait for all netlink command callers to finish working on the 658 * device. 659 */ 660 ib_device_put(device); 661 wait_for_completion(&device->unreg_completion); 662 663 mutex_lock(&device_mutex); 664 665 down_write(&lists_rwsem); 666 list_del(&device->core_list); 667 write_lock_irq(&device->client_data_lock); 668 list_for_each_entry(context, &device->client_data_list, list) 669 context->going_down = true; 670 write_unlock_irq(&device->client_data_lock); 671 downgrade_write(&lists_rwsem); 672 673 list_for_each_entry(context, &device->client_data_list, list) { 674 if (context->client->remove) 675 context->client->remove(device, context->data); 676 } 677 up_read(&lists_rwsem); 678 679 ib_device_unregister_sysfs(device); 680 ib_device_unregister_rdmacg(device); 681 682 mutex_unlock(&device_mutex); 683 684 ib_cache_cleanup_one(device); 685 686 ib_security_destroy_port_pkey_list(device); 687 kfree(device->port_pkey_list); 688 689 down_write(&lists_rwsem); 690 write_lock_irqsave(&device->client_data_lock, flags); 691 list_for_each_entry_safe(context, tmp, &device->client_data_list, 692 list) { 693 list_del(&context->list); 694 kfree(context); 695 } 696 write_unlock_irqrestore(&device->client_data_lock, flags); 697 up_write(&lists_rwsem); 698 699 device->reg_state = IB_DEV_UNREGISTERED; 700 } 701 EXPORT_SYMBOL(ib_unregister_device); 702 703 /** 704 * ib_register_client - Register an IB client 705 * @client:Client to register 706 * 707 * Upper level users of the IB drivers can use ib_register_client() to 708 * register callbacks for IB device addition and removal. When an IB 709 * device is added, each registered client's add method will be called 710 * (in the order the clients were registered), and when a device is 711 * removed, each client's remove method will be called (in the reverse 712 * order that clients were registered). In addition, when 713 * ib_register_client() is called, the client will receive an add 714 * callback for all devices already registered. 715 */ 716 int ib_register_client(struct ib_client *client) 717 { 718 struct ib_device *device; 719 720 mutex_lock(&device_mutex); 721 722 list_for_each_entry(device, &device_list, core_list) 723 if (!add_client_context(device, client) && client->add) 724 client->add(device); 725 726 down_write(&lists_rwsem); 727 list_add_tail(&client->list, &client_list); 728 up_write(&lists_rwsem); 729 730 mutex_unlock(&device_mutex); 731 732 return 0; 733 } 734 EXPORT_SYMBOL(ib_register_client); 735 736 /** 737 * ib_unregister_client - Unregister an IB client 738 * @client:Client to unregister 739 * 740 * Upper level users use ib_unregister_client() to remove their client 741 * registration. When ib_unregister_client() is called, the client 742 * will receive a remove callback for each IB device still registered. 743 */ 744 void ib_unregister_client(struct ib_client *client) 745 { 746 struct ib_client_data *context; 747 struct ib_device *device; 748 749 mutex_lock(&device_mutex); 750 751 down_write(&lists_rwsem); 752 list_del(&client->list); 753 up_write(&lists_rwsem); 754 755 list_for_each_entry(device, &device_list, core_list) { 756 struct ib_client_data *found_context = NULL; 757 758 down_write(&lists_rwsem); 759 write_lock_irq(&device->client_data_lock); 760 list_for_each_entry(context, &device->client_data_list, list) 761 if (context->client == client) { 762 context->going_down = true; 763 found_context = context; 764 break; 765 } 766 write_unlock_irq(&device->client_data_lock); 767 up_write(&lists_rwsem); 768 769 if (client->remove) 770 client->remove(device, found_context ? 771 found_context->data : NULL); 772 773 if (!found_context) { 774 dev_warn(&device->dev, 775 "No client context found for %s\n", 776 client->name); 777 continue; 778 } 779 780 down_write(&lists_rwsem); 781 write_lock_irq(&device->client_data_lock); 782 list_del(&found_context->list); 783 write_unlock_irq(&device->client_data_lock); 784 up_write(&lists_rwsem); 785 kfree(found_context); 786 } 787 788 mutex_unlock(&device_mutex); 789 } 790 EXPORT_SYMBOL(ib_unregister_client); 791 792 /** 793 * ib_get_client_data - Get IB client context 794 * @device:Device to get context for 795 * @client:Client to get context for 796 * 797 * ib_get_client_data() returns client context set with 798 * ib_set_client_data(). 799 */ 800 void *ib_get_client_data(struct ib_device *device, struct ib_client *client) 801 { 802 struct ib_client_data *context; 803 void *ret = NULL; 804 unsigned long flags; 805 806 read_lock_irqsave(&device->client_data_lock, flags); 807 list_for_each_entry(context, &device->client_data_list, list) 808 if (context->client == client) { 809 ret = context->data; 810 break; 811 } 812 read_unlock_irqrestore(&device->client_data_lock, flags); 813 814 return ret; 815 } 816 EXPORT_SYMBOL(ib_get_client_data); 817 818 /** 819 * ib_set_client_data - Set IB client context 820 * @device:Device to set context for 821 * @client:Client to set context for 822 * @data:Context to set 823 * 824 * ib_set_client_data() sets client context that can be retrieved with 825 * ib_get_client_data(). 826 */ 827 void ib_set_client_data(struct ib_device *device, struct ib_client *client, 828 void *data) 829 { 830 struct ib_client_data *context; 831 unsigned long flags; 832 833 write_lock_irqsave(&device->client_data_lock, flags); 834 list_for_each_entry(context, &device->client_data_list, list) 835 if (context->client == client) { 836 context->data = data; 837 goto out; 838 } 839 840 dev_warn(&device->dev, "No client context found for %s\n", 841 client->name); 842 843 out: 844 write_unlock_irqrestore(&device->client_data_lock, flags); 845 } 846 EXPORT_SYMBOL(ib_set_client_data); 847 848 /** 849 * ib_register_event_handler - Register an IB event handler 850 * @event_handler:Handler to register 851 * 852 * ib_register_event_handler() registers an event handler that will be 853 * called back when asynchronous IB events occur (as defined in 854 * chapter 11 of the InfiniBand Architecture Specification). This 855 * callback may occur in interrupt context. 856 */ 857 void ib_register_event_handler(struct ib_event_handler *event_handler) 858 { 859 unsigned long flags; 860 861 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 862 list_add_tail(&event_handler->list, 863 &event_handler->device->event_handler_list); 864 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 865 } 866 EXPORT_SYMBOL(ib_register_event_handler); 867 868 /** 869 * ib_unregister_event_handler - Unregister an event handler 870 * @event_handler:Handler to unregister 871 * 872 * Unregister an event handler registered with 873 * ib_register_event_handler(). 874 */ 875 void ib_unregister_event_handler(struct ib_event_handler *event_handler) 876 { 877 unsigned long flags; 878 879 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 880 list_del(&event_handler->list); 881 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 882 } 883 EXPORT_SYMBOL(ib_unregister_event_handler); 884 885 /** 886 * ib_dispatch_event - Dispatch an asynchronous event 887 * @event:Event to dispatch 888 * 889 * Low-level drivers must call ib_dispatch_event() to dispatch the 890 * event to all registered event handlers when an asynchronous event 891 * occurs. 892 */ 893 void ib_dispatch_event(struct ib_event *event) 894 { 895 unsigned long flags; 896 struct ib_event_handler *handler; 897 898 spin_lock_irqsave(&event->device->event_handler_lock, flags); 899 900 list_for_each_entry(handler, &event->device->event_handler_list, list) 901 handler->handler(handler, event); 902 903 spin_unlock_irqrestore(&event->device->event_handler_lock, flags); 904 } 905 EXPORT_SYMBOL(ib_dispatch_event); 906 907 /** 908 * ib_query_port - Query IB port attributes 909 * @device:Device to query 910 * @port_num:Port number to query 911 * @port_attr:Port attributes 912 * 913 * ib_query_port() returns the attributes of a port through the 914 * @port_attr pointer. 915 */ 916 int ib_query_port(struct ib_device *device, 917 u8 port_num, 918 struct ib_port_attr *port_attr) 919 { 920 union ib_gid gid; 921 int err; 922 923 if (!rdma_is_port_valid(device, port_num)) 924 return -EINVAL; 925 926 memset(port_attr, 0, sizeof(*port_attr)); 927 err = device->ops.query_port(device, port_num, port_attr); 928 if (err || port_attr->subnet_prefix) 929 return err; 930 931 if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND) 932 return 0; 933 934 err = device->ops.query_gid(device, port_num, 0, &gid); 935 if (err) 936 return err; 937 938 port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 939 return 0; 940 } 941 EXPORT_SYMBOL(ib_query_port); 942 943 /** 944 * ib_enum_roce_netdev - enumerate all RoCE ports 945 * @ib_dev : IB device we want to query 946 * @filter: Should we call the callback? 947 * @filter_cookie: Cookie passed to filter 948 * @cb: Callback to call for each found RoCE ports 949 * @cookie: Cookie passed back to the callback 950 * 951 * Enumerates all of the physical RoCE ports of ib_dev 952 * which are related to netdevice and calls callback() on each 953 * device for which filter() function returns non zero. 954 */ 955 void ib_enum_roce_netdev(struct ib_device *ib_dev, 956 roce_netdev_filter filter, 957 void *filter_cookie, 958 roce_netdev_callback cb, 959 void *cookie) 960 { 961 u8 port; 962 963 for (port = rdma_start_port(ib_dev); port <= rdma_end_port(ib_dev); 964 port++) 965 if (rdma_protocol_roce(ib_dev, port)) { 966 struct net_device *idev = NULL; 967 968 if (ib_dev->ops.get_netdev) 969 idev = ib_dev->ops.get_netdev(ib_dev, port); 970 971 if (idev && 972 idev->reg_state >= NETREG_UNREGISTERED) { 973 dev_put(idev); 974 idev = NULL; 975 } 976 977 if (filter(ib_dev, port, idev, filter_cookie)) 978 cb(ib_dev, port, idev, cookie); 979 980 if (idev) 981 dev_put(idev); 982 } 983 } 984 985 /** 986 * ib_enum_all_roce_netdevs - enumerate all RoCE devices 987 * @filter: Should we call the callback? 988 * @filter_cookie: Cookie passed to filter 989 * @cb: Callback to call for each found RoCE ports 990 * @cookie: Cookie passed back to the callback 991 * 992 * Enumerates all RoCE devices' physical ports which are related 993 * to netdevices and calls callback() on each device for which 994 * filter() function returns non zero. 995 */ 996 void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 997 void *filter_cookie, 998 roce_netdev_callback cb, 999 void *cookie) 1000 { 1001 struct ib_device *dev; 1002 1003 down_read(&lists_rwsem); 1004 list_for_each_entry(dev, &device_list, core_list) 1005 ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 1006 up_read(&lists_rwsem); 1007 } 1008 1009 /** 1010 * ib_enum_all_devs - enumerate all ib_devices 1011 * @cb: Callback to call for each found ib_device 1012 * 1013 * Enumerates all ib_devices and calls callback() on each device. 1014 */ 1015 int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 1016 struct netlink_callback *cb) 1017 { 1018 struct ib_device *dev; 1019 unsigned int idx = 0; 1020 int ret = 0; 1021 1022 down_read(&lists_rwsem); 1023 list_for_each_entry(dev, &device_list, core_list) { 1024 ret = nldev_cb(dev, skb, cb, idx); 1025 if (ret) 1026 break; 1027 idx++; 1028 } 1029 1030 up_read(&lists_rwsem); 1031 return ret; 1032 } 1033 1034 /** 1035 * ib_query_pkey - Get P_Key table entry 1036 * @device:Device to query 1037 * @port_num:Port number to query 1038 * @index:P_Key table index to query 1039 * @pkey:Returned P_Key 1040 * 1041 * ib_query_pkey() fetches the specified P_Key table entry. 1042 */ 1043 int ib_query_pkey(struct ib_device *device, 1044 u8 port_num, u16 index, u16 *pkey) 1045 { 1046 if (!rdma_is_port_valid(device, port_num)) 1047 return -EINVAL; 1048 1049 return device->ops.query_pkey(device, port_num, index, pkey); 1050 } 1051 EXPORT_SYMBOL(ib_query_pkey); 1052 1053 /** 1054 * ib_modify_device - Change IB device attributes 1055 * @device:Device to modify 1056 * @device_modify_mask:Mask of attributes to change 1057 * @device_modify:New attribute values 1058 * 1059 * ib_modify_device() changes a device's attributes as specified by 1060 * the @device_modify_mask and @device_modify structure. 1061 */ 1062 int ib_modify_device(struct ib_device *device, 1063 int device_modify_mask, 1064 struct ib_device_modify *device_modify) 1065 { 1066 if (!device->ops.modify_device) 1067 return -ENOSYS; 1068 1069 return device->ops.modify_device(device, device_modify_mask, 1070 device_modify); 1071 } 1072 EXPORT_SYMBOL(ib_modify_device); 1073 1074 /** 1075 * ib_modify_port - Modifies the attributes for the specified port. 1076 * @device: The device to modify. 1077 * @port_num: The number of the port to modify. 1078 * @port_modify_mask: Mask used to specify which attributes of the port 1079 * to change. 1080 * @port_modify: New attribute values for the port. 1081 * 1082 * ib_modify_port() changes a port's attributes as specified by the 1083 * @port_modify_mask and @port_modify structure. 1084 */ 1085 int ib_modify_port(struct ib_device *device, 1086 u8 port_num, int port_modify_mask, 1087 struct ib_port_modify *port_modify) 1088 { 1089 int rc; 1090 1091 if (!rdma_is_port_valid(device, port_num)) 1092 return -EINVAL; 1093 1094 if (device->ops.modify_port) 1095 rc = device->ops.modify_port(device, port_num, 1096 port_modify_mask, 1097 port_modify); 1098 else 1099 rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS; 1100 return rc; 1101 } 1102 EXPORT_SYMBOL(ib_modify_port); 1103 1104 /** 1105 * ib_find_gid - Returns the port number and GID table index where 1106 * a specified GID value occurs. Its searches only for IB link layer. 1107 * @device: The device to query. 1108 * @gid: The GID value to search for. 1109 * @port_num: The port number of the device where the GID value was found. 1110 * @index: The index into the GID table where the GID was found. This 1111 * parameter may be NULL. 1112 */ 1113 int ib_find_gid(struct ib_device *device, union ib_gid *gid, 1114 u8 *port_num, u16 *index) 1115 { 1116 union ib_gid tmp_gid; 1117 int ret, port, i; 1118 1119 for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) { 1120 if (!rdma_protocol_ib(device, port)) 1121 continue; 1122 1123 for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) { 1124 ret = rdma_query_gid(device, port, i, &tmp_gid); 1125 if (ret) 1126 return ret; 1127 if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 1128 *port_num = port; 1129 if (index) 1130 *index = i; 1131 return 0; 1132 } 1133 } 1134 } 1135 1136 return -ENOENT; 1137 } 1138 EXPORT_SYMBOL(ib_find_gid); 1139 1140 /** 1141 * ib_find_pkey - Returns the PKey table index where a specified 1142 * PKey value occurs. 1143 * @device: The device to query. 1144 * @port_num: The port number of the device to search for the PKey. 1145 * @pkey: The PKey value to search for. 1146 * @index: The index into the PKey table where the PKey was found. 1147 */ 1148 int ib_find_pkey(struct ib_device *device, 1149 u8 port_num, u16 pkey, u16 *index) 1150 { 1151 int ret, i; 1152 u16 tmp_pkey; 1153 int partial_ix = -1; 1154 1155 for (i = 0; i < device->port_immutable[port_num].pkey_tbl_len; ++i) { 1156 ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 1157 if (ret) 1158 return ret; 1159 if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 1160 /* if there is full-member pkey take it.*/ 1161 if (tmp_pkey & 0x8000) { 1162 *index = i; 1163 return 0; 1164 } 1165 if (partial_ix < 0) 1166 partial_ix = i; 1167 } 1168 } 1169 1170 /*no full-member, if exists take the limited*/ 1171 if (partial_ix >= 0) { 1172 *index = partial_ix; 1173 return 0; 1174 } 1175 return -ENOENT; 1176 } 1177 EXPORT_SYMBOL(ib_find_pkey); 1178 1179 /** 1180 * ib_get_net_dev_by_params() - Return the appropriate net_dev 1181 * for a received CM request 1182 * @dev: An RDMA device on which the request has been received. 1183 * @port: Port number on the RDMA device. 1184 * @pkey: The Pkey the request came on. 1185 * @gid: A GID that the net_dev uses to communicate. 1186 * @addr: Contains the IP address that the request specified as its 1187 * destination. 1188 */ 1189 struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 1190 u8 port, 1191 u16 pkey, 1192 const union ib_gid *gid, 1193 const struct sockaddr *addr) 1194 { 1195 struct net_device *net_dev = NULL; 1196 struct ib_client_data *context; 1197 1198 if (!rdma_protocol_ib(dev, port)) 1199 return NULL; 1200 1201 down_read(&lists_rwsem); 1202 1203 list_for_each_entry(context, &dev->client_data_list, list) { 1204 struct ib_client *client = context->client; 1205 1206 if (context->going_down) 1207 continue; 1208 1209 if (client->get_net_dev_by_params) { 1210 net_dev = client->get_net_dev_by_params(dev, port, pkey, 1211 gid, addr, 1212 context->data); 1213 if (net_dev) 1214 break; 1215 } 1216 } 1217 1218 up_read(&lists_rwsem); 1219 1220 return net_dev; 1221 } 1222 EXPORT_SYMBOL(ib_get_net_dev_by_params); 1223 1224 void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 1225 { 1226 struct ib_device_ops *dev_ops = &dev->ops; 1227 #define SET_DEVICE_OP(ptr, name) \ 1228 do { \ 1229 if (ops->name) \ 1230 if (!((ptr)->name)) \ 1231 (ptr)->name = ops->name; \ 1232 } while (0) 1233 1234 SET_DEVICE_OP(dev_ops, add_gid); 1235 SET_DEVICE_OP(dev_ops, advise_mr); 1236 SET_DEVICE_OP(dev_ops, alloc_dm); 1237 SET_DEVICE_OP(dev_ops, alloc_fmr); 1238 SET_DEVICE_OP(dev_ops, alloc_hw_stats); 1239 SET_DEVICE_OP(dev_ops, alloc_mr); 1240 SET_DEVICE_OP(dev_ops, alloc_mw); 1241 SET_DEVICE_OP(dev_ops, alloc_pd); 1242 SET_DEVICE_OP(dev_ops, alloc_rdma_netdev); 1243 SET_DEVICE_OP(dev_ops, alloc_ucontext); 1244 SET_DEVICE_OP(dev_ops, alloc_xrcd); 1245 SET_DEVICE_OP(dev_ops, attach_mcast); 1246 SET_DEVICE_OP(dev_ops, check_mr_status); 1247 SET_DEVICE_OP(dev_ops, create_ah); 1248 SET_DEVICE_OP(dev_ops, create_counters); 1249 SET_DEVICE_OP(dev_ops, create_cq); 1250 SET_DEVICE_OP(dev_ops, create_flow); 1251 SET_DEVICE_OP(dev_ops, create_flow_action_esp); 1252 SET_DEVICE_OP(dev_ops, create_qp); 1253 SET_DEVICE_OP(dev_ops, create_rwq_ind_table); 1254 SET_DEVICE_OP(dev_ops, create_srq); 1255 SET_DEVICE_OP(dev_ops, create_wq); 1256 SET_DEVICE_OP(dev_ops, dealloc_dm); 1257 SET_DEVICE_OP(dev_ops, dealloc_fmr); 1258 SET_DEVICE_OP(dev_ops, dealloc_mw); 1259 SET_DEVICE_OP(dev_ops, dealloc_pd); 1260 SET_DEVICE_OP(dev_ops, dealloc_ucontext); 1261 SET_DEVICE_OP(dev_ops, dealloc_xrcd); 1262 SET_DEVICE_OP(dev_ops, del_gid); 1263 SET_DEVICE_OP(dev_ops, dereg_mr); 1264 SET_DEVICE_OP(dev_ops, destroy_ah); 1265 SET_DEVICE_OP(dev_ops, destroy_counters); 1266 SET_DEVICE_OP(dev_ops, destroy_cq); 1267 SET_DEVICE_OP(dev_ops, destroy_flow); 1268 SET_DEVICE_OP(dev_ops, destroy_flow_action); 1269 SET_DEVICE_OP(dev_ops, destroy_qp); 1270 SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table); 1271 SET_DEVICE_OP(dev_ops, destroy_srq); 1272 SET_DEVICE_OP(dev_ops, destroy_wq); 1273 SET_DEVICE_OP(dev_ops, detach_mcast); 1274 SET_DEVICE_OP(dev_ops, disassociate_ucontext); 1275 SET_DEVICE_OP(dev_ops, drain_rq); 1276 SET_DEVICE_OP(dev_ops, drain_sq); 1277 SET_DEVICE_OP(dev_ops, get_dev_fw_str); 1278 SET_DEVICE_OP(dev_ops, get_dma_mr); 1279 SET_DEVICE_OP(dev_ops, get_hw_stats); 1280 SET_DEVICE_OP(dev_ops, get_link_layer); 1281 SET_DEVICE_OP(dev_ops, get_netdev); 1282 SET_DEVICE_OP(dev_ops, get_port_immutable); 1283 SET_DEVICE_OP(dev_ops, get_vector_affinity); 1284 SET_DEVICE_OP(dev_ops, get_vf_config); 1285 SET_DEVICE_OP(dev_ops, get_vf_stats); 1286 SET_DEVICE_OP(dev_ops, map_mr_sg); 1287 SET_DEVICE_OP(dev_ops, map_phys_fmr); 1288 SET_DEVICE_OP(dev_ops, mmap); 1289 SET_DEVICE_OP(dev_ops, modify_ah); 1290 SET_DEVICE_OP(dev_ops, modify_cq); 1291 SET_DEVICE_OP(dev_ops, modify_device); 1292 SET_DEVICE_OP(dev_ops, modify_flow_action_esp); 1293 SET_DEVICE_OP(dev_ops, modify_port); 1294 SET_DEVICE_OP(dev_ops, modify_qp); 1295 SET_DEVICE_OP(dev_ops, modify_srq); 1296 SET_DEVICE_OP(dev_ops, modify_wq); 1297 SET_DEVICE_OP(dev_ops, peek_cq); 1298 SET_DEVICE_OP(dev_ops, poll_cq); 1299 SET_DEVICE_OP(dev_ops, post_recv); 1300 SET_DEVICE_OP(dev_ops, post_send); 1301 SET_DEVICE_OP(dev_ops, post_srq_recv); 1302 SET_DEVICE_OP(dev_ops, process_mad); 1303 SET_DEVICE_OP(dev_ops, query_ah); 1304 SET_DEVICE_OP(dev_ops, query_device); 1305 SET_DEVICE_OP(dev_ops, query_gid); 1306 SET_DEVICE_OP(dev_ops, query_pkey); 1307 SET_DEVICE_OP(dev_ops, query_port); 1308 SET_DEVICE_OP(dev_ops, query_qp); 1309 SET_DEVICE_OP(dev_ops, query_srq); 1310 SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); 1311 SET_DEVICE_OP(dev_ops, read_counters); 1312 SET_DEVICE_OP(dev_ops, reg_dm_mr); 1313 SET_DEVICE_OP(dev_ops, reg_user_mr); 1314 SET_DEVICE_OP(dev_ops, req_ncomp_notif); 1315 SET_DEVICE_OP(dev_ops, req_notify_cq); 1316 SET_DEVICE_OP(dev_ops, rereg_user_mr); 1317 SET_DEVICE_OP(dev_ops, resize_cq); 1318 SET_DEVICE_OP(dev_ops, set_vf_guid); 1319 SET_DEVICE_OP(dev_ops, set_vf_link_state); 1320 SET_DEVICE_OP(dev_ops, unmap_fmr); 1321 } 1322 EXPORT_SYMBOL(ib_set_device_ops); 1323 1324 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 1325 [RDMA_NL_LS_OP_RESOLVE] = { 1326 .doit = ib_nl_handle_resolve_resp, 1327 .flags = RDMA_NL_ADMIN_PERM, 1328 }, 1329 [RDMA_NL_LS_OP_SET_TIMEOUT] = { 1330 .doit = ib_nl_handle_set_timeout, 1331 .flags = RDMA_NL_ADMIN_PERM, 1332 }, 1333 [RDMA_NL_LS_OP_IP_RESOLVE] = { 1334 .doit = ib_nl_handle_ip_res_resp, 1335 .flags = RDMA_NL_ADMIN_PERM, 1336 }, 1337 }; 1338 1339 static int __init ib_core_init(void) 1340 { 1341 int ret; 1342 1343 ib_wq = alloc_workqueue("infiniband", 0, 0); 1344 if (!ib_wq) 1345 return -ENOMEM; 1346 1347 ib_comp_wq = alloc_workqueue("ib-comp-wq", 1348 WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1349 if (!ib_comp_wq) { 1350 ret = -ENOMEM; 1351 goto err; 1352 } 1353 1354 ib_comp_unbound_wq = 1355 alloc_workqueue("ib-comp-unb-wq", 1356 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM | 1357 WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE); 1358 if (!ib_comp_unbound_wq) { 1359 ret = -ENOMEM; 1360 goto err_comp; 1361 } 1362 1363 ret = class_register(&ib_class); 1364 if (ret) { 1365 pr_warn("Couldn't create InfiniBand device class\n"); 1366 goto err_comp_unbound; 1367 } 1368 1369 ret = rdma_nl_init(); 1370 if (ret) { 1371 pr_warn("Couldn't init IB netlink interface: err %d\n", ret); 1372 goto err_sysfs; 1373 } 1374 1375 ret = addr_init(); 1376 if (ret) { 1377 pr_warn("Could't init IB address resolution\n"); 1378 goto err_ibnl; 1379 } 1380 1381 ret = ib_mad_init(); 1382 if (ret) { 1383 pr_warn("Couldn't init IB MAD\n"); 1384 goto err_addr; 1385 } 1386 1387 ret = ib_sa_init(); 1388 if (ret) { 1389 pr_warn("Couldn't init SA\n"); 1390 goto err_mad; 1391 } 1392 1393 ret = register_lsm_notifier(&ibdev_lsm_nb); 1394 if (ret) { 1395 pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 1396 goto err_sa; 1397 } 1398 1399 nldev_init(); 1400 rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 1401 roce_gid_mgmt_init(); 1402 1403 return 0; 1404 1405 err_sa: 1406 ib_sa_cleanup(); 1407 err_mad: 1408 ib_mad_cleanup(); 1409 err_addr: 1410 addr_cleanup(); 1411 err_ibnl: 1412 rdma_nl_exit(); 1413 err_sysfs: 1414 class_unregister(&ib_class); 1415 err_comp_unbound: 1416 destroy_workqueue(ib_comp_unbound_wq); 1417 err_comp: 1418 destroy_workqueue(ib_comp_wq); 1419 err: 1420 destroy_workqueue(ib_wq); 1421 return ret; 1422 } 1423 1424 static void __exit ib_core_cleanup(void) 1425 { 1426 roce_gid_mgmt_cleanup(); 1427 nldev_exit(); 1428 rdma_nl_unregister(RDMA_NL_LS); 1429 unregister_lsm_notifier(&ibdev_lsm_nb); 1430 ib_sa_cleanup(); 1431 ib_mad_cleanup(); 1432 addr_cleanup(); 1433 rdma_nl_exit(); 1434 class_unregister(&ib_class); 1435 destroy_workqueue(ib_comp_unbound_wq); 1436 destroy_workqueue(ib_comp_wq); 1437 /* Make sure that any pending umem accounting work is done. */ 1438 destroy_workqueue(ib_wq); 1439 } 1440 1441 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 1442 1443 subsys_initcall(ib_core_init); 1444 module_exit(ib_core_cleanup); 1445