1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/string.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/slab.h> 39 #include <linux/init.h> 40 #include <linux/netdevice.h> 41 #include <net/net_namespace.h> 42 #include <linux/security.h> 43 #include <linux/notifier.h> 44 #include <linux/hashtable.h> 45 #include <rdma/rdma_netlink.h> 46 #include <rdma/ib_addr.h> 47 #include <rdma/ib_cache.h> 48 #include <rdma/rdma_counter.h> 49 50 #include "core_priv.h" 51 #include "restrack.h" 52 53 MODULE_AUTHOR("Roland Dreier"); 54 MODULE_DESCRIPTION("core kernel InfiniBand API"); 55 MODULE_LICENSE("Dual BSD/GPL"); 56 57 struct workqueue_struct *ib_comp_wq; 58 struct workqueue_struct *ib_comp_unbound_wq; 59 struct workqueue_struct *ib_wq; 60 EXPORT_SYMBOL_GPL(ib_wq); 61 62 /* 63 * Each of the three rwsem locks (devices, clients, client_data) protects the 64 * xarray of the same name. Specifically it allows the caller to assert that 65 * the MARK will/will not be changing under the lock, and for devices and 66 * clients, that the value in the xarray is still a valid pointer. Change of 67 * the MARK is linked to the object state, so holding the lock and testing the 68 * MARK also asserts that the contained object is in a certain state. 69 * 70 * This is used to build a two stage register/unregister flow where objects 71 * can continue to be in the xarray even though they are still in progress to 72 * register/unregister. 73 * 74 * The xarray itself provides additional locking, and restartable iteration, 75 * which is also relied on. 76 * 77 * Locks should not be nested, with the exception of client_data, which is 78 * allowed to nest under the read side of the other two locks. 79 * 80 * The devices_rwsem also protects the device name list, any change or 81 * assignment of device name must also hold the write side to guarantee unique 82 * names. 83 */ 84 85 /* 86 * devices contains devices that have had their names assigned. The 87 * devices may not be registered. Users that care about the registration 88 * status need to call ib_device_try_get() on the device to ensure it is 89 * registered, and keep it registered, for the required duration. 90 * 91 */ 92 static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); 93 static DECLARE_RWSEM(devices_rwsem); 94 #define DEVICE_REGISTERED XA_MARK_1 95 96 static u32 highest_client_id; 97 #define CLIENT_REGISTERED XA_MARK_1 98 static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC); 99 static DECLARE_RWSEM(clients_rwsem); 100 101 static void ib_client_put(struct ib_client *client) 102 { 103 if (refcount_dec_and_test(&client->uses)) 104 complete(&client->uses_zero); 105 } 106 107 /* 108 * If client_data is registered then the corresponding client must also still 109 * be registered. 110 */ 111 #define CLIENT_DATA_REGISTERED XA_MARK_1 112 113 unsigned int rdma_dev_net_id; 114 115 /* 116 * A list of net namespaces is maintained in an xarray. This is necessary 117 * because we can't get the locking right using the existing net ns list. We 118 * would require a init_net callback after the list is updated. 119 */ 120 static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC); 121 /* 122 * rwsem to protect accessing the rdma_nets xarray entries. 123 */ 124 static DECLARE_RWSEM(rdma_nets_rwsem); 125 126 bool ib_devices_shared_netns = true; 127 module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444); 128 MODULE_PARM_DESC(netns_mode, 129 "Share device among net namespaces; default=1 (shared)"); 130 /** 131 * rdma_dev_access_netns() - Return whether an rdma device can be accessed 132 * from a specified net namespace or not. 133 * @dev: Pointer to rdma device which needs to be checked 134 * @net: Pointer to net namesapce for which access to be checked 135 * 136 * When the rdma device is in shared mode, it ignores the net namespace. 137 * When the rdma device is exclusive to a net namespace, rdma device net 138 * namespace is checked against the specified one. 139 */ 140 bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net) 141 { 142 return (ib_devices_shared_netns || 143 net_eq(read_pnet(&dev->coredev.rdma_net), net)); 144 } 145 EXPORT_SYMBOL(rdma_dev_access_netns); 146 147 /* 148 * xarray has this behavior where it won't iterate over NULL values stored in 149 * allocated arrays. So we need our own iterator to see all values stored in 150 * the array. This does the same thing as xa_for_each except that it also 151 * returns NULL valued entries if the array is allocating. Simplified to only 152 * work on simple xarrays. 153 */ 154 static void *xan_find_marked(struct xarray *xa, unsigned long *indexp, 155 xa_mark_t filter) 156 { 157 XA_STATE(xas, xa, *indexp); 158 void *entry; 159 160 rcu_read_lock(); 161 do { 162 entry = xas_find_marked(&xas, ULONG_MAX, filter); 163 if (xa_is_zero(entry)) 164 break; 165 } while (xas_retry(&xas, entry)); 166 rcu_read_unlock(); 167 168 if (entry) { 169 *indexp = xas.xa_index; 170 if (xa_is_zero(entry)) 171 return NULL; 172 return entry; 173 } 174 return XA_ERROR(-ENOENT); 175 } 176 #define xan_for_each_marked(xa, index, entry, filter) \ 177 for (index = 0, entry = xan_find_marked(xa, &(index), filter); \ 178 !xa_is_err(entry); \ 179 (index)++, entry = xan_find_marked(xa, &(index), filter)) 180 181 /* RCU hash table mapping netdevice pointers to struct ib_port_data */ 182 static DEFINE_SPINLOCK(ndev_hash_lock); 183 static DECLARE_HASHTABLE(ndev_hash, 5); 184 185 static void free_netdevs(struct ib_device *ib_dev); 186 static void ib_unregister_work(struct work_struct *work); 187 static void __ib_unregister_device(struct ib_device *device); 188 static int ib_security_change(struct notifier_block *nb, unsigned long event, 189 void *lsm_data); 190 static void ib_policy_change_task(struct work_struct *work); 191 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 192 193 static void __ibdev_printk(const char *level, const struct ib_device *ibdev, 194 struct va_format *vaf) 195 { 196 if (ibdev && ibdev->dev.parent) 197 dev_printk_emit(level[1] - '0', 198 ibdev->dev.parent, 199 "%s %s %s: %pV", 200 dev_driver_string(ibdev->dev.parent), 201 dev_name(ibdev->dev.parent), 202 dev_name(&ibdev->dev), 203 vaf); 204 else if (ibdev) 205 printk("%s%s: %pV", 206 level, dev_name(&ibdev->dev), vaf); 207 else 208 printk("%s(NULL ib_device): %pV", level, vaf); 209 } 210 211 void ibdev_printk(const char *level, const struct ib_device *ibdev, 212 const char *format, ...) 213 { 214 struct va_format vaf; 215 va_list args; 216 217 va_start(args, format); 218 219 vaf.fmt = format; 220 vaf.va = &args; 221 222 __ibdev_printk(level, ibdev, &vaf); 223 224 va_end(args); 225 } 226 EXPORT_SYMBOL(ibdev_printk); 227 228 #define define_ibdev_printk_level(func, level) \ 229 void func(const struct ib_device *ibdev, const char *fmt, ...) \ 230 { \ 231 struct va_format vaf; \ 232 va_list args; \ 233 \ 234 va_start(args, fmt); \ 235 \ 236 vaf.fmt = fmt; \ 237 vaf.va = &args; \ 238 \ 239 __ibdev_printk(level, ibdev, &vaf); \ 240 \ 241 va_end(args); \ 242 } \ 243 EXPORT_SYMBOL(func); 244 245 define_ibdev_printk_level(ibdev_emerg, KERN_EMERG); 246 define_ibdev_printk_level(ibdev_alert, KERN_ALERT); 247 define_ibdev_printk_level(ibdev_crit, KERN_CRIT); 248 define_ibdev_printk_level(ibdev_err, KERN_ERR); 249 define_ibdev_printk_level(ibdev_warn, KERN_WARNING); 250 define_ibdev_printk_level(ibdev_notice, KERN_NOTICE); 251 define_ibdev_printk_level(ibdev_info, KERN_INFO); 252 253 static struct notifier_block ibdev_lsm_nb = { 254 .notifier_call = ib_security_change, 255 }; 256 257 static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net, 258 struct net *net); 259 260 /* Pointer to the RCU head at the start of the ib_port_data array */ 261 struct ib_port_data_rcu { 262 struct rcu_head rcu_head; 263 struct ib_port_data pdata[]; 264 }; 265 266 static void ib_device_check_mandatory(struct ib_device *device) 267 { 268 #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x } 269 static const struct { 270 size_t offset; 271 char *name; 272 } mandatory_table[] = { 273 IB_MANDATORY_FUNC(query_device), 274 IB_MANDATORY_FUNC(query_port), 275 IB_MANDATORY_FUNC(alloc_pd), 276 IB_MANDATORY_FUNC(dealloc_pd), 277 IB_MANDATORY_FUNC(create_qp), 278 IB_MANDATORY_FUNC(modify_qp), 279 IB_MANDATORY_FUNC(destroy_qp), 280 IB_MANDATORY_FUNC(post_send), 281 IB_MANDATORY_FUNC(post_recv), 282 IB_MANDATORY_FUNC(create_cq), 283 IB_MANDATORY_FUNC(destroy_cq), 284 IB_MANDATORY_FUNC(poll_cq), 285 IB_MANDATORY_FUNC(req_notify_cq), 286 IB_MANDATORY_FUNC(get_dma_mr), 287 IB_MANDATORY_FUNC(reg_user_mr), 288 IB_MANDATORY_FUNC(dereg_mr), 289 IB_MANDATORY_FUNC(get_port_immutable) 290 }; 291 int i; 292 293 device->kverbs_provider = true; 294 for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 295 if (!*(void **) ((void *) &device->ops + 296 mandatory_table[i].offset)) { 297 device->kverbs_provider = false; 298 break; 299 } 300 } 301 } 302 303 /* 304 * Caller must perform ib_device_put() to return the device reference count 305 * when ib_device_get_by_index() returns valid device pointer. 306 */ 307 struct ib_device *ib_device_get_by_index(const struct net *net, u32 index) 308 { 309 struct ib_device *device; 310 311 down_read(&devices_rwsem); 312 device = xa_load(&devices, index); 313 if (device) { 314 if (!rdma_dev_access_netns(device, net)) { 315 device = NULL; 316 goto out; 317 } 318 319 if (!ib_device_try_get(device)) 320 device = NULL; 321 } 322 out: 323 up_read(&devices_rwsem); 324 return device; 325 } 326 327 /** 328 * ib_device_put - Release IB device reference 329 * @device: device whose reference to be released 330 * 331 * ib_device_put() releases reference to the IB device to allow it to be 332 * unregistered and eventually free. 333 */ 334 void ib_device_put(struct ib_device *device) 335 { 336 if (refcount_dec_and_test(&device->refcount)) 337 complete(&device->unreg_completion); 338 } 339 EXPORT_SYMBOL(ib_device_put); 340 341 static struct ib_device *__ib_device_get_by_name(const char *name) 342 { 343 struct ib_device *device; 344 unsigned long index; 345 346 xa_for_each (&devices, index, device) 347 if (!strcmp(name, dev_name(&device->dev))) 348 return device; 349 350 return NULL; 351 } 352 353 /** 354 * ib_device_get_by_name - Find an IB device by name 355 * @name: The name to look for 356 * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 357 * 358 * Find and hold an ib_device by its name. The caller must call 359 * ib_device_put() on the returned pointer. 360 */ 361 struct ib_device *ib_device_get_by_name(const char *name, 362 enum rdma_driver_id driver_id) 363 { 364 struct ib_device *device; 365 366 down_read(&devices_rwsem); 367 device = __ib_device_get_by_name(name); 368 if (device && driver_id != RDMA_DRIVER_UNKNOWN && 369 device->ops.driver_id != driver_id) 370 device = NULL; 371 372 if (device) { 373 if (!ib_device_try_get(device)) 374 device = NULL; 375 } 376 up_read(&devices_rwsem); 377 return device; 378 } 379 EXPORT_SYMBOL(ib_device_get_by_name); 380 381 static int rename_compat_devs(struct ib_device *device) 382 { 383 struct ib_core_device *cdev; 384 unsigned long index; 385 int ret = 0; 386 387 mutex_lock(&device->compat_devs_mutex); 388 xa_for_each (&device->compat_devs, index, cdev) { 389 ret = device_rename(&cdev->dev, dev_name(&device->dev)); 390 if (ret) { 391 dev_warn(&cdev->dev, 392 "Fail to rename compatdev to new name %s\n", 393 dev_name(&device->dev)); 394 break; 395 } 396 } 397 mutex_unlock(&device->compat_devs_mutex); 398 return ret; 399 } 400 401 int ib_device_rename(struct ib_device *ibdev, const char *name) 402 { 403 unsigned long index; 404 void *client_data; 405 int ret; 406 407 down_write(&devices_rwsem); 408 if (!strcmp(name, dev_name(&ibdev->dev))) { 409 up_write(&devices_rwsem); 410 return 0; 411 } 412 413 if (__ib_device_get_by_name(name)) { 414 up_write(&devices_rwsem); 415 return -EEXIST; 416 } 417 418 ret = device_rename(&ibdev->dev, name); 419 if (ret) { 420 up_write(&devices_rwsem); 421 return ret; 422 } 423 424 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 425 ret = rename_compat_devs(ibdev); 426 427 downgrade_write(&devices_rwsem); 428 down_read(&ibdev->client_data_rwsem); 429 xan_for_each_marked(&ibdev->client_data, index, client_data, 430 CLIENT_DATA_REGISTERED) { 431 struct ib_client *client = xa_load(&clients, index); 432 433 if (!client || !client->rename) 434 continue; 435 436 client->rename(ibdev, client_data); 437 } 438 up_read(&ibdev->client_data_rwsem); 439 up_read(&devices_rwsem); 440 return 0; 441 } 442 443 int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim) 444 { 445 if (use_dim > 1) 446 return -EINVAL; 447 ibdev->use_cq_dim = use_dim; 448 449 return 0; 450 } 451 452 static int alloc_name(struct ib_device *ibdev, const char *name) 453 { 454 struct ib_device *device; 455 unsigned long index; 456 struct ida inuse; 457 int rc; 458 int i; 459 460 lockdep_assert_held_write(&devices_rwsem); 461 ida_init(&inuse); 462 xa_for_each (&devices, index, device) { 463 char buf[IB_DEVICE_NAME_MAX]; 464 465 if (sscanf(dev_name(&device->dev), name, &i) != 1) 466 continue; 467 if (i < 0 || i >= INT_MAX) 468 continue; 469 snprintf(buf, sizeof buf, name, i); 470 if (strcmp(buf, dev_name(&device->dev)) != 0) 471 continue; 472 473 rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL); 474 if (rc < 0) 475 goto out; 476 } 477 478 rc = ida_alloc(&inuse, GFP_KERNEL); 479 if (rc < 0) 480 goto out; 481 482 rc = dev_set_name(&ibdev->dev, name, rc); 483 out: 484 ida_destroy(&inuse); 485 return rc; 486 } 487 488 static void ib_device_release(struct device *device) 489 { 490 struct ib_device *dev = container_of(device, struct ib_device, dev); 491 492 free_netdevs(dev); 493 WARN_ON(refcount_read(&dev->refcount)); 494 if (dev->port_data) { 495 ib_cache_release_one(dev); 496 ib_security_release_port_pkey_list(dev); 497 rdma_counter_release(dev); 498 kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu, 499 pdata[0]), 500 rcu_head); 501 } 502 503 mutex_destroy(&dev->unregistration_lock); 504 mutex_destroy(&dev->compat_devs_mutex); 505 506 xa_destroy(&dev->compat_devs); 507 xa_destroy(&dev->client_data); 508 kfree_rcu(dev, rcu_head); 509 } 510 511 static int ib_device_uevent(struct device *device, 512 struct kobj_uevent_env *env) 513 { 514 if (add_uevent_var(env, "NAME=%s", dev_name(device))) 515 return -ENOMEM; 516 517 /* 518 * It would be nice to pass the node GUID with the event... 519 */ 520 521 return 0; 522 } 523 524 static const void *net_namespace(struct device *d) 525 { 526 struct ib_core_device *coredev = 527 container_of(d, struct ib_core_device, dev); 528 529 return read_pnet(&coredev->rdma_net); 530 } 531 532 static struct class ib_class = { 533 .name = "infiniband", 534 .dev_release = ib_device_release, 535 .dev_uevent = ib_device_uevent, 536 .ns_type = &net_ns_type_operations, 537 .namespace = net_namespace, 538 }; 539 540 static void rdma_init_coredev(struct ib_core_device *coredev, 541 struct ib_device *dev, struct net *net) 542 { 543 /* This BUILD_BUG_ON is intended to catch layout change 544 * of union of ib_core_device and device. 545 * dev must be the first element as ib_core and providers 546 * driver uses it. Adding anything in ib_core_device before 547 * device will break this assumption. 548 */ 549 BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) != 550 offsetof(struct ib_device, dev)); 551 552 coredev->dev.class = &ib_class; 553 coredev->dev.groups = dev->groups; 554 device_initialize(&coredev->dev); 555 coredev->owner = dev; 556 INIT_LIST_HEAD(&coredev->port_list); 557 write_pnet(&coredev->rdma_net, net); 558 } 559 560 /** 561 * _ib_alloc_device - allocate an IB device struct 562 * @size:size of structure to allocate 563 * 564 * Low-level drivers should use ib_alloc_device() to allocate &struct 565 * ib_device. @size is the size of the structure to be allocated, 566 * including any private data used by the low-level driver. 567 * ib_dealloc_device() must be used to free structures allocated with 568 * ib_alloc_device(). 569 */ 570 struct ib_device *_ib_alloc_device(size_t size) 571 { 572 struct ib_device *device; 573 unsigned int i; 574 575 if (WARN_ON(size < sizeof(struct ib_device))) 576 return NULL; 577 578 device = kzalloc(size, GFP_KERNEL); 579 if (!device) 580 return NULL; 581 582 if (rdma_restrack_init(device)) { 583 kfree(device); 584 return NULL; 585 } 586 587 device->groups[0] = &ib_dev_attr_group; 588 rdma_init_coredev(&device->coredev, device, &init_net); 589 590 INIT_LIST_HEAD(&device->event_handler_list); 591 spin_lock_init(&device->qp_open_list_lock); 592 init_rwsem(&device->event_handler_rwsem); 593 mutex_init(&device->unregistration_lock); 594 /* 595 * client_data needs to be alloc because we don't want our mark to be 596 * destroyed if the user stores NULL in the client data. 597 */ 598 xa_init_flags(&device->client_data, XA_FLAGS_ALLOC); 599 init_rwsem(&device->client_data_rwsem); 600 xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC); 601 mutex_init(&device->compat_devs_mutex); 602 init_completion(&device->unreg_completion); 603 INIT_WORK(&device->unregistration_work, ib_unregister_work); 604 605 spin_lock_init(&device->cq_pools_lock); 606 for (i = 0; i < ARRAY_SIZE(device->cq_pools); i++) 607 INIT_LIST_HEAD(&device->cq_pools[i]); 608 609 device->uverbs_cmd_mask = 610 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | 611 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | 612 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | 613 BIT_ULL(IB_USER_VERBS_CMD_CLOSE_XRCD) | 614 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | 615 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 616 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | 617 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | 618 BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ) | 619 BIT_ULL(IB_USER_VERBS_CMD_CREATE_XSRQ) | 620 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | 621 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | 622 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | 623 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | 624 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | 625 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | 626 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ) | 627 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST) | 628 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | 629 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | 630 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ) | 631 BIT_ULL(IB_USER_VERBS_CMD_OPEN_QP) | 632 BIT_ULL(IB_USER_VERBS_CMD_OPEN_XRCD) | 633 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | 634 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | 635 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | 636 BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ) | 637 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | 638 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) | 639 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ); 640 return device; 641 } 642 EXPORT_SYMBOL(_ib_alloc_device); 643 644 /** 645 * ib_dealloc_device - free an IB device struct 646 * @device:structure to free 647 * 648 * Free a structure allocated with ib_alloc_device(). 649 */ 650 void ib_dealloc_device(struct ib_device *device) 651 { 652 if (device->ops.dealloc_driver) 653 device->ops.dealloc_driver(device); 654 655 /* 656 * ib_unregister_driver() requires all devices to remain in the xarray 657 * while their ops are callable. The last op we call is dealloc_driver 658 * above. This is needed to create a fence on op callbacks prior to 659 * allowing the driver module to unload. 660 */ 661 down_write(&devices_rwsem); 662 if (xa_load(&devices, device->index) == device) 663 xa_erase(&devices, device->index); 664 up_write(&devices_rwsem); 665 666 /* Expedite releasing netdev references */ 667 free_netdevs(device); 668 669 WARN_ON(!xa_empty(&device->compat_devs)); 670 WARN_ON(!xa_empty(&device->client_data)); 671 WARN_ON(refcount_read(&device->refcount)); 672 rdma_restrack_clean(device); 673 /* Balances with device_initialize */ 674 put_device(&device->dev); 675 } 676 EXPORT_SYMBOL(ib_dealloc_device); 677 678 /* 679 * add_client_context() and remove_client_context() must be safe against 680 * parallel calls on the same device - registration/unregistration of both the 681 * device and client can be occurring in parallel. 682 * 683 * The routines need to be a fence, any caller must not return until the add 684 * or remove is fully completed. 685 */ 686 static int add_client_context(struct ib_device *device, 687 struct ib_client *client) 688 { 689 int ret = 0; 690 691 if (!device->kverbs_provider && !client->no_kverbs_req) 692 return 0; 693 694 down_write(&device->client_data_rwsem); 695 /* 696 * So long as the client is registered hold both the client and device 697 * unregistration locks. 698 */ 699 if (!refcount_inc_not_zero(&client->uses)) 700 goto out_unlock; 701 refcount_inc(&device->refcount); 702 703 /* 704 * Another caller to add_client_context got here first and has already 705 * completely initialized context. 706 */ 707 if (xa_get_mark(&device->client_data, client->client_id, 708 CLIENT_DATA_REGISTERED)) 709 goto out; 710 711 ret = xa_err(xa_store(&device->client_data, client->client_id, NULL, 712 GFP_KERNEL)); 713 if (ret) 714 goto out; 715 downgrade_write(&device->client_data_rwsem); 716 if (client->add) { 717 if (client->add(device)) { 718 /* 719 * If a client fails to add then the error code is 720 * ignored, but we won't call any more ops on this 721 * client. 722 */ 723 xa_erase(&device->client_data, client->client_id); 724 up_read(&device->client_data_rwsem); 725 ib_device_put(device); 726 ib_client_put(client); 727 return 0; 728 } 729 } 730 731 /* Readers shall not see a client until add has been completed */ 732 xa_set_mark(&device->client_data, client->client_id, 733 CLIENT_DATA_REGISTERED); 734 up_read(&device->client_data_rwsem); 735 return 0; 736 737 out: 738 ib_device_put(device); 739 ib_client_put(client); 740 out_unlock: 741 up_write(&device->client_data_rwsem); 742 return ret; 743 } 744 745 static void remove_client_context(struct ib_device *device, 746 unsigned int client_id) 747 { 748 struct ib_client *client; 749 void *client_data; 750 751 down_write(&device->client_data_rwsem); 752 if (!xa_get_mark(&device->client_data, client_id, 753 CLIENT_DATA_REGISTERED)) { 754 up_write(&device->client_data_rwsem); 755 return; 756 } 757 client_data = xa_load(&device->client_data, client_id); 758 xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED); 759 client = xa_load(&clients, client_id); 760 up_write(&device->client_data_rwsem); 761 762 /* 763 * Notice we cannot be holding any exclusive locks when calling the 764 * remove callback as the remove callback can recurse back into any 765 * public functions in this module and thus try for any locks those 766 * functions take. 767 * 768 * For this reason clients and drivers should not call the 769 * unregistration functions will holdling any locks. 770 */ 771 if (client->remove) 772 client->remove(device, client_data); 773 774 xa_erase(&device->client_data, client_id); 775 ib_device_put(device); 776 ib_client_put(client); 777 } 778 779 static int alloc_port_data(struct ib_device *device) 780 { 781 struct ib_port_data_rcu *pdata_rcu; 782 unsigned int port; 783 784 if (device->port_data) 785 return 0; 786 787 /* This can only be called once the physical port range is defined */ 788 if (WARN_ON(!device->phys_port_cnt)) 789 return -EINVAL; 790 791 /* 792 * device->port_data is indexed directly by the port number to make 793 * access to this data as efficient as possible. 794 * 795 * Therefore port_data is declared as a 1 based array with potential 796 * empty slots at the beginning. 797 */ 798 pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata, 799 rdma_end_port(device) + 1), 800 GFP_KERNEL); 801 if (!pdata_rcu) 802 return -ENOMEM; 803 /* 804 * The rcu_head is put in front of the port data array and the stored 805 * pointer is adjusted since we never need to see that member until 806 * kfree_rcu. 807 */ 808 device->port_data = pdata_rcu->pdata; 809 810 rdma_for_each_port (device, port) { 811 struct ib_port_data *pdata = &device->port_data[port]; 812 813 pdata->ib_dev = device; 814 spin_lock_init(&pdata->pkey_list_lock); 815 INIT_LIST_HEAD(&pdata->pkey_list); 816 spin_lock_init(&pdata->netdev_lock); 817 INIT_HLIST_NODE(&pdata->ndev_hash_link); 818 } 819 return 0; 820 } 821 822 static int verify_immutable(const struct ib_device *dev, u8 port) 823 { 824 return WARN_ON(!rdma_cap_ib_mad(dev, port) && 825 rdma_max_mad_size(dev, port) != 0); 826 } 827 828 static int setup_port_data(struct ib_device *device) 829 { 830 unsigned int port; 831 int ret; 832 833 ret = alloc_port_data(device); 834 if (ret) 835 return ret; 836 837 rdma_for_each_port (device, port) { 838 struct ib_port_data *pdata = &device->port_data[port]; 839 840 ret = device->ops.get_port_immutable(device, port, 841 &pdata->immutable); 842 if (ret) 843 return ret; 844 845 if (verify_immutable(device, port)) 846 return -EINVAL; 847 } 848 return 0; 849 } 850 851 void ib_get_device_fw_str(struct ib_device *dev, char *str) 852 { 853 if (dev->ops.get_dev_fw_str) 854 dev->ops.get_dev_fw_str(dev, str); 855 else 856 str[0] = '\0'; 857 } 858 EXPORT_SYMBOL(ib_get_device_fw_str); 859 860 static void ib_policy_change_task(struct work_struct *work) 861 { 862 struct ib_device *dev; 863 unsigned long index; 864 865 down_read(&devices_rwsem); 866 xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 867 unsigned int i; 868 869 rdma_for_each_port (dev, i) { 870 u64 sp; 871 int ret = ib_get_cached_subnet_prefix(dev, 872 i, 873 &sp); 874 875 WARN_ONCE(ret, 876 "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 877 ret); 878 if (!ret) 879 ib_security_cache_change(dev, i, sp); 880 } 881 } 882 up_read(&devices_rwsem); 883 } 884 885 static int ib_security_change(struct notifier_block *nb, unsigned long event, 886 void *lsm_data) 887 { 888 if (event != LSM_POLICY_CHANGE) 889 return NOTIFY_DONE; 890 891 schedule_work(&ib_policy_change_work); 892 ib_mad_agent_security_change(); 893 894 return NOTIFY_OK; 895 } 896 897 static void compatdev_release(struct device *dev) 898 { 899 struct ib_core_device *cdev = 900 container_of(dev, struct ib_core_device, dev); 901 902 kfree(cdev); 903 } 904 905 static int add_one_compat_dev(struct ib_device *device, 906 struct rdma_dev_net *rnet) 907 { 908 struct ib_core_device *cdev; 909 int ret; 910 911 lockdep_assert_held(&rdma_nets_rwsem); 912 if (!ib_devices_shared_netns) 913 return 0; 914 915 /* 916 * Create and add compat device in all namespaces other than where it 917 * is currently bound to. 918 */ 919 if (net_eq(read_pnet(&rnet->net), 920 read_pnet(&device->coredev.rdma_net))) 921 return 0; 922 923 /* 924 * The first of init_net() or ib_register_device() to take the 925 * compat_devs_mutex wins and gets to add the device. Others will wait 926 * for completion here. 927 */ 928 mutex_lock(&device->compat_devs_mutex); 929 cdev = xa_load(&device->compat_devs, rnet->id); 930 if (cdev) { 931 ret = 0; 932 goto done; 933 } 934 ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL); 935 if (ret) 936 goto done; 937 938 cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); 939 if (!cdev) { 940 ret = -ENOMEM; 941 goto cdev_err; 942 } 943 944 cdev->dev.parent = device->dev.parent; 945 rdma_init_coredev(cdev, device, read_pnet(&rnet->net)); 946 cdev->dev.release = compatdev_release; 947 ret = dev_set_name(&cdev->dev, "%s", dev_name(&device->dev)); 948 if (ret) 949 goto add_err; 950 951 ret = device_add(&cdev->dev); 952 if (ret) 953 goto add_err; 954 ret = ib_setup_port_attrs(cdev); 955 if (ret) 956 goto port_err; 957 958 ret = xa_err(xa_store(&device->compat_devs, rnet->id, 959 cdev, GFP_KERNEL)); 960 if (ret) 961 goto insert_err; 962 963 mutex_unlock(&device->compat_devs_mutex); 964 return 0; 965 966 insert_err: 967 ib_free_port_attrs(cdev); 968 port_err: 969 device_del(&cdev->dev); 970 add_err: 971 put_device(&cdev->dev); 972 cdev_err: 973 xa_release(&device->compat_devs, rnet->id); 974 done: 975 mutex_unlock(&device->compat_devs_mutex); 976 return ret; 977 } 978 979 static void remove_one_compat_dev(struct ib_device *device, u32 id) 980 { 981 struct ib_core_device *cdev; 982 983 mutex_lock(&device->compat_devs_mutex); 984 cdev = xa_erase(&device->compat_devs, id); 985 mutex_unlock(&device->compat_devs_mutex); 986 if (cdev) { 987 ib_free_port_attrs(cdev); 988 device_del(&cdev->dev); 989 put_device(&cdev->dev); 990 } 991 } 992 993 static void remove_compat_devs(struct ib_device *device) 994 { 995 struct ib_core_device *cdev; 996 unsigned long index; 997 998 xa_for_each (&device->compat_devs, index, cdev) 999 remove_one_compat_dev(device, index); 1000 } 1001 1002 static int add_compat_devs(struct ib_device *device) 1003 { 1004 struct rdma_dev_net *rnet; 1005 unsigned long index; 1006 int ret = 0; 1007 1008 lockdep_assert_held(&devices_rwsem); 1009 1010 down_read(&rdma_nets_rwsem); 1011 xa_for_each (&rdma_nets, index, rnet) { 1012 ret = add_one_compat_dev(device, rnet); 1013 if (ret) 1014 break; 1015 } 1016 up_read(&rdma_nets_rwsem); 1017 return ret; 1018 } 1019 1020 static void remove_all_compat_devs(void) 1021 { 1022 struct ib_compat_device *cdev; 1023 struct ib_device *dev; 1024 unsigned long index; 1025 1026 down_read(&devices_rwsem); 1027 xa_for_each (&devices, index, dev) { 1028 unsigned long c_index = 0; 1029 1030 /* Hold nets_rwsem so that any other thread modifying this 1031 * system param can sync with this thread. 1032 */ 1033 down_read(&rdma_nets_rwsem); 1034 xa_for_each (&dev->compat_devs, c_index, cdev) 1035 remove_one_compat_dev(dev, c_index); 1036 up_read(&rdma_nets_rwsem); 1037 } 1038 up_read(&devices_rwsem); 1039 } 1040 1041 static int add_all_compat_devs(void) 1042 { 1043 struct rdma_dev_net *rnet; 1044 struct ib_device *dev; 1045 unsigned long index; 1046 int ret = 0; 1047 1048 down_read(&devices_rwsem); 1049 xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 1050 unsigned long net_index = 0; 1051 1052 /* Hold nets_rwsem so that any other thread modifying this 1053 * system param can sync with this thread. 1054 */ 1055 down_read(&rdma_nets_rwsem); 1056 xa_for_each (&rdma_nets, net_index, rnet) { 1057 ret = add_one_compat_dev(dev, rnet); 1058 if (ret) 1059 break; 1060 } 1061 up_read(&rdma_nets_rwsem); 1062 } 1063 up_read(&devices_rwsem); 1064 if (ret) 1065 remove_all_compat_devs(); 1066 return ret; 1067 } 1068 1069 int rdma_compatdev_set(u8 enable) 1070 { 1071 struct rdma_dev_net *rnet; 1072 unsigned long index; 1073 int ret = 0; 1074 1075 down_write(&rdma_nets_rwsem); 1076 if (ib_devices_shared_netns == enable) { 1077 up_write(&rdma_nets_rwsem); 1078 return 0; 1079 } 1080 1081 /* enable/disable of compat devices is not supported 1082 * when more than default init_net exists. 1083 */ 1084 xa_for_each (&rdma_nets, index, rnet) { 1085 ret++; 1086 break; 1087 } 1088 if (!ret) 1089 ib_devices_shared_netns = enable; 1090 up_write(&rdma_nets_rwsem); 1091 if (ret) 1092 return -EBUSY; 1093 1094 if (enable) 1095 ret = add_all_compat_devs(); 1096 else 1097 remove_all_compat_devs(); 1098 return ret; 1099 } 1100 1101 static void rdma_dev_exit_net(struct net *net) 1102 { 1103 struct rdma_dev_net *rnet = rdma_net_to_dev_net(net); 1104 struct ib_device *dev; 1105 unsigned long index; 1106 int ret; 1107 1108 down_write(&rdma_nets_rwsem); 1109 /* 1110 * Prevent the ID from being re-used and hide the id from xa_for_each. 1111 */ 1112 ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL)); 1113 WARN_ON(ret); 1114 up_write(&rdma_nets_rwsem); 1115 1116 down_read(&devices_rwsem); 1117 xa_for_each (&devices, index, dev) { 1118 get_device(&dev->dev); 1119 /* 1120 * Release the devices_rwsem so that pontentially blocking 1121 * device_del, doesn't hold the devices_rwsem for too long. 1122 */ 1123 up_read(&devices_rwsem); 1124 1125 remove_one_compat_dev(dev, rnet->id); 1126 1127 /* 1128 * If the real device is in the NS then move it back to init. 1129 */ 1130 rdma_dev_change_netns(dev, net, &init_net); 1131 1132 put_device(&dev->dev); 1133 down_read(&devices_rwsem); 1134 } 1135 up_read(&devices_rwsem); 1136 1137 rdma_nl_net_exit(rnet); 1138 xa_erase(&rdma_nets, rnet->id); 1139 } 1140 1141 static __net_init int rdma_dev_init_net(struct net *net) 1142 { 1143 struct rdma_dev_net *rnet = rdma_net_to_dev_net(net); 1144 unsigned long index; 1145 struct ib_device *dev; 1146 int ret; 1147 1148 write_pnet(&rnet->net, net); 1149 1150 ret = rdma_nl_net_init(rnet); 1151 if (ret) 1152 return ret; 1153 1154 /* No need to create any compat devices in default init_net. */ 1155 if (net_eq(net, &init_net)) 1156 return 0; 1157 1158 ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL); 1159 if (ret) { 1160 rdma_nl_net_exit(rnet); 1161 return ret; 1162 } 1163 1164 down_read(&devices_rwsem); 1165 xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 1166 /* Hold nets_rwsem so that netlink command cannot change 1167 * system configuration for device sharing mode. 1168 */ 1169 down_read(&rdma_nets_rwsem); 1170 ret = add_one_compat_dev(dev, rnet); 1171 up_read(&rdma_nets_rwsem); 1172 if (ret) 1173 break; 1174 } 1175 up_read(&devices_rwsem); 1176 1177 if (ret) 1178 rdma_dev_exit_net(net); 1179 1180 return ret; 1181 } 1182 1183 /* 1184 * Assign the unique string device name and the unique device index. This is 1185 * undone by ib_dealloc_device. 1186 */ 1187 static int assign_name(struct ib_device *device, const char *name) 1188 { 1189 static u32 last_id; 1190 int ret; 1191 1192 down_write(&devices_rwsem); 1193 /* Assign a unique name to the device */ 1194 if (strchr(name, '%')) 1195 ret = alloc_name(device, name); 1196 else 1197 ret = dev_set_name(&device->dev, name); 1198 if (ret) 1199 goto out; 1200 1201 if (__ib_device_get_by_name(dev_name(&device->dev))) { 1202 ret = -ENFILE; 1203 goto out; 1204 } 1205 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); 1206 1207 ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, 1208 &last_id, GFP_KERNEL); 1209 if (ret > 0) 1210 ret = 0; 1211 1212 out: 1213 up_write(&devices_rwsem); 1214 return ret; 1215 } 1216 1217 /* 1218 * setup_device() allocates memory and sets up data that requires calling the 1219 * device ops, this is the only reason these actions are not done during 1220 * ib_alloc_device. It is undone by ib_dealloc_device(). 1221 */ 1222 static int setup_device(struct ib_device *device) 1223 { 1224 struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 1225 int ret; 1226 1227 ib_device_check_mandatory(device); 1228 1229 ret = setup_port_data(device); 1230 if (ret) { 1231 dev_warn(&device->dev, "Couldn't create per-port data\n"); 1232 return ret; 1233 } 1234 1235 memset(&device->attrs, 0, sizeof(device->attrs)); 1236 ret = device->ops.query_device(device, &device->attrs, &uhw); 1237 if (ret) { 1238 dev_warn(&device->dev, 1239 "Couldn't query the device attributes\n"); 1240 return ret; 1241 } 1242 1243 return 0; 1244 } 1245 1246 static void disable_device(struct ib_device *device) 1247 { 1248 u32 cid; 1249 1250 WARN_ON(!refcount_read(&device->refcount)); 1251 1252 down_write(&devices_rwsem); 1253 xa_clear_mark(&devices, device->index, DEVICE_REGISTERED); 1254 up_write(&devices_rwsem); 1255 1256 /* 1257 * Remove clients in LIFO order, see assign_client_id. This could be 1258 * more efficient if xarray learns to reverse iterate. Since no new 1259 * clients can be added to this ib_device past this point we only need 1260 * the maximum possible client_id value here. 1261 */ 1262 down_read(&clients_rwsem); 1263 cid = highest_client_id; 1264 up_read(&clients_rwsem); 1265 while (cid) { 1266 cid--; 1267 remove_client_context(device, cid); 1268 } 1269 1270 ib_cq_pool_cleanup(device); 1271 1272 /* Pairs with refcount_set in enable_device */ 1273 ib_device_put(device); 1274 wait_for_completion(&device->unreg_completion); 1275 1276 /* 1277 * compat devices must be removed after device refcount drops to zero. 1278 * Otherwise init_net() may add more compatdevs after removing compat 1279 * devices and before device is disabled. 1280 */ 1281 remove_compat_devs(device); 1282 } 1283 1284 /* 1285 * An enabled device is visible to all clients and to all the public facing 1286 * APIs that return a device pointer. This always returns with a new get, even 1287 * if it fails. 1288 */ 1289 static int enable_device_and_get(struct ib_device *device) 1290 { 1291 struct ib_client *client; 1292 unsigned long index; 1293 int ret = 0; 1294 1295 /* 1296 * One ref belongs to the xa and the other belongs to this 1297 * thread. This is needed to guard against parallel unregistration. 1298 */ 1299 refcount_set(&device->refcount, 2); 1300 down_write(&devices_rwsem); 1301 xa_set_mark(&devices, device->index, DEVICE_REGISTERED); 1302 1303 /* 1304 * By using downgrade_write() we ensure that no other thread can clear 1305 * DEVICE_REGISTERED while we are completing the client setup. 1306 */ 1307 downgrade_write(&devices_rwsem); 1308 1309 if (device->ops.enable_driver) { 1310 ret = device->ops.enable_driver(device); 1311 if (ret) 1312 goto out; 1313 } 1314 1315 down_read(&clients_rwsem); 1316 xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) { 1317 ret = add_client_context(device, client); 1318 if (ret) 1319 break; 1320 } 1321 up_read(&clients_rwsem); 1322 if (!ret) 1323 ret = add_compat_devs(device); 1324 out: 1325 up_read(&devices_rwsem); 1326 return ret; 1327 } 1328 1329 static void prevent_dealloc_device(struct ib_device *ib_dev) 1330 { 1331 } 1332 1333 /** 1334 * ib_register_device - Register an IB device with IB core 1335 * @device: Device to register 1336 * @name: unique string device name. This may include a '%' which will 1337 * cause a unique index to be added to the passed device name. 1338 * @dma_device: pointer to a DMA-capable device. If %NULL, then the IB 1339 * device will be used. In this case the caller should fully 1340 * setup the ibdev for DMA. This usually means using dma_virt_ops. 1341 * 1342 * Low-level drivers use ib_register_device() to register their 1343 * devices with the IB core. All registered clients will receive a 1344 * callback for each device that is added. @device must be allocated 1345 * with ib_alloc_device(). 1346 * 1347 * If the driver uses ops.dealloc_driver and calls any ib_unregister_device() 1348 * asynchronously then the device pointer may become freed as soon as this 1349 * function returns. 1350 */ 1351 int ib_register_device(struct ib_device *device, const char *name, 1352 struct device *dma_device) 1353 { 1354 int ret; 1355 1356 ret = assign_name(device, name); 1357 if (ret) 1358 return ret; 1359 1360 /* 1361 * If the caller does not provide a DMA capable device then the IB core 1362 * will set up ib_sge and scatterlist structures that stash the kernel 1363 * virtual address into the address field. 1364 */ 1365 WARN_ON(dma_device && !dma_device->dma_parms); 1366 device->dma_device = dma_device; 1367 1368 ret = setup_device(device); 1369 if (ret) 1370 return ret; 1371 1372 ret = ib_cache_setup_one(device); 1373 if (ret) { 1374 dev_warn(&device->dev, 1375 "Couldn't set up InfiniBand P_Key/GID cache\n"); 1376 return ret; 1377 } 1378 1379 ib_device_register_rdmacg(device); 1380 1381 rdma_counter_init(device); 1382 1383 /* 1384 * Ensure that ADD uevent is not fired because it 1385 * is too early amd device is not initialized yet. 1386 */ 1387 dev_set_uevent_suppress(&device->dev, true); 1388 ret = device_add(&device->dev); 1389 if (ret) 1390 goto cg_cleanup; 1391 1392 ret = ib_device_register_sysfs(device); 1393 if (ret) { 1394 dev_warn(&device->dev, 1395 "Couldn't register device with driver model\n"); 1396 goto dev_cleanup; 1397 } 1398 1399 ret = enable_device_and_get(device); 1400 if (ret) { 1401 void (*dealloc_fn)(struct ib_device *); 1402 1403 /* 1404 * If we hit this error flow then we don't want to 1405 * automatically dealloc the device since the caller is 1406 * expected to call ib_dealloc_device() after 1407 * ib_register_device() fails. This is tricky due to the 1408 * possibility for a parallel unregistration along with this 1409 * error flow. Since we have a refcount here we know any 1410 * parallel flow is stopped in disable_device and will see the 1411 * special dealloc_driver pointer, causing the responsibility to 1412 * ib_dealloc_device() to revert back to this thread. 1413 */ 1414 dealloc_fn = device->ops.dealloc_driver; 1415 device->ops.dealloc_driver = prevent_dealloc_device; 1416 ib_device_put(device); 1417 __ib_unregister_device(device); 1418 device->ops.dealloc_driver = dealloc_fn; 1419 dev_set_uevent_suppress(&device->dev, false); 1420 return ret; 1421 } 1422 dev_set_uevent_suppress(&device->dev, false); 1423 /* Mark for userspace that device is ready */ 1424 kobject_uevent(&device->dev.kobj, KOBJ_ADD); 1425 ib_device_put(device); 1426 1427 return 0; 1428 1429 dev_cleanup: 1430 device_del(&device->dev); 1431 cg_cleanup: 1432 dev_set_uevent_suppress(&device->dev, false); 1433 ib_device_unregister_rdmacg(device); 1434 ib_cache_cleanup_one(device); 1435 return ret; 1436 } 1437 EXPORT_SYMBOL(ib_register_device); 1438 1439 /* Callers must hold a get on the device. */ 1440 static void __ib_unregister_device(struct ib_device *ib_dev) 1441 { 1442 /* 1443 * We have a registration lock so that all the calls to unregister are 1444 * fully fenced, once any unregister returns the device is truely 1445 * unregistered even if multiple callers are unregistering it at the 1446 * same time. This also interacts with the registration flow and 1447 * provides sane semantics if register and unregister are racing. 1448 */ 1449 mutex_lock(&ib_dev->unregistration_lock); 1450 if (!refcount_read(&ib_dev->refcount)) 1451 goto out; 1452 1453 disable_device(ib_dev); 1454 1455 /* Expedite removing unregistered pointers from the hash table */ 1456 free_netdevs(ib_dev); 1457 1458 ib_device_unregister_sysfs(ib_dev); 1459 device_del(&ib_dev->dev); 1460 ib_device_unregister_rdmacg(ib_dev); 1461 ib_cache_cleanup_one(ib_dev); 1462 1463 /* 1464 * Drivers using the new flow may not call ib_dealloc_device except 1465 * in error unwind prior to registration success. 1466 */ 1467 if (ib_dev->ops.dealloc_driver && 1468 ib_dev->ops.dealloc_driver != prevent_dealloc_device) { 1469 WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1); 1470 ib_dealloc_device(ib_dev); 1471 } 1472 out: 1473 mutex_unlock(&ib_dev->unregistration_lock); 1474 } 1475 1476 /** 1477 * ib_unregister_device - Unregister an IB device 1478 * @ib_dev: The device to unregister 1479 * 1480 * Unregister an IB device. All clients will receive a remove callback. 1481 * 1482 * Callers should call this routine only once, and protect against races with 1483 * registration. Typically it should only be called as part of a remove 1484 * callback in an implementation of driver core's struct device_driver and 1485 * related. 1486 * 1487 * If ops.dealloc_driver is used then ib_dev will be freed upon return from 1488 * this function. 1489 */ 1490 void ib_unregister_device(struct ib_device *ib_dev) 1491 { 1492 get_device(&ib_dev->dev); 1493 __ib_unregister_device(ib_dev); 1494 put_device(&ib_dev->dev); 1495 } 1496 EXPORT_SYMBOL(ib_unregister_device); 1497 1498 /** 1499 * ib_unregister_device_and_put - Unregister a device while holding a 'get' 1500 * @ib_dev: The device to unregister 1501 * 1502 * This is the same as ib_unregister_device(), except it includes an internal 1503 * ib_device_put() that should match a 'get' obtained by the caller. 1504 * 1505 * It is safe to call this routine concurrently from multiple threads while 1506 * holding the 'get'. When the function returns the device is fully 1507 * unregistered. 1508 * 1509 * Drivers using this flow MUST use the driver_unregister callback to clean up 1510 * their resources associated with the device and dealloc it. 1511 */ 1512 void ib_unregister_device_and_put(struct ib_device *ib_dev) 1513 { 1514 WARN_ON(!ib_dev->ops.dealloc_driver); 1515 get_device(&ib_dev->dev); 1516 ib_device_put(ib_dev); 1517 __ib_unregister_device(ib_dev); 1518 put_device(&ib_dev->dev); 1519 } 1520 EXPORT_SYMBOL(ib_unregister_device_and_put); 1521 1522 /** 1523 * ib_unregister_driver - Unregister all IB devices for a driver 1524 * @driver_id: The driver to unregister 1525 * 1526 * This implements a fence for device unregistration. It only returns once all 1527 * devices associated with the driver_id have fully completed their 1528 * unregistration and returned from ib_unregister_device*(). 1529 * 1530 * If device's are not yet unregistered it goes ahead and starts unregistering 1531 * them. 1532 * 1533 * This does not block creation of new devices with the given driver_id, that 1534 * is the responsibility of the caller. 1535 */ 1536 void ib_unregister_driver(enum rdma_driver_id driver_id) 1537 { 1538 struct ib_device *ib_dev; 1539 unsigned long index; 1540 1541 down_read(&devices_rwsem); 1542 xa_for_each (&devices, index, ib_dev) { 1543 if (ib_dev->ops.driver_id != driver_id) 1544 continue; 1545 1546 get_device(&ib_dev->dev); 1547 up_read(&devices_rwsem); 1548 1549 WARN_ON(!ib_dev->ops.dealloc_driver); 1550 __ib_unregister_device(ib_dev); 1551 1552 put_device(&ib_dev->dev); 1553 down_read(&devices_rwsem); 1554 } 1555 up_read(&devices_rwsem); 1556 } 1557 EXPORT_SYMBOL(ib_unregister_driver); 1558 1559 static void ib_unregister_work(struct work_struct *work) 1560 { 1561 struct ib_device *ib_dev = 1562 container_of(work, struct ib_device, unregistration_work); 1563 1564 __ib_unregister_device(ib_dev); 1565 put_device(&ib_dev->dev); 1566 } 1567 1568 /** 1569 * ib_unregister_device_queued - Unregister a device using a work queue 1570 * @ib_dev: The device to unregister 1571 * 1572 * This schedules an asynchronous unregistration using a WQ for the device. A 1573 * driver should use this to avoid holding locks while doing unregistration, 1574 * such as holding the RTNL lock. 1575 * 1576 * Drivers using this API must use ib_unregister_driver before module unload 1577 * to ensure that all scheduled unregistrations have completed. 1578 */ 1579 void ib_unregister_device_queued(struct ib_device *ib_dev) 1580 { 1581 WARN_ON(!refcount_read(&ib_dev->refcount)); 1582 WARN_ON(!ib_dev->ops.dealloc_driver); 1583 get_device(&ib_dev->dev); 1584 if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work)) 1585 put_device(&ib_dev->dev); 1586 } 1587 EXPORT_SYMBOL(ib_unregister_device_queued); 1588 1589 /* 1590 * The caller must pass in a device that has the kref held and the refcount 1591 * released. If the device is in cur_net and still registered then it is moved 1592 * into net. 1593 */ 1594 static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net, 1595 struct net *net) 1596 { 1597 int ret2 = -EINVAL; 1598 int ret; 1599 1600 mutex_lock(&device->unregistration_lock); 1601 1602 /* 1603 * If a device not under ib_device_get() or if the unregistration_lock 1604 * is not held, the namespace can be changed, or it can be unregistered. 1605 * Check again under the lock. 1606 */ 1607 if (refcount_read(&device->refcount) == 0 || 1608 !net_eq(cur_net, read_pnet(&device->coredev.rdma_net))) { 1609 ret = -ENODEV; 1610 goto out; 1611 } 1612 1613 kobject_uevent(&device->dev.kobj, KOBJ_REMOVE); 1614 disable_device(device); 1615 1616 /* 1617 * At this point no one can be using the device, so it is safe to 1618 * change the namespace. 1619 */ 1620 write_pnet(&device->coredev.rdma_net, net); 1621 1622 down_read(&devices_rwsem); 1623 /* 1624 * Currently rdma devices are system wide unique. So the device name 1625 * is guaranteed free in the new namespace. Publish the new namespace 1626 * at the sysfs level. 1627 */ 1628 ret = device_rename(&device->dev, dev_name(&device->dev)); 1629 up_read(&devices_rwsem); 1630 if (ret) { 1631 dev_warn(&device->dev, 1632 "%s: Couldn't rename device after namespace change\n", 1633 __func__); 1634 /* Try and put things back and re-enable the device */ 1635 write_pnet(&device->coredev.rdma_net, cur_net); 1636 } 1637 1638 ret2 = enable_device_and_get(device); 1639 if (ret2) { 1640 /* 1641 * This shouldn't really happen, but if it does, let the user 1642 * retry at later point. So don't disable the device. 1643 */ 1644 dev_warn(&device->dev, 1645 "%s: Couldn't re-enable device after namespace change\n", 1646 __func__); 1647 } 1648 kobject_uevent(&device->dev.kobj, KOBJ_ADD); 1649 1650 ib_device_put(device); 1651 out: 1652 mutex_unlock(&device->unregistration_lock); 1653 if (ret) 1654 return ret; 1655 return ret2; 1656 } 1657 1658 int ib_device_set_netns_put(struct sk_buff *skb, 1659 struct ib_device *dev, u32 ns_fd) 1660 { 1661 struct net *net; 1662 int ret; 1663 1664 net = get_net_ns_by_fd(ns_fd); 1665 if (IS_ERR(net)) { 1666 ret = PTR_ERR(net); 1667 goto net_err; 1668 } 1669 1670 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) { 1671 ret = -EPERM; 1672 goto ns_err; 1673 } 1674 1675 /* 1676 * Currently supported only for those providers which support 1677 * disassociation and don't do port specific sysfs init. Once a 1678 * port_cleanup infrastructure is implemented, this limitation will be 1679 * removed. 1680 */ 1681 if (!dev->ops.disassociate_ucontext || dev->ops.init_port || 1682 ib_devices_shared_netns) { 1683 ret = -EOPNOTSUPP; 1684 goto ns_err; 1685 } 1686 1687 get_device(&dev->dev); 1688 ib_device_put(dev); 1689 ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net); 1690 put_device(&dev->dev); 1691 1692 put_net(net); 1693 return ret; 1694 1695 ns_err: 1696 put_net(net); 1697 net_err: 1698 ib_device_put(dev); 1699 return ret; 1700 } 1701 1702 static struct pernet_operations rdma_dev_net_ops = { 1703 .init = rdma_dev_init_net, 1704 .exit = rdma_dev_exit_net, 1705 .id = &rdma_dev_net_id, 1706 .size = sizeof(struct rdma_dev_net), 1707 }; 1708 1709 static int assign_client_id(struct ib_client *client) 1710 { 1711 int ret; 1712 1713 down_write(&clients_rwsem); 1714 /* 1715 * The add/remove callbacks must be called in FIFO/LIFO order. To 1716 * achieve this we assign client_ids so they are sorted in 1717 * registration order. 1718 */ 1719 client->client_id = highest_client_id; 1720 ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL); 1721 if (ret) 1722 goto out; 1723 1724 highest_client_id++; 1725 xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED); 1726 1727 out: 1728 up_write(&clients_rwsem); 1729 return ret; 1730 } 1731 1732 static void remove_client_id(struct ib_client *client) 1733 { 1734 down_write(&clients_rwsem); 1735 xa_erase(&clients, client->client_id); 1736 for (; highest_client_id; highest_client_id--) 1737 if (xa_load(&clients, highest_client_id - 1)) 1738 break; 1739 up_write(&clients_rwsem); 1740 } 1741 1742 /** 1743 * ib_register_client - Register an IB client 1744 * @client:Client to register 1745 * 1746 * Upper level users of the IB drivers can use ib_register_client() to 1747 * register callbacks for IB device addition and removal. When an IB 1748 * device is added, each registered client's add method will be called 1749 * (in the order the clients were registered), and when a device is 1750 * removed, each client's remove method will be called (in the reverse 1751 * order that clients were registered). In addition, when 1752 * ib_register_client() is called, the client will receive an add 1753 * callback for all devices already registered. 1754 */ 1755 int ib_register_client(struct ib_client *client) 1756 { 1757 struct ib_device *device; 1758 unsigned long index; 1759 int ret; 1760 1761 refcount_set(&client->uses, 1); 1762 init_completion(&client->uses_zero); 1763 ret = assign_client_id(client); 1764 if (ret) 1765 return ret; 1766 1767 down_read(&devices_rwsem); 1768 xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) { 1769 ret = add_client_context(device, client); 1770 if (ret) { 1771 up_read(&devices_rwsem); 1772 ib_unregister_client(client); 1773 return ret; 1774 } 1775 } 1776 up_read(&devices_rwsem); 1777 return 0; 1778 } 1779 EXPORT_SYMBOL(ib_register_client); 1780 1781 /** 1782 * ib_unregister_client - Unregister an IB client 1783 * @client:Client to unregister 1784 * 1785 * Upper level users use ib_unregister_client() to remove their client 1786 * registration. When ib_unregister_client() is called, the client 1787 * will receive a remove callback for each IB device still registered. 1788 * 1789 * This is a full fence, once it returns no client callbacks will be called, 1790 * or are running in another thread. 1791 */ 1792 void ib_unregister_client(struct ib_client *client) 1793 { 1794 struct ib_device *device; 1795 unsigned long index; 1796 1797 down_write(&clients_rwsem); 1798 ib_client_put(client); 1799 xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED); 1800 up_write(&clients_rwsem); 1801 1802 /* We do not want to have locks while calling client->remove() */ 1803 rcu_read_lock(); 1804 xa_for_each (&devices, index, device) { 1805 if (!ib_device_try_get(device)) 1806 continue; 1807 rcu_read_unlock(); 1808 1809 remove_client_context(device, client->client_id); 1810 1811 ib_device_put(device); 1812 rcu_read_lock(); 1813 } 1814 rcu_read_unlock(); 1815 1816 /* 1817 * remove_client_context() is not a fence, it can return even though a 1818 * removal is ongoing. Wait until all removals are completed. 1819 */ 1820 wait_for_completion(&client->uses_zero); 1821 remove_client_id(client); 1822 } 1823 EXPORT_SYMBOL(ib_unregister_client); 1824 1825 static int __ib_get_global_client_nl_info(const char *client_name, 1826 struct ib_client_nl_info *res) 1827 { 1828 struct ib_client *client; 1829 unsigned long index; 1830 int ret = -ENOENT; 1831 1832 down_read(&clients_rwsem); 1833 xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) { 1834 if (strcmp(client->name, client_name) != 0) 1835 continue; 1836 if (!client->get_global_nl_info) { 1837 ret = -EOPNOTSUPP; 1838 break; 1839 } 1840 ret = client->get_global_nl_info(res); 1841 if (WARN_ON(ret == -ENOENT)) 1842 ret = -EINVAL; 1843 if (!ret && res->cdev) 1844 get_device(res->cdev); 1845 break; 1846 } 1847 up_read(&clients_rwsem); 1848 return ret; 1849 } 1850 1851 static int __ib_get_client_nl_info(struct ib_device *ibdev, 1852 const char *client_name, 1853 struct ib_client_nl_info *res) 1854 { 1855 unsigned long index; 1856 void *client_data; 1857 int ret = -ENOENT; 1858 1859 down_read(&ibdev->client_data_rwsem); 1860 xan_for_each_marked (&ibdev->client_data, index, client_data, 1861 CLIENT_DATA_REGISTERED) { 1862 struct ib_client *client = xa_load(&clients, index); 1863 1864 if (!client || strcmp(client->name, client_name) != 0) 1865 continue; 1866 if (!client->get_nl_info) { 1867 ret = -EOPNOTSUPP; 1868 break; 1869 } 1870 ret = client->get_nl_info(ibdev, client_data, res); 1871 if (WARN_ON(ret == -ENOENT)) 1872 ret = -EINVAL; 1873 1874 /* 1875 * The cdev is guaranteed valid as long as we are inside the 1876 * client_data_rwsem as remove_one can't be called. Keep it 1877 * valid for the caller. 1878 */ 1879 if (!ret && res->cdev) 1880 get_device(res->cdev); 1881 break; 1882 } 1883 up_read(&ibdev->client_data_rwsem); 1884 1885 return ret; 1886 } 1887 1888 /** 1889 * ib_get_client_nl_info - Fetch the nl_info from a client 1890 * @device - IB device 1891 * @client_name - Name of the client 1892 * @res - Result of the query 1893 */ 1894 int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name, 1895 struct ib_client_nl_info *res) 1896 { 1897 int ret; 1898 1899 if (ibdev) 1900 ret = __ib_get_client_nl_info(ibdev, client_name, res); 1901 else 1902 ret = __ib_get_global_client_nl_info(client_name, res); 1903 #ifdef CONFIG_MODULES 1904 if (ret == -ENOENT) { 1905 request_module("rdma-client-%s", client_name); 1906 if (ibdev) 1907 ret = __ib_get_client_nl_info(ibdev, client_name, res); 1908 else 1909 ret = __ib_get_global_client_nl_info(client_name, res); 1910 } 1911 #endif 1912 if (ret) { 1913 if (ret == -ENOENT) 1914 return -EOPNOTSUPP; 1915 return ret; 1916 } 1917 1918 if (WARN_ON(!res->cdev)) 1919 return -EINVAL; 1920 return 0; 1921 } 1922 1923 /** 1924 * ib_set_client_data - Set IB client context 1925 * @device:Device to set context for 1926 * @client:Client to set context for 1927 * @data:Context to set 1928 * 1929 * ib_set_client_data() sets client context data that can be retrieved with 1930 * ib_get_client_data(). This can only be called while the client is 1931 * registered to the device, once the ib_client remove() callback returns this 1932 * cannot be called. 1933 */ 1934 void ib_set_client_data(struct ib_device *device, struct ib_client *client, 1935 void *data) 1936 { 1937 void *rc; 1938 1939 if (WARN_ON(IS_ERR(data))) 1940 data = NULL; 1941 1942 rc = xa_store(&device->client_data, client->client_id, data, 1943 GFP_KERNEL); 1944 WARN_ON(xa_is_err(rc)); 1945 } 1946 EXPORT_SYMBOL(ib_set_client_data); 1947 1948 /** 1949 * ib_register_event_handler - Register an IB event handler 1950 * @event_handler:Handler to register 1951 * 1952 * ib_register_event_handler() registers an event handler that will be 1953 * called back when asynchronous IB events occur (as defined in 1954 * chapter 11 of the InfiniBand Architecture Specification). This 1955 * callback occurs in workqueue context. 1956 */ 1957 void ib_register_event_handler(struct ib_event_handler *event_handler) 1958 { 1959 down_write(&event_handler->device->event_handler_rwsem); 1960 list_add_tail(&event_handler->list, 1961 &event_handler->device->event_handler_list); 1962 up_write(&event_handler->device->event_handler_rwsem); 1963 } 1964 EXPORT_SYMBOL(ib_register_event_handler); 1965 1966 /** 1967 * ib_unregister_event_handler - Unregister an event handler 1968 * @event_handler:Handler to unregister 1969 * 1970 * Unregister an event handler registered with 1971 * ib_register_event_handler(). 1972 */ 1973 void ib_unregister_event_handler(struct ib_event_handler *event_handler) 1974 { 1975 down_write(&event_handler->device->event_handler_rwsem); 1976 list_del(&event_handler->list); 1977 up_write(&event_handler->device->event_handler_rwsem); 1978 } 1979 EXPORT_SYMBOL(ib_unregister_event_handler); 1980 1981 void ib_dispatch_event_clients(struct ib_event *event) 1982 { 1983 struct ib_event_handler *handler; 1984 1985 down_read(&event->device->event_handler_rwsem); 1986 1987 list_for_each_entry(handler, &event->device->event_handler_list, list) 1988 handler->handler(handler, event); 1989 1990 up_read(&event->device->event_handler_rwsem); 1991 } 1992 1993 static int iw_query_port(struct ib_device *device, 1994 u8 port_num, 1995 struct ib_port_attr *port_attr) 1996 { 1997 struct in_device *inetdev; 1998 struct net_device *netdev; 1999 2000 memset(port_attr, 0, sizeof(*port_attr)); 2001 2002 netdev = ib_device_get_netdev(device, port_num); 2003 if (!netdev) 2004 return -ENODEV; 2005 2006 port_attr->max_mtu = IB_MTU_4096; 2007 port_attr->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 2008 2009 if (!netif_carrier_ok(netdev)) { 2010 port_attr->state = IB_PORT_DOWN; 2011 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 2012 } else { 2013 rcu_read_lock(); 2014 inetdev = __in_dev_get_rcu(netdev); 2015 2016 if (inetdev && inetdev->ifa_list) { 2017 port_attr->state = IB_PORT_ACTIVE; 2018 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 2019 } else { 2020 port_attr->state = IB_PORT_INIT; 2021 port_attr->phys_state = 2022 IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING; 2023 } 2024 2025 rcu_read_unlock(); 2026 } 2027 2028 dev_put(netdev); 2029 return device->ops.query_port(device, port_num, port_attr); 2030 } 2031 2032 static int __ib_query_port(struct ib_device *device, 2033 u8 port_num, 2034 struct ib_port_attr *port_attr) 2035 { 2036 union ib_gid gid = {}; 2037 int err; 2038 2039 memset(port_attr, 0, sizeof(*port_attr)); 2040 2041 err = device->ops.query_port(device, port_num, port_attr); 2042 if (err || port_attr->subnet_prefix) 2043 return err; 2044 2045 if (rdma_port_get_link_layer(device, port_num) != 2046 IB_LINK_LAYER_INFINIBAND) 2047 return 0; 2048 2049 err = device->ops.query_gid(device, port_num, 0, &gid); 2050 if (err) 2051 return err; 2052 2053 port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 2054 return 0; 2055 } 2056 2057 /** 2058 * ib_query_port - Query IB port attributes 2059 * @device:Device to query 2060 * @port_num:Port number to query 2061 * @port_attr:Port attributes 2062 * 2063 * ib_query_port() returns the attributes of a port through the 2064 * @port_attr pointer. 2065 */ 2066 int ib_query_port(struct ib_device *device, 2067 u8 port_num, 2068 struct ib_port_attr *port_attr) 2069 { 2070 if (!rdma_is_port_valid(device, port_num)) 2071 return -EINVAL; 2072 2073 if (rdma_protocol_iwarp(device, port_num)) 2074 return iw_query_port(device, port_num, port_attr); 2075 else 2076 return __ib_query_port(device, port_num, port_attr); 2077 } 2078 EXPORT_SYMBOL(ib_query_port); 2079 2080 static void add_ndev_hash(struct ib_port_data *pdata) 2081 { 2082 unsigned long flags; 2083 2084 might_sleep(); 2085 2086 spin_lock_irqsave(&ndev_hash_lock, flags); 2087 if (hash_hashed(&pdata->ndev_hash_link)) { 2088 hash_del_rcu(&pdata->ndev_hash_link); 2089 spin_unlock_irqrestore(&ndev_hash_lock, flags); 2090 /* 2091 * We cannot do hash_add_rcu after a hash_del_rcu until the 2092 * grace period 2093 */ 2094 synchronize_rcu(); 2095 spin_lock_irqsave(&ndev_hash_lock, flags); 2096 } 2097 if (pdata->netdev) 2098 hash_add_rcu(ndev_hash, &pdata->ndev_hash_link, 2099 (uintptr_t)pdata->netdev); 2100 spin_unlock_irqrestore(&ndev_hash_lock, flags); 2101 } 2102 2103 /** 2104 * ib_device_set_netdev - Associate the ib_dev with an underlying net_device 2105 * @ib_dev: Device to modify 2106 * @ndev: net_device to affiliate, may be NULL 2107 * @port: IB port the net_device is connected to 2108 * 2109 * Drivers should use this to link the ib_device to a netdev so the netdev 2110 * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be 2111 * affiliated with any port. 2112 * 2113 * The caller must ensure that the given ndev is not unregistered or 2114 * unregistering, and that either the ib_device is unregistered or 2115 * ib_device_set_netdev() is called with NULL when the ndev sends a 2116 * NETDEV_UNREGISTER event. 2117 */ 2118 int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, 2119 unsigned int port) 2120 { 2121 struct net_device *old_ndev; 2122 struct ib_port_data *pdata; 2123 unsigned long flags; 2124 int ret; 2125 2126 /* 2127 * Drivers wish to call this before ib_register_driver, so we have to 2128 * setup the port data early. 2129 */ 2130 ret = alloc_port_data(ib_dev); 2131 if (ret) 2132 return ret; 2133 2134 if (!rdma_is_port_valid(ib_dev, port)) 2135 return -EINVAL; 2136 2137 pdata = &ib_dev->port_data[port]; 2138 spin_lock_irqsave(&pdata->netdev_lock, flags); 2139 old_ndev = rcu_dereference_protected( 2140 pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 2141 if (old_ndev == ndev) { 2142 spin_unlock_irqrestore(&pdata->netdev_lock, flags); 2143 return 0; 2144 } 2145 2146 if (ndev) 2147 dev_hold(ndev); 2148 rcu_assign_pointer(pdata->netdev, ndev); 2149 spin_unlock_irqrestore(&pdata->netdev_lock, flags); 2150 2151 add_ndev_hash(pdata); 2152 if (old_ndev) 2153 dev_put(old_ndev); 2154 2155 return 0; 2156 } 2157 EXPORT_SYMBOL(ib_device_set_netdev); 2158 2159 static void free_netdevs(struct ib_device *ib_dev) 2160 { 2161 unsigned long flags; 2162 unsigned int port; 2163 2164 if (!ib_dev->port_data) 2165 return; 2166 2167 rdma_for_each_port (ib_dev, port) { 2168 struct ib_port_data *pdata = &ib_dev->port_data[port]; 2169 struct net_device *ndev; 2170 2171 spin_lock_irqsave(&pdata->netdev_lock, flags); 2172 ndev = rcu_dereference_protected( 2173 pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 2174 if (ndev) { 2175 spin_lock(&ndev_hash_lock); 2176 hash_del_rcu(&pdata->ndev_hash_link); 2177 spin_unlock(&ndev_hash_lock); 2178 2179 /* 2180 * If this is the last dev_put there is still a 2181 * synchronize_rcu before the netdev is kfreed, so we 2182 * can continue to rely on unlocked pointer 2183 * comparisons after the put 2184 */ 2185 rcu_assign_pointer(pdata->netdev, NULL); 2186 dev_put(ndev); 2187 } 2188 spin_unlock_irqrestore(&pdata->netdev_lock, flags); 2189 } 2190 } 2191 2192 struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, 2193 unsigned int port) 2194 { 2195 struct ib_port_data *pdata; 2196 struct net_device *res; 2197 2198 if (!rdma_is_port_valid(ib_dev, port)) 2199 return NULL; 2200 2201 pdata = &ib_dev->port_data[port]; 2202 2203 /* 2204 * New drivers should use ib_device_set_netdev() not the legacy 2205 * get_netdev(). 2206 */ 2207 if (ib_dev->ops.get_netdev) 2208 res = ib_dev->ops.get_netdev(ib_dev, port); 2209 else { 2210 spin_lock(&pdata->netdev_lock); 2211 res = rcu_dereference_protected( 2212 pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 2213 if (res) 2214 dev_hold(res); 2215 spin_unlock(&pdata->netdev_lock); 2216 } 2217 2218 /* 2219 * If we are starting to unregister expedite things by preventing 2220 * propagation of an unregistering netdev. 2221 */ 2222 if (res && res->reg_state != NETREG_REGISTERED) { 2223 dev_put(res); 2224 return NULL; 2225 } 2226 2227 return res; 2228 } 2229 2230 /** 2231 * ib_device_get_by_netdev - Find an IB device associated with a netdev 2232 * @ndev: netdev to locate 2233 * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 2234 * 2235 * Find and hold an ib_device that is associated with a netdev via 2236 * ib_device_set_netdev(). The caller must call ib_device_put() on the 2237 * returned pointer. 2238 */ 2239 struct ib_device *ib_device_get_by_netdev(struct net_device *ndev, 2240 enum rdma_driver_id driver_id) 2241 { 2242 struct ib_device *res = NULL; 2243 struct ib_port_data *cur; 2244 2245 rcu_read_lock(); 2246 hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link, 2247 (uintptr_t)ndev) { 2248 if (rcu_access_pointer(cur->netdev) == ndev && 2249 (driver_id == RDMA_DRIVER_UNKNOWN || 2250 cur->ib_dev->ops.driver_id == driver_id) && 2251 ib_device_try_get(cur->ib_dev)) { 2252 res = cur->ib_dev; 2253 break; 2254 } 2255 } 2256 rcu_read_unlock(); 2257 2258 return res; 2259 } 2260 EXPORT_SYMBOL(ib_device_get_by_netdev); 2261 2262 /** 2263 * ib_enum_roce_netdev - enumerate all RoCE ports 2264 * @ib_dev : IB device we want to query 2265 * @filter: Should we call the callback? 2266 * @filter_cookie: Cookie passed to filter 2267 * @cb: Callback to call for each found RoCE ports 2268 * @cookie: Cookie passed back to the callback 2269 * 2270 * Enumerates all of the physical RoCE ports of ib_dev 2271 * which are related to netdevice and calls callback() on each 2272 * device for which filter() function returns non zero. 2273 */ 2274 void ib_enum_roce_netdev(struct ib_device *ib_dev, 2275 roce_netdev_filter filter, 2276 void *filter_cookie, 2277 roce_netdev_callback cb, 2278 void *cookie) 2279 { 2280 unsigned int port; 2281 2282 rdma_for_each_port (ib_dev, port) 2283 if (rdma_protocol_roce(ib_dev, port)) { 2284 struct net_device *idev = 2285 ib_device_get_netdev(ib_dev, port); 2286 2287 if (filter(ib_dev, port, idev, filter_cookie)) 2288 cb(ib_dev, port, idev, cookie); 2289 2290 if (idev) 2291 dev_put(idev); 2292 } 2293 } 2294 2295 /** 2296 * ib_enum_all_roce_netdevs - enumerate all RoCE devices 2297 * @filter: Should we call the callback? 2298 * @filter_cookie: Cookie passed to filter 2299 * @cb: Callback to call for each found RoCE ports 2300 * @cookie: Cookie passed back to the callback 2301 * 2302 * Enumerates all RoCE devices' physical ports which are related 2303 * to netdevices and calls callback() on each device for which 2304 * filter() function returns non zero. 2305 */ 2306 void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 2307 void *filter_cookie, 2308 roce_netdev_callback cb, 2309 void *cookie) 2310 { 2311 struct ib_device *dev; 2312 unsigned long index; 2313 2314 down_read(&devices_rwsem); 2315 xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) 2316 ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 2317 up_read(&devices_rwsem); 2318 } 2319 2320 /** 2321 * ib_enum_all_devs - enumerate all ib_devices 2322 * @cb: Callback to call for each found ib_device 2323 * 2324 * Enumerates all ib_devices and calls callback() on each device. 2325 */ 2326 int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 2327 struct netlink_callback *cb) 2328 { 2329 unsigned long index; 2330 struct ib_device *dev; 2331 unsigned int idx = 0; 2332 int ret = 0; 2333 2334 down_read(&devices_rwsem); 2335 xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 2336 if (!rdma_dev_access_netns(dev, sock_net(skb->sk))) 2337 continue; 2338 2339 ret = nldev_cb(dev, skb, cb, idx); 2340 if (ret) 2341 break; 2342 idx++; 2343 } 2344 up_read(&devices_rwsem); 2345 return ret; 2346 } 2347 2348 /** 2349 * ib_query_pkey - Get P_Key table entry 2350 * @device:Device to query 2351 * @port_num:Port number to query 2352 * @index:P_Key table index to query 2353 * @pkey:Returned P_Key 2354 * 2355 * ib_query_pkey() fetches the specified P_Key table entry. 2356 */ 2357 int ib_query_pkey(struct ib_device *device, 2358 u8 port_num, u16 index, u16 *pkey) 2359 { 2360 if (!rdma_is_port_valid(device, port_num)) 2361 return -EINVAL; 2362 2363 if (!device->ops.query_pkey) 2364 return -EOPNOTSUPP; 2365 2366 return device->ops.query_pkey(device, port_num, index, pkey); 2367 } 2368 EXPORT_SYMBOL(ib_query_pkey); 2369 2370 /** 2371 * ib_modify_device - Change IB device attributes 2372 * @device:Device to modify 2373 * @device_modify_mask:Mask of attributes to change 2374 * @device_modify:New attribute values 2375 * 2376 * ib_modify_device() changes a device's attributes as specified by 2377 * the @device_modify_mask and @device_modify structure. 2378 */ 2379 int ib_modify_device(struct ib_device *device, 2380 int device_modify_mask, 2381 struct ib_device_modify *device_modify) 2382 { 2383 if (!device->ops.modify_device) 2384 return -EOPNOTSUPP; 2385 2386 return device->ops.modify_device(device, device_modify_mask, 2387 device_modify); 2388 } 2389 EXPORT_SYMBOL(ib_modify_device); 2390 2391 /** 2392 * ib_modify_port - Modifies the attributes for the specified port. 2393 * @device: The device to modify. 2394 * @port_num: The number of the port to modify. 2395 * @port_modify_mask: Mask used to specify which attributes of the port 2396 * to change. 2397 * @port_modify: New attribute values for the port. 2398 * 2399 * ib_modify_port() changes a port's attributes as specified by the 2400 * @port_modify_mask and @port_modify structure. 2401 */ 2402 int ib_modify_port(struct ib_device *device, 2403 u8 port_num, int port_modify_mask, 2404 struct ib_port_modify *port_modify) 2405 { 2406 int rc; 2407 2408 if (!rdma_is_port_valid(device, port_num)) 2409 return -EINVAL; 2410 2411 if (device->ops.modify_port) 2412 rc = device->ops.modify_port(device, port_num, 2413 port_modify_mask, 2414 port_modify); 2415 else if (rdma_protocol_roce(device, port_num) && 2416 ((port_modify->set_port_cap_mask & ~IB_PORT_CM_SUP) == 0 || 2417 (port_modify->clr_port_cap_mask & ~IB_PORT_CM_SUP) == 0)) 2418 rc = 0; 2419 else 2420 rc = -EOPNOTSUPP; 2421 return rc; 2422 } 2423 EXPORT_SYMBOL(ib_modify_port); 2424 2425 /** 2426 * ib_find_gid - Returns the port number and GID table index where 2427 * a specified GID value occurs. Its searches only for IB link layer. 2428 * @device: The device to query. 2429 * @gid: The GID value to search for. 2430 * @port_num: The port number of the device where the GID value was found. 2431 * @index: The index into the GID table where the GID was found. This 2432 * parameter may be NULL. 2433 */ 2434 int ib_find_gid(struct ib_device *device, union ib_gid *gid, 2435 u8 *port_num, u16 *index) 2436 { 2437 union ib_gid tmp_gid; 2438 unsigned int port; 2439 int ret, i; 2440 2441 rdma_for_each_port (device, port) { 2442 if (!rdma_protocol_ib(device, port)) 2443 continue; 2444 2445 for (i = 0; i < device->port_data[port].immutable.gid_tbl_len; 2446 ++i) { 2447 ret = rdma_query_gid(device, port, i, &tmp_gid); 2448 if (ret) 2449 return ret; 2450 if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 2451 *port_num = port; 2452 if (index) 2453 *index = i; 2454 return 0; 2455 } 2456 } 2457 } 2458 2459 return -ENOENT; 2460 } 2461 EXPORT_SYMBOL(ib_find_gid); 2462 2463 /** 2464 * ib_find_pkey - Returns the PKey table index where a specified 2465 * PKey value occurs. 2466 * @device: The device to query. 2467 * @port_num: The port number of the device to search for the PKey. 2468 * @pkey: The PKey value to search for. 2469 * @index: The index into the PKey table where the PKey was found. 2470 */ 2471 int ib_find_pkey(struct ib_device *device, 2472 u8 port_num, u16 pkey, u16 *index) 2473 { 2474 int ret, i; 2475 u16 tmp_pkey; 2476 int partial_ix = -1; 2477 2478 for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len; 2479 ++i) { 2480 ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 2481 if (ret) 2482 return ret; 2483 if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 2484 /* if there is full-member pkey take it.*/ 2485 if (tmp_pkey & 0x8000) { 2486 *index = i; 2487 return 0; 2488 } 2489 if (partial_ix < 0) 2490 partial_ix = i; 2491 } 2492 } 2493 2494 /*no full-member, if exists take the limited*/ 2495 if (partial_ix >= 0) { 2496 *index = partial_ix; 2497 return 0; 2498 } 2499 return -ENOENT; 2500 } 2501 EXPORT_SYMBOL(ib_find_pkey); 2502 2503 /** 2504 * ib_get_net_dev_by_params() - Return the appropriate net_dev 2505 * for a received CM request 2506 * @dev: An RDMA device on which the request has been received. 2507 * @port: Port number on the RDMA device. 2508 * @pkey: The Pkey the request came on. 2509 * @gid: A GID that the net_dev uses to communicate. 2510 * @addr: Contains the IP address that the request specified as its 2511 * destination. 2512 * 2513 */ 2514 struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 2515 u8 port, 2516 u16 pkey, 2517 const union ib_gid *gid, 2518 const struct sockaddr *addr) 2519 { 2520 struct net_device *net_dev = NULL; 2521 unsigned long index; 2522 void *client_data; 2523 2524 if (!rdma_protocol_ib(dev, port)) 2525 return NULL; 2526 2527 /* 2528 * Holding the read side guarantees that the client will not become 2529 * unregistered while we are calling get_net_dev_by_params() 2530 */ 2531 down_read(&dev->client_data_rwsem); 2532 xan_for_each_marked (&dev->client_data, index, client_data, 2533 CLIENT_DATA_REGISTERED) { 2534 struct ib_client *client = xa_load(&clients, index); 2535 2536 if (!client || !client->get_net_dev_by_params) 2537 continue; 2538 2539 net_dev = client->get_net_dev_by_params(dev, port, pkey, gid, 2540 addr, client_data); 2541 if (net_dev) 2542 break; 2543 } 2544 up_read(&dev->client_data_rwsem); 2545 2546 return net_dev; 2547 } 2548 EXPORT_SYMBOL(ib_get_net_dev_by_params); 2549 2550 void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 2551 { 2552 struct ib_device_ops *dev_ops = &dev->ops; 2553 #define SET_DEVICE_OP(ptr, name) \ 2554 do { \ 2555 if (ops->name) \ 2556 if (!((ptr)->name)) \ 2557 (ptr)->name = ops->name; \ 2558 } while (0) 2559 2560 #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name) 2561 2562 if (ops->driver_id != RDMA_DRIVER_UNKNOWN) { 2563 WARN_ON(dev_ops->driver_id != RDMA_DRIVER_UNKNOWN && 2564 dev_ops->driver_id != ops->driver_id); 2565 dev_ops->driver_id = ops->driver_id; 2566 } 2567 if (ops->owner) { 2568 WARN_ON(dev_ops->owner && dev_ops->owner != ops->owner); 2569 dev_ops->owner = ops->owner; 2570 } 2571 if (ops->uverbs_abi_ver) 2572 dev_ops->uverbs_abi_ver = ops->uverbs_abi_ver; 2573 2574 dev_ops->uverbs_no_driver_id_binding |= 2575 ops->uverbs_no_driver_id_binding; 2576 2577 SET_DEVICE_OP(dev_ops, add_gid); 2578 SET_DEVICE_OP(dev_ops, advise_mr); 2579 SET_DEVICE_OP(dev_ops, alloc_dm); 2580 SET_DEVICE_OP(dev_ops, alloc_hw_stats); 2581 SET_DEVICE_OP(dev_ops, alloc_mr); 2582 SET_DEVICE_OP(dev_ops, alloc_mr_integrity); 2583 SET_DEVICE_OP(dev_ops, alloc_mw); 2584 SET_DEVICE_OP(dev_ops, alloc_pd); 2585 SET_DEVICE_OP(dev_ops, alloc_rdma_netdev); 2586 SET_DEVICE_OP(dev_ops, alloc_ucontext); 2587 SET_DEVICE_OP(dev_ops, alloc_xrcd); 2588 SET_DEVICE_OP(dev_ops, attach_mcast); 2589 SET_DEVICE_OP(dev_ops, check_mr_status); 2590 SET_DEVICE_OP(dev_ops, counter_alloc_stats); 2591 SET_DEVICE_OP(dev_ops, counter_bind_qp); 2592 SET_DEVICE_OP(dev_ops, counter_dealloc); 2593 SET_DEVICE_OP(dev_ops, counter_unbind_qp); 2594 SET_DEVICE_OP(dev_ops, counter_update_stats); 2595 SET_DEVICE_OP(dev_ops, create_ah); 2596 SET_DEVICE_OP(dev_ops, create_counters); 2597 SET_DEVICE_OP(dev_ops, create_cq); 2598 SET_DEVICE_OP(dev_ops, create_flow); 2599 SET_DEVICE_OP(dev_ops, create_flow_action_esp); 2600 SET_DEVICE_OP(dev_ops, create_qp); 2601 SET_DEVICE_OP(dev_ops, create_rwq_ind_table); 2602 SET_DEVICE_OP(dev_ops, create_srq); 2603 SET_DEVICE_OP(dev_ops, create_user_ah); 2604 SET_DEVICE_OP(dev_ops, create_wq); 2605 SET_DEVICE_OP(dev_ops, dealloc_dm); 2606 SET_DEVICE_OP(dev_ops, dealloc_driver); 2607 SET_DEVICE_OP(dev_ops, dealloc_mw); 2608 SET_DEVICE_OP(dev_ops, dealloc_pd); 2609 SET_DEVICE_OP(dev_ops, dealloc_ucontext); 2610 SET_DEVICE_OP(dev_ops, dealloc_xrcd); 2611 SET_DEVICE_OP(dev_ops, del_gid); 2612 SET_DEVICE_OP(dev_ops, dereg_mr); 2613 SET_DEVICE_OP(dev_ops, destroy_ah); 2614 SET_DEVICE_OP(dev_ops, destroy_counters); 2615 SET_DEVICE_OP(dev_ops, destroy_cq); 2616 SET_DEVICE_OP(dev_ops, destroy_flow); 2617 SET_DEVICE_OP(dev_ops, destroy_flow_action); 2618 SET_DEVICE_OP(dev_ops, destroy_qp); 2619 SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table); 2620 SET_DEVICE_OP(dev_ops, destroy_srq); 2621 SET_DEVICE_OP(dev_ops, destroy_wq); 2622 SET_DEVICE_OP(dev_ops, detach_mcast); 2623 SET_DEVICE_OP(dev_ops, disassociate_ucontext); 2624 SET_DEVICE_OP(dev_ops, drain_rq); 2625 SET_DEVICE_OP(dev_ops, drain_sq); 2626 SET_DEVICE_OP(dev_ops, enable_driver); 2627 SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry); 2628 SET_DEVICE_OP(dev_ops, fill_res_cq_entry); 2629 SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw); 2630 SET_DEVICE_OP(dev_ops, fill_res_mr_entry); 2631 SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw); 2632 SET_DEVICE_OP(dev_ops, fill_res_qp_entry); 2633 SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw); 2634 SET_DEVICE_OP(dev_ops, fill_stat_mr_entry); 2635 SET_DEVICE_OP(dev_ops, get_dev_fw_str); 2636 SET_DEVICE_OP(dev_ops, get_dma_mr); 2637 SET_DEVICE_OP(dev_ops, get_hw_stats); 2638 SET_DEVICE_OP(dev_ops, get_link_layer); 2639 SET_DEVICE_OP(dev_ops, get_netdev); 2640 SET_DEVICE_OP(dev_ops, get_port_immutable); 2641 SET_DEVICE_OP(dev_ops, get_vector_affinity); 2642 SET_DEVICE_OP(dev_ops, get_vf_config); 2643 SET_DEVICE_OP(dev_ops, get_vf_guid); 2644 SET_DEVICE_OP(dev_ops, get_vf_stats); 2645 SET_DEVICE_OP(dev_ops, init_port); 2646 SET_DEVICE_OP(dev_ops, iw_accept); 2647 SET_DEVICE_OP(dev_ops, iw_add_ref); 2648 SET_DEVICE_OP(dev_ops, iw_connect); 2649 SET_DEVICE_OP(dev_ops, iw_create_listen); 2650 SET_DEVICE_OP(dev_ops, iw_destroy_listen); 2651 SET_DEVICE_OP(dev_ops, iw_get_qp); 2652 SET_DEVICE_OP(dev_ops, iw_reject); 2653 SET_DEVICE_OP(dev_ops, iw_rem_ref); 2654 SET_DEVICE_OP(dev_ops, map_mr_sg); 2655 SET_DEVICE_OP(dev_ops, map_mr_sg_pi); 2656 SET_DEVICE_OP(dev_ops, mmap); 2657 SET_DEVICE_OP(dev_ops, mmap_free); 2658 SET_DEVICE_OP(dev_ops, modify_ah); 2659 SET_DEVICE_OP(dev_ops, modify_cq); 2660 SET_DEVICE_OP(dev_ops, modify_device); 2661 SET_DEVICE_OP(dev_ops, modify_flow_action_esp); 2662 SET_DEVICE_OP(dev_ops, modify_port); 2663 SET_DEVICE_OP(dev_ops, modify_qp); 2664 SET_DEVICE_OP(dev_ops, modify_srq); 2665 SET_DEVICE_OP(dev_ops, modify_wq); 2666 SET_DEVICE_OP(dev_ops, peek_cq); 2667 SET_DEVICE_OP(dev_ops, poll_cq); 2668 SET_DEVICE_OP(dev_ops, post_recv); 2669 SET_DEVICE_OP(dev_ops, post_send); 2670 SET_DEVICE_OP(dev_ops, post_srq_recv); 2671 SET_DEVICE_OP(dev_ops, process_mad); 2672 SET_DEVICE_OP(dev_ops, query_ah); 2673 SET_DEVICE_OP(dev_ops, query_device); 2674 SET_DEVICE_OP(dev_ops, query_gid); 2675 SET_DEVICE_OP(dev_ops, query_pkey); 2676 SET_DEVICE_OP(dev_ops, query_port); 2677 SET_DEVICE_OP(dev_ops, query_qp); 2678 SET_DEVICE_OP(dev_ops, query_srq); 2679 SET_DEVICE_OP(dev_ops, query_ucontext); 2680 SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); 2681 SET_DEVICE_OP(dev_ops, read_counters); 2682 SET_DEVICE_OP(dev_ops, reg_dm_mr); 2683 SET_DEVICE_OP(dev_ops, reg_user_mr); 2684 SET_DEVICE_OP(dev_ops, req_ncomp_notif); 2685 SET_DEVICE_OP(dev_ops, req_notify_cq); 2686 SET_DEVICE_OP(dev_ops, rereg_user_mr); 2687 SET_DEVICE_OP(dev_ops, resize_cq); 2688 SET_DEVICE_OP(dev_ops, set_vf_guid); 2689 SET_DEVICE_OP(dev_ops, set_vf_link_state); 2690 2691 SET_OBJ_SIZE(dev_ops, ib_ah); 2692 SET_OBJ_SIZE(dev_ops, ib_counters); 2693 SET_OBJ_SIZE(dev_ops, ib_cq); 2694 SET_OBJ_SIZE(dev_ops, ib_mw); 2695 SET_OBJ_SIZE(dev_ops, ib_pd); 2696 SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table); 2697 SET_OBJ_SIZE(dev_ops, ib_srq); 2698 SET_OBJ_SIZE(dev_ops, ib_ucontext); 2699 SET_OBJ_SIZE(dev_ops, ib_xrcd); 2700 } 2701 EXPORT_SYMBOL(ib_set_device_ops); 2702 2703 #ifdef CONFIG_INFINIBAND_VIRT_DMA 2704 int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents) 2705 { 2706 struct scatterlist *s; 2707 int i; 2708 2709 for_each_sg(sg, s, nents, i) { 2710 sg_dma_address(s) = (uintptr_t)sg_virt(s); 2711 sg_dma_len(s) = s->length; 2712 } 2713 return nents; 2714 } 2715 EXPORT_SYMBOL(ib_dma_virt_map_sg); 2716 #endif /* CONFIG_INFINIBAND_VIRT_DMA */ 2717 2718 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 2719 [RDMA_NL_LS_OP_RESOLVE] = { 2720 .doit = ib_nl_handle_resolve_resp, 2721 .flags = RDMA_NL_ADMIN_PERM, 2722 }, 2723 [RDMA_NL_LS_OP_SET_TIMEOUT] = { 2724 .doit = ib_nl_handle_set_timeout, 2725 .flags = RDMA_NL_ADMIN_PERM, 2726 }, 2727 [RDMA_NL_LS_OP_IP_RESOLVE] = { 2728 .doit = ib_nl_handle_ip_res_resp, 2729 .flags = RDMA_NL_ADMIN_PERM, 2730 }, 2731 }; 2732 2733 static int __init ib_core_init(void) 2734 { 2735 int ret; 2736 2737 ib_wq = alloc_workqueue("infiniband", 0, 0); 2738 if (!ib_wq) 2739 return -ENOMEM; 2740 2741 ib_comp_wq = alloc_workqueue("ib-comp-wq", 2742 WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 2743 if (!ib_comp_wq) { 2744 ret = -ENOMEM; 2745 goto err; 2746 } 2747 2748 ib_comp_unbound_wq = 2749 alloc_workqueue("ib-comp-unb-wq", 2750 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM | 2751 WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE); 2752 if (!ib_comp_unbound_wq) { 2753 ret = -ENOMEM; 2754 goto err_comp; 2755 } 2756 2757 ret = class_register(&ib_class); 2758 if (ret) { 2759 pr_warn("Couldn't create InfiniBand device class\n"); 2760 goto err_comp_unbound; 2761 } 2762 2763 rdma_nl_init(); 2764 2765 ret = addr_init(); 2766 if (ret) { 2767 pr_warn("Couldn't init IB address resolution\n"); 2768 goto err_ibnl; 2769 } 2770 2771 ret = ib_mad_init(); 2772 if (ret) { 2773 pr_warn("Couldn't init IB MAD\n"); 2774 goto err_addr; 2775 } 2776 2777 ret = ib_sa_init(); 2778 if (ret) { 2779 pr_warn("Couldn't init SA\n"); 2780 goto err_mad; 2781 } 2782 2783 ret = register_blocking_lsm_notifier(&ibdev_lsm_nb); 2784 if (ret) { 2785 pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 2786 goto err_sa; 2787 } 2788 2789 ret = register_pernet_device(&rdma_dev_net_ops); 2790 if (ret) { 2791 pr_warn("Couldn't init compat dev. ret %d\n", ret); 2792 goto err_compat; 2793 } 2794 2795 nldev_init(); 2796 rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 2797 roce_gid_mgmt_init(); 2798 2799 return 0; 2800 2801 err_compat: 2802 unregister_blocking_lsm_notifier(&ibdev_lsm_nb); 2803 err_sa: 2804 ib_sa_cleanup(); 2805 err_mad: 2806 ib_mad_cleanup(); 2807 err_addr: 2808 addr_cleanup(); 2809 err_ibnl: 2810 class_unregister(&ib_class); 2811 err_comp_unbound: 2812 destroy_workqueue(ib_comp_unbound_wq); 2813 err_comp: 2814 destroy_workqueue(ib_comp_wq); 2815 err: 2816 destroy_workqueue(ib_wq); 2817 return ret; 2818 } 2819 2820 static void __exit ib_core_cleanup(void) 2821 { 2822 roce_gid_mgmt_cleanup(); 2823 nldev_exit(); 2824 rdma_nl_unregister(RDMA_NL_LS); 2825 unregister_pernet_device(&rdma_dev_net_ops); 2826 unregister_blocking_lsm_notifier(&ibdev_lsm_nb); 2827 ib_sa_cleanup(); 2828 ib_mad_cleanup(); 2829 addr_cleanup(); 2830 rdma_nl_exit(); 2831 class_unregister(&ib_class); 2832 destroy_workqueue(ib_comp_unbound_wq); 2833 destroy_workqueue(ib_comp_wq); 2834 /* Make sure that any pending umem accounting work is done. */ 2835 destroy_workqueue(ib_wq); 2836 flush_workqueue(system_unbound_wq); 2837 WARN_ON(!xa_empty(&clients)); 2838 WARN_ON(!xa_empty(&devices)); 2839 } 2840 2841 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 2842 2843 /* ib core relies on netdev stack to first register net_ns_type_operations 2844 * ns kobject type before ib_core initialization. 2845 */ 2846 fs_initcall(ib_core_init); 2847 module_exit(ib_core_cleanup); 2848