1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 */ 6 7 #include <net/genetlink.h> 8 #define CREATE_TRACE_POINTS 9 #include <trace/events/devlink.h> 10 11 #include "devl_internal.h" 12 13 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); 14 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); 15 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); 16 17 DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); 18 19 static struct devlink *devlinks_xa_get(unsigned long index) 20 { 21 struct devlink *devlink; 22 23 rcu_read_lock(); 24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED); 25 if (!devlink || !devlink_try_get(devlink)) 26 devlink = NULL; 27 rcu_read_unlock(); 28 return devlink; 29 } 30 31 /* devlink_rels xarray contains 1:1 relationships between 32 * devlink object and related nested devlink instance. 33 * The xarray index is used to get the nested object from 34 * the nested-in object code. 35 */ 36 static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); 37 38 #define DEVLINK_REL_IN_USE XA_MARK_0 39 40 struct devlink_rel { 41 u32 index; 42 refcount_t refcount; 43 u32 devlink_index; 44 struct { 45 u32 devlink_index; 46 u32 obj_index; 47 devlink_rel_notify_cb_t *notify_cb; 48 devlink_rel_cleanup_cb_t *cleanup_cb; 49 struct delayed_work notify_work; 50 } nested_in; 51 }; 52 53 static void devlink_rel_free(struct devlink_rel *rel) 54 { 55 xa_erase(&devlink_rels, rel->index); 56 kfree(rel); 57 } 58 59 static void __devlink_rel_get(struct devlink_rel *rel) 60 { 61 refcount_inc(&rel->refcount); 62 } 63 64 static void __devlink_rel_put(struct devlink_rel *rel) 65 { 66 if (refcount_dec_and_test(&rel->refcount)) 67 devlink_rel_free(rel); 68 } 69 70 struct devlink *__must_check devlink_nested_in_get_lock(struct devlink *devlink) 71 { 72 devl_assert_locked(devlink); 73 if (!devlink->rel) 74 return NULL; 75 devlink = devlinks_xa_get(devlink->rel->nested_in.devlink_index); 76 if (!devlink) 77 return NULL; 78 devl_lock(devlink); 79 if (devl_is_registered(devlink)) 80 return devlink; 81 devl_unlock(devlink); 82 devlink_put(devlink); 83 return NULL; 84 } 85 86 static void devlink_rel_nested_in_notify_work(struct work_struct *work) 87 { 88 struct devlink_rel *rel = container_of(work, struct devlink_rel, 89 nested_in.notify_work.work); 90 struct devlink *devlink; 91 92 devlink = devlinks_xa_get(rel->nested_in.devlink_index); 93 if (!devlink) 94 goto rel_put; 95 if (!devl_trylock(devlink)) { 96 devlink_put(devlink); 97 goto reschedule_work; 98 } 99 if (!devl_is_registered(devlink)) { 100 devl_unlock(devlink); 101 devlink_put(devlink); 102 goto rel_put; 103 } 104 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE)) 105 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); 106 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); 107 devl_unlock(devlink); 108 devlink_put(devlink); 109 110 rel_put: 111 __devlink_rel_put(rel); 112 return; 113 114 reschedule_work: 115 /* The work may have been queued again meanwhile, which took its own 116 * reference. Drop ours in that case. 117 */ 118 if (!schedule_delayed_work(&rel->nested_in.notify_work, 1)) 119 __devlink_rel_put(rel); 120 } 121 122 static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) 123 { 124 __devlink_rel_get(rel); 125 /* The pending work holds a reference already, drop the new one. */ 126 if (!schedule_delayed_work(&rel->nested_in.notify_work, 0)) 127 __devlink_rel_put(rel); 128 } 129 130 static struct devlink_rel *devlink_rel_alloc(void) 131 { 132 struct devlink_rel *rel; 133 static u32 next; 134 int err; 135 136 rel = kzalloc_obj(*rel); 137 if (!rel) 138 return ERR_PTR(-ENOMEM); 139 140 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel, 141 xa_limit_32b, &next, GFP_KERNEL); 142 if (err < 0) { 143 kfree(rel); 144 return ERR_PTR(err); 145 } 146 147 refcount_set(&rel->refcount, 1); 148 INIT_DELAYED_WORK(&rel->nested_in.notify_work, 149 &devlink_rel_nested_in_notify_work); 150 return rel; 151 } 152 153 static void devlink_rel_put(struct devlink *devlink) 154 { 155 struct devlink_rel *rel = devlink->rel; 156 157 if (!rel) 158 return; 159 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 160 devlink_rel_nested_in_notify_work_schedule(rel); 161 __devlink_rel_put(rel); 162 devlink->rel = NULL; 163 } 164 165 void devlink_rel_nested_in_clear(u32 rel_index) 166 { 167 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE); 168 } 169 170 int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, 171 u32 obj_index, devlink_rel_notify_cb_t *notify_cb, 172 devlink_rel_cleanup_cb_t *cleanup_cb, 173 struct devlink *devlink) 174 { 175 struct devlink_rel *rel = devlink_rel_alloc(); 176 177 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 178 179 if (IS_ERR(rel)) 180 return PTR_ERR(rel); 181 182 rel->devlink_index = devlink->index; 183 rel->nested_in.devlink_index = devlink_index; 184 rel->nested_in.obj_index = obj_index; 185 rel->nested_in.notify_cb = notify_cb; 186 rel->nested_in.cleanup_cb = cleanup_cb; 187 *rel_index = rel->index; 188 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 189 devlink->rel = rel; 190 return 0; 191 } 192 193 /** 194 * devlink_rel_nested_in_notify - Notify the object this devlink 195 * instance is nested in. 196 * @devlink: devlink 197 * 198 * This is called upon network namespace change of devlink instance. 199 * In case this devlink instance is nested in another devlink object, 200 * a notification of a change of this object should be sent 201 * over netlink. The parent devlink instance lock needs to be 202 * taken during the notification preparation. 203 * Since the parent may or may not be locked, 'work' is utilized. 204 */ 205 void devlink_rel_nested_in_notify(struct devlink *devlink) 206 { 207 struct devlink_rel *rel = devlink->rel; 208 209 if (!rel) 210 return; 211 devlink_rel_nested_in_notify_work_schedule(rel); 212 } 213 214 static struct devlink_rel *devlink_rel_find(unsigned long rel_index) 215 { 216 return xa_find(&devlink_rels, &rel_index, rel_index, 217 DEVLINK_REL_IN_USE); 218 } 219 220 static struct devlink *devlink_rel_devlink_get(u32 rel_index) 221 { 222 struct devlink_rel *rel; 223 u32 devlink_index; 224 225 if (!rel_index) 226 return NULL; 227 xa_lock(&devlink_rels); 228 rel = devlink_rel_find(rel_index); 229 if (rel) 230 devlink_index = rel->devlink_index; 231 xa_unlock(&devlink_rels); 232 if (!rel) 233 return NULL; 234 return devlinks_xa_get(devlink_index); 235 } 236 237 int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, 238 u32 rel_index, int attrtype, 239 bool *msg_updated) 240 { 241 struct net *net = devlink_net(devlink); 242 struct devlink *rel_devlink; 243 int err; 244 245 rel_devlink = devlink_rel_devlink_get(rel_index); 246 if (!rel_devlink) 247 return 0; 248 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype); 249 devlink_put(rel_devlink); 250 if (!err && msg_updated) 251 *msg_updated = true; 252 return err; 253 } 254 255 void *devlink_priv(struct devlink *devlink) 256 { 257 return &devlink->priv; 258 } 259 EXPORT_SYMBOL_GPL(devlink_priv); 260 261 struct devlink *priv_to_devlink(void *priv) 262 { 263 return container_of(priv, struct devlink, priv); 264 } 265 EXPORT_SYMBOL_GPL(priv_to_devlink); 266 267 struct device *devlink_to_dev(const struct devlink *devlink) 268 { 269 return devlink->dev; 270 } 271 EXPORT_SYMBOL_GPL(devlink_to_dev); 272 273 const char *devlink_bus_name(const struct devlink *devlink) 274 { 275 return devlink->dev ? devlink->dev->bus->name : DEVLINK_INDEX_BUS_NAME; 276 } 277 EXPORT_SYMBOL_GPL(devlink_bus_name); 278 279 const char *devlink_dev_name(const struct devlink *devlink) 280 { 281 return devlink->dev ? dev_name(devlink->dev) : devlink->dev_name_index; 282 } 283 EXPORT_SYMBOL_GPL(devlink_dev_name); 284 285 const char *devlink_dev_driver_name(const struct devlink *devlink) 286 { 287 return devlink->dev_driver->name; 288 } 289 EXPORT_SYMBOL_GPL(devlink_dev_driver_name); 290 291 struct net *devlink_net(const struct devlink *devlink) 292 { 293 return read_pnet(&devlink->_net); 294 } 295 EXPORT_SYMBOL_GPL(devlink_net); 296 297 void devl_assert_locked(struct devlink *devlink) 298 { 299 lockdep_assert_held(&devlink->lock); 300 } 301 EXPORT_SYMBOL_GPL(devl_assert_locked); 302 303 #ifdef CONFIG_LOCKDEP 304 /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */ 305 bool devl_lock_is_held(struct devlink *devlink) 306 { 307 return lockdep_is_held(&devlink->lock); 308 } 309 EXPORT_SYMBOL_GPL(devl_lock_is_held); 310 #endif 311 312 void devl_lock(struct devlink *devlink) 313 { 314 mutex_lock(&devlink->lock); 315 } 316 EXPORT_SYMBOL_GPL(devl_lock); 317 318 int devl_trylock(struct devlink *devlink) 319 { 320 return mutex_trylock(&devlink->lock); 321 } 322 EXPORT_SYMBOL_GPL(devl_trylock); 323 324 void devl_unlock(struct devlink *devlink) 325 { 326 mutex_unlock(&devlink->lock); 327 } 328 EXPORT_SYMBOL_GPL(devl_unlock); 329 330 /** 331 * devlink_try_get() - try to obtain a reference on a devlink instance 332 * @devlink: instance to reference 333 * 334 * Obtain a reference on a devlink instance. A reference on a devlink instance 335 * only implies that it's safe to take the instance lock. It does not imply 336 * that the instance is registered, use devl_is_registered() after taking 337 * the instance lock to check registration status. 338 */ 339 struct devlink *__must_check devlink_try_get(struct devlink *devlink) 340 { 341 if (refcount_inc_not_zero(&devlink->refcount)) 342 return devlink; 343 return NULL; 344 } 345 346 static void devlink_release(struct work_struct *work) 347 { 348 struct devlink *devlink; 349 350 devlink = container_of(to_rcu_work(work), struct devlink, rwork); 351 352 mutex_destroy(&devlink->lock); 353 lockdep_unregister_key(&devlink->lock_key); 354 if (devlink->dev) 355 put_device(devlink->dev); 356 else 357 kfree(devlink->dev_name_index); 358 kvfree(devlink); 359 } 360 361 void devlink_put(struct devlink *devlink) 362 { 363 if (refcount_dec_and_test(&devlink->refcount)) 364 queue_rcu_work(system_percpu_wq, &devlink->rwork); 365 } 366 367 static struct devlink *__devlinks_xa_find_get(struct net *net, 368 unsigned long *indexp, 369 unsigned long end) 370 { 371 struct devlink *devlink = NULL; 372 373 rcu_read_lock(); 374 retry: 375 devlink = xa_find(&devlinks, indexp, end, DEVLINK_REGISTERED); 376 if (!devlink) 377 goto unlock; 378 379 if (!devlink_try_get(devlink)) 380 goto next; 381 if (!net_eq(devlink_net(devlink), net)) { 382 devlink_put(devlink); 383 goto next; 384 } 385 unlock: 386 rcu_read_unlock(); 387 return devlink; 388 389 next: 390 (*indexp)++; 391 goto retry; 392 } 393 394 struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp) 395 { 396 return __devlinks_xa_find_get(net, indexp, ULONG_MAX); 397 } 398 399 struct devlink *devlinks_xa_lookup_get(struct net *net, unsigned long index) 400 { 401 return __devlinks_xa_find_get(net, &index, index); 402 } 403 404 /** 405 * devl_register - Register devlink instance 406 * @devlink: devlink 407 */ 408 int devl_register(struct devlink *devlink) 409 { 410 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 411 devl_assert_locked(devlink); 412 413 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 414 devlink_notify_register(devlink); 415 devlink_rel_nested_in_notify(devlink); 416 417 return 0; 418 } 419 EXPORT_SYMBOL_GPL(devl_register); 420 421 void devlink_register(struct devlink *devlink) 422 { 423 devl_lock(devlink); 424 devl_register(devlink); 425 devl_unlock(devlink); 426 } 427 EXPORT_SYMBOL_GPL(devlink_register); 428 429 /** 430 * devl_unregister - Unregister devlink instance 431 * @devlink: devlink 432 */ 433 void devl_unregister(struct devlink *devlink) 434 { 435 ASSERT_DEVLINK_REGISTERED(devlink); 436 devl_assert_locked(devlink); 437 438 devlink_notify_unregister(devlink); 439 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 440 devlink_rel_put(devlink); 441 } 442 EXPORT_SYMBOL_GPL(devl_unregister); 443 444 void devlink_unregister(struct devlink *devlink) 445 { 446 devl_lock(devlink); 447 devl_unregister(devlink); 448 devl_unlock(devlink); 449 } 450 EXPORT_SYMBOL_GPL(devlink_unregister); 451 452 struct devlink *__devlink_alloc(const struct devlink_ops *ops, size_t priv_size, 453 struct net *net, struct device *dev, 454 const struct device_driver *dev_driver) 455 { 456 struct devlink *devlink; 457 static u32 last_id; 458 int ret; 459 460 WARN_ON(!ops || !dev_driver); 461 if (!devlink_reload_actions_valid(ops)) 462 return NULL; 463 464 devlink = kvzalloc_flex(*devlink, priv, priv_size); 465 if (!devlink) 466 return NULL; 467 468 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b, 469 &last_id, GFP_KERNEL); 470 if (ret < 0) 471 goto err_xa_alloc; 472 473 if (dev) { 474 devlink->dev = get_device(dev); 475 } else { 476 devlink->dev_name_index = kasprintf(GFP_KERNEL, "%u", devlink->index); 477 if (!devlink->dev_name_index) 478 goto err_kasprintf; 479 } 480 481 devlink->ops = ops; 482 devlink->dev_driver = dev_driver; 483 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); 484 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC); 485 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); 486 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC); 487 write_pnet(&devlink->_net, net); 488 INIT_LIST_HEAD(&devlink->rate_list); 489 INIT_LIST_HEAD(&devlink->linecard_list); 490 INIT_LIST_HEAD(&devlink->sb_list); 491 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list); 492 INIT_LIST_HEAD(&devlink->resource_list); 493 INIT_LIST_HEAD(&devlink->region_list); 494 INIT_LIST_HEAD(&devlink->reporter_list); 495 INIT_LIST_HEAD(&devlink->trap_list); 496 INIT_LIST_HEAD(&devlink->trap_group_list); 497 INIT_LIST_HEAD(&devlink->trap_policer_list); 498 INIT_RCU_WORK(&devlink->rwork, devlink_release); 499 lockdep_register_key(&devlink->lock_key); 500 mutex_init(&devlink->lock); 501 lockdep_set_class(&devlink->lock, &devlink->lock_key); 502 refcount_set(&devlink->refcount, 1); 503 504 return devlink; 505 506 err_kasprintf: 507 xa_erase(&devlinks, devlink->index); 508 err_xa_alloc: 509 kvfree(devlink); 510 return NULL; 511 } 512 513 /** 514 * devlink_alloc_ns - Allocate new devlink instance resources 515 * in specific namespace 516 * 517 * @ops: ops 518 * @priv_size: size of user private data 519 * @net: net namespace 520 * @dev: parent device 521 * 522 * Allocate new devlink instance resources, including devlink index 523 * and name. 524 */ 525 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, 526 size_t priv_size, struct net *net, 527 struct device *dev) 528 { 529 WARN_ON(!dev); 530 return __devlink_alloc(ops, priv_size, net, dev, dev->driver); 531 } 532 EXPORT_SYMBOL_GPL(devlink_alloc_ns); 533 534 /** 535 * devlink_free - Free devlink instance resources 536 * 537 * @devlink: devlink 538 */ 539 void devlink_free(struct devlink *devlink) 540 { 541 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 542 543 devl_lock(devlink); 544 WARN_ON(devlink_rates_check(devlink, NULL, NULL)); 545 devl_unlock(devlink); 546 devlink_rel_put(devlink); 547 548 WARN_ON(!list_empty(&devlink->trap_policer_list)); 549 WARN_ON(!list_empty(&devlink->trap_group_list)); 550 WARN_ON(!list_empty(&devlink->trap_list)); 551 WARN_ON(!list_empty(&devlink->reporter_list)); 552 WARN_ON(!list_empty(&devlink->region_list)); 553 WARN_ON(!list_empty(&devlink->resource_list)); 554 WARN_ON(!list_empty(&devlink->dpipe_table_list)); 555 WARN_ON(!list_empty(&devlink->sb_list)); 556 WARN_ON(!list_empty(&devlink->linecard_list)); 557 WARN_ON(!xa_empty(&devlink->ports)); 558 559 xa_destroy(&devlink->nested_rels); 560 xa_destroy(&devlink->snapshot_ids); 561 xa_destroy(&devlink->params); 562 xa_destroy(&devlink->ports); 563 564 xa_erase(&devlinks, devlink->index); 565 566 devlink_put(devlink); 567 } 568 EXPORT_SYMBOL_GPL(devlink_free); 569 570 static void __net_exit devlink_pernet_pre_exit(struct net *net) 571 { 572 struct devlink *devlink; 573 u32 actions_performed; 574 unsigned long index; 575 int err; 576 577 /* In case network namespace is getting destroyed, reload 578 * all devlink instances from this namespace into init_net. 579 */ 580 devlinks_xa_for_each_registered_get(net, index, devlink) { 581 devl_dev_lock(devlink, true); 582 err = 0; 583 if (devl_is_registered(devlink)) 584 err = devlink_reload(devlink, &init_net, 585 DEVLINK_RELOAD_ACTION_DRIVER_REINIT, 586 DEVLINK_RELOAD_LIMIT_UNSPEC, 587 &actions_performed, NULL); 588 devl_dev_unlock(devlink, true); 589 devlink_put(devlink); 590 if (err && err != -EOPNOTSUPP) 591 pr_warn("Failed to reload devlink instance into init_net\n"); 592 } 593 } 594 595 static struct pernet_operations devlink_pernet_ops __net_initdata = { 596 .pre_exit = devlink_pernet_pre_exit, 597 }; 598 599 static struct notifier_block devlink_port_netdevice_nb = { 600 .notifier_call = devlink_port_netdevice_event, 601 }; 602 603 static int __init devlink_init(void) 604 { 605 int err; 606 607 err = register_pernet_subsys(&devlink_pernet_ops); 608 if (err) 609 goto out; 610 err = genl_register_family(&devlink_nl_family); 611 if (err) 612 goto out_unreg_pernet_subsys; 613 err = register_netdevice_notifier(&devlink_port_netdevice_nb); 614 if (!err) 615 return 0; 616 617 genl_unregister_family(&devlink_nl_family); 618 619 out_unreg_pernet_subsys: 620 unregister_pernet_subsys(&devlink_pernet_ops); 621 out: 622 WARN_ON(err); 623 return err; 624 } 625 626 subsys_initcall(devlink_init); 627