1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 */ 6 7 #include <net/genetlink.h> 8 #define CREATE_TRACE_POINTS 9 #include <trace/events/devlink.h> 10 11 #include "devl_internal.h" 12 13 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); 14 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); 15 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); 16 17 DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); 18 19 static struct devlink *devlinks_xa_get(unsigned long index) 20 { 21 struct devlink *devlink; 22 23 rcu_read_lock(); 24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED); 25 if (!devlink || !devlink_try_get(devlink)) 26 devlink = NULL; 27 rcu_read_unlock(); 28 return devlink; 29 } 30 31 /* devlink_rels xarray contains 1:1 relationships between 32 * devlink object and related nested devlink instance. 33 * The xarray index is used to get the nested object from 34 * the nested-in object code. 35 */ 36 static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); 37 38 #define DEVLINK_REL_IN_USE XA_MARK_0 39 40 struct devlink_rel { 41 u32 index; 42 refcount_t refcount; 43 u32 devlink_index; 44 struct { 45 u32 devlink_index; 46 u32 obj_index; 47 devlink_rel_notify_cb_t *notify_cb; 48 devlink_rel_cleanup_cb_t *cleanup_cb; 49 struct delayed_work notify_work; 50 } nested_in; 51 }; 52 53 static void devlink_rel_free(struct devlink_rel *rel) 54 { 55 xa_erase(&devlink_rels, rel->index); 56 kfree(rel); 57 } 58 59 static void __devlink_rel_get(struct devlink_rel *rel) 60 { 61 refcount_inc(&rel->refcount); 62 } 63 64 static void __devlink_rel_put(struct devlink_rel *rel) 65 { 66 if (refcount_dec_and_test(&rel->refcount)) 67 devlink_rel_free(rel); 68 } 69 70 struct devlink *__must_check devlink_nested_in_get_lock(struct devlink *devlink) 71 { 72 devl_assert_locked(devlink); 73 if (!devlink->rel) 74 return NULL; 75 devlink = devlinks_xa_get(devlink->rel->nested_in.devlink_index); 76 if (!devlink) 77 return NULL; 78 devl_lock(devlink); 79 if (devl_is_registered(devlink)) 80 return devlink; 81 devl_unlock(devlink); 82 devlink_put(devlink); 83 return NULL; 84 } 85 86 static void devlink_rel_nested_in_notify_work(struct work_struct *work) 87 { 88 struct devlink_rel *rel = container_of(work, struct devlink_rel, 89 nested_in.notify_work.work); 90 struct devlink *devlink; 91 92 devlink = devlinks_xa_get(rel->nested_in.devlink_index); 93 if (!devlink) 94 goto rel_put; 95 if (!devl_trylock(devlink)) { 96 devlink_put(devlink); 97 goto reschedule_work; 98 } 99 if (!devl_is_registered(devlink)) { 100 devl_unlock(devlink); 101 devlink_put(devlink); 102 goto rel_put; 103 } 104 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE)) 105 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); 106 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); 107 devl_unlock(devlink); 108 devlink_put(devlink); 109 110 rel_put: 111 __devlink_rel_put(rel); 112 return; 113 114 reschedule_work: 115 schedule_delayed_work(&rel->nested_in.notify_work, 1); 116 } 117 118 static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) 119 { 120 __devlink_rel_get(rel); 121 schedule_delayed_work(&rel->nested_in.notify_work, 0); 122 } 123 124 static struct devlink_rel *devlink_rel_alloc(void) 125 { 126 struct devlink_rel *rel; 127 static u32 next; 128 int err; 129 130 rel = kzalloc_obj(*rel); 131 if (!rel) 132 return ERR_PTR(-ENOMEM); 133 134 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel, 135 xa_limit_32b, &next, GFP_KERNEL); 136 if (err < 0) { 137 kfree(rel); 138 return ERR_PTR(err); 139 } 140 141 refcount_set(&rel->refcount, 1); 142 INIT_DELAYED_WORK(&rel->nested_in.notify_work, 143 &devlink_rel_nested_in_notify_work); 144 return rel; 145 } 146 147 static void devlink_rel_put(struct devlink *devlink) 148 { 149 struct devlink_rel *rel = devlink->rel; 150 151 if (!rel) 152 return; 153 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 154 devlink_rel_nested_in_notify_work_schedule(rel); 155 __devlink_rel_put(rel); 156 devlink->rel = NULL; 157 } 158 159 void devlink_rel_nested_in_clear(u32 rel_index) 160 { 161 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE); 162 } 163 164 int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, 165 u32 obj_index, devlink_rel_notify_cb_t *notify_cb, 166 devlink_rel_cleanup_cb_t *cleanup_cb, 167 struct devlink *devlink) 168 { 169 struct devlink_rel *rel = devlink_rel_alloc(); 170 171 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 172 173 if (IS_ERR(rel)) 174 return PTR_ERR(rel); 175 176 rel->devlink_index = devlink->index; 177 rel->nested_in.devlink_index = devlink_index; 178 rel->nested_in.obj_index = obj_index; 179 rel->nested_in.notify_cb = notify_cb; 180 rel->nested_in.cleanup_cb = cleanup_cb; 181 *rel_index = rel->index; 182 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 183 devlink->rel = rel; 184 return 0; 185 } 186 187 /** 188 * devlink_rel_nested_in_notify - Notify the object this devlink 189 * instance is nested in. 190 * @devlink: devlink 191 * 192 * This is called upon network namespace change of devlink instance. 193 * In case this devlink instance is nested in another devlink object, 194 * a notification of a change of this object should be sent 195 * over netlink. The parent devlink instance lock needs to be 196 * taken during the notification preparation. 197 * Since the parent may or may not be locked, 'work' is utilized. 198 */ 199 void devlink_rel_nested_in_notify(struct devlink *devlink) 200 { 201 struct devlink_rel *rel = devlink->rel; 202 203 if (!rel) 204 return; 205 devlink_rel_nested_in_notify_work_schedule(rel); 206 } 207 208 static struct devlink_rel *devlink_rel_find(unsigned long rel_index) 209 { 210 return xa_find(&devlink_rels, &rel_index, rel_index, 211 DEVLINK_REL_IN_USE); 212 } 213 214 static struct devlink *devlink_rel_devlink_get(u32 rel_index) 215 { 216 struct devlink_rel *rel; 217 u32 devlink_index; 218 219 if (!rel_index) 220 return NULL; 221 xa_lock(&devlink_rels); 222 rel = devlink_rel_find(rel_index); 223 if (rel) 224 devlink_index = rel->devlink_index; 225 xa_unlock(&devlink_rels); 226 if (!rel) 227 return NULL; 228 return devlinks_xa_get(devlink_index); 229 } 230 231 int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, 232 u32 rel_index, int attrtype, 233 bool *msg_updated) 234 { 235 struct net *net = devlink_net(devlink); 236 struct devlink *rel_devlink; 237 int err; 238 239 rel_devlink = devlink_rel_devlink_get(rel_index); 240 if (!rel_devlink) 241 return 0; 242 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype); 243 devlink_put(rel_devlink); 244 if (!err && msg_updated) 245 *msg_updated = true; 246 return err; 247 } 248 249 void *devlink_priv(struct devlink *devlink) 250 { 251 return &devlink->priv; 252 } 253 EXPORT_SYMBOL_GPL(devlink_priv); 254 255 struct devlink *priv_to_devlink(void *priv) 256 { 257 return container_of(priv, struct devlink, priv); 258 } 259 EXPORT_SYMBOL_GPL(priv_to_devlink); 260 261 struct device *devlink_to_dev(const struct devlink *devlink) 262 { 263 return devlink->dev; 264 } 265 EXPORT_SYMBOL_GPL(devlink_to_dev); 266 267 const char *devlink_bus_name(const struct devlink *devlink) 268 { 269 return devlink->dev ? devlink->dev->bus->name : DEVLINK_INDEX_BUS_NAME; 270 } 271 EXPORT_SYMBOL_GPL(devlink_bus_name); 272 273 const char *devlink_dev_name(const struct devlink *devlink) 274 { 275 return devlink->dev ? dev_name(devlink->dev) : devlink->dev_name_index; 276 } 277 EXPORT_SYMBOL_GPL(devlink_dev_name); 278 279 const char *devlink_dev_driver_name(const struct devlink *devlink) 280 { 281 return devlink->dev_driver->name; 282 } 283 EXPORT_SYMBOL_GPL(devlink_dev_driver_name); 284 285 struct net *devlink_net(const struct devlink *devlink) 286 { 287 return read_pnet(&devlink->_net); 288 } 289 EXPORT_SYMBOL_GPL(devlink_net); 290 291 void devl_assert_locked(struct devlink *devlink) 292 { 293 lockdep_assert_held(&devlink->lock); 294 } 295 EXPORT_SYMBOL_GPL(devl_assert_locked); 296 297 #ifdef CONFIG_LOCKDEP 298 /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */ 299 bool devl_lock_is_held(struct devlink *devlink) 300 { 301 return lockdep_is_held(&devlink->lock); 302 } 303 EXPORT_SYMBOL_GPL(devl_lock_is_held); 304 #endif 305 306 void devl_lock(struct devlink *devlink) 307 { 308 mutex_lock(&devlink->lock); 309 } 310 EXPORT_SYMBOL_GPL(devl_lock); 311 312 int devl_trylock(struct devlink *devlink) 313 { 314 return mutex_trylock(&devlink->lock); 315 } 316 EXPORT_SYMBOL_GPL(devl_trylock); 317 318 void devl_unlock(struct devlink *devlink) 319 { 320 mutex_unlock(&devlink->lock); 321 } 322 EXPORT_SYMBOL_GPL(devl_unlock); 323 324 /** 325 * devlink_try_get() - try to obtain a reference on a devlink instance 326 * @devlink: instance to reference 327 * 328 * Obtain a reference on a devlink instance. A reference on a devlink instance 329 * only implies that it's safe to take the instance lock. It does not imply 330 * that the instance is registered, use devl_is_registered() after taking 331 * the instance lock to check registration status. 332 */ 333 struct devlink *__must_check devlink_try_get(struct devlink *devlink) 334 { 335 if (refcount_inc_not_zero(&devlink->refcount)) 336 return devlink; 337 return NULL; 338 } 339 340 static void devlink_release(struct work_struct *work) 341 { 342 struct devlink *devlink; 343 344 devlink = container_of(to_rcu_work(work), struct devlink, rwork); 345 346 mutex_destroy(&devlink->lock); 347 lockdep_unregister_key(&devlink->lock_key); 348 if (devlink->dev) 349 put_device(devlink->dev); 350 else 351 kfree(devlink->dev_name_index); 352 kvfree(devlink); 353 } 354 355 void devlink_put(struct devlink *devlink) 356 { 357 if (refcount_dec_and_test(&devlink->refcount)) 358 queue_rcu_work(system_percpu_wq, &devlink->rwork); 359 } 360 361 static struct devlink *__devlinks_xa_find_get(struct net *net, 362 unsigned long *indexp, 363 unsigned long end) 364 { 365 struct devlink *devlink = NULL; 366 367 rcu_read_lock(); 368 retry: 369 devlink = xa_find(&devlinks, indexp, end, DEVLINK_REGISTERED); 370 if (!devlink) 371 goto unlock; 372 373 if (!devlink_try_get(devlink)) 374 goto next; 375 if (!net_eq(devlink_net(devlink), net)) { 376 devlink_put(devlink); 377 goto next; 378 } 379 unlock: 380 rcu_read_unlock(); 381 return devlink; 382 383 next: 384 (*indexp)++; 385 goto retry; 386 } 387 388 struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp) 389 { 390 return __devlinks_xa_find_get(net, indexp, ULONG_MAX); 391 } 392 393 struct devlink *devlinks_xa_lookup_get(struct net *net, unsigned long index) 394 { 395 return __devlinks_xa_find_get(net, &index, index); 396 } 397 398 /** 399 * devl_register - Register devlink instance 400 * @devlink: devlink 401 */ 402 int devl_register(struct devlink *devlink) 403 { 404 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 405 devl_assert_locked(devlink); 406 407 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 408 devlink_notify_register(devlink); 409 devlink_rel_nested_in_notify(devlink); 410 411 return 0; 412 } 413 EXPORT_SYMBOL_GPL(devl_register); 414 415 void devlink_register(struct devlink *devlink) 416 { 417 devl_lock(devlink); 418 devl_register(devlink); 419 devl_unlock(devlink); 420 } 421 EXPORT_SYMBOL_GPL(devlink_register); 422 423 /** 424 * devl_unregister - Unregister devlink instance 425 * @devlink: devlink 426 */ 427 void devl_unregister(struct devlink *devlink) 428 { 429 ASSERT_DEVLINK_REGISTERED(devlink); 430 devl_assert_locked(devlink); 431 432 devlink_notify_unregister(devlink); 433 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 434 devlink_rel_put(devlink); 435 } 436 EXPORT_SYMBOL_GPL(devl_unregister); 437 438 void devlink_unregister(struct devlink *devlink) 439 { 440 devl_lock(devlink); 441 devl_unregister(devlink); 442 devl_unlock(devlink); 443 } 444 EXPORT_SYMBOL_GPL(devlink_unregister); 445 446 struct devlink *__devlink_alloc(const struct devlink_ops *ops, size_t priv_size, 447 struct net *net, struct device *dev, 448 const struct device_driver *dev_driver) 449 { 450 struct devlink *devlink; 451 static u32 last_id; 452 int ret; 453 454 WARN_ON(!ops || !dev_driver); 455 if (!devlink_reload_actions_valid(ops)) 456 return NULL; 457 458 devlink = kvzalloc_flex(*devlink, priv, priv_size); 459 if (!devlink) 460 return NULL; 461 462 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b, 463 &last_id, GFP_KERNEL); 464 if (ret < 0) 465 goto err_xa_alloc; 466 467 if (dev) { 468 devlink->dev = get_device(dev); 469 } else { 470 devlink->dev_name_index = kasprintf(GFP_KERNEL, "%u", devlink->index); 471 if (!devlink->dev_name_index) 472 goto err_kasprintf; 473 } 474 475 devlink->ops = ops; 476 devlink->dev_driver = dev_driver; 477 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); 478 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC); 479 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); 480 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC); 481 write_pnet(&devlink->_net, net); 482 INIT_LIST_HEAD(&devlink->rate_list); 483 INIT_LIST_HEAD(&devlink->linecard_list); 484 INIT_LIST_HEAD(&devlink->sb_list); 485 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list); 486 INIT_LIST_HEAD(&devlink->resource_list); 487 INIT_LIST_HEAD(&devlink->region_list); 488 INIT_LIST_HEAD(&devlink->reporter_list); 489 INIT_LIST_HEAD(&devlink->trap_list); 490 INIT_LIST_HEAD(&devlink->trap_group_list); 491 INIT_LIST_HEAD(&devlink->trap_policer_list); 492 INIT_RCU_WORK(&devlink->rwork, devlink_release); 493 lockdep_register_key(&devlink->lock_key); 494 mutex_init(&devlink->lock); 495 lockdep_set_class(&devlink->lock, &devlink->lock_key); 496 refcount_set(&devlink->refcount, 1); 497 498 return devlink; 499 500 err_kasprintf: 501 xa_erase(&devlinks, devlink->index); 502 err_xa_alloc: 503 kvfree(devlink); 504 return NULL; 505 } 506 507 /** 508 * devlink_alloc_ns - Allocate new devlink instance resources 509 * in specific namespace 510 * 511 * @ops: ops 512 * @priv_size: size of user private data 513 * @net: net namespace 514 * @dev: parent device 515 * 516 * Allocate new devlink instance resources, including devlink index 517 * and name. 518 */ 519 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, 520 size_t priv_size, struct net *net, 521 struct device *dev) 522 { 523 WARN_ON(!dev); 524 return __devlink_alloc(ops, priv_size, net, dev, dev->driver); 525 } 526 EXPORT_SYMBOL_GPL(devlink_alloc_ns); 527 528 /** 529 * devlink_free - Free devlink instance resources 530 * 531 * @devlink: devlink 532 */ 533 void devlink_free(struct devlink *devlink) 534 { 535 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 536 537 devl_lock(devlink); 538 WARN_ON(devlink_rates_check(devlink, NULL, NULL)); 539 devl_unlock(devlink); 540 devlink_rel_put(devlink); 541 542 WARN_ON(!list_empty(&devlink->trap_policer_list)); 543 WARN_ON(!list_empty(&devlink->trap_group_list)); 544 WARN_ON(!list_empty(&devlink->trap_list)); 545 WARN_ON(!list_empty(&devlink->reporter_list)); 546 WARN_ON(!list_empty(&devlink->region_list)); 547 WARN_ON(!list_empty(&devlink->resource_list)); 548 WARN_ON(!list_empty(&devlink->dpipe_table_list)); 549 WARN_ON(!list_empty(&devlink->sb_list)); 550 WARN_ON(!list_empty(&devlink->linecard_list)); 551 WARN_ON(!xa_empty(&devlink->ports)); 552 553 xa_destroy(&devlink->nested_rels); 554 xa_destroy(&devlink->snapshot_ids); 555 xa_destroy(&devlink->params); 556 xa_destroy(&devlink->ports); 557 558 xa_erase(&devlinks, devlink->index); 559 560 devlink_put(devlink); 561 } 562 EXPORT_SYMBOL_GPL(devlink_free); 563 564 static void __net_exit devlink_pernet_pre_exit(struct net *net) 565 { 566 struct devlink *devlink; 567 u32 actions_performed; 568 unsigned long index; 569 int err; 570 571 /* In case network namespace is getting destroyed, reload 572 * all devlink instances from this namespace into init_net. 573 */ 574 devlinks_xa_for_each_registered_get(net, index, devlink) { 575 devl_dev_lock(devlink, true); 576 err = 0; 577 if (devl_is_registered(devlink)) 578 err = devlink_reload(devlink, &init_net, 579 DEVLINK_RELOAD_ACTION_DRIVER_REINIT, 580 DEVLINK_RELOAD_LIMIT_UNSPEC, 581 &actions_performed, NULL); 582 devl_dev_unlock(devlink, true); 583 devlink_put(devlink); 584 if (err && err != -EOPNOTSUPP) 585 pr_warn("Failed to reload devlink instance into init_net\n"); 586 } 587 } 588 589 static struct pernet_operations devlink_pernet_ops __net_initdata = { 590 .pre_exit = devlink_pernet_pre_exit, 591 }; 592 593 static struct notifier_block devlink_port_netdevice_nb = { 594 .notifier_call = devlink_port_netdevice_event, 595 }; 596 597 static int __init devlink_init(void) 598 { 599 int err; 600 601 err = register_pernet_subsys(&devlink_pernet_ops); 602 if (err) 603 goto out; 604 err = genl_register_family(&devlink_nl_family); 605 if (err) 606 goto out_unreg_pernet_subsys; 607 err = register_netdevice_notifier(&devlink_port_netdevice_nb); 608 if (!err) 609 return 0; 610 611 genl_unregister_family(&devlink_nl_family); 612 613 out_unreg_pernet_subsys: 614 unregister_pernet_subsys(&devlink_pernet_ops); 615 out: 616 WARN_ON(err); 617 return err; 618 } 619 620 subsys_initcall(devlink_init); 621