1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 */ 6 7 #include <net/genetlink.h> 8 #define CREATE_TRACE_POINTS 9 #include <trace/events/devlink.h> 10 11 #include "devl_internal.h" 12 13 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); 14 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); 15 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); 16 17 DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); 18 19 static struct devlink *devlinks_xa_get(unsigned long index) 20 { 21 struct devlink *devlink; 22 23 rcu_read_lock(); 24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED); 25 if (!devlink || !devlink_try_get(devlink)) 26 devlink = NULL; 27 rcu_read_unlock(); 28 return devlink; 29 } 30 31 /* devlink_rels xarray contains 1:1 relationships between 32 * devlink object and related nested devlink instance. 33 * The xarray index is used to get the nested object from 34 * the nested-in object code. 35 */ 36 static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); 37 38 #define DEVLINK_REL_IN_USE XA_MARK_0 39 40 struct devlink_rel { 41 u32 index; 42 refcount_t refcount; 43 u32 devlink_index; 44 struct { 45 u32 devlink_index; 46 u32 obj_index; 47 devlink_rel_notify_cb_t *notify_cb; 48 devlink_rel_cleanup_cb_t *cleanup_cb; 49 struct delayed_work notify_work; 50 } nested_in; 51 }; 52 53 static void devlink_rel_free(struct devlink_rel *rel) 54 { 55 xa_erase(&devlink_rels, rel->index); 56 kfree(rel); 57 } 58 59 static void __devlink_rel_get(struct devlink_rel *rel) 60 { 61 refcount_inc(&rel->refcount); 62 } 63 64 static void __devlink_rel_put(struct devlink_rel *rel) 65 { 66 if (refcount_dec_and_test(&rel->refcount)) 67 devlink_rel_free(rel); 68 } 69 70 static void devlink_rel_nested_in_notify_work(struct work_struct *work) 71 { 72 struct devlink_rel *rel = container_of(work, struct devlink_rel, 73 nested_in.notify_work.work); 74 struct devlink *devlink; 75 76 devlink = devlinks_xa_get(rel->nested_in.devlink_index); 77 if (!devlink) 78 goto rel_put; 79 if (!devl_trylock(devlink)) { 80 devlink_put(devlink); 81 goto reschedule_work; 82 } 83 if (!devl_is_registered(devlink)) { 84 devl_unlock(devlink); 85 devlink_put(devlink); 86 goto rel_put; 87 } 88 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE)) 89 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); 90 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); 91 devl_unlock(devlink); 92 devlink_put(devlink); 93 94 rel_put: 95 __devlink_rel_put(rel); 96 return; 97 98 reschedule_work: 99 schedule_delayed_work(&rel->nested_in.notify_work, 1); 100 } 101 102 static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) 103 { 104 __devlink_rel_get(rel); 105 schedule_delayed_work(&rel->nested_in.notify_work, 0); 106 } 107 108 static struct devlink_rel *devlink_rel_alloc(void) 109 { 110 struct devlink_rel *rel; 111 static u32 next; 112 int err; 113 114 rel = kzalloc(sizeof(*rel), GFP_KERNEL); 115 if (!rel) 116 return ERR_PTR(-ENOMEM); 117 118 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel, 119 xa_limit_32b, &next, GFP_KERNEL); 120 if (err) { 121 kfree(rel); 122 return ERR_PTR(err); 123 } 124 125 refcount_set(&rel->refcount, 1); 126 INIT_DELAYED_WORK(&rel->nested_in.notify_work, 127 &devlink_rel_nested_in_notify_work); 128 return rel; 129 } 130 131 static void devlink_rel_put(struct devlink *devlink) 132 { 133 struct devlink_rel *rel = devlink->rel; 134 135 if (!rel) 136 return; 137 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 138 devlink_rel_nested_in_notify_work_schedule(rel); 139 __devlink_rel_put(rel); 140 devlink->rel = NULL; 141 } 142 143 void devlink_rel_nested_in_clear(u32 rel_index) 144 { 145 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE); 146 } 147 148 int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, 149 u32 obj_index, devlink_rel_notify_cb_t *notify_cb, 150 devlink_rel_cleanup_cb_t *cleanup_cb, 151 struct devlink *devlink) 152 { 153 struct devlink_rel *rel = devlink_rel_alloc(); 154 155 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 156 157 if (IS_ERR(rel)) 158 return PTR_ERR(rel); 159 160 rel->devlink_index = devlink->index; 161 rel->nested_in.devlink_index = devlink_index; 162 rel->nested_in.obj_index = obj_index; 163 rel->nested_in.notify_cb = notify_cb; 164 rel->nested_in.cleanup_cb = cleanup_cb; 165 *rel_index = rel->index; 166 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); 167 devlink->rel = rel; 168 return 0; 169 } 170 171 /** 172 * devlink_rel_nested_in_notify - Notify the object this devlink 173 * instance is nested in. 174 * @devlink: devlink 175 * 176 * This is called upon network namespace change of devlink instance. 177 * In case this devlink instance is nested in another devlink object, 178 * a notification of a change of this object should be sent 179 * over netlink. The parent devlink instance lock needs to be 180 * taken during the notification preparation. 181 * However, since the devlink lock of nested instance is held here, 182 * we would end with wrong devlink instance lock ordering and 183 * deadlock. Therefore the work is utilized to avoid that. 184 */ 185 void devlink_rel_nested_in_notify(struct devlink *devlink) 186 { 187 struct devlink_rel *rel = devlink->rel; 188 189 if (!rel) 190 return; 191 devlink_rel_nested_in_notify_work_schedule(rel); 192 } 193 194 static struct devlink_rel *devlink_rel_find(unsigned long rel_index) 195 { 196 return xa_find(&devlink_rels, &rel_index, rel_index, 197 DEVLINK_REL_IN_USE); 198 } 199 200 static struct devlink *devlink_rel_devlink_get(u32 rel_index) 201 { 202 struct devlink_rel *rel; 203 u32 devlink_index; 204 205 if (!rel_index) 206 return NULL; 207 xa_lock(&devlink_rels); 208 rel = devlink_rel_find(rel_index); 209 if (rel) 210 devlink_index = rel->devlink_index; 211 xa_unlock(&devlink_rels); 212 if (!rel) 213 return NULL; 214 return devlinks_xa_get(devlink_index); 215 } 216 217 int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, 218 u32 rel_index, int attrtype, 219 bool *msg_updated) 220 { 221 struct net *net = devlink_net(devlink); 222 struct devlink *rel_devlink; 223 int err; 224 225 rel_devlink = devlink_rel_devlink_get(rel_index); 226 if (!rel_devlink) 227 return 0; 228 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype); 229 devlink_put(rel_devlink); 230 if (!err && msg_updated) 231 *msg_updated = true; 232 return err; 233 } 234 235 void *devlink_priv(struct devlink *devlink) 236 { 237 return &devlink->priv; 238 } 239 EXPORT_SYMBOL_GPL(devlink_priv); 240 241 struct devlink *priv_to_devlink(void *priv) 242 { 243 return container_of(priv, struct devlink, priv); 244 } 245 EXPORT_SYMBOL_GPL(priv_to_devlink); 246 247 struct device *devlink_to_dev(const struct devlink *devlink) 248 { 249 return devlink->dev; 250 } 251 EXPORT_SYMBOL_GPL(devlink_to_dev); 252 253 struct net *devlink_net(const struct devlink *devlink) 254 { 255 return read_pnet(&devlink->_net); 256 } 257 EXPORT_SYMBOL_GPL(devlink_net); 258 259 void devl_assert_locked(struct devlink *devlink) 260 { 261 lockdep_assert_held(&devlink->lock); 262 } 263 EXPORT_SYMBOL_GPL(devl_assert_locked); 264 265 #ifdef CONFIG_LOCKDEP 266 /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */ 267 bool devl_lock_is_held(struct devlink *devlink) 268 { 269 return lockdep_is_held(&devlink->lock); 270 } 271 EXPORT_SYMBOL_GPL(devl_lock_is_held); 272 #endif 273 274 void devl_lock(struct devlink *devlink) 275 { 276 mutex_lock(&devlink->lock); 277 } 278 EXPORT_SYMBOL_GPL(devl_lock); 279 280 int devl_trylock(struct devlink *devlink) 281 { 282 return mutex_trylock(&devlink->lock); 283 } 284 EXPORT_SYMBOL_GPL(devl_trylock); 285 286 void devl_unlock(struct devlink *devlink) 287 { 288 mutex_unlock(&devlink->lock); 289 } 290 EXPORT_SYMBOL_GPL(devl_unlock); 291 292 /** 293 * devlink_try_get() - try to obtain a reference on a devlink instance 294 * @devlink: instance to reference 295 * 296 * Obtain a reference on a devlink instance. A reference on a devlink instance 297 * only implies that it's safe to take the instance lock. It does not imply 298 * that the instance is registered, use devl_is_registered() after taking 299 * the instance lock to check registration status. 300 */ 301 struct devlink *__must_check devlink_try_get(struct devlink *devlink) 302 { 303 if (refcount_inc_not_zero(&devlink->refcount)) 304 return devlink; 305 return NULL; 306 } 307 308 static void devlink_release(struct work_struct *work) 309 { 310 struct devlink *devlink; 311 312 devlink = container_of(to_rcu_work(work), struct devlink, rwork); 313 314 mutex_destroy(&devlink->lock); 315 lockdep_unregister_key(&devlink->lock_key); 316 put_device(devlink->dev); 317 kvfree(devlink); 318 } 319 320 void devlink_put(struct devlink *devlink) 321 { 322 if (refcount_dec_and_test(&devlink->refcount)) 323 queue_rcu_work(system_wq, &devlink->rwork); 324 } 325 326 struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp) 327 { 328 struct devlink *devlink = NULL; 329 330 rcu_read_lock(); 331 retry: 332 devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED); 333 if (!devlink) 334 goto unlock; 335 336 if (!devlink_try_get(devlink)) 337 goto next; 338 if (!net_eq(devlink_net(devlink), net)) { 339 devlink_put(devlink); 340 goto next; 341 } 342 unlock: 343 rcu_read_unlock(); 344 return devlink; 345 346 next: 347 (*indexp)++; 348 goto retry; 349 } 350 351 /** 352 * devl_register - Register devlink instance 353 * @devlink: devlink 354 */ 355 int devl_register(struct devlink *devlink) 356 { 357 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 358 devl_assert_locked(devlink); 359 360 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 361 devlink_notify_register(devlink); 362 devlink_rel_nested_in_notify(devlink); 363 364 return 0; 365 } 366 EXPORT_SYMBOL_GPL(devl_register); 367 368 void devlink_register(struct devlink *devlink) 369 { 370 devl_lock(devlink); 371 devl_register(devlink); 372 devl_unlock(devlink); 373 } 374 EXPORT_SYMBOL_GPL(devlink_register); 375 376 /** 377 * devl_unregister - Unregister devlink instance 378 * @devlink: devlink 379 */ 380 void devl_unregister(struct devlink *devlink) 381 { 382 ASSERT_DEVLINK_REGISTERED(devlink); 383 devl_assert_locked(devlink); 384 385 devlink_notify_unregister(devlink); 386 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); 387 devlink_rel_put(devlink); 388 } 389 EXPORT_SYMBOL_GPL(devl_unregister); 390 391 void devlink_unregister(struct devlink *devlink) 392 { 393 devl_lock(devlink); 394 devl_unregister(devlink); 395 devl_unlock(devlink); 396 } 397 EXPORT_SYMBOL_GPL(devlink_unregister); 398 399 /** 400 * devlink_alloc_ns - Allocate new devlink instance resources 401 * in specific namespace 402 * 403 * @ops: ops 404 * @priv_size: size of user private data 405 * @net: net namespace 406 * @dev: parent device 407 * 408 * Allocate new devlink instance resources, including devlink index 409 * and name. 410 */ 411 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, 412 size_t priv_size, struct net *net, 413 struct device *dev) 414 { 415 struct devlink *devlink; 416 static u32 last_id; 417 int ret; 418 419 WARN_ON(!ops || !dev); 420 if (!devlink_reload_actions_valid(ops)) 421 return NULL; 422 423 devlink = kvzalloc(struct_size(devlink, priv, priv_size), GFP_KERNEL); 424 if (!devlink) 425 return NULL; 426 427 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b, 428 &last_id, GFP_KERNEL); 429 if (ret < 0) 430 goto err_xa_alloc; 431 432 devlink->dev = get_device(dev); 433 devlink->ops = ops; 434 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); 435 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC); 436 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); 437 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC); 438 write_pnet(&devlink->_net, net); 439 INIT_LIST_HEAD(&devlink->rate_list); 440 INIT_LIST_HEAD(&devlink->linecard_list); 441 INIT_LIST_HEAD(&devlink->sb_list); 442 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list); 443 INIT_LIST_HEAD(&devlink->resource_list); 444 INIT_LIST_HEAD(&devlink->region_list); 445 INIT_LIST_HEAD(&devlink->reporter_list); 446 INIT_LIST_HEAD(&devlink->trap_list); 447 INIT_LIST_HEAD(&devlink->trap_group_list); 448 INIT_LIST_HEAD(&devlink->trap_policer_list); 449 INIT_RCU_WORK(&devlink->rwork, devlink_release); 450 lockdep_register_key(&devlink->lock_key); 451 mutex_init(&devlink->lock); 452 lockdep_set_class(&devlink->lock, &devlink->lock_key); 453 refcount_set(&devlink->refcount, 1); 454 455 return devlink; 456 457 err_xa_alloc: 458 kvfree(devlink); 459 return NULL; 460 } 461 EXPORT_SYMBOL_GPL(devlink_alloc_ns); 462 463 /** 464 * devlink_free - Free devlink instance resources 465 * 466 * @devlink: devlink 467 */ 468 void devlink_free(struct devlink *devlink) 469 { 470 ASSERT_DEVLINK_NOT_REGISTERED(devlink); 471 472 WARN_ON(!list_empty(&devlink->trap_policer_list)); 473 WARN_ON(!list_empty(&devlink->trap_group_list)); 474 WARN_ON(!list_empty(&devlink->trap_list)); 475 WARN_ON(!list_empty(&devlink->reporter_list)); 476 WARN_ON(!list_empty(&devlink->region_list)); 477 WARN_ON(!list_empty(&devlink->resource_list)); 478 WARN_ON(!list_empty(&devlink->dpipe_table_list)); 479 WARN_ON(!list_empty(&devlink->sb_list)); 480 WARN_ON(!list_empty(&devlink->rate_list)); 481 WARN_ON(!list_empty(&devlink->linecard_list)); 482 WARN_ON(!xa_empty(&devlink->ports)); 483 484 xa_destroy(&devlink->nested_rels); 485 xa_destroy(&devlink->snapshot_ids); 486 xa_destroy(&devlink->params); 487 xa_destroy(&devlink->ports); 488 489 xa_erase(&devlinks, devlink->index); 490 491 devlink_put(devlink); 492 } 493 EXPORT_SYMBOL_GPL(devlink_free); 494 495 static void __net_exit devlink_pernet_pre_exit(struct net *net) 496 { 497 struct devlink *devlink; 498 u32 actions_performed; 499 unsigned long index; 500 int err; 501 502 /* In case network namespace is getting destroyed, reload 503 * all devlink instances from this namespace into init_net. 504 */ 505 devlinks_xa_for_each_registered_get(net, index, devlink) { 506 devl_dev_lock(devlink, true); 507 err = 0; 508 if (devl_is_registered(devlink)) 509 err = devlink_reload(devlink, &init_net, 510 DEVLINK_RELOAD_ACTION_DRIVER_REINIT, 511 DEVLINK_RELOAD_LIMIT_UNSPEC, 512 &actions_performed, NULL); 513 devl_dev_unlock(devlink, true); 514 devlink_put(devlink); 515 if (err && err != -EOPNOTSUPP) 516 pr_warn("Failed to reload devlink instance into init_net\n"); 517 } 518 } 519 520 static struct pernet_operations devlink_pernet_ops __net_initdata = { 521 .pre_exit = devlink_pernet_pre_exit, 522 }; 523 524 static struct notifier_block devlink_port_netdevice_nb = { 525 .notifier_call = devlink_port_netdevice_event, 526 }; 527 528 static int __init devlink_init(void) 529 { 530 int err; 531 532 err = register_pernet_subsys(&devlink_pernet_ops); 533 if (err) 534 goto out; 535 err = genl_register_family(&devlink_nl_family); 536 if (err) 537 goto out_unreg_pernet_subsys; 538 err = register_netdevice_notifier(&devlink_port_netdevice_nb); 539 if (!err) 540 return 0; 541 542 genl_unregister_family(&devlink_nl_family); 543 544 out_unreg_pernet_subsys: 545 unregister_pernet_subsys(&devlink_pernet_ops); 546 out: 547 WARN_ON(err); 548 return err; 549 } 550 551 subsys_initcall(devlink_init); 552