1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Generic nexthop implementation 4 * 5 * Copyright (c) 2017-19 Cumulus Networks 6 * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com> 7 */ 8 9 #ifndef __LINUX_NEXTHOP_H 10 #define __LINUX_NEXTHOP_H 11 12 #include <linux/netdevice.h> 13 #include <linux/notifier.h> 14 #include <linux/route.h> 15 #include <linux/types.h> 16 #include <net/ip_fib.h> 17 #include <net/ip6_fib.h> 18 #include <net/netlink.h> 19 20 #define NEXTHOP_VALID_USER_FLAGS RTNH_F_ONLINK 21 22 struct nexthop; 23 24 struct nh_config { 25 u32 nh_id; 26 27 u8 nh_family; 28 u8 nh_protocol; 29 u8 nh_blackhole; 30 u8 nh_fdb; 31 __be16 nh_dst_port; 32 u32 nh_flags; 33 34 int nh_ifindex; 35 struct net_device *dev; 36 37 union { 38 __be32 ipv4; 39 struct in6_addr ipv6; 40 } gw; 41 42 struct nlattr *nh_grp; 43 u16 nh_grp_type; 44 u16 nh_grp_res_num_buckets; 45 unsigned long nh_grp_res_idle_timer; 46 unsigned long nh_grp_res_unbalanced_timer; 47 bool nh_grp_res_has_num_buckets; 48 bool nh_grp_res_has_idle_timer; 49 bool nh_grp_res_has_unbalanced_timer; 50 51 bool nh_hw_stats; 52 53 struct nlattr *nh_encap; 54 u16 nh_encap_type; 55 56 u32 nlflags; 57 struct nl_info nlinfo; 58 }; 59 60 struct nh_info { 61 struct hlist_node dev_hash; /* entry on netns devhash */ 62 struct nexthop *nh_parent; 63 64 u8 family; 65 bool reject_nh; 66 bool fdb_nh; 67 __be16 dst_port; 68 69 union { 70 struct fib_nh_common fib_nhc; 71 struct fib_nh fib_nh; 72 struct fib6_nh fib6_nh; 73 }; 74 }; 75 76 struct nh_res_bucket { 77 struct nh_grp_entry __rcu *nh_entry; 78 atomic_long_t used_time; 79 unsigned long migrated_time; 80 bool occupied; 81 u8 nh_flags; 82 }; 83 84 struct nh_res_table { 85 struct net *net; 86 u32 nhg_id; 87 struct delayed_work upkeep_dw; 88 89 /* List of NHGEs that have too few buckets ("uw" for underweight). 90 * Reclaimed buckets will be given to entries in this list. 91 */ 92 struct list_head uw_nh_entries; 93 unsigned long unbalanced_since; 94 95 u32 idle_timer; 96 u32 unbalanced_timer; 97 98 u16 num_nh_buckets; 99 struct nh_res_bucket nh_buckets[] __counted_by(num_nh_buckets); 100 }; 101 102 struct nh_grp_entry_stats { 103 u64_stats_t packets; 104 struct u64_stats_sync syncp; 105 }; 106 107 struct nh_grp_entry { 108 struct nexthop *nh; 109 struct nh_grp_entry_stats __percpu *stats; 110 u16 weight; 111 112 union { 113 struct { 114 atomic_t upper_bound; 115 } hthr; 116 struct { 117 /* Member on uw_nh_entries. */ 118 struct list_head uw_nh_entry; 119 120 u16 count_buckets; 121 u16 wants_buckets; 122 } res; 123 }; 124 125 struct list_head nh_list; 126 struct nexthop *nh_parent; /* nexthop of group with this entry */ 127 u64 packets_hw; 128 }; 129 130 struct nh_group { 131 struct nh_group *spare; /* spare group for removals */ 132 u16 num_nh; 133 bool is_multipath; 134 bool hash_threshold; 135 bool resilient; 136 bool fdb_nh; 137 bool has_v4; 138 bool hw_stats; 139 140 struct nh_res_table __rcu *res_table; 141 struct nh_grp_entry nh_entries[] __counted_by(num_nh); 142 }; 143 144 struct nexthop { 145 struct rb_node rb_node; /* entry on netns rbtree */ 146 struct list_head fi_list; /* v4 entries using nh */ 147 struct list_head f6i_list; /* v6 entries using nh */ 148 struct list_head fdb_list; /* fdb entries using this nh */ 149 struct list_head grp_list; /* nh group entries using this nh */ 150 struct net *net; 151 152 u32 id; 153 154 u8 protocol; /* app managing this nh */ 155 u8 nh_flags; 156 bool is_group; 157 bool dead; 158 spinlock_t lock; /* protect dead and f6i_list */ 159 160 refcount_t refcnt; 161 struct rcu_head rcu; 162 163 union { 164 struct nh_info __rcu *nh_info; 165 struct nh_group __rcu *nh_grp; 166 }; 167 }; 168 169 enum nexthop_event_type { 170 NEXTHOP_EVENT_DEL, 171 NEXTHOP_EVENT_REPLACE, 172 NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE, 173 NEXTHOP_EVENT_BUCKET_REPLACE, 174 NEXTHOP_EVENT_HW_STATS_REPORT_DELTA, 175 }; 176 177 enum nh_notifier_info_type { 178 NH_NOTIFIER_INFO_TYPE_SINGLE, 179 NH_NOTIFIER_INFO_TYPE_GRP, 180 NH_NOTIFIER_INFO_TYPE_RES_TABLE, 181 NH_NOTIFIER_INFO_TYPE_RES_BUCKET, 182 NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS, 183 }; 184 185 struct nh_notifier_single_info { 186 struct net_device *dev; 187 u8 gw_family; 188 union { 189 __be32 ipv4; 190 struct in6_addr ipv6; 191 }; 192 u32 id; 193 u8 is_reject:1, 194 is_fdb:1, 195 has_encap:1; 196 }; 197 198 struct nh_notifier_grp_entry_info { 199 u16 weight; 200 struct nh_notifier_single_info nh; 201 }; 202 203 struct nh_notifier_grp_info { 204 u16 num_nh; 205 bool is_fdb; 206 bool hw_stats; 207 struct nh_notifier_grp_entry_info nh_entries[] __counted_by(num_nh); 208 }; 209 210 struct nh_notifier_res_bucket_info { 211 u16 bucket_index; 212 unsigned int idle_timer_ms; 213 bool force; 214 struct nh_notifier_single_info old_nh; 215 struct nh_notifier_single_info new_nh; 216 }; 217 218 struct nh_notifier_res_table_info { 219 u16 num_nh_buckets; 220 bool hw_stats; 221 struct nh_notifier_single_info nhs[] __counted_by(num_nh_buckets); 222 }; 223 224 struct nh_notifier_grp_hw_stats_entry_info { 225 u32 id; 226 u64 packets; 227 }; 228 229 struct nh_notifier_grp_hw_stats_info { 230 u16 num_nh; 231 bool hw_stats_used; 232 struct nh_notifier_grp_hw_stats_entry_info stats[] __counted_by(num_nh); 233 }; 234 235 struct nh_notifier_info { 236 struct net *net; 237 struct netlink_ext_ack *extack; 238 u32 id; 239 enum nh_notifier_info_type type; 240 union { 241 struct nh_notifier_single_info *nh; 242 struct nh_notifier_grp_info *nh_grp; 243 struct nh_notifier_res_table_info *nh_res_table; 244 struct nh_notifier_res_bucket_info *nh_res_bucket; 245 struct nh_notifier_grp_hw_stats_info *nh_grp_hw_stats; 246 }; 247 }; 248 249 int register_nexthop_notifier(struct net *net, struct notifier_block *nb, 250 struct netlink_ext_ack *extack); 251 int __unregister_nexthop_notifier(struct net *net, struct notifier_block *nb); 252 int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb); 253 void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap); 254 void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index, 255 bool offload, bool trap); 256 void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets, 257 unsigned long *activity); 258 void nh_grp_hw_stats_report_delta(struct nh_notifier_grp_hw_stats_info *info, 259 unsigned int nh_idx, 260 u64 delta_packets); 261 262 /* caller is holding rcu or rtnl; no reference taken to nexthop */ 263 struct nexthop *nexthop_find_by_id(struct net *net, u32 id); 264 void nexthop_free_rcu(struct rcu_head *head); 265 266 static inline bool nexthop_get(struct nexthop *nh) 267 { 268 return refcount_inc_not_zero(&nh->refcnt); 269 } 270 271 static inline void nexthop_put(struct nexthop *nh) 272 { 273 if (refcount_dec_and_test(&nh->refcnt)) 274 call_rcu_hurry(&nh->rcu, nexthop_free_rcu); 275 } 276 277 static inline bool nexthop_cmp(const struct nexthop *nh1, 278 const struct nexthop *nh2) 279 { 280 return nh1 == nh2; 281 } 282 283 static inline bool nexthop_is_fdb(const struct nexthop *nh) 284 { 285 if (nh->is_group) { 286 const struct nh_group *nh_grp; 287 288 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 289 return nh_grp->fdb_nh; 290 } else { 291 const struct nh_info *nhi; 292 293 nhi = rcu_dereference_rtnl(nh->nh_info); 294 return nhi->fdb_nh; 295 } 296 } 297 298 static inline bool nexthop_has_v4(const struct nexthop *nh) 299 { 300 if (nh->is_group) { 301 struct nh_group *nh_grp; 302 303 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 304 return nh_grp->has_v4; 305 } 306 return false; 307 } 308 309 static inline bool nexthop_is_multipath(const struct nexthop *nh) 310 { 311 if (nh->is_group) { 312 struct nh_group *nh_grp; 313 314 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 315 return nh_grp->is_multipath; 316 } 317 return false; 318 } 319 320 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash); 321 322 static inline unsigned int nexthop_num_path(const struct nexthop *nh) 323 { 324 unsigned int rc = 1; 325 326 if (nh->is_group) { 327 struct nh_group *nh_grp; 328 329 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 330 if (nh_grp->is_multipath) 331 rc = nh_grp->num_nh; 332 } 333 334 return rc; 335 } 336 337 static inline 338 struct nexthop *nexthop_mpath_select(const struct nh_group *nhg, int nhsel) 339 { 340 /* for_nexthops macros in fib_semantics.c grabs a pointer to 341 * the nexthop before checking nhsel 342 */ 343 if (nhsel >= nhg->num_nh) 344 return NULL; 345 346 return nhg->nh_entries[nhsel].nh; 347 } 348 349 static inline 350 int nexthop_mpath_fill_node(struct sk_buff *skb, struct nexthop *nh, 351 u8 rt_family) 352 { 353 struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp); 354 int i; 355 356 for (i = 0; i < nhg->num_nh; i++) { 357 struct nexthop *nhe = nhg->nh_entries[i].nh; 358 struct nh_info *nhi = rcu_dereference_rtnl(nhe->nh_info); 359 struct fib_nh_common *nhc = &nhi->fib_nhc; 360 int weight = nhg->nh_entries[i].weight; 361 362 if (fib_add_nexthop(skb, nhc, weight, rt_family, 0) < 0) 363 return -EMSGSIZE; 364 } 365 366 return 0; 367 } 368 369 /* called with rcu lock */ 370 static inline bool nexthop_is_blackhole(const struct nexthop *nh) 371 { 372 const struct nh_info *nhi; 373 374 if (nh->is_group) { 375 struct nh_group *nh_grp; 376 377 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 378 if (nh_grp->num_nh > 1) 379 return false; 380 381 nh = nh_grp->nh_entries[0].nh; 382 } 383 384 nhi = rcu_dereference_rtnl(nh->nh_info); 385 return nhi->reject_nh; 386 } 387 388 static inline void nexthop_path_fib_result(struct fib_result *res, int hash) 389 { 390 struct nh_info *nhi; 391 struct nexthop *nh; 392 393 nh = nexthop_select_path(res->fi->nh, hash); 394 nhi = rcu_dereference(nh->nh_info); 395 res->nhc = &nhi->fib_nhc; 396 } 397 398 /* called with rcu read lock or rtnl held */ 399 static inline 400 struct fib_nh_common *nexthop_fib_nhc(struct nexthop *nh, int nhsel) 401 { 402 struct nh_info *nhi; 403 404 BUILD_BUG_ON(offsetof(struct fib_nh, nh_common) != 0); 405 BUILD_BUG_ON(offsetof(struct fib6_nh, nh_common) != 0); 406 407 if (nh->is_group) { 408 struct nh_group *nh_grp; 409 410 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 411 if (nh_grp->is_multipath) { 412 nh = nexthop_mpath_select(nh_grp, nhsel); 413 if (!nh) 414 return NULL; 415 } 416 } 417 418 nhi = rcu_dereference_rtnl(nh->nh_info); 419 return &nhi->fib_nhc; 420 } 421 422 /* called from fib_table_lookup with rcu_lock */ 423 static inline 424 struct fib_nh_common *nexthop_get_nhc_lookup(const struct nexthop *nh, 425 int fib_flags, 426 const struct flowi4 *flp, 427 int *nhsel) 428 { 429 struct nh_info *nhi; 430 431 if (nh->is_group) { 432 struct nh_group *nhg = rcu_dereference(nh->nh_grp); 433 int i; 434 435 for (i = 0; i < nhg->num_nh; i++) { 436 struct nexthop *nhe = nhg->nh_entries[i].nh; 437 438 nhi = rcu_dereference(nhe->nh_info); 439 if (fib_lookup_good_nhc(&nhi->fib_nhc, fib_flags, flp)) { 440 *nhsel = i; 441 return &nhi->fib_nhc; 442 } 443 } 444 } else { 445 nhi = rcu_dereference(nh->nh_info); 446 if (fib_lookup_good_nhc(&nhi->fib_nhc, fib_flags, flp)) { 447 *nhsel = 0; 448 return &nhi->fib_nhc; 449 } 450 } 451 452 return NULL; 453 } 454 455 static inline bool nexthop_uses_dev(const struct nexthop *nh, 456 const struct net_device *dev) 457 { 458 struct nh_info *nhi; 459 460 if (nh->is_group) { 461 struct nh_group *nhg = rcu_dereference(nh->nh_grp); 462 int i; 463 464 for (i = 0; i < nhg->num_nh; i++) { 465 struct nexthop *nhe = nhg->nh_entries[i].nh; 466 467 nhi = rcu_dereference(nhe->nh_info); 468 if (nhc_l3mdev_matches_dev(&nhi->fib_nhc, dev)) 469 return true; 470 } 471 } else { 472 nhi = rcu_dereference(nh->nh_info); 473 if (nhc_l3mdev_matches_dev(&nhi->fib_nhc, dev)) 474 return true; 475 } 476 477 return false; 478 } 479 480 static inline unsigned int fib_info_num_path(const struct fib_info *fi) 481 { 482 if (unlikely(fi->nh)) 483 return nexthop_num_path(fi->nh); 484 485 return fi->fib_nhs; 486 } 487 488 int fib_check_nexthop(struct nexthop *nh, u8 scope, 489 struct netlink_ext_ack *extack); 490 491 static inline struct fib_nh_common *fib_info_nhc(struct fib_info *fi, int nhsel) 492 { 493 if (unlikely(fi->nh)) 494 return nexthop_fib_nhc(fi->nh, nhsel); 495 496 return &fi->fib_nh[nhsel].nh_common; 497 } 498 499 /* only used when fib_nh is built into fib_info */ 500 static inline struct fib_nh *fib_info_nh(struct fib_info *fi, int nhsel) 501 { 502 WARN_ON(fi->nh); 503 504 return &fi->fib_nh[nhsel]; 505 } 506 507 /* 508 * IPv6 variants 509 */ 510 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg, 511 struct netlink_ext_ack *extack); 512 513 /* Caller should either hold rcu_read_lock(), or RTNL. */ 514 static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh) 515 { 516 struct nh_info *nhi; 517 518 if (nh->is_group) { 519 struct nh_group *nh_grp; 520 521 nh_grp = rcu_dereference_rtnl(nh->nh_grp); 522 nh = nexthop_mpath_select(nh_grp, 0); 523 if (!nh) 524 return NULL; 525 } 526 527 nhi = rcu_dereference_rtnl(nh->nh_info); 528 if (nhi->family == AF_INET6) 529 return &nhi->fib6_nh; 530 531 return NULL; 532 } 533 534 static inline struct net_device *fib6_info_nh_dev(struct fib6_info *f6i) 535 { 536 struct fib6_nh *fib6_nh; 537 538 fib6_nh = f6i->nh ? nexthop_fib6_nh(f6i->nh) : f6i->fib6_nh; 539 return fib6_nh->fib_nh_dev; 540 } 541 542 static inline void nexthop_path_fib6_result(struct fib6_result *res, int hash) 543 { 544 struct nexthop *nh = res->f6i->nh; 545 struct nh_info *nhi; 546 547 nh = nexthop_select_path(nh, hash); 548 549 nhi = rcu_dereference_rtnl(nh->nh_info); 550 if (nhi->reject_nh) { 551 res->fib6_type = RTN_BLACKHOLE; 552 res->fib6_flags |= RTF_REJECT; 553 res->nh = nexthop_fib6_nh(nh); 554 } else { 555 res->nh = &nhi->fib6_nh; 556 } 557 } 558 559 int nexthop_for_each_fib6_nh(struct nexthop *nh, 560 int (*cb)(struct fib6_nh *nh, void *arg), 561 void *arg); 562 563 static inline int nexthop_get_family(struct nexthop *nh) 564 { 565 struct nh_info *nhi = rcu_dereference_rtnl(nh->nh_info); 566 567 return nhi->family; 568 } 569 570 static inline 571 struct fib_nh_common *nexthop_fdb_nhc(struct nexthop *nh) 572 { 573 struct nh_info *nhi = rcu_dereference_rtnl(nh->nh_info); 574 575 return &nhi->fib_nhc; 576 } 577 578 static inline struct fib_nh_common *nexthop_path_fdb_result(struct nexthop *nh, 579 int hash, 580 __be16 *dst_port) 581 { 582 struct nh_info *nhi; 583 struct nexthop *nhp; 584 585 nhp = nexthop_select_path(nh, hash); 586 if (unlikely(!nhp)) 587 return NULL; 588 nhi = rcu_dereference(nhp->nh_info); 589 *dst_port = nhi->dst_port; 590 return &nhi->fib_nhc; 591 } 592 #endif 593