1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * IPv4 Forwarding Information Base: semantics. 8 * 9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 10 */ 11 12 #include <linux/uaccess.h> 13 #include <linux/bitops.h> 14 #include <linux/types.h> 15 #include <linux/kernel.h> 16 #include <linux/jiffies.h> 17 #include <linux/mm.h> 18 #include <linux/string.h> 19 #include <linux/socket.h> 20 #include <linux/sockios.h> 21 #include <linux/errno.h> 22 #include <linux/in.h> 23 #include <linux/inet.h> 24 #include <linux/inetdevice.h> 25 #include <linux/netdevice.h> 26 #include <linux/if_arp.h> 27 #include <linux/proc_fs.h> 28 #include <linux/skbuff.h> 29 #include <linux/init.h> 30 #include <linux/slab.h> 31 #include <linux/netlink.h> 32 #include <linux/hash.h> 33 #include <linux/nospec.h> 34 35 #include <net/arp.h> 36 #include <net/inet_dscp.h> 37 #include <net/ip.h> 38 #include <net/protocol.h> 39 #include <net/route.h> 40 #include <net/tcp.h> 41 #include <net/sock.h> 42 #include <net/ip_fib.h> 43 #include <net/ip6_fib.h> 44 #include <net/nexthop.h> 45 #include <net/netlink.h> 46 #include <net/rtnh.h> 47 #include <net/lwtunnel.h> 48 #include <net/fib_notifier.h> 49 #include <net/addrconf.h> 50 51 #include "fib_lookup.h" 52 53 /* for_nexthops and change_nexthops only used when nexthop object 54 * is not set in a fib_info. The logic within can reference fib_nh. 55 */ 56 #ifdef CONFIG_IP_ROUTE_MULTIPATH 57 58 #define for_nexthops(fi) { \ 59 int nhsel; const struct fib_nh *nh; \ 60 for (nhsel = 0, nh = (fi)->fib_nh; \ 61 nhsel < fib_info_num_path((fi)); \ 62 nh++, nhsel++) 63 64 #define change_nexthops(fi) { \ 65 int nhsel; struct fib_nh *nexthop_nh; \ 66 for (nhsel = 0, nexthop_nh = (struct fib_nh *)((fi)->fib_nh); \ 67 nhsel < fib_info_num_path((fi)); \ 68 nexthop_nh++, nhsel++) 69 70 #else /* CONFIG_IP_ROUTE_MULTIPATH */ 71 72 /* Hope, that gcc will optimize it to get rid of dummy loop */ 73 74 #define for_nexthops(fi) { \ 75 int nhsel; const struct fib_nh *nh = (fi)->fib_nh; \ 76 for (nhsel = 0; nhsel < 1; nhsel++) 77 78 #define change_nexthops(fi) { \ 79 int nhsel; \ 80 struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh); \ 81 for (nhsel = 0; nhsel < 1; nhsel++) 82 83 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 84 85 #define endfor_nexthops(fi) } 86 87 88 const struct fib_prop fib_props[RTN_MAX + 1] = { 89 [RTN_UNSPEC] = { 90 .error = 0, 91 .scope = RT_SCOPE_NOWHERE, 92 }, 93 [RTN_UNICAST] = { 94 .error = 0, 95 .scope = RT_SCOPE_UNIVERSE, 96 }, 97 [RTN_LOCAL] = { 98 .error = 0, 99 .scope = RT_SCOPE_HOST, 100 }, 101 [RTN_BROADCAST] = { 102 .error = 0, 103 .scope = RT_SCOPE_LINK, 104 }, 105 [RTN_ANYCAST] = { 106 .error = 0, 107 .scope = RT_SCOPE_LINK, 108 }, 109 [RTN_MULTICAST] = { 110 .error = 0, 111 .scope = RT_SCOPE_UNIVERSE, 112 }, 113 [RTN_BLACKHOLE] = { 114 .error = -EINVAL, 115 .scope = RT_SCOPE_UNIVERSE, 116 }, 117 [RTN_UNREACHABLE] = { 118 .error = -EHOSTUNREACH, 119 .scope = RT_SCOPE_UNIVERSE, 120 }, 121 [RTN_PROHIBIT] = { 122 .error = -EACCES, 123 .scope = RT_SCOPE_UNIVERSE, 124 }, 125 [RTN_THROW] = { 126 .error = -EAGAIN, 127 .scope = RT_SCOPE_UNIVERSE, 128 }, 129 [RTN_NAT] = { 130 .error = -EINVAL, 131 .scope = RT_SCOPE_NOWHERE, 132 }, 133 [RTN_XRESOLVE] = { 134 .error = -EINVAL, 135 .scope = RT_SCOPE_NOWHERE, 136 }, 137 }; 138 139 static void rt_fibinfo_free(struct rtable __rcu **rtp) 140 { 141 struct rtable *rt = rcu_dereference_protected(*rtp, 1); 142 143 if (!rt) 144 return; 145 146 /* Not even needed : RCU_INIT_POINTER(*rtp, NULL); 147 * because we waited an RCU grace period before calling 148 * free_fib_info_rcu() 149 */ 150 151 dst_dev_put(&rt->dst); 152 dst_release_immediate(&rt->dst); 153 } 154 155 static void free_nh_exceptions(struct fib_nh_common *nhc) 156 { 157 struct fnhe_hash_bucket *hash; 158 int i; 159 160 hash = rcu_dereference_protected(nhc->nhc_exceptions, 1); 161 if (!hash) 162 return; 163 for (i = 0; i < FNHE_HASH_SIZE; i++) { 164 struct fib_nh_exception *fnhe; 165 166 fnhe = rcu_dereference_protected(hash[i].chain, 1); 167 while (fnhe) { 168 struct fib_nh_exception *next; 169 170 next = rcu_dereference_protected(fnhe->fnhe_next, 1); 171 172 rt_fibinfo_free(&fnhe->fnhe_rth_input); 173 rt_fibinfo_free(&fnhe->fnhe_rth_output); 174 175 kfree(fnhe); 176 177 fnhe = next; 178 } 179 } 180 kfree(hash); 181 } 182 183 static void rt_fibinfo_free_cpus(struct rtable __rcu * __percpu *rtp) 184 { 185 int cpu; 186 187 if (!rtp) 188 return; 189 190 for_each_possible_cpu(cpu) { 191 struct rtable *rt; 192 193 rt = rcu_dereference_protected(*per_cpu_ptr(rtp, cpu), 1); 194 if (rt) { 195 dst_dev_put(&rt->dst); 196 dst_release_immediate(&rt->dst); 197 } 198 } 199 free_percpu(rtp); 200 } 201 202 void fib_nh_common_release(struct fib_nh_common *nhc) 203 { 204 netdev_put(nhc->nhc_dev, &nhc->nhc_dev_tracker); 205 lwtstate_put(nhc->nhc_lwtstate); 206 rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output); 207 rt_fibinfo_free(&nhc->nhc_rth_input); 208 free_nh_exceptions(nhc); 209 } 210 211 void fib_nh_release(struct net *net, struct fib_nh *fib_nh) 212 { 213 #ifdef CONFIG_IP_ROUTE_CLASSID 214 if (fib_nh->nh_tclassid) 215 atomic_dec(&net->ipv4.fib_num_tclassid_users); 216 #endif 217 fib_nh_common_release(&fib_nh->nh_common); 218 } 219 220 /* Release a nexthop info record */ 221 static void free_fib_info_rcu(struct rcu_head *head) 222 { 223 struct fib_info *fi = container_of(head, struct fib_info, rcu); 224 225 if (fi->nh) { 226 nexthop_put(fi->nh); 227 } else { 228 change_nexthops(fi) { 229 fib_nh_release(fi->fib_net, nexthop_nh); 230 } endfor_nexthops(fi); 231 } 232 233 ip_fib_metrics_put(fi->fib_metrics); 234 235 kfree(fi); 236 } 237 238 void free_fib_info(struct fib_info *fi) 239 { 240 if (fi->fib_dead == 0) { 241 pr_warn("Freeing alive fib_info %p\n", fi); 242 return; 243 } 244 245 call_rcu_hurry(&fi->rcu, free_fib_info_rcu); 246 } 247 EXPORT_SYMBOL_GPL(free_fib_info); 248 249 void fib_release_info(struct fib_info *fi) 250 { 251 ASSERT_RTNL(); 252 if (fi && refcount_dec_and_test(&fi->fib_treeref)) { 253 hlist_del(&fi->fib_hash); 254 fi->fib_net->ipv4.fib_info_cnt--; 255 256 if (fi->fib_prefsrc) 257 hlist_del(&fi->fib_lhash); 258 if (fi->nh) { 259 list_del(&fi->nh_list); 260 } else { 261 change_nexthops(fi) { 262 if (!nexthop_nh->fib_nh_dev) 263 continue; 264 hlist_del_rcu(&nexthop_nh->nh_hash); 265 } endfor_nexthops(fi) 266 } 267 /* Paired with READ_ONCE() from fib_table_lookup() */ 268 WRITE_ONCE(fi->fib_dead, 1); 269 fib_info_put(fi); 270 } 271 } 272 273 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi) 274 { 275 const struct fib_nh *onh; 276 277 if (fi->nh || ofi->nh) 278 return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1; 279 280 if (ofi->fib_nhs == 0) 281 return 0; 282 283 for_nexthops(fi) { 284 onh = fib_info_nh(ofi, nhsel); 285 286 if (nh->fib_nh_oif != onh->fib_nh_oif || 287 nh->fib_nh_gw_family != onh->fib_nh_gw_family || 288 nh->fib_nh_scope != onh->fib_nh_scope || 289 #ifdef CONFIG_IP_ROUTE_MULTIPATH 290 nh->fib_nh_weight != onh->fib_nh_weight || 291 #endif 292 #ifdef CONFIG_IP_ROUTE_CLASSID 293 nh->nh_tclassid != onh->nh_tclassid || 294 #endif 295 lwtunnel_cmp_encap(nh->fib_nh_lws, onh->fib_nh_lws) || 296 ((nh->fib_nh_flags ^ onh->fib_nh_flags) & ~RTNH_COMPARE_MASK)) 297 return -1; 298 299 if (nh->fib_nh_gw_family == AF_INET && 300 nh->fib_nh_gw4 != onh->fib_nh_gw4) 301 return -1; 302 303 if (nh->fib_nh_gw_family == AF_INET6 && 304 ipv6_addr_cmp(&nh->fib_nh_gw6, &onh->fib_nh_gw6)) 305 return -1; 306 } endfor_nexthops(fi); 307 return 0; 308 } 309 310 static struct hlist_head *fib_nh_head(struct net_device *dev) 311 { 312 return &dev->fib_nh_head; 313 } 314 315 static unsigned int fib_info_hashfn_1(int init_val, u8 protocol, u8 scope, 316 u32 prefsrc, u32 priority) 317 { 318 unsigned int val = init_val; 319 320 val ^= (protocol << 8) | scope; 321 val ^= prefsrc; 322 val ^= priority; 323 324 return val; 325 } 326 327 static unsigned int fib_info_hashfn_result(const struct net *net, 328 unsigned int val) 329 { 330 return hash_32(val ^ net_hash_mix(net), net->ipv4.fib_info_hash_bits); 331 } 332 333 static struct hlist_head *fib_info_hash_bucket(struct fib_info *fi) 334 { 335 struct net *net = fi->fib_net; 336 unsigned int val; 337 338 val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol, 339 fi->fib_scope, (__force u32)fi->fib_prefsrc, 340 fi->fib_priority); 341 342 if (fi->nh) { 343 val ^= fi->nh->id; 344 } else { 345 for_nexthops(fi) { 346 val ^= nh->fib_nh_oif; 347 } endfor_nexthops(fi) 348 } 349 350 return &net->ipv4.fib_info_hash[fib_info_hashfn_result(net, val)]; 351 } 352 353 static struct hlist_head *fib_info_laddrhash_bucket(const struct net *net, 354 __be32 val) 355 { 356 unsigned int hash_bits = net->ipv4.fib_info_hash_bits; 357 u32 slot; 358 359 slot = hash_32(net_hash_mix(net) ^ (__force u32)val, hash_bits); 360 361 return &net->ipv4.fib_info_hash[(1 << hash_bits) + slot]; 362 } 363 364 static struct hlist_head *fib_info_hash_alloc(unsigned int hash_bits) 365 { 366 /* The second half is used for prefsrc */ 367 return kvzalloc_objs(struct hlist_head, (1 << hash_bits) * 2); 368 } 369 370 static void fib_info_hash_free(struct hlist_head *head) 371 { 372 kvfree(head); 373 } 374 375 static void fib_info_hash_grow(struct net *net) 376 { 377 unsigned int old_size = 1 << net->ipv4.fib_info_hash_bits; 378 struct hlist_head *new_info_hash, *old_info_hash; 379 unsigned int i; 380 381 if (net->ipv4.fib_info_cnt < old_size) 382 return; 383 384 new_info_hash = fib_info_hash_alloc(net->ipv4.fib_info_hash_bits + 1); 385 if (!new_info_hash) 386 return; 387 388 old_info_hash = net->ipv4.fib_info_hash; 389 net->ipv4.fib_info_hash = new_info_hash; 390 net->ipv4.fib_info_hash_bits += 1; 391 392 for (i = 0; i < old_size; i++) { 393 struct hlist_head *head = &old_info_hash[i]; 394 struct hlist_node *n; 395 struct fib_info *fi; 396 397 hlist_for_each_entry_safe(fi, n, head, fib_hash) 398 hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi)); 399 } 400 401 for (i = 0; i < old_size; i++) { 402 struct hlist_head *lhead = &old_info_hash[old_size + i]; 403 struct hlist_node *n; 404 struct fib_info *fi; 405 406 hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) 407 hlist_add_head(&fi->fib_lhash, 408 fib_info_laddrhash_bucket(fi->fib_net, 409 fi->fib_prefsrc)); 410 } 411 412 fib_info_hash_free(old_info_hash); 413 } 414 415 /* no metrics, only nexthop id */ 416 static struct fib_info *fib_find_info_nh(struct net *net, 417 const struct fib_config *cfg) 418 { 419 struct hlist_head *head; 420 struct fib_info *fi; 421 unsigned int hash; 422 423 hash = fib_info_hashfn_1(cfg->fc_nh_id, 424 cfg->fc_protocol, cfg->fc_scope, 425 (__force u32)cfg->fc_prefsrc, 426 cfg->fc_priority); 427 hash = fib_info_hashfn_result(net, hash); 428 head = &net->ipv4.fib_info_hash[hash]; 429 430 hlist_for_each_entry(fi, head, fib_hash) { 431 if (!fi->nh || fi->nh->id != cfg->fc_nh_id) 432 continue; 433 434 if (cfg->fc_protocol == fi->fib_protocol && 435 cfg->fc_scope == fi->fib_scope && 436 cfg->fc_prefsrc == fi->fib_prefsrc && 437 cfg->fc_priority == fi->fib_priority && 438 cfg->fc_type == fi->fib_type && 439 cfg->fc_table == fi->fib_tb_id && 440 !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK)) 441 return fi; 442 } 443 444 return NULL; 445 } 446 447 static struct fib_info *fib_find_info(struct fib_info *nfi) 448 { 449 struct hlist_head *head = fib_info_hash_bucket(nfi); 450 struct fib_info *fi; 451 452 hlist_for_each_entry(fi, head, fib_hash) { 453 if (fi->fib_nhs != nfi->fib_nhs) 454 continue; 455 456 if (nfi->fib_protocol == fi->fib_protocol && 457 nfi->fib_scope == fi->fib_scope && 458 nfi->fib_prefsrc == fi->fib_prefsrc && 459 nfi->fib_priority == fi->fib_priority && 460 nfi->fib_type == fi->fib_type && 461 nfi->fib_tb_id == fi->fib_tb_id && 462 memcmp(nfi->fib_metrics, fi->fib_metrics, 463 sizeof(u32) * RTAX_MAX) == 0 && 464 !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) && 465 nh_comp(fi, nfi) == 0) 466 return fi; 467 } 468 469 return NULL; 470 } 471 472 /* Check, that the gateway is already configured. 473 * Used only by redirect accept routine, under rcu_read_lock(); 474 */ 475 int ip_fib_check_default(__be32 gw, struct net_device *dev) 476 { 477 struct hlist_head *head; 478 struct fib_nh *nh; 479 480 head = fib_nh_head(dev); 481 482 hlist_for_each_entry_rcu(nh, head, nh_hash) { 483 DEBUG_NET_WARN_ON_ONCE(nh->fib_nh_dev != dev); 484 if (nh->fib_nh_gw4 == gw && 485 !(nh->fib_nh_flags & RTNH_F_DEAD)) { 486 return 0; 487 } 488 } 489 490 return -1; 491 } 492 493 static size_t fib_nexthop_nlmsg_size(const struct fib_nh_common *nhc, 494 bool skip_oif) 495 { 496 size_t nhsize = 0; 497 498 switch (nhc->nhc_gw_family) { 499 case AF_INET: 500 nhsize += nla_total_size(4); /* RTA_GATEWAY */ 501 break; 502 case AF_INET6: 503 nhsize += nla_total_size(sizeof(struct rtvia) + 504 sizeof(struct in6_addr)); 505 break; 506 } 507 508 if (!skip_oif && nhc->nhc_dev) 509 nhsize += nla_total_size(4); /* RTA_OIF */ 510 511 if (nhc->nhc_lwtstate) { 512 /* RTA_ENCAP */ 513 nhsize += lwtunnel_get_encap_size(nhc->nhc_lwtstate); 514 /* RTA_ENCAP_TYPE */ 515 nhsize += nla_total_size(2); 516 } 517 518 return nhsize; 519 } 520 521 size_t fib_nlmsg_size(struct fib_info *fi) 522 { 523 size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg)) 524 + nla_total_size(4) /* RTA_TABLE */ 525 + nla_total_size(4) /* RTA_DST */ 526 + nla_total_size(4) /* RTA_PRIORITY */ 527 + nla_total_size(4) /* RTA_PREFSRC */ 528 + nla_total_size(TCP_CA_NAME_MAX); /* RTAX_CC_ALGO */ 529 unsigned int nhs = fib_info_num_path(fi); 530 531 /* space for nested metrics */ 532 payload += nla_total_size((RTAX_MAX * nla_total_size(4))); 533 534 if (fi->nh) 535 payload += nla_total_size(4); /* RTA_NH_ID */ 536 537 if (nhs) { 538 size_t mpsize = 0; 539 unsigned int i; 540 541 for (i = 0; i < fib_info_num_path(fi); i++) { 542 struct fib_nh_common *nhc = fib_info_nhc(fi, i); 543 size_t nhsize; 544 545 nhsize = fib_nexthop_nlmsg_size(nhc, nhs != 1); 546 547 if (nhs != 1) 548 nhsize += NLA_ALIGN(sizeof(struct rtnexthop)); 549 550 #ifdef CONFIG_IP_ROUTE_CLASSID 551 if (nhc->nhc_family == AF_INET) { 552 struct fib_nh *nh; 553 554 nh = container_of(nhc, struct fib_nh, nh_common); 555 if (nh->nh_tclassid) 556 nhsize += nla_total_size(4); 557 } 558 #endif 559 if (nhs == 1) 560 payload += nhsize; 561 else 562 mpsize += nhsize; 563 } 564 565 if (nhs != 1) 566 payload += nla_total_size(mpsize); 567 } 568 569 return payload; 570 } 571 572 void rtmsg_fib(int event, __be32 key, struct fib_alias *fa, 573 int dst_len, u32 tb_id, const struct nl_info *info, 574 unsigned int nlm_flags) 575 { 576 struct fib_rt_info fri; 577 struct sk_buff *skb; 578 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 579 int err = -ENOBUFS; 580 581 skb = nlmsg_new(fib_nlmsg_size(fa->fa_info), GFP_KERNEL); 582 if (!skb) 583 goto errout; 584 585 fri.fi = fa->fa_info; 586 fri.tb_id = tb_id; 587 fri.dst = key; 588 fri.dst_len = dst_len; 589 fri.dscp = fa->fa_dscp; 590 fri.type = fa->fa_type; 591 fri.offload = READ_ONCE(fa->offload); 592 fri.trap = READ_ONCE(fa->trap); 593 fri.offload_failed = READ_ONCE(fa->offload_failed); 594 err = fib_dump_info(skb, info->portid, seq, event, &fri, nlm_flags); 595 if (err < 0) { 596 /* -EMSGSIZE implies BUG in fib_nlmsg_size() */ 597 WARN_ON(err == -EMSGSIZE); 598 kfree_skb(skb); 599 goto errout; 600 } 601 rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_IPV4_ROUTE, 602 info->nlh, GFP_KERNEL); 603 return; 604 errout: 605 rtnl_set_sk_err(info->nl_net, RTNLGRP_IPV4_ROUTE, err); 606 } 607 608 static int fib_detect_death(struct fib_info *fi, int order, 609 struct fib_info **last_resort, int *last_idx, 610 int dflt) 611 { 612 const struct fib_nh_common *nhc = fib_info_nhc(fi, 0); 613 struct net *net = fi->fib_net; 614 int state = NUD_NONE; 615 struct neighbour *n; 616 617 if (likely(nhc->nhc_gw_family == AF_INET)) 618 n = neigh_lookup(arp_table(net), &nhc->nhc_gw.ipv4, nhc->nhc_dev); 619 else if (IS_ENABLED(CONFIG_IPV6) && nhc->nhc_gw_family == AF_INET6) 620 n = neigh_lookup(nd_table(net), &nhc->nhc_gw.ipv6, nhc->nhc_dev); 621 else 622 n = NULL; 623 624 if (n) { 625 state = READ_ONCE(n->nud_state); 626 neigh_release(n); 627 } else { 628 return 0; 629 } 630 if (state == NUD_REACHABLE) 631 return 0; 632 if ((state & NUD_VALID) && order != dflt) 633 return 0; 634 if ((state & NUD_VALID) || 635 (*last_idx < 0 && order > dflt && state != NUD_INCOMPLETE)) { 636 *last_resort = fi; 637 *last_idx = order; 638 } 639 return 1; 640 } 641 642 int fib_nh_common_init(struct net *net, struct fib_nh_common *nhc, 643 struct nlattr *encap, u16 encap_type, 644 void *cfg, gfp_t gfp_flags, 645 struct netlink_ext_ack *extack) 646 { 647 int err; 648 649 nhc->nhc_pcpu_rth_output = alloc_percpu_gfp(struct rtable __rcu *, 650 gfp_flags); 651 if (!nhc->nhc_pcpu_rth_output) 652 return -ENOMEM; 653 654 if (encap) { 655 struct lwtunnel_state *lwtstate; 656 657 err = lwtunnel_build_state(net, encap_type, encap, 658 nhc->nhc_family, cfg, &lwtstate, 659 extack); 660 if (err) 661 goto lwt_failure; 662 663 nhc->nhc_lwtstate = lwtstate_get(lwtstate); 664 } 665 666 return 0; 667 668 lwt_failure: 669 rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output); 670 nhc->nhc_pcpu_rth_output = NULL; 671 return err; 672 } 673 674 int fib_nh_init(struct net *net, struct fib_nh *nh, 675 struct fib_config *cfg, int nh_weight, 676 struct netlink_ext_ack *extack) 677 { 678 int err; 679 680 nh->fib_nh_family = AF_INET; 681 682 err = fib_nh_common_init(net, &nh->nh_common, cfg->fc_encap, 683 cfg->fc_encap_type, cfg, GFP_KERNEL, extack); 684 if (err) 685 return err; 686 687 nh->fib_nh_oif = cfg->fc_oif; 688 nh->fib_nh_gw_family = cfg->fc_gw_family; 689 if (cfg->fc_gw_family == AF_INET) 690 nh->fib_nh_gw4 = cfg->fc_gw4; 691 else if (cfg->fc_gw_family == AF_INET6) 692 nh->fib_nh_gw6 = cfg->fc_gw6; 693 694 nh->fib_nh_flags = cfg->fc_flags; 695 696 #ifdef CONFIG_IP_ROUTE_CLASSID 697 nh->nh_tclassid = cfg->fc_flow; 698 if (nh->nh_tclassid) 699 atomic_inc(&net->ipv4.fib_num_tclassid_users); 700 #endif 701 #ifdef CONFIG_IP_ROUTE_MULTIPATH 702 nh->fib_nh_weight = nh_weight; 703 #endif 704 return 0; 705 } 706 707 #ifdef CONFIG_IP_ROUTE_MULTIPATH 708 709 static int fib_count_nexthops(struct rtnexthop *rtnh, int remaining, 710 struct netlink_ext_ack *extack) 711 { 712 int nhs = 0; 713 714 while (rtnh_ok(rtnh, remaining)) { 715 nhs++; 716 rtnh = rtnh_next(rtnh, &remaining); 717 } 718 719 /* leftover implies invalid nexthop configuration, discard it */ 720 if (remaining > 0) { 721 NL_SET_ERR_MSG(extack, 722 "Invalid nexthop configuration - extra data after nexthops"); 723 nhs = 0; 724 } 725 726 return nhs; 727 } 728 729 static int fib_gw_from_attr(__be32 *gw, struct nlattr *nla, 730 struct netlink_ext_ack *extack) 731 { 732 if (nla_len(nla) < sizeof(*gw)) { 733 NL_SET_ERR_MSG(extack, "Invalid IPv4 address in RTA_GATEWAY"); 734 return -EINVAL; 735 } 736 737 *gw = nla_get_in_addr(nla); 738 739 return 0; 740 } 741 742 /* only called when fib_nh is integrated into fib_info */ 743 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, 744 int remaining, struct fib_config *cfg, 745 struct netlink_ext_ack *extack) 746 { 747 struct net *net = fi->fib_net; 748 struct fib_config fib_cfg; 749 struct fib_nh *nh; 750 int ret; 751 752 change_nexthops(fi) { 753 int attrlen; 754 755 memset(&fib_cfg, 0, sizeof(fib_cfg)); 756 757 if (!rtnh_ok(rtnh, remaining)) { 758 NL_SET_ERR_MSG(extack, 759 "Invalid nexthop configuration - extra data after nexthop"); 760 return -EINVAL; 761 } 762 763 if (rtnh->rtnh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) { 764 NL_SET_ERR_MSG(extack, 765 "Invalid flags for nexthop - can not contain DEAD or LINKDOWN"); 766 return -EINVAL; 767 } 768 769 fib_cfg.fc_flags = (cfg->fc_flags & ~0xFF) | rtnh->rtnh_flags; 770 fib_cfg.fc_oif = rtnh->rtnh_ifindex; 771 772 attrlen = rtnh_attrlen(rtnh); 773 if (attrlen > 0) { 774 struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh); 775 776 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 777 nlav = nla_find(attrs, attrlen, RTA_VIA); 778 if (nla && nlav) { 779 NL_SET_ERR_MSG(extack, 780 "Nexthop configuration can not contain both GATEWAY and VIA"); 781 return -EINVAL; 782 } 783 if (nla) { 784 ret = fib_gw_from_attr(&fib_cfg.fc_gw4, nla, 785 extack); 786 if (ret) 787 goto errout; 788 789 if (fib_cfg.fc_gw4) 790 fib_cfg.fc_gw_family = AF_INET; 791 } else if (nlav) { 792 ret = fib_gw_from_via(&fib_cfg, nlav, extack); 793 if (ret) 794 goto errout; 795 } 796 797 nla = nla_find(attrs, attrlen, RTA_FLOW); 798 if (nla) { 799 if (nla_len(nla) < sizeof(u32)) { 800 NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW"); 801 return -EINVAL; 802 } 803 fib_cfg.fc_flow = nla_get_u32(nla); 804 } 805 806 fib_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 807 /* RTA_ENCAP_TYPE length checked in 808 * lwtunnel_valid_encap_type_attr 809 */ 810 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 811 if (nla) 812 fib_cfg.fc_encap_type = nla_get_u16(nla); 813 } 814 815 ret = fib_nh_init(net, nexthop_nh, &fib_cfg, 816 rtnh->rtnh_hops + 1, extack); 817 if (ret) 818 goto errout; 819 820 rtnh = rtnh_next(rtnh, &remaining); 821 } endfor_nexthops(fi); 822 823 ret = -EINVAL; 824 nh = fib_info_nh(fi, 0); 825 if (cfg->fc_oif && nh->fib_nh_oif != cfg->fc_oif) { 826 NL_SET_ERR_MSG(extack, 827 "Nexthop device index does not match RTA_OIF"); 828 goto errout; 829 } 830 if (cfg->fc_gw_family) { 831 if (cfg->fc_gw_family != nh->fib_nh_gw_family || 832 (cfg->fc_gw_family == AF_INET && 833 nh->fib_nh_gw4 != cfg->fc_gw4) || 834 (cfg->fc_gw_family == AF_INET6 && 835 ipv6_addr_cmp(&nh->fib_nh_gw6, &cfg->fc_gw6))) { 836 NL_SET_ERR_MSG(extack, 837 "Nexthop gateway does not match RTA_GATEWAY or RTA_VIA"); 838 goto errout; 839 } 840 } 841 #ifdef CONFIG_IP_ROUTE_CLASSID 842 if (cfg->fc_flow && nh->nh_tclassid != cfg->fc_flow) { 843 NL_SET_ERR_MSG(extack, 844 "Nexthop class id does not match RTA_FLOW"); 845 goto errout; 846 } 847 #endif 848 ret = 0; 849 errout: 850 return ret; 851 } 852 853 /* only called when fib_nh is integrated into fib_info */ 854 static void fib_rebalance(struct fib_info *fi) 855 { 856 int total; 857 int w; 858 859 if (fib_info_num_path(fi) < 2) 860 return; 861 862 total = 0; 863 for_nexthops(fi) { 864 if (nh->fib_nh_flags & RTNH_F_DEAD) 865 continue; 866 867 if (ip_ignore_linkdown(nh->fib_nh_dev) && 868 nh->fib_nh_flags & RTNH_F_LINKDOWN) 869 continue; 870 871 total += nh->fib_nh_weight; 872 } endfor_nexthops(fi); 873 874 w = 0; 875 change_nexthops(fi) { 876 int upper_bound; 877 878 if (!total || nexthop_nh->fib_nh_flags & RTNH_F_DEAD) { 879 upper_bound = -1; 880 } else if (ip_ignore_linkdown(nexthop_nh->fib_nh_dev) && 881 nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 882 upper_bound = -1; 883 } else { 884 w += nexthop_nh->fib_nh_weight; 885 upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, 886 total) - 1; 887 } 888 889 atomic_set(&nexthop_nh->fib_nh_upper_bound, upper_bound); 890 } endfor_nexthops(fi); 891 } 892 #else /* CONFIG_IP_ROUTE_MULTIPATH */ 893 894 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, 895 int remaining, struct fib_config *cfg, 896 struct netlink_ext_ack *extack) 897 { 898 NL_SET_ERR_MSG(extack, "Multipath support not enabled in kernel"); 899 900 return -EINVAL; 901 } 902 903 #define fib_rebalance(fi) do { } while (0) 904 905 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 906 907 static int fib_encap_match(struct net *net, u16 encap_type, 908 struct nlattr *encap, 909 const struct fib_nh *nh, 910 const struct fib_config *cfg, 911 struct netlink_ext_ack *extack) 912 { 913 struct lwtunnel_state *lwtstate; 914 int ret, result = 0; 915 916 if (encap_type == LWTUNNEL_ENCAP_NONE) 917 return 0; 918 919 ret = lwtunnel_build_state(net, encap_type, encap, AF_INET, 920 cfg, &lwtstate, extack); 921 if (!ret) { 922 result = lwtunnel_cmp_encap(lwtstate, nh->fib_nh_lws); 923 lwtstate_free(lwtstate); 924 } 925 926 return result; 927 } 928 929 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi, 930 struct netlink_ext_ack *extack) 931 { 932 #ifdef CONFIG_IP_ROUTE_MULTIPATH 933 struct rtnexthop *rtnh; 934 int remaining; 935 #endif 936 937 if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority) 938 return 1; 939 940 if (cfg->fc_nh_id) { 941 if (fi->nh && cfg->fc_nh_id == fi->nh->id) 942 return 0; 943 return 1; 944 } 945 946 if (fi->nh) { 947 if (cfg->fc_oif || cfg->fc_gw_family || cfg->fc_mp) 948 return 1; 949 return 0; 950 } 951 952 if (cfg->fc_oif || cfg->fc_gw_family) { 953 struct fib_nh *nh; 954 955 nh = fib_info_nh(fi, 0); 956 if (cfg->fc_encap) { 957 if (fib_encap_match(net, cfg->fc_encap_type, 958 cfg->fc_encap, nh, cfg, extack)) 959 return 1; 960 } 961 #ifdef CONFIG_IP_ROUTE_CLASSID 962 if (cfg->fc_flow && 963 cfg->fc_flow != nh->nh_tclassid) 964 return 1; 965 #endif 966 if ((cfg->fc_oif && cfg->fc_oif != nh->fib_nh_oif) || 967 (cfg->fc_gw_family && 968 cfg->fc_gw_family != nh->fib_nh_gw_family)) 969 return 1; 970 971 if (cfg->fc_gw_family == AF_INET && 972 cfg->fc_gw4 != nh->fib_nh_gw4) 973 return 1; 974 975 if (cfg->fc_gw_family == AF_INET6 && 976 ipv6_addr_cmp(&cfg->fc_gw6, &nh->fib_nh_gw6)) 977 return 1; 978 979 return 0; 980 } 981 982 #ifdef CONFIG_IP_ROUTE_MULTIPATH 983 if (!cfg->fc_mp) 984 return 0; 985 986 rtnh = cfg->fc_mp; 987 remaining = cfg->fc_mp_len; 988 989 for_nexthops(fi) { 990 int attrlen; 991 992 if (!rtnh_ok(rtnh, remaining)) 993 return -EINVAL; 994 995 if (rtnh->rtnh_ifindex && rtnh->rtnh_ifindex != nh->fib_nh_oif) 996 return 1; 997 998 attrlen = rtnh_attrlen(rtnh); 999 if (attrlen > 0) { 1000 struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh); 1001 int err; 1002 1003 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 1004 nlav = nla_find(attrs, attrlen, RTA_VIA); 1005 if (nla && nlav) { 1006 NL_SET_ERR_MSG(extack, 1007 "Nexthop configuration can not contain both GATEWAY and VIA"); 1008 return -EINVAL; 1009 } 1010 1011 if (nla) { 1012 __be32 gw; 1013 1014 err = fib_gw_from_attr(&gw, nla, extack); 1015 if (err) 1016 return err; 1017 1018 if (nh->fib_nh_gw_family != AF_INET || 1019 gw != nh->fib_nh_gw4) 1020 return 1; 1021 } else if (nlav) { 1022 struct fib_config cfg2; 1023 1024 err = fib_gw_from_via(&cfg2, nlav, extack); 1025 if (err) 1026 return err; 1027 1028 switch (nh->fib_nh_gw_family) { 1029 case AF_INET: 1030 if (cfg2.fc_gw_family != AF_INET || 1031 cfg2.fc_gw4 != nh->fib_nh_gw4) 1032 return 1; 1033 break; 1034 case AF_INET6: 1035 if (cfg2.fc_gw_family != AF_INET6 || 1036 ipv6_addr_cmp(&cfg2.fc_gw6, 1037 &nh->fib_nh_gw6)) 1038 return 1; 1039 break; 1040 } 1041 } 1042 1043 #ifdef CONFIG_IP_ROUTE_CLASSID 1044 nla = nla_find(attrs, attrlen, RTA_FLOW); 1045 if (nla) { 1046 if (nla_len(nla) < sizeof(u32)) { 1047 NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW"); 1048 return -EINVAL; 1049 } 1050 if (nla_get_u32(nla) != nh->nh_tclassid) 1051 return 1; 1052 } 1053 #endif 1054 } 1055 1056 rtnh = rtnh_next(rtnh, &remaining); 1057 } endfor_nexthops(fi); 1058 #endif 1059 return 0; 1060 } 1061 1062 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi) 1063 { 1064 struct nlattr *nla; 1065 int remaining; 1066 1067 if (!cfg->fc_mx) 1068 return true; 1069 1070 nla_for_each_attr(nla, cfg->fc_mx, cfg->fc_mx_len, remaining) { 1071 int type = nla_type(nla); 1072 u32 fi_val, val; 1073 1074 if (!type) 1075 continue; 1076 if (type > RTAX_MAX) 1077 return false; 1078 1079 type = array_index_nospec(type, RTAX_MAX + 1); 1080 if (type == RTAX_CC_ALGO) { 1081 char tmp[TCP_CA_NAME_MAX]; 1082 bool ecn_ca = false; 1083 1084 nla_strscpy(tmp, nla, sizeof(tmp)); 1085 val = tcp_ca_get_key_by_name(tmp, &ecn_ca); 1086 } else { 1087 if (nla_len(nla) != sizeof(u32)) 1088 return false; 1089 val = nla_get_u32(nla); 1090 } 1091 1092 fi_val = fi->fib_metrics->metrics[type - 1]; 1093 if (type == RTAX_FEATURES) 1094 fi_val &= ~DST_FEATURE_ECN_CA; 1095 1096 if (fi_val != val) 1097 return false; 1098 } 1099 1100 return true; 1101 } 1102 1103 static int fib_check_nh_v6_gw(struct net *net, struct fib_nh *nh, 1104 u32 table, struct netlink_ext_ack *extack) 1105 { 1106 struct fib6_config cfg = { 1107 .fc_table = table, 1108 .fc_flags = nh->fib_nh_flags | RTF_GATEWAY, 1109 .fc_ifindex = nh->fib_nh_oif, 1110 .fc_gateway = nh->fib_nh_gw6, 1111 }; 1112 struct fib6_nh fib6_nh = {}; 1113 int err; 1114 1115 err = fib6_nh_init(net, &fib6_nh, &cfg, GFP_KERNEL, extack); 1116 if (!err) { 1117 nh->fib_nh_dev = fib6_nh.fib_nh_dev; 1118 netdev_hold(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, 1119 GFP_KERNEL); 1120 nh->fib_nh_oif = nh->fib_nh_dev->ifindex; 1121 nh->fib_nh_scope = RT_SCOPE_LINK; 1122 1123 fib6_nh_release(&fib6_nh); 1124 } 1125 1126 return err; 1127 } 1128 1129 /* 1130 * Picture 1131 * ------- 1132 * 1133 * Semantics of nexthop is very messy by historical reasons. 1134 * We have to take into account, that: 1135 * a) gateway can be actually local interface address, 1136 * so that gatewayed route is direct. 1137 * b) gateway must be on-link address, possibly 1138 * described not by an ifaddr, but also by a direct route. 1139 * c) If both gateway and interface are specified, they should not 1140 * contradict. 1141 * d) If we use tunnel routes, gateway could be not on-link. 1142 * 1143 * Attempt to reconcile all of these (alas, self-contradictory) conditions 1144 * results in pretty ugly and hairy code with obscure logic. 1145 * 1146 * I chose to generalized it instead, so that the size 1147 * of code does not increase practically, but it becomes 1148 * much more general. 1149 * Every prefix is assigned a "scope" value: "host" is local address, 1150 * "link" is direct route, 1151 * [ ... "site" ... "interior" ... ] 1152 * and "universe" is true gateway route with global meaning. 1153 * 1154 * Every prefix refers to a set of "nexthop"s (gw, oif), 1155 * where gw must have narrower scope. This recursion stops 1156 * when gw has LOCAL scope or if "nexthop" is declared ONLINK, 1157 * which means that gw is forced to be on link. 1158 * 1159 * Code is still hairy, but now it is apparently logically 1160 * consistent and very flexible. F.e. as by-product it allows 1161 * to co-exists in peace independent exterior and interior 1162 * routing processes. 1163 * 1164 * Normally it looks as following. 1165 * 1166 * {universe prefix} -> (gw, oif) [scope link] 1167 * | 1168 * |-> {link prefix} -> (gw, oif) [scope local] 1169 * | 1170 * |-> {local prefix} (terminal node) 1171 */ 1172 static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table, 1173 u8 scope, struct netlink_ext_ack *extack) 1174 { 1175 struct net_device *dev; 1176 struct fib_result res; 1177 int err = 0; 1178 1179 if (nh->fib_nh_flags & RTNH_F_ONLINK) { 1180 unsigned int addr_type; 1181 1182 if (scope >= RT_SCOPE_LINK) { 1183 NL_SET_ERR_MSG(extack, "Nexthop has invalid scope"); 1184 return -EINVAL; 1185 } 1186 dev = __dev_get_by_index(net, nh->fib_nh_oif); 1187 if (!dev) { 1188 NL_SET_ERR_MSG(extack, "Nexthop device required for onlink"); 1189 return -ENODEV; 1190 } 1191 if (!(dev->flags & IFF_UP)) { 1192 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 1193 return -ENETDOWN; 1194 } 1195 addr_type = inet_addr_type_dev_table(net, dev, nh->fib_nh_gw4); 1196 if (addr_type != RTN_UNICAST) { 1197 NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway"); 1198 return -EINVAL; 1199 } 1200 if (!netif_carrier_ok(dev)) 1201 nh->fib_nh_flags |= RTNH_F_LINKDOWN; 1202 nh->fib_nh_dev = dev; 1203 netdev_hold(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); 1204 nh->fib_nh_scope = RT_SCOPE_LINK; 1205 return 0; 1206 } 1207 rcu_read_lock(); 1208 { 1209 struct fib_table *tbl = NULL; 1210 struct flowi4 fl4 = { 1211 .daddr = nh->fib_nh_gw4, 1212 .flowi4_scope = scope + 1, 1213 .flowi4_oif = nh->fib_nh_oif, 1214 .flowi4_iif = LOOPBACK_IFINDEX, 1215 }; 1216 1217 /* It is not necessary, but requires a bit of thinking */ 1218 if (fl4.flowi4_scope < RT_SCOPE_LINK) 1219 fl4.flowi4_scope = RT_SCOPE_LINK; 1220 1221 if (table && table != RT_TABLE_MAIN) 1222 tbl = fib_get_table(net, table); 1223 1224 if (tbl) 1225 err = fib_table_lookup(tbl, &fl4, &res, 1226 FIB_LOOKUP_IGNORE_LINKSTATE | 1227 FIB_LOOKUP_NOREF); 1228 1229 /* on error or if no table given do full lookup. This 1230 * is needed for example when nexthops are in the local 1231 * table rather than the given table 1232 */ 1233 if (!tbl || err) { 1234 err = fib_lookup(net, &fl4, &res, 1235 FIB_LOOKUP_IGNORE_LINKSTATE); 1236 } 1237 1238 if (err) { 1239 NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway"); 1240 goto out; 1241 } 1242 } 1243 1244 err = -EINVAL; 1245 if (res.type != RTN_UNICAST && res.type != RTN_LOCAL) { 1246 NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway"); 1247 goto out; 1248 } 1249 nh->fib_nh_scope = res.scope; 1250 nh->fib_nh_oif = FIB_RES_OIF(res); 1251 nh->fib_nh_dev = dev = FIB_RES_DEV(res); 1252 if (!dev) { 1253 NL_SET_ERR_MSG(extack, 1254 "No egress device for nexthop gateway"); 1255 goto out; 1256 } 1257 netdev_hold(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); 1258 if (!netif_carrier_ok(dev)) 1259 nh->fib_nh_flags |= RTNH_F_LINKDOWN; 1260 err = (dev->flags & IFF_UP) ? 0 : -ENETDOWN; 1261 out: 1262 rcu_read_unlock(); 1263 return err; 1264 } 1265 1266 static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh, 1267 struct netlink_ext_ack *extack) 1268 { 1269 struct in_device *in_dev; 1270 int err; 1271 1272 if (nh->fib_nh_flags & (RTNH_F_PERVASIVE | RTNH_F_ONLINK)) { 1273 NL_SET_ERR_MSG(extack, 1274 "Invalid flags for nexthop - PERVASIVE and ONLINK can not be set"); 1275 return -EINVAL; 1276 } 1277 1278 rcu_read_lock(); 1279 1280 err = -ENODEV; 1281 in_dev = inetdev_by_index(net, nh->fib_nh_oif); 1282 if (!in_dev) 1283 goto out; 1284 err = -ENETDOWN; 1285 if (!(in_dev->dev->flags & IFF_UP)) { 1286 NL_SET_ERR_MSG(extack, "Device for nexthop is not up"); 1287 goto out; 1288 } 1289 1290 nh->fib_nh_dev = in_dev->dev; 1291 netdev_hold(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); 1292 nh->fib_nh_scope = RT_SCOPE_HOST; 1293 if (!netif_carrier_ok(nh->fib_nh_dev)) 1294 nh->fib_nh_flags |= RTNH_F_LINKDOWN; 1295 err = 0; 1296 out: 1297 rcu_read_unlock(); 1298 return err; 1299 } 1300 1301 int fib_check_nh(struct net *net, struct fib_nh *nh, u32 table, u8 scope, 1302 struct netlink_ext_ack *extack) 1303 { 1304 int err; 1305 1306 if (nh->fib_nh_gw_family == AF_INET) 1307 err = fib_check_nh_v4_gw(net, nh, table, scope, extack); 1308 else if (nh->fib_nh_gw_family == AF_INET6) 1309 err = fib_check_nh_v6_gw(net, nh, table, extack); 1310 else 1311 err = fib_check_nh_nongw(net, nh, extack); 1312 1313 return err; 1314 } 1315 1316 __be32 fib_info_update_nhc_saddr(struct net *net, struct fib_nh_common *nhc, 1317 unsigned char scope) 1318 { 1319 struct fib_nh *nh; 1320 __be32 saddr; 1321 1322 if (nhc->nhc_family != AF_INET) 1323 return inet_select_addr(nhc->nhc_dev, 0, scope); 1324 1325 nh = container_of(nhc, struct fib_nh, nh_common); 1326 saddr = inet_select_addr(nh->fib_nh_dev, nh->fib_nh_gw4, scope); 1327 1328 WRITE_ONCE(nh->nh_saddr, saddr); 1329 WRITE_ONCE(nh->nh_saddr_genid, atomic_read(&net->ipv4.dev_addr_genid)); 1330 1331 return saddr; 1332 } 1333 1334 __be32 fib_result_prefsrc(struct net *net, struct fib_result *res) 1335 { 1336 struct fib_nh_common *nhc = res->nhc; 1337 1338 if (res->fi->fib_prefsrc) 1339 return res->fi->fib_prefsrc; 1340 1341 if (nhc->nhc_family == AF_INET) { 1342 struct fib_nh *nh; 1343 1344 nh = container_of(nhc, struct fib_nh, nh_common); 1345 if (READ_ONCE(nh->nh_saddr_genid) == 1346 atomic_read(&net->ipv4.dev_addr_genid)) 1347 return READ_ONCE(nh->nh_saddr); 1348 } 1349 1350 return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope); 1351 } 1352 1353 static bool fib_valid_prefsrc(struct fib_config *cfg, __be32 fib_prefsrc) 1354 { 1355 if (cfg->fc_type != RTN_LOCAL || !cfg->fc_dst || 1356 fib_prefsrc != cfg->fc_dst) { 1357 u32 tb_id = cfg->fc_table; 1358 int rc; 1359 1360 if (tb_id == RT_TABLE_MAIN) 1361 tb_id = RT_TABLE_LOCAL; 1362 1363 rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net, 1364 fib_prefsrc, tb_id); 1365 1366 if (rc != RTN_LOCAL && tb_id != RT_TABLE_LOCAL) { 1367 rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net, 1368 fib_prefsrc, RT_TABLE_LOCAL); 1369 } 1370 1371 if (rc != RTN_LOCAL) 1372 return false; 1373 } 1374 return true; 1375 } 1376 1377 struct fib_info *fib_create_info(struct fib_config *cfg, 1378 struct netlink_ext_ack *extack) 1379 { 1380 int err; 1381 struct fib_info *fi = NULL; 1382 struct nexthop *nh = NULL; 1383 struct fib_info *ofi; 1384 int nhs = 1; 1385 struct net *net = cfg->fc_nlinfo.nl_net; 1386 1387 ASSERT_RTNL(); 1388 if (cfg->fc_type > RTN_MAX) 1389 goto err_inval; 1390 1391 /* Fast check to catch the most weird cases */ 1392 if (fib_props[cfg->fc_type].scope > cfg->fc_scope) { 1393 NL_SET_ERR_MSG(extack, "Invalid scope"); 1394 goto err_inval; 1395 } 1396 1397 if (cfg->fc_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) { 1398 NL_SET_ERR_MSG(extack, 1399 "Invalid rtm_flags - can not contain DEAD or LINKDOWN"); 1400 goto err_inval; 1401 } 1402 1403 if (cfg->fc_nh_id) { 1404 if (!cfg->fc_mx) { 1405 fi = fib_find_info_nh(net, cfg); 1406 if (fi) { 1407 refcount_inc(&fi->fib_treeref); 1408 return fi; 1409 } 1410 } 1411 1412 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 1413 if (!nh) { 1414 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 1415 goto err_inval; 1416 } 1417 nhs = 0; 1418 } 1419 1420 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1421 if (cfg->fc_mp) { 1422 nhs = fib_count_nexthops(cfg->fc_mp, cfg->fc_mp_len, extack); 1423 if (nhs == 0) 1424 goto err_inval; 1425 } 1426 #endif 1427 1428 fib_info_hash_grow(net); 1429 1430 fi = kzalloc_flex(*fi, fib_nh, nhs); 1431 if (!fi) { 1432 err = -ENOBUFS; 1433 goto failure; 1434 } 1435 1436 fi->fib_metrics = ip_fib_metrics_init(cfg->fc_mx, cfg->fc_mx_len, extack); 1437 if (IS_ERR(fi->fib_metrics)) { 1438 err = PTR_ERR(fi->fib_metrics); 1439 kfree(fi); 1440 return ERR_PTR(err); 1441 } 1442 1443 fi->fib_net = net; 1444 fi->fib_protocol = cfg->fc_protocol; 1445 fi->fib_scope = cfg->fc_scope; 1446 fi->fib_flags = cfg->fc_flags; 1447 fi->fib_priority = cfg->fc_priority; 1448 fi->fib_prefsrc = cfg->fc_prefsrc; 1449 fi->fib_type = cfg->fc_type; 1450 fi->fib_tb_id = cfg->fc_table; 1451 1452 fi->fib_nhs = nhs; 1453 if (nh) { 1454 if (!nexthop_get(nh)) { 1455 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 1456 err = -EINVAL; 1457 } else { 1458 err = 0; 1459 fi->nh = nh; 1460 } 1461 } else { 1462 change_nexthops(fi) { 1463 nexthop_nh->nh_parent = fi; 1464 } endfor_nexthops(fi) 1465 1466 if (cfg->fc_mp) 1467 err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg, 1468 extack); 1469 else 1470 err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack); 1471 } 1472 1473 if (err != 0) 1474 goto failure; 1475 1476 if (fib_props[cfg->fc_type].error) { 1477 if (cfg->fc_gw_family || cfg->fc_oif || cfg->fc_mp) { 1478 NL_SET_ERR_MSG(extack, 1479 "Gateway, device and multipath can not be specified for this route type"); 1480 goto err_inval; 1481 } 1482 goto link_it; 1483 } else { 1484 switch (cfg->fc_type) { 1485 case RTN_UNICAST: 1486 case RTN_LOCAL: 1487 case RTN_BROADCAST: 1488 case RTN_ANYCAST: 1489 case RTN_MULTICAST: 1490 break; 1491 default: 1492 NL_SET_ERR_MSG(extack, "Invalid route type"); 1493 goto err_inval; 1494 } 1495 } 1496 1497 if (cfg->fc_scope > RT_SCOPE_HOST) { 1498 NL_SET_ERR_MSG(extack, "Invalid scope"); 1499 goto err_inval; 1500 } 1501 1502 if (fi->nh) { 1503 err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack); 1504 if (err) 1505 goto failure; 1506 } else if (cfg->fc_scope == RT_SCOPE_HOST) { 1507 struct fib_nh *nh = fi->fib_nh; 1508 1509 /* Local address is added. */ 1510 if (nhs != 1) { 1511 NL_SET_ERR_MSG(extack, 1512 "Route with host scope can not have multiple nexthops"); 1513 goto err_inval; 1514 } 1515 if (nh->fib_nh_gw_family) { 1516 NL_SET_ERR_MSG(extack, 1517 "Route with host scope can not have a gateway"); 1518 goto err_inval; 1519 } 1520 nh->fib_nh_scope = RT_SCOPE_NOWHERE; 1521 nh->fib_nh_dev = dev_get_by_index(net, nh->fib_nh_oif); 1522 err = -ENODEV; 1523 if (!nh->fib_nh_dev) 1524 goto failure; 1525 netdev_tracker_alloc(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, 1526 GFP_KERNEL); 1527 } else { 1528 int linkdown = 0; 1529 1530 change_nexthops(fi) { 1531 err = fib_check_nh(cfg->fc_nlinfo.nl_net, nexthop_nh, 1532 cfg->fc_table, cfg->fc_scope, 1533 extack); 1534 if (err != 0) 1535 goto failure; 1536 if (nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN) 1537 linkdown++; 1538 } endfor_nexthops(fi) 1539 if (linkdown == fi->fib_nhs) 1540 fi->fib_flags |= RTNH_F_LINKDOWN; 1541 } 1542 1543 if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) { 1544 NL_SET_ERR_MSG(extack, "Invalid prefsrc address"); 1545 goto err_inval; 1546 } 1547 1548 if (!fi->nh) { 1549 change_nexthops(fi) { 1550 fib_info_update_nhc_saddr(net, &nexthop_nh->nh_common, 1551 fi->fib_scope); 1552 if (nexthop_nh->fib_nh_gw_family == AF_INET6) 1553 fi->fib_nh_is_v6 = true; 1554 } endfor_nexthops(fi) 1555 1556 fib_rebalance(fi); 1557 } 1558 1559 link_it: 1560 ofi = fib_find_info(fi); 1561 if (ofi) { 1562 /* fib_table_lookup() should not see @fi yet. */ 1563 fi->fib_dead = 1; 1564 free_fib_info(fi); 1565 refcount_inc(&ofi->fib_treeref); 1566 return ofi; 1567 } 1568 1569 refcount_set(&fi->fib_treeref, 1); 1570 refcount_set(&fi->fib_clntref, 1); 1571 1572 net->ipv4.fib_info_cnt++; 1573 hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi)); 1574 1575 if (fi->fib_prefsrc) { 1576 struct hlist_head *head; 1577 1578 head = fib_info_laddrhash_bucket(net, fi->fib_prefsrc); 1579 hlist_add_head(&fi->fib_lhash, head); 1580 } 1581 if (fi->nh) { 1582 list_add(&fi->nh_list, &nh->fi_list); 1583 } else { 1584 change_nexthops(fi) { 1585 struct hlist_head *head; 1586 1587 if (!nexthop_nh->fib_nh_dev) 1588 continue; 1589 head = fib_nh_head(nexthop_nh->fib_nh_dev); 1590 hlist_add_head_rcu(&nexthop_nh->nh_hash, head); 1591 } endfor_nexthops(fi) 1592 } 1593 return fi; 1594 1595 err_inval: 1596 err = -EINVAL; 1597 1598 failure: 1599 if (fi) { 1600 /* fib_table_lookup() should not see @fi yet. */ 1601 fi->fib_dead = 1; 1602 free_fib_info(fi); 1603 } 1604 1605 return ERR_PTR(err); 1606 } 1607 1608 int fib_nexthop_info(struct sk_buff *skb, const struct fib_nh_common *nhc, 1609 u8 rt_family, unsigned char *flags, bool skip_oif) 1610 { 1611 if (nhc->nhc_flags & RTNH_F_DEAD) 1612 *flags |= RTNH_F_DEAD; 1613 1614 if (nhc->nhc_flags & RTNH_F_LINKDOWN) { 1615 *flags |= RTNH_F_LINKDOWN; 1616 1617 rcu_read_lock(); 1618 switch (nhc->nhc_family) { 1619 case AF_INET: 1620 if (ip_ignore_linkdown(nhc->nhc_dev)) 1621 *flags |= RTNH_F_DEAD; 1622 break; 1623 case AF_INET6: 1624 if (ip6_ignore_linkdown(nhc->nhc_dev)) 1625 *flags |= RTNH_F_DEAD; 1626 break; 1627 } 1628 rcu_read_unlock(); 1629 } 1630 1631 switch (nhc->nhc_gw_family) { 1632 case AF_INET: 1633 if (nla_put_in_addr(skb, RTA_GATEWAY, nhc->nhc_gw.ipv4)) 1634 goto nla_put_failure; 1635 break; 1636 case AF_INET6: 1637 /* if gateway family does not match nexthop family 1638 * gateway is encoded as RTA_VIA 1639 */ 1640 if (rt_family != nhc->nhc_gw_family) { 1641 int alen = sizeof(struct in6_addr); 1642 struct nlattr *nla; 1643 struct rtvia *via; 1644 1645 nla = nla_reserve(skb, RTA_VIA, alen + 2); 1646 if (!nla) 1647 goto nla_put_failure; 1648 1649 via = nla_data(nla); 1650 via->rtvia_family = AF_INET6; 1651 memcpy(via->rtvia_addr, &nhc->nhc_gw.ipv6, alen); 1652 } else if (nla_put_in6_addr(skb, RTA_GATEWAY, 1653 &nhc->nhc_gw.ipv6) < 0) { 1654 goto nla_put_failure; 1655 } 1656 break; 1657 } 1658 1659 *flags |= (nhc->nhc_flags & 1660 (RTNH_F_ONLINK | RTNH_F_OFFLOAD | RTNH_F_TRAP)); 1661 1662 if (!skip_oif && nhc->nhc_dev && 1663 nla_put_u32(skb, RTA_OIF, nhc->nhc_dev->ifindex)) 1664 goto nla_put_failure; 1665 1666 if (lwtunnel_fill_encap(skb, nhc->nhc_lwtstate, 1667 RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 1668 goto nla_put_failure; 1669 1670 return 0; 1671 1672 nla_put_failure: 1673 return -EMSGSIZE; 1674 } 1675 1676 #if IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) || IS_ENABLED(CONFIG_IPV6) 1677 int fib_add_nexthop(struct sk_buff *skb, const struct fib_nh_common *nhc, 1678 int nh_weight, u8 rt_family, u32 nh_tclassid) 1679 { 1680 const struct net_device *dev = nhc->nhc_dev; 1681 struct rtnexthop *rtnh; 1682 unsigned char flags = 0; 1683 1684 rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh)); 1685 if (!rtnh) 1686 goto nla_put_failure; 1687 1688 rtnh->rtnh_hops = nh_weight - 1; 1689 rtnh->rtnh_ifindex = dev ? dev->ifindex : 0; 1690 1691 if (fib_nexthop_info(skb, nhc, rt_family, &flags, true) < 0) 1692 goto nla_put_failure; 1693 1694 rtnh->rtnh_flags = flags; 1695 1696 if (nh_tclassid && nla_put_u32(skb, RTA_FLOW, nh_tclassid)) 1697 goto nla_put_failure; 1698 1699 /* length of rtnetlink header + attributes */ 1700 rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh; 1701 1702 return 0; 1703 1704 nla_put_failure: 1705 return -EMSGSIZE; 1706 } 1707 #endif 1708 1709 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1710 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi) 1711 { 1712 struct nlattr *mp; 1713 1714 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 1715 if (!mp) 1716 goto nla_put_failure; 1717 1718 if (unlikely(fi->nh)) { 1719 if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0) 1720 goto nla_put_failure; 1721 goto mp_end; 1722 } 1723 1724 for_nexthops(fi) { 1725 u32 nh_tclassid = 0; 1726 #ifdef CONFIG_IP_ROUTE_CLASSID 1727 nh_tclassid = nh->nh_tclassid; 1728 #endif 1729 if (fib_add_nexthop(skb, &nh->nh_common, nh->fib_nh_weight, 1730 AF_INET, nh_tclassid) < 0) 1731 goto nla_put_failure; 1732 } endfor_nexthops(fi); 1733 1734 mp_end: 1735 nla_nest_end(skb, mp); 1736 1737 return 0; 1738 1739 nla_put_failure: 1740 return -EMSGSIZE; 1741 } 1742 #else 1743 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi) 1744 { 1745 return 0; 1746 } 1747 #endif 1748 1749 int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, 1750 const struct fib_rt_info *fri, unsigned int flags) 1751 { 1752 unsigned int nhs = fib_info_num_path(fri->fi); 1753 struct fib_info *fi = fri->fi; 1754 u32 tb_id = fri->tb_id; 1755 struct nlmsghdr *nlh; 1756 struct rtmsg *rtm; 1757 1758 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags); 1759 if (!nlh) 1760 return -EMSGSIZE; 1761 1762 rtm = nlmsg_data(nlh); 1763 rtm->rtm_family = AF_INET; 1764 rtm->rtm_dst_len = fri->dst_len; 1765 rtm->rtm_src_len = 0; 1766 rtm->rtm_tos = inet_dscp_to_dsfield(fri->dscp); 1767 if (tb_id < 256) 1768 rtm->rtm_table = tb_id; 1769 else 1770 rtm->rtm_table = RT_TABLE_COMPAT; 1771 if (nla_put_u32(skb, RTA_TABLE, tb_id)) 1772 goto nla_put_failure; 1773 rtm->rtm_type = fri->type; 1774 rtm->rtm_flags = fi->fib_flags; 1775 rtm->rtm_scope = fi->fib_scope; 1776 rtm->rtm_protocol = fi->fib_protocol; 1777 1778 if (rtm->rtm_dst_len && 1779 nla_put_in_addr(skb, RTA_DST, fri->dst)) 1780 goto nla_put_failure; 1781 if (fi->fib_priority && 1782 nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority)) 1783 goto nla_put_failure; 1784 if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0) 1785 goto nla_put_failure; 1786 1787 if (fi->fib_prefsrc && 1788 nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc)) 1789 goto nla_put_failure; 1790 1791 if (fi->nh) { 1792 if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id)) 1793 goto nla_put_failure; 1794 if (nexthop_is_blackhole(fi->nh)) 1795 rtm->rtm_type = RTN_BLACKHOLE; 1796 if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode)) 1797 goto offload; 1798 } 1799 1800 if (nhs == 1) { 1801 const struct fib_nh_common *nhc = fib_info_nhc(fi, 0); 1802 unsigned char flags = 0; 1803 1804 if (fib_nexthop_info(skb, nhc, AF_INET, &flags, false) < 0) 1805 goto nla_put_failure; 1806 1807 rtm->rtm_flags = flags; 1808 #ifdef CONFIG_IP_ROUTE_CLASSID 1809 if (nhc->nhc_family == AF_INET) { 1810 struct fib_nh *nh; 1811 1812 nh = container_of(nhc, struct fib_nh, nh_common); 1813 if (nh->nh_tclassid && 1814 nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid)) 1815 goto nla_put_failure; 1816 } 1817 #endif 1818 } else { 1819 if (fib_add_multipath(skb, fi) < 0) 1820 goto nla_put_failure; 1821 } 1822 1823 offload: 1824 if (fri->offload) 1825 rtm->rtm_flags |= RTM_F_OFFLOAD; 1826 if (fri->trap) 1827 rtm->rtm_flags |= RTM_F_TRAP; 1828 if (fri->offload_failed) 1829 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 1830 1831 nlmsg_end(skb, nlh); 1832 return 0; 1833 1834 nla_put_failure: 1835 nlmsg_cancel(skb, nlh); 1836 return -EMSGSIZE; 1837 } 1838 1839 /* 1840 * Update FIB if: 1841 * - local address disappeared -> we must delete all the entries 1842 * referring to it. 1843 * - device went down -> we must shutdown all nexthops going via it. 1844 */ 1845 int fib_sync_down_addr(struct net_device *dev, __be32 local) 1846 { 1847 int tb_id = l3mdev_fib_table(dev) ? : RT_TABLE_MAIN; 1848 struct net *net = dev_net(dev); 1849 struct hlist_head *head; 1850 struct fib_info *fi; 1851 int ret = 0; 1852 1853 if (!local) 1854 return 0; 1855 1856 head = fib_info_laddrhash_bucket(net, local); 1857 hlist_for_each_entry(fi, head, fib_lhash) { 1858 if (!net_eq(fi->fib_net, net) || 1859 fi->fib_tb_id != tb_id) 1860 continue; 1861 if (fi->fib_prefsrc == local) { 1862 fi->fib_flags |= RTNH_F_DEAD; 1863 fi->pfsrc_removed = true; 1864 ret++; 1865 } 1866 } 1867 return ret; 1868 } 1869 1870 static int call_fib_nh_notifiers(struct fib_nh *nh, 1871 enum fib_event_type event_type) 1872 { 1873 bool ignore_link_down = ip_ignore_linkdown(nh->fib_nh_dev); 1874 struct fib_nh_notifier_info info = { 1875 .fib_nh = nh, 1876 }; 1877 1878 switch (event_type) { 1879 case FIB_EVENT_NH_ADD: 1880 if (nh->fib_nh_flags & RTNH_F_DEAD) 1881 break; 1882 if (ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN) 1883 break; 1884 return call_fib4_notifiers(dev_net(nh->fib_nh_dev), event_type, 1885 &info.info); 1886 case FIB_EVENT_NH_DEL: 1887 if ((ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN) || 1888 (nh->fib_nh_flags & RTNH_F_DEAD)) 1889 return call_fib4_notifiers(dev_net(nh->fib_nh_dev), 1890 event_type, &info.info); 1891 break; 1892 default: 1893 break; 1894 } 1895 1896 return NOTIFY_DONE; 1897 } 1898 1899 /* Walk the exceptions of a nexthop after its first hop MTU changed. The 1900 * chain is RCU protected here, while fnhe_update_pmtu() takes fnhe_lock 1901 * for the update of each entry. 1902 */ 1903 void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig) 1904 { 1905 struct fnhe_hash_bucket *bucket; 1906 int i; 1907 1908 rcu_read_lock(); 1909 bucket = rcu_dereference(nhc->nhc_exceptions); 1910 if (!bucket) 1911 goto out; 1912 1913 for (i = 0; i < FNHE_HASH_SIZE; i++) { 1914 struct fib_nh_exception *fnhe; 1915 1916 for (fnhe = rcu_dereference(bucket[i].chain); 1917 fnhe; 1918 fnhe = rcu_dereference(fnhe->fnhe_next)) 1919 fnhe_update_pmtu(fnhe, new, orig); 1920 } 1921 out: 1922 rcu_read_unlock(); 1923 } 1924 1925 void fib_sync_mtu(struct net_device *dev, u32 orig_mtu) 1926 { 1927 struct hlist_head *head = fib_nh_head(dev); 1928 struct fib_nh *nh; 1929 1930 hlist_for_each_entry(nh, head, nh_hash) { 1931 DEBUG_NET_WARN_ON_ONCE(nh->fib_nh_dev != dev); 1932 fib_nhc_update_mtu(&nh->nh_common, dev->mtu, orig_mtu); 1933 } 1934 } 1935 1936 /* Event force Flags Description 1937 * NETDEV_CHANGE 0 LINKDOWN Carrier OFF, not for scope host 1938 * NETDEV_DOWN 0 LINKDOWN|DEAD Link down, not for scope host 1939 * NETDEV_DOWN 1 LINKDOWN|DEAD Last address removed 1940 * NETDEV_UNREGISTER 1 LINKDOWN|DEAD Device removed 1941 * 1942 * only used when fib_nh is built into fib_info 1943 */ 1944 int fib_sync_down_dev(struct net_device *dev, unsigned long event, bool force) 1945 { 1946 struct hlist_head *head = fib_nh_head(dev); 1947 struct fib_info *prev_fi = NULL; 1948 int scope = RT_SCOPE_NOWHERE; 1949 struct fib_nh *nh; 1950 int ret = 0; 1951 1952 if (force) 1953 scope = -1; 1954 1955 hlist_for_each_entry(nh, head, nh_hash) { 1956 struct fib_info *fi = nh->nh_parent; 1957 int dead; 1958 1959 BUG_ON(!fi->fib_nhs); 1960 DEBUG_NET_WARN_ON_ONCE(nh->fib_nh_dev != dev); 1961 if (fi == prev_fi) 1962 continue; 1963 prev_fi = fi; 1964 dead = 0; 1965 change_nexthops(fi) { 1966 if (nexthop_nh->fib_nh_flags & RTNH_F_DEAD) 1967 dead++; 1968 else if (nexthop_nh->fib_nh_dev == dev && 1969 nexthop_nh->fib_nh_scope != scope) { 1970 switch (event) { 1971 case NETDEV_DOWN: 1972 case NETDEV_UNREGISTER: 1973 nexthop_nh->fib_nh_flags |= RTNH_F_DEAD; 1974 fallthrough; 1975 case NETDEV_CHANGE: 1976 nexthop_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 1977 break; 1978 } 1979 call_fib_nh_notifiers(nexthop_nh, 1980 FIB_EVENT_NH_DEL); 1981 dead++; 1982 } 1983 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1984 if (event == NETDEV_UNREGISTER && 1985 nexthop_nh->fib_nh_dev == dev) { 1986 dead = fi->fib_nhs; 1987 break; 1988 } 1989 #endif 1990 } endfor_nexthops(fi) 1991 if (dead == fi->fib_nhs) { 1992 switch (event) { 1993 case NETDEV_DOWN: 1994 case NETDEV_UNREGISTER: 1995 fi->fib_flags |= RTNH_F_DEAD; 1996 fallthrough; 1997 case NETDEV_CHANGE: 1998 fi->fib_flags |= RTNH_F_LINKDOWN; 1999 break; 2000 } 2001 ret++; 2002 } 2003 2004 fib_rebalance(fi); 2005 } 2006 2007 return ret; 2008 } 2009 2010 /* Must be invoked inside of an RCU protected region. */ 2011 static void fib_select_default(const struct flowi4 *flp, struct fib_result *res) 2012 { 2013 struct fib_info *fi = NULL, *last_resort = NULL; 2014 struct hlist_head *fa_head = res->fa_head; 2015 struct fib_table *tb = res->table; 2016 u8 slen = 32 - res->prefixlen; 2017 int order = -1, last_idx = -1; 2018 struct fib_alias *fa, *fa1 = NULL; 2019 u32 last_prio = res->fi->fib_priority; 2020 dscp_t last_dscp = 0; 2021 2022 hlist_for_each_entry_rcu(fa, fa_head, fa_list) { 2023 struct fib_info *next_fi = fa->fa_info; 2024 struct fib_nh_common *nhc; 2025 2026 if (fa->fa_slen != slen) 2027 continue; 2028 if (fa->fa_dscp && !fib_dscp_masked_match(fa->fa_dscp, flp)) 2029 continue; 2030 if (fa->tb_id != tb->tb_id) 2031 continue; 2032 if (next_fi->fib_priority > last_prio && 2033 fa->fa_dscp == last_dscp) { 2034 if (last_dscp) 2035 continue; 2036 break; 2037 } 2038 if (next_fi->fib_flags & RTNH_F_DEAD) 2039 continue; 2040 last_dscp = fa->fa_dscp; 2041 last_prio = next_fi->fib_priority; 2042 2043 if (next_fi->fib_scope != res->scope || 2044 fa->fa_type != RTN_UNICAST) 2045 continue; 2046 2047 nhc = fib_info_nhc(next_fi, 0); 2048 if (!nhc->nhc_gw_family || nhc->nhc_scope != RT_SCOPE_LINK) 2049 continue; 2050 2051 fib_alias_accessed(fa); 2052 2053 if (!fi) { 2054 if (next_fi != res->fi) 2055 break; 2056 fa1 = fa; 2057 } else if (!fib_detect_death(fi, order, &last_resort, 2058 &last_idx, fa1->fa_default)) { 2059 fib_result_assign(res, fi); 2060 fa1->fa_default = order; 2061 goto out; 2062 } 2063 fi = next_fi; 2064 order++; 2065 } 2066 2067 if (order <= 0 || !fi) { 2068 if (fa1) 2069 fa1->fa_default = -1; 2070 goto out; 2071 } 2072 2073 if (!fib_detect_death(fi, order, &last_resort, &last_idx, 2074 fa1->fa_default)) { 2075 fib_result_assign(res, fi); 2076 fa1->fa_default = order; 2077 goto out; 2078 } 2079 2080 if (last_idx >= 0) 2081 fib_result_assign(res, last_resort); 2082 fa1->fa_default = last_idx; 2083 out: 2084 return; 2085 } 2086 2087 /* 2088 * Dead device goes up. We wake up dead nexthops. 2089 * It takes sense only on multipath routes. 2090 * 2091 * only used when fib_nh is built into fib_info 2092 */ 2093 int fib_sync_up(struct net_device *dev, unsigned char nh_flags) 2094 { 2095 struct fib_info *prev_fi; 2096 struct hlist_head *head; 2097 struct fib_nh *nh; 2098 int ret; 2099 2100 if (!(dev->flags & IFF_UP)) 2101 return 0; 2102 2103 if (nh_flags & RTNH_F_DEAD) { 2104 unsigned int flags = netif_get_flags(dev); 2105 2106 if (flags & (IFF_RUNNING | IFF_LOWER_UP)) 2107 nh_flags |= RTNH_F_LINKDOWN; 2108 } 2109 2110 prev_fi = NULL; 2111 head = fib_nh_head(dev); 2112 ret = 0; 2113 2114 hlist_for_each_entry(nh, head, nh_hash) { 2115 struct fib_info *fi = nh->nh_parent; 2116 int alive; 2117 2118 BUG_ON(!fi->fib_nhs); 2119 DEBUG_NET_WARN_ON_ONCE(nh->fib_nh_dev != dev); 2120 if (fi == prev_fi) 2121 continue; 2122 2123 prev_fi = fi; 2124 alive = 0; 2125 change_nexthops(fi) { 2126 if (!(nexthop_nh->fib_nh_flags & nh_flags)) { 2127 alive++; 2128 continue; 2129 } 2130 if (!nexthop_nh->fib_nh_dev || 2131 !(nexthop_nh->fib_nh_dev->flags & IFF_UP)) 2132 continue; 2133 if (nexthop_nh->fib_nh_dev != dev || 2134 !__in_dev_get_rtnl(dev)) 2135 continue; 2136 alive++; 2137 nexthop_nh->fib_nh_flags &= ~nh_flags; 2138 call_fib_nh_notifiers(nexthop_nh, FIB_EVENT_NH_ADD); 2139 } endfor_nexthops(fi) 2140 2141 if (alive > 0) { 2142 fi->fib_flags &= ~nh_flags; 2143 ret++; 2144 } 2145 2146 fib_rebalance(fi); 2147 } 2148 2149 return ret; 2150 } 2151 2152 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2153 static bool fib_good_nh(const struct fib_nh *nh) 2154 { 2155 int state = NUD_REACHABLE; 2156 2157 if (nh->fib_nh_scope == RT_SCOPE_LINK) { 2158 struct neighbour *n; 2159 2160 rcu_read_lock(); 2161 2162 if (likely(nh->fib_nh_gw_family == AF_INET)) 2163 n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev, 2164 (__force u32)nh->fib_nh_gw4); 2165 else if (IS_ENABLED(CONFIG_IPV6) && 2166 nh->fib_nh_gw_family == AF_INET6) 2167 n = __ipv6_neigh_lookup_noref(nh->fib_nh_dev, 2168 &nh->fib_nh_gw6); 2169 else 2170 n = NULL; 2171 if (n) 2172 state = READ_ONCE(n->nud_state); 2173 2174 rcu_read_unlock(); 2175 } 2176 2177 return !!(state & NUD_VALID); 2178 } 2179 2180 static __be32 fib_nh_saddr(struct net *net, const struct fib_info *fi, 2181 struct fib_nh *nh, int genid) 2182 { 2183 if (READ_ONCE(nh->nh_saddr_genid) == genid) 2184 return READ_ONCE(nh->nh_saddr); 2185 2186 return fib_info_update_nhc_saddr(net, &nh->nh_common, fi->fib_scope); 2187 } 2188 2189 void fib_select_multipath(struct fib_result *res, int hash, 2190 const struct flowi4 *fl4) 2191 { 2192 struct fib_info *fi = res->fi; 2193 struct net *net = fi->fib_net; 2194 bool use_neigh; 2195 int score = -1; 2196 __be32 saddr; 2197 int genid; 2198 2199 if (unlikely(res->fi->nh)) { 2200 nexthop_path_fib_result(res, hash); 2201 return; 2202 } 2203 2204 use_neigh = READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh); 2205 saddr = fl4 ? fl4->saddr : 0; 2206 genid = saddr ? atomic_read(&net->ipv4.dev_addr_genid) : 0; 2207 2208 change_nexthops(fi) { 2209 int nh_upper_bound, nh_score = 0; 2210 2211 /* Nexthops without a carrier are assigned an upper bound of 2212 * minus one when "ignore_routes_with_linkdown" is set. 2213 */ 2214 nh_upper_bound = atomic_read(&nexthop_nh->fib_nh_upper_bound); 2215 if (nh_upper_bound == -1 || 2216 (use_neigh && !fib_good_nh(nexthop_nh))) 2217 continue; 2218 2219 if (saddr && fib_nh_saddr(net, fi, nexthop_nh, genid) == saddr) 2220 nh_score += 2; 2221 if (hash <= nh_upper_bound) 2222 nh_score++; 2223 if (score < nh_score) { 2224 res->nh_sel = nhsel; 2225 res->nhc = &nexthop_nh->nh_common; 2226 if (nh_score == 3 || (!saddr && nh_score == 1)) 2227 return; 2228 score = nh_score; 2229 } 2230 2231 } endfor_nexthops(fi); 2232 } 2233 #endif 2234 2235 void fib_select_path(struct net *net, struct fib_result *res, 2236 struct flowi4 *fl4, const struct sk_buff *skb) 2237 { 2238 if (fl4->flowi4_oif) 2239 goto check_saddr; 2240 2241 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2242 if (fib_info_num_path(res->fi) > 1) { 2243 int h = fib_multipath_hash(net, fl4, skb, NULL); 2244 2245 fib_select_multipath(res, h, fl4); 2246 } 2247 else 2248 #endif 2249 if (!res->prefixlen && 2250 res->table->tb_num_default > 1 && 2251 res->type == RTN_UNICAST) 2252 fib_select_default(fl4, res); 2253 2254 check_saddr: 2255 if (!fl4->saddr) { 2256 struct net_device *l3mdev; 2257 2258 l3mdev = dev_get_by_index_rcu(net, fl4->flowi4_l3mdev); 2259 2260 if (!l3mdev || 2261 l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) == l3mdev) 2262 fl4->saddr = fib_result_prefsrc(net, res); 2263 else 2264 fl4->saddr = inet_select_addr(l3mdev, 0, RT_SCOPE_LINK); 2265 } 2266 } 2267 2268 int __net_init fib4_semantics_init(struct net *net) 2269 { 2270 unsigned int hash_bits = 4; 2271 2272 net->ipv4.fib_info_hash = fib_info_hash_alloc(hash_bits); 2273 if (!net->ipv4.fib_info_hash) 2274 return -ENOMEM; 2275 2276 net->ipv4.fib_info_hash_bits = hash_bits; 2277 net->ipv4.fib_info_cnt = 0; 2278 2279 return 0; 2280 } 2281 2282 void __net_exit fib4_semantics_exit(struct net *net) 2283 { 2284 fib_info_hash_free(net->ipv4.fib_info_hash); 2285 } 2286