Lines Matching +full:use +full:- +full:rtm
1 // SPDX-License-Identifier: GPL-2.0-or-later
76 list_for_each_entry(w, &(net)->ipv6.fib6_walkers, lh)
80 write_lock_bh(&net->ipv6.fib6_walker_lock);
81 list_add(&w->lh, &net->ipv6.fib6_walkers);
82 write_unlock_bh(&net->ipv6.fib6_walker_lock);
87 write_lock_bh(&net->ipv6.fib6_walker_lock);
88 list_del(&w->lh);
89 write_unlock_bh(&net->ipv6.fib6_walker_lock);
94 int new, old = atomic_read(&net->ipv6.fib6_sernum);
98 } while (!atomic_try_cmpxchg(&net->ipv6.fib6_sernum, &old, new));
111 fn = rcu_dereference_protected(f6i->fib6_node,
112 lockdep_is_held(&f6i->fib6_table->tb6_lock));
114 WRITE_ONCE(fn->fn_sernum, fib6_new_sernum(net));
141 * See include/asm-generic/bitops/le.h.
160 INIT_LIST_HEAD(&f6i->fib6_siblings);
161 refcount_set(&f6i->fib6_ref, 1);
163 INIT_HLIST_NODE(&f6i->gc_link);
172 WARN_ON(f6i->fib6_node);
174 if (f6i->nh)
175 nexthop_put(f6i->nh);
177 fib6_nh_release(f6i->fib6_nh);
179 ip_fib_metrics_put(f6i->fib6_metrics);
190 net->ipv6.rt6_stats->fib_nodes++;
198 net->ipv6.rt6_stats->fib_nodes--;
204 net->ipv6.rt6_stats->fib_nodes--;
209 inetpeer_invalidate_tree(&table->tb6_peers);
221 spin_lock_init(&tb->tb6_lock);
222 h = tb->tb6_id & (FIB6_TABLE_HASHSZ - 1);
228 hlist_add_head_rcu(&tb->tb6_hlist, &net->ipv6.fib_table_hash[h]);
239 table->tb6_id = id;
240 rcu_assign_pointer(table->tb6_root.leaf,
241 net->ipv6.fib6_null_entry);
242 table->tb6_root.fn_flags = RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
243 inet_peer_base_init(&table->tb6_peers);
244 INIT_HLIST_HEAD(&table->tb6_gc_hlist);
265 spin_lock_bh(&net->ipv6.fib_table_hash_lock);
269 spin_unlock_bh(&net->ipv6.fib_table_hash_lock);
276 spin_unlock_bh(&net->ipv6.fib_table_hash_lock);
290 head = &net->ipv6.fib_table_hash[id & (FIB6_TABLE_HASHSZ - 1)];
293 * but rcu_dereference_raw() is used to avoid data-race.
296 if (tb->tb6_id == id)
305 fib6_link_table(net, net->ipv6.fib6_main_tbl);
306 fib6_link_table(net, net->ipv6.fib6_local_tbl);
317 return net->ipv6.fib6_main_tbl;
327 net, net->ipv6.fib6_main_tbl, fl6, skb, flags);
328 if (rt->dst.error == -EAGAIN) {
330 rt = net->ipv6.ip6_null_entry;
332 dst_hold(&rt->dst);
335 return &rt->dst;
342 return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6,
348 fib6_link_table(net, net->ipv6.fib6_main_tbl);
359 const struct hlist_head *head = &net->ipv6.fib_table_hash[h];
363 fib_seq += READ_ONCE(tb->fib_seq);
408 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
424 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
432 .nsiblings = rt->fib6_nsiblings,
435 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
451 if (!rt || rt == arg->net->ipv6.fib6_null_entry)
454 nsiblings = READ_ONCE(rt->fib6_nsiblings);
456 err = call_fib6_multipath_entry_notifier(arg->nb, fib_event,
459 arg->extack);
461 err = call_fib6_entry_notifier(arg->nb, fib_event, rt,
462 arg->extack);
471 err = fib6_rt_dump(w->leaf, w->args);
472 w->leaf = NULL;
481 w->root = &tb->tb6_root;
482 spin_lock_bh(&tb->tb6_lock);
484 spin_unlock_bh(&tb->tb6_lock);
499 return -ENOMEM;
501 w->func = fib6_node_dump;
505 w->args = &arg;
508 struct hlist_head *head = &net->ipv6.fib_table_hash[h];
522 return err > 0 ? -EINVAL : err;
531 res = rt6_dump_route(rt, w->args, w->skip_in_node);
534 w->leaf = rt;
539 w->skip_in_node += res;
543 w->skip_in_node = 0;
550 if (rt->fib6_nsiblings)
551 rt = list_last_entry(&rt->fib6_siblings,
555 w->leaf = NULL;
561 struct net *net = sock_net(cb->skb->sk);
562 struct fib6_walker *w = (void *)cb->args[2];
565 if (cb->args[4]) {
566 cb->args[4] = 0;
569 cb->args[2] = 0;
572 cb->done = (void *)cb->args[3];
573 cb->args[1] = 3;
579 return cb->done ? cb->done(cb) : 0;
585 struct net *net = sock_net(skb->sk);
589 w = (void *)cb->args[2];
590 w->root = &table->tb6_root;
592 if (cb->args[4] == 0) {
593 w->count = 0;
594 w->skip = 0;
595 w->skip_in_node = 0;
597 spin_lock_bh(&table->tb6_lock);
599 spin_unlock_bh(&table->tb6_lock);
601 cb->args[4] = 1;
602 cb->args[5] = READ_ONCE(w->root->fn_sernum);
605 int sernum = READ_ONCE(w->root->fn_sernum);
606 if (cb->args[5] != sernum) {
608 cb->args[5] = sernum;
609 w->state = FWS_INIT;
610 w->node = w->root;
611 w->skip = w->count;
612 w->skip_in_node = 0;
614 w->skip = 0;
616 spin_lock_bh(&table->tb6_lock);
618 spin_unlock_bh(&table->tb6_lock);
621 cb->args[4] = 0;
635 const struct nlmsghdr *nlh = cb->nlh;
636 struct net *net = sock_net(skb->sk);
645 if (cb->strict_check) {
650 struct rtmsg *rtm = nlmsg_data(nlh);
652 if (rtm->rtm_flags & RTM_F_PREFIX)
656 w = (void *)cb->args[2];
664 err = -ENOMEM;
667 w->func = fib6_dump_node;
668 cb->args[2] = (long)w;
672 cb->args[3] = (long)cb->done;
673 cb->done = fib6_dump_done;
680 w->args = &arg;
685 if (rtnl_msg_family(cb->nlh) != PF_INET6)
688 NL_SET_ERR_MSG_MOD(cb->extack, "FIB table does not exist");
689 err = -ENOENT;
693 if (!cb->args[0]) {
696 cb->args[0] = 1;
701 s_h = cb->args[0];
702 s_e = cb->args[1];
706 head = &net->ipv6.fib_table_hash[h];
718 cb->args[1] = e;
719 cb->args[0] = h;
733 if (f6i->fib6_metrics == &dst_default_metrics) {
739 refcount_set(&p->refcnt, 1);
740 f6i->fib6_metrics = p;
743 f6i->fib6_metrics->metrics[metric - 1] = val;
773 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
774 lockdep_is_held(&table->tb6_lock));
780 if (plen < fn->fn_bit ||
781 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
785 "Can not replace route - no match found");
787 return ERR_PTR(-ENOENT);
798 if (plen == fn->fn_bit) {
800 if (!(fn->fn_flags & RTN_RTINFO)) {
801 RCU_INIT_POINTER(fn->leaf, NULL);
804 } else if (fn->fn_flags & RTN_TL_ROOT &&
805 rcu_access_pointer(fn->leaf) ==
806 net->ipv6.fib6_null_entry) {
807 RCU_INIT_POINTER(fn->leaf, NULL);
818 dir = addr_bit_set(addr, fn->fn_bit);
821 rcu_dereference_protected(fn->right,
822 lockdep_is_held(&table->tb6_lock)) :
823 rcu_dereference_protected(fn->left,
824 lockdep_is_held(&table->tb6_lock));
839 "Can not replace route - no match found");
841 return ERR_PTR(-ENOENT);
853 return ERR_PTR(-ENOMEM);
854 ln->fn_bit = plen;
855 RCU_INIT_POINTER(ln->parent, pn);
858 rcu_assign_pointer(pn->right, ln);
860 rcu_assign_pointer(pn->left, ln);
874 pn = rcu_dereference_protected(fn->parent,
875 lockdep_is_held(&table->tb6_lock));
883 bit = __ipv6_addr_diff(addr, &key->addr, sizeof(*addr));
899 return ERR_PTR(-ENOMEM);
910 in->fn_bit = bit;
912 RCU_INIT_POINTER(in->parent, pn);
913 in->leaf = fn->leaf;
914 fib6_info_hold(rcu_dereference_protected(in->leaf,
915 lockdep_is_held(&table->tb6_lock)));
919 rcu_assign_pointer(pn->right, in);
921 rcu_assign_pointer(pn->left, in);
923 ln->fn_bit = plen;
925 RCU_INIT_POINTER(ln->parent, in);
926 rcu_assign_pointer(fn->parent, in);
929 rcu_assign_pointer(in->right, ln);
930 rcu_assign_pointer(in->left, fn);
932 rcu_assign_pointer(in->left, ln);
933 rcu_assign_pointer(in->right, fn);
946 return ERR_PTR(-ENOMEM);
948 ln->fn_bit = plen;
950 RCU_INIT_POINTER(ln->parent, pn);
952 if (addr_bit_set(&key->addr, plen))
953 RCU_INIT_POINTER(ln->right, fn);
955 RCU_INIT_POINTER(ln->left, fn);
957 rcu_assign_pointer(fn->parent, ln);
960 rcu_assign_pointer(pn->right, ln);
962 rcu_assign_pointer(pn->left, ln);
972 if (!fib6_nh->rt6i_pcpu)
983 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
989 * is using 'match'. The cached pcpu_rt->from only changes
993 if (pcpu_rt && rcu_access_pointer(pcpu_rt->from) == match) {
996 from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
1016 f6i->fib6_destroying = 1;
1019 if (f6i->nh) {
1021 nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_drop_pcpu_from, f6i);
1026 fib6_nh = f6i->fib6_nh;
1034 struct fib6_table *table = rt->fib6_table;
1040 if (rt->nh) {
1041 spin_lock(&rt->nh->lock);
1043 if (!list_empty(&rt->nh_list))
1044 list_del_init(&rt->nh_list);
1046 spin_unlock(&rt->nh->lock);
1049 if (refcount_read(&rt->fib6_ref) != 1) {
1057 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
1058 lockdep_is_held(&table->tb6_lock));
1060 if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {
1064 rcu_assign_pointer(fn->leaf, new_leaf);
1067 fn = rcu_dereference_protected(fn->parent,
1068 lockdep_is_held(&table->tb6_lock));
1084 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
1085 lockdep_is_held(&rt->fib6_table->tb6_lock));
1089 int replace = (info->nlh &&
1090 (info->nlh->nlmsg_flags & NLM_F_REPLACE));
1091 int add = (!info->nlh ||
1092 (info->nlh->nlmsg_flags & NLM_F_CREATE));
1099 if (info->nlh && (info->nlh->nlmsg_flags & NLM_F_APPEND))
1102 ins = &fn->leaf;
1105 iter = rcu_dereference_protected(iter->fib6_next,
1106 lockdep_is_held(&rt->fib6_table->tb6_lock))) {
1111 if (iter->fib6_metric == rt->fib6_metric) {
1115 if (info->nlh &&
1116 (info->nlh->nlmsg_flags & NLM_F_EXCL))
1117 return -EEXIST;
1130 if (rt->fib6_nsiblings)
1131 WRITE_ONCE(rt->fib6_nsiblings, 0);
1132 if (!(iter->fib6_flags & RTF_EXPIRES))
1133 return -EEXIST;
1134 if (!(rt->fib6_flags & RTF_EXPIRES)) {
1138 fib6_set_expires(iter, rt->expires);
1141 if (!(rt->fib6_flags & (RTF_ADDRCONF | RTF_PREFIX_RT))) {
1142 iter->fib6_flags &= ~RTF_ADDRCONF;
1143 iter->fib6_flags &= ~RTF_PREFIX_RT;
1146 if (rt->fib6_pmtu)
1148 rt->fib6_pmtu);
1149 return -EEXIST;
1164 WRITE_ONCE(rt->fib6_nsiblings,
1165 rt->fib6_nsiblings + 1);
1168 if (iter->fib6_metric > rt->fib6_metric)
1172 ins = &iter->fib6_next;
1176 /* No matching route with same ecmp-able-ness found, replace
1181 lockdep_is_held(&rt->fib6_table->tb6_lock));
1185 /* Reset round-robin state, if necessary */
1186 if (ins == &fn->leaf)
1187 fn->rr_ptr = NULL;
1190 if (rt->fib6_nsiblings) {
1198 if (sibling->fib6_metric == rt->fib6_metric &&
1200 list_add_tail_rcu(&rt->fib6_siblings,
1201 &sibling->fib6_siblings);
1204 sibling = rcu_dereference_protected(sibling->fib6_next,
1205 lockdep_is_held(&rt->fib6_table->tb6_lock));
1214 &rt->fib6_siblings, fib6_siblings) {
1215 WRITE_ONCE(sibling->fib6_nsiblings,
1216 sibling->fib6_nsiblings + 1);
1217 BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings);
1220 BUG_ON(fib6_nsiblings != rt->fib6_nsiblings);
1240 if (!info->skip_notify_kernel &&
1241 (notify_sibling_rt || ins == &fn->leaf)) {
1248 err = call_fib6_entry_notifiers(info->nl_net,
1257 if (!rt->fib6_nsiblings)
1261 &rt->fib6_siblings,
1263 WRITE_ONCE(sibling->fib6_nsiblings,
1264 sibling->fib6_nsiblings - 1);
1265 WRITE_ONCE(rt->fib6_nsiblings, 0);
1266 list_del_rcu(&rt->fib6_siblings);
1274 rcu_assign_pointer(rt->fib6_next, iter);
1276 rcu_assign_pointer(rt->fib6_node, fn);
1278 if (!info->skip_notify)
1280 info->nl_net->ipv6.rt6_stats->fib_rt_entries++;
1282 if (!(fn->fn_flags & RTN_RTINFO)) {
1283 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1284 fn->fn_flags |= RTN_RTINFO;
1294 return -ENOENT;
1297 if (!info->skip_notify_kernel && ins == &fn->leaf) {
1298 err = call_fib6_entry_notifiers(info->nl_net,
1306 rcu_assign_pointer(rt->fib6_node, fn);
1307 rt->fib6_next = iter->fib6_next;
1309 if (!info->skip_notify)
1311 if (!(fn->fn_flags & RTN_RTINFO)) {
1312 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1313 fn->fn_flags |= RTN_RTINFO;
1315 nsiblings = iter->fib6_nsiblings;
1316 iter->fib6_node = NULL;
1317 list_add(&iter->purge_link, purge_list);
1318 if (rcu_access_pointer(fn->rr_ptr) == iter)
1319 fn->rr_ptr = NULL;
1323 ins = &rt->fib6_next;
1325 lockdep_is_held(&rt->fib6_table->tb6_lock));
1327 if (iter->fib6_metric > rt->fib6_metric)
1330 *ins = iter->fib6_next;
1331 iter->fib6_node = NULL;
1332 list_add(&iter->purge_link, purge_list);
1333 if (rcu_access_pointer(fn->rr_ptr) == iter)
1334 fn->rr_ptr = NULL;
1335 nsiblings--;
1336 info->nl_net->ipv6.rt6_stats->fib_rt_entries--;
1338 ins = &iter->fib6_next;
1341 lockdep_is_held(&rt->fib6_table->tb6_lock));
1356 spin_lock(&rt->nh->lock);
1358 if (rt->nh->dead) {
1360 err = -EINVAL;
1364 list_add(&rt->nh_list, &rt->nh->f6i_list);
1367 spin_unlock(&rt->nh->lock);
1374 if (!timer_pending(&net->ipv6.ip6_fib_timer) &&
1375 (rt->fib6_flags & RTF_EXPIRES))
1376 mod_timer(&net->ipv6.ip6_fib_timer,
1377 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1382 if (!timer_pending(&net->ipv6.ip6_fib_timer))
1383 mod_timer(&net->ipv6.ip6_fib_timer,
1384 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1390 struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,
1391 lockdep_is_held(&rt->fib6_table->tb6_lock));
1396 WRITE_ONCE(fn->fn_sernum, sernum);
1397 fn = rcu_dereference_protected(fn->parent,
1398 lockdep_is_held(&rt->fib6_table->tb6_lock));
1410 spin_lock_bh(&f6i->fib6_table->tb6_lock);
1412 spin_unlock_bh(&f6i->fib6_table->tb6_lock);
1418 * with source addr info in sub-trees
1419 * Need to own table->tb6_lock
1425 struct fib6_table *table = rt->fib6_table;
1431 int err = -ENOMEM;
1435 if (info->nlh) {
1436 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
1438 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1444 fn = fib6_add_1(info->nl_net, table, root,
1445 &rt->fib6_dst.addr, rt->fib6_dst.plen,
1457 if (rt->fib6_src.plen) {
1460 if (!rcu_access_pointer(fn->subtree)) {
1474 sfn = node_alloc(info->nl_net);
1478 fib6_info_hold(info->nl_net->ipv6.fib6_null_entry);
1479 rcu_assign_pointer(sfn->leaf,
1480 info->nl_net->ipv6.fib6_null_entry);
1481 sfn->fn_flags = RTN_ROOT;
1485 sn = fib6_add_1(info->nl_net, table, sfn,
1486 &rt->fib6_src.addr, rt->fib6_src.plen,
1495 node_free_immediate(info->nl_net, sfn);
1501 rcu_assign_pointer(sfn->parent, fn);
1502 rcu_assign_pointer(fn->subtree, sfn);
1504 sn = fib6_add_1(info->nl_net, table, FIB6_SUBTREE(fn),
1505 &rt->fib6_src.addr, rt->fib6_src.plen,
1515 if (!rcu_access_pointer(fn->leaf)) {
1516 if (fn->fn_flags & RTN_TL_ROOT) {
1518 rcu_assign_pointer(fn->leaf,
1519 info->nl_net->ipv6.fib6_null_entry);
1522 rcu_assign_pointer(fn->leaf, rt);
1529 if (rt->nh)
1537 list_del(&iter->purge_link);
1538 fib6_purge_rt(iter, fn, info->nl_net);
1542 __fib6_update_sernum_upto_root(rt, fib6_new_sernum(info->nl_net));
1544 if (rt->fib6_flags & RTF_EXPIRES)
1547 fib6_start_gc(info->nl_net, rt);
1555 * super-tree leaf node we have to find a new one for it.
1559 rcu_dereference_protected(pn->leaf,
1560 lockdep_is_held(&table->tb6_lock));
1563 RCU_INIT_POINTER(pn->leaf, NULL);
1566 if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) {
1567 pn_leaf = fib6_find_prefix(info->nl_net, table,
1571 info->nl_net->ipv6.fib6_null_entry;
1573 rcu_assign_pointer(pn->leaf, pn_leaf);
1579 fib6_routes_require_src_inc(info->nl_net);
1584 /* fn->leaf could be NULL and fib6_repair_tree() needs to be called if:
1592 (!(fn->fn_flags & (RTN_RTINFO|RTN_ROOT)) ||
1593 (fn->fn_flags & RTN_TL_ROOT &&
1594 !rcu_access_pointer(fn->leaf))))
1595 fib6_repair_tree(info->nl_net, table, fn);
1615 if (unlikely(args->offset == 0))
1627 dir = addr_bit_set(args->addr, fn->fn_bit);
1629 next = dir ? rcu_dereference(fn->right) :
1630 rcu_dereference(fn->left);
1642 if (subtree || fn->fn_flags & RTN_RTINFO) {
1643 struct fib6_info *leaf = rcu_dereference(fn->leaf);
1649 key = (struct rt6key *) ((u8 *)leaf + args->offset);
1651 if (ipv6_prefix_equal(&key->addr, args->addr, key->plen)) {
1662 if (fn->fn_flags & RTN_RTINFO)
1667 if (fn->fn_flags & RTN_ROOT)
1670 fn = rcu_dereference(fn->parent);
1700 if (!fn || fn->fn_flags & RTN_TL_ROOT)
1726 struct fib6_info *leaf = rcu_dereference(fn->leaf);
1731 if (plen <= fn->fn_bit)
1742 if (plen < fn->fn_bit ||
1743 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit))
1746 if (plen == fn->fn_bit)
1749 if (fn->fn_flags & RTN_RTINFO)
1756 if (addr_bit_set(addr, fn->fn_bit))
1757 fn = rcu_dereference(fn->right);
1759 fn = rcu_dereference(fn->left);
1794 if (fn && fn->fn_flags & RTN_RTINFO)
1812 if (fn->fn_flags & RTN_ROOT)
1813 return net->ipv6.fib6_null_entry;
1816 child_left = rcu_dereference_protected(fn->left,
1817 lockdep_is_held(&table->tb6_lock));
1818 child_right = rcu_dereference_protected(fn->right,
1819 lockdep_is_held(&table->tb6_lock));
1821 return rcu_dereference_protected(child_left->leaf,
1822 lockdep_is_held(&table->tb6_lock));
1824 return rcu_dereference_protected(child_right->leaf,
1825 lockdep_is_held(&table->tb6_lock));
1835 * Need to own table->tb6_lock
1848 /* Set fn->leaf to null_entry for root node. */
1849 if (fn->fn_flags & RTN_TL_ROOT) {
1850 rcu_assign_pointer(fn->leaf, net->ipv6.fib6_null_entry);
1855 struct fib6_node *fn_r = rcu_dereference_protected(fn->right,
1856 lockdep_is_held(&table->tb6_lock));
1857 struct fib6_node *fn_l = rcu_dereference_protected(fn->left,
1858 lockdep_is_held(&table->tb6_lock));
1859 struct fib6_node *pn = rcu_dereference_protected(fn->parent,
1860 lockdep_is_held(&table->tb6_lock));
1861 struct fib6_node *pn_r = rcu_dereference_protected(pn->right,
1862 lockdep_is_held(&table->tb6_lock));
1863 struct fib6_node *pn_l = rcu_dereference_protected(pn->left,
1864 lockdep_is_held(&table->tb6_lock));
1865 struct fib6_info *fn_leaf = rcu_dereference_protected(fn->leaf,
1866 lockdep_is_held(&table->tb6_lock));
1867 struct fib6_info *pn_leaf = rcu_dereference_protected(pn->leaf,
1868 lockdep_is_held(&table->tb6_lock));
1871 pr_debug("fixing tree: plen=%d iter=%d\n", fn->fn_bit, iter);
1874 WARN_ON(fn->fn_flags & RTN_RTINFO);
1875 WARN_ON(fn->fn_flags & RTN_TL_ROOT);
1892 || (children && fn->fn_flags & RTN_ROOT)
1899 new_fn_leaf = net->ipv6.fib6_null_entry;
1903 rcu_assign_pointer(fn->leaf, new_fn_leaf);
1909 WARN_ON(!(fn->fn_flags & RTN_ROOT));
1910 RCU_INIT_POINTER(pn->subtree, NULL);
1913 WARN_ON(fn->fn_flags & RTN_ROOT);
1916 rcu_assign_pointer(pn->right, child);
1918 rcu_assign_pointer(pn->left, child);
1924 rcu_assign_pointer(child->parent, pn);
1930 read_lock(&net->ipv6.fib6_walker_lock);
1933 if (w->node == fn) {
1935 w, w->state, nstate);
1936 w->node = pn;
1937 w->state = nstate;
1940 if (w->node == fn) {
1941 w->node = child;
1944 w, w->state);
1945 w->state = w->state >= FWS_R ? FWS_U : FWS_INIT;
1948 w, w->state);
1949 w->state = w->state >= FWS_C ? FWS_U : FWS_INIT;
1954 read_unlock(&net->ipv6.fib6_walker_lock);
1957 if (pn->fn_flags & RTN_RTINFO || FIB6_SUBTREE(pn))
1960 RCU_INIT_POINTER(pn->leaf, NULL);
1972 lockdep_is_held(&table->tb6_lock));
1973 struct net *net = info->nl_net;
1980 leaf = rcu_dereference_protected(fn->leaf,
1981 lockdep_is_held(&table->tb6_lock));
1982 if (leaf == rt && !rt->fib6_nsiblings) {
1983 if (rcu_access_pointer(rt->fib6_next))
1984 replace_rt = rcu_dereference_protected(rt->fib6_next,
1985 lockdep_is_held(&table->tb6_lock));
1991 *rtp = rt->fib6_next;
1992 rt->fib6_node = NULL;
1993 net->ipv6.rt6_stats->fib_rt_entries--;
1994 net->ipv6.rt6_stats->fib_discarded_routes++;
1996 /* Reset round-robin state, if necessary */
1997 if (rcu_access_pointer(fn->rr_ptr) == rt)
1998 fn->rr_ptr = NULL;
2001 if (rt->fib6_nsiblings) {
2009 if (rt->fib6_metric == leaf->fib6_metric &&
2013 &rt->fib6_siblings, fib6_siblings)
2014 WRITE_ONCE(sibling->fib6_nsiblings,
2015 sibling->fib6_nsiblings - 1);
2016 WRITE_ONCE(rt->fib6_nsiblings, 0);
2017 list_del_rcu(&rt->fib6_siblings);
2022 read_lock(&net->ipv6.fib6_walker_lock);
2024 if (w->state == FWS_C && w->leaf == rt) {
2026 w->leaf = rcu_dereference_protected(rt->fib6_next,
2027 lockdep_is_held(&table->tb6_lock));
2028 if (!w->leaf)
2029 w->state = FWS_U;
2032 read_unlock(&net->ipv6.fib6_walker_lock);
2038 if (!rcu_access_pointer(fn->leaf)) {
2039 if (!(fn->fn_flags & RTN_TL_ROOT)) {
2040 fn->fn_flags &= ~RTN_RTINFO;
2041 net->ipv6.rt6_stats->fib_route_nodes--;
2048 if (!info->skip_notify_kernel) {
2055 if (!info->skip_notify)
2061 /* Need to own table->tb6_lock */
2064 struct net *net = info->nl_net;
2070 if (rt == net->ipv6.fib6_null_entry)
2071 return -ENOENT;
2073 table = rt->fib6_table;
2074 fn = rcu_dereference_protected(rt->fib6_node,
2075 lockdep_is_held(&table->tb6_lock));
2077 return -ENOENT;
2079 WARN_ON(!(fn->fn_flags & RTN_RTINFO));
2085 for (rtp = &fn->leaf; *rtp; rtp = rtp_next) {
2087 lockdep_is_held(&table->tb6_lock));
2090 fib6_routes_require_src_dec(info->nl_net);
2094 rtp_next = &cur->fib6_next;
2096 return -ENOENT;
2105 * and use this function for garbage collection, clone pruning,
2111 * Callback function w->func may return:
2112 * 0 -> continue walking.
2113 * positive value -> walking is suspended (used by tree dumps,
2115 * negative value -> terminate walking.
2118 * 0 -> walk is complete.
2119 * >0 -> walk is incomplete (i.e. suspended)
2120 * <0 -> walk is terminated by an error.
2129 /* w->root should always be table->tb6_root */
2130 WARN_ON_ONCE(!(w->root->fn_flags & RTN_TL_ROOT));
2133 fn = w->node;
2137 switch (w->state) {
2141 w->node = FIB6_SUBTREE(fn);
2144 w->state = FWS_L;
2148 left = rcu_dereference_protected(fn->left, 1);
2150 w->node = left;
2151 w->state = FWS_INIT;
2154 w->state = FWS_R;
2157 right = rcu_dereference_protected(fn->right, 1);
2159 w->node = right;
2160 w->state = FWS_INIT;
2163 w->state = FWS_C;
2164 w->leaf = rcu_dereference_protected(fn->leaf, 1);
2167 if (w->leaf && fn->fn_flags & RTN_RTINFO) {
2170 if (w->skip) {
2171 w->skip--;
2175 err = w->func(w);
2179 w->count++;
2183 w->state = FWS_U;
2186 if (fn == w->root)
2188 pn = rcu_dereference_protected(fn->parent, 1);
2189 left = rcu_dereference_protected(pn->left, 1);
2190 right = rcu_dereference_protected(pn->right, 1);
2191 w->node = pn;
2194 WARN_ON(!(fn->fn_flags & RTN_ROOT));
2195 w->state = FWS_L;
2200 w->state = FWS_R;
2204 w->state = FWS_C;
2205 w->leaf = rcu_dereference_protected(w->node->leaf, 1);
2219 w->state = FWS_INIT;
2220 w->node = w->root;
2235 .nl_net = c->net,
2236 .skip_notify = c->skip_notify,
2239 if (c->sernum != FIB6_NO_SERNUM_CHANGE &&
2240 READ_ONCE(w->node->fn_sernum) != c->sernum)
2241 WRITE_ONCE(w->node->fn_sernum, c->sernum);
2243 if (!c->func) {
2244 WARN_ON_ONCE(c->sernum == FIB6_NO_SERNUM_CHANGE);
2245 w->leaf = NULL;
2250 res = c->func(rt, c->arg);
2251 if (res == -1) {
2252 w->leaf = rt;
2258 rcu_access_pointer(rt->fib6_node),
2264 } else if (res == -2) {
2265 if (WARN_ON(!rt->fib6_nsiblings))
2267 rt = list_last_entry(&rt->fib6_siblings,
2273 w->leaf = rt;
2281 * It may return -2 -> skip multipath route.
2282 * -1 -> delete this route.
2283 * 0 -> continue walking
2316 head = &net->ipv6.fib_table_hash[h];
2318 spin_lock_bh(&table->tb6_lock);
2319 fib6_clean_tree(net, &table->tb6_root,
2321 spin_unlock_bh(&table->tb6_lock);
2357 * Routes are expired even if they are in use.
2360 if (rt->fib6_flags & RTF_EXPIRES && rt->expires) {
2361 if (time_after(now, rt->expires)) {
2363 return -1;
2365 gc_args->more++;
2370 * only if they are not in use now.
2388 hlist_for_each_entry_safe(rt, n, &tb6->tb6_gc_hlist, gc_link)
2389 if (fib6_age(rt, gc_args) == -1)
2401 head = &net->ipv6.fib_table_hash[h];
2403 spin_lock_bh(&table->tb6_lock);
2407 spin_unlock_bh(&table->tb6_lock);
2419 spin_lock_bh(&net->ipv6.fib6_gc_lock);
2420 } else if (!spin_trylock_bh(&net->ipv6.fib6_gc_lock)) {
2421 mod_timer(&net->ipv6.ip6_fib_timer, jiffies + HZ);
2425 net->ipv6.sysctl.ip6_rt_gc_interval;
2430 net->ipv6.ip6_rt_last_gc = now;
2433 mod_timer(&net->ipv6.ip6_fib_timer,
2435 + net->ipv6.sysctl.ip6_rt_gc_interval));
2437 timer_delete(&net->ipv6.ip6_fib_timer);
2438 spin_unlock_bh(&net->ipv6.fib6_gc_lock);
2457 /* Default to 3-tuple */
2458 net->ipv6.sysctl.multipath_hash_fields =
2461 spin_lock_init(&net->ipv6.fib6_gc_lock);
2462 rwlock_init(&net->ipv6.fib6_walker_lock);
2463 INIT_LIST_HEAD(&net->ipv6.fib6_walkers);
2464 timer_setup(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, 0);
2466 net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);
2467 if (!net->ipv6.rt6_stats)
2470 /* Avoid false sharing : Use at least a full cache line */
2473 net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);
2474 if (!net->ipv6.fib_table_hash)
2477 spin_lock_init(&net->ipv6.fib_table_hash_lock);
2479 net->ipv6.fib6_main_tbl = kzalloc(sizeof(*net->ipv6.fib6_main_tbl),
2481 if (!net->ipv6.fib6_main_tbl)
2484 net->ipv6.fib6_main_tbl->tb6_id = RT6_TABLE_MAIN;
2485 rcu_assign_pointer(net->ipv6.fib6_main_tbl->tb6_root.leaf,
2486 net->ipv6.fib6_null_entry);
2487 net->ipv6.fib6_main_tbl->tb6_root.fn_flags =
2489 inet_peer_base_init(&net->ipv6.fib6_main_tbl->tb6_peers);
2490 INIT_HLIST_HEAD(&net->ipv6.fib6_main_tbl->tb6_gc_hlist);
2493 net->ipv6.fib6_local_tbl = kzalloc(sizeof(*net->ipv6.fib6_local_tbl),
2495 if (!net->ipv6.fib6_local_tbl)
2497 net->ipv6.fib6_local_tbl->tb6_id = RT6_TABLE_LOCAL;
2498 rcu_assign_pointer(net->ipv6.fib6_local_tbl->tb6_root.leaf,
2499 net->ipv6.fib6_null_entry);
2500 net->ipv6.fib6_local_tbl->tb6_root.fn_flags =
2502 inet_peer_base_init(&net->ipv6.fib6_local_tbl->tb6_peers);
2503 INIT_HLIST_HEAD(&net->ipv6.fib6_local_tbl->tb6_gc_hlist);
2511 kfree(net->ipv6.fib6_main_tbl);
2514 kfree(net->ipv6.fib_table_hash);
2516 kfree(net->ipv6.rt6_stats);
2519 return -ENOMEM;
2526 timer_delete_sync(&net->ipv6.ip6_fib_timer);
2529 struct hlist_head *head = &net->ipv6.fib_table_hash[i];
2534 hlist_del(&tb->tb6_hlist);
2539 kfree(net->ipv6.fib_table_hash);
2540 kfree(net->ipv6.rt6_stats);
2557 int ret = -ENOMEM;
2593 struct ipv6_route_iter *iter = seq->private;
2594 struct fib6_nh *fib6_nh = rt->fib6_nh;
2595 unsigned int flags = rt->fib6_flags;
2598 if (rt->nh)
2599 fib6_nh = nexthop_fib6_nh(rt->nh);
2601 seq_printf(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);
2604 seq_printf(seq, "%pi6 %02x ", &rt->fib6_src.addr, rt->fib6_src.plen);
2608 if (fib6_nh->fib_nh_gw_family) {
2610 seq_printf(seq, "%pi6", &fib6_nh->fib_nh_gw6);
2615 dev = fib6_nh->fib_nh_dev;
2617 rt->fib6_metric, refcount_read(&rt->fib6_ref), 0,
2618 flags, dev ? dev->name : "");
2619 iter->w.leaf = NULL;
2625 struct ipv6_route_iter *iter = w->args;
2627 if (!iter->skip)
2631 iter->w.leaf = rcu_dereference_protected(
2632 iter->w.leaf->fib6_next,
2633 lockdep_is_held(&iter->tbl->tb6_lock));
2634 iter->skip--;
2635 if (!iter->skip && iter->w.leaf)
2637 } while (iter->w.leaf);
2645 memset(&iter->w, 0, sizeof(iter->w));
2646 iter->w.func = ipv6_route_yield;
2647 iter->w.root = &iter->tbl->tb6_root;
2648 iter->w.state = FWS_INIT;
2649 iter->w.node = iter->w.root;
2650 iter->w.args = iter;
2651 iter->sernum = READ_ONCE(iter->w.root->fn_sernum);
2652 INIT_LIST_HEAD(&iter->w.lh);
2653 fib6_walker_link(net, &iter->w);
2663 h = (tbl->tb6_id & (FIB6_TABLE_HASHSZ - 1)) + 1;
2664 node = rcu_dereference(hlist_next_rcu(&tbl->tb6_hlist));
2672 hlist_first_rcu(&net->ipv6.fib_table_hash[h++]));
2679 int sernum = READ_ONCE(iter->w.root->fn_sernum);
2681 if (iter->sernum != sernum) {
2682 iter->sernum = sernum;
2683 iter->w.state = FWS_INIT;
2684 iter->w.node = iter->w.root;
2685 WARN_ON(iter->w.skip);
2686 iter->w.skip = iter->w.count;
2695 struct ipv6_route_iter *iter = seq->private;
2701 n = rcu_dereference(((struct fib6_info *)v)->fib6_next);
2707 spin_lock_bh(&iter->tbl->tb6_lock);
2708 r = fib6_walk_continue(&iter->w);
2709 spin_unlock_bh(&iter->tbl->tb6_lock);
2711 return iter->w.leaf;
2713 fib6_walker_unlink(net, &iter->w);
2716 fib6_walker_unlink(net, &iter->w);
2718 iter->tbl = ipv6_route_seq_next_table(iter->tbl, net);
2719 if (!iter->tbl)
2730 struct ipv6_route_iter *iter = seq->private;
2733 iter->tbl = ipv6_route_seq_next_table(NULL, net);
2734 iter->skip = *pos;
2736 if (iter->tbl) {
2748 struct fib6_walker *w = &iter->w;
2749 return w->node && !(w->state == FWS_U && w->node == w->root);
2756 struct ipv6_route_iter *iter = seq->private;
2759 fib6_walker_unlink(net, &iter->w);
2778 struct ipv6_route_iter *iter = seq->private;
2789 iter->w.leaf = NULL;