21 #include <linux/net.h>
29 #include <net/ip.h>
30 #include <net/ipv6.h>
31 #include <net/ndisc.h>
32 #include <net/addrconf.h>
33 #include <net/lwtunnel.h>
34 #include <net/fib_notifier.h>
36 #include <net/ip_fib.h>
37 #include <net/ip6_fib.h>
38 #include <net/ip6_route.h>
44 	struct net *net;  member
57 static struct fib6_info *fib6_find_prefix(struct net *net,
60 static struct fib6_node *fib6_repair_tree(struct net *net,
63 static int fib6_walk(struct net *net, struct fib6_walker *w);
75 #define FOR_WALKERS(net, w) \  argument
76 	list_for_each_entry(w, &(net)->ipv6.fib6_walkers, lh)
78 static void fib6_walker_link(struct net *net, struct fib6_walker *w)  in fib6_walker_link()  argument
80 	write_lock_bh(&net->ipv6.fib6_walker_lock);  in fib6_walker_link()
81 	list_add(&w->lh, &net->ipv6.fib6_walkers);  in fib6_walker_link()
82 	write_unlock_bh(&net->ipv6.fib6_walker_lock);  in fib6_walker_link()
85 static void fib6_walker_unlink(struct net *net, struct fib6_walker *w)  in fib6_walker_unlink()  argument
87 	write_lock_bh(&net->ipv6.fib6_walker_lock);  in fib6_walker_unlink()
89 	write_unlock_bh(&net->ipv6.fib6_walker_lock);  in fib6_walker_unlink()
92 static int fib6_new_sernum(struct net *net)  in fib6_new_sernum()  argument
94 	int new, old = atomic_read(&net->ipv6.fib6_sernum);  in fib6_new_sernum()
98 	} while (!atomic_try_cmpxchg(&net->ipv6.fib6_sernum, &old, new));  in fib6_new_sernum()
107 void fib6_update_sernum(struct net *net, struct fib6_info *f6i)  in fib6_update_sernum()  argument
114 		WRITE_ONCE(fn->fn_sernum, fib6_new_sernum(net));  in fib6_update_sernum()
184 static struct fib6_node *node_alloc(struct net *net)  in node_alloc()  argument
190 		net->ipv6.rt6_stats->fib_nodes++;  in node_alloc()
195 static void node_free_immediate(struct net *net, struct fib6_node *fn)  in node_free_immediate()  argument
198 	net->ipv6.rt6_stats->fib_nodes--;  in node_free_immediate()
201 static void node_free(struct net *net, struct fib6_node *fn)  in node_free()  argument
204 	net->ipv6.rt6_stats->fib_nodes--;  in node_free()
213 static void fib6_link_table(struct net *net, struct fib6_table *tb)  in fib6_link_table()  argument
228 	hlist_add_head_rcu(&tb->tb6_hlist, &net->ipv6.fib_table_hash[h]);  in fib6_link_table()
233 static struct fib6_table *fib6_alloc_table(struct net *net, u32 id)  in fib6_alloc_table()  argument
241 				   net->ipv6.fib6_null_entry);  in fib6_alloc_table()
250 struct fib6_table *fib6_new_table(struct net *net, u32 id)  in fib6_new_table()  argument
257 	tb = fib6_get_table(net, id);  in fib6_new_table()
261 	new_tb = fib6_alloc_table(net, id);  in fib6_new_table()
265 	spin_lock_bh(&net->ipv6.fib_table_hash_lock);  in fib6_new_table()
267 	tb = fib6_get_table(net, id);  in fib6_new_table()
269 		spin_unlock_bh(&net->ipv6.fib_table_hash_lock);  in fib6_new_table()
274 	fib6_link_table(net, new_tb);  in fib6_new_table()
276 	spin_unlock_bh(&net->ipv6.fib_table_hash_lock);  in fib6_new_table()
282 struct fib6_table *fib6_get_table(struct net *net, u32 id)  in fib6_get_table()  argument
290 	head = &net->ipv6.fib_table_hash[id & (FIB6_TABLE_HASHSZ - 1)];  in fib6_get_table()
303 static void __net_init fib6_tables_init(struct net *net)  in fib6_tables_init()  argument
305 	fib6_link_table(net, net->ipv6.fib6_main_tbl);  in fib6_tables_init()
306 	fib6_link_table(net, net->ipv6.fib6_local_tbl);  in fib6_tables_init()
310 struct fib6_table *fib6_new_table(struct net *net, u32 id)  in fib6_new_table()  argument
312 	return fib6_get_table(net, id);  in fib6_new_table()
315 struct fib6_table *fib6_get_table(struct net *net, u32 id)  in fib6_get_table()  argument
317 	  return net->ipv6.fib6_main_tbl;  in fib6_get_table()
320 struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,  in fib6_rule_lookup()  argument
327 			net, net->ipv6.fib6_main_tbl, fl6, skb, flags);  in fib6_rule_lookup()
330 		rt = net->ipv6.ip6_null_entry;  in fib6_rule_lookup()
339 int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6,  in fib6_lookup()  argument
342 	return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6,  in fib6_lookup()
346 static void __net_init fib6_tables_init(struct net *net)  in fib6_tables_init()  argument
348 	fib6_link_table(net, net->ipv6.fib6_main_tbl);  in fib6_tables_init()
353 unsigned int fib6_tables_seq_read(const struct net *net)  in fib6_tables_seq_read()  argument
359 		const struct hlist_head *head = &net->ipv6.fib_table_hash[h];  in fib6_tables_seq_read()
398 int call_fib6_entry_notifiers(struct net *net,  in call_fib6_entry_notifiers()  argument
409 	return call_fib6_notifiers(net, event_type, &info.info);  in call_fib6_entry_notifiers()
412 int call_fib6_multipath_entry_notifiers(struct net *net,  in call_fib6_multipath_entry_notifiers()  argument
425 	return call_fib6_notifiers(net, event_type, &info.info);  in call_fib6_multipath_entry_notifiers()
428 int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt)  in call_fib6_entry_notifiers_replace()  argument
436 	return call_fib6_notifiers(net, FIB_EVENT_ENTRY_REPLACE, &info.info);  in call_fib6_entry_notifiers_replace()
440 	struct net *net;  member
451 	if (!rt || rt == arg->net->ipv6.fib6_null_entry)  in fib6_rt_dump()
476 static int fib6_table_dump(struct net *net, struct fib6_table *tb,  in fib6_table_dump()  argument
483 	err = fib6_walk(net, w);  in fib6_table_dump()
489 int fib6_tables_dump(struct net *net, struct notifier_block *nb,  in fib6_tables_dump()  argument
502 	arg.net = net;  in fib6_tables_dump()
508 		struct hlist_head *head = &net->ipv6.fib_table_hash[h];  in fib6_tables_dump()
512 			err = fib6_table_dump(net, tb, w);  in fib6_tables_dump()
561 	struct net *net = sock_net(cb->skb->sk);  in fib6_dump_end()  local
567 			fib6_walker_unlink(net, w);  in fib6_dump_end()
585 	struct net *net = sock_net(skb->sk);  in fib6_dump_table()  local
598 		res = fib6_walk(net, w);  in fib6_dump_table()
620 			fib6_walker_unlink(net, w);  in fib6_dump_table()
636 	struct net *net = sock_net(skb->sk);  in inet6_dump_fib()  local
646 		err = ip_valid_fib_dump_req(net, nlh, &arg.filter, cb);  in inet6_dump_fib()
679 	arg.net = net;  in inet6_dump_fib()
683 		tb = fib6_get_table(net, arg.filter.table_id);  in inet6_dump_fib()
706 		head = &net->ipv6.fib_table_hash[h];  in inet6_dump_fib()
754 static struct fib6_node *fib6_add_1(struct net *net,  in fib6_add_1()  argument
806 				   net->ipv6.fib6_null_entry) {  in fib6_add_1()
850 	ln = node_alloc(net);  in fib6_add_1()
891 		in = node_alloc(net);  in fib6_add_1()
892 		ln = node_alloc(net);  in fib6_add_1()
896 				node_free_immediate(net, in);  in fib6_add_1()
898 				node_free_immediate(net, ln);  in fib6_add_1()
943 		ln = node_alloc(net);  in fib6_add_1()
1032 			  struct net *net)  in fib6_purge_rt()  argument
1061 				new_leaf = fib6_find_prefix(net, table, fn);  in fib6_purge_rt()
1368 static void fib6_start_gc(struct net *net, struct fib6_info *rt)  in fib6_start_gc()  argument
1370 	if (!timer_pending(&net->ipv6.ip6_fib_timer) &&  in fib6_start_gc()
1372 		mod_timer(&net->ipv6.ip6_fib_timer,  in fib6_start_gc()
1373 			  jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);  in fib6_start_gc()
1376 void fib6_force_start_gc(struct net *net)  in fib6_force_start_gc()  argument
1378 	if (!timer_pending(&net->ipv6.ip6_fib_timer))  in fib6_force_start_gc()
1379 		mod_timer(&net->ipv6.ip6_fib_timer,  in fib6_force_start_gc()
1380 			  jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);  in fib6_force_start_gc()
1398 void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt)  in fib6_update_sernum_upto_root()  argument
1400 	__fib6_update_sernum_upto_root(rt, fib6_new_sernum(net));  in fib6_update_sernum_upto_root()
1404 void fib6_update_sernum_stub(struct net *net, struct fib6_info *f6i)  in fib6_update_sernum_stub()  argument
1407 	fib6_update_sernum_upto_root(net, f6i);  in fib6_update_sernum_stub()
1802 static struct fib6_info *fib6_find_prefix(struct net *net,  in fib6_find_prefix()  argument
1809 		return net->ipv6.fib6_null_entry;  in fib6_find_prefix()
1834 static struct fib6_node *fib6_repair_tree(struct net *net,  in fib6_repair_tree()  argument
1846 		rcu_assign_pointer(fn->leaf, net->ipv6.fib6_null_entry);  in fib6_repair_tree()
1891 			new_fn_leaf = fib6_find_prefix(net, table, fn);  in fib6_repair_tree()
1895 				new_fn_leaf = net->ipv6.fib6_null_entry;  in fib6_repair_tree()
1926 		read_lock(&net->ipv6.fib6_walker_lock);  in fib6_repair_tree()
1927 		FOR_WALKERS(net, w) {  in fib6_repair_tree()
1950 		read_unlock(&net->ipv6.fib6_walker_lock);  in fib6_repair_tree()
1952 		node_free(net, fn);  in fib6_repair_tree()
1969 	struct net *net = info->nl_net;  in fib6_del_route()  local
1989 	net->ipv6.rt6_stats->fib_rt_entries--;  in fib6_del_route()
1990 	net->ipv6.rt6_stats->fib_discarded_routes++;  in fib6_del_route()
2018 	read_lock(&net->ipv6.fib6_walker_lock);  in fib6_del_route()
2019 	FOR_WALKERS(net, w) {  in fib6_del_route()
2028 	read_unlock(&net->ipv6.fib6_walker_lock);  in fib6_del_route()
2037 			net->ipv6.rt6_stats->fib_route_nodes--;  in fib6_del_route()
2039 		fn = fib6_repair_tree(net, table, fn);  in fib6_del_route()
2042 	fib6_purge_rt(rt, fn, net);  in fib6_del_route()
2046 			call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL,  in fib6_del_route()
2049 			call_fib6_entry_notifiers_replace(net, replace_rt);  in fib6_del_route()
2060 	struct net *net = info->nl_net;  in fib6_del()  local
2066 	if (rt == net->ipv6.fib6_null_entry)  in fib6_del()
2211 static int fib6_walk(struct net *net, struct fib6_walker *w)  in fib6_walk()  argument
2218 	fib6_walker_link(net, w);  in fib6_walk()
2221 		fib6_walker_unlink(net, w);  in fib6_walk()
2231 		.nl_net = c->net,  in fib6_clean_node()
2282 static void fib6_clean_tree(struct net *net, struct fib6_node *root,  in fib6_clean_tree()  argument
2296 	c.net = net;  in fib6_clean_tree()
2299 	fib6_walk(net, &c.w);  in fib6_clean_tree()
2302 static void __fib6_clean_all(struct net *net,  in __fib6_clean_all()  argument
2312 		head = &net->ipv6.fib_table_hash[h];  in __fib6_clean_all()
2315 			fib6_clean_tree(net, &table->tb6_root,  in __fib6_clean_all()
2323 void fib6_clean_all(struct net *net, int (*func)(struct fib6_info *, void *),  in fib6_clean_all()  argument
2326 	__fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, false);  in fib6_clean_all()
2329 void fib6_clean_all_skip_notify(struct net *net,  in fib6_clean_all_skip_notify()  argument
2333 	__fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, true);  in fib6_clean_all_skip_notify()
2336 static void fib6_flush_trees(struct net *net)  in fib6_flush_trees()  argument
2338 	int new_sernum = fib6_new_sernum(net);  in fib6_flush_trees()
2340 	__fib6_clean_all(net, NULL, new_sernum, NULL, false);  in fib6_flush_trees()
2373 static void fib6_gc_table(struct net *net,  in fib6_gc_table()  argument
2380 		.nl_net = net,  in fib6_gc_table()
2389 static void fib6_gc_all(struct net *net, struct fib6_gc_args *gc_args)  in fib6_gc_all()  argument
2397 		head = &net->ipv6.fib_table_hash[h];  in fib6_gc_all()
2401 			fib6_gc_table(net, table, gc_args);  in fib6_gc_all()
2409 void fib6_run_gc(unsigned long expires, struct net *net, bool force)  in fib6_run_gc()  argument
2415 		spin_lock_bh(&net->ipv6.fib6_gc_lock);  in fib6_run_gc()
2416 	} else if (!spin_trylock_bh(&net->ipv6.fib6_gc_lock)) {  in fib6_run_gc()
2417 		mod_timer(&net->ipv6.ip6_fib_timer, jiffies + HZ);  in fib6_run_gc()
2421 			  net->ipv6.sysctl.ip6_rt_gc_interval;  in fib6_run_gc()
2424 	fib6_gc_all(net, &gc_args);  in fib6_run_gc()
2426 	net->ipv6.ip6_rt_last_gc = now;  in fib6_run_gc()
2429 		mod_timer(&net->ipv6.ip6_fib_timer,  in fib6_run_gc()
2431 					+ net->ipv6.sysctl.ip6_rt_gc_interval));  in fib6_run_gc()
2433 		timer_delete(&net->ipv6.ip6_fib_timer);  in fib6_run_gc()
2434 	spin_unlock_bh(&net->ipv6.fib6_gc_lock);  in fib6_run_gc()
2439 	struct net *arg = timer_container_of(arg, t, ipv6.ip6_fib_timer);  in fib6_gc_timer_cb()
2444 static int __net_init fib6_net_init(struct net *net)  in fib6_net_init()  argument
2449 	err = fib6_notifier_init(net);  in fib6_net_init()
2454 	net->ipv6.sysctl.multipath_hash_fields =  in fib6_net_init()
2457 	spin_lock_init(&net->ipv6.fib6_gc_lock);  in fib6_net_init()
2458 	rwlock_init(&net->ipv6.fib6_walker_lock);  in fib6_net_init()
2459 	INIT_LIST_HEAD(&net->ipv6.fib6_walkers);  in fib6_net_init()
2460 	timer_setup(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, 0);  in fib6_net_init()
2462 	net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);  in fib6_net_init()
2463 	if (!net->ipv6.rt6_stats)  in fib6_net_init()
2469 	net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);  in fib6_net_init()
2470 	if (!net->ipv6.fib_table_hash)  in fib6_net_init()
2473 	spin_lock_init(&net->ipv6.fib_table_hash_lock);  in fib6_net_init()
2475 	net->ipv6.fib6_main_tbl = kzalloc(sizeof(*net->ipv6.fib6_main_tbl),  in fib6_net_init()
2477 	if (!net->ipv6.fib6_main_tbl)  in fib6_net_init()
2480 	net->ipv6.fib6_main_tbl->tb6_id = RT6_TABLE_MAIN;  in fib6_net_init()
2481 	rcu_assign_pointer(net->ipv6.fib6_main_tbl->tb6_root.leaf,  in fib6_net_init()
2482 			   net->ipv6.fib6_null_entry);  in fib6_net_init()
2483 	net->ipv6.fib6_main_tbl->tb6_root.fn_flags =  in fib6_net_init()
2485 	inet_peer_base_init(&net->ipv6.fib6_main_tbl->tb6_peers);  in fib6_net_init()
2486 	INIT_HLIST_HEAD(&net->ipv6.fib6_main_tbl->tb6_gc_hlist);  in fib6_net_init()
2489 	net->ipv6.fib6_local_tbl = kzalloc(sizeof(*net->ipv6.fib6_local_tbl),  in fib6_net_init()
2491 	if (!net->ipv6.fib6_local_tbl)  in fib6_net_init()
2493 	net->ipv6.fib6_local_tbl->tb6_id = RT6_TABLE_LOCAL;  in fib6_net_init()
2494 	rcu_assign_pointer(net->ipv6.fib6_local_tbl->tb6_root.leaf,  in fib6_net_init()
2495 			   net->ipv6.fib6_null_entry);  in fib6_net_init()
2496 	net->ipv6.fib6_local_tbl->tb6_root.fn_flags =  in fib6_net_init()
2498 	inet_peer_base_init(&net->ipv6.fib6_local_tbl->tb6_peers);  in fib6_net_init()
2499 	INIT_HLIST_HEAD(&net->ipv6.fib6_local_tbl->tb6_gc_hlist);  in fib6_net_init()
2501 	fib6_tables_init(net);  in fib6_net_init()
2507 	kfree(net->ipv6.fib6_main_tbl);  in fib6_net_init()
2510 	kfree(net->ipv6.fib_table_hash);  in fib6_net_init()
2512 	kfree(net->ipv6.rt6_stats);  in fib6_net_init()
2514 	fib6_notifier_exit(net);  in fib6_net_init()
2518 static void fib6_net_exit(struct net *net)  in fib6_net_exit()  argument
2522 	timer_delete_sync(&net->ipv6.ip6_fib_timer);  in fib6_net_exit()
2525 		struct hlist_head *head = &net->ipv6.fib_table_hash[i];  in fib6_net_exit()
2535 	kfree(net->ipv6.fib_table_hash);  in fib6_net_exit()
2536 	kfree(net->ipv6.rt6_stats);  in fib6_net_exit()
2537 	fib6_notifier_exit(net);  in fib6_net_exit()
2639 				      struct net *net)  in ipv6_route_seq_setup_walk()  argument
2649 	fib6_walker_link(net, &iter->w);  in ipv6_route_seq_setup_walk()
2653 						    struct net *net)  in ipv6_route_seq_next_table()  argument
2668 			hlist_first_rcu(&net->ipv6.fib_table_hash[h++]));  in ipv6_route_seq_next_table()
2690 	struct net *net = seq_file_net(seq);  in ipv6_route_seq_next()  local
2709 		fib6_walker_unlink(net, &iter->w);  in ipv6_route_seq_next()
2712 	fib6_walker_unlink(net, &iter->w);  in ipv6_route_seq_next()
2714 	iter->tbl = ipv6_route_seq_next_table(iter->tbl, net);  in ipv6_route_seq_next()
2718 	ipv6_route_seq_setup_walk(iter, net);  in ipv6_route_seq_next()
2725 	struct net *net = seq_file_net(seq);  in ipv6_route_seq_start()  local
2729 	iter->tbl = ipv6_route_seq_next_table(NULL, net);  in ipv6_route_seq_start()
2735 		ipv6_route_seq_setup_walk(iter, net);  in ipv6_route_seq_start()
2751 	struct net *net = seq_file_net(seq);  in ipv6_route_native_seq_stop()  local
2755 		fib6_walker_unlink(net, &iter->w);  in ipv6_route_native_seq_stop()