1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * ROUTE - implementation of the IP router. 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Alan Cox, <gw4pts@gw4pts.ampr.org> 11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi> 12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 13 * 14 * Fixes: 15 * Alan Cox : Verify area fixes. 16 * Alan Cox : cli() protects routing changes 17 * Rui Oliveira : ICMP routing table updates 18 * (rco@di.uminho.pt) Routing table insertion and update 19 * Linus Torvalds : Rewrote bits to be sensible 20 * Alan Cox : Added BSD route gw semantics 21 * Alan Cox : Super /proc >4K 22 * Alan Cox : MTU in route table 23 * Alan Cox : MSS actually. Also added the window 24 * clamper. 25 * Sam Lantinga : Fixed route matching in rt_del() 26 * Alan Cox : Routing cache support. 27 * Alan Cox : Removed compatibility cruft. 28 * Alan Cox : RTF_REJECT support. 29 * Alan Cox : TCP irtt support. 30 * Jonathan Naylor : Added Metric support. 31 * Miquel van Smoorenburg : BSD API fixes. 32 * Miquel van Smoorenburg : Metrics. 33 * Alan Cox : Use __u32 properly 34 * Alan Cox : Aligned routing errors more closely with BSD 35 * our system is still very different. 36 * Alan Cox : Faster /proc handling 37 * Alexey Kuznetsov : Massive rework to support tree based routing, 38 * routing caches and better behaviour. 39 * 40 * Olaf Erb : irtt wasn't being copied right. 41 * Bjorn Ekwall : Kerneld route support. 42 * Alan Cox : Multicast fixed (I hope) 43 * Pavel Krauz : Limited broadcast fixed 44 * Mike McLagan : Routing by source 45 * Alexey Kuznetsov : End of old history. Split to fib.c and 46 * route.c and rewritten from scratch. 47 * Andi Kleen : Load-limit warning messages. 48 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow. 50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow. 51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful. 52 * Marc Boucher : routing by fwmark 53 * Robert Olsson : Added rt_cache statistics 54 * Arnaldo C. Melo : Convert proc stuff to seq_file 55 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes. 56 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect 57 * Ilia Sotnikov : Removed TOS from hash calculations 58 * 59 * This program is free software; you can redistribute it and/or 60 * modify it under the terms of the GNU General Public License 61 * as published by the Free Software Foundation; either version 62 * 2 of the License, or (at your option) any later version. 63 */ 64 65 #include <linux/module.h> 66 #include <asm/uaccess.h> 67 #include <asm/system.h> 68 #include <linux/bitops.h> 69 #include <linux/types.h> 70 #include <linux/kernel.h> 71 #include <linux/mm.h> 72 #include <linux/bootmem.h> 73 #include <linux/string.h> 74 #include <linux/socket.h> 75 #include <linux/sockios.h> 76 #include <linux/errno.h> 77 #include <linux/in.h> 78 #include <linux/inet.h> 79 #include <linux/netdevice.h> 80 #include <linux/proc_fs.h> 81 #include <linux/init.h> 82 #include <linux/workqueue.h> 83 #include <linux/skbuff.h> 84 #include <linux/inetdevice.h> 85 #include <linux/igmp.h> 86 #include <linux/pkt_sched.h> 87 #include <linux/mroute.h> 88 #include <linux/netfilter_ipv4.h> 89 #include <linux/random.h> 90 #include <linux/jhash.h> 91 #include <linux/rcupdate.h> 92 #include <linux/times.h> 93 #include <linux/slab.h> 94 #include <net/dst.h> 95 #include <net/net_namespace.h> 96 #include <net/protocol.h> 97 #include <net/ip.h> 98 #include <net/route.h> 99 #include <net/inetpeer.h> 100 #include <net/sock.h> 101 #include <net/ip_fib.h> 102 #include <net/arp.h> 103 #include <net/tcp.h> 104 #include <net/icmp.h> 105 #include <net/xfrm.h> 106 #include <net/netevent.h> 107 #include <net/rtnetlink.h> 108 #ifdef CONFIG_SYSCTL 109 #include <linux/sysctl.h> 110 #endif 111 112 #define RT_FL_TOS(oldflp) \ 113 ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK))) 114 115 #define IP_MAX_MTU 0xFFF0 116 117 #define RT_GC_TIMEOUT (300*HZ) 118 119 static int ip_rt_max_size; 120 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT; 121 static int ip_rt_gc_interval __read_mostly = 60 * HZ; 122 static int ip_rt_gc_min_interval __read_mostly = HZ / 2; 123 static int ip_rt_redirect_number __read_mostly = 9; 124 static int ip_rt_redirect_load __read_mostly = HZ / 50; 125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1)); 126 static int ip_rt_error_cost __read_mostly = HZ; 127 static int ip_rt_error_burst __read_mostly = 5 * HZ; 128 static int ip_rt_gc_elasticity __read_mostly = 8; 129 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ; 130 static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20; 131 static int ip_rt_min_advmss __read_mostly = 256; 132 static int rt_chain_length_max __read_mostly = 20; 133 134 static struct delayed_work expires_work; 135 static unsigned long expires_ljiffies; 136 137 /* 138 * Interface to generic destination cache. 139 */ 140 141 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie); 142 static unsigned int ipv4_default_advmss(const struct dst_entry *dst); 143 static unsigned int ipv4_default_mtu(const struct dst_entry *dst); 144 static void ipv4_dst_destroy(struct dst_entry *dst); 145 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst); 146 static void ipv4_link_failure(struct sk_buff *skb); 147 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu); 148 static int rt_garbage_collect(struct dst_ops *ops); 149 150 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev, 151 int how) 152 { 153 } 154 155 static struct dst_ops ipv4_dst_ops = { 156 .family = AF_INET, 157 .protocol = cpu_to_be16(ETH_P_IP), 158 .gc = rt_garbage_collect, 159 .check = ipv4_dst_check, 160 .default_advmss = ipv4_default_advmss, 161 .default_mtu = ipv4_default_mtu, 162 .destroy = ipv4_dst_destroy, 163 .ifdown = ipv4_dst_ifdown, 164 .negative_advice = ipv4_negative_advice, 165 .link_failure = ipv4_link_failure, 166 .update_pmtu = ip_rt_update_pmtu, 167 .local_out = __ip_local_out, 168 }; 169 170 #define ECN_OR_COST(class) TC_PRIO_##class 171 172 const __u8 ip_tos2prio[16] = { 173 TC_PRIO_BESTEFFORT, 174 ECN_OR_COST(FILLER), 175 TC_PRIO_BESTEFFORT, 176 ECN_OR_COST(BESTEFFORT), 177 TC_PRIO_BULK, 178 ECN_OR_COST(BULK), 179 TC_PRIO_BULK, 180 ECN_OR_COST(BULK), 181 TC_PRIO_INTERACTIVE, 182 ECN_OR_COST(INTERACTIVE), 183 TC_PRIO_INTERACTIVE, 184 ECN_OR_COST(INTERACTIVE), 185 TC_PRIO_INTERACTIVE_BULK, 186 ECN_OR_COST(INTERACTIVE_BULK), 187 TC_PRIO_INTERACTIVE_BULK, 188 ECN_OR_COST(INTERACTIVE_BULK) 189 }; 190 191 192 /* 193 * Route cache. 194 */ 195 196 /* The locking scheme is rather straight forward: 197 * 198 * 1) Read-Copy Update protects the buckets of the central route hash. 199 * 2) Only writers remove entries, and they hold the lock 200 * as they look at rtable reference counts. 201 * 3) Only readers acquire references to rtable entries, 202 * they do so with atomic increments and with the 203 * lock held. 204 */ 205 206 struct rt_hash_bucket { 207 struct rtable __rcu *chain; 208 }; 209 210 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \ 211 defined(CONFIG_PROVE_LOCKING) 212 /* 213 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks 214 * The size of this table is a power of two and depends on the number of CPUS. 215 * (on lockdep we have a quite big spinlock_t, so keep the size down there) 216 */ 217 #ifdef CONFIG_LOCKDEP 218 # define RT_HASH_LOCK_SZ 256 219 #else 220 # if NR_CPUS >= 32 221 # define RT_HASH_LOCK_SZ 4096 222 # elif NR_CPUS >= 16 223 # define RT_HASH_LOCK_SZ 2048 224 # elif NR_CPUS >= 8 225 # define RT_HASH_LOCK_SZ 1024 226 # elif NR_CPUS >= 4 227 # define RT_HASH_LOCK_SZ 512 228 # else 229 # define RT_HASH_LOCK_SZ 256 230 # endif 231 #endif 232 233 static spinlock_t *rt_hash_locks; 234 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)] 235 236 static __init void rt_hash_lock_init(void) 237 { 238 int i; 239 240 rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ, 241 GFP_KERNEL); 242 if (!rt_hash_locks) 243 panic("IP: failed to allocate rt_hash_locks\n"); 244 245 for (i = 0; i < RT_HASH_LOCK_SZ; i++) 246 spin_lock_init(&rt_hash_locks[i]); 247 } 248 #else 249 # define rt_hash_lock_addr(slot) NULL 250 251 static inline void rt_hash_lock_init(void) 252 { 253 } 254 #endif 255 256 static struct rt_hash_bucket *rt_hash_table __read_mostly; 257 static unsigned rt_hash_mask __read_mostly; 258 static unsigned int rt_hash_log __read_mostly; 259 260 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); 261 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field) 262 263 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx, 264 int genid) 265 { 266 return jhash_3words((__force u32)daddr, (__force u32)saddr, 267 idx, genid) 268 & rt_hash_mask; 269 } 270 271 static inline int rt_genid(struct net *net) 272 { 273 return atomic_read(&net->ipv4.rt_genid); 274 } 275 276 #ifdef CONFIG_PROC_FS 277 struct rt_cache_iter_state { 278 struct seq_net_private p; 279 int bucket; 280 int genid; 281 }; 282 283 static struct rtable *rt_cache_get_first(struct seq_file *seq) 284 { 285 struct rt_cache_iter_state *st = seq->private; 286 struct rtable *r = NULL; 287 288 for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) { 289 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain)) 290 continue; 291 rcu_read_lock_bh(); 292 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain); 293 while (r) { 294 if (dev_net(r->dst.dev) == seq_file_net(seq) && 295 r->rt_genid == st->genid) 296 return r; 297 r = rcu_dereference_bh(r->dst.rt_next); 298 } 299 rcu_read_unlock_bh(); 300 } 301 return r; 302 } 303 304 static struct rtable *__rt_cache_get_next(struct seq_file *seq, 305 struct rtable *r) 306 { 307 struct rt_cache_iter_state *st = seq->private; 308 309 r = rcu_dereference_bh(r->dst.rt_next); 310 while (!r) { 311 rcu_read_unlock_bh(); 312 do { 313 if (--st->bucket < 0) 314 return NULL; 315 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain)); 316 rcu_read_lock_bh(); 317 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain); 318 } 319 return r; 320 } 321 322 static struct rtable *rt_cache_get_next(struct seq_file *seq, 323 struct rtable *r) 324 { 325 struct rt_cache_iter_state *st = seq->private; 326 while ((r = __rt_cache_get_next(seq, r)) != NULL) { 327 if (dev_net(r->dst.dev) != seq_file_net(seq)) 328 continue; 329 if (r->rt_genid == st->genid) 330 break; 331 } 332 return r; 333 } 334 335 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos) 336 { 337 struct rtable *r = rt_cache_get_first(seq); 338 339 if (r) 340 while (pos && (r = rt_cache_get_next(seq, r))) 341 --pos; 342 return pos ? NULL : r; 343 } 344 345 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) 346 { 347 struct rt_cache_iter_state *st = seq->private; 348 if (*pos) 349 return rt_cache_get_idx(seq, *pos - 1); 350 st->genid = rt_genid(seq_file_net(seq)); 351 return SEQ_START_TOKEN; 352 } 353 354 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) 355 { 356 struct rtable *r; 357 358 if (v == SEQ_START_TOKEN) 359 r = rt_cache_get_first(seq); 360 else 361 r = rt_cache_get_next(seq, v); 362 ++*pos; 363 return r; 364 } 365 366 static void rt_cache_seq_stop(struct seq_file *seq, void *v) 367 { 368 if (v && v != SEQ_START_TOKEN) 369 rcu_read_unlock_bh(); 370 } 371 372 static int rt_cache_seq_show(struct seq_file *seq, void *v) 373 { 374 if (v == SEQ_START_TOKEN) 375 seq_printf(seq, "%-127s\n", 376 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t" 377 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t" 378 "HHUptod\tSpecDst"); 379 else { 380 struct rtable *r = v; 381 int len; 382 383 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t" 384 "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n", 385 r->dst.dev ? r->dst.dev->name : "*", 386 (__force u32)r->rt_dst, 387 (__force u32)r->rt_gateway, 388 r->rt_flags, atomic_read(&r->dst.__refcnt), 389 r->dst.__use, 0, (__force u32)r->rt_src, 390 dst_metric_advmss(&r->dst) + 40, 391 dst_metric(&r->dst, RTAX_WINDOW), 392 (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) + 393 dst_metric(&r->dst, RTAX_RTTVAR)), 394 r->fl.fl4_tos, 395 r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1, 396 r->dst.hh ? (r->dst.hh->hh_output == 397 dev_queue_xmit) : 0, 398 r->rt_spec_dst, &len); 399 400 seq_printf(seq, "%*s\n", 127 - len, ""); 401 } 402 return 0; 403 } 404 405 static const struct seq_operations rt_cache_seq_ops = { 406 .start = rt_cache_seq_start, 407 .next = rt_cache_seq_next, 408 .stop = rt_cache_seq_stop, 409 .show = rt_cache_seq_show, 410 }; 411 412 static int rt_cache_seq_open(struct inode *inode, struct file *file) 413 { 414 return seq_open_net(inode, file, &rt_cache_seq_ops, 415 sizeof(struct rt_cache_iter_state)); 416 } 417 418 static const struct file_operations rt_cache_seq_fops = { 419 .owner = THIS_MODULE, 420 .open = rt_cache_seq_open, 421 .read = seq_read, 422 .llseek = seq_lseek, 423 .release = seq_release_net, 424 }; 425 426 427 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos) 428 { 429 int cpu; 430 431 if (*pos == 0) 432 return SEQ_START_TOKEN; 433 434 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) { 435 if (!cpu_possible(cpu)) 436 continue; 437 *pos = cpu+1; 438 return &per_cpu(rt_cache_stat, cpu); 439 } 440 return NULL; 441 } 442 443 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) 444 { 445 int cpu; 446 447 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { 448 if (!cpu_possible(cpu)) 449 continue; 450 *pos = cpu+1; 451 return &per_cpu(rt_cache_stat, cpu); 452 } 453 return NULL; 454 455 } 456 457 static void rt_cpu_seq_stop(struct seq_file *seq, void *v) 458 { 459 460 } 461 462 static int rt_cpu_seq_show(struct seq_file *seq, void *v) 463 { 464 struct rt_cache_stat *st = v; 465 466 if (v == SEQ_START_TOKEN) { 467 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n"); 468 return 0; 469 } 470 471 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x " 472 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n", 473 dst_entries_get_slow(&ipv4_dst_ops), 474 st->in_hit, 475 st->in_slow_tot, 476 st->in_slow_mc, 477 st->in_no_route, 478 st->in_brd, 479 st->in_martian_dst, 480 st->in_martian_src, 481 482 st->out_hit, 483 st->out_slow_tot, 484 st->out_slow_mc, 485 486 st->gc_total, 487 st->gc_ignored, 488 st->gc_goal_miss, 489 st->gc_dst_overflow, 490 st->in_hlist_search, 491 st->out_hlist_search 492 ); 493 return 0; 494 } 495 496 static const struct seq_operations rt_cpu_seq_ops = { 497 .start = rt_cpu_seq_start, 498 .next = rt_cpu_seq_next, 499 .stop = rt_cpu_seq_stop, 500 .show = rt_cpu_seq_show, 501 }; 502 503 504 static int rt_cpu_seq_open(struct inode *inode, struct file *file) 505 { 506 return seq_open(file, &rt_cpu_seq_ops); 507 } 508 509 static const struct file_operations rt_cpu_seq_fops = { 510 .owner = THIS_MODULE, 511 .open = rt_cpu_seq_open, 512 .read = seq_read, 513 .llseek = seq_lseek, 514 .release = seq_release, 515 }; 516 517 #ifdef CONFIG_NET_CLS_ROUTE 518 static int rt_acct_proc_show(struct seq_file *m, void *v) 519 { 520 struct ip_rt_acct *dst, *src; 521 unsigned int i, j; 522 523 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL); 524 if (!dst) 525 return -ENOMEM; 526 527 for_each_possible_cpu(i) { 528 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i); 529 for (j = 0; j < 256; j++) { 530 dst[j].o_bytes += src[j].o_bytes; 531 dst[j].o_packets += src[j].o_packets; 532 dst[j].i_bytes += src[j].i_bytes; 533 dst[j].i_packets += src[j].i_packets; 534 } 535 } 536 537 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct)); 538 kfree(dst); 539 return 0; 540 } 541 542 static int rt_acct_proc_open(struct inode *inode, struct file *file) 543 { 544 return single_open(file, rt_acct_proc_show, NULL); 545 } 546 547 static const struct file_operations rt_acct_proc_fops = { 548 .owner = THIS_MODULE, 549 .open = rt_acct_proc_open, 550 .read = seq_read, 551 .llseek = seq_lseek, 552 .release = single_release, 553 }; 554 #endif 555 556 static int __net_init ip_rt_do_proc_init(struct net *net) 557 { 558 struct proc_dir_entry *pde; 559 560 pde = proc_net_fops_create(net, "rt_cache", S_IRUGO, 561 &rt_cache_seq_fops); 562 if (!pde) 563 goto err1; 564 565 pde = proc_create("rt_cache", S_IRUGO, 566 net->proc_net_stat, &rt_cpu_seq_fops); 567 if (!pde) 568 goto err2; 569 570 #ifdef CONFIG_NET_CLS_ROUTE 571 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops); 572 if (!pde) 573 goto err3; 574 #endif 575 return 0; 576 577 #ifdef CONFIG_NET_CLS_ROUTE 578 err3: 579 remove_proc_entry("rt_cache", net->proc_net_stat); 580 #endif 581 err2: 582 remove_proc_entry("rt_cache", net->proc_net); 583 err1: 584 return -ENOMEM; 585 } 586 587 static void __net_exit ip_rt_do_proc_exit(struct net *net) 588 { 589 remove_proc_entry("rt_cache", net->proc_net_stat); 590 remove_proc_entry("rt_cache", net->proc_net); 591 #ifdef CONFIG_NET_CLS_ROUTE 592 remove_proc_entry("rt_acct", net->proc_net); 593 #endif 594 } 595 596 static struct pernet_operations ip_rt_proc_ops __net_initdata = { 597 .init = ip_rt_do_proc_init, 598 .exit = ip_rt_do_proc_exit, 599 }; 600 601 static int __init ip_rt_proc_init(void) 602 { 603 return register_pernet_subsys(&ip_rt_proc_ops); 604 } 605 606 #else 607 static inline int ip_rt_proc_init(void) 608 { 609 return 0; 610 } 611 #endif /* CONFIG_PROC_FS */ 612 613 static inline void rt_free(struct rtable *rt) 614 { 615 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free); 616 } 617 618 static inline void rt_drop(struct rtable *rt) 619 { 620 ip_rt_put(rt); 621 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free); 622 } 623 624 static inline int rt_fast_clean(struct rtable *rth) 625 { 626 /* Kill broadcast/multicast entries very aggresively, if they 627 collide in hash table with more useful entries */ 628 return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) && 629 rt_is_input_route(rth) && rth->dst.rt_next; 630 } 631 632 static inline int rt_valuable(struct rtable *rth) 633 { 634 return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) || 635 rth->dst.expires; 636 } 637 638 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2) 639 { 640 unsigned long age; 641 int ret = 0; 642 643 if (atomic_read(&rth->dst.__refcnt)) 644 goto out; 645 646 ret = 1; 647 if (rth->dst.expires && 648 time_after_eq(jiffies, rth->dst.expires)) 649 goto out; 650 651 age = jiffies - rth->dst.lastuse; 652 ret = 0; 653 if ((age <= tmo1 && !rt_fast_clean(rth)) || 654 (age <= tmo2 && rt_valuable(rth))) 655 goto out; 656 ret = 1; 657 out: return ret; 658 } 659 660 /* Bits of score are: 661 * 31: very valuable 662 * 30: not quite useless 663 * 29..0: usage counter 664 */ 665 static inline u32 rt_score(struct rtable *rt) 666 { 667 u32 score = jiffies - rt->dst.lastuse; 668 669 score = ~score & ~(3<<30); 670 671 if (rt_valuable(rt)) 672 score |= (1<<31); 673 674 if (rt_is_output_route(rt) || 675 !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL))) 676 score |= (1<<30); 677 678 return score; 679 } 680 681 static inline bool rt_caching(const struct net *net) 682 { 683 return net->ipv4.current_rt_cache_rebuild_count <= 684 net->ipv4.sysctl_rt_cache_rebuild_count; 685 } 686 687 static inline bool compare_hash_inputs(const struct flowi *fl1, 688 const struct flowi *fl2) 689 { 690 return ((((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) | 691 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) | 692 (fl1->iif ^ fl2->iif)) == 0); 693 } 694 695 static inline int compare_keys(struct flowi *fl1, struct flowi *fl2) 696 { 697 return (((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) | 698 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) | 699 (fl1->mark ^ fl2->mark) | 700 (*(u16 *)&fl1->fl4_tos ^ *(u16 *)&fl2->fl4_tos) | 701 (fl1->oif ^ fl2->oif) | 702 (fl1->iif ^ fl2->iif)) == 0; 703 } 704 705 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2) 706 { 707 return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev)); 708 } 709 710 static inline int rt_is_expired(struct rtable *rth) 711 { 712 return rth->rt_genid != rt_genid(dev_net(rth->dst.dev)); 713 } 714 715 /* 716 * Perform a full scan of hash table and free all entries. 717 * Can be called by a softirq or a process. 718 * In the later case, we want to be reschedule if necessary 719 */ 720 static void rt_do_flush(struct net *net, int process_context) 721 { 722 unsigned int i; 723 struct rtable *rth, *next; 724 725 for (i = 0; i <= rt_hash_mask; i++) { 726 struct rtable __rcu **pprev; 727 struct rtable *list; 728 729 if (process_context && need_resched()) 730 cond_resched(); 731 rth = rcu_dereference_raw(rt_hash_table[i].chain); 732 if (!rth) 733 continue; 734 735 spin_lock_bh(rt_hash_lock_addr(i)); 736 737 list = NULL; 738 pprev = &rt_hash_table[i].chain; 739 rth = rcu_dereference_protected(*pprev, 740 lockdep_is_held(rt_hash_lock_addr(i))); 741 742 while (rth) { 743 next = rcu_dereference_protected(rth->dst.rt_next, 744 lockdep_is_held(rt_hash_lock_addr(i))); 745 746 if (!net || 747 net_eq(dev_net(rth->dst.dev), net)) { 748 rcu_assign_pointer(*pprev, next); 749 rcu_assign_pointer(rth->dst.rt_next, list); 750 list = rth; 751 } else { 752 pprev = &rth->dst.rt_next; 753 } 754 rth = next; 755 } 756 757 spin_unlock_bh(rt_hash_lock_addr(i)); 758 759 for (; list; list = next) { 760 next = rcu_dereference_protected(list->dst.rt_next, 1); 761 rt_free(list); 762 } 763 } 764 } 765 766 /* 767 * While freeing expired entries, we compute average chain length 768 * and standard deviation, using fixed-point arithmetic. 769 * This to have an estimation of rt_chain_length_max 770 * rt_chain_length_max = max(elasticity, AVG + 4*SD) 771 * We use 3 bits for frational part, and 29 (or 61) for magnitude. 772 */ 773 774 #define FRACT_BITS 3 775 #define ONE (1UL << FRACT_BITS) 776 777 /* 778 * Given a hash chain and an item in this hash chain, 779 * find if a previous entry has the same hash_inputs 780 * (but differs on tos, mark or oif) 781 * Returns 0 if an alias is found. 782 * Returns ONE if rth has no alias before itself. 783 */ 784 static int has_noalias(const struct rtable *head, const struct rtable *rth) 785 { 786 const struct rtable *aux = head; 787 788 while (aux != rth) { 789 if (compare_hash_inputs(&aux->fl, &rth->fl)) 790 return 0; 791 aux = rcu_dereference_protected(aux->dst.rt_next, 1); 792 } 793 return ONE; 794 } 795 796 static void rt_check_expire(void) 797 { 798 static unsigned int rover; 799 unsigned int i = rover, goal; 800 struct rtable *rth; 801 struct rtable __rcu **rthp; 802 unsigned long samples = 0; 803 unsigned long sum = 0, sum2 = 0; 804 unsigned long delta; 805 u64 mult; 806 807 delta = jiffies - expires_ljiffies; 808 expires_ljiffies = jiffies; 809 mult = ((u64)delta) << rt_hash_log; 810 if (ip_rt_gc_timeout > 1) 811 do_div(mult, ip_rt_gc_timeout); 812 goal = (unsigned int)mult; 813 if (goal > rt_hash_mask) 814 goal = rt_hash_mask + 1; 815 for (; goal > 0; goal--) { 816 unsigned long tmo = ip_rt_gc_timeout; 817 unsigned long length; 818 819 i = (i + 1) & rt_hash_mask; 820 rthp = &rt_hash_table[i].chain; 821 822 if (need_resched()) 823 cond_resched(); 824 825 samples++; 826 827 if (rcu_dereference_raw(*rthp) == NULL) 828 continue; 829 length = 0; 830 spin_lock_bh(rt_hash_lock_addr(i)); 831 while ((rth = rcu_dereference_protected(*rthp, 832 lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) { 833 prefetch(rth->dst.rt_next); 834 if (rt_is_expired(rth)) { 835 *rthp = rth->dst.rt_next; 836 rt_free(rth); 837 continue; 838 } 839 if (rth->dst.expires) { 840 /* Entry is expired even if it is in use */ 841 if (time_before_eq(jiffies, rth->dst.expires)) { 842 nofree: 843 tmo >>= 1; 844 rthp = &rth->dst.rt_next; 845 /* 846 * We only count entries on 847 * a chain with equal hash inputs once 848 * so that entries for different QOS 849 * levels, and other non-hash input 850 * attributes don't unfairly skew 851 * the length computation 852 */ 853 length += has_noalias(rt_hash_table[i].chain, rth); 854 continue; 855 } 856 } else if (!rt_may_expire(rth, tmo, ip_rt_gc_timeout)) 857 goto nofree; 858 859 /* Cleanup aged off entries. */ 860 *rthp = rth->dst.rt_next; 861 rt_free(rth); 862 } 863 spin_unlock_bh(rt_hash_lock_addr(i)); 864 sum += length; 865 sum2 += length*length; 866 } 867 if (samples) { 868 unsigned long avg = sum / samples; 869 unsigned long sd = int_sqrt(sum2 / samples - avg*avg); 870 rt_chain_length_max = max_t(unsigned long, 871 ip_rt_gc_elasticity, 872 (avg + 4*sd) >> FRACT_BITS); 873 } 874 rover = i; 875 } 876 877 /* 878 * rt_worker_func() is run in process context. 879 * we call rt_check_expire() to scan part of the hash table 880 */ 881 static void rt_worker_func(struct work_struct *work) 882 { 883 rt_check_expire(); 884 schedule_delayed_work(&expires_work, ip_rt_gc_interval); 885 } 886 887 /* 888 * Pertubation of rt_genid by a small quantity [1..256] 889 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate() 890 * many times (2^24) without giving recent rt_genid. 891 * Jenkins hash is strong enough that litle changes of rt_genid are OK. 892 */ 893 static void rt_cache_invalidate(struct net *net) 894 { 895 unsigned char shuffle; 896 897 get_random_bytes(&shuffle, sizeof(shuffle)); 898 atomic_add(shuffle + 1U, &net->ipv4.rt_genid); 899 } 900 901 /* 902 * delay < 0 : invalidate cache (fast : entries will be deleted later) 903 * delay >= 0 : invalidate & flush cache (can be long) 904 */ 905 void rt_cache_flush(struct net *net, int delay) 906 { 907 rt_cache_invalidate(net); 908 if (delay >= 0) 909 rt_do_flush(net, !in_softirq()); 910 } 911 912 /* Flush previous cache invalidated entries from the cache */ 913 void rt_cache_flush_batch(struct net *net) 914 { 915 rt_do_flush(net, !in_softirq()); 916 } 917 918 static void rt_emergency_hash_rebuild(struct net *net) 919 { 920 if (net_ratelimit()) 921 printk(KERN_WARNING "Route hash chain too long!\n"); 922 rt_cache_invalidate(net); 923 } 924 925 /* 926 Short description of GC goals. 927 928 We want to build algorithm, which will keep routing cache 929 at some equilibrium point, when number of aged off entries 930 is kept approximately equal to newly generated ones. 931 932 Current expiration strength is variable "expire". 933 We try to adjust it dynamically, so that if networking 934 is idle expires is large enough to keep enough of warm entries, 935 and when load increases it reduces to limit cache size. 936 */ 937 938 static int rt_garbage_collect(struct dst_ops *ops) 939 { 940 static unsigned long expire = RT_GC_TIMEOUT; 941 static unsigned long last_gc; 942 static int rover; 943 static int equilibrium; 944 struct rtable *rth; 945 struct rtable __rcu **rthp; 946 unsigned long now = jiffies; 947 int goal; 948 int entries = dst_entries_get_fast(&ipv4_dst_ops); 949 950 /* 951 * Garbage collection is pretty expensive, 952 * do not make it too frequently. 953 */ 954 955 RT_CACHE_STAT_INC(gc_total); 956 957 if (now - last_gc < ip_rt_gc_min_interval && 958 entries < ip_rt_max_size) { 959 RT_CACHE_STAT_INC(gc_ignored); 960 goto out; 961 } 962 963 entries = dst_entries_get_slow(&ipv4_dst_ops); 964 /* Calculate number of entries, which we want to expire now. */ 965 goal = entries - (ip_rt_gc_elasticity << rt_hash_log); 966 if (goal <= 0) { 967 if (equilibrium < ipv4_dst_ops.gc_thresh) 968 equilibrium = ipv4_dst_ops.gc_thresh; 969 goal = entries - equilibrium; 970 if (goal > 0) { 971 equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1); 972 goal = entries - equilibrium; 973 } 974 } else { 975 /* We are in dangerous area. Try to reduce cache really 976 * aggressively. 977 */ 978 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1); 979 equilibrium = entries - goal; 980 } 981 982 if (now - last_gc >= ip_rt_gc_min_interval) 983 last_gc = now; 984 985 if (goal <= 0) { 986 equilibrium += goal; 987 goto work_done; 988 } 989 990 do { 991 int i, k; 992 993 for (i = rt_hash_mask, k = rover; i >= 0; i--) { 994 unsigned long tmo = expire; 995 996 k = (k + 1) & rt_hash_mask; 997 rthp = &rt_hash_table[k].chain; 998 spin_lock_bh(rt_hash_lock_addr(k)); 999 while ((rth = rcu_dereference_protected(*rthp, 1000 lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) { 1001 if (!rt_is_expired(rth) && 1002 !rt_may_expire(rth, tmo, expire)) { 1003 tmo >>= 1; 1004 rthp = &rth->dst.rt_next; 1005 continue; 1006 } 1007 *rthp = rth->dst.rt_next; 1008 rt_free(rth); 1009 goal--; 1010 } 1011 spin_unlock_bh(rt_hash_lock_addr(k)); 1012 if (goal <= 0) 1013 break; 1014 } 1015 rover = k; 1016 1017 if (goal <= 0) 1018 goto work_done; 1019 1020 /* Goal is not achieved. We stop process if: 1021 1022 - if expire reduced to zero. Otherwise, expire is halfed. 1023 - if table is not full. 1024 - if we are called from interrupt. 1025 - jiffies check is just fallback/debug loop breaker. 1026 We will not spin here for long time in any case. 1027 */ 1028 1029 RT_CACHE_STAT_INC(gc_goal_miss); 1030 1031 if (expire == 0) 1032 break; 1033 1034 expire >>= 1; 1035 #if RT_CACHE_DEBUG >= 2 1036 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire, 1037 dst_entries_get_fast(&ipv4_dst_ops), goal, i); 1038 #endif 1039 1040 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size) 1041 goto out; 1042 } while (!in_softirq() && time_before_eq(jiffies, now)); 1043 1044 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size) 1045 goto out; 1046 if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size) 1047 goto out; 1048 if (net_ratelimit()) 1049 printk(KERN_WARNING "dst cache overflow\n"); 1050 RT_CACHE_STAT_INC(gc_dst_overflow); 1051 return 1; 1052 1053 work_done: 1054 expire += ip_rt_gc_min_interval; 1055 if (expire > ip_rt_gc_timeout || 1056 dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh || 1057 dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh) 1058 expire = ip_rt_gc_timeout; 1059 #if RT_CACHE_DEBUG >= 2 1060 printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire, 1061 dst_entries_get_fast(&ipv4_dst_ops), goal, rover); 1062 #endif 1063 out: return 0; 1064 } 1065 1066 /* 1067 * Returns number of entries in a hash chain that have different hash_inputs 1068 */ 1069 static int slow_chain_length(const struct rtable *head) 1070 { 1071 int length = 0; 1072 const struct rtable *rth = head; 1073 1074 while (rth) { 1075 length += has_noalias(head, rth); 1076 rth = rcu_dereference_protected(rth->dst.rt_next, 1); 1077 } 1078 return length >> FRACT_BITS; 1079 } 1080 1081 static int rt_intern_hash(unsigned hash, struct rtable *rt, 1082 struct rtable **rp, struct sk_buff *skb, int ifindex) 1083 { 1084 struct rtable *rth, *cand; 1085 struct rtable __rcu **rthp, **candp; 1086 unsigned long now; 1087 u32 min_score; 1088 int chain_length; 1089 int attempts = !in_softirq(); 1090 1091 restart: 1092 chain_length = 0; 1093 min_score = ~(u32)0; 1094 cand = NULL; 1095 candp = NULL; 1096 now = jiffies; 1097 1098 if (!rt_caching(dev_net(rt->dst.dev))) { 1099 /* 1100 * If we're not caching, just tell the caller we 1101 * were successful and don't touch the route. The 1102 * caller hold the sole reference to the cache entry, and 1103 * it will be released when the caller is done with it. 1104 * If we drop it here, the callers have no way to resolve routes 1105 * when we're not caching. Instead, just point *rp at rt, so 1106 * the caller gets a single use out of the route 1107 * Note that we do rt_free on this new route entry, so that 1108 * once its refcount hits zero, we are still able to reap it 1109 * (Thanks Alexey) 1110 * Note: To avoid expensive rcu stuff for this uncached dst, 1111 * we set DST_NOCACHE so that dst_release() can free dst without 1112 * waiting a grace period. 1113 */ 1114 1115 rt->dst.flags |= DST_NOCACHE; 1116 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) { 1117 int err = arp_bind_neighbour(&rt->dst); 1118 if (err) { 1119 if (net_ratelimit()) 1120 printk(KERN_WARNING 1121 "Neighbour table failure & not caching routes.\n"); 1122 ip_rt_put(rt); 1123 return err; 1124 } 1125 } 1126 1127 goto skip_hashing; 1128 } 1129 1130 rthp = &rt_hash_table[hash].chain; 1131 1132 spin_lock_bh(rt_hash_lock_addr(hash)); 1133 while ((rth = rcu_dereference_protected(*rthp, 1134 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) { 1135 if (rt_is_expired(rth)) { 1136 *rthp = rth->dst.rt_next; 1137 rt_free(rth); 1138 continue; 1139 } 1140 if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) { 1141 /* Put it first */ 1142 *rthp = rth->dst.rt_next; 1143 /* 1144 * Since lookup is lockfree, the deletion 1145 * must be visible to another weakly ordered CPU before 1146 * the insertion at the start of the hash chain. 1147 */ 1148 rcu_assign_pointer(rth->dst.rt_next, 1149 rt_hash_table[hash].chain); 1150 /* 1151 * Since lookup is lockfree, the update writes 1152 * must be ordered for consistency on SMP. 1153 */ 1154 rcu_assign_pointer(rt_hash_table[hash].chain, rth); 1155 1156 dst_use(&rth->dst, now); 1157 spin_unlock_bh(rt_hash_lock_addr(hash)); 1158 1159 rt_drop(rt); 1160 if (rp) 1161 *rp = rth; 1162 else 1163 skb_dst_set(skb, &rth->dst); 1164 return 0; 1165 } 1166 1167 if (!atomic_read(&rth->dst.__refcnt)) { 1168 u32 score = rt_score(rth); 1169 1170 if (score <= min_score) { 1171 cand = rth; 1172 candp = rthp; 1173 min_score = score; 1174 } 1175 } 1176 1177 chain_length++; 1178 1179 rthp = &rth->dst.rt_next; 1180 } 1181 1182 if (cand) { 1183 /* ip_rt_gc_elasticity used to be average length of chain 1184 * length, when exceeded gc becomes really aggressive. 1185 * 1186 * The second limit is less certain. At the moment it allows 1187 * only 2 entries per bucket. We will see. 1188 */ 1189 if (chain_length > ip_rt_gc_elasticity) { 1190 *candp = cand->dst.rt_next; 1191 rt_free(cand); 1192 } 1193 } else { 1194 if (chain_length > rt_chain_length_max && 1195 slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) { 1196 struct net *net = dev_net(rt->dst.dev); 1197 int num = ++net->ipv4.current_rt_cache_rebuild_count; 1198 if (!rt_caching(net)) { 1199 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n", 1200 rt->dst.dev->name, num); 1201 } 1202 rt_emergency_hash_rebuild(net); 1203 spin_unlock_bh(rt_hash_lock_addr(hash)); 1204 1205 hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src, 1206 ifindex, rt_genid(net)); 1207 goto restart; 1208 } 1209 } 1210 1211 /* Try to bind route to arp only if it is output 1212 route or unicast forwarding path. 1213 */ 1214 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) { 1215 int err = arp_bind_neighbour(&rt->dst); 1216 if (err) { 1217 spin_unlock_bh(rt_hash_lock_addr(hash)); 1218 1219 if (err != -ENOBUFS) { 1220 rt_drop(rt); 1221 return err; 1222 } 1223 1224 /* Neighbour tables are full and nothing 1225 can be released. Try to shrink route cache, 1226 it is most likely it holds some neighbour records. 1227 */ 1228 if (attempts-- > 0) { 1229 int saved_elasticity = ip_rt_gc_elasticity; 1230 int saved_int = ip_rt_gc_min_interval; 1231 ip_rt_gc_elasticity = 1; 1232 ip_rt_gc_min_interval = 0; 1233 rt_garbage_collect(&ipv4_dst_ops); 1234 ip_rt_gc_min_interval = saved_int; 1235 ip_rt_gc_elasticity = saved_elasticity; 1236 goto restart; 1237 } 1238 1239 if (net_ratelimit()) 1240 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n"); 1241 rt_drop(rt); 1242 return -ENOBUFS; 1243 } 1244 } 1245 1246 rt->dst.rt_next = rt_hash_table[hash].chain; 1247 1248 #if RT_CACHE_DEBUG >= 2 1249 if (rt->dst.rt_next) { 1250 struct rtable *trt; 1251 printk(KERN_DEBUG "rt_cache @%02x: %pI4", 1252 hash, &rt->rt_dst); 1253 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next) 1254 printk(" . %pI4", &trt->rt_dst); 1255 printk("\n"); 1256 } 1257 #endif 1258 /* 1259 * Since lookup is lockfree, we must make sure 1260 * previous writes to rt are comitted to memory 1261 * before making rt visible to other CPUS. 1262 */ 1263 rcu_assign_pointer(rt_hash_table[hash].chain, rt); 1264 1265 spin_unlock_bh(rt_hash_lock_addr(hash)); 1266 1267 skip_hashing: 1268 if (rp) 1269 *rp = rt; 1270 else 1271 skb_dst_set(skb, &rt->dst); 1272 return 0; 1273 } 1274 1275 void rt_bind_peer(struct rtable *rt, int create) 1276 { 1277 struct inet_peer *peer; 1278 1279 peer = inet_getpeer_v4(rt->rt_dst, create); 1280 1281 if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL) 1282 inet_putpeer(peer); 1283 } 1284 1285 /* 1286 * Peer allocation may fail only in serious out-of-memory conditions. However 1287 * we still can generate some output. 1288 * Random ID selection looks a bit dangerous because we have no chances to 1289 * select ID being unique in a reasonable period of time. 1290 * But broken packet identifier may be better than no packet at all. 1291 */ 1292 static void ip_select_fb_ident(struct iphdr *iph) 1293 { 1294 static DEFINE_SPINLOCK(ip_fb_id_lock); 1295 static u32 ip_fallback_id; 1296 u32 salt; 1297 1298 spin_lock_bh(&ip_fb_id_lock); 1299 salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr); 1300 iph->id = htons(salt & 0xFFFF); 1301 ip_fallback_id = salt; 1302 spin_unlock_bh(&ip_fb_id_lock); 1303 } 1304 1305 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more) 1306 { 1307 struct rtable *rt = (struct rtable *) dst; 1308 1309 if (rt) { 1310 if (rt->peer == NULL) 1311 rt_bind_peer(rt, 1); 1312 1313 /* If peer is attached to destination, it is never detached, 1314 so that we need not to grab a lock to dereference it. 1315 */ 1316 if (rt->peer) { 1317 iph->id = htons(inet_getid(rt->peer, more)); 1318 return; 1319 } 1320 } else 1321 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n", 1322 __builtin_return_address(0)); 1323 1324 ip_select_fb_ident(iph); 1325 } 1326 EXPORT_SYMBOL(__ip_select_ident); 1327 1328 static void rt_del(unsigned hash, struct rtable *rt) 1329 { 1330 struct rtable __rcu **rthp; 1331 struct rtable *aux; 1332 1333 rthp = &rt_hash_table[hash].chain; 1334 spin_lock_bh(rt_hash_lock_addr(hash)); 1335 ip_rt_put(rt); 1336 while ((aux = rcu_dereference_protected(*rthp, 1337 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) { 1338 if (aux == rt || rt_is_expired(aux)) { 1339 *rthp = aux->dst.rt_next; 1340 rt_free(aux); 1341 continue; 1342 } 1343 rthp = &aux->dst.rt_next; 1344 } 1345 spin_unlock_bh(rt_hash_lock_addr(hash)); 1346 } 1347 1348 /* called in rcu_read_lock() section */ 1349 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw, 1350 __be32 saddr, struct net_device *dev) 1351 { 1352 int i, k; 1353 struct in_device *in_dev = __in_dev_get_rcu(dev); 1354 struct rtable *rth; 1355 struct rtable __rcu **rthp; 1356 __be32 skeys[2] = { saddr, 0 }; 1357 int ikeys[2] = { dev->ifindex, 0 }; 1358 struct netevent_redirect netevent; 1359 struct net *net; 1360 1361 if (!in_dev) 1362 return; 1363 1364 net = dev_net(dev); 1365 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) || 1366 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) || 1367 ipv4_is_zeronet(new_gw)) 1368 goto reject_redirect; 1369 1370 if (!rt_caching(net)) 1371 goto reject_redirect; 1372 1373 if (!IN_DEV_SHARED_MEDIA(in_dev)) { 1374 if (!inet_addr_onlink(in_dev, new_gw, old_gw)) 1375 goto reject_redirect; 1376 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev)) 1377 goto reject_redirect; 1378 } else { 1379 if (inet_addr_type(net, new_gw) != RTN_UNICAST) 1380 goto reject_redirect; 1381 } 1382 1383 for (i = 0; i < 2; i++) { 1384 for (k = 0; k < 2; k++) { 1385 unsigned hash = rt_hash(daddr, skeys[i], ikeys[k], 1386 rt_genid(net)); 1387 1388 rthp = &rt_hash_table[hash].chain; 1389 1390 while ((rth = rcu_dereference(*rthp)) != NULL) { 1391 struct rtable *rt; 1392 1393 if (rth->fl.fl4_dst != daddr || 1394 rth->fl.fl4_src != skeys[i] || 1395 rth->fl.oif != ikeys[k] || 1396 rt_is_input_route(rth) || 1397 rt_is_expired(rth) || 1398 !net_eq(dev_net(rth->dst.dev), net)) { 1399 rthp = &rth->dst.rt_next; 1400 continue; 1401 } 1402 1403 if (rth->rt_dst != daddr || 1404 rth->rt_src != saddr || 1405 rth->dst.error || 1406 rth->rt_gateway != old_gw || 1407 rth->dst.dev != dev) 1408 break; 1409 1410 dst_hold(&rth->dst); 1411 1412 rt = dst_alloc(&ipv4_dst_ops); 1413 if (rt == NULL) { 1414 ip_rt_put(rth); 1415 return; 1416 } 1417 1418 /* Copy all the information. */ 1419 *rt = *rth; 1420 rt->dst.__use = 1; 1421 atomic_set(&rt->dst.__refcnt, 1); 1422 rt->dst.child = NULL; 1423 if (rt->dst.dev) 1424 dev_hold(rt->dst.dev); 1425 rt->dst.obsolete = -1; 1426 rt->dst.lastuse = jiffies; 1427 rt->dst.path = &rt->dst; 1428 rt->dst.neighbour = NULL; 1429 rt->dst.hh = NULL; 1430 #ifdef CONFIG_XFRM 1431 rt->dst.xfrm = NULL; 1432 #endif 1433 rt->rt_genid = rt_genid(net); 1434 rt->rt_flags |= RTCF_REDIRECTED; 1435 1436 /* Gateway is different ... */ 1437 rt->rt_gateway = new_gw; 1438 1439 /* Redirect received -> path was valid */ 1440 dst_confirm(&rth->dst); 1441 1442 if (rt->peer) 1443 atomic_inc(&rt->peer->refcnt); 1444 1445 if (arp_bind_neighbour(&rt->dst) || 1446 !(rt->dst.neighbour->nud_state & 1447 NUD_VALID)) { 1448 if (rt->dst.neighbour) 1449 neigh_event_send(rt->dst.neighbour, NULL); 1450 ip_rt_put(rth); 1451 rt_drop(rt); 1452 goto do_next; 1453 } 1454 1455 netevent.old = &rth->dst; 1456 netevent.new = &rt->dst; 1457 call_netevent_notifiers(NETEVENT_REDIRECT, 1458 &netevent); 1459 1460 rt_del(hash, rth); 1461 if (!rt_intern_hash(hash, rt, &rt, NULL, rt->fl.oif)) 1462 ip_rt_put(rt); 1463 goto do_next; 1464 } 1465 do_next: 1466 ; 1467 } 1468 } 1469 return; 1470 1471 reject_redirect: 1472 #ifdef CONFIG_IP_ROUTE_VERBOSE 1473 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) 1474 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n" 1475 " Advised path = %pI4 -> %pI4\n", 1476 &old_gw, dev->name, &new_gw, 1477 &saddr, &daddr); 1478 #endif 1479 ; 1480 } 1481 1482 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst) 1483 { 1484 struct rtable *rt = (struct rtable *)dst; 1485 struct dst_entry *ret = dst; 1486 1487 if (rt) { 1488 if (dst->obsolete > 0) { 1489 ip_rt_put(rt); 1490 ret = NULL; 1491 } else if ((rt->rt_flags & RTCF_REDIRECTED) || 1492 (rt->dst.expires && 1493 time_after_eq(jiffies, rt->dst.expires))) { 1494 unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src, 1495 rt->fl.oif, 1496 rt_genid(dev_net(dst->dev))); 1497 #if RT_CACHE_DEBUG >= 1 1498 printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n", 1499 &rt->rt_dst, rt->fl.fl4_tos); 1500 #endif 1501 rt_del(hash, rt); 1502 ret = NULL; 1503 } 1504 } 1505 return ret; 1506 } 1507 1508 /* 1509 * Algorithm: 1510 * 1. The first ip_rt_redirect_number redirects are sent 1511 * with exponential backoff, then we stop sending them at all, 1512 * assuming that the host ignores our redirects. 1513 * 2. If we did not see packets requiring redirects 1514 * during ip_rt_redirect_silence, we assume that the host 1515 * forgot redirected route and start to send redirects again. 1516 * 1517 * This algorithm is much cheaper and more intelligent than dumb load limiting 1518 * in icmp.c. 1519 * 1520 * NOTE. Do not forget to inhibit load limiting for redirects (redundant) 1521 * and "frag. need" (breaks PMTU discovery) in icmp.c. 1522 */ 1523 1524 void ip_rt_send_redirect(struct sk_buff *skb) 1525 { 1526 struct rtable *rt = skb_rtable(skb); 1527 struct in_device *in_dev; 1528 int log_martians; 1529 1530 rcu_read_lock(); 1531 in_dev = __in_dev_get_rcu(rt->dst.dev); 1532 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) { 1533 rcu_read_unlock(); 1534 return; 1535 } 1536 log_martians = IN_DEV_LOG_MARTIANS(in_dev); 1537 rcu_read_unlock(); 1538 1539 /* No redirected packets during ip_rt_redirect_silence; 1540 * reset the algorithm. 1541 */ 1542 if (time_after(jiffies, rt->dst.rate_last + ip_rt_redirect_silence)) 1543 rt->dst.rate_tokens = 0; 1544 1545 /* Too many ignored redirects; do not send anything 1546 * set dst.rate_last to the last seen redirected packet. 1547 */ 1548 if (rt->dst.rate_tokens >= ip_rt_redirect_number) { 1549 rt->dst.rate_last = jiffies; 1550 return; 1551 } 1552 1553 /* Check for load limit; set rate_last to the latest sent 1554 * redirect. 1555 */ 1556 if (rt->dst.rate_tokens == 0 || 1557 time_after(jiffies, 1558 (rt->dst.rate_last + 1559 (ip_rt_redirect_load << rt->dst.rate_tokens)))) { 1560 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway); 1561 rt->dst.rate_last = jiffies; 1562 ++rt->dst.rate_tokens; 1563 #ifdef CONFIG_IP_ROUTE_VERBOSE 1564 if (log_martians && 1565 rt->dst.rate_tokens == ip_rt_redirect_number && 1566 net_ratelimit()) 1567 printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n", 1568 &rt->rt_src, rt->rt_iif, 1569 &rt->rt_dst, &rt->rt_gateway); 1570 #endif 1571 } 1572 } 1573 1574 static int ip_error(struct sk_buff *skb) 1575 { 1576 struct rtable *rt = skb_rtable(skb); 1577 unsigned long now; 1578 int code; 1579 1580 switch (rt->dst.error) { 1581 case EINVAL: 1582 default: 1583 goto out; 1584 case EHOSTUNREACH: 1585 code = ICMP_HOST_UNREACH; 1586 break; 1587 case ENETUNREACH: 1588 code = ICMP_NET_UNREACH; 1589 IP_INC_STATS_BH(dev_net(rt->dst.dev), 1590 IPSTATS_MIB_INNOROUTES); 1591 break; 1592 case EACCES: 1593 code = ICMP_PKT_FILTERED; 1594 break; 1595 } 1596 1597 now = jiffies; 1598 rt->dst.rate_tokens += now - rt->dst.rate_last; 1599 if (rt->dst.rate_tokens > ip_rt_error_burst) 1600 rt->dst.rate_tokens = ip_rt_error_burst; 1601 rt->dst.rate_last = now; 1602 if (rt->dst.rate_tokens >= ip_rt_error_cost) { 1603 rt->dst.rate_tokens -= ip_rt_error_cost; 1604 icmp_send(skb, ICMP_DEST_UNREACH, code, 0); 1605 } 1606 1607 out: kfree_skb(skb); 1608 return 0; 1609 } 1610 1611 /* 1612 * The last two values are not from the RFC but 1613 * are needed for AMPRnet AX.25 paths. 1614 */ 1615 1616 static const unsigned short mtu_plateau[] = 1617 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 }; 1618 1619 static inline unsigned short guess_mtu(unsigned short old_mtu) 1620 { 1621 int i; 1622 1623 for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++) 1624 if (old_mtu > mtu_plateau[i]) 1625 return mtu_plateau[i]; 1626 return 68; 1627 } 1628 1629 unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph, 1630 unsigned short new_mtu, 1631 struct net_device *dev) 1632 { 1633 int i, k; 1634 unsigned short old_mtu = ntohs(iph->tot_len); 1635 struct rtable *rth; 1636 int ikeys[2] = { dev->ifindex, 0 }; 1637 __be32 skeys[2] = { iph->saddr, 0, }; 1638 __be32 daddr = iph->daddr; 1639 unsigned short est_mtu = 0; 1640 1641 for (k = 0; k < 2; k++) { 1642 for (i = 0; i < 2; i++) { 1643 unsigned hash = rt_hash(daddr, skeys[i], ikeys[k], 1644 rt_genid(net)); 1645 1646 rcu_read_lock(); 1647 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth; 1648 rth = rcu_dereference(rth->dst.rt_next)) { 1649 unsigned short mtu = new_mtu; 1650 1651 if (rth->fl.fl4_dst != daddr || 1652 rth->fl.fl4_src != skeys[i] || 1653 rth->rt_dst != daddr || 1654 rth->rt_src != iph->saddr || 1655 rth->fl.oif != ikeys[k] || 1656 rt_is_input_route(rth) || 1657 dst_metric_locked(&rth->dst, RTAX_MTU) || 1658 !net_eq(dev_net(rth->dst.dev), net) || 1659 rt_is_expired(rth)) 1660 continue; 1661 1662 if (new_mtu < 68 || new_mtu >= old_mtu) { 1663 1664 /* BSD 4.2 compatibility hack :-( */ 1665 if (mtu == 0 && 1666 old_mtu >= dst_mtu(&rth->dst) && 1667 old_mtu >= 68 + (iph->ihl << 2)) 1668 old_mtu -= iph->ihl << 2; 1669 1670 mtu = guess_mtu(old_mtu); 1671 } 1672 if (mtu <= dst_mtu(&rth->dst)) { 1673 if (mtu < dst_mtu(&rth->dst)) { 1674 dst_confirm(&rth->dst); 1675 if (mtu < ip_rt_min_pmtu) { 1676 u32 lock = dst_metric(&rth->dst, 1677 RTAX_LOCK); 1678 mtu = ip_rt_min_pmtu; 1679 lock |= (1 << RTAX_MTU); 1680 dst_metric_set(&rth->dst, RTAX_LOCK, 1681 lock); 1682 } 1683 dst_metric_set(&rth->dst, RTAX_MTU, mtu); 1684 dst_set_expires(&rth->dst, 1685 ip_rt_mtu_expires); 1686 } 1687 est_mtu = mtu; 1688 } 1689 } 1690 rcu_read_unlock(); 1691 } 1692 } 1693 return est_mtu ? : new_mtu; 1694 } 1695 1696 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu) 1697 { 1698 if (dst_mtu(dst) > mtu && mtu >= 68 && 1699 !(dst_metric_locked(dst, RTAX_MTU))) { 1700 if (mtu < ip_rt_min_pmtu) { 1701 u32 lock = dst_metric(dst, RTAX_LOCK); 1702 mtu = ip_rt_min_pmtu; 1703 dst_metric_set(dst, RTAX_LOCK, lock | (1 << RTAX_MTU)); 1704 } 1705 dst_metric_set(dst, RTAX_MTU, mtu); 1706 dst_set_expires(dst, ip_rt_mtu_expires); 1707 call_netevent_notifiers(NETEVENT_PMTU_UPDATE, dst); 1708 } 1709 } 1710 1711 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie) 1712 { 1713 if (rt_is_expired((struct rtable *)dst)) 1714 return NULL; 1715 return dst; 1716 } 1717 1718 static void ipv4_dst_destroy(struct dst_entry *dst) 1719 { 1720 struct rtable *rt = (struct rtable *) dst; 1721 struct inet_peer *peer = rt->peer; 1722 1723 if (peer) { 1724 rt->peer = NULL; 1725 inet_putpeer(peer); 1726 } 1727 } 1728 1729 1730 static void ipv4_link_failure(struct sk_buff *skb) 1731 { 1732 struct rtable *rt; 1733 1734 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 1735 1736 rt = skb_rtable(skb); 1737 if (rt) 1738 dst_set_expires(&rt->dst, 0); 1739 } 1740 1741 static int ip_rt_bug(struct sk_buff *skb) 1742 { 1743 printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n", 1744 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, 1745 skb->dev ? skb->dev->name : "?"); 1746 kfree_skb(skb); 1747 return 0; 1748 } 1749 1750 /* 1751 We do not cache source address of outgoing interface, 1752 because it is used only by IP RR, TS and SRR options, 1753 so that it out of fast path. 1754 1755 BTW remember: "addr" is allowed to be not aligned 1756 in IP options! 1757 */ 1758 1759 void ip_rt_get_source(u8 *addr, struct rtable *rt) 1760 { 1761 __be32 src; 1762 struct fib_result res; 1763 1764 if (rt_is_output_route(rt)) 1765 src = rt->rt_src; 1766 else { 1767 rcu_read_lock(); 1768 if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0) 1769 src = FIB_RES_PREFSRC(res); 1770 else 1771 src = inet_select_addr(rt->dst.dev, rt->rt_gateway, 1772 RT_SCOPE_UNIVERSE); 1773 rcu_read_unlock(); 1774 } 1775 memcpy(addr, &src, 4); 1776 } 1777 1778 #ifdef CONFIG_NET_CLS_ROUTE 1779 static void set_class_tag(struct rtable *rt, u32 tag) 1780 { 1781 if (!(rt->dst.tclassid & 0xFFFF)) 1782 rt->dst.tclassid |= tag & 0xFFFF; 1783 if (!(rt->dst.tclassid & 0xFFFF0000)) 1784 rt->dst.tclassid |= tag & 0xFFFF0000; 1785 } 1786 #endif 1787 1788 static unsigned int ipv4_default_advmss(const struct dst_entry *dst) 1789 { 1790 unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS); 1791 1792 if (advmss == 0) { 1793 advmss = max_t(unsigned int, dst->dev->mtu - 40, 1794 ip_rt_min_advmss); 1795 if (advmss > 65535 - 40) 1796 advmss = 65535 - 40; 1797 } 1798 return advmss; 1799 } 1800 1801 static unsigned int ipv4_default_mtu(const struct dst_entry *dst) 1802 { 1803 unsigned int mtu = dst->dev->mtu; 1804 1805 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) { 1806 const struct rtable *rt = (const struct rtable *) dst; 1807 1808 if (rt->rt_gateway != rt->rt_dst && mtu > 576) 1809 mtu = 576; 1810 } 1811 1812 if (mtu > IP_MAX_MTU) 1813 mtu = IP_MAX_MTU; 1814 1815 return mtu; 1816 } 1817 1818 static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag) 1819 { 1820 struct dst_entry *dst = &rt->dst; 1821 struct fib_info *fi = res->fi; 1822 1823 if (fi) { 1824 if (FIB_RES_GW(*res) && 1825 FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK) 1826 rt->rt_gateway = FIB_RES_GW(*res); 1827 dst_import_metrics(dst, fi->fib_metrics); 1828 #ifdef CONFIG_NET_CLS_ROUTE 1829 dst->tclassid = FIB_RES_NH(*res).nh_tclassid; 1830 #endif 1831 } 1832 1833 if (dst_mtu(dst) > IP_MAX_MTU) 1834 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU); 1835 if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40) 1836 dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40); 1837 1838 #ifdef CONFIG_NET_CLS_ROUTE 1839 #ifdef CONFIG_IP_MULTIPLE_TABLES 1840 set_class_tag(rt, fib_rules_tclass(res)); 1841 #endif 1842 set_class_tag(rt, itag); 1843 #endif 1844 rt->rt_type = res->type; 1845 } 1846 1847 /* called in rcu_read_lock() section */ 1848 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1849 u8 tos, struct net_device *dev, int our) 1850 { 1851 unsigned int hash; 1852 struct rtable *rth; 1853 __be32 spec_dst; 1854 struct in_device *in_dev = __in_dev_get_rcu(dev); 1855 u32 itag = 0; 1856 int err; 1857 1858 /* Primary sanity checks. */ 1859 1860 if (in_dev == NULL) 1861 return -EINVAL; 1862 1863 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1864 ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP)) 1865 goto e_inval; 1866 1867 if (ipv4_is_zeronet(saddr)) { 1868 if (!ipv4_is_local_multicast(daddr)) 1869 goto e_inval; 1870 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK); 1871 } else { 1872 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst, 1873 &itag, 0); 1874 if (err < 0) 1875 goto e_err; 1876 } 1877 rth = dst_alloc(&ipv4_dst_ops); 1878 if (!rth) 1879 goto e_nobufs; 1880 1881 rth->dst.output = ip_rt_bug; 1882 rth->dst.obsolete = -1; 1883 1884 atomic_set(&rth->dst.__refcnt, 1); 1885 rth->dst.flags= DST_HOST; 1886 if (IN_DEV_CONF_GET(in_dev, NOPOLICY)) 1887 rth->dst.flags |= DST_NOPOLICY; 1888 rth->fl.fl4_dst = daddr; 1889 rth->rt_dst = daddr; 1890 rth->fl.fl4_tos = tos; 1891 rth->fl.mark = skb->mark; 1892 rth->fl.fl4_src = saddr; 1893 rth->rt_src = saddr; 1894 #ifdef CONFIG_NET_CLS_ROUTE 1895 rth->dst.tclassid = itag; 1896 #endif 1897 rth->rt_iif = 1898 rth->fl.iif = dev->ifindex; 1899 rth->dst.dev = init_net.loopback_dev; 1900 dev_hold(rth->dst.dev); 1901 rth->fl.oif = 0; 1902 rth->rt_gateway = daddr; 1903 rth->rt_spec_dst= spec_dst; 1904 rth->rt_genid = rt_genid(dev_net(dev)); 1905 rth->rt_flags = RTCF_MULTICAST; 1906 rth->rt_type = RTN_MULTICAST; 1907 if (our) { 1908 rth->dst.input= ip_local_deliver; 1909 rth->rt_flags |= RTCF_LOCAL; 1910 } 1911 1912 #ifdef CONFIG_IP_MROUTE 1913 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev)) 1914 rth->dst.input = ip_mr_input; 1915 #endif 1916 RT_CACHE_STAT_INC(in_slow_mc); 1917 1918 hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev))); 1919 return rt_intern_hash(hash, rth, NULL, skb, dev->ifindex); 1920 1921 e_nobufs: 1922 return -ENOBUFS; 1923 e_inval: 1924 return -EINVAL; 1925 e_err: 1926 return err; 1927 } 1928 1929 1930 static void ip_handle_martian_source(struct net_device *dev, 1931 struct in_device *in_dev, 1932 struct sk_buff *skb, 1933 __be32 daddr, 1934 __be32 saddr) 1935 { 1936 RT_CACHE_STAT_INC(in_martian_src); 1937 #ifdef CONFIG_IP_ROUTE_VERBOSE 1938 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) { 1939 /* 1940 * RFC1812 recommendation, if source is martian, 1941 * the only hint is MAC header. 1942 */ 1943 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n", 1944 &daddr, &saddr, dev->name); 1945 if (dev->hard_header_len && skb_mac_header_was_set(skb)) { 1946 int i; 1947 const unsigned char *p = skb_mac_header(skb); 1948 printk(KERN_WARNING "ll header: "); 1949 for (i = 0; i < dev->hard_header_len; i++, p++) { 1950 printk("%02x", *p); 1951 if (i < (dev->hard_header_len - 1)) 1952 printk(":"); 1953 } 1954 printk("\n"); 1955 } 1956 } 1957 #endif 1958 } 1959 1960 /* called in rcu_read_lock() section */ 1961 static int __mkroute_input(struct sk_buff *skb, 1962 struct fib_result *res, 1963 struct in_device *in_dev, 1964 __be32 daddr, __be32 saddr, u32 tos, 1965 struct rtable **result) 1966 { 1967 struct rtable *rth; 1968 int err; 1969 struct in_device *out_dev; 1970 unsigned int flags = 0; 1971 __be32 spec_dst; 1972 u32 itag; 1973 1974 /* get a working reference to the output device */ 1975 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res)); 1976 if (out_dev == NULL) { 1977 if (net_ratelimit()) 1978 printk(KERN_CRIT "Bug in ip_route_input" \ 1979 "_slow(). Please, report\n"); 1980 return -EINVAL; 1981 } 1982 1983 1984 err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res), 1985 in_dev->dev, &spec_dst, &itag, skb->mark); 1986 if (err < 0) { 1987 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr, 1988 saddr); 1989 1990 goto cleanup; 1991 } 1992 1993 if (err) 1994 flags |= RTCF_DIRECTSRC; 1995 1996 if (out_dev == in_dev && err && 1997 (IN_DEV_SHARED_MEDIA(out_dev) || 1998 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) 1999 flags |= RTCF_DOREDIRECT; 2000 2001 if (skb->protocol != htons(ETH_P_IP)) { 2002 /* Not IP (i.e. ARP). Do not create route, if it is 2003 * invalid for proxy arp. DNAT routes are always valid. 2004 * 2005 * Proxy arp feature have been extended to allow, ARP 2006 * replies back to the same interface, to support 2007 * Private VLAN switch technologies. See arp.c. 2008 */ 2009 if (out_dev == in_dev && 2010 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) { 2011 err = -EINVAL; 2012 goto cleanup; 2013 } 2014 } 2015 2016 2017 rth = dst_alloc(&ipv4_dst_ops); 2018 if (!rth) { 2019 err = -ENOBUFS; 2020 goto cleanup; 2021 } 2022 2023 atomic_set(&rth->dst.__refcnt, 1); 2024 rth->dst.flags= DST_HOST; 2025 if (IN_DEV_CONF_GET(in_dev, NOPOLICY)) 2026 rth->dst.flags |= DST_NOPOLICY; 2027 if (IN_DEV_CONF_GET(out_dev, NOXFRM)) 2028 rth->dst.flags |= DST_NOXFRM; 2029 rth->fl.fl4_dst = daddr; 2030 rth->rt_dst = daddr; 2031 rth->fl.fl4_tos = tos; 2032 rth->fl.mark = skb->mark; 2033 rth->fl.fl4_src = saddr; 2034 rth->rt_src = saddr; 2035 rth->rt_gateway = daddr; 2036 rth->rt_iif = 2037 rth->fl.iif = in_dev->dev->ifindex; 2038 rth->dst.dev = (out_dev)->dev; 2039 dev_hold(rth->dst.dev); 2040 rth->fl.oif = 0; 2041 rth->rt_spec_dst= spec_dst; 2042 2043 rth->dst.obsolete = -1; 2044 rth->dst.input = ip_forward; 2045 rth->dst.output = ip_output; 2046 rth->rt_genid = rt_genid(dev_net(rth->dst.dev)); 2047 2048 rt_set_nexthop(rth, res, itag); 2049 2050 rth->rt_flags = flags; 2051 2052 *result = rth; 2053 err = 0; 2054 cleanup: 2055 return err; 2056 } 2057 2058 static int ip_mkroute_input(struct sk_buff *skb, 2059 struct fib_result *res, 2060 const struct flowi *fl, 2061 struct in_device *in_dev, 2062 __be32 daddr, __be32 saddr, u32 tos) 2063 { 2064 struct rtable* rth = NULL; 2065 int err; 2066 unsigned hash; 2067 2068 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2069 if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0) 2070 fib_select_multipath(fl, res); 2071 #endif 2072 2073 /* create a routing cache entry */ 2074 err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth); 2075 if (err) 2076 return err; 2077 2078 /* put it into the cache */ 2079 hash = rt_hash(daddr, saddr, fl->iif, 2080 rt_genid(dev_net(rth->dst.dev))); 2081 return rt_intern_hash(hash, rth, NULL, skb, fl->iif); 2082 } 2083 2084 /* 2085 * NOTE. We drop all the packets that has local source 2086 * addresses, because every properly looped back packet 2087 * must have correct destination already attached by output routine. 2088 * 2089 * Such approach solves two big problems: 2090 * 1. Not simplex devices are handled properly. 2091 * 2. IP spoofing attempts are filtered with 100% of guarantee. 2092 * called with rcu_read_lock() 2093 */ 2094 2095 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2096 u8 tos, struct net_device *dev) 2097 { 2098 struct fib_result res; 2099 struct in_device *in_dev = __in_dev_get_rcu(dev); 2100 struct flowi fl = { .fl4_dst = daddr, 2101 .fl4_src = saddr, 2102 .fl4_tos = tos, 2103 .fl4_scope = RT_SCOPE_UNIVERSE, 2104 .mark = skb->mark, 2105 .iif = dev->ifindex }; 2106 unsigned flags = 0; 2107 u32 itag = 0; 2108 struct rtable * rth; 2109 unsigned hash; 2110 __be32 spec_dst; 2111 int err = -EINVAL; 2112 struct net * net = dev_net(dev); 2113 2114 /* IP on this device is disabled. */ 2115 2116 if (!in_dev) 2117 goto out; 2118 2119 /* Check for the most weird martians, which can be not detected 2120 by fib_lookup. 2121 */ 2122 2123 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 2124 ipv4_is_loopback(saddr)) 2125 goto martian_source; 2126 2127 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) 2128 goto brd_input; 2129 2130 /* Accept zero addresses only to limited broadcast; 2131 * I even do not know to fix it or not. Waiting for complains :-) 2132 */ 2133 if (ipv4_is_zeronet(saddr)) 2134 goto martian_source; 2135 2136 if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr)) 2137 goto martian_destination; 2138 2139 /* 2140 * Now we are ready to route packet. 2141 */ 2142 err = fib_lookup(net, &fl, &res); 2143 if (err != 0) { 2144 if (!IN_DEV_FORWARD(in_dev)) 2145 goto e_hostunreach; 2146 goto no_route; 2147 } 2148 2149 RT_CACHE_STAT_INC(in_slow_tot); 2150 2151 if (res.type == RTN_BROADCAST) 2152 goto brd_input; 2153 2154 if (res.type == RTN_LOCAL) { 2155 err = fib_validate_source(saddr, daddr, tos, 2156 net->loopback_dev->ifindex, 2157 dev, &spec_dst, &itag, skb->mark); 2158 if (err < 0) 2159 goto martian_source_keep_err; 2160 if (err) 2161 flags |= RTCF_DIRECTSRC; 2162 spec_dst = daddr; 2163 goto local_input; 2164 } 2165 2166 if (!IN_DEV_FORWARD(in_dev)) 2167 goto e_hostunreach; 2168 if (res.type != RTN_UNICAST) 2169 goto martian_destination; 2170 2171 err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos); 2172 out: return err; 2173 2174 brd_input: 2175 if (skb->protocol != htons(ETH_P_IP)) 2176 goto e_inval; 2177 2178 if (ipv4_is_zeronet(saddr)) 2179 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK); 2180 else { 2181 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst, 2182 &itag, skb->mark); 2183 if (err < 0) 2184 goto martian_source_keep_err; 2185 if (err) 2186 flags |= RTCF_DIRECTSRC; 2187 } 2188 flags |= RTCF_BROADCAST; 2189 res.type = RTN_BROADCAST; 2190 RT_CACHE_STAT_INC(in_brd); 2191 2192 local_input: 2193 rth = dst_alloc(&ipv4_dst_ops); 2194 if (!rth) 2195 goto e_nobufs; 2196 2197 rth->dst.output= ip_rt_bug; 2198 rth->dst.obsolete = -1; 2199 rth->rt_genid = rt_genid(net); 2200 2201 atomic_set(&rth->dst.__refcnt, 1); 2202 rth->dst.flags= DST_HOST; 2203 if (IN_DEV_CONF_GET(in_dev, NOPOLICY)) 2204 rth->dst.flags |= DST_NOPOLICY; 2205 rth->fl.fl4_dst = daddr; 2206 rth->rt_dst = daddr; 2207 rth->fl.fl4_tos = tos; 2208 rth->fl.mark = skb->mark; 2209 rth->fl.fl4_src = saddr; 2210 rth->rt_src = saddr; 2211 #ifdef CONFIG_NET_CLS_ROUTE 2212 rth->dst.tclassid = itag; 2213 #endif 2214 rth->rt_iif = 2215 rth->fl.iif = dev->ifindex; 2216 rth->dst.dev = net->loopback_dev; 2217 dev_hold(rth->dst.dev); 2218 rth->rt_gateway = daddr; 2219 rth->rt_spec_dst= spec_dst; 2220 rth->dst.input= ip_local_deliver; 2221 rth->rt_flags = flags|RTCF_LOCAL; 2222 if (res.type == RTN_UNREACHABLE) { 2223 rth->dst.input= ip_error; 2224 rth->dst.error= -err; 2225 rth->rt_flags &= ~RTCF_LOCAL; 2226 } 2227 rth->rt_type = res.type; 2228 hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net)); 2229 err = rt_intern_hash(hash, rth, NULL, skb, fl.iif); 2230 goto out; 2231 2232 no_route: 2233 RT_CACHE_STAT_INC(in_no_route); 2234 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE); 2235 res.type = RTN_UNREACHABLE; 2236 if (err == -ESRCH) 2237 err = -ENETUNREACH; 2238 goto local_input; 2239 2240 /* 2241 * Do not cache martian addresses: they should be logged (RFC1812) 2242 */ 2243 martian_destination: 2244 RT_CACHE_STAT_INC(in_martian_dst); 2245 #ifdef CONFIG_IP_ROUTE_VERBOSE 2246 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) 2247 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n", 2248 &daddr, &saddr, dev->name); 2249 #endif 2250 2251 e_hostunreach: 2252 err = -EHOSTUNREACH; 2253 goto out; 2254 2255 e_inval: 2256 err = -EINVAL; 2257 goto out; 2258 2259 e_nobufs: 2260 err = -ENOBUFS; 2261 goto out; 2262 2263 martian_source: 2264 err = -EINVAL; 2265 martian_source_keep_err: 2266 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2267 goto out; 2268 } 2269 2270 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2271 u8 tos, struct net_device *dev, bool noref) 2272 { 2273 struct rtable * rth; 2274 unsigned hash; 2275 int iif = dev->ifindex; 2276 struct net *net; 2277 int res; 2278 2279 net = dev_net(dev); 2280 2281 rcu_read_lock(); 2282 2283 if (!rt_caching(net)) 2284 goto skip_cache; 2285 2286 tos &= IPTOS_RT_MASK; 2287 hash = rt_hash(daddr, saddr, iif, rt_genid(net)); 2288 2289 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth; 2290 rth = rcu_dereference(rth->dst.rt_next)) { 2291 if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) | 2292 ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) | 2293 (rth->fl.iif ^ iif) | 2294 rth->fl.oif | 2295 (rth->fl.fl4_tos ^ tos)) == 0 && 2296 rth->fl.mark == skb->mark && 2297 net_eq(dev_net(rth->dst.dev), net) && 2298 !rt_is_expired(rth)) { 2299 if (noref) { 2300 dst_use_noref(&rth->dst, jiffies); 2301 skb_dst_set_noref(skb, &rth->dst); 2302 } else { 2303 dst_use(&rth->dst, jiffies); 2304 skb_dst_set(skb, &rth->dst); 2305 } 2306 RT_CACHE_STAT_INC(in_hit); 2307 rcu_read_unlock(); 2308 return 0; 2309 } 2310 RT_CACHE_STAT_INC(in_hlist_search); 2311 } 2312 2313 skip_cache: 2314 /* Multicast recognition logic is moved from route cache to here. 2315 The problem was that too many Ethernet cards have broken/missing 2316 hardware multicast filters :-( As result the host on multicasting 2317 network acquires a lot of useless route cache entries, sort of 2318 SDR messages from all the world. Now we try to get rid of them. 2319 Really, provided software IP multicast filter is organized 2320 reasonably (at least, hashed), it does not result in a slowdown 2321 comparing with route cache reject entries. 2322 Note, that multicast routers are not affected, because 2323 route cache entry is created eventually. 2324 */ 2325 if (ipv4_is_multicast(daddr)) { 2326 struct in_device *in_dev = __in_dev_get_rcu(dev); 2327 2328 if (in_dev) { 2329 int our = ip_check_mc(in_dev, daddr, saddr, 2330 ip_hdr(skb)->protocol); 2331 if (our 2332 #ifdef CONFIG_IP_MROUTE 2333 || 2334 (!ipv4_is_local_multicast(daddr) && 2335 IN_DEV_MFORWARD(in_dev)) 2336 #endif 2337 ) { 2338 int res = ip_route_input_mc(skb, daddr, saddr, 2339 tos, dev, our); 2340 rcu_read_unlock(); 2341 return res; 2342 } 2343 } 2344 rcu_read_unlock(); 2345 return -EINVAL; 2346 } 2347 res = ip_route_input_slow(skb, daddr, saddr, tos, dev); 2348 rcu_read_unlock(); 2349 return res; 2350 } 2351 EXPORT_SYMBOL(ip_route_input_common); 2352 2353 /* called with rcu_read_lock() */ 2354 static int __mkroute_output(struct rtable **result, 2355 struct fib_result *res, 2356 const struct flowi *fl, 2357 const struct flowi *oldflp, 2358 struct net_device *dev_out, 2359 unsigned flags) 2360 { 2361 struct rtable *rth; 2362 struct in_device *in_dev; 2363 u32 tos = RT_FL_TOS(oldflp); 2364 2365 if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK)) 2366 return -EINVAL; 2367 2368 if (ipv4_is_lbcast(fl->fl4_dst)) 2369 res->type = RTN_BROADCAST; 2370 else if (ipv4_is_multicast(fl->fl4_dst)) 2371 res->type = RTN_MULTICAST; 2372 else if (ipv4_is_zeronet(fl->fl4_dst)) 2373 return -EINVAL; 2374 2375 if (dev_out->flags & IFF_LOOPBACK) 2376 flags |= RTCF_LOCAL; 2377 2378 in_dev = __in_dev_get_rcu(dev_out); 2379 if (!in_dev) 2380 return -EINVAL; 2381 2382 if (res->type == RTN_BROADCAST) { 2383 flags |= RTCF_BROADCAST | RTCF_LOCAL; 2384 res->fi = NULL; 2385 } else if (res->type == RTN_MULTICAST) { 2386 flags |= RTCF_MULTICAST | RTCF_LOCAL; 2387 if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src, 2388 oldflp->proto)) 2389 flags &= ~RTCF_LOCAL; 2390 /* If multicast route do not exist use 2391 * default one, but do not gateway in this case. 2392 * Yes, it is hack. 2393 */ 2394 if (res->fi && res->prefixlen < 4) 2395 res->fi = NULL; 2396 } 2397 2398 2399 rth = dst_alloc(&ipv4_dst_ops); 2400 if (!rth) 2401 return -ENOBUFS; 2402 2403 atomic_set(&rth->dst.__refcnt, 1); 2404 rth->dst.flags= DST_HOST; 2405 if (IN_DEV_CONF_GET(in_dev, NOXFRM)) 2406 rth->dst.flags |= DST_NOXFRM; 2407 if (IN_DEV_CONF_GET(in_dev, NOPOLICY)) 2408 rth->dst.flags |= DST_NOPOLICY; 2409 2410 rth->fl.fl4_dst = oldflp->fl4_dst; 2411 rth->fl.fl4_tos = tos; 2412 rth->fl.fl4_src = oldflp->fl4_src; 2413 rth->fl.oif = oldflp->oif; 2414 rth->fl.mark = oldflp->mark; 2415 rth->rt_dst = fl->fl4_dst; 2416 rth->rt_src = fl->fl4_src; 2417 rth->rt_iif = oldflp->oif ? : dev_out->ifindex; 2418 /* get references to the devices that are to be hold by the routing 2419 cache entry */ 2420 rth->dst.dev = dev_out; 2421 dev_hold(dev_out); 2422 rth->rt_gateway = fl->fl4_dst; 2423 rth->rt_spec_dst= fl->fl4_src; 2424 2425 rth->dst.output=ip_output; 2426 rth->dst.obsolete = -1; 2427 rth->rt_genid = rt_genid(dev_net(dev_out)); 2428 2429 RT_CACHE_STAT_INC(out_slow_tot); 2430 2431 if (flags & RTCF_LOCAL) { 2432 rth->dst.input = ip_local_deliver; 2433 rth->rt_spec_dst = fl->fl4_dst; 2434 } 2435 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) { 2436 rth->rt_spec_dst = fl->fl4_src; 2437 if (flags & RTCF_LOCAL && 2438 !(dev_out->flags & IFF_LOOPBACK)) { 2439 rth->dst.output = ip_mc_output; 2440 RT_CACHE_STAT_INC(out_slow_mc); 2441 } 2442 #ifdef CONFIG_IP_MROUTE 2443 if (res->type == RTN_MULTICAST) { 2444 if (IN_DEV_MFORWARD(in_dev) && 2445 !ipv4_is_local_multicast(oldflp->fl4_dst)) { 2446 rth->dst.input = ip_mr_input; 2447 rth->dst.output = ip_mc_output; 2448 } 2449 } 2450 #endif 2451 } 2452 2453 rt_set_nexthop(rth, res, 0); 2454 2455 rth->rt_flags = flags; 2456 *result = rth; 2457 return 0; 2458 } 2459 2460 /* called with rcu_read_lock() */ 2461 static int ip_mkroute_output(struct rtable **rp, 2462 struct fib_result *res, 2463 const struct flowi *fl, 2464 const struct flowi *oldflp, 2465 struct net_device *dev_out, 2466 unsigned flags) 2467 { 2468 struct rtable *rth = NULL; 2469 int err = __mkroute_output(&rth, res, fl, oldflp, dev_out, flags); 2470 unsigned hash; 2471 if (err == 0) { 2472 hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif, 2473 rt_genid(dev_net(dev_out))); 2474 err = rt_intern_hash(hash, rth, rp, NULL, oldflp->oif); 2475 } 2476 2477 return err; 2478 } 2479 2480 /* 2481 * Major route resolver routine. 2482 * called with rcu_read_lock(); 2483 */ 2484 2485 static int ip_route_output_slow(struct net *net, struct rtable **rp, 2486 const struct flowi *oldflp) 2487 { 2488 u32 tos = RT_FL_TOS(oldflp); 2489 struct flowi fl = { .fl4_dst = oldflp->fl4_dst, 2490 .fl4_src = oldflp->fl4_src, 2491 .fl4_tos = tos & IPTOS_RT_MASK, 2492 .fl4_scope = ((tos & RTO_ONLINK) ? 2493 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE), 2494 .mark = oldflp->mark, 2495 .iif = net->loopback_dev->ifindex, 2496 .oif = oldflp->oif }; 2497 struct fib_result res; 2498 unsigned int flags = 0; 2499 struct net_device *dev_out = NULL; 2500 int err; 2501 2502 2503 res.fi = NULL; 2504 #ifdef CONFIG_IP_MULTIPLE_TABLES 2505 res.r = NULL; 2506 #endif 2507 2508 if (oldflp->fl4_src) { 2509 err = -EINVAL; 2510 if (ipv4_is_multicast(oldflp->fl4_src) || 2511 ipv4_is_lbcast(oldflp->fl4_src) || 2512 ipv4_is_zeronet(oldflp->fl4_src)) 2513 goto out; 2514 2515 /* I removed check for oif == dev_out->oif here. 2516 It was wrong for two reasons: 2517 1. ip_dev_find(net, saddr) can return wrong iface, if saddr 2518 is assigned to multiple interfaces. 2519 2. Moreover, we are allowed to send packets with saddr 2520 of another iface. --ANK 2521 */ 2522 2523 if (oldflp->oif == 0 && 2524 (ipv4_is_multicast(oldflp->fl4_dst) || 2525 ipv4_is_lbcast(oldflp->fl4_dst))) { 2526 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2527 dev_out = __ip_dev_find(net, oldflp->fl4_src, false); 2528 if (dev_out == NULL) 2529 goto out; 2530 2531 /* Special hack: user can direct multicasts 2532 and limited broadcast via necessary interface 2533 without fiddling with IP_MULTICAST_IF or IP_PKTINFO. 2534 This hack is not just for fun, it allows 2535 vic,vat and friends to work. 2536 They bind socket to loopback, set ttl to zero 2537 and expect that it will work. 2538 From the viewpoint of routing cache they are broken, 2539 because we are not allowed to build multicast path 2540 with loopback source addr (look, routing cache 2541 cannot know, that ttl is zero, so that packet 2542 will not leave this host and route is valid). 2543 Luckily, this hack is good workaround. 2544 */ 2545 2546 fl.oif = dev_out->ifindex; 2547 goto make_route; 2548 } 2549 2550 if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) { 2551 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2552 if (!__ip_dev_find(net, oldflp->fl4_src, false)) 2553 goto out; 2554 } 2555 } 2556 2557 2558 if (oldflp->oif) { 2559 dev_out = dev_get_by_index_rcu(net, oldflp->oif); 2560 err = -ENODEV; 2561 if (dev_out == NULL) 2562 goto out; 2563 2564 /* RACE: Check return value of inet_select_addr instead. */ 2565 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) { 2566 err = -ENETUNREACH; 2567 goto out; 2568 } 2569 if (ipv4_is_local_multicast(oldflp->fl4_dst) || 2570 ipv4_is_lbcast(oldflp->fl4_dst)) { 2571 if (!fl.fl4_src) 2572 fl.fl4_src = inet_select_addr(dev_out, 0, 2573 RT_SCOPE_LINK); 2574 goto make_route; 2575 } 2576 if (!fl.fl4_src) { 2577 if (ipv4_is_multicast(oldflp->fl4_dst)) 2578 fl.fl4_src = inet_select_addr(dev_out, 0, 2579 fl.fl4_scope); 2580 else if (!oldflp->fl4_dst) 2581 fl.fl4_src = inet_select_addr(dev_out, 0, 2582 RT_SCOPE_HOST); 2583 } 2584 } 2585 2586 if (!fl.fl4_dst) { 2587 fl.fl4_dst = fl.fl4_src; 2588 if (!fl.fl4_dst) 2589 fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK); 2590 dev_out = net->loopback_dev; 2591 fl.oif = net->loopback_dev->ifindex; 2592 res.type = RTN_LOCAL; 2593 flags |= RTCF_LOCAL; 2594 goto make_route; 2595 } 2596 2597 if (fib_lookup(net, &fl, &res)) { 2598 res.fi = NULL; 2599 if (oldflp->oif) { 2600 /* Apparently, routing tables are wrong. Assume, 2601 that the destination is on link. 2602 2603 WHY? DW. 2604 Because we are allowed to send to iface 2605 even if it has NO routes and NO assigned 2606 addresses. When oif is specified, routing 2607 tables are looked up with only one purpose: 2608 to catch if destination is gatewayed, rather than 2609 direct. Moreover, if MSG_DONTROUTE is set, 2610 we send packet, ignoring both routing tables 2611 and ifaddr state. --ANK 2612 2613 2614 We could make it even if oif is unknown, 2615 likely IPv6, but we do not. 2616 */ 2617 2618 if (fl.fl4_src == 0) 2619 fl.fl4_src = inet_select_addr(dev_out, 0, 2620 RT_SCOPE_LINK); 2621 res.type = RTN_UNICAST; 2622 goto make_route; 2623 } 2624 err = -ENETUNREACH; 2625 goto out; 2626 } 2627 2628 if (res.type == RTN_LOCAL) { 2629 if (!fl.fl4_src) { 2630 if (res.fi->fib_prefsrc) 2631 fl.fl4_src = res.fi->fib_prefsrc; 2632 else 2633 fl.fl4_src = fl.fl4_dst; 2634 } 2635 dev_out = net->loopback_dev; 2636 fl.oif = dev_out->ifindex; 2637 res.fi = NULL; 2638 flags |= RTCF_LOCAL; 2639 goto make_route; 2640 } 2641 2642 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2643 if (res.fi->fib_nhs > 1 && fl.oif == 0) 2644 fib_select_multipath(&fl, &res); 2645 else 2646 #endif 2647 if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif) 2648 fib_select_default(net, &fl, &res); 2649 2650 if (!fl.fl4_src) 2651 fl.fl4_src = FIB_RES_PREFSRC(res); 2652 2653 dev_out = FIB_RES_DEV(res); 2654 fl.oif = dev_out->ifindex; 2655 2656 2657 make_route: 2658 err = ip_mkroute_output(rp, &res, &fl, oldflp, dev_out, flags); 2659 2660 out: return err; 2661 } 2662 2663 int __ip_route_output_key(struct net *net, struct rtable **rp, 2664 const struct flowi *flp) 2665 { 2666 unsigned int hash; 2667 int res; 2668 struct rtable *rth; 2669 2670 if (!rt_caching(net)) 2671 goto slow_output; 2672 2673 hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net)); 2674 2675 rcu_read_lock_bh(); 2676 for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth; 2677 rth = rcu_dereference_bh(rth->dst.rt_next)) { 2678 if (rth->fl.fl4_dst == flp->fl4_dst && 2679 rth->fl.fl4_src == flp->fl4_src && 2680 rt_is_output_route(rth) && 2681 rth->fl.oif == flp->oif && 2682 rth->fl.mark == flp->mark && 2683 !((rth->fl.fl4_tos ^ flp->fl4_tos) & 2684 (IPTOS_RT_MASK | RTO_ONLINK)) && 2685 net_eq(dev_net(rth->dst.dev), net) && 2686 !rt_is_expired(rth)) { 2687 dst_use(&rth->dst, jiffies); 2688 RT_CACHE_STAT_INC(out_hit); 2689 rcu_read_unlock_bh(); 2690 *rp = rth; 2691 return 0; 2692 } 2693 RT_CACHE_STAT_INC(out_hlist_search); 2694 } 2695 rcu_read_unlock_bh(); 2696 2697 slow_output: 2698 rcu_read_lock(); 2699 res = ip_route_output_slow(net, rp, flp); 2700 rcu_read_unlock(); 2701 return res; 2702 } 2703 EXPORT_SYMBOL_GPL(__ip_route_output_key); 2704 2705 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie) 2706 { 2707 return NULL; 2708 } 2709 2710 static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst) 2711 { 2712 return 0; 2713 } 2714 2715 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu) 2716 { 2717 } 2718 2719 static struct dst_ops ipv4_dst_blackhole_ops = { 2720 .family = AF_INET, 2721 .protocol = cpu_to_be16(ETH_P_IP), 2722 .destroy = ipv4_dst_destroy, 2723 .check = ipv4_blackhole_dst_check, 2724 .default_mtu = ipv4_blackhole_default_mtu, 2725 .update_pmtu = ipv4_rt_blackhole_update_pmtu, 2726 }; 2727 2728 2729 static int ipv4_dst_blackhole(struct net *net, struct rtable **rp, struct flowi *flp) 2730 { 2731 struct rtable *ort = *rp; 2732 struct rtable *rt = (struct rtable *) 2733 dst_alloc(&ipv4_dst_blackhole_ops); 2734 2735 if (rt) { 2736 struct dst_entry *new = &rt->dst; 2737 2738 atomic_set(&new->__refcnt, 1); 2739 new->__use = 1; 2740 new->input = dst_discard; 2741 new->output = dst_discard; 2742 dst_copy_metrics(new, &ort->dst); 2743 2744 new->dev = ort->dst.dev; 2745 if (new->dev) 2746 dev_hold(new->dev); 2747 2748 rt->fl = ort->fl; 2749 2750 rt->rt_genid = rt_genid(net); 2751 rt->rt_flags = ort->rt_flags; 2752 rt->rt_type = ort->rt_type; 2753 rt->rt_dst = ort->rt_dst; 2754 rt->rt_src = ort->rt_src; 2755 rt->rt_iif = ort->rt_iif; 2756 rt->rt_gateway = ort->rt_gateway; 2757 rt->rt_spec_dst = ort->rt_spec_dst; 2758 rt->peer = ort->peer; 2759 if (rt->peer) 2760 atomic_inc(&rt->peer->refcnt); 2761 2762 dst_free(new); 2763 } 2764 2765 dst_release(&(*rp)->dst); 2766 *rp = rt; 2767 return rt ? 0 : -ENOMEM; 2768 } 2769 2770 int ip_route_output_flow(struct net *net, struct rtable **rp, struct flowi *flp, 2771 struct sock *sk, int flags) 2772 { 2773 int err; 2774 2775 if ((err = __ip_route_output_key(net, rp, flp)) != 0) 2776 return err; 2777 2778 if (flp->proto) { 2779 if (!flp->fl4_src) 2780 flp->fl4_src = (*rp)->rt_src; 2781 if (!flp->fl4_dst) 2782 flp->fl4_dst = (*rp)->rt_dst; 2783 err = __xfrm_lookup(net, (struct dst_entry **)rp, flp, sk, 2784 flags ? XFRM_LOOKUP_WAIT : 0); 2785 if (err == -EREMOTE) 2786 err = ipv4_dst_blackhole(net, rp, flp); 2787 2788 return err; 2789 } 2790 2791 return 0; 2792 } 2793 EXPORT_SYMBOL_GPL(ip_route_output_flow); 2794 2795 int ip_route_output_key(struct net *net, struct rtable **rp, struct flowi *flp) 2796 { 2797 return ip_route_output_flow(net, rp, flp, NULL, 0); 2798 } 2799 EXPORT_SYMBOL(ip_route_output_key); 2800 2801 static int rt_fill_info(struct net *net, 2802 struct sk_buff *skb, u32 pid, u32 seq, int event, 2803 int nowait, unsigned int flags) 2804 { 2805 struct rtable *rt = skb_rtable(skb); 2806 struct rtmsg *r; 2807 struct nlmsghdr *nlh; 2808 long expires; 2809 u32 id = 0, ts = 0, tsage = 0, error; 2810 2811 nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags); 2812 if (nlh == NULL) 2813 return -EMSGSIZE; 2814 2815 r = nlmsg_data(nlh); 2816 r->rtm_family = AF_INET; 2817 r->rtm_dst_len = 32; 2818 r->rtm_src_len = 0; 2819 r->rtm_tos = rt->fl.fl4_tos; 2820 r->rtm_table = RT_TABLE_MAIN; 2821 NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN); 2822 r->rtm_type = rt->rt_type; 2823 r->rtm_scope = RT_SCOPE_UNIVERSE; 2824 r->rtm_protocol = RTPROT_UNSPEC; 2825 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; 2826 if (rt->rt_flags & RTCF_NOTIFY) 2827 r->rtm_flags |= RTM_F_NOTIFY; 2828 2829 NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst); 2830 2831 if (rt->fl.fl4_src) { 2832 r->rtm_src_len = 32; 2833 NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src); 2834 } 2835 if (rt->dst.dev) 2836 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex); 2837 #ifdef CONFIG_NET_CLS_ROUTE 2838 if (rt->dst.tclassid) 2839 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid); 2840 #endif 2841 if (rt_is_input_route(rt)) 2842 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst); 2843 else if (rt->rt_src != rt->fl.fl4_src) 2844 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src); 2845 2846 if (rt->rt_dst != rt->rt_gateway) 2847 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway); 2848 2849 if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0) 2850 goto nla_put_failure; 2851 2852 if (rt->fl.mark) 2853 NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark); 2854 2855 error = rt->dst.error; 2856 expires = rt->dst.expires ? rt->dst.expires - jiffies : 0; 2857 if (rt->peer) { 2858 inet_peer_refcheck(rt->peer); 2859 id = atomic_read(&rt->peer->ip_id_count) & 0xffff; 2860 if (rt->peer->tcp_ts_stamp) { 2861 ts = rt->peer->tcp_ts; 2862 tsage = get_seconds() - rt->peer->tcp_ts_stamp; 2863 } 2864 } 2865 2866 if (rt_is_input_route(rt)) { 2867 #ifdef CONFIG_IP_MROUTE 2868 __be32 dst = rt->rt_dst; 2869 2870 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) && 2871 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { 2872 int err = ipmr_get_route(net, skb, r, nowait); 2873 if (err <= 0) { 2874 if (!nowait) { 2875 if (err == 0) 2876 return 0; 2877 goto nla_put_failure; 2878 } else { 2879 if (err == -EMSGSIZE) 2880 goto nla_put_failure; 2881 error = err; 2882 } 2883 } 2884 } else 2885 #endif 2886 NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif); 2887 } 2888 2889 if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage, 2890 expires, error) < 0) 2891 goto nla_put_failure; 2892 2893 return nlmsg_end(skb, nlh); 2894 2895 nla_put_failure: 2896 nlmsg_cancel(skb, nlh); 2897 return -EMSGSIZE; 2898 } 2899 2900 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg) 2901 { 2902 struct net *net = sock_net(in_skb->sk); 2903 struct rtmsg *rtm; 2904 struct nlattr *tb[RTA_MAX+1]; 2905 struct rtable *rt = NULL; 2906 __be32 dst = 0; 2907 __be32 src = 0; 2908 u32 iif; 2909 int err; 2910 int mark; 2911 struct sk_buff *skb; 2912 2913 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy); 2914 if (err < 0) 2915 goto errout; 2916 2917 rtm = nlmsg_data(nlh); 2918 2919 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2920 if (skb == NULL) { 2921 err = -ENOBUFS; 2922 goto errout; 2923 } 2924 2925 /* Reserve room for dummy headers, this skb can pass 2926 through good chunk of routing engine. 2927 */ 2928 skb_reset_mac_header(skb); 2929 skb_reset_network_header(skb); 2930 2931 /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */ 2932 ip_hdr(skb)->protocol = IPPROTO_ICMP; 2933 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr)); 2934 2935 src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0; 2936 dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0; 2937 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; 2938 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; 2939 2940 if (iif) { 2941 struct net_device *dev; 2942 2943 dev = __dev_get_by_index(net, iif); 2944 if (dev == NULL) { 2945 err = -ENODEV; 2946 goto errout_free; 2947 } 2948 2949 skb->protocol = htons(ETH_P_IP); 2950 skb->dev = dev; 2951 skb->mark = mark; 2952 local_bh_disable(); 2953 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev); 2954 local_bh_enable(); 2955 2956 rt = skb_rtable(skb); 2957 if (err == 0 && rt->dst.error) 2958 err = -rt->dst.error; 2959 } else { 2960 struct flowi fl = { 2961 .fl4_dst = dst, 2962 .fl4_src = src, 2963 .fl4_tos = rtm->rtm_tos, 2964 .oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0, 2965 .mark = mark, 2966 }; 2967 err = ip_route_output_key(net, &rt, &fl); 2968 } 2969 2970 if (err) 2971 goto errout_free; 2972 2973 skb_dst_set(skb, &rt->dst); 2974 if (rtm->rtm_flags & RTM_F_NOTIFY) 2975 rt->rt_flags |= RTCF_NOTIFY; 2976 2977 err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq, 2978 RTM_NEWROUTE, 0, 0); 2979 if (err <= 0) 2980 goto errout_free; 2981 2982 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid); 2983 errout: 2984 return err; 2985 2986 errout_free: 2987 kfree_skb(skb); 2988 goto errout; 2989 } 2990 2991 int ip_rt_dump(struct sk_buff *skb, struct netlink_callback *cb) 2992 { 2993 struct rtable *rt; 2994 int h, s_h; 2995 int idx, s_idx; 2996 struct net *net; 2997 2998 net = sock_net(skb->sk); 2999 3000 s_h = cb->args[0]; 3001 if (s_h < 0) 3002 s_h = 0; 3003 s_idx = idx = cb->args[1]; 3004 for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) { 3005 if (!rt_hash_table[h].chain) 3006 continue; 3007 rcu_read_lock_bh(); 3008 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt; 3009 rt = rcu_dereference_bh(rt->dst.rt_next), idx++) { 3010 if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx) 3011 continue; 3012 if (rt_is_expired(rt)) 3013 continue; 3014 skb_dst_set_noref(skb, &rt->dst); 3015 if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid, 3016 cb->nlh->nlmsg_seq, RTM_NEWROUTE, 3017 1, NLM_F_MULTI) <= 0) { 3018 skb_dst_drop(skb); 3019 rcu_read_unlock_bh(); 3020 goto done; 3021 } 3022 skb_dst_drop(skb); 3023 } 3024 rcu_read_unlock_bh(); 3025 } 3026 3027 done: 3028 cb->args[0] = h; 3029 cb->args[1] = idx; 3030 return skb->len; 3031 } 3032 3033 void ip_rt_multicast_event(struct in_device *in_dev) 3034 { 3035 rt_cache_flush(dev_net(in_dev->dev), 0); 3036 } 3037 3038 #ifdef CONFIG_SYSCTL 3039 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write, 3040 void __user *buffer, 3041 size_t *lenp, loff_t *ppos) 3042 { 3043 if (write) { 3044 int flush_delay; 3045 ctl_table ctl; 3046 struct net *net; 3047 3048 memcpy(&ctl, __ctl, sizeof(ctl)); 3049 ctl.data = &flush_delay; 3050 proc_dointvec(&ctl, write, buffer, lenp, ppos); 3051 3052 net = (struct net *)__ctl->extra1; 3053 rt_cache_flush(net, flush_delay); 3054 return 0; 3055 } 3056 3057 return -EINVAL; 3058 } 3059 3060 static ctl_table ipv4_route_table[] = { 3061 { 3062 .procname = "gc_thresh", 3063 .data = &ipv4_dst_ops.gc_thresh, 3064 .maxlen = sizeof(int), 3065 .mode = 0644, 3066 .proc_handler = proc_dointvec, 3067 }, 3068 { 3069 .procname = "max_size", 3070 .data = &ip_rt_max_size, 3071 .maxlen = sizeof(int), 3072 .mode = 0644, 3073 .proc_handler = proc_dointvec, 3074 }, 3075 { 3076 /* Deprecated. Use gc_min_interval_ms */ 3077 3078 .procname = "gc_min_interval", 3079 .data = &ip_rt_gc_min_interval, 3080 .maxlen = sizeof(int), 3081 .mode = 0644, 3082 .proc_handler = proc_dointvec_jiffies, 3083 }, 3084 { 3085 .procname = "gc_min_interval_ms", 3086 .data = &ip_rt_gc_min_interval, 3087 .maxlen = sizeof(int), 3088 .mode = 0644, 3089 .proc_handler = proc_dointvec_ms_jiffies, 3090 }, 3091 { 3092 .procname = "gc_timeout", 3093 .data = &ip_rt_gc_timeout, 3094 .maxlen = sizeof(int), 3095 .mode = 0644, 3096 .proc_handler = proc_dointvec_jiffies, 3097 }, 3098 { 3099 .procname = "gc_interval", 3100 .data = &ip_rt_gc_interval, 3101 .maxlen = sizeof(int), 3102 .mode = 0644, 3103 .proc_handler = proc_dointvec_jiffies, 3104 }, 3105 { 3106 .procname = "redirect_load", 3107 .data = &ip_rt_redirect_load, 3108 .maxlen = sizeof(int), 3109 .mode = 0644, 3110 .proc_handler = proc_dointvec, 3111 }, 3112 { 3113 .procname = "redirect_number", 3114 .data = &ip_rt_redirect_number, 3115 .maxlen = sizeof(int), 3116 .mode = 0644, 3117 .proc_handler = proc_dointvec, 3118 }, 3119 { 3120 .procname = "redirect_silence", 3121 .data = &ip_rt_redirect_silence, 3122 .maxlen = sizeof(int), 3123 .mode = 0644, 3124 .proc_handler = proc_dointvec, 3125 }, 3126 { 3127 .procname = "error_cost", 3128 .data = &ip_rt_error_cost, 3129 .maxlen = sizeof(int), 3130 .mode = 0644, 3131 .proc_handler = proc_dointvec, 3132 }, 3133 { 3134 .procname = "error_burst", 3135 .data = &ip_rt_error_burst, 3136 .maxlen = sizeof(int), 3137 .mode = 0644, 3138 .proc_handler = proc_dointvec, 3139 }, 3140 { 3141 .procname = "gc_elasticity", 3142 .data = &ip_rt_gc_elasticity, 3143 .maxlen = sizeof(int), 3144 .mode = 0644, 3145 .proc_handler = proc_dointvec, 3146 }, 3147 { 3148 .procname = "mtu_expires", 3149 .data = &ip_rt_mtu_expires, 3150 .maxlen = sizeof(int), 3151 .mode = 0644, 3152 .proc_handler = proc_dointvec_jiffies, 3153 }, 3154 { 3155 .procname = "min_pmtu", 3156 .data = &ip_rt_min_pmtu, 3157 .maxlen = sizeof(int), 3158 .mode = 0644, 3159 .proc_handler = proc_dointvec, 3160 }, 3161 { 3162 .procname = "min_adv_mss", 3163 .data = &ip_rt_min_advmss, 3164 .maxlen = sizeof(int), 3165 .mode = 0644, 3166 .proc_handler = proc_dointvec, 3167 }, 3168 { } 3169 }; 3170 3171 static struct ctl_table empty[1]; 3172 3173 static struct ctl_table ipv4_skeleton[] = 3174 { 3175 { .procname = "route", 3176 .mode = 0555, .child = ipv4_route_table}, 3177 { .procname = "neigh", 3178 .mode = 0555, .child = empty}, 3179 { } 3180 }; 3181 3182 static __net_initdata struct ctl_path ipv4_path[] = { 3183 { .procname = "net", }, 3184 { .procname = "ipv4", }, 3185 { }, 3186 }; 3187 3188 static struct ctl_table ipv4_route_flush_table[] = { 3189 { 3190 .procname = "flush", 3191 .maxlen = sizeof(int), 3192 .mode = 0200, 3193 .proc_handler = ipv4_sysctl_rtcache_flush, 3194 }, 3195 { }, 3196 }; 3197 3198 static __net_initdata struct ctl_path ipv4_route_path[] = { 3199 { .procname = "net", }, 3200 { .procname = "ipv4", }, 3201 { .procname = "route", }, 3202 { }, 3203 }; 3204 3205 static __net_init int sysctl_route_net_init(struct net *net) 3206 { 3207 struct ctl_table *tbl; 3208 3209 tbl = ipv4_route_flush_table; 3210 if (!net_eq(net, &init_net)) { 3211 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL); 3212 if (tbl == NULL) 3213 goto err_dup; 3214 } 3215 tbl[0].extra1 = net; 3216 3217 net->ipv4.route_hdr = 3218 register_net_sysctl_table(net, ipv4_route_path, tbl); 3219 if (net->ipv4.route_hdr == NULL) 3220 goto err_reg; 3221 return 0; 3222 3223 err_reg: 3224 if (tbl != ipv4_route_flush_table) 3225 kfree(tbl); 3226 err_dup: 3227 return -ENOMEM; 3228 } 3229 3230 static __net_exit void sysctl_route_net_exit(struct net *net) 3231 { 3232 struct ctl_table *tbl; 3233 3234 tbl = net->ipv4.route_hdr->ctl_table_arg; 3235 unregister_net_sysctl_table(net->ipv4.route_hdr); 3236 BUG_ON(tbl == ipv4_route_flush_table); 3237 kfree(tbl); 3238 } 3239 3240 static __net_initdata struct pernet_operations sysctl_route_ops = { 3241 .init = sysctl_route_net_init, 3242 .exit = sysctl_route_net_exit, 3243 }; 3244 #endif 3245 3246 static __net_init int rt_genid_init(struct net *net) 3247 { 3248 get_random_bytes(&net->ipv4.rt_genid, 3249 sizeof(net->ipv4.rt_genid)); 3250 return 0; 3251 } 3252 3253 static __net_initdata struct pernet_operations rt_genid_ops = { 3254 .init = rt_genid_init, 3255 }; 3256 3257 3258 #ifdef CONFIG_NET_CLS_ROUTE 3259 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; 3260 #endif /* CONFIG_NET_CLS_ROUTE */ 3261 3262 static __initdata unsigned long rhash_entries; 3263 static int __init set_rhash_entries(char *str) 3264 { 3265 if (!str) 3266 return 0; 3267 rhash_entries = simple_strtoul(str, &str, 0); 3268 return 1; 3269 } 3270 __setup("rhash_entries=", set_rhash_entries); 3271 3272 int __init ip_rt_init(void) 3273 { 3274 int rc = 0; 3275 3276 #ifdef CONFIG_NET_CLS_ROUTE 3277 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); 3278 if (!ip_rt_acct) 3279 panic("IP: failed to allocate ip_rt_acct\n"); 3280 #endif 3281 3282 ipv4_dst_ops.kmem_cachep = 3283 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0, 3284 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3285 3286 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep; 3287 3288 if (dst_entries_init(&ipv4_dst_ops) < 0) 3289 panic("IP: failed to allocate ipv4_dst_ops counter\n"); 3290 3291 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0) 3292 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n"); 3293 3294 rt_hash_table = (struct rt_hash_bucket *) 3295 alloc_large_system_hash("IP route cache", 3296 sizeof(struct rt_hash_bucket), 3297 rhash_entries, 3298 (totalram_pages >= 128 * 1024) ? 3299 15 : 17, 3300 0, 3301 &rt_hash_log, 3302 &rt_hash_mask, 3303 rhash_entries ? 0 : 512 * 1024); 3304 memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket)); 3305 rt_hash_lock_init(); 3306 3307 ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1); 3308 ip_rt_max_size = (rt_hash_mask + 1) * 16; 3309 3310 devinet_init(); 3311 ip_fib_init(); 3312 3313 /* All the timers, started at system startup tend 3314 to synchronize. Perturb it a bit. 3315 */ 3316 INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func); 3317 expires_ljiffies = jiffies; 3318 schedule_delayed_work(&expires_work, 3319 net_random() % ip_rt_gc_interval + ip_rt_gc_interval); 3320 3321 if (ip_rt_proc_init()) 3322 printk(KERN_ERR "Unable to create route proc files\n"); 3323 #ifdef CONFIG_XFRM 3324 xfrm_init(); 3325 xfrm4_init(ip_rt_max_size); 3326 #endif 3327 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL); 3328 3329 #ifdef CONFIG_SYSCTL 3330 register_pernet_subsys(&sysctl_route_ops); 3331 #endif 3332 register_pernet_subsys(&rt_genid_ops); 3333 return rc; 3334 } 3335 3336 #ifdef CONFIG_SYSCTL 3337 /* 3338 * We really need to sanitize the damn ipv4 init order, then all 3339 * this nonsense will go away. 3340 */ 3341 void __init ip_static_sysctl_init(void) 3342 { 3343 register_sysctl_paths(ipv4_path, ipv4_skeleton); 3344 } 3345 #endif 3346