?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xen mmu: fix a race window causing leave_mm BUG()
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / net / ipv4 / route.c
blobfabfe8168b909d87f875e3467ad3c1fd854a3dbd
1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * ROUTE - implementation of the IP router.
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 *
14 * Fixes:
15 * Alan Cox : Verify area fixes.
16 * Alan Cox : cli() protects routing changes
17 * Rui Oliveira : ICMP routing table updates
18 * (rco@di.uminho.pt) Routing table insertion and update
19 * Linus Torvalds : Rewrote bits to be sensible
20 * Alan Cox : Added BSD route gw semantics
21 * Alan Cox : Super /proc >4K
22 * Alan Cox : MTU in route table
23 * Alan Cox : MSS actually. Also added the window
24 * clamper.
25 * Sam Lantinga : Fixed route matching in rt_del()
26 * Alan Cox : Routing cache support.
27 * Alan Cox : Removed compatibility cruft.
28 * Alan Cox : RTF_REJECT support.
29 * Alan Cox : TCP irtt support.
30 * Jonathan Naylor : Added Metric support.
31 * Miquel van Smoorenburg : BSD API fixes.
32 * Miquel van Smoorenburg : Metrics.
33 * Alan Cox : Use __u32 properly
34 * Alan Cox : Aligned routing errors more closely with BSD
35 * our system is still very different.
36 * Alan Cox : Faster /proc handling
37 * Alexey Kuznetsov : Massive rework to support tree based routing,
38 * routing caches and better behaviour.
39 *
40 * Olaf Erb : irtt wasn't being copied right.
41 * Bjorn Ekwall : Kerneld route support.
42 * Alan Cox : Multicast fixed (I hope)
43 * Pavel Krauz : Limited broadcast fixed
44 * Mike McLagan : Routing by source
45 * Alexey Kuznetsov : End of old history. Split to fib.c and
46 * route.c and rewritten from scratch.
47 * Andi Kleen : Load-limit warning messages.
48 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow.
50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful.
52 * Marc Boucher : routing by fwmark
53 * Robert Olsson : Added rt_cache statistics
54 * Arnaldo C. Melo : Convert proc stuff to seq_file
55 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
56 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
57 * Ilia Sotnikov : Removed TOS from hash calculations
58 *
59 * This program is free software; you can redistribute it and/or
60 * modify it under the terms of the GNU General Public License
61 * as published by the Free Software Foundation; either version
62 * 2 of the License, or (at your option) any later version.
63 */
64
65 #include <linux/module.h>
66 #include <asm/uaccess.h>
67 #include <asm/system.h>
68 #include <linux/bitops.h>
69 #include <linux/types.h>
70 #include <linux/kernel.h>
71 #include <linux/mm.h>
72 #include <linux/bootmem.h>
73 #include <linux/string.h>
74 #include <linux/socket.h>
75 #include <linux/sockios.h>
76 #include <linux/errno.h>
77 #include <linux/in.h>
78 #include <linux/inet.h>
79 #include <linux/netdevice.h>
80 #include <linux/proc_fs.h>
81 #include <linux/init.h>
82 #include <linux/workqueue.h>
83 #include <linux/skbuff.h>
84 #include <linux/inetdevice.h>
85 #include <linux/igmp.h>
86 #include <linux/pkt_sched.h>
87 #include <linux/mroute.h>
88 #include <linux/netfilter_ipv4.h>
89 #include <linux/random.h>
90 #include <linux/jhash.h>
91 #include <linux/rcupdate.h>
92 #include <linux/times.h>
93 #include <linux/slab.h>
94 #include <net/dst.h>
95 #include <net/net_namespace.h>
96 #include <net/protocol.h>
97 #include <net/ip.h>
98 #include <net/route.h>
99 #include <net/inetpeer.h>
100 #include <net/sock.h>
101 #include <net/ip_fib.h>
102 #include <net/arp.h>
103 #include <net/tcp.h>
104 #include <net/icmp.h>
105 #include <net/xfrm.h>
106 #include <net/netevent.h>
107 #include <net/rtnetlink.h>
108 #ifdef CONFIG_SYSCTL
109 #include <linux/sysctl.h>
110 #endif
111
112 #define RT_FL_TOS(oldflp) \
113 ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
114
115 #define IP_MAX_MTU 0xFFF0
116
117 #define RT_GC_TIMEOUT (300*HZ)
118
119 static int ip_rt_max_size;
120 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
121 static int ip_rt_gc_interval __read_mostly = 60 * HZ;
122 static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
123 static int ip_rt_redirect_number __read_mostly = 9;
124 static int ip_rt_redirect_load __read_mostly = HZ / 50;
125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
126 static int ip_rt_error_cost __read_mostly = HZ;
127 static int ip_rt_error_burst __read_mostly = 5 * HZ;
128 static int ip_rt_gc_elasticity __read_mostly = 8;
129 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
130 static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
131 static int ip_rt_min_advmss __read_mostly = 256;
132 static int rt_chain_length_max __read_mostly = 20;
133
134 static struct delayed_work expires_work;
135 static unsigned long expires_ljiffies;
136
137 /*
138 * Interface to generic destination cache.
139 */
140
141 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
142 static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
143 static unsigned int ipv4_default_mtu(const struct dst_entry *dst);
144 static void ipv4_dst_destroy(struct dst_entry *dst);
145 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
146 static void ipv4_link_failure(struct sk_buff *skb);
147 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
148 static int rt_garbage_collect(struct dst_ops *ops);
149
150 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
151 int how)
152 {
153 }
154
155 static struct dst_ops ipv4_dst_ops = {
156 .family = AF_INET,
157 .protocol = cpu_to_be16(ETH_P_IP),
158 .gc = rt_garbage_collect,
159 .check = ipv4_dst_check,
160 .default_advmss = ipv4_default_advmss,
161 .default_mtu = ipv4_default_mtu,
162 .destroy = ipv4_dst_destroy,
163 .ifdown = ipv4_dst_ifdown,
164 .negative_advice = ipv4_negative_advice,
165 .link_failure = ipv4_link_failure,
166 .update_pmtu = ip_rt_update_pmtu,
167 .local_out = __ip_local_out,
168 };
169
170 #define ECN_OR_COST(class) TC_PRIO_##class
171
172 const __u8 ip_tos2prio[16] = {
173 TC_PRIO_BESTEFFORT,
174 ECN_OR_COST(BESTEFFORT),
175 TC_PRIO_BESTEFFORT,
176 ECN_OR_COST(BESTEFFORT),
177 TC_PRIO_BULK,
178 ECN_OR_COST(BULK),
179 TC_PRIO_BULK,
180 ECN_OR_COST(BULK),
181 TC_PRIO_INTERACTIVE,
182 ECN_OR_COST(INTERACTIVE),
183 TC_PRIO_INTERACTIVE,
184 ECN_OR_COST(INTERACTIVE),
185 TC_PRIO_INTERACTIVE_BULK,
186 ECN_OR_COST(INTERACTIVE_BULK),
187 TC_PRIO_INTERACTIVE_BULK,
188 ECN_OR_COST(INTERACTIVE_BULK)
189 };
190
191
192 /*
193 * Route cache.
194 */
195
196 /* The locking scheme is rather straight forward:
197 *
198 * 1) Read-Copy Update protects the buckets of the central route hash.
199 * 2) Only writers remove entries, and they hold the lock
200 * as they look at rtable reference counts.
201 * 3) Only readers acquire references to rtable entries,
202 * they do so with atomic increments and with the
203 * lock held.
204 */
205
206 struct rt_hash_bucket {
207 struct rtable __rcu *chain;
208 };
209
210 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
211 defined(CONFIG_PROVE_LOCKING)
212 /*
213 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
214 * The size of this table is a power of two and depends on the number of CPUS.
215 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
216 */
217 #ifdef CONFIG_LOCKDEP
218 # define RT_HASH_LOCK_SZ 256
219 #else
220 # if NR_CPUS >= 32
221 # define RT_HASH_LOCK_SZ 4096
222 # elif NR_CPUS >= 16
223 # define RT_HASH_LOCK_SZ 2048
224 # elif NR_CPUS >= 8
225 # define RT_HASH_LOCK_SZ 1024
226 # elif NR_CPUS >= 4
227 # define RT_HASH_LOCK_SZ 512
228 # else
229 # define RT_HASH_LOCK_SZ 256
230 # endif
231 #endif
232
233 static spinlock_t *rt_hash_locks;
234 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
235
236 static __init void rt_hash_lock_init(void)
237 {
238 int i;
239
240 rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
241 GFP_KERNEL);
242 if (!rt_hash_locks)
243 panic("IP: failed to allocate rt_hash_locks\n");
244
245 for (i = 0; i < RT_HASH_LOCK_SZ; i++)
246 spin_lock_init(&rt_hash_locks[i]);
247 }
248 #else
249 # define rt_hash_lock_addr(slot) NULL
250
251 static inline void rt_hash_lock_init(void)
252 {
253 }
254 #endif
255
256 static struct rt_hash_bucket *rt_hash_table __read_mostly;
257 static unsigned rt_hash_mask __read_mostly;
258 static unsigned int rt_hash_log __read_mostly;
259
260 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
261 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
262
263 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
264 int genid)
265 {
266 return jhash_3words((__force u32)daddr, (__force u32)saddr,
267 idx, genid)
268 & rt_hash_mask;
269 }
270
271 static inline int rt_genid(struct net *net)
272 {
273 return atomic_read(&net->ipv4.rt_genid);
274 }
275
276 #ifdef CONFIG_PROC_FS
277 struct rt_cache_iter_state {
278 struct seq_net_private p;
279 int bucket;
280 int genid;
281 };
282
283 static struct rtable *rt_cache_get_first(struct seq_file *seq)
284 {
285 struct rt_cache_iter_state *st = seq->private;
286 struct rtable *r = NULL;
287
288 for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
289 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
290 continue;
291 rcu_read_lock_bh();
292 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
293 while (r) {
294 if (dev_net(r->dst.dev) == seq_file_net(seq) &&
295 r->rt_genid == st->genid)
296 return r;
297 r = rcu_dereference_bh(r->dst.rt_next);
298 }
299 rcu_read_unlock_bh();
300 }
301 return r;
302 }
303
304 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
305 struct rtable *r)
306 {
307 struct rt_cache_iter_state *st = seq->private;
308
309 r = rcu_dereference_bh(r->dst.rt_next);
310 while (!r) {
311 rcu_read_unlock_bh();
312 do {
313 if (--st->bucket < 0)
314 return NULL;
315 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
316 rcu_read_lock_bh();
317 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
318 }
319 return r;
320 }
321
322 static struct rtable *rt_cache_get_next(struct seq_file *seq,
323 struct rtable *r)
324 {
325 struct rt_cache_iter_state *st = seq->private;
326 while ((r = __rt_cache_get_next(seq, r)) != NULL) {
327 if (dev_net(r->dst.dev) != seq_file_net(seq))
328 continue;
329 if (r->rt_genid == st->genid)
330 break;
331 }
332 return r;
333 }
334
335 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
336 {
337 struct rtable *r = rt_cache_get_first(seq);
338
339 if (r)
340 while (pos && (r = rt_cache_get_next(seq, r)))
341 --pos;
342 return pos ? NULL : r;
343 }
344
345 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
346 {
347 struct rt_cache_iter_state *st = seq->private;
348 if (*pos)
349 return rt_cache_get_idx(seq, *pos - 1);
350 st->genid = rt_genid(seq_file_net(seq));
351 return SEQ_START_TOKEN;
352 }
353
354 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
355 {
356 struct rtable *r;
357
358 if (v == SEQ_START_TOKEN)
359 r = rt_cache_get_first(seq);
360 else
361 r = rt_cache_get_next(seq, v);
362 ++*pos;
363 return r;
364 }
365
366 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
367 {
368 if (v && v != SEQ_START_TOKEN)
369 rcu_read_unlock_bh();
370 }
371
372 static int rt_cache_seq_show(struct seq_file *seq, void *v)
373 {
374 if (v == SEQ_START_TOKEN)
375 seq_printf(seq, "%-127s\n",
376 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
377 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
378 "HHUptod\tSpecDst");
379 else {
380 struct rtable *r = v;
381 int len;
382
383 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
384 "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
385 r->dst.dev ? r->dst.dev->name : "*",
386 (__force u32)r->rt_dst,
387 (__force u32)r->rt_gateway,
388 r->rt_flags, atomic_read(&r->dst.__refcnt),
389 r->dst.__use, 0, (__force u32)r->rt_src,
390 dst_metric_advmss(&r->dst) + 40,
391 dst_metric(&r->dst, RTAX_WINDOW),
392 (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
393 dst_metric(&r->dst, RTAX_RTTVAR)),
394 r->fl.fl4_tos,
395 r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
396 r->dst.hh ? (r->dst.hh->hh_output ==
397 dev_queue_xmit) : 0,
398 r->rt_spec_dst, &len);
399
400 seq_printf(seq, "%*s\n", 127 - len, "");
401 }
402 return 0;
403 }
404
405 static const struct seq_operations rt_cache_seq_ops = {
406 .start = rt_cache_seq_start,
407 .next = rt_cache_seq_next,
408 .stop = rt_cache_seq_stop,
409 .show = rt_cache_seq_show,
410 };
411
412 static int rt_cache_seq_open(struct inode *inode, struct file *file)
413 {
414 return seq_open_net(inode, file, &rt_cache_seq_ops,
415 sizeof(struct rt_cache_iter_state));
416 }
417
418 static const struct file_operations rt_cache_seq_fops = {
419 .owner = THIS_MODULE,
420 .open = rt_cache_seq_open,
421 .read = seq_read,
422 .llseek = seq_lseek,
423 .release = seq_release_net,
424 };
425
426
427 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
428 {
429 int cpu;
430
431 if (*pos == 0)
432 return SEQ_START_TOKEN;
433
434 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
435 if (!cpu_possible(cpu))
436 continue;
437 *pos = cpu+1;
438 return &per_cpu(rt_cache_stat, cpu);
439 }
440 return NULL;
441 }
442
443 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
444 {
445 int cpu;
446
447 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
448 if (!cpu_possible(cpu))
449 continue;
450 *pos = cpu+1;
451 return &per_cpu(rt_cache_stat, cpu);
452 }
453 return NULL;
454
455 }
456
457 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
458 {
459
460 }
461
462 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
463 {
464 struct rt_cache_stat *st = v;
465
466 if (v == SEQ_START_TOKEN) {
467 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
468 return 0;
469 }
470
471 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x "
472 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
473 dst_entries_get_slow(&ipv4_dst_ops),
474 st->in_hit,
475 st->in_slow_tot,
476 st->in_slow_mc,
477 st->in_no_route,
478 st->in_brd,
479 st->in_martian_dst,
480 st->in_martian_src,
481
482 st->out_hit,
483 st->out_slow_tot,
484 st->out_slow_mc,
485
486 st->gc_total,
487 st->gc_ignored,
488 st->gc_goal_miss,
489 st->gc_dst_overflow,
490 st->in_hlist_search,
491 st->out_hlist_search
492 );
493 return 0;
494 }
495
496 static const struct seq_operations rt_cpu_seq_ops = {
497 .start = rt_cpu_seq_start,
498 .next = rt_cpu_seq_next,
499 .stop = rt_cpu_seq_stop,
500 .show = rt_cpu_seq_show,
501 };
502
503
504 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
505 {
506 return seq_open(file, &rt_cpu_seq_ops);
507 }
508
509 static const struct file_operations rt_cpu_seq_fops = {
510 .owner = THIS_MODULE,
511 .open = rt_cpu_seq_open,
512 .read = seq_read,
513 .llseek = seq_lseek,
514 .release = seq_release,
515 };
516
517 #ifdef CONFIG_NET_CLS_ROUTE
518 static int rt_acct_proc_show(struct seq_file *m, void *v)
519 {
520 struct ip_rt_acct *dst, *src;
521 unsigned int i, j;
522
523 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
524 if (!dst)
525 return -ENOMEM;
526
527 for_each_possible_cpu(i) {
528 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
529 for (j = 0; j < 256; j++) {
530 dst[j].o_bytes += src[j].o_bytes;
531 dst[j].o_packets += src[j].o_packets;
532 dst[j].i_bytes += src[j].i_bytes;
533 dst[j].i_packets += src[j].i_packets;
534 }
535 }
536
537 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
538 kfree(dst);
539 return 0;
540 }
541
542 static int rt_acct_proc_open(struct inode *inode, struct file *file)
543 {
544 return single_open(file, rt_acct_proc_show, NULL);
545 }
546
547 static const struct file_operations rt_acct_proc_fops = {
548 .owner = THIS_MODULE,
549 .open = rt_acct_proc_open,
550 .read = seq_read,
551 .llseek = seq_lseek,
552 .release = single_release,
553 };
554 #endif
555
556 static int __net_init ip_rt_do_proc_init(struct net *net)
557 {
558 struct proc_dir_entry *pde;
559
560 pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
561 &rt_cache_seq_fops);
562 if (!pde)
563 goto err1;
564
565 pde = proc_create("rt_cache", S_IRUGO,
566 net->proc_net_stat, &rt_cpu_seq_fops);
567 if (!pde)
568 goto err2;
569
570 #ifdef CONFIG_NET_CLS_ROUTE
571 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
572 if (!pde)
573 goto err3;
574 #endif
575 return 0;
576
577 #ifdef CONFIG_NET_CLS_ROUTE
578 err3:
579 remove_proc_entry("rt_cache", net->proc_net_stat);
580 #endif
581 err2:
582 remove_proc_entry("rt_cache", net->proc_net);
583 err1:
584 return -ENOMEM;
585 }
586
587 static void __net_exit ip_rt_do_proc_exit(struct net *net)
588 {
589 remove_proc_entry("rt_cache", net->proc_net_stat);
590 remove_proc_entry("rt_cache", net->proc_net);
591 #ifdef CONFIG_NET_CLS_ROUTE
592 remove_proc_entry("rt_acct", net->proc_net);
593 #endif
594 }
595
596 static struct pernet_operations ip_rt_proc_ops __net_initdata = {
597 .init = ip_rt_do_proc_init,
598 .exit = ip_rt_do_proc_exit,
599 };
600
601 static int __init ip_rt_proc_init(void)
602 {
603 return register_pernet_subsys(&ip_rt_proc_ops);
604 }
605
606 #else
607 static inline int ip_rt_proc_init(void)
608 {
609 return 0;
610 }
611 #endif /* CONFIG_PROC_FS */
612
613 static inline void rt_free(struct rtable *rt)
614 {
615 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
616 }
617
618 static inline void rt_drop(struct rtable *rt)
619 {
620 ip_rt_put(rt);
621 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
622 }
623
624 static inline int rt_fast_clean(struct rtable *rth)
625 {
626 /* Kill broadcast/multicast entries very aggresively, if they
627 collide in hash table with more useful entries */
628 return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
629 rt_is_input_route(rth) && rth->dst.rt_next;
630 }
631
632 static inline int rt_valuable(struct rtable *rth)
633 {
634 return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
635 rth->dst.expires;
636 }
637
638 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
639 {
640 unsigned long age;
641 int ret = 0;
642
643 if (atomic_read(&rth->dst.__refcnt))
644 goto out;
645
646 ret = 1;
647 if (rth->dst.expires &&
648 time_after_eq(jiffies, rth->dst.expires))
649 goto out;
650
651 age = jiffies - rth->dst.lastuse;
652 ret = 0;
653 if ((age <= tmo1 && !rt_fast_clean(rth)) ||
654 (age <= tmo2 && rt_valuable(rth)))
655 goto out;
656 ret = 1;
657 out: return ret;
658 }
659
660 /* Bits of score are:
661 * 31: very valuable
662 * 30: not quite useless
663 * 29..0: usage counter
664 */
665 static inline u32 rt_score(struct rtable *rt)
666 {
667 u32 score = jiffies - rt->dst.lastuse;
668
669 score = ~score & ~(3<<30);
670
671 if (rt_valuable(rt))
672 score |= (1<<31);
673
674 if (rt_is_output_route(rt) ||
675 !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
676 score |= (1<<30);
677
678 return score;
679 }
680
681 static inline bool rt_caching(const struct net *net)
682 {
683 return net->ipv4.current_rt_cache_rebuild_count <=
684 net->ipv4.sysctl_rt_cache_rebuild_count;
685 }
686
687 static inline bool compare_hash_inputs(const struct flowi *fl1,
688 const struct flowi *fl2)
689 {
690 return ((((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
691 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
692 (fl1->iif ^ fl2->iif)) == 0);
693 }
694
695 static inline int compare_keys(struct flowi *fl1, struct flowi *fl2)
696 {
697 return (((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
698 ((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
699 (fl1->mark ^ fl2->mark) |
700 (*(u16 *)&fl1->fl4_tos ^ *(u16 *)&fl2->fl4_tos) |
701 (fl1->oif ^ fl2->oif) |
702 (fl1->iif ^ fl2->iif)) == 0;
703 }
704
705 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
706 {
707 return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
708 }
709
710 static inline int rt_is_expired(struct rtable *rth)
711 {
712 return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
713 }
714
715 /*
716 * Perform a full scan of hash table and free all entries.
717 * Can be called by a softirq or a process.
718 * In the later case, we want to be reschedule if necessary
719 */
720 static void rt_do_flush(struct net *net, int process_context)
721 {
722 unsigned int i;
723 struct rtable *rth, *next;
724
725 for (i = 0; i <= rt_hash_mask; i++) {
726 struct rtable __rcu **pprev;
727 struct rtable *list;
728
729 if (process_context && need_resched())
730 cond_resched();
731 rth = rcu_dereference_raw(rt_hash_table[i].chain);
732 if (!rth)
733 continue;
734
735 spin_lock_bh(rt_hash_lock_addr(i));
736
737 list = NULL;
738 pprev = &rt_hash_table[i].chain;
739 rth = rcu_dereference_protected(*pprev,
740 lockdep_is_held(rt_hash_lock_addr(i)));
741
742 while (rth) {
743 next = rcu_dereference_protected(rth->dst.rt_next,
744 lockdep_is_held(rt_hash_lock_addr(i)));
745
746 if (!net ||
747 net_eq(dev_net(rth->dst.dev), net)) {
748 rcu_assign_pointer(*pprev, next);
749 rcu_assign_pointer(rth->dst.rt_next, list);
750 list = rth;
751 } else {
752 pprev = &rth->dst.rt_next;
753 }
754 rth = next;
755 }
756
757 spin_unlock_bh(rt_hash_lock_addr(i));
758
759 for (; list; list = next) {
760 next = rcu_dereference_protected(list->dst.rt_next, 1);
761 rt_free(list);
762 }
763 }
764 }
765
766 /*
767 * While freeing expired entries, we compute average chain length
768 * and standard deviation, using fixed-point arithmetic.
769 * This to have an estimation of rt_chain_length_max
770 * rt_chain_length_max = max(elasticity, AVG + 4*SD)
771 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
772 */
773
774 #define FRACT_BITS 3
775 #define ONE (1UL << FRACT_BITS)
776
777 /*
778 * Given a hash chain and an item in this hash chain,
779 * find if a previous entry has the same hash_inputs
780 * (but differs on tos, mark or oif)
781 * Returns 0 if an alias is found.
782 * Returns ONE if rth has no alias before itself.
783 */
784 static int has_noalias(const struct rtable *head, const struct rtable *rth)
785 {
786 const struct rtable *aux = head;
787
788 while (aux != rth) {
789 if (compare_hash_inputs(&aux->fl, &rth->fl))
790 return 0;
791 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
792 }
793 return ONE;
794 }
795
796 static void rt_check_expire(void)
797 {
798 static unsigned int rover;
799 unsigned int i = rover, goal;
800 struct rtable *rth;
801 struct rtable __rcu **rthp;
802 unsigned long samples = 0;
803 unsigned long sum = 0, sum2 = 0;
804 unsigned long delta;
805 u64 mult;
806
807 delta = jiffies - expires_ljiffies;
808 expires_ljiffies = jiffies;
809 mult = ((u64)delta) << rt_hash_log;
810 if (ip_rt_gc_timeout > 1)
811 do_div(mult, ip_rt_gc_timeout);
812 goal = (unsigned int)mult;
813 if (goal > rt_hash_mask)
814 goal = rt_hash_mask + 1;
815 for (; goal > 0; goal--) {
816 unsigned long tmo = ip_rt_gc_timeout;
817 unsigned long length;
818
819 i = (i + 1) & rt_hash_mask;
820 rthp = &rt_hash_table[i].chain;
821
822 if (need_resched())
823 cond_resched();
824
825 samples++;
826
827 if (rcu_dereference_raw(*rthp) == NULL)
828 continue;
829 length = 0;
830 spin_lock_bh(rt_hash_lock_addr(i));
831 while ((rth = rcu_dereference_protected(*rthp,
832 lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
833 prefetch(rth->dst.rt_next);
834 if (rt_is_expired(rth)) {
835 *rthp = rth->dst.rt_next;
836 rt_free(rth);
837 continue;
838 }
839 if (rth->dst.expires) {
840 /* Entry is expired even if it is in use */
841 if (time_before_eq(jiffies, rth->dst.expires)) {
842 nofree:
843 tmo >>= 1;
844 rthp = &rth->dst.rt_next;
845 /*
846 * We only count entries on
847 * a chain with equal hash inputs once
848 * so that entries for different QOS
849 * levels, and other non-hash input
850 * attributes don't unfairly skew
851 * the length computation
852 */
853 length += has_noalias(rt_hash_table[i].chain, rth);
854 continue;
855 }
856 } else if (!rt_may_expire(rth, tmo, ip_rt_gc_timeout))
857 goto nofree;
858
859 /* Cleanup aged off entries. */
860 *rthp = rth->dst.rt_next;
861 rt_free(rth);
862 }
863 spin_unlock_bh(rt_hash_lock_addr(i));
864 sum += length;
865 sum2 += length*length;
866 }
867 if (samples) {
868 unsigned long avg = sum / samples;
869 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
870 rt_chain_length_max = max_t(unsigned long,
871 ip_rt_gc_elasticity,
872 (avg + 4*sd) >> FRACT_BITS);
873 }
874 rover = i;
875 }
876
877 /*
878 * rt_worker_func() is run in process context.
879 * we call rt_check_expire() to scan part of the hash table
880 */
881 static void rt_worker_func(struct work_struct *work)
882 {
883 rt_check_expire();
884 schedule_delayed_work(&expires_work, ip_rt_gc_interval);
885 }
886
887 /*
888 * Pertubation of rt_genid by a small quantity [1..256]
889 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
890 * many times (2^24) without giving recent rt_genid.
891 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
892 */
893 static void rt_cache_invalidate(struct net *net)
894 {
895 unsigned char shuffle;
896
897 get_random_bytes(&shuffle, sizeof(shuffle));
898 atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
899 }
900
901 /*
902 * delay < 0 : invalidate cache (fast : entries will be deleted later)
903 * delay >= 0 : invalidate & flush cache (can be long)
904 */
905 void rt_cache_flush(struct net *net, int delay)
906 {
907 rt_cache_invalidate(net);
908 if (delay >= 0)
909 rt_do_flush(net, !in_softirq());
910 }
911
912 /* Flush previous cache invalidated entries from the cache */
913 void rt_cache_flush_batch(struct net *net)
914 {
915 rt_do_flush(net, !in_softirq());
916 }
917
918 static void rt_emergency_hash_rebuild(struct net *net)
919 {
920 if (net_ratelimit())
921 printk(KERN_WARNING "Route hash chain too long!\n");
922 rt_cache_invalidate(net);
923 }
924
925 /*
926 Short description of GC goals.
927
928 We want to build algorithm, which will keep routing cache
929 at some equilibrium point, when number of aged off entries
930 is kept approximately equal to newly generated ones.
931
932 Current expiration strength is variable "expire".
933 We try to adjust it dynamically, so that if networking
934 is idle expires is large enough to keep enough of warm entries,
935 and when load increases it reduces to limit cache size.
936 */
937
938 static int rt_garbage_collect(struct dst_ops *ops)
939 {
940 static unsigned long expire = RT_GC_TIMEOUT;
941 static unsigned long last_gc;
942 static int rover;
943 static int equilibrium;
944 struct rtable *rth;
945 struct rtable __rcu **rthp;
946 unsigned long now = jiffies;
947 int goal;
948 int entries = dst_entries_get_fast(&ipv4_dst_ops);
949
950 /*
951 * Garbage collection is pretty expensive,
952 * do not make it too frequently.
953 */
954
955 RT_CACHE_STAT_INC(gc_total);
956
957 if (now - last_gc < ip_rt_gc_min_interval &&
958 entries < ip_rt_max_size) {
959 RT_CACHE_STAT_INC(gc_ignored);
960 goto out;
961 }
962
963 entries = dst_entries_get_slow(&ipv4_dst_ops);
964 /* Calculate number of entries, which we want to expire now. */
965 goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
966 if (goal <= 0) {
967 if (equilibrium < ipv4_dst_ops.gc_thresh)
968 equilibrium = ipv4_dst_ops.gc_thresh;
969 goal = entries - equilibrium;
970 if (goal > 0) {
971 equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
972 goal = entries - equilibrium;
973 }
974 } else {
975 /* We are in dangerous area. Try to reduce cache really
976 * aggressively.
977 */
978 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
979 equilibrium = entries - goal;
980 }
981
982 if (now - last_gc >= ip_rt_gc_min_interval)
983 last_gc = now;
984
985 if (goal <= 0) {
986 equilibrium += goal;
987 goto work_done;
988 }
989
990 do {
991 int i, k;
992
993 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
994 unsigned long tmo = expire;
995
996 k = (k + 1) & rt_hash_mask;
997 rthp = &rt_hash_table[k].chain;
998 spin_lock_bh(rt_hash_lock_addr(k));
999 while ((rth = rcu_dereference_protected(*rthp,
1000 lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
1001 if (!rt_is_expired(rth) &&
1002 !rt_may_expire(rth, tmo, expire)) {
1003 tmo >>= 1;
1004 rthp = &rth->dst.rt_next;
1005 continue;
1006 }
1007 *rthp = rth->dst.rt_next;
1008 rt_free(rth);
1009 goal--;
1010 }
1011 spin_unlock_bh(rt_hash_lock_addr(k));
1012 if (goal <= 0)
1013 break;
1014 }
1015 rover = k;
1016
1017 if (goal <= 0)
1018 goto work_done;
1019
1020 /* Goal is not achieved. We stop process if:
1021
1022 - if expire reduced to zero. Otherwise, expire is halfed.
1023 - if table is not full.
1024 - if we are called from interrupt.
1025 - jiffies check is just fallback/debug loop breaker.
1026 We will not spin here for long time in any case.
1027 */
1028
1029 RT_CACHE_STAT_INC(gc_goal_miss);
1030
1031 if (expire == 0)
1032 break;
1033
1034 expire >>= 1;
1035 #if RT_CACHE_DEBUG >= 2
1036 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
1037 dst_entries_get_fast(&ipv4_dst_ops), goal, i);
1038 #endif
1039
1040 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1041 goto out;
1042 } while (!in_softirq() && time_before_eq(jiffies, now));
1043
1044 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1045 goto out;
1046 if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
1047 goto out;
1048 if (net_ratelimit())
1049 printk(KERN_WARNING "dst cache overflow\n");
1050 RT_CACHE_STAT_INC(gc_dst_overflow);
1051 return 1;
1052
1053 work_done:
1054 expire += ip_rt_gc_min_interval;
1055 if (expire > ip_rt_gc_timeout ||
1056 dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1057 dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
1058 expire = ip_rt_gc_timeout;
1059 #if RT_CACHE_DEBUG >= 2
1060 printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
1061 dst_entries_get_fast(&ipv4_dst_ops), goal, rover);
1062 #endif
1063 out: return 0;
1064 }
1065
1066 /*
1067 * Returns number of entries in a hash chain that have different hash_inputs
1068 */
1069 static int slow_chain_length(const struct rtable *head)
1070 {
1071 int length = 0;
1072 const struct rtable *rth = head;
1073
1074 while (rth) {
1075 length += has_noalias(head, rth);
1076 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1077 }
1078 return length >> FRACT_BITS;
1079 }
1080
1081 static int rt_intern_hash(unsigned hash, struct rtable *rt,
1082 struct rtable **rp, struct sk_buff *skb, int ifindex)
1083 {
1084 struct rtable *rth, *cand;
1085 struct rtable __rcu **rthp, **candp;
1086 unsigned long now;
1087 u32 min_score;
1088 int chain_length;
1089 int attempts = !in_softirq();
1090
1091 restart:
1092 chain_length = 0;
1093 min_score = ~(u32)0;
1094 cand = NULL;
1095 candp = NULL;
1096 now = jiffies;
1097
1098 if (!rt_caching(dev_net(rt->dst.dev))) {
1099 /*
1100 * If we're not caching, just tell the caller we
1101 * were successful and don't touch the route. The
1102 * caller hold the sole reference to the cache entry, and
1103 * it will be released when the caller is done with it.
1104 * If we drop it here, the callers have no way to resolve routes
1105 * when we're not caching. Instead, just point *rp at rt, so
1106 * the caller gets a single use out of the route
1107 * Note that we do rt_free on this new route entry, so that
1108 * once its refcount hits zero, we are still able to reap it
1109 * (Thanks Alexey)
1110 * Note: To avoid expensive rcu stuff for this uncached dst,
1111 * we set DST_NOCACHE so that dst_release() can free dst without
1112 * waiting a grace period.
1113 */
1114
1115 rt->dst.flags |= DST_NOCACHE;
1116 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1117 int err = arp_bind_neighbour(&rt->dst);
1118 if (err) {
1119 if (net_ratelimit())
1120 printk(KERN_WARNING
1121 "Neighbour table failure & not caching routes.\n");
1122 ip_rt_put(rt);
1123 return err;
1124 }
1125 }
1126
1127 goto skip_hashing;
1128 }
1129
1130 rthp = &rt_hash_table[hash].chain;
1131
1132 spin_lock_bh(rt_hash_lock_addr(hash));
1133 while ((rth = rcu_dereference_protected(*rthp,
1134 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1135 if (rt_is_expired(rth)) {
1136 *rthp = rth->dst.rt_next;
1137 rt_free(rth);
1138 continue;
1139 }
1140 if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) {
1141 /* Put it first */
1142 *rthp = rth->dst.rt_next;
1143 /*
1144 * Since lookup is lockfree, the deletion
1145 * must be visible to another weakly ordered CPU before
1146 * the insertion at the start of the hash chain.
1147 */
1148 rcu_assign_pointer(rth->dst.rt_next,
1149 rt_hash_table[hash].chain);
1150 /*
1151 * Since lookup is lockfree, the update writes
1152 * must be ordered for consistency on SMP.
1153 */
1154 rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1155
1156 dst_use(&rth->dst, now);
1157 spin_unlock_bh(rt_hash_lock_addr(hash));
1158
1159 rt_drop(rt);
1160 if (rp)
1161 *rp = rth;
1162 else
1163 skb_dst_set(skb, &rth->dst);
1164 return 0;
1165 }
1166
1167 if (!atomic_read(&rth->dst.__refcnt)) {
1168 u32 score = rt_score(rth);
1169
1170 if (score <= min_score) {
1171 cand = rth;
1172 candp = rthp;
1173 min_score = score;
1174 }
1175 }
1176
1177 chain_length++;
1178
1179 rthp = &rth->dst.rt_next;
1180 }
1181
1182 if (cand) {
1183 /* ip_rt_gc_elasticity used to be average length of chain
1184 * length, when exceeded gc becomes really aggressive.
1185 *
1186 * The second limit is less certain. At the moment it allows
1187 * only 2 entries per bucket. We will see.
1188 */
1189 if (chain_length > ip_rt_gc_elasticity) {
1190 *candp = cand->dst.rt_next;
1191 rt_free(cand);
1192 }
1193 } else {
1194 if (chain_length > rt_chain_length_max &&
1195 slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1196 struct net *net = dev_net(rt->dst.dev);
1197 int num = ++net->ipv4.current_rt_cache_rebuild_count;
1198 if (!rt_caching(net)) {
1199 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1200 rt->dst.dev->name, num);
1201 }
1202 rt_emergency_hash_rebuild(net);
1203 spin_unlock_bh(rt_hash_lock_addr(hash));
1204
1205 hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1206 ifindex, rt_genid(net));
1207 goto restart;
1208 }
1209 }
1210
1211 /* Try to bind route to arp only if it is output
1212 route or unicast forwarding path.
1213 */
1214 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1215 int err = arp_bind_neighbour(&rt->dst);
1216 if (err) {
1217 spin_unlock_bh(rt_hash_lock_addr(hash));
1218
1219 if (err != -ENOBUFS) {
1220 rt_drop(rt);
1221 return err;
1222 }
1223
1224 /* Neighbour tables are full and nothing
1225 can be released. Try to shrink route cache,
1226 it is most likely it holds some neighbour records.
1227 */
1228 if (attempts-- > 0) {
1229 int saved_elasticity = ip_rt_gc_elasticity;
1230 int saved_int = ip_rt_gc_min_interval;
1231 ip_rt_gc_elasticity = 1;
1232 ip_rt_gc_min_interval = 0;
1233 rt_garbage_collect(&ipv4_dst_ops);
1234 ip_rt_gc_min_interval = saved_int;
1235 ip_rt_gc_elasticity = saved_elasticity;
1236 goto restart;
1237 }
1238
1239 if (net_ratelimit())
1240 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
1241 rt_drop(rt);
1242 return -ENOBUFS;
1243 }
1244 }
1245
1246 rt->dst.rt_next = rt_hash_table[hash].chain;
1247
1248 #if RT_CACHE_DEBUG >= 2
1249 if (rt->dst.rt_next) {
1250 struct rtable *trt;
1251 printk(KERN_DEBUG "rt_cache @%02x: %pI4",
1252 hash, &rt->rt_dst);
1253 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
1254 printk(" . %pI4", &trt->rt_dst);
1255 printk("\n");
1256 }
1257 #endif
1258 /*
1259 * Since lookup is lockfree, we must make sure
1260 * previous writes to rt are comitted to memory
1261 * before making rt visible to other CPUS.
1262 */
1263 rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1264
1265 spin_unlock_bh(rt_hash_lock_addr(hash));
1266
1267 skip_hashing:
1268 if (rp)
1269 *rp = rt;
1270 else
1271 skb_dst_set(skb, &rt->dst);
1272 return 0;
1273 }
1274
1275 void rt_bind_peer(struct rtable *rt, int create)
1276 {
1277 struct inet_peer *peer;
1278
1279 peer = inet_getpeer_v4(rt->rt_dst, create);
1280
1281 if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
1282 inet_putpeer(peer);
1283 }
1284
1285 /*
1286 * Peer allocation may fail only in serious out-of-memory conditions. However
1287 * we still can generate some output.
1288 * Random ID selection looks a bit dangerous because we have no chances to
1289 * select ID being unique in a reasonable period of time.
1290 * But broken packet identifier may be better than no packet at all.
1291 */
1292 static void ip_select_fb_ident(struct iphdr *iph)
1293 {
1294 static DEFINE_SPINLOCK(ip_fb_id_lock);
1295 static u32 ip_fallback_id;
1296 u32 salt;
1297
1298 spin_lock_bh(&ip_fb_id_lock);
1299 salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1300 iph->id = htons(salt & 0xFFFF);
1301 ip_fallback_id = salt;
1302 spin_unlock_bh(&ip_fb_id_lock);
1303 }
1304
1305 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1306 {
1307 struct rtable *rt = (struct rtable *) dst;
1308
1309 if (rt) {
1310 if (rt->peer == NULL)
1311 rt_bind_peer(rt, 1);
1312
1313 /* If peer is attached to destination, it is never detached,
1314 so that we need not to grab a lock to dereference it.
1315 */
1316 if (rt->peer) {
1317 iph->id = htons(inet_getid(rt->peer, more));
1318 return;
1319 }
1320 } else
1321 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1322 __builtin_return_address(0));
1323
1324 ip_select_fb_ident(iph);
1325 }
1326 EXPORT_SYMBOL(__ip_select_ident);
1327
1328 static void rt_del(unsigned hash, struct rtable *rt)
1329 {
1330 struct rtable __rcu **rthp;
1331 struct rtable *aux;
1332
1333 rthp = &rt_hash_table[hash].chain;
1334 spin_lock_bh(rt_hash_lock_addr(hash));
1335 ip_rt_put(rt);
1336 while ((aux = rcu_dereference_protected(*rthp,
1337 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1338 if (aux == rt || rt_is_expired(aux)) {
1339 *rthp = aux->dst.rt_next;
1340 rt_free(aux);
1341 continue;
1342 }
1343 rthp = &aux->dst.rt_next;
1344 }
1345 spin_unlock_bh(rt_hash_lock_addr(hash));
1346 }
1347
1348 /* called in rcu_read_lock() section */
1349 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1350 __be32 saddr, struct net_device *dev)
1351 {
1352 int i, k;
1353 struct in_device *in_dev = __in_dev_get_rcu(dev);
1354 struct rtable *rth;
1355 struct rtable __rcu **rthp;
1356 __be32 skeys[2] = { saddr, 0 };
1357 int ikeys[2] = { dev->ifindex, 0 };
1358 struct netevent_redirect netevent;
1359 struct net *net;
1360
1361 if (!in_dev)
1362 return;
1363
1364 net = dev_net(dev);
1365 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1366 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1367 ipv4_is_zeronet(new_gw))
1368 goto reject_redirect;
1369
1370 if (!rt_caching(net))
1371 goto reject_redirect;
1372
1373 if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1374 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1375 goto reject_redirect;
1376 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1377 goto reject_redirect;
1378 } else {
1379 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1380 goto reject_redirect;
1381 }
1382
1383 for (i = 0; i < 2; i++) {
1384 for (k = 0; k < 2; k++) {
1385 unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1386 rt_genid(net));
1387
1388 rthp = &rt_hash_table[hash].chain;
1389
1390 while ((rth = rcu_dereference(*rthp)) != NULL) {
1391 struct rtable *rt;
1392
1393 if (rth->fl.fl4_dst != daddr ||
1394 rth->fl.fl4_src != skeys[i] ||
1395 rth->fl.oif != ikeys[k] ||
1396 rt_is_input_route(rth) ||
1397 rt_is_expired(rth) ||
1398 !net_eq(dev_net(rth->dst.dev), net)) {
1399 rthp = &rth->dst.rt_next;
1400 continue;
1401 }
1402
1403 if (rth->rt_dst != daddr ||
1404 rth->rt_src != saddr ||
1405 rth->dst.error ||
1406 rth->rt_gateway != old_gw ||
1407 rth->dst.dev != dev)
1408 break;
1409
1410 dst_hold(&rth->dst);
1411
1412 rt = dst_alloc(&ipv4_dst_ops);
1413 if (rt == NULL) {
1414 ip_rt_put(rth);
1415 return;
1416 }
1417
1418 /* Copy all the information. */
1419 *rt = *rth;
1420 rt->dst.__use = 1;
1421 atomic_set(&rt->dst.__refcnt, 1);
1422 rt->dst.child = NULL;
1423 if (rt->dst.dev)
1424 dev_hold(rt->dst.dev);
1425 rt->dst.obsolete = -1;
1426 rt->dst.lastuse = jiffies;
1427 rt->dst.path = &rt->dst;
1428 rt->dst.neighbour = NULL;
1429 rt->dst.hh = NULL;
1430 #ifdef CONFIG_XFRM
1431 rt->dst.xfrm = NULL;
1432 #endif
1433 rt->rt_genid = rt_genid(net);
1434 rt->rt_flags |= RTCF_REDIRECTED;
1435
1436 /* Gateway is different ... */
1437 rt->rt_gateway = new_gw;
1438
1439 /* Redirect received -> path was valid */
1440 dst_confirm(&rth->dst);
1441
1442 if (rt->peer)
1443 atomic_inc(&rt->peer->refcnt);
1444
1445 if (arp_bind_neighbour(&rt->dst) ||
1446 !(rt->dst.neighbour->nud_state &
1447 NUD_VALID)) {
1448 if (rt->dst.neighbour)
1449 neigh_event_send(rt->dst.neighbour, NULL);
1450 ip_rt_put(rth);
1451 rt_drop(rt);
1452 goto do_next;
1453 }
1454
1455 netevent.old = &rth->dst;
1456 netevent.new = &rt->dst;
1457 call_netevent_notifiers(NETEVENT_REDIRECT,
1458 &netevent);
1459
1460 rt_del(hash, rth);
1461 if (!rt_intern_hash(hash, rt, &rt, NULL, rt->fl.oif))
1462 ip_rt_put(rt);
1463 goto do_next;
1464 }
1465 do_next:
1466 ;
1467 }
1468 }
1469 return;
1470
1471 reject_redirect:
1472 #ifdef CONFIG_IP_ROUTE_VERBOSE
1473 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1474 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1475 " Advised path = %pI4 -> %pI4\n",
1476 &old_gw, dev->name, &new_gw,
1477 &saddr, &daddr);
1478 #endif
1479 ;
1480 }
1481
1482 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1483 {
1484 struct rtable *rt = (struct rtable *)dst;
1485 struct dst_entry *ret = dst;
1486
1487 if (rt) {
1488 if (dst->obsolete > 0) {
1489 ip_rt_put(rt);
1490 ret = NULL;
1491 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
1492 (rt->dst.expires &&
1493 time_after_eq(jiffies, rt->dst.expires))) {
1494 unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1495 rt->fl.oif,
1496 rt_genid(dev_net(dst->dev)));
1497 #if RT_CACHE_DEBUG >= 1
1498 printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
1499 &rt->rt_dst, rt->fl.fl4_tos);
1500 #endif
1501 rt_del(hash, rt);
1502 ret = NULL;
1503 }
1504 }
1505 return ret;
1506 }
1507
1508 /*
1509 * Algorithm:
1510 * 1. The first ip_rt_redirect_number redirects are sent
1511 * with exponential backoff, then we stop sending them at all,
1512 * assuming that the host ignores our redirects.
1513 * 2. If we did not see packets requiring redirects
1514 * during ip_rt_redirect_silence, we assume that the host
1515 * forgot redirected route and start to send redirects again.
1516 *
1517 * This algorithm is much cheaper and more intelligent than dumb load limiting
1518 * in icmp.c.
1519 *
1520 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1521 * and "frag. need" (breaks PMTU discovery) in icmp.c.
1522 */
1523
1524 void ip_rt_send_redirect(struct sk_buff *skb)
1525 {
1526 struct rtable *rt = skb_rtable(skb);
1527 struct in_device *in_dev;
1528 int log_martians;
1529
1530 rcu_read_lock();
1531 in_dev = __in_dev_get_rcu(rt->dst.dev);
1532 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1533 rcu_read_unlock();
1534 return;
1535 }
1536 log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1537 rcu_read_unlock();
1538
1539 /* No redirected packets during ip_rt_redirect_silence;
1540 * reset the algorithm.
1541 */
1542 if (time_after(jiffies, rt->dst.rate_last + ip_rt_redirect_silence))
1543 rt->dst.rate_tokens = 0;
1544
1545 /* Too many ignored redirects; do not send anything
1546 * set dst.rate_last to the last seen redirected packet.
1547 */
1548 if (rt->dst.rate_tokens >= ip_rt_redirect_number) {
1549 rt->dst.rate_last = jiffies;
1550 return;
1551 }
1552
1553 /* Check for load limit; set rate_last to the latest sent
1554 * redirect.
1555 */
1556 if (rt->dst.rate_tokens == 0 ||
1557 time_after(jiffies,
1558 (rt->dst.rate_last +
1559 (ip_rt_redirect_load << rt->dst.rate_tokens)))) {
1560 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1561 rt->dst.rate_last = jiffies;
1562 ++rt->dst.rate_tokens;
1563 #ifdef CONFIG_IP_ROUTE_VERBOSE
1564 if (log_martians &&
1565 rt->dst.rate_tokens == ip_rt_redirect_number &&
1566 net_ratelimit())
1567 printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1568 &rt->rt_src, rt->rt_iif,
1569 &rt->rt_dst, &rt->rt_gateway);
1570 #endif
1571 }
1572 }
1573
1574 static int ip_error(struct sk_buff *skb)
1575 {
1576 struct rtable *rt = skb_rtable(skb);
1577 unsigned long now;
1578 int code;
1579
1580 switch (rt->dst.error) {
1581 case EINVAL:
1582 default:
1583 goto out;
1584 case EHOSTUNREACH:
1585 code = ICMP_HOST_UNREACH;
1586 break;
1587 case ENETUNREACH:
1588 code = ICMP_NET_UNREACH;
1589 IP_INC_STATS_BH(dev_net(rt->dst.dev),
1590 IPSTATS_MIB_INNOROUTES);
1591 break;
1592 case EACCES:
1593 code = ICMP_PKT_FILTERED;
1594 break;
1595 }
1596
1597 now = jiffies;
1598 rt->dst.rate_tokens += now - rt->dst.rate_last;
1599 if (rt->dst.rate_tokens > ip_rt_error_burst)
1600 rt->dst.rate_tokens = ip_rt_error_burst;
1601 rt->dst.rate_last = now;
1602 if (rt->dst.rate_tokens >= ip_rt_error_cost) {
1603 rt->dst.rate_tokens -= ip_rt_error_cost;
1604 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1605 }
1606
1607 out: kfree_skb(skb);
1608 return 0;
1609 }
1610
1611 /*
1612 * The last two values are not from the RFC but
1613 * are needed for AMPRnet AX.25 paths.
1614 */
1615
1616 static const unsigned short mtu_plateau[] =
1617 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1618
1619 static inline unsigned short guess_mtu(unsigned short old_mtu)
1620 {
1621 int i;
1622
1623 for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1624 if (old_mtu > mtu_plateau[i])
1625 return mtu_plateau[i];
1626 return 68;
1627 }
1628
1629 unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
1630 unsigned short new_mtu,
1631 struct net_device *dev)
1632 {
1633 int i, k;
1634 unsigned short old_mtu = ntohs(iph->tot_len);
1635 struct rtable *rth;
1636 int ikeys[2] = { dev->ifindex, 0 };
1637 __be32 skeys[2] = { iph->saddr, 0, };
1638 __be32 daddr = iph->daddr;
1639 unsigned short est_mtu = 0;
1640
1641 for (k = 0; k < 2; k++) {
1642 for (i = 0; i < 2; i++) {
1643 unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1644 rt_genid(net));
1645
1646 rcu_read_lock();
1647 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
1648 rth = rcu_dereference(rth->dst.rt_next)) {
1649 unsigned short mtu = new_mtu;
1650
1651 if (rth->fl.fl4_dst != daddr ||
1652 rth->fl.fl4_src != skeys[i] ||
1653 rth->rt_dst != daddr ||
1654 rth->rt_src != iph->saddr ||
1655 rth->fl.oif != ikeys[k] ||
1656 rt_is_input_route(rth) ||
1657 dst_metric_locked(&rth->dst, RTAX_MTU) ||
1658 !net_eq(dev_net(rth->dst.dev), net) ||
1659 rt_is_expired(rth))
1660 continue;
1661
1662 if (new_mtu < 68 || new_mtu >= old_mtu) {
1663
1664 /* BSD 4.2 compatibility hack :-( */
1665 if (mtu == 0 &&
1666 old_mtu >= dst_mtu(&rth->dst) &&
1667 old_mtu >= 68 + (iph->ihl << 2))
1668 old_mtu -= iph->ihl << 2;
1669
1670 mtu = guess_mtu(old_mtu);
1671 }
1672 if (mtu <= dst_mtu(&rth->dst)) {
1673 if (mtu < dst_mtu(&rth->dst)) {
1674 dst_confirm(&rth->dst);
1675 if (mtu < ip_rt_min_pmtu) {
1676 u32 lock = dst_metric(&rth->dst,
1677 RTAX_LOCK);
1678 mtu = ip_rt_min_pmtu;
1679 lock |= (1 << RTAX_MTU);
1680 dst_metric_set(&rth->dst, RTAX_LOCK,
1681 lock);
1682 }
1683 dst_metric_set(&rth->dst, RTAX_MTU, mtu);
1684 dst_set_expires(&rth->dst,
1685 ip_rt_mtu_expires);
1686 }
1687 est_mtu = mtu;
1688 }
1689 }
1690 rcu_read_unlock();
1691 }
1692 }
1693 return est_mtu ? : new_mtu;
1694 }
1695
1696 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1697 {
1698 if (dst_mtu(dst) > mtu && mtu >= 68 &&
1699 !(dst_metric_locked(dst, RTAX_MTU))) {
1700 if (mtu < ip_rt_min_pmtu) {
1701 u32 lock = dst_metric(dst, RTAX_LOCK);
1702 mtu = ip_rt_min_pmtu;
1703 dst_metric_set(dst, RTAX_LOCK, lock | (1 << RTAX_MTU));
1704 }
1705 dst_metric_set(dst, RTAX_MTU, mtu);
1706 dst_set_expires(dst, ip_rt_mtu_expires);
1707 call_netevent_notifiers(NETEVENT_PMTU_UPDATE, dst);
1708 }
1709 }
1710
1711 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1712 {
1713 if (rt_is_expired((struct rtable *)dst))
1714 return NULL;
1715 return dst;
1716 }
1717
1718 static void ipv4_dst_destroy(struct dst_entry *dst)
1719 {
1720 struct rtable *rt = (struct rtable *) dst;
1721 struct inet_peer *peer = rt->peer;
1722
1723 if (peer) {
1724 rt->peer = NULL;
1725 inet_putpeer(peer);
1726 }
1727 }
1728
1729
1730 static void ipv4_link_failure(struct sk_buff *skb)
1731 {
1732 struct rtable *rt;
1733
1734 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1735
1736 rt = skb_rtable(skb);
1737 if (rt)
1738 dst_set_expires(&rt->dst, 0);
1739 }
1740
1741 static int ip_rt_bug(struct sk_buff *skb)
1742 {
1743 printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1744 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1745 skb->dev ? skb->dev->name : "?");
1746 kfree_skb(skb);
1747 return 0;
1748 }
1749
1750 /*
1751 We do not cache source address of outgoing interface,
1752 because it is used only by IP RR, TS and SRR options,
1753 so that it out of fast path.
1754
1755 BTW remember: "addr" is allowed to be not aligned
1756 in IP options!
1757 */
1758
1759 void ip_rt_get_source(u8 *addr, struct rtable *rt)
1760 {
1761 __be32 src;
1762 struct fib_result res;
1763
1764 if (rt_is_output_route(rt))
1765 src = rt->rt_src;
1766 else {
1767 rcu_read_lock();
1768 if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0)
1769 src = FIB_RES_PREFSRC(res);
1770 else
1771 src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1772 RT_SCOPE_UNIVERSE);
1773 rcu_read_unlock();
1774 }
1775 memcpy(addr, &src, 4);
1776 }
1777
1778 #ifdef CONFIG_NET_CLS_ROUTE
1779 static void set_class_tag(struct rtable *rt, u32 tag)
1780 {
1781 if (!(rt->dst.tclassid & 0xFFFF))
1782 rt->dst.tclassid |= tag & 0xFFFF;
1783 if (!(rt->dst.tclassid & 0xFFFF0000))
1784 rt->dst.tclassid |= tag & 0xFFFF0000;
1785 }
1786 #endif
1787
1788 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1789 {
1790 unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1791
1792 if (advmss == 0) {
1793 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1794 ip_rt_min_advmss);
1795 if (advmss > 65535 - 40)
1796 advmss = 65535 - 40;
1797 }
1798 return advmss;
1799 }
1800
1801 static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
1802 {
1803 unsigned int mtu = dst->dev->mtu;
1804
1805 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1806 const struct rtable *rt = (const struct rtable *) dst;
1807
1808 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1809 mtu = 576;
1810 }
1811
1812 if (mtu > IP_MAX_MTU)
1813 mtu = IP_MAX_MTU;
1814
1815 return mtu;
1816 }
1817
1818 static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
1819 {
1820 struct dst_entry *dst = &rt->dst;
1821 struct fib_info *fi = res->fi;
1822
1823 if (fi) {
1824 if (FIB_RES_GW(*res) &&
1825 FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1826 rt->rt_gateway = FIB_RES_GW(*res);
1827 dst_import_metrics(dst, fi->fib_metrics);
1828 #ifdef CONFIG_NET_CLS_ROUTE
1829 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1830 #endif
1831 }
1832
1833 if (dst_mtu(dst) > IP_MAX_MTU)
1834 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1835 if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40)
1836 dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
1837
1838 #ifdef CONFIG_NET_CLS_ROUTE
1839 #ifdef CONFIG_IP_MULTIPLE_TABLES
1840 set_class_tag(rt, fib_rules_tclass(res));
1841 #endif
1842 set_class_tag(rt, itag);
1843 #endif
1844 rt->rt_type = res->type;
1845 }
1846
1847 /* called in rcu_read_lock() section */
1848 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1849 u8 tos, struct net_device *dev, int our)
1850 {
1851 unsigned int hash;
1852 struct rtable *rth;
1853 __be32 spec_dst;
1854 struct in_device *in_dev = __in_dev_get_rcu(dev);
1855 u32 itag = 0;
1856 int err;
1857
1858 /* Primary sanity checks. */
1859
1860 if (in_dev == NULL)
1861 return -EINVAL;
1862
1863 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1864 ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
1865 goto e_inval;
1866
1867 if (ipv4_is_zeronet(saddr)) {
1868 if (!ipv4_is_local_multicast(daddr))
1869 goto e_inval;
1870 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1871 } else {
1872 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
1873 &itag, 0);
1874 if (err < 0)
1875 goto e_err;
1876 }
1877 rth = dst_alloc(&ipv4_dst_ops);
1878 if (!rth)
1879 goto e_nobufs;
1880
1881 rth->dst.output = ip_rt_bug;
1882 rth->dst.obsolete = -1;
1883
1884 atomic_set(&rth->dst.__refcnt, 1);
1885 rth->dst.flags= DST_HOST;
1886 if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
1887 rth->dst.flags |= DST_NOPOLICY;
1888 rth->fl.fl4_dst = daddr;
1889 rth->rt_dst = daddr;
1890 rth->fl.fl4_tos = tos;
1891 rth->fl.mark = skb->mark;
1892 rth->fl.fl4_src = saddr;
1893 rth->rt_src = saddr;
1894 #ifdef CONFIG_NET_CLS_ROUTE
1895 rth->dst.tclassid = itag;
1896 #endif
1897 rth->rt_iif =
1898 rth->fl.iif = dev->ifindex;
1899 rth->dst.dev = init_net.loopback_dev;
1900 dev_hold(rth->dst.dev);
1901 rth->fl.oif = 0;
1902 rth->rt_gateway = daddr;
1903 rth->rt_spec_dst= spec_dst;
1904 rth->rt_genid = rt_genid(dev_net(dev));
1905 rth->rt_flags = RTCF_MULTICAST;
1906 rth->rt_type = RTN_MULTICAST;
1907 if (our) {
1908 rth->dst.input= ip_local_deliver;
1909 rth->rt_flags |= RTCF_LOCAL;
1910 }
1911
1912 #ifdef CONFIG_IP_MROUTE
1913 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1914 rth->dst.input = ip_mr_input;
1915 #endif
1916 RT_CACHE_STAT_INC(in_slow_mc);
1917
1918 hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1919 return rt_intern_hash(hash, rth, NULL, skb, dev->ifindex);
1920
1921 e_nobufs:
1922 return -ENOBUFS;
1923 e_inval:
1924 return -EINVAL;
1925 e_err:
1926 return err;
1927 }
1928
1929
1930 static void ip_handle_martian_source(struct net_device *dev,
1931 struct in_device *in_dev,
1932 struct sk_buff *skb,
1933 __be32 daddr,
1934 __be32 saddr)
1935 {
1936 RT_CACHE_STAT_INC(in_martian_src);
1937 #ifdef CONFIG_IP_ROUTE_VERBOSE
1938 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1939 /*
1940 * RFC1812 recommendation, if source is martian,
1941 * the only hint is MAC header.
1942 */
1943 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1944 &daddr, &saddr, dev->name);
1945 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1946 int i;
1947 const unsigned char *p = skb_mac_header(skb);
1948 printk(KERN_WARNING "ll header: ");
1949 for (i = 0; i < dev->hard_header_len; i++, p++) {
1950 printk("%02x", *p);
1951 if (i < (dev->hard_header_len - 1))
1952 printk(":");
1953 }
1954 printk("\n");
1955 }
1956 }
1957 #endif
1958 }
1959
1960 /* called in rcu_read_lock() section */
1961 static int __mkroute_input(struct sk_buff *skb,
1962 struct fib_result *res,
1963 struct in_device *in_dev,
1964 __be32 daddr, __be32 saddr, u32 tos,
1965 struct rtable **result)
1966 {
1967 struct rtable *rth;
1968 int err;
1969 struct in_device *out_dev;
1970 unsigned int flags = 0;
1971 __be32 spec_dst;
1972 u32 itag;
1973
1974 /* get a working reference to the output device */
1975 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1976 if (out_dev == NULL) {
1977 if (net_ratelimit())
1978 printk(KERN_CRIT "Bug in ip_route_input" \
1979 "_slow(). Please, report\n");
1980 return -EINVAL;
1981 }
1982
1983
1984 err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
1985 in_dev->dev, &spec_dst, &itag, skb->mark);
1986 if (err < 0) {
1987 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1988 saddr);
1989
1990 goto cleanup;
1991 }
1992
1993 if (err)
1994 flags |= RTCF_DIRECTSRC;
1995
1996 if (out_dev == in_dev && err &&
1997 (IN_DEV_SHARED_MEDIA(out_dev) ||
1998 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1999 flags |= RTCF_DOREDIRECT;
2000
2001 if (skb->protocol != htons(ETH_P_IP)) {
2002 /* Not IP (i.e. ARP). Do not create route, if it is
2003 * invalid for proxy arp. DNAT routes are always valid.
2004 *
2005 * Proxy arp feature have been extended to allow, ARP
2006 * replies back to the same interface, to support
2007 * Private VLAN switch technologies. See arp.c.
2008 */
2009 if (out_dev == in_dev &&
2010 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2011 err = -EINVAL;
2012 goto cleanup;
2013 }
2014 }
2015
2016
2017 rth = dst_alloc(&ipv4_dst_ops);
2018 if (!rth) {
2019 err = -ENOBUFS;
2020 goto cleanup;
2021 }
2022
2023 atomic_set(&rth->dst.__refcnt, 1);
2024 rth->dst.flags= DST_HOST;
2025 if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2026 rth->dst.flags |= DST_NOPOLICY;
2027 if (IN_DEV_CONF_GET(out_dev, NOXFRM))
2028 rth->dst.flags |= DST_NOXFRM;
2029 rth->fl.fl4_dst = daddr;
2030 rth->rt_dst = daddr;
2031 rth->fl.fl4_tos = tos;
2032 rth->fl.mark = skb->mark;
2033 rth->fl.fl4_src = saddr;
2034 rth->rt_src = saddr;
2035 rth->rt_gateway = daddr;
2036 rth->rt_iif =
2037 rth->fl.iif = in_dev->dev->ifindex;
2038 rth->dst.dev = (out_dev)->dev;
2039 dev_hold(rth->dst.dev);
2040 rth->fl.oif = 0;
2041 rth->rt_spec_dst= spec_dst;
2042
2043 rth->dst.obsolete = -1;
2044 rth->dst.input = ip_forward;
2045 rth->dst.output = ip_output;
2046 rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2047
2048 rt_set_nexthop(rth, res, itag);
2049
2050 rth->rt_flags = flags;
2051
2052 *result = rth;
2053 err = 0;
2054 cleanup:
2055 return err;
2056 }
2057
2058 static int ip_mkroute_input(struct sk_buff *skb,
2059 struct fib_result *res,
2060 const struct flowi *fl,
2061 struct in_device *in_dev,
2062 __be32 daddr, __be32 saddr, u32 tos)
2063 {
2064 struct rtable* rth = NULL;
2065 int err;
2066 unsigned hash;
2067
2068 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2069 if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0)
2070 fib_select_multipath(fl, res);
2071 #endif
2072
2073 /* create a routing cache entry */
2074 err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2075 if (err)
2076 return err;
2077
2078 /* put it into the cache */
2079 hash = rt_hash(daddr, saddr, fl->iif,
2080 rt_genid(dev_net(rth->dst.dev)));
2081 return rt_intern_hash(hash, rth, NULL, skb, fl->iif);
2082 }
2083
2084 /*
2085 * NOTE. We drop all the packets that has local source
2086 * addresses, because every properly looped back packet
2087 * must have correct destination already attached by output routine.
2088 *
2089 * Such approach solves two big problems:
2090 * 1. Not simplex devices are handled properly.
2091 * 2. IP spoofing attempts are filtered with 100% of guarantee.
2092 * called with rcu_read_lock()
2093 */
2094
2095 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2096 u8 tos, struct net_device *dev)
2097 {
2098 struct fib_result res;
2099 struct in_device *in_dev = __in_dev_get_rcu(dev);
2100 struct flowi fl = { .fl4_dst = daddr,
2101 .fl4_src = saddr,
2102 .fl4_tos = tos,
2103 .fl4_scope = RT_SCOPE_UNIVERSE,
2104 .mark = skb->mark,
2105 .iif = dev->ifindex };
2106 unsigned flags = 0;
2107 u32 itag = 0;
2108 struct rtable * rth;
2109 unsigned hash;
2110 __be32 spec_dst;
2111 int err = -EINVAL;
2112 struct net * net = dev_net(dev);
2113
2114 /* IP on this device is disabled. */
2115
2116 if (!in_dev)
2117 goto out;
2118
2119 /* Check for the most weird martians, which can be not detected
2120 by fib_lookup.
2121 */
2122
2123 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2124 ipv4_is_loopback(saddr))
2125 goto martian_source;
2126
2127 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2128 goto brd_input;
2129
2130 /* Accept zero addresses only to limited broadcast;
2131 * I even do not know to fix it or not. Waiting for complains :-)
2132 */
2133 if (ipv4_is_zeronet(saddr))
2134 goto martian_source;
2135
2136 if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
2137 goto martian_destination;
2138
2139 /*
2140 * Now we are ready to route packet.
2141 */
2142 err = fib_lookup(net, &fl, &res);
2143 if (err != 0) {
2144 if (!IN_DEV_FORWARD(in_dev))
2145 goto e_hostunreach;
2146 goto no_route;
2147 }
2148
2149 RT_CACHE_STAT_INC(in_slow_tot);
2150
2151 if (res.type == RTN_BROADCAST)
2152 goto brd_input;
2153
2154 if (res.type == RTN_LOCAL) {
2155 err = fib_validate_source(saddr, daddr, tos,
2156 net->loopback_dev->ifindex,
2157 dev, &spec_dst, &itag, skb->mark);
2158 if (err < 0)
2159 goto martian_source_keep_err;
2160 if (err)
2161 flags |= RTCF_DIRECTSRC;
2162 spec_dst = daddr;
2163 goto local_input;
2164 }
2165
2166 if (!IN_DEV_FORWARD(in_dev))
2167 goto e_hostunreach;
2168 if (res.type != RTN_UNICAST)
2169 goto martian_destination;
2170
2171 err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos);
2172 out: return err;
2173
2174 brd_input:
2175 if (skb->protocol != htons(ETH_P_IP))
2176 goto e_inval;
2177
2178 if (ipv4_is_zeronet(saddr))
2179 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2180 else {
2181 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
2182 &itag, skb->mark);
2183 if (err < 0)
2184 goto martian_source_keep_err;
2185 if (err)
2186 flags |= RTCF_DIRECTSRC;
2187 }
2188 flags |= RTCF_BROADCAST;
2189 res.type = RTN_BROADCAST;
2190 RT_CACHE_STAT_INC(in_brd);
2191
2192 local_input:
2193 rth = dst_alloc(&ipv4_dst_ops);
2194 if (!rth)
2195 goto e_nobufs;
2196
2197 rth->dst.output= ip_rt_bug;
2198 rth->dst.obsolete = -1;
2199 rth->rt_genid = rt_genid(net);
2200
2201 atomic_set(&rth->dst.__refcnt, 1);
2202 rth->dst.flags= DST_HOST;
2203 if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2204 rth->dst.flags |= DST_NOPOLICY;
2205 rth->fl.fl4_dst = daddr;
2206 rth->rt_dst = daddr;
2207 rth->fl.fl4_tos = tos;
2208 rth->fl.mark = skb->mark;
2209 rth->fl.fl4_src = saddr;
2210 rth->rt_src = saddr;
2211 #ifdef CONFIG_NET_CLS_ROUTE
2212 rth->dst.tclassid = itag;
2213 #endif
2214 rth->rt_iif =
2215 rth->fl.iif = dev->ifindex;
2216 rth->dst.dev = net->loopback_dev;
2217 dev_hold(rth->dst.dev);
2218 rth->rt_gateway = daddr;
2219 rth->rt_spec_dst= spec_dst;
2220 rth->dst.input= ip_local_deliver;
2221 rth->rt_flags = flags|RTCF_LOCAL;
2222 if (res.type == RTN_UNREACHABLE) {
2223 rth->dst.input= ip_error;
2224 rth->dst.error= -err;
2225 rth->rt_flags &= ~RTCF_LOCAL;
2226 }
2227 rth->rt_type = res.type;
2228 hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net));
2229 err = rt_intern_hash(hash, rth, NULL, skb, fl.iif);
2230 goto out;
2231
2232 no_route:
2233 RT_CACHE_STAT_INC(in_no_route);
2234 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2235 res.type = RTN_UNREACHABLE;
2236 if (err == -ESRCH)
2237 err = -ENETUNREACH;
2238 goto local_input;
2239
2240 /*
2241 * Do not cache martian addresses: they should be logged (RFC1812)
2242 */
2243 martian_destination:
2244 RT_CACHE_STAT_INC(in_martian_dst);
2245 #ifdef CONFIG_IP_ROUTE_VERBOSE
2246 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2247 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2248 &daddr, &saddr, dev->name);
2249 #endif
2250
2251 e_hostunreach:
2252 err = -EHOSTUNREACH;
2253 goto out;
2254
2255 e_inval:
2256 err = -EINVAL;
2257 goto out;
2258
2259 e_nobufs:
2260 err = -ENOBUFS;
2261 goto out;
2262
2263 martian_source:
2264 err = -EINVAL;
2265 martian_source_keep_err:
2266 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2267 goto out;
2268 }
2269
2270 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2271 u8 tos, struct net_device *dev, bool noref)
2272 {
2273 struct rtable * rth;
2274 unsigned hash;
2275 int iif = dev->ifindex;
2276 struct net *net;
2277 int res;
2278
2279 net = dev_net(dev);
2280
2281 rcu_read_lock();
2282
2283 if (!rt_caching(net))
2284 goto skip_cache;
2285
2286 tos &= IPTOS_RT_MASK;
2287 hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2288
2289 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2290 rth = rcu_dereference(rth->dst.rt_next)) {
2291 if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) |
2292 ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) |
2293 (rth->fl.iif ^ iif) |
2294 rth->fl.oif |
2295 (rth->fl.fl4_tos ^ tos)) == 0 &&
2296 rth->fl.mark == skb->mark &&
2297 net_eq(dev_net(rth->dst.dev), net) &&
2298 !rt_is_expired(rth)) {
2299 if (noref) {
2300 dst_use_noref(&rth->dst, jiffies);
2301 skb_dst_set_noref(skb, &rth->dst);
2302 } else {
2303 dst_use(&rth->dst, jiffies);
2304 skb_dst_set(skb, &rth->dst);
2305 }
2306 RT_CACHE_STAT_INC(in_hit);
2307 rcu_read_unlock();
2308 return 0;
2309 }
2310 RT_CACHE_STAT_INC(in_hlist_search);
2311 }
2312
2313 skip_cache:
2314 /* Multicast recognition logic is moved from route cache to here.
2315 The problem was that too many Ethernet cards have broken/missing
2316 hardware multicast filters :-( As result the host on multicasting
2317 network acquires a lot of useless route cache entries, sort of
2318 SDR messages from all the world. Now we try to get rid of them.
2319 Really, provided software IP multicast filter is organized
2320 reasonably (at least, hashed), it does not result in a slowdown
2321 comparing with route cache reject entries.
2322 Note, that multicast routers are not affected, because
2323 route cache entry is created eventually.
2324 */
2325 if (ipv4_is_multicast(daddr)) {
2326 struct in_device *in_dev = __in_dev_get_rcu(dev);
2327
2328 if (in_dev) {
2329 int our = ip_check_mc(in_dev, daddr, saddr,
2330 ip_hdr(skb)->protocol);
2331 if (our
2332 #ifdef CONFIG_IP_MROUTE
2333 ||
2334 (!ipv4_is_local_multicast(daddr) &&
2335 IN_DEV_MFORWARD(in_dev))
2336 #endif
2337 ) {
2338 int res = ip_route_input_mc(skb, daddr, saddr,
2339 tos, dev, our);
2340 rcu_read_unlock();
2341 return res;
2342 }
2343 }
2344 rcu_read_unlock();
2345 return -EINVAL;
2346 }
2347 res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2348 rcu_read_unlock();
2349 return res;
2350 }
2351 EXPORT_SYMBOL(ip_route_input_common);
2352
2353 /* called with rcu_read_lock() */
2354 static int __mkroute_output(struct rtable **result,
2355 struct fib_result *res,
2356 const struct flowi *fl,
2357 const struct flowi *oldflp,
2358 struct net_device *dev_out,
2359 unsigned flags)
2360 {
2361 struct rtable *rth;
2362 struct in_device *in_dev;
2363 u32 tos = RT_FL_TOS(oldflp);
2364
2365 if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))
2366 return -EINVAL;
2367
2368 if (ipv4_is_lbcast(fl->fl4_dst))
2369 res->type = RTN_BROADCAST;
2370 else if (ipv4_is_multicast(fl->fl4_dst))
2371 res->type = RTN_MULTICAST;
2372 else if (ipv4_is_zeronet(fl->fl4_dst))
2373 return -EINVAL;
2374
2375 if (dev_out->flags & IFF_LOOPBACK)
2376 flags |= RTCF_LOCAL;
2377
2378 in_dev = __in_dev_get_rcu(dev_out);
2379 if (!in_dev)
2380 return -EINVAL;
2381
2382 if (res->type == RTN_BROADCAST) {
2383 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2384 res->fi = NULL;
2385 } else if (res->type == RTN_MULTICAST) {
2386 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2387 if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src,
2388 oldflp->proto))
2389 flags &= ~RTCF_LOCAL;
2390 /* If multicast route do not exist use
2391 * default one, but do not gateway in this case.
2392 * Yes, it is hack.
2393 */
2394 if (res->fi && res->prefixlen < 4)
2395 res->fi = NULL;
2396 }
2397
2398
2399 rth = dst_alloc(&ipv4_dst_ops);
2400 if (!rth)
2401 return -ENOBUFS;
2402
2403 atomic_set(&rth->dst.__refcnt, 1);
2404 rth->dst.flags= DST_HOST;
2405 if (IN_DEV_CONF_GET(in_dev, NOXFRM))
2406 rth->dst.flags |= DST_NOXFRM;
2407 if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2408 rth->dst.flags |= DST_NOPOLICY;
2409
2410 rth->fl.fl4_dst = oldflp->fl4_dst;
2411 rth->fl.fl4_tos = tos;
2412 rth->fl.fl4_src = oldflp->fl4_src;
2413 rth->fl.oif = oldflp->oif;
2414 rth->fl.mark = oldflp->mark;
2415 rth->rt_dst = fl->fl4_dst;
2416 rth->rt_src = fl->fl4_src;
2417 rth->rt_iif = oldflp->oif ? : dev_out->ifindex;
2418 /* get references to the devices that are to be hold by the routing
2419 cache entry */
2420 rth->dst.dev = dev_out;
2421 dev_hold(dev_out);
2422 rth->rt_gateway = fl->fl4_dst;
2423 rth->rt_spec_dst= fl->fl4_src;
2424
2425 rth->dst.output=ip_output;
2426 rth->dst.obsolete = -1;
2427 rth->rt_genid = rt_genid(dev_net(dev_out));
2428
2429 RT_CACHE_STAT_INC(out_slow_tot);
2430
2431 if (flags & RTCF_LOCAL) {
2432 rth->dst.input = ip_local_deliver;
2433 rth->rt_spec_dst = fl->fl4_dst;
2434 }
2435 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2436 rth->rt_spec_dst = fl->fl4_src;
2437 if (flags & RTCF_LOCAL &&
2438 !(dev_out->flags & IFF_LOOPBACK)) {
2439 rth->dst.output = ip_mc_output;
2440 RT_CACHE_STAT_INC(out_slow_mc);
2441 }
2442 #ifdef CONFIG_IP_MROUTE
2443 if (res->type == RTN_MULTICAST) {
2444 if (IN_DEV_MFORWARD(in_dev) &&
2445 !ipv4_is_local_multicast(oldflp->fl4_dst)) {
2446 rth->dst.input = ip_mr_input;
2447 rth->dst.output = ip_mc_output;
2448 }
2449 }
2450 #endif
2451 }
2452
2453 rt_set_nexthop(rth, res, 0);
2454
2455 rth->rt_flags = flags;
2456 *result = rth;
2457 return 0;
2458 }
2459
2460 /* called with rcu_read_lock() */
2461 static int ip_mkroute_output(struct rtable **rp,
2462 struct fib_result *res,
2463 const struct flowi *fl,
2464 const struct flowi *oldflp,
2465 struct net_device *dev_out,
2466 unsigned flags)
2467 {
2468 struct rtable *rth = NULL;
2469 int err = __mkroute_output(&rth, res, fl, oldflp, dev_out, flags);
2470 unsigned hash;
2471 if (err == 0) {
2472 hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif,
2473 rt_genid(dev_net(dev_out)));
2474 err = rt_intern_hash(hash, rth, rp, NULL, oldflp->oif);
2475 }
2476
2477 return err;
2478 }
2479
2480 /*
2481 * Major route resolver routine.
2482 * called with rcu_read_lock();
2483 */
2484
2485 static int ip_route_output_slow(struct net *net, struct rtable **rp,
2486 const struct flowi *oldflp)
2487 {
2488 u32 tos = RT_FL_TOS(oldflp);
2489 struct flowi fl = { .fl4_dst = oldflp->fl4_dst,
2490 .fl4_src = oldflp->fl4_src,
2491 .fl4_tos = tos & IPTOS_RT_MASK,
2492 .fl4_scope = ((tos & RTO_ONLINK) ?
2493 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE),
2494 .mark = oldflp->mark,
2495 .iif = net->loopback_dev->ifindex,
2496 .oif = oldflp->oif };
2497 struct fib_result res;
2498 unsigned int flags = 0;
2499 struct net_device *dev_out = NULL;
2500 int err;
2501
2502
2503 res.fi = NULL;
2504 #ifdef CONFIG_IP_MULTIPLE_TABLES
2505 res.r = NULL;
2506 #endif
2507
2508 if (oldflp->fl4_src) {
2509 err = -EINVAL;
2510 if (ipv4_is_multicast(oldflp->fl4_src) ||
2511 ipv4_is_lbcast(oldflp->fl4_src) ||
2512 ipv4_is_zeronet(oldflp->fl4_src))
2513 goto out;
2514
2515 /* I removed check for oif == dev_out->oif here.
2516 It was wrong for two reasons:
2517 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2518 is assigned to multiple interfaces.
2519 2. Moreover, we are allowed to send packets with saddr
2520 of another iface. --ANK
2521 */
2522
2523 if (oldflp->oif == 0 &&
2524 (ipv4_is_multicast(oldflp->fl4_dst) ||
2525 ipv4_is_lbcast(oldflp->fl4_dst))) {
2526 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2527 dev_out = __ip_dev_find(net, oldflp->fl4_src, false);
2528 if (dev_out == NULL)
2529 goto out;
2530
2531 /* Special hack: user can direct multicasts
2532 and limited broadcast via necessary interface
2533 without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2534 This hack is not just for fun, it allows
2535 vic,vat and friends to work.
2536 They bind socket to loopback, set ttl to zero
2537 and expect that it will work.
2538 From the viewpoint of routing cache they are broken,
2539 because we are not allowed to build multicast path
2540 with loopback source addr (look, routing cache
2541 cannot know, that ttl is zero, so that packet
2542 will not leave this host and route is valid).
2543 Luckily, this hack is good workaround.
2544 */
2545
2546 fl.oif = dev_out->ifindex;
2547 goto make_route;
2548 }
2549
2550 if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) {
2551 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2552 if (!__ip_dev_find(net, oldflp->fl4_src, false))
2553 goto out;
2554 }
2555 }
2556
2557
2558 if (oldflp->oif) {
2559 dev_out = dev_get_by_index_rcu(net, oldflp->oif);
2560 err = -ENODEV;
2561 if (dev_out == NULL)
2562 goto out;
2563
2564 /* RACE: Check return value of inet_select_addr instead. */
2565 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2566 err = -ENETUNREACH;
2567 goto out;
2568 }
2569 if (ipv4_is_local_multicast(oldflp->fl4_dst) ||
2570 ipv4_is_lbcast(oldflp->fl4_dst)) {
2571 if (!fl.fl4_src)
2572 fl.fl4_src = inet_select_addr(dev_out, 0,
2573 RT_SCOPE_LINK);
2574 goto make_route;
2575 }
2576 if (!fl.fl4_src) {
2577 if (ipv4_is_multicast(oldflp->fl4_dst))
2578 fl.fl4_src = inet_select_addr(dev_out, 0,
2579 fl.fl4_scope);
2580 else if (!oldflp->fl4_dst)
2581 fl.fl4_src = inet_select_addr(dev_out, 0,
2582 RT_SCOPE_HOST);
2583 }
2584 }
2585
2586 if (!fl.fl4_dst) {
2587 fl.fl4_dst = fl.fl4_src;
2588 if (!fl.fl4_dst)
2589 fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK);
2590 dev_out = net->loopback_dev;
2591 fl.oif = net->loopback_dev->ifindex;
2592 res.type = RTN_LOCAL;
2593 flags |= RTCF_LOCAL;
2594 goto make_route;
2595 }
2596
2597 if (fib_lookup(net, &fl, &res)) {
2598 res.fi = NULL;
2599 if (oldflp->oif) {
2600 /* Apparently, routing tables are wrong. Assume,
2601 that the destination is on link.
2602
2603 WHY? DW.
2604 Because we are allowed to send to iface
2605 even if it has NO routes and NO assigned
2606 addresses. When oif is specified, routing
2607 tables are looked up with only one purpose:
2608 to catch if destination is gatewayed, rather than
2609 direct. Moreover, if MSG_DONTROUTE is set,
2610 we send packet, ignoring both routing tables
2611 and ifaddr state. --ANK
2612
2613
2614 We could make it even if oif is unknown,
2615 likely IPv6, but we do not.
2616 */
2617
2618 if (fl.fl4_src == 0)
2619 fl.fl4_src = inet_select_addr(dev_out, 0,
2620 RT_SCOPE_LINK);
2621 res.type = RTN_UNICAST;
2622 goto make_route;
2623 }
2624 err = -ENETUNREACH;
2625 goto out;
2626 }
2627
2628 if (res.type == RTN_LOCAL) {
2629 if (!fl.fl4_src) {
2630 if (res.fi->fib_prefsrc)
2631 fl.fl4_src = res.fi->fib_prefsrc;
2632 else
2633 fl.fl4_src = fl.fl4_dst;
2634 }
2635 dev_out = net->loopback_dev;
2636 fl.oif = dev_out->ifindex;
2637 res.fi = NULL;
2638 flags |= RTCF_LOCAL;
2639 goto make_route;
2640 }
2641
2642 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2643 if (res.fi->fib_nhs > 1 && fl.oif == 0)
2644 fib_select_multipath(&fl, &res);
2645 else
2646 #endif
2647 if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif)
2648 fib_select_default(net, &fl, &res);
2649
2650 if (!fl.fl4_src)
2651 fl.fl4_src = FIB_RES_PREFSRC(res);
2652
2653 dev_out = FIB_RES_DEV(res);
2654 fl.oif = dev_out->ifindex;
2655
2656
2657 make_route:
2658 err = ip_mkroute_output(rp, &res, &fl, oldflp, dev_out, flags);
2659
2660 out: return err;
2661 }
2662
2663 int __ip_route_output_key(struct net *net, struct rtable **rp,
2664 const struct flowi *flp)
2665 {
2666 unsigned int hash;
2667 int res;
2668 struct rtable *rth;
2669
2670 if (!rt_caching(net))
2671 goto slow_output;
2672
2673 hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net));
2674
2675 rcu_read_lock_bh();
2676 for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2677 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2678 if (rth->fl.fl4_dst == flp->fl4_dst &&
2679 rth->fl.fl4_src == flp->fl4_src &&
2680 rt_is_output_route(rth) &&
2681 rth->fl.oif == flp->oif &&
2682 rth->fl.mark == flp->mark &&
2683 !((rth->fl.fl4_tos ^ flp->fl4_tos) &
2684 (IPTOS_RT_MASK | RTO_ONLINK)) &&
2685 net_eq(dev_net(rth->dst.dev), net) &&
2686 !rt_is_expired(rth)) {
2687 dst_use(&rth->dst, jiffies);
2688 RT_CACHE_STAT_INC(out_hit);
2689 rcu_read_unlock_bh();
2690 *rp = rth;
2691 return 0;
2692 }
2693 RT_CACHE_STAT_INC(out_hlist_search);
2694 }
2695 rcu_read_unlock_bh();
2696
2697 slow_output:
2698 rcu_read_lock();
2699 res = ip_route_output_slow(net, rp, flp);
2700 rcu_read_unlock();
2701 return res;
2702 }
2703 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2704
2705 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2706 {
2707 return NULL;
2708 }
2709
2710 static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst)
2711 {
2712 return 0;
2713 }
2714
2715 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2716 {
2717 }
2718
2719 static struct dst_ops ipv4_dst_blackhole_ops = {
2720 .family = AF_INET,
2721 .protocol = cpu_to_be16(ETH_P_IP),
2722 .destroy = ipv4_dst_destroy,
2723 .check = ipv4_blackhole_dst_check,
2724 .default_mtu = ipv4_blackhole_default_mtu,
2725 .default_advmss = ipv4_default_advmss,
2726 .update_pmtu = ipv4_rt_blackhole_update_pmtu,
2727 };
2728
2729
2730 static int ipv4_dst_blackhole(struct net *net, struct rtable **rp, struct flowi *flp)
2731 {
2732 struct rtable *ort = *rp;
2733 struct rtable *rt = (struct rtable *)
2734 dst_alloc(&ipv4_dst_blackhole_ops);
2735
2736 if (rt) {
2737 struct dst_entry *new = &rt->dst;
2738
2739 atomic_set(&new->__refcnt, 1);
2740 new->__use = 1;
2741 new->input = dst_discard;
2742 new->output = dst_discard;
2743 dst_copy_metrics(new, &ort->dst);
2744
2745 new->dev = ort->dst.dev;
2746 if (new->dev)
2747 dev_hold(new->dev);
2748
2749 rt->fl = ort->fl;
2750
2751 rt->rt_genid = rt_genid(net);
2752 rt->rt_flags = ort->rt_flags;
2753 rt->rt_type = ort->rt_type;
2754 rt->rt_dst = ort->rt_dst;
2755 rt->rt_src = ort->rt_src;
2756 rt->rt_iif = ort->rt_iif;
2757 rt->rt_gateway = ort->rt_gateway;
2758 rt->rt_spec_dst = ort->rt_spec_dst;
2759 rt->peer = ort->peer;
2760 if (rt->peer)
2761 atomic_inc(&rt->peer->refcnt);
2762
2763 dst_free(new);
2764 }
2765
2766 dst_release(&(*rp)->dst);
2767 *rp = rt;
2768 return rt ? 0 : -ENOMEM;
2769 }
2770
2771 int ip_route_output_flow(struct net *net, struct rtable **rp, struct flowi *flp,
2772 struct sock *sk, int flags)
2773 {
2774 int err;
2775
2776 if ((err = __ip_route_output_key(net, rp, flp)) != 0)
2777 return err;
2778
2779 if (flp->proto) {
2780 if (!flp->fl4_src)
2781 flp->fl4_src = (*rp)->rt_src;
2782 if (!flp->fl4_dst)
2783 flp->fl4_dst = (*rp)->rt_dst;
2784 err = __xfrm_lookup(net, (struct dst_entry **)rp, flp, sk,
2785 flags ? XFRM_LOOKUP_WAIT : 0);
2786 if (err == -EREMOTE)
2787 err = ipv4_dst_blackhole(net, rp, flp);
2788
2789 return err;
2790 }
2791
2792 return 0;
2793 }
2794 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2795
2796 int ip_route_output_key(struct net *net, struct rtable **rp, struct flowi *flp)
2797 {
2798 return ip_route_output_flow(net, rp, flp, NULL, 0);
2799 }
2800 EXPORT_SYMBOL(ip_route_output_key);
2801
2802 static int rt_fill_info(struct net *net,
2803 struct sk_buff *skb, u32 pid, u32 seq, int event,
2804 int nowait, unsigned int flags)
2805 {
2806 struct rtable *rt = skb_rtable(skb);
2807 struct rtmsg *r;
2808 struct nlmsghdr *nlh;
2809 long expires;
2810 u32 id = 0, ts = 0, tsage = 0, error;
2811
2812 nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2813 if (nlh == NULL)
2814 return -EMSGSIZE;
2815
2816 r = nlmsg_data(nlh);
2817 r->rtm_family = AF_INET;
2818 r->rtm_dst_len = 32;
2819 r->rtm_src_len = 0;
2820 r->rtm_tos = rt->fl.fl4_tos;
2821 r->rtm_table = RT_TABLE_MAIN;
2822 NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
2823 r->rtm_type = rt->rt_type;
2824 r->rtm_scope = RT_SCOPE_UNIVERSE;
2825 r->rtm_protocol = RTPROT_UNSPEC;
2826 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2827 if (rt->rt_flags & RTCF_NOTIFY)
2828 r->rtm_flags |= RTM_F_NOTIFY;
2829
2830 NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2831
2832 if (rt->fl.fl4_src) {
2833 r->rtm_src_len = 32;
2834 NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src);
2835 }
2836 if (rt->dst.dev)
2837 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2838 #ifdef CONFIG_NET_CLS_ROUTE
2839 if (rt->dst.tclassid)
2840 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
2841 #endif
2842 if (rt_is_input_route(rt))
2843 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2844 else if (rt->rt_src != rt->fl.fl4_src)
2845 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2846
2847 if (rt->rt_dst != rt->rt_gateway)
2848 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2849
2850 if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2851 goto nla_put_failure;
2852
2853 if (rt->fl.mark)
2854 NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark);
2855
2856 error = rt->dst.error;
2857 expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
2858 if (rt->peer) {
2859 inet_peer_refcheck(rt->peer);
2860 id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
2861 if (rt->peer->tcp_ts_stamp) {
2862 ts = rt->peer->tcp_ts;
2863 tsage = get_seconds() - rt->peer->tcp_ts_stamp;
2864 }
2865 }
2866
2867 if (rt_is_input_route(rt)) {
2868 #ifdef CONFIG_IP_MROUTE
2869 __be32 dst = rt->rt_dst;
2870
2871 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2872 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2873 int err = ipmr_get_route(net, skb, r, nowait);
2874 if (err <= 0) {
2875 if (!nowait) {
2876 if (err == 0)
2877 return 0;
2878 goto nla_put_failure;
2879 } else {
2880 if (err == -EMSGSIZE)
2881 goto nla_put_failure;
2882 error = err;
2883 }
2884 }
2885 } else
2886 #endif
2887 NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif);
2888 }
2889
2890 if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2891 expires, error) < 0)
2892 goto nla_put_failure;
2893
2894 return nlmsg_end(skb, nlh);
2895
2896 nla_put_failure:
2897 nlmsg_cancel(skb, nlh);
2898 return -EMSGSIZE;
2899 }
2900
2901 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
2902 {
2903 struct net *net = sock_net(in_skb->sk);
2904 struct rtmsg *rtm;
2905 struct nlattr *tb[RTA_MAX+1];
2906 struct rtable *rt = NULL;
2907 __be32 dst = 0;
2908 __be32 src = 0;
2909 u32 iif;
2910 int err;
2911 int mark;
2912 struct sk_buff *skb;
2913
2914 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2915 if (err < 0)
2916 goto errout;
2917
2918 rtm = nlmsg_data(nlh);
2919
2920 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2921 if (skb == NULL) {
2922 err = -ENOBUFS;
2923 goto errout;
2924 }
2925
2926 /* Reserve room for dummy headers, this skb can pass
2927 through good chunk of routing engine.
2928 */
2929 skb_reset_mac_header(skb);
2930 skb_reset_network_header(skb);
2931
2932 /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2933 ip_hdr(skb)->protocol = IPPROTO_ICMP;
2934 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2935
2936 src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2937 dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2938 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2939 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2940
2941 if (iif) {
2942 struct net_device *dev;
2943
2944 dev = __dev_get_by_index(net, iif);
2945 if (dev == NULL) {
2946 err = -ENODEV;
2947 goto errout_free;
2948 }
2949
2950 skb->protocol = htons(ETH_P_IP);
2951 skb->dev = dev;
2952 skb->mark = mark;
2953 local_bh_disable();
2954 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2955 local_bh_enable();
2956
2957 rt = skb_rtable(skb);
2958 if (err == 0 && rt->dst.error)
2959 err = -rt->dst.error;
2960 } else {
2961 struct flowi fl = {
2962 .fl4_dst = dst,
2963 .fl4_src = src,
2964 .fl4_tos = rtm->rtm_tos,
2965 .oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2966 .mark = mark,
2967 };
2968 err = ip_route_output_key(net, &rt, &fl);
2969 }
2970
2971 if (err)
2972 goto errout_free;
2973
2974 skb_dst_set(skb, &rt->dst);
2975 if (rtm->rtm_flags & RTM_F_NOTIFY)
2976 rt->rt_flags |= RTCF_NOTIFY;
2977
2978 err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2979 RTM_NEWROUTE, 0, 0);
2980 if (err <= 0)
2981 goto errout_free;
2982
2983 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2984 errout:
2985 return err;
2986
2987 errout_free:
2988 kfree_skb(skb);
2989 goto errout;
2990 }
2991
2992 int ip_rt_dump(struct sk_buff *skb, struct netlink_callback *cb)
2993 {
2994 struct rtable *rt;
2995 int h, s_h;
2996 int idx, s_idx;
2997 struct net *net;
2998
2999 net = sock_net(skb->sk);
3000
3001 s_h = cb->args[0];
3002 if (s_h < 0)
3003 s_h = 0;
3004 s_idx = idx = cb->args[1];
3005 for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3006 if (!rt_hash_table[h].chain)
3007 continue;
3008 rcu_read_lock_bh();
3009 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
3010 rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3011 if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3012 continue;
3013 if (rt_is_expired(rt))
3014 continue;
3015 skb_dst_set_noref(skb, &rt->dst);
3016 if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3017 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3018 1, NLM_F_MULTI) <= 0) {
3019 skb_dst_drop(skb);
3020 rcu_read_unlock_bh();
3021 goto done;
3022 }
3023 skb_dst_drop(skb);
3024 }
3025 rcu_read_unlock_bh();
3026 }
3027
3028 done:
3029 cb->args[0] = h;
3030 cb->args[1] = idx;
3031 return skb->len;
3032 }
3033
3034 void ip_rt_multicast_event(struct in_device *in_dev)
3035 {
3036 rt_cache_flush(dev_net(in_dev->dev), 0);
3037 }
3038
3039 #ifdef CONFIG_SYSCTL
3040 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3041 void __user *buffer,
3042 size_t *lenp, loff_t *ppos)
3043 {
3044 if (write) {
3045 int flush_delay;
3046 ctl_table ctl;
3047 struct net *net;
3048
3049 memcpy(&ctl, __ctl, sizeof(ctl));
3050 ctl.data = &flush_delay;
3051 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3052
3053 net = (struct net *)__ctl->extra1;
3054 rt_cache_flush(net, flush_delay);
3055 return 0;
3056 }
3057
3058 return -EINVAL;
3059 }
3060
3061 static ctl_table ipv4_route_table[] = {
3062 {
3063 .procname = "gc_thresh",
3064 .data = &ipv4_dst_ops.gc_thresh,
3065 .maxlen = sizeof(int),
3066 .mode = 0644,
3067 .proc_handler = proc_dointvec,
3068 },
3069 {
3070 .procname = "max_size",
3071 .data = &ip_rt_max_size,
3072 .maxlen = sizeof(int),
3073 .mode = 0644,
3074 .proc_handler = proc_dointvec,
3075 },
3076 {
3077 /* Deprecated. Use gc_min_interval_ms */
3078
3079 .procname = "gc_min_interval",
3080 .data = &ip_rt_gc_min_interval,
3081 .maxlen = sizeof(int),
3082 .mode = 0644,
3083 .proc_handler = proc_dointvec_jiffies,
3084 },
3085 {
3086 .procname = "gc_min_interval_ms",
3087 .data = &ip_rt_gc_min_interval,
3088 .maxlen = sizeof(int),
3089 .mode = 0644,
3090 .proc_handler = proc_dointvec_ms_jiffies,
3091 },
3092 {
3093 .procname = "gc_timeout",
3094 .data = &ip_rt_gc_timeout,
3095 .maxlen = sizeof(int),
3096 .mode = 0644,
3097 .proc_handler = proc_dointvec_jiffies,
3098 },
3099 {
3100 .procname = "gc_interval",
3101 .data = &ip_rt_gc_interval,
3102 .maxlen = sizeof(int),
3103 .mode = 0644,
3104 .proc_handler = proc_dointvec_jiffies,
3105 },
3106 {
3107 .procname = "redirect_load",
3108 .data = &ip_rt_redirect_load,
3109 .maxlen = sizeof(int),
3110 .mode = 0644,
3111 .proc_handler = proc_dointvec,
3112 },
3113 {
3114 .procname = "redirect_number",
3115 .data = &ip_rt_redirect_number,
3116 .maxlen = sizeof(int),
3117 .mode = 0644,
3118 .proc_handler = proc_dointvec,
3119 },
3120 {
3121 .procname = "redirect_silence",
3122 .data = &ip_rt_redirect_silence,
3123 .maxlen = sizeof(int),
3124 .mode = 0644,
3125 .proc_handler = proc_dointvec,
3126 },
3127 {
3128 .procname = "error_cost",
3129 .data = &ip_rt_error_cost,
3130 .maxlen = sizeof(int),
3131 .mode = 0644,
3132 .proc_handler = proc_dointvec,
3133 },
3134 {
3135 .procname = "error_burst",
3136 .data = &ip_rt_error_burst,
3137 .maxlen = sizeof(int),
3138 .mode = 0644,
3139 .proc_handler = proc_dointvec,
3140 },
3141 {
3142 .procname = "gc_elasticity",
3143 .data = &ip_rt_gc_elasticity,
3144 .maxlen = sizeof(int),
3145 .mode = 0644,
3146 .proc_handler = proc_dointvec,
3147 },
3148 {
3149 .procname = "mtu_expires",
3150 .data = &ip_rt_mtu_expires,
3151 .maxlen = sizeof(int),
3152 .mode = 0644,
3153 .proc_handler = proc_dointvec_jiffies,
3154 },
3155 {
3156 .procname = "min_pmtu",
3157 .data = &ip_rt_min_pmtu,
3158 .maxlen = sizeof(int),
3159 .mode = 0644,
3160 .proc_handler = proc_dointvec,
3161 },
3162 {
3163 .procname = "min_adv_mss",
3164 .data = &ip_rt_min_advmss,
3165 .maxlen = sizeof(int),
3166 .mode = 0644,
3167 .proc_handler = proc_dointvec,
3168 },
3169 { }
3170 };
3171
3172 static struct ctl_table empty[1];
3173
3174 static struct ctl_table ipv4_skeleton[] =
3175 {
3176 { .procname = "route",
3177 .mode = 0555, .child = ipv4_route_table},
3178 { .procname = "neigh",
3179 .mode = 0555, .child = empty},
3180 { }
3181 };
3182
3183 static __net_initdata struct ctl_path ipv4_path[] = {
3184 { .procname = "net", },
3185 { .procname = "ipv4", },
3186 { },
3187 };
3188
3189 static struct ctl_table ipv4_route_flush_table[] = {
3190 {
3191 .procname = "flush",
3192 .maxlen = sizeof(int),
3193 .mode = 0200,
3194 .proc_handler = ipv4_sysctl_rtcache_flush,
3195 },
3196 { },
3197 };
3198
3199 static __net_initdata struct ctl_path ipv4_route_path[] = {
3200 { .procname = "net", },
3201 { .procname = "ipv4", },
3202 { .procname = "route", },
3203 { },
3204 };
3205
3206 static __net_init int sysctl_route_net_init(struct net *net)
3207 {
3208 struct ctl_table *tbl;
3209
3210 tbl = ipv4_route_flush_table;
3211 if (!net_eq(net, &init_net)) {
3212 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3213 if (tbl == NULL)
3214 goto err_dup;
3215 }
3216 tbl[0].extra1 = net;
3217
3218 net->ipv4.route_hdr =
3219 register_net_sysctl_table(net, ipv4_route_path, tbl);
3220 if (net->ipv4.route_hdr == NULL)
3221 goto err_reg;
3222 return 0;
3223
3224 err_reg:
3225 if (tbl != ipv4_route_flush_table)
3226 kfree(tbl);
3227 err_dup:
3228 return -ENOMEM;
3229 }
3230
3231 static __net_exit void sysctl_route_net_exit(struct net *net)
3232 {
3233 struct ctl_table *tbl;
3234
3235 tbl = net->ipv4.route_hdr->ctl_table_arg;
3236 unregister_net_sysctl_table(net->ipv4.route_hdr);
3237 BUG_ON(tbl == ipv4_route_flush_table);
3238 kfree(tbl);
3239 }
3240
3241 static __net_initdata struct pernet_operations sysctl_route_ops = {
3242 .init = sysctl_route_net_init,
3243 .exit = sysctl_route_net_exit,
3244 };
3245 #endif
3246
3247 static __net_init int rt_genid_init(struct net *net)
3248 {
3249 get_random_bytes(&net->ipv4.rt_genid,
3250 sizeof(net->ipv4.rt_genid));
3251 return 0;
3252 }
3253
3254 static __net_initdata struct pernet_operations rt_genid_ops = {
3255 .init = rt_genid_init,
3256 };
3257
3258
3259 #ifdef CONFIG_NET_CLS_ROUTE
3260 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3261 #endif /* CONFIG_NET_CLS_ROUTE */
3262
3263 static __initdata unsigned long rhash_entries;
3264 static int __init set_rhash_entries(char *str)
3265 {
3266 if (!str)
3267 return 0;
3268 rhash_entries = simple_strtoul(str, &str, 0);
3269 return 1;
3270 }
3271 __setup("rhash_entries=", set_rhash_entries);
3272
3273 int __init ip_rt_init(void)
3274 {
3275 int rc = 0;
3276
3277 #ifdef CONFIG_NET_CLS_ROUTE
3278 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3279 if (!ip_rt_acct)
3280 panic("IP: failed to allocate ip_rt_acct\n");
3281 #endif
3282
3283 ipv4_dst_ops.kmem_cachep =
3284 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3285 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3286
3287 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3288
3289 if (dst_entries_init(&ipv4_dst_ops) < 0)
3290 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3291
3292 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3293 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3294
3295 rt_hash_table = (struct rt_hash_bucket *)
3296 alloc_large_system_hash("IP route cache",
3297 sizeof(struct rt_hash_bucket),
3298 rhash_entries,
3299 (totalram_pages >= 128 * 1024) ?
3300 15 : 17,
3301 0,
3302 &rt_hash_log,
3303 &rt_hash_mask,
3304 rhash_entries ? 0 : 512 * 1024);
3305 memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3306 rt_hash_lock_init();
3307
3308 ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3309 ip_rt_max_size = (rt_hash_mask + 1) * 16;
3310
3311 devinet_init();
3312 ip_fib_init();
3313
3314 /* All the timers, started at system startup tend
3315 to synchronize. Perturb it a bit.
3316 */
3317 INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3318 expires_ljiffies = jiffies;
3319 schedule_delayed_work(&expires_work,
3320 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3321
3322 if (ip_rt_proc_init())
3323 printk(KERN_ERR "Unable to create route proc files\n");
3324 #ifdef CONFIG_XFRM
3325 xfrm_init();
3326 xfrm4_init(ip_rt_max_size);
3327 #endif
3328 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);
3329
3330 #ifdef CONFIG_SYSCTL
3331 register_pernet_subsys(&sysctl_route_ops);
3332 #endif
3333 register_pernet_subsys(&rt_genid_ops);
3334 return rc;
3335 }
3336
3337 #ifdef CONFIG_SYSCTL
3338 /*
3339 * We really need to sanitize the damn ipv4 init order, then all
3340 * this nonsense will go away.
3341 */
3342 void __init ip_static_sysctl_init(void)
3343 {
3344 register_sysctl_paths(ipv4_path, ipv4_skeleton);
3345 }
3346 #endif
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree