net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 891                 peer->rate_tokens = 0;
 892
 893         /* Too many ignored redirects; do not send anything
 894          * set dst.rate_last to the last seen redirected packet.
 895          */
 896         if (peer->rate_tokens >= ip_rt_redirect_number) {
 897                 peer->rate_last = jiffies;
 898                 goto out_put_peer;
 899         }
 900
 901         /* Check for load limit; set rate_last to the latest sent
 902          * redirect.
 903          */
 904         if (peer->rate_tokens == 0 ||
 905             time_after(jiffies,
 906                        (peer->rate_last +
 907                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 908                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 909
 910                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 911                 peer->rate_last = jiffies;
 912                 ++peer->rate_tokens;
 913 #ifdef CONFIG_IP_ROUTE_VERBOSE
 914                 if (log_martians &&
 915                     peer->rate_tokens == ip_rt_redirect_number)
 916                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 917                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 918                                              &ip_hdr(skb)->daddr, &gw);
 919 #endif
 920         }
 921 out_put_peer:
 922         inet_putpeer(peer);
 923 }
 924
 925 static int ip_error(struct sk_buff *skb)
 926 {
 927         struct rtable *rt = skb_rtable(skb);
 928         struct net_device *dev = skb->dev;
 929         struct in_device *in_dev;
 930         struct inet_peer *peer;
 931         unsigned long now;
 932         struct net *net;
 933         bool send;
 934         int code;
 935
 936         if (netif_is_l3_master(skb->dev)) {
 937                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 938                 if (!dev)
 939                         goto out;
 940         }
 941
 942         in_dev = __in_dev_get_rcu(dev);
 943
 944         /* IP on this device is disabled. */
 945         if (!in_dev)
 946                 goto out;
 947
 948         net = dev_net(rt->dst.dev);
 949         if (!IN_DEV_FORWARD(in_dev)) {
 950                 switch (rt->dst.error) {
 951                 case EHOSTUNREACH:
 952                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 953                         break;
 954
 955                 case ENETUNREACH:
 956                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 957                         break;
 958                 }
 959                 goto out;
 960         }
 961
 962         switch (rt->dst.error) {
 963         case EINVAL:
 964         default:
 965                 goto out;
 966         case EHOSTUNREACH:
 967                 code = ICMP_HOST_UNREACH;
 968                 break;
 969         case ENETUNREACH:
 970                 code = ICMP_NET_UNREACH;
 971                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 972                 break;
 973         case EACCES:
 974                 code = ICMP_PKT_FILTERED;
 975                 break;
 976         }
 977
 978         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 979                                l3mdev_master_ifindex(skb->dev), 1);
 980
 981         send = true;
 982         if (peer) {
 983                 now = jiffies;
 984                 peer->rate_tokens += now - peer->rate_last;
 985                 if (peer->rate_tokens > ip_rt_error_burst)
 986                         peer->rate_tokens = ip_rt_error_burst;
 987                 peer->rate_last = now;
 988                 if (peer->rate_tokens >= ip_rt_error_cost)
 989                         peer->rate_tokens -= ip_rt_error_cost;
 990                 else
 991                         send = false;
 992                 inet_putpeer(peer);
 993         }
 994         if (send)
 995                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 996
 997 out:    kfree_skb(skb);
 998         return 0;
 999 }
1000
1001 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1002 {
1003         struct dst_entry *dst = &rt->dst;
1004         struct fib_result res;
1005         bool lock = false;
1006
1007         if (ip_mtu_locked(dst))
1008                 return;
1009
1010         if (ipv4_mtu(dst) < mtu)
1011                 return;
1012
1013         if (mtu < ip_rt_min_pmtu) {
1014                 lock = true;
1015                 mtu = ip_rt_min_pmtu;
1016         }
1017
1018         if (rt->rt_pmtu == mtu &&
1019             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1020                 return;
1021
1022         rcu_read_lock();
1023         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1024                 struct fib_nh *nh = &FIB_RES_NH(res);
1025
1026                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1027                                       jiffies + ip_rt_mtu_expires);
1028         }
1029         rcu_read_unlock();
1030 }
1031
1032 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1033                               struct sk_buff *skb, u32 mtu)
1034 {
1035         struct rtable *rt = (struct rtable *) dst;
1036         struct flowi4 fl4;
1037
1038         ip_rt_build_flow_key(&fl4, sk, skb);
1039         __ip_rt_update_pmtu(rt, &fl4, mtu);
1040 }
1041
1042 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1043                       int oif, u32 mark, u8 protocol, int flow_flags)
1044 {
1045         const struct iphdr *iph = (const struct iphdr *) skb->data;
1046         struct flowi4 fl4;
1047         struct rtable *rt;
1048
1049         if (!mark)
1050                 mark = IP4_REPLY_MARK(net, skb->mark);
1051
1052         __build_flow_key(net, &fl4, NULL, iph, oif,
1053                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1054         rt = __ip_route_output_key(net, &fl4);
1055         if (!IS_ERR(rt)) {
1056                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1057                 ip_rt_put(rt);
1058         }
1059 }
1060 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1061
1062 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1063 {
1064         const struct iphdr *iph = (const struct iphdr *) skb->data;
1065         struct flowi4 fl4;
1066         struct rtable *rt;
1067
1068         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1069
1070         if (!fl4.flowi4_mark)
1071                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1072
1073         rt = __ip_route_output_key(sock_net(sk), &fl4);
1074         if (!IS_ERR(rt)) {
1075                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1076                 ip_rt_put(rt);
1077         }
1078 }
1079
1080 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1081 {
1082         const struct iphdr *iph = (const struct iphdr *) skb->data;
1083         struct flowi4 fl4;
1084         struct rtable *rt;
1085         struct dst_entry *odst = NULL;
1086         bool new = false;
1087         struct net *net = sock_net(sk);
1088
1089         bh_lock_sock(sk);
1090
1091         if (!ip_sk_accept_pmtu(sk))
1092                 goto out;
1093
1094         odst = sk_dst_get(sk);
1095
1096         if (sock_owned_by_user(sk) || !odst) {
1097                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1098                 goto out;
1099         }
1100
1101         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1102
1103         rt = (struct rtable *)odst;
1104         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1105                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1106                 if (IS_ERR(rt))
1107                         goto out;
1108
1109                 new = true;
1110         }
1111
1112         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1113
1114         if (!dst_check(&rt->dst, 0)) {
1115                 if (new)
1116                         dst_release(&rt->dst);
1117
1118                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1119                 if (IS_ERR(rt))
1120                         goto out;
1121
1122                 new = true;
1123         }
1124
1125         if (new)
1126                 sk_dst_set(sk, &rt->dst);
1127
1128 out:
1129         bh_unlock_sock(sk);
1130         dst_release(odst);
1131 }
1132 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1133
1134 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1135                    int oif, u32 mark, u8 protocol, int flow_flags)
1136 {
1137         const struct iphdr *iph = (const struct iphdr *) skb->data;
1138         struct flowi4 fl4;
1139         struct rtable *rt;
1140
1141         __build_flow_key(net, &fl4, NULL, iph, oif,
1142                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1143         rt = __ip_route_output_key(net, &fl4);
1144         if (!IS_ERR(rt)) {
1145                 __ip_do_redirect(rt, skb, &fl4, false);
1146                 ip_rt_put(rt);
1147         }
1148 }
1149 EXPORT_SYMBOL_GPL(ipv4_redirect);
1150
1151 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1152 {
1153         const struct iphdr *iph = (const struct iphdr *) skb->data;
1154         struct flowi4 fl4;
1155         struct rtable *rt;
1156         struct net *net = sock_net(sk);
1157
1158         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1159         rt = __ip_route_output_key(net, &fl4);
1160         if (!IS_ERR(rt)) {
1161                 __ip_do_redirect(rt, skb, &fl4, false);
1162                 ip_rt_put(rt);
1163         }
1164 }
1165 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1166
1167 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1168 {
1169         struct rtable *rt = (struct rtable *) dst;
1170
1171         /* All IPV4 dsts are created with ->obsolete set to the value
1172          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1173          * into this function always.
1174          *
1175          * When a PMTU/redirect information update invalidates a route,
1176          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1177          * DST_OBSOLETE_DEAD by dst_free().
1178          */
1179         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1180                 return NULL;
1181         return dst;
1182 }
1183
1184 static void ipv4_link_failure(struct sk_buff *skb)
1185 {
1186         struct rtable *rt;
1187
1188         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1189
1190         rt = skb_rtable(skb);
1191         if (rt)
1192                 dst_set_expires(&rt->dst, 0);
1193 }
1194
1195 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1196 {
1197         pr_debug("%s: %pI4 -> %pI4, %s\n",
1198                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1199                  skb->dev ? skb->dev->name : "?");
1200         kfree_skb(skb);
1201         WARN_ON(1);
1202         return 0;
1203 }
1204
1205 /*
1206    We do not cache source address of outgoing interface,
1207    because it is used only by IP RR, TS and SRR options,
1208    so that it out of fast path.
1209
1210    BTW remember: "addr" is allowed to be not aligned
1211    in IP options!
1212  */
1213
1214 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1215 {
1216         __be32 src;
1217
1218         if (rt_is_output_route(rt))
1219                 src = ip_hdr(skb)->saddr;
1220         else {
1221                 struct fib_result res;
1222                 struct flowi4 fl4;
1223                 struct iphdr *iph;
1224
1225                 iph = ip_hdr(skb);
1226
1227                 memset(&fl4, 0, sizeof(fl4));
1228                 fl4.daddr = iph->daddr;
1229                 fl4.saddr = iph->saddr;
1230                 fl4.flowi4_tos = RT_TOS(iph->tos);
1231                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1232                 fl4.flowi4_iif = skb->dev->ifindex;
1233                 fl4.flowi4_mark = skb->mark;
1234
1235                 rcu_read_lock();
1236                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1237                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1238                 else
1239                         src = inet_select_addr(rt->dst.dev,
1240                                                rt_nexthop(rt, iph->daddr),
1241                                                RT_SCOPE_UNIVERSE);
1242                 rcu_read_unlock();
1243         }
1244         memcpy(addr, &src, 4);
1245 }
1246
1247 #ifdef CONFIG_IP_ROUTE_CLASSID
1248 static void set_class_tag(struct rtable *rt, u32 tag)
1249 {
1250         if (!(rt->dst.tclassid & 0xFFFF))
1251                 rt->dst.tclassid |= tag & 0xFFFF;
1252         if (!(rt->dst.tclassid & 0xFFFF0000))
1253                 rt->dst.tclassid |= tag & 0xFFFF0000;
1254 }
1255 #endif
1256
1257 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1258 {
1259         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1260         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1261                                     ip_rt_min_advmss);
1262
1263         return min(advmss, IPV4_MAX_PMTU - header_size);
1264 }
1265
1266 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1267 {
1268         const struct rtable *rt = (const struct rtable *) dst;
1269         unsigned int mtu = rt->rt_pmtu;
1270
1271         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1272                 mtu = dst_metric_raw(dst, RTAX_MTU);
1273
1274         if (mtu)
1275                 return mtu;
1276
1277         mtu = READ_ONCE(dst->dev->mtu);
1278
1279         if (unlikely(ip_mtu_locked(dst))) {
1280                 if (rt->rt_uses_gateway && mtu > 576)
1281                         mtu = 576;
1282         }
1283
1284         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1285
1286         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1287 }
1288
1289 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1290 {
1291         struct fnhe_hash_bucket *hash;
1292         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1293         u32 hval = fnhe_hashfun(daddr);
1294
1295         spin_lock_bh(&fnhe_lock);
1296
1297         hash = rcu_dereference_protected(nh->nh_exceptions,
1298                                          lockdep_is_held(&fnhe_lock));
1299         hash += hval;
1300
1301         fnhe_p = &hash->chain;
1302         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1303         while (fnhe) {
1304                 if (fnhe->fnhe_daddr == daddr) {
1305                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1306                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1307                         fnhe_flush_routes(fnhe);
1308                         kfree_rcu(fnhe, rcu);
1309                         break;
1310                 }
1311                 fnhe_p = &fnhe->fnhe_next;
1312                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1313                                                  lockdep_is_held(&fnhe_lock));
1314         }
1315
1316         spin_unlock_bh(&fnhe_lock);
1317 }
1318
1319 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1320 {
1321         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1322         struct fib_nh_exception *fnhe;
1323         u32 hval;
1324
1325         if (!hash)
1326                 return NULL;
1327
1328         hval = fnhe_hashfun(daddr);
1329
1330         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1331              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1332                 if (fnhe->fnhe_daddr == daddr) {
1333                         if (fnhe->fnhe_expires &&
1334                             time_after(jiffies, fnhe->fnhe_expires)) {
1335                                 ip_del_fnhe(nh, daddr);
1336                                 break;
1337                         }
1338                         return fnhe;
1339                 }
1340         }
1341         return NULL;
1342 }
1343
1344 /* MTU selection:
1345  * 1. mtu on route is locked - use it
1346  * 2. mtu from nexthop exception
1347  * 3. mtu from egress device
1348  */
1349
1350 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1351 {
1352         struct fib_info *fi = res->fi;
1353         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1354         struct net_device *dev = nh->nh_dev;
1355         u32 mtu = 0;
1356
1357         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1358             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1359                 mtu = fi->fib_mtu;
1360
1361         if (likely(!mtu)) {
1362                 struct fib_nh_exception *fnhe;
1363
1364                 fnhe = find_exception(nh, daddr);
1365                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1366                         mtu = fnhe->fnhe_pmtu;
1367         }
1368
1369         if (likely(!mtu))
1370                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1371
1372         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1373 }
1374
1375 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1376                               __be32 daddr, const bool do_cache)
1377 {
1378         bool ret = false;
1379
1380         spin_lock_bh(&fnhe_lock);
1381
1382         if (daddr == fnhe->fnhe_daddr) {
1383                 struct rtable __rcu **porig;
1384                 struct rtable *orig;
1385                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1386
1387                 if (rt_is_input_route(rt))
1388                         porig = &fnhe->fnhe_rth_input;
1389                 else
1390                         porig = &fnhe->fnhe_rth_output;
1391                 orig = rcu_dereference(*porig);
1392
1393                 if (fnhe->fnhe_genid != genid) {
1394                         fnhe->fnhe_genid = genid;
1395                         fnhe->fnhe_gw = 0;
1396                         fnhe->fnhe_pmtu = 0;
1397                         fnhe->fnhe_expires = 0;
1398                         fnhe->fnhe_mtu_locked = false;
1399                         fnhe_flush_routes(fnhe);
1400                         orig = NULL;
1401                 }
1402                 fill_route_from_fnhe(rt, fnhe);
1403                 if (!rt->rt_gateway)
1404                         rt->rt_gateway = daddr;
1405
1406                 if (do_cache) {
1407                         dst_hold(&rt->dst);
1408                         rcu_assign_pointer(*porig, rt);
1409                         if (orig) {
1410                                 dst_dev_put(&orig->dst);
1411                                 dst_release(&orig->dst);
1412                         }
1413                         ret = true;
1414                 }
1415
1416                 fnhe->fnhe_stamp = jiffies;
1417         }
1418         spin_unlock_bh(&fnhe_lock);
1419
1420         return ret;
1421 }
1422
1423 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1424 {
1425         struct rtable *orig, *prev, **p;
1426         bool ret = true;
1427
1428         if (rt_is_input_route(rt)) {
1429                 p = (struct rtable **)&nh->nh_rth_input;
1430         } else {
1431                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1432         }
1433         orig = *p;
1434
1435         /* hold dst before doing cmpxchg() to avoid race condition
1436          * on this dst
1437          */
1438         dst_hold(&rt->dst);
1439         prev = cmpxchg(p, orig, rt);
1440         if (prev == orig) {
1441                 if (orig) {
1442                         dst_dev_put(&orig->dst);
1443                         dst_release(&orig->dst);
1444                 }
1445         } else {
1446                 dst_release(&rt->dst);
1447                 ret = false;
1448         }
1449
1450         return ret;
1451 }
1452
1453 struct uncached_list {
1454         spinlock_t              lock;
1455         struct list_head        head;
1456 };
1457
1458 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1459
1460 void rt_add_uncached_list(struct rtable *rt)
1461 {
1462         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1463
1464         rt->rt_uncached_list = ul;
1465
1466         spin_lock_bh(&ul->lock);
1467         list_add_tail(&rt->rt_uncached, &ul->head);
1468         spin_unlock_bh(&ul->lock);
1469 }
1470
1471 void rt_del_uncached_list(struct rtable *rt)
1472 {
1473         if (!list_empty(&rt->rt_uncached)) {
1474                 struct uncached_list *ul = rt->rt_uncached_list;
1475
1476                 spin_lock_bh(&ul->lock);
1477                 list_del(&rt->rt_uncached);
1478                 spin_unlock_bh(&ul->lock);
1479         }
1480 }
1481
1482 static void ipv4_dst_destroy(struct dst_entry *dst)
1483 {
1484         struct dst_metrics *p = (struct dst_metrics *)DST_METRICS_PTR(dst);
1485         struct rtable *rt = (struct rtable *)dst;
1486
1487         if (p != &dst_default_metrics && refcount_dec_and_test(&p->refcnt))
1488                 kfree(p);
1489
1490         rt_del_uncached_list(rt);
1491 }
1492
1493 void rt_flush_dev(struct net_device *dev)
1494 {
1495         struct net *net = dev_net(dev);
1496         struct rtable *rt;
1497         int cpu;
1498
1499         for_each_possible_cpu(cpu) {
1500                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1501
1502                 spin_lock_bh(&ul->lock);
1503                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1504                         if (rt->dst.dev != dev)
1505                                 continue;
1506                         rt->dst.dev = net->loopback_dev;
1507                         dev_hold(rt->dst.dev);
1508                         dev_put(dev);
1509                 }
1510                 spin_unlock_bh(&ul->lock);
1511         }
1512 }
1513
1514 static bool rt_cache_valid(const struct rtable *rt)
1515 {
1516         return  rt &&
1517                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1518                 !rt_is_expired(rt);
1519 }
1520
1521 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1522                            const struct fib_result *res,
1523                            struct fib_nh_exception *fnhe,
1524                            struct fib_info *fi, u16 type, u32 itag,
1525                            const bool do_cache)
1526 {
1527         bool cached = false;
1528
1529         if (fi) {
1530                 struct fib_nh *nh = &FIB_RES_NH(*res);
1531
1532                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1533                         rt->rt_gateway = nh->nh_gw;
1534                         rt->rt_uses_gateway = 1;
1535                 }
1536                 dst_init_metrics(&rt->dst, fi->fib_metrics->metrics, true);
1537                 if (fi->fib_metrics != &dst_default_metrics) {
1538                         rt->dst._metrics |= DST_METRICS_REFCOUNTED;
1539                         refcount_inc(&fi->fib_metrics->refcnt);
1540                 }
1541 #ifdef CONFIG_IP_ROUTE_CLASSID
1542                 rt->dst.tclassid = nh->nh_tclassid;
1543 #endif
1544                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1545                 if (unlikely(fnhe))
1546                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1547                 else if (do_cache)
1548                         cached = rt_cache_route(nh, rt);
1549                 if (unlikely(!cached)) {
1550                         /* Routes we intend to cache in nexthop exception or
1551                          * FIB nexthop have the DST_NOCACHE bit clear.
1552                          * However, if we are unsuccessful at storing this
1553                          * route into the cache we really need to set it.
1554                          */
1555                         if (!rt->rt_gateway)
1556                                 rt->rt_gateway = daddr;
1557                         rt_add_uncached_list(rt);
1558                 }
1559         } else
1560                 rt_add_uncached_list(rt);
1561
1562 #ifdef CONFIG_IP_ROUTE_CLASSID
1563 #ifdef CONFIG_IP_MULTIPLE_TABLES
1564         set_class_tag(rt, res->tclassid);
1565 #endif
1566         set_class_tag(rt, itag);
1567 #endif
1568 }
1569
1570 struct rtable *rt_dst_alloc(struct net_device *dev,
1571                             unsigned int flags, u16 type,
1572                             bool nopolicy, bool noxfrm, bool will_cache)
1573 {
1574         struct rtable *rt;
1575
1576         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1577                        (will_cache ? 0 : DST_HOST) |
1578                        (nopolicy ? DST_NOPOLICY : 0) |
1579                        (noxfrm ? DST_NOXFRM : 0));
1580
1581         if (rt) {
1582                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1583                 rt->rt_flags = flags;
1584                 rt->rt_type = type;
1585                 rt->rt_is_input = 0;
1586                 rt->rt_iif = 0;
1587                 rt->rt_pmtu = 0;
1588                 rt->rt_mtu_locked = 0;
1589                 rt->rt_gateway = 0;
1590                 rt->rt_uses_gateway = 0;
1591                 INIT_LIST_HEAD(&rt->rt_uncached);
1592
1593                 rt->dst.output = ip_output;
1594                 if (flags & RTCF_LOCAL)
1595                         rt->dst.input = ip_local_deliver;
1596         }
1597
1598         return rt;
1599 }
1600 EXPORT_SYMBOL(rt_dst_alloc);
1601
1602 /* called in rcu_read_lock() section */
1603 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1604                           u8 tos, struct net_device *dev,
1605                           struct in_device *in_dev, u32 *itag)
1606 {
1607         int err;
1608
1609         /* Primary sanity checks. */
1610         if (!in_dev)
1611                 return -EINVAL;
1612
1613         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1614             skb->protocol != htons(ETH_P_IP))
1615                 return -EINVAL;
1616
1617         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1618                 return -EINVAL;
1619
1620         if (ipv4_is_zeronet(saddr)) {
1621                 if (!ipv4_is_local_multicast(daddr))
1622                         return -EINVAL;
1623         } else {
1624                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1625                                           in_dev, itag);
1626                 if (err < 0)
1627                         return err;
1628         }
1629         return 0;
1630 }
1631
1632 /* called in rcu_read_lock() section */
1633 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1634                              u8 tos, struct net_device *dev, int our)
1635 {
1636         struct in_device *in_dev = __in_dev_get_rcu(dev);
1637         unsigned int flags = RTCF_MULTICAST;
1638         struct rtable *rth;
1639         u32 itag = 0;
1640         int err;
1641
1642         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1643         if (err)
1644                 return err;
1645
1646         if (our)
1647                 flags |= RTCF_LOCAL;
1648
1649         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1650                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1651         if (!rth)
1652                 return -ENOBUFS;
1653
1654 #ifdef CONFIG_IP_ROUTE_CLASSID
1655         rth->dst.tclassid = itag;
1656 #endif
1657         rth->dst.output = ip_rt_bug;
1658         rth->rt_is_input= 1;
1659
1660 #ifdef CONFIG_IP_MROUTE
1661         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1662                 rth->dst.input = ip_mr_input;
1663 #endif
1664         RT_CACHE_STAT_INC(in_slow_mc);
1665
1666         skb_dst_set(skb, &rth->dst);
1667         return 0;
1668 }
1669
1670
1671 static void ip_handle_martian_source(struct net_device *dev,
1672                                      struct in_device *in_dev,
1673                                      struct sk_buff *skb,
1674                                      __be32 daddr,
1675                                      __be32 saddr)
1676 {
1677         RT_CACHE_STAT_INC(in_martian_src);
1678 #ifdef CONFIG_IP_ROUTE_VERBOSE
1679         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1680                 /*
1681                  *      RFC1812 recommendation, if source is martian,
1682                  *      the only hint is MAC header.
1683                  */
1684                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1685                         &daddr, &saddr, dev->name);
1686                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1687                         print_hex_dump(KERN_WARNING, "ll header: ",
1688                                        DUMP_PREFIX_OFFSET, 16, 1,
1689                                        skb_mac_header(skb),
1690                                        dev->hard_header_len, true);
1691                 }
1692         }
1693 #endif
1694 }
1695
1696 /* called in rcu_read_lock() section */
1697 static int __mkroute_input(struct sk_buff *skb,
1698                            const struct fib_result *res,
1699                            struct in_device *in_dev,
1700                            __be32 daddr, __be32 saddr, u32 tos)
1701 {
1702         struct fib_nh_exception *fnhe;
1703         struct rtable *rth;
1704         int err;
1705         struct in_device *out_dev;
1706         bool do_cache;
1707         u32 itag = 0;
1708
1709         /* get a working reference to the output device */
1710         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1711         if (!out_dev) {
1712                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1713                 return -EINVAL;
1714         }
1715
1716         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1717                                   in_dev->dev, in_dev, &itag);
1718         if (err < 0) {
1719                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1720                                          saddr);
1721
1722                 goto cleanup;
1723         }
1724
1725         do_cache = res->fi && !itag;
1726         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1727             skb->protocol == htons(ETH_P_IP) &&
1728             (IN_DEV_SHARED_MEDIA(out_dev) ||
1729              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1730                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1731
1732         if (skb->protocol != htons(ETH_P_IP)) {
1733                 /* Not IP (i.e. ARP). Do not create route, if it is
1734                  * invalid for proxy arp. DNAT routes are always valid.
1735                  *
1736                  * Proxy arp feature have been extended to allow, ARP
1737                  * replies back to the same interface, to support
1738                  * Private VLAN switch technologies. See arp.c.
1739                  */
1740                 if (out_dev == in_dev &&
1741                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1742                         err = -EINVAL;
1743                         goto cleanup;
1744                 }
1745         }
1746
1747         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1748         if (do_cache) {
1749                 if (fnhe)
1750                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1751                 else
1752                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1753                 if (rt_cache_valid(rth)) {
1754                         skb_dst_set_noref(skb, &rth->dst);
1755                         goto out;
1756                 }
1757         }
1758
1759         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1760                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1761                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1762         if (!rth) {
1763                 err = -ENOBUFS;
1764                 goto cleanup;
1765         }
1766
1767         rth->rt_is_input = 1;
1768         RT_CACHE_STAT_INC(in_slow_tot);
1769
1770         rth->dst.input = ip_forward;
1771
1772         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1773                        do_cache);
1774         lwtunnel_set_redirect(&rth->dst);
1775         skb_dst_set(skb, &rth->dst);
1776 out:
1777         err = 0;
1778  cleanup:
1779         return err;
1780 }
1781
1782 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1783 /* To make ICMP packets follow the right flow, the multipath hash is
1784  * calculated from the inner IP addresses.
1785  */
1786 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1787                                  struct flow_keys *hash_keys)
1788 {
1789         const struct iphdr *outer_iph = ip_hdr(skb);
1790         const struct iphdr *key_iph = outer_iph;
1791         const struct iphdr *inner_iph;
1792         const struct icmphdr *icmph;
1793         struct iphdr _inner_iph;
1794         struct icmphdr _icmph;
1795
1796         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1797                 goto out;
1798
1799         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1800                 goto out;
1801
1802         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1803                                    &_icmph);
1804         if (!icmph)
1805                 goto out;
1806
1807         if (icmph->type != ICMP_DEST_UNREACH &&
1808             icmph->type != ICMP_REDIRECT &&
1809             icmph->type != ICMP_TIME_EXCEEDED &&
1810             icmph->type != ICMP_PARAMETERPROB)
1811                 goto out;
1812
1813         inner_iph = skb_header_pointer(skb,
1814                                        outer_iph->ihl * 4 + sizeof(_icmph),
1815                                        sizeof(_inner_iph), &_inner_iph);
1816         if (!inner_iph)
1817                 goto out;
1818
1819         key_iph = inner_iph;
1820 out:
1821         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1822         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1823 }
1824
1825 /* if skb is set it will be used and fl4 can be NULL */
1826 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1827                        const struct sk_buff *skb, struct flow_keys *flkeys)
1828 {
1829         struct flow_keys hash_keys;
1830         u32 mhash;
1831
1832         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1833         case 0:
1834                 memset(&hash_keys, 0, sizeof(hash_keys));
1835                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1836                 if (skb) {
1837                         ip_multipath_l3_keys(skb, &hash_keys);
1838                 } else {
1839                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1840                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1841                 }
1842                 break;
1843         case 1:
1844                 /* skb is currently provided only when forwarding */
1845                 if (skb) {
1846                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1847                         struct flow_keys keys;
1848
1849                         /* short-circuit if we already have L4 hash present */
1850                         if (skb->l4_hash)
1851                                 return skb_get_hash_raw(skb) >> 1;
1852
1853                         memset(&hash_keys, 0, sizeof(hash_keys));
1854
1855                         if (!flkeys) {
1856                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1857                                 flkeys = &keys;
1858                         }
1859
1860                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1861                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1862                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1863                         hash_keys.ports.src = flkeys->ports.src;
1864                         hash_keys.ports.dst = flkeys->ports.dst;
1865                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1866                 } else {
1867                         memset(&hash_keys, 0, sizeof(hash_keys));
1868                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1869                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1870                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1871                         hash_keys.ports.src = fl4->fl4_sport;
1872                         hash_keys.ports.dst = fl4->fl4_dport;
1873                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1874                 }
1875                 break;
1876         }
1877         mhash = flow_hash_from_keys(&hash_keys);
1878
1879         return mhash >> 1;
1880 }
1881 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1882
1883 static int ip_mkroute_input(struct sk_buff *skb,
1884                             struct fib_result *res,
1885                             struct in_device *in_dev,
1886                             __be32 daddr, __be32 saddr, u32 tos,
1887                             struct flow_keys *hkeys)
1888 {
1889 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1890         if (res->fi && res->fi->fib_nhs > 1) {
1891                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1892
1893                 fib_select_multipath(res, h);
1894         }
1895 #endif
1896
1897         /* create a routing cache entry */
1898         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1899 }
1900
1901 /*
1902  *      NOTE. We drop all the packets that has local source
1903  *      addresses, because every properly looped back packet
1904  *      must have correct destination already attached by output routine.
1905  *
1906  *      Such approach solves two big problems:
1907  *      1. Not simplex devices are handled properly.
1908  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1909  *      called with rcu_read_lock()
1910  */
1911
1912 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1913                                u8 tos, struct net_device *dev,
1914                                struct fib_result *res)
1915 {
1916         struct in_device *in_dev = __in_dev_get_rcu(dev);
1917         struct flow_keys *flkeys = NULL, _flkeys;
1918         struct net    *net = dev_net(dev);
1919         struct ip_tunnel_info *tun_info;
1920         int             err = -EINVAL;
1921         unsigned int    flags = 0;
1922         u32             itag = 0;
1923         struct rtable   *rth;
1924         struct flowi4   fl4;
1925         bool do_cache;
1926
1927         /* IP on this device is disabled. */
1928
1929         if (!in_dev)
1930                 goto out;
1931
1932         /* Check for the most weird martians, which can be not detected
1933            by fib_lookup.
1934          */
1935
1936         tun_info = skb_tunnel_info(skb);
1937         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1938                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1939         else
1940                 fl4.flowi4_tun_key.tun_id = 0;
1941         skb_dst_drop(skb);
1942
1943         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1944                 goto martian_source;
1945
1946         res->fi = NULL;
1947         res->table = NULL;
1948         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1949                 goto brd_input;
1950
1951         /* Accept zero addresses only to limited broadcast;
1952          * I even do not know to fix it or not. Waiting for complains :-)
1953          */
1954         if (ipv4_is_zeronet(saddr))
1955                 goto martian_source;
1956
1957         if (ipv4_is_zeronet(daddr))
1958                 goto martian_destination;
1959
1960         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1961          * and call it once if daddr or/and saddr are loopback addresses
1962          */
1963         if (ipv4_is_loopback(daddr)) {
1964                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1965                         goto martian_destination;
1966         } else if (ipv4_is_loopback(saddr)) {
1967                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1968                         goto martian_source;
1969         }
1970
1971         /*
1972          *      Now we are ready to route packet.
1973          */
1974         fl4.flowi4_oif = 0;
1975         fl4.flowi4_iif = dev->ifindex;
1976         fl4.flowi4_mark = skb->mark;
1977         fl4.flowi4_tos = tos;
1978         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1979         fl4.flowi4_flags = 0;
1980         fl4.daddr = daddr;
1981         fl4.saddr = saddr;
1982         fl4.flowi4_uid = sock_net_uid(net, NULL);
1983
1984         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
1985                 flkeys = &_flkeys;
1986         } else {
1987                 fl4.flowi4_proto = 0;
1988                 fl4.fl4_sport = 0;
1989                 fl4.fl4_dport = 0;
1990         }
1991
1992         err = fib_lookup(net, &fl4, res, 0);
1993         if (err != 0) {
1994                 if (!IN_DEV_FORWARD(in_dev))
1995                         err = -EHOSTUNREACH;
1996                 goto no_route;
1997         }
1998
1999         if (res->type == RTN_BROADCAST) {
2000                 if (IN_DEV_BFORWARD(in_dev))
2001                         goto make_route;
2002                 goto brd_input;
2003         }
2004
2005         if (res->type == RTN_LOCAL) {
2006                 err = fib_validate_source(skb, saddr, daddr, tos,
2007                                           0, dev, in_dev, &itag);
2008                 if (err < 0)
2009                         goto martian_source;
2010                 goto local_input;
2011         }
2012
2013         if (!IN_DEV_FORWARD(in_dev)) {
2014                 err = -EHOSTUNREACH;
2015                 goto no_route;
2016         }
2017         if (res->type != RTN_UNICAST)
2018                 goto martian_destination;
2019
2020 make_route:
2021         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2022 out:    return err;
2023
2024 brd_input:
2025         if (skb->protocol != htons(ETH_P_IP))
2026                 goto e_inval;
2027
2028         if (!ipv4_is_zeronet(saddr)) {
2029                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2030                                           in_dev, &itag);
2031                 if (err < 0)
2032                         goto martian_source;
2033         }
2034         flags |= RTCF_BROADCAST;
2035         res->type = RTN_BROADCAST;
2036         RT_CACHE_STAT_INC(in_brd);
2037
2038 local_input:
2039         do_cache = false;
2040         if (res->fi) {
2041                 if (!itag) {
2042                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2043                         if (rt_cache_valid(rth)) {
2044                                 skb_dst_set_noref(skb, &rth->dst);
2045                                 err = 0;
2046                                 goto out;
2047                         }
2048                         do_cache = true;
2049                 }
2050         }
2051
2052         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2053                            flags | RTCF_LOCAL, res->type,
2054                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2055         if (!rth)
2056                 goto e_nobufs;
2057
2058         rth->dst.output= ip_rt_bug;
2059 #ifdef CONFIG_IP_ROUTE_CLASSID
2060         rth->dst.tclassid = itag;
2061 #endif
2062         rth->rt_is_input = 1;
2063
2064         RT_CACHE_STAT_INC(in_slow_tot);
2065         if (res->type == RTN_UNREACHABLE) {
2066                 rth->dst.input= ip_error;
2067                 rth->dst.error= -err;
2068                 rth->rt_flags   &= ~RTCF_LOCAL;
2069         }
2070
2071         if (do_cache) {
2072                 struct fib_nh *nh = &FIB_RES_NH(*res);
2073
2074                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2075                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2076                         WARN_ON(rth->dst.input == lwtunnel_input);
2077                         rth->dst.lwtstate->orig_input = rth->dst.input;
2078                         rth->dst.input = lwtunnel_input;
2079                 }
2080
2081                 if (unlikely(!rt_cache_route(nh, rth)))
2082                         rt_add_uncached_list(rth);
2083         }
2084         skb_dst_set(skb, &rth->dst);
2085         err = 0;
2086         goto out;
2087
2088 no_route:
2089         RT_CACHE_STAT_INC(in_no_route);
2090         res->type = RTN_UNREACHABLE;
2091         res->fi = NULL;
2092         res->table = NULL;
2093         goto local_input;
2094
2095         /*
2096          *      Do not cache martian addresses: they should be logged (RFC1812)
2097          */
2098 martian_destination:
2099         RT_CACHE_STAT_INC(in_martian_dst);
2100 #ifdef CONFIG_IP_ROUTE_VERBOSE
2101         if (IN_DEV_LOG_MARTIANS(in_dev))
2102                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2103                                      &daddr, &saddr, dev->name);
2104 #endif
2105
2106 e_inval:
2107         err = -EINVAL;
2108         goto out;
2109
2110 e_nobufs:
2111         err = -ENOBUFS;
2112         goto out;
2113
2114 martian_source:
2115         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2116         goto out;
2117 }
2118
2119 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2120                          u8 tos, struct net_device *dev)
2121 {
2122         struct fib_result res;
2123         int err;
2124
2125         tos &= IPTOS_RT_MASK;
2126         rcu_read_lock();
2127         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2128         rcu_read_unlock();
2129
2130         return err;
2131 }
2132 EXPORT_SYMBOL(ip_route_input_noref);
2133
2134 /* called with rcu_read_lock held */
2135 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2136                        u8 tos, struct net_device *dev, struct fib_result *res)
2137 {
2138         /* Multicast recognition logic is moved from route cache to here.
2139            The problem was that too many Ethernet cards have broken/missing
2140            hardware multicast filters :-( As result the host on multicasting
2141            network acquires a lot of useless route cache entries, sort of
2142            SDR messages from all the world. Now we try to get rid of them.
2143            Really, provided software IP multicast filter is organized
2144            reasonably (at least, hashed), it does not result in a slowdown
2145            comparing with route cache reject entries.
2146            Note, that multicast routers are not affected, because
2147            route cache entry is created eventually.
2148          */
2149         if (ipv4_is_multicast(daddr)) {
2150                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2151                 int our = 0;
2152                 int err = -EINVAL;
2153
2154                 if (in_dev)
2155                         our = ip_check_mc_rcu(in_dev, daddr, saddr,
2156                                               ip_hdr(skb)->protocol);
2157
2158                 /* check l3 master if no match yet */
2159                 if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
2160                         struct in_device *l3_in_dev;
2161
2162                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2163                         if (l3_in_dev)
2164                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2165                                                       ip_hdr(skb)->protocol);
2166                 }
2167
2168                 if (our
2169 #ifdef CONFIG_IP_MROUTE
2170                         ||
2171                     (!ipv4_is_local_multicast(daddr) &&
2172                      IN_DEV_MFORWARD(in_dev))
2173 #endif
2174                    ) {
2175                         err = ip_route_input_mc(skb, daddr, saddr,
2176                                                 tos, dev, our);
2177                 }
2178                 return err;
2179         }
2180
2181         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2182 }
2183
2184 /* called with rcu_read_lock() */
2185 static struct rtable *__mkroute_output(const struct fib_result *res,
2186                                        const struct flowi4 *fl4, int orig_oif,
2187                                        struct net_device *dev_out,
2188                                        unsigned int flags)
2189 {
2190         struct fib_info *fi = res->fi;
2191         struct fib_nh_exception *fnhe;
2192         struct in_device *in_dev;
2193         u16 type = res->type;
2194         struct rtable *rth;
2195         bool do_cache;
2196
2197         in_dev = __in_dev_get_rcu(dev_out);
2198         if (!in_dev)
2199                 return ERR_PTR(-EINVAL);
2200
2201         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2202                 if (ipv4_is_loopback(fl4->saddr) &&
2203                     !(dev_out->flags & IFF_LOOPBACK) &&
2204                     !netif_is_l3_master(dev_out))
2205                         return ERR_PTR(-EINVAL);
2206
2207         if (ipv4_is_lbcast(fl4->daddr))
2208                 type = RTN_BROADCAST;
2209         else if (ipv4_is_multicast(fl4->daddr))
2210                 type = RTN_MULTICAST;
2211         else if (ipv4_is_zeronet(fl4->daddr))
2212                 return ERR_PTR(-EINVAL);
2213
2214         if (dev_out->flags & IFF_LOOPBACK)
2215                 flags |= RTCF_LOCAL;
2216
2217         do_cache = true;
2218         if (type == RTN_BROADCAST) {
2219                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2220                 fi = NULL;
2221         } else if (type == RTN_MULTICAST) {
2222                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2223                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2224                                      fl4->flowi4_proto))
2225                         flags &= ~RTCF_LOCAL;
2226                 else
2227                         do_cache = false;
2228                 /* If multicast route do not exist use
2229                  * default one, but do not gateway in this case.
2230                  * Yes, it is hack.
2231                  */
2232                 if (fi && res->prefixlen < 4)
2233                         fi = NULL;
2234         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2235                    (orig_oif != dev_out->ifindex)) {
2236                 /* For local routes that require a particular output interface
2237                  * we do not want to cache the result.  Caching the result
2238                  * causes incorrect behaviour when there are multiple source
2239                  * addresses on the interface, the end result being that if the
2240                  * intended recipient is waiting on that interface for the
2241                  * packet he won't receive it because it will be delivered on
2242                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2243                  * be set to the loopback interface as well.
2244                  */
2245                 do_cache = false;
2246         }
2247
2248         fnhe = NULL;
2249         do_cache &= fi != NULL;
2250         if (fi) {
2251                 struct rtable __rcu **prth;
2252                 struct fib_nh *nh = &FIB_RES_NH(*res);
2253
2254                 fnhe = find_exception(nh, fl4->daddr);
2255                 if (!do_cache)
2256                         goto add;
2257                 if (fnhe) {
2258                         prth = &fnhe->fnhe_rth_output;
2259                 } else {
2260                         if (unlikely(fl4->flowi4_flags &
2261                                      FLOWI_FLAG_KNOWN_NH &&
2262                                      !(nh->nh_gw &&
2263                                        nh->nh_scope == RT_SCOPE_LINK))) {
2264                                 do_cache = false;
2265                                 goto add;
2266                         }
2267                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2268                 }
2269                 rth = rcu_dereference(*prth);
2270                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2271                         return rth;
2272         }
2273
2274 add:
2275         rth = rt_dst_alloc(dev_out, flags, type,
2276                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2277                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2278                            do_cache);
2279         if (!rth)
2280                 return ERR_PTR(-ENOBUFS);
2281
2282         rth->rt_iif = orig_oif;
2283
2284         RT_CACHE_STAT_INC(out_slow_tot);
2285
2286         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2287                 if (flags & RTCF_LOCAL &&
2288                     !(dev_out->flags & IFF_LOOPBACK)) {
2289                         rth->dst.output = ip_mc_output;
2290                         RT_CACHE_STAT_INC(out_slow_mc);
2291                 }
2292 #ifdef CONFIG_IP_MROUTE
2293                 if (type == RTN_MULTICAST) {
2294                         if (IN_DEV_MFORWARD(in_dev) &&
2295                             !ipv4_is_local_multicast(fl4->daddr)) {
2296                                 rth->dst.input = ip_mr_input;
2297                                 rth->dst.output = ip_mc_output;
2298                         }
2299                 }
2300 #endif
2301         }
2302
2303         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2304         lwtunnel_set_redirect(&rth->dst);
2305
2306         return rth;
2307 }
2308
2309 /*
2310  * Major route resolver routine.
2311  */
2312
2313 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2314                                         const struct sk_buff *skb)
2315 {
2316         __u8 tos = RT_FL_TOS(fl4);
2317         struct fib_result res = {
2318                 .type           = RTN_UNSPEC,
2319                 .fi             = NULL,
2320                 .table          = NULL,
2321                 .tclassid       = 0,
2322         };
2323         struct rtable *rth;
2324
2325         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2326         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2327         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2328                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2329
2330         rcu_read_lock();
2331         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2332         rcu_read_unlock();
2333
2334         return rth;
2335 }
2336 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2337
2338 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2339                                             struct fib_result *res,
2340                                             const struct sk_buff *skb)
2341 {
2342         struct net_device *dev_out = NULL;
2343         int orig_oif = fl4->flowi4_oif;
2344         unsigned int flags = 0;
2345         struct rtable *rth;
2346         int err = -ENETUNREACH;
2347
2348         if (fl4->saddr) {
2349                 rth = ERR_PTR(-EINVAL);
2350                 if (ipv4_is_multicast(fl4->saddr) ||
2351                     ipv4_is_lbcast(fl4->saddr) ||
2352                     ipv4_is_zeronet(fl4->saddr))
2353                         goto out;
2354
2355                 /* I removed check for oif == dev_out->oif here.
2356                    It was wrong for two reasons:
2357                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2358                       is assigned to multiple interfaces.
2359                    2. Moreover, we are allowed to send packets with saddr
2360                       of another iface. --ANK
2361                  */
2362
2363                 if (fl4->flowi4_oif == 0 &&
2364                     (ipv4_is_multicast(fl4->daddr) ||
2365                      ipv4_is_lbcast(fl4->daddr))) {
2366                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2367                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2368                         if (!dev_out)
2369                                 goto out;
2370
2371                         /* Special hack: user can direct multicasts
2372                            and limited broadcast via necessary interface
2373                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2374                            This hack is not just for fun, it allows
2375                            vic,vat and friends to work.
2376                            They bind socket to loopback, set ttl to zero
2377                            and expect that it will work.
2378                            From the viewpoint of routing cache they are broken,
2379                            because we are not allowed to build multicast path
2380                            with loopback source addr (look, routing cache
2381                            cannot know, that ttl is zero, so that packet
2382                            will not leave this host and route is valid).
2383                            Luckily, this hack is good workaround.
2384                          */
2385
2386                         fl4->flowi4_oif = dev_out->ifindex;
2387                         goto make_route;
2388                 }
2389
2390                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2391                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2392                         if (!__ip_dev_find(net, fl4->saddr, false))
2393                                 goto out;
2394                 }
2395         }
2396
2397
2398         if (fl4->flowi4_oif) {
2399                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2400                 rth = ERR_PTR(-ENODEV);
2401                 if (!dev_out)
2402                         goto out;
2403
2404                 /* RACE: Check return value of inet_select_addr instead. */
2405                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2406                         rth = ERR_PTR(-ENETUNREACH);
2407                         goto out;
2408                 }
2409                 if (ipv4_is_local_multicast(fl4->daddr) ||
2410                     ipv4_is_lbcast(fl4->daddr) ||
2411                     fl4->flowi4_proto == IPPROTO_IGMP) {
2412                         if (!fl4->saddr)
2413                                 fl4->saddr = inet_select_addr(dev_out, 0,
2414                                                               RT_SCOPE_LINK);
2415                         goto make_route;
2416                 }
2417                 if (!fl4->saddr) {
2418                         if (ipv4_is_multicast(fl4->daddr))
2419                                 fl4->saddr = inet_select_addr(dev_out, 0,
2420                                                               fl4->flowi4_scope);
2421                         else if (!fl4->daddr)
2422                                 fl4->saddr = inet_select_addr(dev_out, 0,
2423                                                               RT_SCOPE_HOST);
2424                 }
2425         }
2426
2427         if (!fl4->daddr) {
2428                 fl4->daddr = fl4->saddr;
2429                 if (!fl4->daddr)
2430                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2431                 dev_out = net->loopback_dev;
2432                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2433                 res->type = RTN_LOCAL;
2434                 flags |= RTCF_LOCAL;
2435                 goto make_route;
2436         }
2437
2438         err = fib_lookup(net, fl4, res, 0);
2439         if (err) {
2440                 res->fi = NULL;
2441                 res->table = NULL;
2442                 if (fl4->flowi4_oif &&
2443                     (ipv4_is_multicast(fl4->daddr) ||
2444                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2445                         /* Apparently, routing tables are wrong. Assume,
2446                            that the destination is on link.
2447
2448                            WHY? DW.
2449                            Because we are allowed to send to iface
2450                            even if it has NO routes and NO assigned
2451                            addresses. When oif is specified, routing
2452                            tables are looked up with only one purpose:
2453                            to catch if destination is gatewayed, rather than
2454                            direct. Moreover, if MSG_DONTROUTE is set,
2455                            we send packet, ignoring both routing tables
2456                            and ifaddr state. --ANK
2457
2458
2459                            We could make it even if oif is unknown,
2460                            likely IPv6, but we do not.
2461                          */
2462
2463                         if (fl4->saddr == 0)
2464                                 fl4->saddr = inet_select_addr(dev_out, 0,
2465                                                               RT_SCOPE_LINK);
2466                         res->type = RTN_UNICAST;
2467                         goto make_route;
2468                 }
2469                 rth = ERR_PTR(err);
2470                 goto out;
2471         }
2472
2473         if (res->type == RTN_LOCAL) {
2474                 if (!fl4->saddr) {
2475                         if (res->fi->fib_prefsrc)
2476                                 fl4->saddr = res->fi->fib_prefsrc;
2477                         else
2478                                 fl4->saddr = fl4->daddr;
2479                 }
2480
2481                 /* L3 master device is the loopback for that domain */
2482                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2483                         net->loopback_dev;
2484
2485                 /* make sure orig_oif points to fib result device even
2486                  * though packet rx/tx happens over loopback or l3mdev
2487                  */
2488                 orig_oif = FIB_RES_OIF(*res);
2489
2490                 fl4->flowi4_oif = dev_out->ifindex;
2491                 flags |= RTCF_LOCAL;
2492                 goto make_route;
2493         }
2494
2495         fib_select_path(net, res, fl4, skb);
2496
2497         dev_out = FIB_RES_DEV(*res);
2498         fl4->flowi4_oif = dev_out->ifindex;
2499
2500
2501 make_route:
2502         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2503
2504 out:
2505         return rth;
2506 }
2507
2508 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2509 {
2510         return NULL;
2511 }
2512
2513 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2514 {
2515         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2516
2517         return mtu ? : dst->dev->mtu;
2518 }
2519
2520 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2521                                           struct sk_buff *skb, u32 mtu)
2522 {
2523 }
2524
2525 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2526                                        struct sk_buff *skb)
2527 {
2528 }
2529
2530 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2531                                           unsigned long old)
2532 {
2533         return NULL;
2534 }
2535
2536 static struct dst_ops ipv4_dst_blackhole_ops = {
2537         .family                 =       AF_INET,
2538         .check                  =       ipv4_blackhole_dst_check,
2539         .mtu                    =       ipv4_blackhole_mtu,
2540         .default_advmss         =       ipv4_default_advmss,
2541         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2542         .redirect               =       ipv4_rt_blackhole_redirect,
2543         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2544         .neigh_lookup           =       ipv4_neigh_lookup,
2545 };
2546
2547 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2548 {
2549         struct rtable *ort = (struct rtable *) dst_orig;
2550         struct rtable *rt;
2551
2552         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2553         if (rt) {
2554                 struct dst_entry *new = &rt->dst;
2555
2556                 new->__use = 1;
2557                 new->input = dst_discard;
2558                 new->output = dst_discard_out;
2559
2560                 new->dev = net->loopback_dev;
2561                 if (new->dev)
2562                         dev_hold(new->dev);
2563
2564                 rt->rt_is_input = ort->rt_is_input;
2565                 rt->rt_iif = ort->rt_iif;
2566                 rt->rt_pmtu = ort->rt_pmtu;
2567                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2568
2569                 rt->rt_genid = rt_genid_ipv4(net);
2570                 rt->rt_flags = ort->rt_flags;
2571                 rt->rt_type = ort->rt_type;
2572                 rt->rt_gateway = ort->rt_gateway;
2573                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2574
2575                 INIT_LIST_HEAD(&rt->rt_uncached);
2576         }
2577
2578         dst_release(dst_orig);
2579
2580         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2581 }
2582
2583 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2584                                     const struct sock *sk)
2585 {
2586         struct rtable *rt = __ip_route_output_key(net, flp4);
2587
2588         if (IS_ERR(rt))
2589                 return rt;
2590
2591         if (flp4->flowi4_proto)
2592                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2593                                                         flowi4_to_flowi(flp4),
2594                                                         sk, 0);
2595
2596         return rt;
2597 }
2598 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2599
2600 /* called with rcu_read_lock held */
2601 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2602                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2603                         struct sk_buff *skb, u32 portid, u32 seq)
2604 {
2605         struct rtmsg *r;
2606         struct nlmsghdr *nlh;
2607         unsigned long expires = 0;
2608         u32 error;
2609         u32 metrics[RTAX_MAX];
2610
2611         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2612         if (!nlh)
2613                 return -EMSGSIZE;
2614
2615         r = nlmsg_data(nlh);
2616         r->rtm_family    = AF_INET;
2617         r->rtm_dst_len  = 32;
2618         r->rtm_src_len  = 0;
2619         r->rtm_tos      = fl4->flowi4_tos;
2620         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2621         if (nla_put_u32(skb, RTA_TABLE, table_id))
2622                 goto nla_put_failure;
2623         r->rtm_type     = rt->rt_type;
2624         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2625         r->rtm_protocol = RTPROT_UNSPEC;
2626         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2627         if (rt->rt_flags & RTCF_NOTIFY)
2628                 r->rtm_flags |= RTM_F_NOTIFY;
2629         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2630                 r->rtm_flags |= RTCF_DOREDIRECT;
2631
2632         if (nla_put_in_addr(skb, RTA_DST, dst))
2633                 goto nla_put_failure;
2634         if (src) {
2635                 r->rtm_src_len = 32;
2636                 if (nla_put_in_addr(skb, RTA_SRC, src))
2637                         goto nla_put_failure;
2638         }
2639         if (rt->dst.dev &&
2640             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2641                 goto nla_put_failure;
2642 #ifdef CONFIG_IP_ROUTE_CLASSID
2643         if (rt->dst.tclassid &&
2644             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2645                 goto nla_put_failure;
2646 #endif
2647         if (!rt_is_input_route(rt) &&
2648             fl4->saddr != src) {
2649                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2650                         goto nla_put_failure;
2651         }
2652         if (rt->rt_uses_gateway &&
2653             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2654                 goto nla_put_failure;
2655
2656         expires = rt->dst.expires;
2657         if (expires) {
2658                 unsigned long now = jiffies;
2659
2660                 if (time_before(now, expires))
2661                         expires -= now;
2662                 else
2663                         expires = 0;
2664         }
2665
2666         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2667         if (rt->rt_pmtu && expires)
2668                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2669         if (rt->rt_mtu_locked && expires)
2670                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2671         if (rtnetlink_put_metrics(skb, metrics) < 0)
2672                 goto nla_put_failure;
2673
2674         if (fl4->flowi4_mark &&
2675             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2676                 goto nla_put_failure;
2677
2678         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2679             nla_put_u32(skb, RTA_UID,
2680                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2681                 goto nla_put_failure;
2682
2683         error = rt->dst.error;
2684
2685         if (rt_is_input_route(rt)) {
2686 #ifdef CONFIG_IP_MROUTE
2687                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2688                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2689                         int err = ipmr_get_route(net, skb,
2690                                                  fl4->saddr, fl4->daddr,
2691                                                  r, portid);
2692
2693                         if (err <= 0) {
2694                                 if (err == 0)
2695                                         return 0;
2696                                 goto nla_put_failure;
2697                         }
2698                 } else
2699 #endif
2700                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2701                                 goto nla_put_failure;
2702         }
2703
2704         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2705                 goto nla_put_failure;
2706
2707         nlmsg_end(skb, nlh);
2708         return 0;
2709
2710 nla_put_failure:
2711         nlmsg_cancel(skb, nlh);
2712         return -EMSGSIZE;
2713 }
2714
2715 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2716                                                    u8 ip_proto, __be16 sport,
2717                                                    __be16 dport)
2718 {
2719         struct sk_buff *skb;
2720         struct iphdr *iph;
2721
2722         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2723         if (!skb)
2724                 return NULL;
2725
2726         /* Reserve room for dummy headers, this skb can pass
2727          * through good chunk of routing engine.
2728          */
2729         skb_reset_mac_header(skb);
2730         skb_reset_network_header(skb);
2731         skb->protocol = htons(ETH_P_IP);
2732         iph = skb_put(skb, sizeof(struct iphdr));
2733         iph->protocol = ip_proto;
2734         iph->saddr = src;
2735         iph->daddr = dst;
2736         iph->version = 0x4;
2737         iph->frag_off = 0;
2738         iph->ihl = 0x5;
2739         skb_set_transport_header(skb, skb->len);
2740
2741         switch (iph->protocol) {
2742         case IPPROTO_UDP: {
2743                 struct udphdr *udph;
2744
2745                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2746                 udph->source = sport;
2747                 udph->dest = dport;
2748                 udph->len = sizeof(struct udphdr);
2749                 udph->check = 0;
2750                 break;
2751         }
2752         case IPPROTO_TCP: {
2753                 struct tcphdr *tcph;
2754
2755                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2756                 tcph->source    = sport;
2757                 tcph->dest      = dport;
2758                 tcph->doff      = sizeof(struct tcphdr) / 4;
2759                 tcph->rst = 1;
2760                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2761                                             src, dst, 0);
2762                 break;
2763         }
2764         case IPPROTO_ICMP: {
2765                 struct icmphdr *icmph;
2766
2767                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2768                 icmph->type = ICMP_ECHO;
2769                 icmph->code = 0;
2770         }
2771         }
2772
2773         return skb;
2774 }
2775
2776 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2777                              struct netlink_ext_ack *extack)
2778 {
2779         struct net *net = sock_net(in_skb->sk);
2780         struct nlattr *tb[RTA_MAX+1];
2781         u32 table_id = RT_TABLE_MAIN;
2782         __be16 sport = 0, dport = 0;
2783         struct fib_result res = {};
2784         u8 ip_proto = IPPROTO_UDP;
2785         struct rtable *rt = NULL;
2786         struct sk_buff *skb;
2787         struct rtmsg *rtm;
2788         struct flowi4 fl4;
2789         __be32 dst = 0;
2790         __be32 src = 0;
2791         kuid_t uid;
2792         u32 iif;
2793         int err;
2794         int mark;
2795
2796         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2797                           extack);
2798         if (err < 0)
2799                 return err;
2800
2801         rtm = nlmsg_data(nlh);
2802         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2803         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2804         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2805         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2806         if (tb[RTA_UID])
2807                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2808         else
2809                 uid = (iif ? INVALID_UID : current_uid());
2810
2811         if (tb[RTA_IP_PROTO]) {
2812                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2813                                                   &ip_proto, extack);
2814                 if (err)
2815                         return err;
2816         }
2817
2818         if (tb[RTA_SPORT])
2819                 sport = nla_get_be16(tb[RTA_SPORT]);
2820
2821         if (tb[RTA_DPORT])
2822                 dport = nla_get_be16(tb[RTA_DPORT]);
2823
2824         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2825         if (!skb)
2826                 return -ENOBUFS;
2827
2828         memset(&fl4, 0, sizeof(fl4));
2829         fl4.daddr = dst;
2830         fl4.saddr = src;
2831         fl4.flowi4_tos = rtm->rtm_tos;
2832         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2833         fl4.flowi4_mark = mark;
2834         fl4.flowi4_uid = uid;
2835         if (sport)
2836                 fl4.fl4_sport = sport;
2837         if (dport)
2838                 fl4.fl4_dport = dport;
2839         fl4.flowi4_proto = ip_proto;
2840
2841         rcu_read_lock();
2842
2843         if (iif) {
2844                 struct net_device *dev;
2845
2846                 dev = dev_get_by_index_rcu(net, iif);
2847                 if (!dev) {
2848                         err = -ENODEV;
2849                         goto errout_rcu;
2850                 }
2851
2852                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2853                 skb->dev        = dev;
2854                 skb->mark       = mark;
2855                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2856                                          dev, &res);
2857
2858                 rt = skb_rtable(skb);
2859                 if (err == 0 && rt->dst.error)
2860                         err = -rt->dst.error;
2861         } else {
2862                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2863                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2864                 err = 0;
2865                 if (IS_ERR(rt))
2866                         err = PTR_ERR(rt);
2867                 else
2868                         skb_dst_set(skb, &rt->dst);
2869         }
2870
2871         if (err)
2872                 goto errout_rcu;
2873
2874         if (rtm->rtm_flags & RTM_F_NOTIFY)
2875                 rt->rt_flags |= RTCF_NOTIFY;
2876
2877         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2878                 table_id = res.table ? res.table->tb_id : 0;
2879
2880         /* reset skb for netlink reply msg */
2881         skb_trim(skb, 0);
2882         skb_reset_network_header(skb);
2883         skb_reset_transport_header(skb);
2884         skb_reset_mac_header(skb);
2885
2886         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2887                 if (!res.fi) {
2888                         err = fib_props[res.type].error;
2889                         if (!err)
2890                                 err = -EHOSTUNREACH;
2891                         goto errout_rcu;
2892                 }
2893                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2894                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2895                                     rt->rt_type, res.prefix, res.prefixlen,
2896                                     fl4.flowi4_tos, res.fi, 0);
2897         } else {
2898                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2899                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2900         }
2901         if (err < 0)
2902                 goto errout_rcu;
2903
2904         rcu_read_unlock();
2905
2906         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2907
2908 errout_free:
2909         return err;
2910 errout_rcu:
2911         rcu_read_unlock();
2912         kfree_skb(skb);
2913         goto errout_free;
2914 }
2915
2916 void ip_rt_multicast_event(struct in_device *in_dev)
2917 {
2918         rt_cache_flush(dev_net(in_dev->dev));
2919 }
2920
2921 #ifdef CONFIG_SYSCTL
2922 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2923 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2924 static int ip_rt_gc_elasticity __read_mostly    = 8;
2925 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
2926
2927 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2928                                         void __user *buffer,
2929                                         size_t *lenp, loff_t *ppos)
2930 {
2931         struct net *net = (struct net *)__ctl->extra1;
2932
2933         if (write) {
2934                 rt_cache_flush(net);
2935                 fnhe_genid_bump(net);
2936                 return 0;
2937         }
2938
2939         return -EINVAL;
2940 }
2941
2942 static struct ctl_table ipv4_route_table[] = {
2943         {
2944                 .procname       = "gc_thresh",
2945                 .data           = &ipv4_dst_ops.gc_thresh,
2946                 .maxlen         = sizeof(int),
2947                 .mode           = 0644,
2948                 .proc_handler   = proc_dointvec,
2949         },
2950         {
2951                 .procname       = "max_size",
2952                 .data           = &ip_rt_max_size,
2953                 .maxlen         = sizeof(int),
2954                 .mode           = 0644,
2955                 .proc_handler   = proc_dointvec,
2956         },
2957         {
2958                 /*  Deprecated. Use gc_min_interval_ms */
2959
2960                 .procname       = "gc_min_interval",
2961                 .data           = &ip_rt_gc_min_interval,
2962                 .maxlen         = sizeof(int),
2963                 .mode           = 0644,
2964                 .proc_handler   = proc_dointvec_jiffies,
2965         },
2966         {
2967                 .procname       = "gc_min_interval_ms",
2968                 .data           = &ip_rt_gc_min_interval,
2969                 .maxlen         = sizeof(int),
2970                 .mode           = 0644,
2971                 .proc_handler   = proc_dointvec_ms_jiffies,
2972         },
2973         {
2974                 .procname       = "gc_timeout",
2975                 .data           = &ip_rt_gc_timeout,
2976                 .maxlen         = sizeof(int),
2977                 .mode           = 0644,
2978                 .proc_handler   = proc_dointvec_jiffies,
2979         },
2980         {
2981                 .procname       = "gc_interval",
2982                 .data           = &ip_rt_gc_interval,
2983                 .maxlen         = sizeof(int),
2984                 .mode           = 0644,
2985                 .proc_handler   = proc_dointvec_jiffies,
2986         },
2987         {
2988                 .procname       = "redirect_load",
2989                 .data           = &ip_rt_redirect_load,
2990                 .maxlen         = sizeof(int),
2991                 .mode           = 0644,
2992                 .proc_handler   = proc_dointvec,
2993         },
2994         {
2995                 .procname       = "redirect_number",
2996                 .data           = &ip_rt_redirect_number,
2997                 .maxlen         = sizeof(int),
2998                 .mode           = 0644,
2999                 .proc_handler   = proc_dointvec,
3000         },
3001         {
3002                 .procname       = "redirect_silence",
3003                 .data           = &ip_rt_redirect_silence,
3004                 .maxlen         = sizeof(int),
3005                 .mode           = 0644,
3006                 .proc_handler   = proc_dointvec,
3007         },
3008         {
3009                 .procname       = "error_cost",
3010                 .data           = &ip_rt_error_cost,
3011                 .maxlen         = sizeof(int),
3012                 .mode           = 0644,
3013                 .proc_handler   = proc_dointvec,
3014         },
3015         {
3016                 .procname       = "error_burst",
3017                 .data           = &ip_rt_error_burst,
3018                 .maxlen         = sizeof(int),
3019                 .mode           = 0644,
3020                 .proc_handler   = proc_dointvec,
3021         },
3022         {
3023                 .procname       = "gc_elasticity",
3024                 .data           = &ip_rt_gc_elasticity,
3025                 .maxlen         = sizeof(int),
3026                 .mode           = 0644,
3027                 .proc_handler   = proc_dointvec,
3028         },
3029         {
3030                 .procname       = "mtu_expires",
3031                 .data           = &ip_rt_mtu_expires,
3032                 .maxlen         = sizeof(int),
3033                 .mode           = 0644,
3034                 .proc_handler   = proc_dointvec_jiffies,
3035         },
3036         {
3037                 .procname       = "min_pmtu",
3038                 .data           = &ip_rt_min_pmtu,
3039                 .maxlen         = sizeof(int),
3040                 .mode           = 0644,
3041                 .proc_handler   = proc_dointvec_minmax,
3042                 .extra1         = &ip_min_valid_pmtu,
3043         },
3044         {
3045                 .procname       = "min_adv_mss",
3046                 .data           = &ip_rt_min_advmss,
3047                 .maxlen         = sizeof(int),
3048                 .mode           = 0644,
3049                 .proc_handler   = proc_dointvec,
3050         },
3051         { }
3052 };
3053
3054 static struct ctl_table ipv4_route_flush_table[] = {
3055         {
3056                 .procname       = "flush",
3057                 .maxlen         = sizeof(int),
3058                 .mode           = 0200,
3059                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3060         },
3061         { },
3062 };
3063
3064 static __net_init int sysctl_route_net_init(struct net *net)
3065 {
3066         struct ctl_table *tbl;
3067
3068         tbl = ipv4_route_flush_table;
3069         if (!net_eq(net, &init_net)) {
3070                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3071                 if (!tbl)
3072                         goto err_dup;
3073
3074                 /* Don't export sysctls to unprivileged users */
3075                 if (net->user_ns != &init_user_ns)
3076                         tbl[0].procname = NULL;
3077         }
3078         tbl[0].extra1 = net;
3079
3080         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3081         if (!net->ipv4.route_hdr)
3082                 goto err_reg;
3083         return 0;
3084
3085 err_reg:
3086         if (tbl != ipv4_route_flush_table)
3087                 kfree(tbl);
3088 err_dup:
3089         return -ENOMEM;
3090 }
3091
3092 static __net_exit void sysctl_route_net_exit(struct net *net)
3093 {
3094         struct ctl_table *tbl;
3095
3096         tbl = net->ipv4.route_hdr->ctl_table_arg;
3097         unregister_net_sysctl_table(net->ipv4.route_hdr);
3098         BUG_ON(tbl == ipv4_route_flush_table);
3099         kfree(tbl);
3100 }
3101
3102 static __net_initdata struct pernet_operations sysctl_route_ops = {
3103         .init = sysctl_route_net_init,
3104         .exit = sysctl_route_net_exit,
3105 };
3106 #endif
3107
3108 static __net_init int rt_genid_init(struct net *net)
3109 {
3110         atomic_set(&net->ipv4.rt_genid, 0);
3111         atomic_set(&net->fnhe_genid, 0);
3112         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3113         return 0;
3114 }
3115
3116 static __net_initdata struct pernet_operations rt_genid_ops = {
3117         .init = rt_genid_init,
3118 };
3119
3120 static int __net_init ipv4_inetpeer_init(struct net *net)
3121 {
3122         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3123
3124         if (!bp)
3125                 return -ENOMEM;
3126         inet_peer_base_init(bp);
3127         net->ipv4.peers = bp;
3128         return 0;
3129 }
3130
3131 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3132 {
3133         struct inet_peer_base *bp = net->ipv4.peers;
3134
3135         net->ipv4.peers = NULL;
3136         inetpeer_invalidate_tree(bp);
3137         kfree(bp);
3138 }
3139
3140 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3141         .init   =       ipv4_inetpeer_init,
3142         .exit   =       ipv4_inetpeer_exit,
3143 };
3144
3145 #ifdef CONFIG_IP_ROUTE_CLASSID
3146 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3147 #endif /* CONFIG_IP_ROUTE_CLASSID */
3148
3149 int __init ip_rt_init(void)
3150 {
3151         int cpu;
3152
3153         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3154                                   GFP_KERNEL);
3155         if (!ip_idents)
3156                 panic("IP: failed to allocate ip_idents\n");
3157
3158         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3159
3160         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3161         if (!ip_tstamps)
3162                 panic("IP: failed to allocate ip_tstamps\n");
3163
3164         for_each_possible_cpu(cpu) {
3165                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3166
3167                 INIT_LIST_HEAD(&ul->head);
3168                 spin_lock_init(&ul->lock);
3169         }
3170 #ifdef CONFIG_IP_ROUTE_CLASSID
3171         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3172         if (!ip_rt_acct)
3173                 panic("IP: failed to allocate ip_rt_acct\n");
3174 #endif
3175
3176         ipv4_dst_ops.kmem_cachep =
3177                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3178                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3179
3180         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3181
3182         if (dst_entries_init(&ipv4_dst_ops) < 0)
3183                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3184
3185         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3186                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3187
3188         ipv4_dst_ops.gc_thresh = ~0;
3189         ip_rt_max_size = INT_MAX;
3190
3191         devinet_init();
3192         ip_fib_init();
3193
3194         if (ip_rt_proc_init())
3195                 pr_err("Unable to create route proc files\n");
3196 #ifdef CONFIG_XFRM
3197         xfrm_init();
3198         xfrm4_init();
3199 #endif
3200         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3201                       RTNL_FLAG_DOIT_UNLOCKED);
3202
3203 #ifdef CONFIG_SYSCTL
3204         register_pernet_subsys(&sysctl_route_ops);
3205 #endif
3206         register_pernet_subsys(&rt_genid_ops);
3207         register_pernet_subsys(&ipv4_inetpeer_ops);
3208         return 0;
3209 }
3210
3211 #ifdef CONFIG_SYSCTL
3212 /*
3213  * We really need to sanitize the damn ipv4 init order, then all
3214  * this nonsense will go away.
3215  */
3216 void __init ip_static_sysctl_init(void)
3217 {
3218         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3219 }
3220 #endif